From 3a5cba5c4016c5803f7d8df15b4c3eba9b55464f Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 13:57:22 +0300 Subject: [PATCH 01/88] docs: design unified MCP image and atomic corpus delivery --- .../2026-09-10-mcp-atomic-site-snapshots.md | 72 ++++ ...26-09-10-mcp-published-combined-runtime.md | 73 ++++ ...09-10-mcp-recoverable-container-release.md | 66 ++++ spec/contracts/mcp-api-v2-r4.md | 97 ++++++ spec/contracts/mcp-corpus-snapshot-v1-r0.md | 245 ++++++++++++++ spec/contracts/mcp-distribution-v1-r0.md | 145 ++++++++ spec/contracts/mcp-release-runtime-v1-r0.md | 139 ++++++++ ...6-09-10-mcp-ci-deployment-policy-design.md | 118 +++++++ ...09-10-mcp-container-distribution-design.md | 320 ++++++++++++++++++ .../mcp-active-snapshot-is-complete.md | 36 ++ .../mcp-published-runtime-is-one-service.md | 39 +++ ...mcp-release-has-recoverable-predecessor.md | 34 ++ ...cp-site-override-has-no-public-fallback.md | 36 ++ ...6-09-09-markdown-rendering-verification.md | 5 +- spec/operations/2026-09-09-mcp-incident.md | 8 +- ...026-07-22-unified-diagnostic-chips-plan.md | 4 +- spec/process/architecture-artifacts-v2.md | 93 +++++ 17 files changed, 1521 insertions(+), 9 deletions(-) create mode 100644 spec/adr/2026-09-10-mcp-atomic-site-snapshots.md create mode 100644 spec/adr/2026-09-10-mcp-published-combined-runtime.md create mode 100644 spec/adr/2026-09-10-mcp-recoverable-container-release.md create mode 100644 spec/contracts/mcp-api-v2-r4.md create mode 100644 spec/contracts/mcp-corpus-snapshot-v1-r0.md create mode 100644 spec/contracts/mcp-distribution-v1-r0.md create mode 100644 spec/contracts/mcp-release-runtime-v1-r0.md create mode 100644 spec/designs/2026-09-10-mcp-ci-deployment-policy-design.md create mode 100644 spec/designs/2026-09-10-mcp-container-distribution-design.md create mode 100644 spec/invariants/mcp-active-snapshot-is-complete.md create mode 100644 spec/invariants/mcp-published-runtime-is-one-service.md create mode 100644 spec/invariants/mcp-release-has-recoverable-predecessor.md create mode 100644 spec/invariants/mcp-site-override-has-no-public-fallback.md create mode 100644 spec/process/architecture-artifacts-v2.md diff --git a/spec/adr/2026-09-10-mcp-atomic-site-snapshots.md b/spec/adr/2026-09-10-mcp-atomic-site-snapshots.md new file mode 100644 index 0000000..89e528c --- /dev/null +++ b/spec/adr/2026-09-10-mcp-atomic-site-snapshots.md @@ -0,0 +1,72 @@ +--- +schema_version: 1 +kind: adr +id: MCP_ATOMIC_SITE_SNAPSHOTS +scope: product +design: design:mcp-container-distribution +requirements: + - MCP_RUNTIME_AND_CORPUS_RELEASE_INDEPENDENTLY + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_SNAPSHOT_LOAD_IS_ATOMIC + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNAPSHOT_IO_IS_BOUNDED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + - MCP_OFFLINE_USES_VERIFIED_CACHE +aliases: [] +supersedes: [] +cancels: [] +invariants: + introduces: + - invariant:MCP_ACTIVE_SNAPSHOT_IS_COMPLETE + - invariant:MCP_SITE_OVERRIDE_HAS_NO_PUBLIC_FALLBACK + preserves: [] + replaces: {} + cancels: [] +contracts: + introduces: + - contract:MCP_CORPUS_SNAPSHOT@1.0 + preserves: + - contract:MCP_API@2.4 + replaces: {} + cancels: [] +--- + +# Атомарные snapshots от выбранного сайта + +## Входные требования + +Независимый выпуск данных, единственная настройка источника и адресов, +ограниченный background I/O, целостность поколения и локальная автономность. + +## Решение + +Выбранный сайт публикует manifest одного неизменяемого snapshot. Runtime +скачивает, проверяет и подготавливает его вне request path; одним коротким +переключением ссылки активирует готовый immutable index. Каждый запрос +удерживает свою ссылку до завершения. Cache разделён по нормализованному сайту +и schema; активная и предыдущая копии сохраняются для offline/recovery. + +Публичный manifest явно ссылается на nginx `ai.v8std.ru/indexes/`, локальный — +на файл своего сайта. Второй source/base URL не вводится. Доставка файлов +отделена от вычислительного MCP, но не является вторым MCP-сервисом. + +## Влияние на инварианты + +Вводятся полнота active snapshot и запрет скрытого public fallback при site +override. Последняя рабочая копия важнее свежести: transient source failure +не делает готовый runtime неготовым. Смена источника не наследует чужой cache. + +## Влияние на контракты + +Snapshot 1.0 определяет manifest/archive, URL trust boundary, hash и resource +budgets. API 2.4 использует одно поколение для поиска и чтения, сохраняя текущий +surface; неподготовленный instance сообщает явную неготовность. + +## Отклонённые альтернативы + +- Обход сайта или скачивание данных при каждом tool call. +- Независимые mutable pages/vectors с записью в рабочий cache до проверки. +- Второй environment variable для ссылок или неявный fallback на v8std.ru. +- Python endpoint для статических индексов: связывает доступность с runtime. +- Обязательное скачивание corpus при каждом старте агента без persistent cache. diff --git a/spec/adr/2026-09-10-mcp-published-combined-runtime.md b/spec/adr/2026-09-10-mcp-published-combined-runtime.md new file mode 100644 index 0000000..5803f4e --- /dev/null +++ b/spec/adr/2026-09-10-mcp-published-combined-runtime.md @@ -0,0 +1,73 @@ +--- +schema_version: 1 +kind: adr +id: MCP_PUBLISHED_COMBINED_RUNTIME +scope: product +design: design:mcp-container-distribution +requirements: + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_COMBINED_PAGE_READING_COMPATIBLE + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + - MCP_DISTRIBUTION_PROVENANCE_IS_VERIFIABLE +aliases: [] +supersedes: + - adr:MCP_COMBINED_ENDPOINT +cancels: [] +invariants: + introduces: + - invariant:MCP_PUBLISHED_RUNTIME_IS_ONE_SERVICE + preserves: + - invariant:MCP_LEGACY_ENDPOINT_STABILITY + - invariant:MCP_RESOURCE_LINKS_ARE_LISTABLE + - invariant:MCP_SNIPPET_SIGNALS_SURVIVE_TEXT_BUDGET + replaces: + invariant:MCP_COMBINED_ENDPOINT_IS_SINGLE_RUNTIME: invariant:MCP_PUBLISHED_RUNTIME_IS_ONE_SERVICE + cancels: [] +contracts: + introduces: + - contract:MCP_API@2.4 + - contract:MCP_DISTRIBUTION@1.0 + preserves: + - contract:MCP_API@2.0 + - contract:MCP_API@2.1 + replaces: + contract:MCP_API@2.2: contract:MCP_API@2.4 + contract:MCP_API@2.3: contract:MCP_API@2.4 + cancels: [] +--- + +# Один опубликованный runtime для всех способов запуска + +## Входные требования + +Единый логический MCP-сервис, совместимое чтение страниц, lifecycle кодового +агента и проверяемое происхождение артефакта. + +## Решение + +Один multi-platform runtime image запускается в stdio либо HTTP-режиме. +Production, локальный запуск и каталог используют его опубликованный digest, +не отдельный Docker-built fork и не docs-builder. Steady state production — +один активный runtime; максимум два однотипных экземпляра при переключении, +без самостоятельного v3 endpoint. Внешняя граница остаётся `/mcp`. + +## Влияние на инварианты + +Буквальная привязка к одному Python-процессу и старому systemd unit заменена +инвариантом одного сервиса из опубликованного образа. Legacy API, разрешимость +Resource links и работа snippet-сигналов сохранены. Исторические ссылки на +single-runtime invariant не означают запрета ограниченного overlap при rollout. + +## Влияние на контракты + +API 2.4 наследует 2.3, не меняет tool arguments/result shapes и дополняет +транспорты и lifecycle. Distribution 1.0 задаёт наблюдаемый интерфейс образа. +Ссылки на 2.0/2.1 сохраняют историческую совместимость, а не второй действующий +runtime. Ранее отклонённый API 3.0 не возобновляется. + +## Отклонённые альтернативы + +- Раздельные production/local/catalog образы: разные пути выпуска и проверки. +- Отдельный сервис v3: отсутствует необходимость в изоляции пользователей v3. +- Остановка старого до проверки нового: увеличивает риск аварии при rollout. +- Snapshot внутри runtime image: связывает выпуск статей с рестартом сервера. diff --git a/spec/adr/2026-09-10-mcp-recoverable-container-release.md b/spec/adr/2026-09-10-mcp-recoverable-container-release.md new file mode 100644 index 0000000..f877577 --- /dev/null +++ b/spec/adr/2026-09-10-mcp-recoverable-container-release.md @@ -0,0 +1,66 @@ +--- +schema_version: 1 +kind: adr +id: MCP_RECOVERABLE_CONTAINER_RELEASE +scope: product +design: design:mcp-container-distribution +requirements: + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_SHARED_HOST_LOAD_IS_MEASURED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART +aliases: [] +supersedes: [] +cancels: [] +invariants: + introduces: + - invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR + preserves: + - invariant:MCP_PUBLISHED_RUNTIME_IS_ONE_SERVICE + - invariant:MCP_ACTIVE_SNAPSHOT_IS_COMPLETE + replaces: {} + cancels: [] +contracts: + introduces: + - contract:MCP_RELEASE_RUNTIME@1.0 + preserves: + - contract:MCP_API@2.4 + - contract:MCP_CORPUS_SNAPSHOT@1.0 + - contract:MCP_DISTRIBUTION@1.0 + replaces: {} + cancels: [] +--- + +# Восстанавливаемая транзакция смены runtime + +## Входные требования + +Обратимость переключения, измеренное совместное потребление host и независимая +раздача индексов во время перезапуска MCP. + +## Решение + +Host выполняет ограниченную по времени идемпотентную release-транзакцию: +проверить envelope → подготовить образ и corpus → readiness на внутреннем +порту → переключить nginx → публичный smoke → bounded drain старого. +До commit хранятся predecessor image/config и совместимые данные. Сбой после +switch приводит к rollback. SSH служит каналом передачи задания, не владельцем +жизни транзакции; host восстанавливает состояние после потери соединения. + +## Влияние на инварианты + +Добавляется проверяемый recoverable predecessor; единый endpoint и атомарность +snapshot сохраняются. Capacity gate включает overlap двух runtime и staging +индекса: нельзя обещать бесшовность, если они не помещаются в память host. + +## Влияние на контракты + +Release runtime 1.0 задаёт состояния, idempotency, наблюдаемые результаты и +fault-injection. API/distribution/snapshot contracts не меняются. Правила +выдачи CI полномочий принадлежат отдельному process design, не этому ADR. + +## Отклонённые альтернативы + +- `pull latest && restart` без predecessor и post-switch smoke. +- Произвольный shell из входа deploy job или обычная SSH-сессия как транзакция. +- GC прежнего image/cache до успешного переключения. +- Считать активный systemd wrapper доказательством готовности контейнера. diff --git a/spec/contracts/mcp-api-v2-r4.md b/spec/contracts/mcp-api-v2-r4.md new file mode 100644 index 0000000..975e44b --- /dev/null +++ b/spec/contracts/mcp-api-v2-r4.md @@ -0,0 +1,97 @@ +--- +schema_version: 1 +kind: contract +id: MCP_API +scope: product +version: 2 +revision: 4 +compatibility: backward-compatible +design: design:mcp-container-distribution +producer: one combined MCP runtime over Streamable HTTP or stdio +consumers: + - existing MCP v2 clients + - coding agents using tools and Resources + - local container and Docker Gateway clients + - runtime readiness probes +requirements: + - MCP_LEGACY_VERSION_REMAINS_COMPATIBLE + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_COMBINED_PAGE_READING_COMPATIBLE + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNIPPET_ACCEPTED_INPUT_IS_SCANNED + - MCP_SNIPPET_TARGETS_SURVIVE_QUERY_BUDGET + - MCP_SNIPPET_INSTANCE_LIMIT_IS_DISCOVERABLE + - MCP_SNIPPET_RETRIEVAL_WORK_IS_BOUNDED + - MCP_SNIPPET_RESPONSE_STAYS_COMPACT +governs: + - scripts/v8std_mcp_server.py + - scripts/v8std_mcp_index.py + - scripts/v8std_retrieval_rules.py + - scripts/run_v8std_mcp.sh + - deploy/nginx/server-v8std-mcp.conf + - deploy/systemd/v8std-mcp.service + - docker-compose/docker-compose.yml + - docs/mcp.md + - docs/support.md + - tests/test_v8std_mcp_server.py + - tests/test_v8std_mcp_index.py + - tests/test_v8std_mcp_snippet.py + - tests/test_v8std_mcp_combined.py + - tests/test_v8std_mcp_distribution.py +conformance: + module: tests.test_v8std_mcp_distribution + command: .venv/bin/python -m unittest tests.test_v8std_mcp_distribution tests.test_v8std_mcp_snippet tests.test_v8std_mcp_server tests.test_v8std_mcp_index tests.test_v8std_mcp_combined -v +required_when: implemented +supersedes: + - contract:MCP_API@2.3 +deprecates: [] +--- + +# MCP API v2.4: поставка и lifecycle + +## Совместимость + +Нормативно сохраняется [API 2.3](mcp-api-v2-r3.md) целиком, кроме способа +инициализации/обновления index и буквальной привязки к одному process/unit. +Ни версия MCP protocol, ни application API version в `/version` не становятся +`2.4` из-за номера этого внутреннего контракта. Нет `/v3/mcp`, новых обязательных +tool arguments, эвристики по имени агента или удаления `v8std_get_page`. + +Сохраняются пять tools и три текущих bulk Resources, без нового bulk discovery. +Результаты, ошибки ввода, лимиты snippet 4000 по умолчанию / 32000 максимум, +search query 500, preview 1000, tokens 80 и суммарно 4000, один hybrid search +и compact top-K нормативно наследуются. Refresh не меняет ranking model. + +## Транспорт и жизненный цикл + +HTTP — stateless Streamable HTTP с JSON-ответами и текущей POST-only политикой +edge. Долгоживущий GET/SSE для уведомлений не появляется. Stdio запускает тот +же набор tools/Resources; stdout только MCP, журналы в stderr. SIGTERM и EOF +ограниченно завершают процессы/задачи, без orphan refresh worker. + +Initialize и discovery не ждут сети: schema и допустимые лимиты известны из +конфигурации. До первого валидного snapshot обращения, которым нужны данные, +получают MCP tool/resource error с кодом в тексте `INDEX_NOT_READY` и указанием +повторить позже; SDK-обёртка не фиксируется. Не выдаются пустые успешные результаты +и не запускается синхронная загрузка от этого обращения. Ошибки ввода проверяются +до готовности данных там, где проверка не требует corpus. + +Существующий HTTP `/healthz` возвращает 200 только при готовом active snapshot, +иначе 503. Additive `/livez` проверяет жизнь runtime, не сеть. +Freshness отдельно от readiness: валидный stale snapshot остаётся готовым. +Health/version metadata добавляют runtime SHA, corpus ID, время последней +успешной проверки и компактную категорию refresh error; secrets, raw code, +локальные filesystem paths и содержимое corpus туда не попадают. + +Каждый data request захватывает одну ссылку на immutable snapshot. Поиск, +сигналы, страницы и Resources внутри запроса не смешивают поколения. Внешний +I/O отсутствует на request path; подготовка следующего index не держит query +lock. Параметры refresh и source берутся при запуске, не из tool input. + +## Проверки + +Нужны wire tests stdio/HTTP, initialize до доступности источника, два агента, +warm/cold startup, clean EOF/SIGTERM, совместимость прежних tools/Resources +и ответы во время медленного/повреждённого refresh. Новый conformance module +является задачей реализации, а не существующим доказательством. diff --git a/spec/contracts/mcp-corpus-snapshot-v1-r0.md b/spec/contracts/mcp-corpus-snapshot-v1-r0.md new file mode 100644 index 0000000..761079f --- /dev/null +++ b/spec/contracts/mcp-corpus-snapshot-v1-r0.md @@ -0,0 +1,245 @@ +--- +schema_version: 1 +kind: contract +id: MCP_CORPUS_SNAPSHOT +scope: product +version: 1 +revision: 0 +compatibility: backward-compatible +design: design:mcp-container-distribution +producer: site artifact builder and static index publisher +consumers: + - MCP background snapshot loader + - local static-site image builder + - release verifier and operators +requirements: + - MCP_RUNTIME_AND_CORPUS_RELEASE_INDEPENDENTLY + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_SNAPSHOT_LOAD_IS_ATOMIC + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNAPSHOT_IO_IS_BOUNDED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + - MCP_OFFLINE_USES_VERIFIED_CACHE +governs: + - scripts/generate_ai_artifacts.py + - scripts/generate_search_vectors.py + - scripts/v8std_mcp_index.py + - scripts/v8std_mcp_snapshots.py + - scripts/generate_mcp_snapshot.py + - deploy/nginx + - .github/workflows + - tests/test_v8std_mcp_snapshots.py +conformance: + module: tests.test_v8std_mcp_snapshots + command: .venv/bin/python -m unittest tests.test_v8std_mcp_snapshots -v +required_when: implemented +supersedes: [] +deprecates: [] +--- + +# Corpus snapshot 1.0 + +## Источник и адреса + +Единственная публичная настройка — `V8STD_MCP_SITE_URL`, по умолчанию +`https://v8std.ru/`. CLI `--site-url` имеет приоритет над env; пустое явно +заданное значение является ошибкой конфигурации. Удаляются внешние пробелы, +схема/host нормализуются, default port убирается, base path сохраняется и +завершается `/`. Запрещены userinfo, query, fragment, неподдерживаемые схемы, +неоднозначные encoded separators и выход из base path через dot segments. +HTTP допускается только при явно заданном site URL для локальной установки; +HTTPS не понижается при redirect. TLS certificate verification обязательна. + +Bootstrap — `ai/mcp/v1/manifest.json`. Так, base `/knowledge/` +даёт `/knowledge/ai/mcp/v1/manifest.json`, а не URL от корня host. Для любой +нестандартной установки manifest, archive и redirects остаются в том же +origin и base path. Ни manifest, ни его redirect не выбирают новый base URL +ответов. Запросы не используют credentials из URL и не пересылают secrets. + +Только для нормализованного стандартного `https://v8std.ru/` допускается +явный archive URL manifest на `https://ai.v8std.ru/indexes/v1/`. Этот фиксированный +delivery origin не является пользовательским вторым source setting. Redirect +из него допускается только внутри того же origin/path. Для local-site даже +такой public archive URL отклоняется. Не более трёх redirects с проверкой +каждого перехода; `//foreign-host`, traversal и смена схемы не обходят правила. +Отсутствующий manifest даёт ошибку обновления, не fallback к старым public URLs. + +SITE_URL должен быть доступен и контейнеру, и пользователю, открывающему ссылки. +Адрес `http://site:8080/`, видимый только Docker DNS, для браузера не подходит. +Compose-документация предоставляет проверенный общий адрес и сетевое разрешение +для host/контейнера, не вводит скрытый второй URL. Если такая маршрутизация +отсутствует, setup должен сообщить об этом, а не выдавать неработающие ссылки. + +## Manifest + +UTF-8 JSON object, schema major 1; обязательные поля: + +| Поле | Значение | +|---|---| +| `schema_version` | Целое `1`. | +| `source_sha` | Полный Git SHA входов публикации. | +| `corpus_id` | SHA-256 content descriptor, определённый ниже. | +| `archive.path` | Относительный URL от directory manifest либо разрешённый абсолютный URL. | +| `archive.sha256` | SHA-256 точных сжатых байтов архива. | +| `archive.bytes` | Точное число сжатых байтов. | +| `archive.unpacked_bytes` | Сумма размеров пяти обычных файлов без tar padding. | +| `vector_model` | `v8std-hash-embeddings-v1`. | +| `vector_dim` | Целое `256`. | + +Неизвестные optional поля игнорируются; неизвестная major, duplicate JSON keys, +неверный тип или unsupported model/dim отклоняются. Hash имеет 64 lowercase hex +символа. Archive path публичной публикации равен +`https://ai.v8std.ru/indexes/v1//snapshot.tar.gz`; локальной — +`/snapshot.tar.gz` относительно directory manifest. +SHA не считается цифровой подписью: доверие к manifest задаётся настроенным +сайтом и TLS, а hash обеспечивает целостность указанного им содержимого. + +## Архив и воспроизводимость + +Один детерминированный `tar.gz`, ровно пять regular-file members в заданном порядке: + +1. `metadata.json`; +2. `pages.jsonl`; +3. `search-vectors.jsonl`; +4. `llms.txt`; +5. `llms-full.txt`. + +Нет каталогов, дополнительных entries, duplicates, PAX/GNU extensions, +symlinks/hardlinks, device files, absolute paths или traversal. Metadata UID/GID, +permissions и tar mtime фиксированы; gzip mtime = 0, filename отсутствует. +Generator и compression implementation закреплены входами сборки. Распаковка +потоковая в новый private staging directory, не `extractall` в рабочий cache. +Дополнительные gzip members/trailing payload отклоняются. + +`metadata.json` содержит `schema_version`, `source_sha`, `corpus_id`, +`canonical_site_url`, `vector_model`, `vector_dim` и `files`. `files` — object +с ровно четырьмя ключами остальных имён; каждое значение содержит `sha256`, +`bytes` и для JSONL `rows` (непустые строки). Canonical site — происхождение +входных ссылок, не URL локальной установки и не новый источник сети. + +Content descriptor — весь metadata object **без** `corpus_id`; его canonical +JSON encoding: UTF-8, ключи sorted, separators `,` и `:`, без ASCII escaping, +без float/NaN и без завершающего newline. SHA-256 этих байтов — `corpus_id`. +Metadata не включает hash самого metadata или архива: цикла самохеширования +нет. Archive hash — отдельный delivery ID. Public/local publication одного +corpus используют одинаковые archive bytes; только manifest path отличается. + +## Семантическая проверка + +Pages и vectors сохраняют текущий формат строк, дополненный переносимыми +`site_path` и `markdown_path` страницы. Это относительные пути от SITE_URL без +origin, traversal или scheme. Существующие `url`/`markdown_url` канонического +сайта остаются в артефакте для совместимых потребителей. Runtime формирует +выходные URL по site paths и выбранному SITE_URL, а не доверяет этим origin. + +Внутренние ссылки в Markdown/HTML body, llms и выдаваемом pages Resource +перепривязываются по разобранным ссылочным узлам и catalog canonical paths. +Сохраняются fragment/query при допустимом локальном path. Внешние +`source_urls`, текст кода, строковые литералы, inline/fenced code не меняются. +Нельзя выполнять глобальный replace домена. Publisher проверяет отсутствие +неразрешённых внутренних ссылок; reader проверяет границы путей. Старые stable +ID, aliases, relations и фильтрация support pages сохраняются. + +Pages IDs уникальны; vector identity `(id, field, chunk_index)` уникальна и +ссылается на существующую страницу/чанк. Model/dim всех строк совпадают с +metadata и реализацией reader. Base64 имеет точную длину 256 float32, +компоненты конечны; `text_sha256` совпадает с соответствующим чанком по +сохранённым правилам генератора. Hash/size/count каждого файла совпадает с +metadata, а metadata — с manifest. Empty corpus, dangling vectors и частично +валидные поколения целиком отклоняются, а не подмешиваются к старому index. + +Presentation URL rebasing не меняет retrieval input и text hashes: +поиск/проверка векторов работают с каноническими данными, ссылки преобразуются +только на границе ответа. Локальные и публичные установки дают одинаковые +scores/IDs для одного запроса и corpus; различаются только site URLs. + +## Бюджеты загрузчика + +Стартовые safety limits, уточняемые только согласованной ревизией: + +| Объект | Предел | +|---|---| +| Manifest и metadata каждый | 64 KiB | +| Archive compressed | 16 MiB | +| Сумма распакованных файлов | 64 MiB | +| Pages / vectors | 16 / 32 MiB | +| llms / llms-full | 4 / 16 MiB | +| Одна JSONL строка | 1 MiB UTF-8 | +| Pages / vector rows | По 100 000 | +| Вся попытка обновления | 60 секунд monotonic deadline | +| Блокирующий сетевой read | Не более 20 секунд и остатка общего deadline | +| Cache на экземпляр volume | 256 MiB, включая staging и pinned generations | + +Content-Length проверяется, но не заменяет счётчик фактически прочитанных +байтов. Число JSON nesting levels ограничено 32; строки, числа и arrays +проверяются до построения index. Сжатый файл отдаётся как `application/gzip` +без `Content-Encoding: gzip`, чтобы HTTP decompression не меняла hash и бюджет. + +Если места недостаточно, refresh прекращается с ограниченной ошибкой. Active +и rollback-pinned файлы не удаляются ради загрузки нового; temporary/stale +unreferenced files убираются только в собственном cache namespace. Внутрипроцессное +потребление RAM staging+active измеряется отдельно: byte-limit архива не +является доказательством memory-limit Python index. + +## Cache, update и readiness + +Namespace = SHA-256 нормализованного SITE_URL плюс schema major. Startup +проверяет last-good generation и его файловые hashes до использования; при +повреждении пробует предыдущий валидный **того же** namespace. Проверенный +cache позволяет стать ready без доступной сети, refresh запускается фоном. + +Staging → verification → index construction → durable cache commit → atomic +process pointer swap. Query lock защищает только короткий swap, не сеть/парсинг/CPU build. +Файловый commit использует fsync и atomic rename; после crash active pointer +указывает либо на старую, либо на полностью записанную новую копию. Дисковый +pointer записывается до process pointer swap, а при ошибке записи старый +active сохраняется. Отдельный refcount удерживает поколение для текущих запросов. + +Один background updater на процесс; межпроцессная блокировка namespace +сериализует скачивание/commit при общем volume. Query не берёт этот lock. +Процесс, проигравший lock, повторно читает готовый cache вместо повторной +загрузки archive. Каждая копия процесса строит свой in-memory index. Блокировка +имеет timeout; crash владельца освобождает её средствами ОС. + +Refresh interval по умолчанию 3600 секунд с jitter ±20%; `--refresh-seconds 0` +отключает периодические обновления после bootstrap. Bootstrap при отсутствии +данных может повторяться с backoff. Ошибки используют backoff 30…3600 секунд +с jitter ±20%, один in-flight refresh, без бесконечного tight loop. Manifest +проверяется условным GET с ETag/Last-Modified. `304` применяется только при +наличии связанного валидного cache; иначе выполняется обычный GET. При +неизменном archive hash повторно скачивать архив не нужно. + +Stale last-good остаётся ready без искусственного срока годности. Exported +metadata отличает `loaded_at`, `last_checked_at`, `last_success_at` и +`refresh_error_code`. Логи не содержат raw corpus, procedure или секреты URL. +Для операционного rollback можно закрепить поколение локальным release state; +это внутренний механизм host, не второй public source variable. + +## Статическая доставка и публикация + +nginx читает отдельный read-only каталог immutable объектов; publisher имеет +право атомарно добавлять файлы, но runtime не может менять раздачу. GET/HEAD +поддерживают Content-Length, ETag и immutable cache headers. Directory listing +выключен. GET файла не создаёт backend connection к MCP. Префикс `/indexes/` +не проксируется в runtime и не получает его POST-only правила. + +Manifest публикуется с revalidation (`max-age=0, must-revalidate` там, где +публикатор управляет headers); на Pages проверяется фактический cache profile, +краткая задержка freshness допустима. Архивы хранятся минимум семь суток после +последней ссылки опубликованного manifest и дольше при rollback pin. GC +учитывает историю публикаций и pinned releases, а не только mtime. Текущий +manifest и rollback target никогда не ссылаются на удалённый объект. + +Download connection/byte budgets отделены от MCP admission, с учётом общего +NAT. Конкретные nginx/host лимиты выбираются по смешанному load test; лимит +RPS сам по себе не защищает полосу от большого файла. Публикация на том же host +переносит стоимость трафика с Pages, а не устраняет её. + +## Conformance + +Будущий module проверяет валидную пару source/public и source/local-prefix; +hostile manifest/redirect/archive; version/model/hash/row mismatches; +atomicity и crash на каждой cache стадии; запрет query-path network; +offline/source-switch/304 recovery; concurrent readers/updaters; URL rebasing +без порчи code/external provenance; nginx restart-independence и bounded I/O. diff --git a/spec/contracts/mcp-distribution-v1-r0.md b/spec/contracts/mcp-distribution-v1-r0.md new file mode 100644 index 0000000..daf8b11 --- /dev/null +++ b/spec/contracts/mcp-distribution-v1-r0.md @@ -0,0 +1,145 @@ +--- +schema_version: 1 +kind: contract +id: MCP_DISTRIBUTION +scope: product +version: 1 +revision: 0 +compatibility: backward-compatible +design: design:mcp-container-distribution +producer: v8std image publication pipeline +consumers: + - production runtime host + - docker run and Compose operators + - Docker MCP Catalog and Gateway +requirements: + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + - MCP_OFFLINE_USES_VERIFIED_CACHE + - MCP_DISTRIBUTION_PROVENANCE_IS_VERIFIABLE +governs: + - docker-compose + - Dockerfile.mcp + - Dockerfile.site + - deploy + - .github/workflows + - scripts/run_v8std_mcp.sh + - scripts/v8std_mcp_server.py + - overrides/main.html + - zensical.toml + - docs/mcp.md + - docs/support.md + - tests/test_v8std_mcp_distribution.py +conformance: + module: tests.test_v8std_mcp_distribution + command: .venv/bin/python -m unittest tests.test_v8std_mcp_distribution -v +required_when: implemented +supersedes: [] +deprecates: [] +--- + +# Официальная поставка проекта: образы и запуск + +## Артефакты + +Предлагаемые registry coordinates — `ghcr.io/zeegin/v8std-mcp` и +`ghcr.io/zeegin/v8std-site`; право организации публиковать public packages +проверяется до выпуска. Если реестр недоступен, это release gate, а не повод +создать неподтверждённый альтернативный production image. Образы поддерживают +`linux/amd64` и `linux/arm64`. Immutable `sha-` и release tags указывают +на multi-platform image index; production и release evidence используют digest +этого index с записью выбранного platform child digest. Mutable `latest` +допустим для discovery, но не доказательство точного выпуска. + +MCP image содержит только runtime, его закреплённые зависимости, лицензии и +OCI metadata исходного SHA. Не содержит Git checkout, docs generator, Zensical, +Pillow, corpus, private keys или build credentials. Публикация включает SBOM, +build provenance и проверяемую подпись/attestation доверенной workflow identity. +Production verifier не доверяет тегу или произвольной подписи из чужого workflow. +Base images и Actions закреплены по digest/SHA; обновление зависимостей проходит +тот же release путь. Лицензионная проверка данных отделена от лицензии runtime. + +Static-site image содержит готовый сайт local publication profile, manifest +и archive, HTTP server без docs build на старте. Такой же исходный corpus +идёт в Pages/public snapshot. Local profile исключает публичную аналитику, +рекордер и загрузку внешних fonts/assets; build-time получение зависимостей +не обещается офлайн. Runtime сайт после установки работает без интернета. + +## Интерфейс MCP image + +Entrypoint непосредственно запускает runtime, без генерации индексов и shell +install на старте. Default transport — `stdio`, explicit +`--transport streamable-http` — production/local HTTP. CLI имеет приоритет +над одноимённым env; невалидный explicit input завершает процесс с ненулевым +кодом до работы с сетью. CLI `--site-url`/env `V8STD_MCP_SITE_URL` имеют единый +смысл из snapshot contract; второго public base setting нет. + +Поддерживаются существующая настройка snippet limit и её порядок CLI/env, +`--cache-dir`/`V8STD_MCP_CACHE_DIR` и `--refresh-seconds`. Точное существующее +имя snippet env и диапазон наследуются из design крупных процедур; не вводится +параллельный alias. Cache path в образе — `/var/lib/v8std-mcp`, persistent volume +с документированным UID/GID. Образ работает non-root, read-only rootfs, +`cap-drop ALL`, `no-new-privileges`, без Docker socket, с writable cache и +ограниченным tmpfs. Stdio stdout чистый; healthcheck HTTP не применяется к stdio. + +HTTP слушает внутренний port 8000 на `0.0.0.0`; production публикует его только +на loopback для host nginx, local Compose по умолчанию только на loopback host. +TLS/public access остаются на nginx; open proxy к произвольному SITE_URL не +создаётся. SIGTERM прекращает admission и выполняет bounded drain; EOF stdio +завершает runtime и фоновые задачи. Docker использует init для дочерних процессов. + +Legacy direct Python developer/test entrypoints не удаляются в рамках смены +поставки. Старые независимые `--index-url`/`--vectors-url` не становятся +конфигурацией нового образа: их миграция явно описывается, совместное задание с +`--site-url` отклоняется. Внутренние unit-test fixtures могут использовать +локальные files; production loader не получает незаявленный legacy fallback. + +## Compose и общий адрес локального сайта + +Поставляется один документированный Compose без исходного bind mount: +static-site image + опциональный MCP HTTP image + persistent cache. Для сайта +без MCP не требуется запуск Python. Для одного MCP достаточно `docker run` или +каталога, сайт в Compose не обязателен при использовании публичного source. +Production использует тот же MCP image; nginx/TLS/storage host — окружение, +не другой MCP Dockerfile. Старый docs development Compose остаётся явно dev. + +Оператор выбирает один site URL, достижимый с host и из контейнера. Для desktop +пример использует опубликованный host port и проверенное разрешение +`host.docker.internal` на выбранной платформе; для LAN — общий DNS/адрес host. +Именно этот адрес виден и в ссылках ответов. Linux example включает явную +проверку host-gateway, а не предполагает desktop DNS. Site base-prefix и +redirects проверяются end-to-end. Никакой скрытой подмены source на `site` +с сохранением другого public URL не допускается. + +## Docker MCP Catalog + +Catalog entry ссылается на наш опубликованный image digest и точный source SHA; +в нём объявлены site URL, snippet limit и persistent cache volume. Требуется +реальный прогон через Gateway: initialize/tools/list/tool call, продолжительная +сессия, повторный запуск с warm cache и несколько сессий. Жизнь контейнера +не должна ограничиваться одним tool call; настройка `longLived` проверяется +по актуальной версии Gateway. Нельзя предполагать один контейнер на всех +агентов: Gateway может изолировать сессии. Shared volume экономит downloads, +не объединяет автоматически Python heaps независимых процессов. + +Каталог может временно указывать предыдущую проверенную версию, пока Docker +рассматривает обновление entry. Для каждого release image digest одинаков во +всех каналах этой версии; одновременное равенство версий каталога и production +не требуется. Новый production release не ожидает внешнего catalog review. + +Проверки каталога/лицензий и принятие Docker team — внешний gate. Наш образ +официальный для проекта v8std, но без отдельного принятия не называется +Docker Official Image или Docker-built. Отказ каталога не блокирует исправный +direct Docker и production путь и не даёт оснований сообщить о публикации там. + +## Приёмка + +Будущие container tests проверяют оба platform artifacts, non-root/read-only, +чистый stdio, реальный HTTP POST, сохранение лимитов и cache, cold-offline +неготовность, warm-offline работу и локальный сайт с запрещённым internet egress. +Egress-проверка браузера включает fonts/analytics, а не только MCP socket trace. +CI сравнивает image digests deployment references с опубликованным +артефактом соответствующей версии. Документация показывает проверенные команды, не пример сборки +пользователем ещё одного production образа. diff --git a/spec/contracts/mcp-release-runtime-v1-r0.md b/spec/contracts/mcp-release-runtime-v1-r0.md new file mode 100644 index 0000000..a2784fe --- /dev/null +++ b/spec/contracts/mcp-release-runtime-v1-r0.md @@ -0,0 +1,139 @@ +--- +schema_version: 1 +kind: contract +id: MCP_RELEASE_RUNTIME +scope: product +version: 1 +revision: 0 +compatibility: backward-compatible +design: design:mcp-container-distribution +producer: restricted host release controller +consumers: + - release invoker + - nginx edge + - operators and monitoring +requirements: + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_SHARED_HOST_LOAD_IS_MEASURED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE +governs: + - deploy + - scripts/v8std_mcp_server.py + - tests/test_v8std_mcp_release.py +conformance: + module: tests.test_v8std_mcp_release + command: .venv/bin/python -m unittest tests.test_v8std_mcp_release -v +required_when: implemented +supersedes: [] +deprecates: [] +--- + +# Транзакция выпуска MCP + +## Входная граница + +Invoker передаёт небольшой versioned JSON envelope: schema major, release ID, +монотонный sequence, полный source SHA, immutable image index digest, platform +child digest, configuration digest, совместимый corpus ID и deadline. Образ и +конфигурация проверяются по разрешённому namespace и происхождению; envelope +не содержит shell commands, arbitrary mount paths, environment passthrough +или writable path в static index store. Unknown schema и невалидные поля +отклоняются до побочных эффектов. + +Host credential разрешает только валидируемый release command и отдельную +ограниченную публикацию static artifacts, не общий root shell или docker group. +CI permissions и правила выдачи credential определены process design. Host +проверяет ожидаемую publisher identity/attestation, digest и конфигурацию; +произвольный образ из доверенного registry сам по себе не разрешён. + +Release ID идемпотентен: повтор с теми же полями возвращает текущий/конечный +результат; повтор с другими полями отклоняется. Sequence ниже уже принятого +не активируется. Rollback предыдущего image — часть текущей транзакции, не +старый CI job; ручной новый rollback request имеет новый sequence. Lock +сериализует releases и static manifest pointer updates, но не file GET. + +## Состояния и переходы + +```text +RECEIVED → VERIFIED → PREPARED → READY → SWITCHED → COMMITTED + └──────── до SWITCHED: FAILED, старый runtime продолжает работать + └─ smoke error → ROLLED_BACK +``` + +В VERIFIED сверяются authority, digest, compatibility и capacity. PREPARED +означает pull image, подготовленную конфигурацию и валидный corpus нового +runtime; старые image/config/data закреплены. В READY новый runtime на +внутреннем candidate port прошёл healthz и реальные initialize/tools/list/ +search/get_page/explain_snippet. Проверяются runtime SHA и corpus ID. + +SWITCHED — атомарная замена управляемого upstream include, успешный `nginx -t` +и reload. Если проверка конфигурации не прошла, старый include восстанавливается +до reload. Публичный smoke проверяет TLS, `/mcp`, правильный image/corpus и +сохранение static index download. COMMITTED наступает только после smoke, +затем старый runtime завершает ограниченный drain. + +Предлагаемые safety budgets: вся host-транзакция до 5 минут; readiness до +90 секунд в пределах общего deadline; post-switch smoke до 30 секунд; +drain старого runtime до 30 секунд и окончательный stop до 45 секунд. +Исчерпание бюджета не оставляет candidate активным без результата. Значения +проверяются на production-подобном стенде до включения автоматизации. + +Ошибка после switch возвращает predecessor config/upstream, reload и +проверяет старый endpoint; candidate прекращает admission и завершается. +Rollback failure — отдельный terminal `RECOVERY_REQUIRED` с alert и сохранёнными +артефактами; нельзя обозначать его как успешный rollback. Предыдущий runtime +не останавливается до успешного post-switch smoke нового. + +## Crash recovery и данные + +Release controller исполняется как ограниченная host job независимо от жизни +SSH. Durable journal фиксирует intent и каждый переход с fsync до необратимой +операции. На restart reconciler сравнивает journal, реальные контейнеры и nginx +upstream; непринятый SWITCHED release возвращается к predecessor. Два controller +не выполняют операции одновременно. Нельзя счесть отмену CI успешным deploy. + +Pinned generations и текущие cache pointers разделены. Candidate может скачать +и проверить новый corpus, но не меняет данные, нужные старому для rollback. +На время release автоматическое следование manifest не отбирает совместимый +rollback snapshot. После commit normal refresh возобновляется; unsupported +schema/model оставляет последний валидный index и выдаёт сигнал оператору. + +Predecessor включает не только image, но и configuration digest, cache schema +и валидный corpus; предыдущий image не обязан читать будущую schema. Retention +хранит хотя бы один успешно работавший predecessor и все in-flight pins. +Первые migration/startup не выдают отсутствие predecessor за гарантию rollback: +до начального container cutover сохранены Python deployment и его конфигурация, +данные и проверенный путь возврата. После проверки первого container release +формируется обычная цепочка predecessors. + +## Host и нагрузка + +nginx static store не находится внутри runtime container или MCP cache. +Контейнеры слушают loopback; наружу открыты только необходимые TLS/MCP и +административные порты согласно host inventory. Сертификаты, renewal и +мониторинг проверяются независимо от application migration. Systemd active +wrapper без готового контейнера не удовлетворяет readiness. + +Перед переключением capacity check проверяет disk headroom для pull/staging, +RAM для old+new runtime и index preparation, file descriptors и сетевой бюджет. +При дефиците switch не начинается. Admission ограничивает одновременно +выполняемые MCP запросы, idle keep-alive и большие downloads раздельно; +перегрузка отвечает retryable status по прежней edge policy. Конкретные +настройки допускаются в production только после mixed-load evidence. + +## Наблюдаемость и conformance + +Structured result содержит release ID/sequence, exact digests/SHA/corpus, +время, конечное состояние и компактный error code. Secrets, raw procedures и +arbitrary command output не попадают в public status. Мониторинг различает +runtime liveness, readiness, stale corpus, refresh failures, deploy/rollback +failures, memory/FD exhaustion и static egress. Расширенный публичный dashboard +из других designs не объявляется реализованным этим контрактом. + +Будущий module плюс стенд проверяют duplicate/stale/concurrent jobs, failed +pull/signature/config, corrupt corpus, readiness timeout, invalid nginx config, +post-switch failure, SSH drop, CI cancellation, kill/reboot между каждым +переходом и rollback failure. Сравниваются реальные endpoint/digest/state, +а не только exit code controller. Отдельный mixed-load отчёт фиксирует +предельную проверенную нагрузку; 100 000 подключений пока являются целью. diff --git a/spec/designs/2026-09-10-mcp-ci-deployment-policy-design.md b/spec/designs/2026-09-10-mcp-ci-deployment-policy-design.md new file mode 100644 index 0000000..10ec445 --- /dev/null +++ b/spec/designs/2026-09-10-mcp-ci-deployment-policy-design.md @@ -0,0 +1,118 @@ +--- +schema_version: 1 +kind: design +id: mcp-ci-deployment-policy +scope: process +requirements: + introduces: + - MCP_SERVER_DEPLOYS_AUTOMATICALLY_FROM_VERIFIED_MAIN + - MCP_AUTODEPLOY_ACTIVATION_IS_CONTROLLED + uses: + - ALL_CHANGES_USE_BRANCHES + - MAIN_ACCEPTS_ONLY_VALIDATED_MERGES + - TRIVIALITY_IS_ASSESSED_NOT_ASSUMED + - ARCHITECTURE_IMPACT_IS_RECHECKED + - REQUIREMENTS_ARE_TRACEABLE + - ARCHITECTURE_DECISIONS_ARE_ATOMIC + - ADR_IDENTITIES_ARE_SEMANTIC + - ARCHITECTURE_INVARIANTS_ARE_SEMANTIC + - OBSERVABLE_BOUNDARIES_ARE_CONTRACTED + - ARCHITECTURE_DOCUMENTS_ARE_IMMUTABLE + - PROJECT_ERRORS_TRIGGER_COMPREHENSIVE_REVIEW + - DESIGN_APPROVAL_PRECEDES_IMPLEMENTATION + - ARCHITECTURE_PROCESS_IS_MACHINE_VALIDATED + - INTERNAL_SPECIFICATIONS_STAY_UNPUBLISHED + - SITE_DEPLOYS_AUTOMATICALLY_FROM_MAIN + - EXISTING_SPECIFICATIONS_USE_ONE_MODEL + - SUPERPOWERS_DRIVES_DESIGN_AND_PLANNING + - PRODUCT_ARCHITECTURE_EXCLUDES_DEVELOPMENT_PROCESS + replaces: + MCP_SERVER_DEPLOYMENT_REQUIRES_EXPLICIT_REQUEST: MCP_SERVER_DEPLOYS_AUTOMATICALLY_FROM_VERIFIED_MAIN + cancels: [] +decisions: [] +invariants: [] +contracts: [] +supersedes: + - design:v8std-architecture-process +cancels: [] +--- + +# Политика автоматической поставки MCP из CI + +## Согласованное изменение + +Пользователь прямо выбрал автоматическое обновление ai.v8std.ru из GitHub CI +и использование того же опубликованного образа, что для локальной поставки. +Это меняет прежнее правило отдельного ручного deploy на каждый выпуск. +Нельзя одновременно сохранить запрет автодеплоя и обещать автоматическую +поставку после push. Новый process v2 фиксирует явного преемника, не редактируя +замороженные design/process v1. + +Продуктовая топология и rollback описаны в +[продуктовом design](2026-09-10-mcp-container-distribution-design.md). +Git rules, разрешения CI и approval gates не становятся продуктовыми ADR. + +## Требования + +### MCP_SERVER_DEPLOYS_AUTOMATICALLY_FROM_VERIFIED_MAIN + +После управляемой активации разрешённый push проверенного `main` запускает +публикацию и при необходимости автоматический rollout MCP без отдельной +ручной команды на каждый SHA. CI использует только прошедший gates SHA из +защищённого `main` и точный опубликованный digest. PR, fork, tag, имя ветки +из входного параметра и mutable `latest` не являются полномочием deploy. +Контентные изменения обновляют corpus без рестарта runtime. + +Проверка: интеграционная матрица main/PR/fork/tag/устаревший run; негативные +случаи не получают production credentials и не меняют host. Успешный release +проверяется на том же digest, который поступил в registry. + +### MCP_AUTODEPLOY_ACTIVATION_IS_CONTROLLED + +Согласование design не устанавливает Docker, не удаляет старый сайт и не +выдаёт secrets. Первичное включение требует отдельного операционного этапа: +проверить host, backup/restore, точные targets, restricted deploy identity, +protected main и production environment, затем подтвердить первый rollout +и возможность отключить автоматизацию. До выполнения этапа действует прежняя +операционная граница: не деплоить без явного запроса. + +Проверка: до activation marker/секретов workflow может собрать и проверить +артефакты, но не может менять production; после активации failed gates, stale +release и не-main источники блокируются. Kill switch запрещает новые releases, +не уничтожает работающий endpoint и не прерывает host rollback. + +## Нормативный преемник и активация + +[Process v2](../process/architecture-artifacts-v2.md) сохраняет схему графа, +заморозку документов, отдельную ветку, основной checkout, согласование design, +plan, semantic impact, tests и strict build. Push локального main по-прежнему +требует явного запроса; PR и worktree — только по явному запросу. Здесь такой +отдельный PR для поставки был запрошен, но его создание не входит в запись +этого пакета. + +Текущий CLI загружает schema из `architecture-artifacts-v1.md`. Новая версия +имеет ту же schema; только документация v2 не переключает CLI или поведение CI. +В implementation plan должны войти согласованные изменения указателя loader, +`AGENTS.md`, `spec/README.md`, repo skill, policy tests и workflows. Их единый +diff обязан исключить одновременно действующие противоречивые инструкции. +Старые structured files, включая завершённый deployment-boundary plan, остаются +историческими свидетельствами, не редактируются. + +## Порядок CI и доказательства + +PR gates выполняются без production secrets. Сборка из main использует +закреплённые actions и минимальные permissions; доступ к registry/deploy +выдаётся только соответствующему job. GitHub-hosted runner не исполняет +непроверенный PR на production host. Production environment ограничивает main; +защита main требует успешных обязательных checks и контролируемых bypass. + +Image публикуется до deploy и проверяется по digest. Состояние host хранит SHA, +digest, config hash, corpus ID, монотонный release sequence и результат smoke. +Один deploy за раз; устаревший job не отменяет более новый. Отмена CI не +обрывает ограниченную host-транзакцию переключения/rollback. + +Merge-ready plan содержит только реализуемые до merge задачи и проверяемые +на стенде доказательства. Последующие merge, разрешённый push, первичная +активация и реальные внешние deployments отмечаются операционными результатами +отдельно. Нельзя отметить host migration или принятие Docker Catalog как +сделанные по зелёным unit tests. diff --git a/spec/designs/2026-09-10-mcp-container-distribution-design.md b/spec/designs/2026-09-10-mcp-container-distribution-design.md new file mode 100644 index 0000000..8238abd --- /dev/null +++ b/spec/designs/2026-09-10-mcp-container-distribution-design.md @@ -0,0 +1,320 @@ +--- +schema_version: 1 +kind: design +id: mcp-container-distribution +scope: product +requirements: + introduces: + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_RUNTIME_AND_CORPUS_RELEASE_INDEPENDENTLY + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_SNAPSHOT_LOAD_IS_ATOMIC + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNAPSHOT_IO_IS_BOUNDED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_SHARED_HOST_LOAD_IS_MEASURED + - MCP_OFFLINE_USES_VERIFIED_CACHE + - MCP_DISTRIBUTION_PROVENANCE_IS_VERIFIABLE + uses: + - MCP_COMBINED_PAGE_READING_COMPATIBLE + - MCP_LEGACY_VERSION_REMAINS_COMPATIBLE + - MCP_POST_ONLY_AGENT_TRANSPORT + - MCP_EDGE_CONNECTION_CAPACITY + - MCP_AGENT_REQUESTS_SCALE_HORIZONTALLY + - MCP_OVERLOAD_RETURNS_RETRYABLE_STATUS + - MCP_WORKER_DRAIN_IS_BOUNDED + - MCP_RESOURCE_VERSION_PAGE_READING_USES_RESOURCES + - MCP_RESOURCE_CATALOG_IS_PAGINATED + - MCP_RESOURCE_LIST_USES_STABLE_SNAPSHOTS + - MCP_RESOURCE_NOTIFICATIONS_ARE_OMITTED + - MCP_RESOURCE_LINKS_RESOLVE_TO_LISTED_RESOURCES + - MCP_RESOURCES_EXCLUDE_SUPPORT_PAGES + - MCP_TEMPLATES_EXCLUDE_LANGUAGE_AND_METHOD_SOURCES + - MCP_SNIPPET_ACCEPTED_INPUT_IS_SCANNED + - MCP_SNIPPET_TARGETS_SURVIVE_QUERY_BUDGET + - MCP_SNIPPET_INSTANCE_LIMIT_IS_DISCOVERABLE + - MCP_SNIPPET_RETRIEVAL_WORK_IS_BOUNDED + - MCP_SNIPPET_RESPONSE_STAYS_COMPACT + replaces: + MCP_COMBINED_ENDPOINT_CAPABILITIES: MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + cancels: [] +decisions: + - adr:MCP_PUBLISHED_COMBINED_RUNTIME + - adr:MCP_ATOMIC_SITE_SNAPSHOTS + - adr:MCP_RECOVERABLE_CONTAINER_RELEASE + - adr:SNIPPET_SIGNALS_OUTSIDE_TEXT_QUERY +invariants: + - invariant:MCP_PUBLISHED_RUNTIME_IS_ONE_SERVICE + - invariant:MCP_ACTIVE_SNAPSHOT_IS_COMPLETE + - invariant:MCP_SITE_OVERRIDE_HAS_NO_PUBLIC_FALLBACK + - invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR + - invariant:MCP_SNIPPET_SIGNALS_SURVIVE_TEXT_BUDGET +contracts: + - contract:MCP_API@2.4 + - contract:MCP_CORPUS_SNAPSHOT@1.0 + - contract:MCP_DISTRIBUTION@1.0 + - contract:MCP_RELEASE_RUNTIME@1.0 +supersedes: + - design:mcp-combined-endpoint + - design:mcp-large-procedure-retrieval +cancels: [] +--- + +# Единая поставка MCP, сайта и индексов + +## Решение и граница этой работы + +Пользователь согласовал собственный опубликованный MCP-образ для локальных +агентов и production, локальный сайт в Compose, одну настройку сайта, +раздачу индексов с `ai.v8std.ru` и автоматический production deployment из CI. +Этот пакет переводит согласование в проверяемый проект. Он ещё не является +реализацией, заявкой в Docker Catalog или разрешением удалить данные сервера. +Числовые бюджеты загрузчика ниже — предлагаемые параметры письменного пакета, +не результаты нагрузочных испытаний. + +База анализа: `main` `b7bef11e145a188b30e7a7b17df2be4cb1acbd0c`. +В этой поставке нет отдельного v3, нового поискового движка, расширения +Resources или обработки целых репозиториев. Сохраняются алгоритм крупных +процедур, ограничения ответа и пять существующих tools. Политика автоматического +деплоя вынесена в [процессный design](2026-09-10-mcp-ci-deployment-policy-design.md). + +## Потоки артефактов + +```text +проверенный SHA main + ├─ сборка runtime → опубликованный MCP image digest + │ ├─ production HTTP /mcp + │ ├─ локальный stdio / HTTP + │ └─ Docker Catalog → тот же образ + └─ сборка контента → неизменяемый snapshot + ├─ nginx ai.v8std.ru/indexes/v1//snapshot.tar.gz + │ ↑ manifest публичного сайта на GitHub Pages + └─ static-site image с локальным manifest и snapshot + ↑ локальный сайт в Compose + +MCP → V8STD_MCP_SITE_URL → manifest → проверенный локальный snapshot + └─ URL ответов от того же SITE_URL +``` + +MCP скачивает подготовленный набор данных, а не обходит HTML сайта. +Обычные tool calls не обращаются к сайту или серверу индексов. nginx раздаёт +готовые файлы, не запускает генератор и не вызывает Python MCP. + +## Почему нужна смена механизма, а не только Dockerfile + +Текущий индекс обновляется синхронно под блокировкой из поискового пути, +pages и vectors загружаются независимо, cache не разделён по источнику. +Упаковка этого кода в контейнер сохраняет сетевые задержки, смешение поколений +и риск использования данных другого сайта. Новый загрузчик устраняет эти +причины до переключения production. + +Текущий docs-builder содержит генераторы, зависимости сайта и исходники. +Runtime-образ получает только сервер и runtime-зависимости; corpus не зашит +в него. Поэтому обновление статьи не пересобирает MCP и не перезапускает его. +Самодостаточный первый запуск без сети достигается локальным сайтом либо +заранее проверенным cache, а не пустым thin image. Это отличие от предложения +PR #33 и ожидания cold-offline в issue #32 необходимо честно указать при закрытии. + +## Требования + +### MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + +Все способы контейнерного запуска используют один релизный multi-platform +image index в рамках одного выпуска; production закрепляет digest, а не mutable +tag. Один публичный `/mcp` сохраняет совмещённые возможности. Проверка +сопоставляет каждый способ запуска с опубликованным digest его версии. +Каталог может отставать по версии из-за внешнего review: это не другая сборка +той же версии и не причина задерживать production security update. +В устойчивом production-состоянии один +активный экземпляр; при переключении допускаются старый и новый на внутренних +портах с ограниченным drain. Это явная замена буквального требования одного +процесса/systemd unit, а не возвращение разделения v2/v3. + +### MCP_RUNTIME_AND_CORPUS_RELEASE_INDEPENDENTLY + +Контентный commit публикует новый snapshot без перезапуска MCP; изменение +runtime публикует образ без обязательной смены corpus. Матрица двух типов +commit подтверждает отсутствие лишнего rollout. Совместимость schema/model +проверяется до активации; изменение формата сначала выпускает совместимый +reader, затем новый manifest. + +### MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + +`V8STD_MCP_SITE_URL` задаёт bootstrap manifest и основу URL ответов. Значение +по умолчанию — `https://v8std.ru/`. Второй `V8STD_MCP_PUBLIC_BASE_URL` не вводится. +Тест с сайтом под `/knowledge/` проверяет источник, article/Markdown URLs и +внутренние ссылки в телах ответов. Внешние ссылки-первоисточники не переписываются. + +### MCP_SNAPSHOT_LOAD_IS_ATOMIC + +Pages, vectors и тексты Resources относятся к одному проверенному поколению. +Запрос удерживает один immutable snapshot на всё время обработки. Повреждённый +архив, неизвестная схема или исчезнувший файл не меняют active snapshot. +Параллельные запросы во время refresh видят либо старое, либо новое поколение. + +### MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + +Обновление выполняет один фоновый координатор на процесс, а не поиск, +`resources/read` или `get_page`. Тест замедляет источник на 60 секунд и +проверяет отсутствие сетевого I/O и ожидания refresh в рабочих запросах. +Тяжёлая подготовка поколения также не блокирует event loop; влияние CPU/GIL +и пик памяти проверяются нагрузкой, выбор executor фиксируется в plan. + +### MCP_SNAPSHOT_IO_IS_BOUNDED + +Manifest, скачивание, распаковка, строки JSONL, количество записей, векторы, +время и место на диске имеют отдельные пределы из snapshot contract. Тесты +включают gzip bomb, traversal, медленный поток, oversized record и повторные +ошибки; последняя рабочая копия не повреждается. + +### MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + +GET опубликованного immutable snapshot обслуживается nginx при остановленном +MCP. Он не зависит от MCP cache или временного каталога контейнера. Проверка +останавливает только runtime и сравнивает status, hash и длину файла. + +### MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + +Локальная публикация поставляет свой manifest, snapshot и необходимые assets. +Заблокированный внешний интернет не мешает MCP работать с доступным локальным +сайтом и браузеру открывать его: нет фоновой аналитики, remote fonts и скрытого +fallback на публичный corpus. Пользовательский переход по внешнему источнику +не является фоновым запросом и не запрещается. + +### MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + +Образ поддерживает stdio для контейнерных агентов и stateless Streamable HTTP +для production. stdout в stdio содержит только протокол; завершение stdin и +SIGTERM прекращают фоновые задачи. Warm cache переиспользуется между запусками. +Проверки включают cold/warm start, reconnect и несколько независимых сессий +агента, в том числе через Docker Gateway; контейнер на каждый tool call не нужен. + +### MCP_RELEASE_SWITCH_IS_REVERSIBLE + +Ошибки pull, подготовки corpus, readiness, переключения nginx и smoke после +переключения сохраняют или восстанавливают предыдущие image/config/data. +Потеря SSH и отмена CI не бросают host в промежуточном состоянии. Это +подтверждается fault-injection по каждому переходу release contract. + +### MCP_SHARED_HOST_LOAD_IS_MEASURED + +На production-подобном стенде одновременно воспроизводятся короткие MCP POST, +idle keep-alive, reconnect, общий NAT, refresh, скачивание индексов и rollout. +Публикуются hardware, число соединений и активных запросов, RPS, p95/p99, +ошибки, CPU/RAM, file descriptors и полоса. Цель 100 000 подключений не считается +достигнутой по значению `worker_connections` или по числу idle TCP sockets. + +### MCP_OFFLINE_USES_VERIFIED_CACHE + +После успешной синхронизации тот же источник работает без сети из persistent +cache. Cold start без доступного источника и без валидного cache возвращает +явную неготовность, не подменяет данные другим сайтом. Тест проверяет отдельно +cache другого origin, повреждённый cache, stale cache и recovery после сети. + +### MCP_DISTRIBUTION_PROVENANCE_IS_VERIFIABLE + +Образ и snapshot связываются с проверенным SHA и воспроизводимыми входами +сборки; runtime dependencies закреплены, образ имеет SBOM и provenance. +Проверяется поставляемая лицензия кода, данных и зависимостей. Публикация +собственного образа не выдаётся за статус Docker-built/Docker Official Image. + +## Контракты и сохранённые решения + +- [API 2.4](../contracts/mcp-api-v2-r4.md): прежний surface, stdio и readiness. +- [Snapshot 1.0](../contracts/mcp-corpus-snapshot-v1-r0.md): формат, URL, cache, + лимиты, consistency и правила переключения поколения. +- [Distribution 1.0](../contracts/mcp-distribution-v1-r0.md): образы, параметры, + локальный Compose и Docker Catalog. +- [Release runtime 1.0](../contracts/mcp-release-runtime-v1-r0.md): транзакция + переключения и восстановление. + +Из предшествующих combined и large-procedure designs принимаются без +изменения правила чтения страниц и весь алгоритм snippet: полный анализ +допустимого входа, отдельные сигналы, один ограниченный поиск, приоритет целей, +лимиты ответа, discovery и отсутствие исходного кода в usage logs. Их +математическая и тестовая спецификация остаётся в исторических документах. +Смена design нужна из-за ссылки на заменённое топологическое требование, а не +из-за отмены исправления PR #31. ADR `SNIPPET_SIGNALS_OUTSIDE_TEXT_QUERY` +сохраняется; его ссылка на прежний single-runtime invariant читается через +явного преемника этого инварианта. + +Существующие три bulk Resources остаются совместимыми, но новые полные +каталоги и greedy discovery в эту поставку не добавляются. Их удаление или +смена политики потребует отдельного согласования. Новые snapshots — служебная +доставка corpus экземпляру MCP, а не новые MCP Resources для агента. + +## Последовательность публикации и отказов + +Для контента CI сначала строит и проверяет snapshot, загружает неизменяемый +файл на ai-host, проверяет его извне, затем публикует Pages с manifest этого +файла. Если загрузка или проверка неудачны, новый Pages manifest не публикуется. +Если Pages deployment не состоялся, остаётся безопасный неиспользуемый объект. +Local-site image содержит ту же версию corpus и локальную ссылку на архив; +публичный и локальный HTML могут различаться только publication profile. +Это необходимо для удаления публичной аналитики и внешних fonts локально. + +Работающий MCP при временном отказе manifest или archive продолжает использовать +last-good snapshot. Runtime rollout не требует одновременного Pages deployment; +активное поколение и поколение для rollback остаются совместимыми с обоими +образами на период переключения. + +## Production и внешние условия выпуска + +Docker migration требует отдельной подготовки целевого сервера и свежего +замера ресурсов до начала работ. +Одновременные old/new runtime плюс staging нового index должны поместиться +с запасом; иначе до включения rollout нужен более ёмкий host. Увеличение лимита +nginx не устраняет дефицит памяти или полосы. + +На выделенном host остаются ai.v8std.ru, SSH, TLS renewal, защита и мониторинг. +Посторонний vhost нельзя удалить вслепую: сначала backup вне host, проверка +зависимости default TLS от его сертификата, затем согласованный cleanup и smoke +ai/TLS/monitoring. Никакая стадия CI не удаляет посторонние сайты автоматически. + +Три границы приёмки нельзя смешивать: + +1. Локальные gates: воспроизводимая сборка, новый код и contracts, fixture и + container tests, security checks, смешанная нагрузка на стенде. +2. Управляемая активация: production protection/secrets, Docker, storage, + firewall, backup/rollback, начальный digest и отдельное включение CI deploy. +3. Внешняя поставка: доступный registry artifact, post-deploy evidence, review + Docker Catalog. Его сроки и принятие не зависят только от нашего PR. + +Новый PR в v8std оформляется отдельно от PR #33. Закрывать #33 как альтернативно +решённый следует после merge, публикации и проверки локального/production +пути, с благодарностью автору и точным отличием cold-offline. Принятие в Docker +Catalog отмечается отдельно; issue #32 не закрывается автоматически. Внешние +комментарии содержат результат, а не внутренние неудачи проверок. + +## Влияние и следующий gate + +Архитектурное влияние нетривиально: lifecycle источника, формат corpus, +границы сети и cache, процесс агента, схема release и его авторизация. +Будущие пути: MCP server/index, генераторы AI artifacts, Docker/Compose, +nginx/systemd, workflows, local-site profile, docs и tests. В этой ветке +изменяются только новые файлы `spec/`; старые structured documents не правятся. + +После просмотра письменного пакета создаётся отдельный implementation plan +со срезами loader → containers → publication → host migration → release +verification. Это порядок проектирования, не готовый plan и не разрешение +начать implementation до следующего gate. Объявленные будущие fitness-модули +в contracts/invariants ещё не существуют; `required_when: implemented` не +позволяет представить их как доказательства готовой реализации. + +## Проверенные внешние границы + +На 2026-09-10 Docker Registry допускает собственный образ через `--image`, +но включение требует review команды Docker; параметры надо объявлять в +catalog config. Перед отправкой нужен повторный live-check правил: +[Docker contribution guide](https://github.com/docker/mcp-registry/blob/main/CONTRIBUTING.md). +Выбран self-built путь, поэтому происхождение/SBOM обеспечивает наш CI. + +GitHub environment и branch restrictions — отдельные защитные настройки, +их нельзя считать существующими из наличия YAML: +[GitHub deployment environments](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments). +Для файловой раздачи используется nginx: +[nginx sendfile](https://nginx.org/en/docs/http/ngx_http_core_module.html#sendfile). +Сам перенос трафика с Pages на целевой сервер не увеличивает доступную полосу host. diff --git a/spec/invariants/mcp-active-snapshot-is-complete.md b/spec/invariants/mcp-active-snapshot-is-complete.md new file mode 100644 index 0000000..a7fc8c6 --- /dev/null +++ b/spec/invariants/mcp-active-snapshot-is-complete.md @@ -0,0 +1,36 @@ +--- +schema_version: 1 +kind: invariant +id: MCP_ACTIVE_SNAPSHOT_IS_COMPLETE +scope: product +introduced_by: adr:MCP_ATOMIC_SITE_SNAPSHOTS +requirements: + - MCP_SNAPSHOT_LOAD_IS_ATOMIC + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNAPSHOT_IO_IS_BOUNDED + - MCP_OFFLINE_USES_VERIFIED_CACHE +owner: v8std maintainers +governs: + - scripts/v8std_mcp_index.py + - scripts/v8std_mcp_snapshots.py +check: + module: tests.test_v8std_mcp_snapshots + command: .venv/bin/python -m unittest tests.test_v8std_mcp_snapshots -v +required_when: implemented +--- + +# Готовый MCP обслуживает запрос одним полным поколением + +Ready runtime имеет валидный неизменяемый snapshot одного source namespace: +pages, vectors и Resource texts проверены по одному descriptor. Каждый запрос +удерживает это поколение до завершения. Ни один запрос не инициирует сетевое +обновление и не ждёт его lock. Background verification/parse/build не +модифицируют active generation. + +Сбой загрузки, превышение любого бюджета, неизвестная schema и crash записи +не подменяют active частичным или чужим corpus. Если рабочего поколения нет, +runtime явно не готов; пустой corpus не имитирует успешную готовность. + +Fitness — будущие concurrent-query generation assertions и fault-injection +каждой стадии fetch/verify/build/commit, с offline restart и bounded-resource +проверками. Медленный источник не добавляет сетевое ожидание в tool latency. diff --git a/spec/invariants/mcp-published-runtime-is-one-service.md b/spec/invariants/mcp-published-runtime-is-one-service.md new file mode 100644 index 0000000..63fa1b8 --- /dev/null +++ b/spec/invariants/mcp-published-runtime-is-one-service.md @@ -0,0 +1,39 @@ +--- +schema_version: 1 +kind: invariant +id: MCP_PUBLISHED_RUNTIME_IS_ONE_SERVICE +scope: product +introduced_by: adr:MCP_PUBLISHED_COMBINED_RUNTIME +requirements: + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE +owner: v8std maintainers +governs: + - scripts/v8std_mcp_server.py + - Dockerfile.mcp + - docker-compose + - deploy +check: + module: tests.test_v8std_mcp_distribution + command: .venv/bin/python -m unittest tests.test_v8std_mcp_distribution tests.test_v8std_mcp_combined -v +required_when: implemented +--- + +# Один логический сервис из опубликованного runtime image + +В production существует один публичный `/mcp` без самостоятельного v3 +endpoint/listener/profile. В устойчивом состоянии один активный runtime; +ограниченный overlap old/new при release не создаёт второго пользовательского +сервиса. Каждый serving container относится к проверенному опубликованному +release digest, старому или новому; произвольная локальная сборка не участвует. + +Local stdio/HTTP и catalog используют тот же multi-platform image для той же +версии, с допустимым различием platform child digest. Отставание версии catalog +из-за внешнего review не создаёт вторую сборку и не блокирует production release. +Пять tools, существующие Resources и +snippet-поведение совпадают. CPU architecture, транспорт и конфигурация сайта +не выбирают другую кодовую реализацию MCP. + +Fitness — будущие проверки digest/reference parity, surface parity, отсутствие +v3 route, warm agent session и ограниченный old/new overlap. Наличие этого +файла не означает, что контейнеры уже собраны или опубликованы. diff --git a/spec/invariants/mcp-release-has-recoverable-predecessor.md b/spec/invariants/mcp-release-has-recoverable-predecessor.md new file mode 100644 index 0000000..fd1db74 --- /dev/null +++ b/spec/invariants/mcp-release-has-recoverable-predecessor.md @@ -0,0 +1,34 @@ +--- +schema_version: 1 +kind: invariant +id: MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR +scope: product +introduced_by: adr:MCP_RECOVERABLE_CONTAINER_RELEASE +requirements: + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_SHARED_HOST_LOAD_IS_MEASURED +owner: v8std maintainers +governs: + - deploy +check: + module: tests.test_v8std_mcp_release + command: .venv/bin/python -m unittest tests.test_v8std_mcp_release -v +required_when: implemented +--- + +# Переключение сохраняет восстанавливаемого предшественника + +До успешного public smoke новый runtime не уничтожает прежние image/config +и совместимый corpus. Одновременно выполняется не более одной host release +transaction. Повтор и устаревшее задание не откатывают более новый успешный +release. Потеря invoker не отменяет recovery на host. + +Сбой до switch оставляет старый serving runtime. Сбой после switch приводит +к проверяемому rollback либо явному `RECOVERY_REQUIRED`, но не ложному успеху. +При нехватке памяти для old/new overlap или диска для pinned data переключение +не начинается. Для первого container cutover predecessor — сохранённый и +проверенный старый Python deployment; он не считается обычным container release. + +Fitness — будущая матрица faults на переходах release contract, с проверкой +реального endpoint, exact digest и данных после восстановления. Unit mock +успешного `docker restart` не доказывает этот инвариант. diff --git a/spec/invariants/mcp-site-override-has-no-public-fallback.md b/spec/invariants/mcp-site-override-has-no-public-fallback.md new file mode 100644 index 0000000..3e31ba3 --- /dev/null +++ b/spec/invariants/mcp-site-override-has-no-public-fallback.md @@ -0,0 +1,36 @@ +--- +schema_version: 1 +kind: invariant +id: MCP_SITE_OVERRIDE_HAS_NO_PUBLIC_FALLBACK +scope: product +introduced_by: adr:MCP_ATOMIC_SITE_SNAPSHOTS +requirements: + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS +owner: v8std maintainers +governs: + - scripts/v8std_mcp_snapshots.py + - scripts/v8std_mcp_index.py + - docker-compose + - overrides/main.html + - zensical.toml +check: + module: tests.test_v8std_mcp_distribution + command: .venv/bin/python -m unittest tests.test_v8std_mcp_distribution tests.test_v8std_mcp_snapshots -v +required_when: implemented +--- + +# Выбранный локальный сайт определяет источник и ссылки + +Для нестандартного SITE_URL bootstrap, archive и их redirects остаются в +разрешённых origin/base path выбранного сайта. Внутренние article/Markdown +ссылки выдачи используют этот же base. Отказ local source не разрешает запрос +к публичному корпусу, reuse его cache или возврат публичных article URLs. + +Local-site publication не выполняет фоновые запросы analytics/recorder/fonts +в public internet. Внешние provenance links остаются ссылками; их явное +открытие пользователем и build-time dependency downloads не входят в запрет. + +Fitness — будущий network-deny integration test local-site+MCP+browser, +cross-origin redirect rejection, смена SITE_URL с cache и body/code link fixtures. +Проверка только env или только верхнего `url` недостаточна. diff --git a/spec/operations/2026-09-09-markdown-rendering-verification.md b/spec/operations/2026-09-09-markdown-rendering-verification.md index 8295e04..7f1fd52 100644 --- a/spec/operations/2026-09-09-markdown-rendering-verification.md +++ b/spec/operations/2026-09-09-markdown-rendering-verification.md @@ -126,9 +126,8 @@ HTTP body before/after SHA-256: `06cf99bbcab3a96a58f8afe006d594c86eabdaae2e342d133c3aa8a077e79876`. Evidence: `serve.json`, `serve.log`, `serve-initial.html`, `serve-rebuilt.html`. -Docker context was explicitly `desktop-linux`, endpoint -`unix:///Users/ingvarvilkman/.docker/run/docker.sock`; client/server preflight -29.7.2. Real local Docker build, image import from `/tmp` with +Docker context was explicitly `desktop-linux`, using its local Unix socket; +client/server preflight 29.7.2. Real local Docker build, image import from `/tmp` with `PYTHONPATH=/opt/v8std`, and project-config render through a read-only `/docs` mount all exited 0. The rendered compact BSL fixture contained standalone before/code/after blocks and passed `check_html` (1 article, 0 violations). diff --git a/spec/operations/2026-09-09-mcp-incident.md b/spec/operations/2026-09-09-mcp-incident.md index ac28737..f1d0697 100644 --- a/spec/operations/2026-09-09-mcp-incident.md +++ b/spec/operations/2026-09-09-mcp-incident.md @@ -79,17 +79,17 @@ References: [nginx core directives](https://nginx.org/en/docs/ngx_core_module.ht - Deployed configuration source: local main commit `d707daa9b6d487ae4cf9d93d6c34ff86fa6b5793`. Only the emergency nginx patch and map were applied. No Git push or application update occurred. -- Backup/evidence directory (root-only): - `/root/v8std-incident-20260909-kkl40C` on ai.v8std.ru. +- Backup/evidence was retained in a root-only directory; its location is + recorded privately. - At 17:59:50, exact patch dry-run and host nginx 1.24.0 `nginx -t` passed. - Existing unrelated OCSP warning for 0x1c.ru remained. + An existing OCSP warning unrelated to MCP remained. - Restart began at 17:59:58 and completed at 18:00:08 UTC (21:00:08 MSK). systemd MainPID and nginx pid-file both became 643435. Two workers stuck in shutdown and the obsolete master were cleared by the unit restart. - Established TCP connections dropped from 1,133 to 6 immediately, then 41 after 33 seconds of live reconnects. Python FDs dropped from 384 to 8. - External health, browser GET, HEAD, initialize, tools/list, search and page - retrieval passed. Monitoring and 0x1c.ru returned 200. SSE GET returned 405 + retrieval passed. Monitoring and unrelated service checks returned 200. SSE GET returned 405 with `Allow: POST, HEAD` in 0.364 seconds. - Real Python MCP SDK 1.27.0 completed initialize -> list five tools -> search with protocol 2025-11-25 and no tool error. diff --git a/spec/plans/2026-07-22-unified-diagnostic-chips-plan.md b/spec/plans/2026-07-22-unified-diagnostic-chips-plan.md index 440a6b5..dc7167b 100644 --- a/spec/plans/2026-07-22-unified-diagnostic-chips-plan.md +++ b/spec/plans/2026-07-22-unified-diagnostic-chips-plan.md @@ -25,7 +25,7 @@ implements: - Existing `diagnostic-backlinks:start` and `diagnostic-backlinks:end` markers remain unchanged. - Search attributes and other invisible technical values are not converted into components. - Light theme, dark theme, keyboard focus, and narrow screens remain readable. -- Before running Python commands, set `VIRTUAL_ENV=/Users/ingvarvilkman/Documents/git/v8std/.venv`; use `$VIRTUAL_ENV/bin/python` so project dependencies are available from the linked worktree. +- Before running Python commands, set `VIRTUAL_ENV="$PWD/.venv"` from the checkout root; use `$VIRTUAL_ENV/bin/python` so project dependencies are available from the linked worktree. --- @@ -236,7 +236,7 @@ Run: ```bash $VIRTUAL_ENV/bin/python scripts/generate_diagnostic_standard_links.py --check -VIRTUAL_ENV="/Users/ingvarvilkman/Documents/git/v8std/.venv" ./scripts/zensical_docs.sh build --strict +VIRTUAL_ENV="$PWD/.venv" ./scripts/zensical_docs.sh build --strict ``` Expected: both generator checks succeed and Zensical reports no strict-build errors. diff --git a/spec/process/architecture-artifacts-v2.md b/spec/process/architecture-artifacts-v2.md new file mode 100644 index 0000000..d857c2f --- /dev/null +++ b/spec/process/architecture-artifacts-v2.md @@ -0,0 +1,93 @@ +--- +schema_version: 1 +kind: process +id: architecture-artifacts +version: 2 +supersedes: + - process:architecture-artifacts@1 +schema: + semantic_id_pattern: '^[A-Z][A-Z_]*$' + document_id_pattern: '^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$' + typed_reference_pattern: '^(design|adr|invariant|contract|plan|process):' + contract_reference_pattern: '^contract:[A-Z][A-Z_]*@\d+\.\d+$' + adr_filename_pattern: '^\d{4}-\d{2}-\d{2}-[a-z]+(?:-[a-z]+)*\.md$' + directories: + design: spec/designs + adr: spec/adr + invariant: spec/invariants + contract: spec/contracts + plan: spec/plans + process: spec/process + frozen_kinds: [design, adr, invariant, contract, plan, process] + forbidden_fields: [status] +--- + +# Архитектурные артефакты v8std, версия 2 + +## Область изменения + +Версия 2 является нормативным преемником версии 1 с неизменной машинной схемой. +Она принимается вместе с `design:mcp-ci-deployment-policy`. Из +[неизменяемой версии 1](architecture-artifacts-v1.md) нормативно сохраняются +разделы «Виды документов», «Структурные данные и поясняющий текст», +«Типизированные ссылки», «Требования», «Решения, инварианты и контракты», +«Даты, состояния и заморозка», «Безопасность деклараций». +Правила остальных разделов заменены ниже, а не применяются одновременно. + +## Последовательность работы + +1. До первой записи создать отдельную ветку в основном checkout. PR/worktree + применять только по явному запросу пользователя. +2. Выполнить ручной semantic impact check по намерению и предполагаемым путям. + Пустой diff или отсутствие совпадений `governs` не доказывают тривиальность. +3. Если влияние найдено или не исключено — использовать brainstorming, + письменно согласовать design-пакет и создать implementation plan. +4. При опровержении design остановить реализацию и пересмотреть граф целиком. +5. Перед локальным merge повторить ручной impact по фактическому diff, CLI + `impact`, `validate --merge-ready`, fitness, полный suite и strict build. + Прямые коммиты в main и push feature напрямую в remote main запрещены. +6. Push локального main выполнять только по явному запросу. Разрешённый push + запускает проверку/публикацию сайта, corpus и runtime по затронутым входам. +7. После первичной активации MCP автоматически обновляется из CI только + проверенным SHA main и опубликованным digest, с собственной post-deploy + проверкой и rollback. До активации обновление требует отдельного запроса. + +## Plan и готовность к merge + +Plan содержит хотя бы один Markdown checkbox. Для merge-ready все checkboxes +candidate plans должны быть завершены. Незавершённый plan допустим при обычной +validate. Design/ADR/invariant/contract без plan могут быть ACCEPTED; только +завершённый принятый plan с явным `implements` добавляет IMPLEMENTED. + +Checkboxes описывают реализацию и доказательства, доступные до merge. Merge, +разрешённый push, внешняя публикация, первичная активация и последующие +автоматические production deployments — отдельные результаты интеграции, +а не условие завершённости такого plan. Факт включения автоматизации и факт +успешного rollout записываются раздельно. + +Fitness evidence обязательно в момент `required_when: accepted|implemented`. +Декларация будущего теста не является выполненным тестом. Наличие design без +plan допустимо; запись design не означает готовый продукт. + +## Границы публикации и MCP deployment + +Первичная активация требует согласованного операционного этапа: safety inventory +и backup host, защита main и production environment, изолированные credentials, +проверенный recovery и явное включение. Она не выводится из merge design-файлов. +Указатель schema loader, repo instructions, skill, tests и CI обновляются +согласованно в реализации; до этого действующие инструменты остаются на v1. + +После активации разрешённый push проверенного main означает разрешение +автоматической доставки соответствующих артефактов этим настроенным CI. +Отдельное разрешение на каждый обычный MCP rollout не требуется. CI не даёт +агенту полномочий вручную менять production вне этого пути, удалять чужие +приложения или выдавать новые доступы. + +Контентная поставка сначала размещает immutable snapshot, затем публикует +ссылающийся на него Pages manifest. Runtime поставляется из опубликованного +image digest. Некорректный или устаревший release не активируется; неуспешный +smoke возвращает предыдущую работоспособную комбинацию image/config/data. + +Отключение автоматизации прекращает новые production jobs, но не останавливает +работающий MCP. Аварийное ручное вмешательство требует отдельного запроса, +фиксации SHA/digest и последующего возвращения host к управляемому состоянию. From 33ca471be2a75e069883a08662fa0f85c07962b4 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:14:56 +0300 Subject: [PATCH 02/88] docs: plan unified MCP container implementation --- ...6-09-10-mcp-container-distribution-plan.md | 393 ++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 spec/plans/2026-09-10-mcp-container-distribution-plan.md diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md new file mode 100644 index 0000000..014e321 --- /dev/null +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -0,0 +1,393 @@ +--- +schema_version: 1 +kind: plan +id: mcp-container-distribution +design: design:mcp-container-distribution +implements: + - adr:MCP_PUBLISHED_COMBINED_RUNTIME + - adr:MCP_ATOMIC_SITE_SNAPSHOTS + - contract:MCP_API@2.4 + - contract:MCP_CORPUS_SNAPSHOT@1.0 + - invariant:MCP_ACTIVE_SNAPSHOT_IS_COMPLETE +requirements: + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + - MCP_RUNTIME_AND_CORPUS_RELEASE_INDEPENDENTLY + - MCP_SITE_SETTING_CONTROLS_SOURCE_AND_LINKS + - MCP_SNAPSHOT_LOAD_IS_ATOMIC + - MCP_REFRESH_STAYS_OUTSIDE_TOOL_CALLS + - MCP_SNAPSHOT_IO_IS_BOUNDED + - MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART + - MCP_LOCAL_SITE_HAS_NO_BACKGROUND_PUBLIC_EGRESS + - MCP_CONTAINER_DISTRIBUTION_SUPPORTS_AGENT_LIFECYCLE + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_SHARED_HOST_LOAD_IS_MEASURED + - MCP_OFFLINE_USES_VERIFIED_CACHE + - MCP_DISTRIBUTION_PROVENANCE_IS_VERIFIABLE +--- + +# MCP Container Distribution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Реализовать единый контейнерный MCP с безопасной доставкой corpus и проверяемой автоматической поставкой, не активируя production в ходе локальной разработки. + +**Architecture:** Подготовленный snapshot загружается фоном в отдельное поколение индекса. Запрос удерживает готовое поколение, а ссылки преобразуются только на границе ответа. Один опубликованный runtime image используется всеми каналами; отдельный статический сайт и host-controller обеспечивают локальную установку и обратимый rollout. + +**Tech Stack:** Python 3.12+, существующий mcp==1.27.0/FastMCP, unittest, Docker/Compose, nginx, GitHub Actions. Формат snapshot — deterministic tar.gz, SHA-256, JSON/JSONL. + +**Spec:** [Продуктовый design](../designs/2026-09-10-mcp-container-distribution-design.md), [процессный design](../designs/2026-09-10-mcp-ci-deployment-policy-design.md), [snapshot contract](../contracts/mcp-corpus-snapshot-v1-r0.md), [API contract](../contracts/mcp-api-v2-r4.md), [distribution contract](../contracts/mcp-distribution-v1-r0.md), [release contract](../contracts/mcp-release-runtime-v1-r0.md). + +## Global Constraints + +- Один `/mcp`, пять существующих tools, три существующих bulk Resources; никакого `/v3/mcp` или нового bulk discovery. +- Единственная публичная настройка — `V8STD_MCP_SITE_URL`, по умолчанию `https://v8std.ru/`; CLI `--site-url` имеет приоритет. Второй public-base URL не вводится. +- Snapshot schema major 1, model `v8std-hash-embeddings-v1`, dim 256; допустимые размеры/схемы/URL определены snapshot contract и не ослабляются ради fixture. +- Snippet: 4000 default, 32000 maximum; query 500, preview 1000, tokens 80 и суммарно 4000; один hybrid search, прежний ranking и отсутствие raw procedure в usage logs. +- Tool/resource request не скачивает данные и не ждёт refresh; одновременно видит одно валидное поколение. +- Один multi-platform image на версию, `linux/amd64` и `linux/arm64`; catalog review может отставать, но не создаёт другую сборку той же версии. +- Local site и MCP не делают background public egress. Cold-offline thin image без cache не готов; warm-offline с проверенным cache работает. +- Основной checkout, существующая ветка `codex/mcp-container-distribution-design`; не создавать worktree, не менять main, не пушить и не деплоить во время реализации. +- TDD и focused tests для каждого поведения. Strict build выполняется **до** полного suite: tests читают `site/LICENSES`, параллельная пересборка разрушает их вход. +- Утверждение о 100 000 подключений запрещено без production-like mixed-load evidence. + +## Scope and acceptance boundaries + +Это один связанный release-пакет с шестью проверяемыми задачами в трёх блоках: +corpus/runtime (1–3), container distribution (4), delivery (5–6). Их interfaces +зафиксированы ниже. Запуск CI, registry publication, production setup/cleanup, +первичный rollout и внешний Docker Catalog PR — последующие операции интеграции, +не checkboxes этого plan. + +`implements` намеренно не включает весь design, distribution/release contracts +и инварианты, требующие внешнего artifact/digest/host evidence. Эти артефакты +остаются принятым намерением до отдельного evidence-backed plan. Завершение +кода release-controller само по себе не доказывает rollout на целевом сервере или 100k. +Процессная реализация получает отдельный process plan, чтобы product plan +не объявлял Git policy продуктовым инвариантом. + +## File ownership and interfaces + +| Task | Write scope | Responsibility | +|---|---|---| +| 1 | `scripts/v8std_mcp_snapshot_format.py`, `scripts/generate_mcp_snapshot.py`, `tests/test_v8std_mcp_snapshot_format.py`, `tests/mcp_snapshot_fixtures.py` | Pure format validation, deterministic producer; no network/index refresh. | +| 2 | `scripts/v8std_mcp_snapshots.py`, `tests/test_v8std_mcp_snapshots.py` | URL trust boundary, HTTP/cache transaction, background coordinator. | +| 3 | `scripts/v8std_mcp_runtime.py`, `scripts/v8std_mcp_presentation.py`, `scripts/v8std_mcp_index.py`, `scripts/v8std_mcp_server.py`, runtime tests | Frozen generation construction, request facade, stdio/HTTP lifecycle. | +| 4 | Dockerfiles/Compose/lock, local-profile script, tests, docs | Build and exercise the two images and local site. | +| 5 | `deploy/container/`, `scripts/v8std_mcp_release.py`, `tests/test_v8std_mcp_release.py` | Typed host transaction, nginx index store and recovery. | +| 6 | workflows, publication scripts, architecture policy/test references, docs/operations | Fail-closed CI delivery, process v2 synchronization, integration evidence. | + +### Task 1: Deterministic corpus format and producer + +**Files:** create format/producer/fixture/test files listed in ownership table. +Read the full snapshot contract and existing `generate_search_vectors.py`; +do not change the ranking or introduce a runtime dependency on docs builders. + +**Interfaces produced:** + +```python +class SnapshotError(ValueError): + # str(error) is a bounded stable category, never untrusted payload. + code: str + +@dataclass(frozen=True) +class VerifiedSnapshot: + metadata: dict + files: dict[str, bytes] + archive_sha256: str + +def normalize_site_url(value: str) -> str: ... +def validate_manifest(payload: bytes) -> dict: ... +def verify_archive(payload: bytes, manifest: dict) -> VerifiedSnapshot: ... +def build_snapshot(docs_dir: Path, source_sha: str, canonical_site_url: str) -> tuple[bytes, dict]: ... +def publish_snapshot(docs_dir: Path, output_dir: Path, source_sha: str, + canonical_site_url: str, *, public_delivery: bool = False) -> Path: ... +``` + +The ellipses above describe signatures, not implementation steps. Keep format +validation in one module so network/cache code reuses exactly the same checks. +Producer converts canonical page URLs into additional `site_path`/`markdown_path` +without rewriting canonical body text; generated vectors and text hashes remain +valid. Site-level generator outputs currently consumed elsewhere stay intact. + +- [ ] **RED:** Add behavior tests with hand-authored one-page docs and independently + constructed hostile tar members. The first test asserts the producer module + exists using `importlib.util.find_spec`, then execute the producer twice and + require exact archive byte equality. Example expected path is literal: + +```python +self.assertEqual(page["site_path"], "std/437/") +self.assertEqual(page["markdown_path"], "std/437.md") +self.assertEqual(verified.metadata["vector_dim"], 256) +with self.assertRaisesRegex(SnapshotError, "archive_member"): + verify_archive(archive_with_symlink, matching_manifest) +``` + + Run `.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format -v`; + missing implementation must fail before production code is written. +- [ ] **GREEN format:** Implement strict JSON with duplicate-key/depth/type + guards; exact five regular members; deterministic gzip/tar; independent + descriptor/archive hashes; compressed/decompressed/member/row/line bounds; + no symlink, trailing gzip data, PAX, duplicate or traversal member. Count all + decompressed bytes including tar framing, not only declared member sizes. + Validate page/vector IDs, finite components, model/dim and chunk text hashes + against the existing chunk rules. Reuse those pure rules without importing + docs/Pillow. Normalize site URL and preserve base prefix. +- [ ] **GREEN producer:** CLI accepts `--docs`, `--output`, `--source-sha`, + `--site-url`, `--public-delivery`; publishes immutable hash directory first, + manifest last with atomic write. In local mode archive path is relative; + public mode uses the fixed ai delivery origin. Existing same-hash bytes are + verified, never overwritten with different contents. +- [ ] **Verify:** Real current docs corpus round-trip, deterministic rebuild, + bad vectors/schema/count/hash and all archive budget cases pass. Record exact + focused command and results, run existing vector/index tests, self-review, + then commit only this task's files and submit for spec+quality review. + +### Task 2: Bounded snapshot cache and background coordinator + +**Files:** create `scripts/v8std_mcp_snapshots.py` and +`tests/test_v8std_mcp_snapshots.py`; consume Task 1 format module. + +**Interfaces produced:** + +```python +class SnapshotStore: + def __init__(self, site_url: str, cache_dir: Path): ... + def cached(self) -> VerifiedSnapshot | None: ... + def refresh(self) -> VerifiedSnapshot: ... + +class SnapshotCoordinator: + def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600): ... + def start(self) -> None: ... + def current(self): ... # returns one built generation or raises INDEX_NOT_READY + def status(self) -> dict: ... + def close(self) -> None: ... +``` + +`build: Callable[[VerifiedSnapshot], Any]` constructs an immutable generation. +The coordinator owns the active reference and stores network/parse work outside +the request path. Prepare in a bounded worker with a lifecycle that is stopped +on close; no network/CPU build on the ASGI event loop. Test/store internals may +inject monotonic clock/transport at their actual dependency boundary, never +test-only methods on production classes. + +- [ ] **RED:** ThreadingHTTPServer fixtures count GETs, return delayed chunks, + corrupt archives, foreign redirects and 304. Cold failure must produce explicit + not-ready; warm store must keep its previous corpus ID. Example contract test: + +```python +before = coordinator.current() +source.begin_blocking_response() +source.wait_until_requested() +self.assertIs(coordinator.current(), before) +self.assertEqual(coordinator.current().corpus_id, "fixture-generation-a") +``` + + Fixture generations in this test are small builder results independent of the + format hash; real-format tests use actual corpus IDs. Run focused tests to RED. +- [ ] **GREEN URL/network:** Resolve manifest below selected base prefix; + permit fixed ai archive origin only for default public site; validate every + redirect, no downgrade, no credentials and three redirects maximum. Stream + reads enforce byte caps plus 60s whole attempt/20s read timeout. Validate + Content-Encoding and length; reuse ETag/Last-Modified only with a valid cache. +- [ ] **GREEN cache/lifecycle:** Namespace by normalized source/schema; verify + cache before use; retain active+previous and pins. File lock serializes + download/commit between processes, query never takes it. Atomic durable pointer + update precedes in-process swap; crash, disk-full, corrupt current fall back to + previous same-source generation. Cache total includes staging and is bounded + at 256 MiB. One background updater, 3600s refresh ±20%, 30…3600s error backoff, + zero disables periodic refresh after bootstrap. Close interrupts workers with + a bounded deadline, without orphan processes/threads holding interpreter exit. +- [ ] **Verify:** Test same/different sources, prefix, zero interval, 304 without + cache, delayed read, repeated faults, crash stages, two processes/shared volume, + cache budget and no query blocking. Run Task 1+2 tests, record RED/GREEN and + commit task files; request spec+quality review before integration. + +### Task 3: Frozen index generations, presentation and MCP lifecycle + +**Files:** create `scripts/v8std_mcp_runtime.py`, +`scripts/v8std_mcp_presentation.py`, `tests/test_v8std_mcp_runtime.py`, +`tests/test_v8std_mcp_presentation.py`; modify index/server and focused tests. + +**Interfaces produced:** + +```python +@dataclass(frozen=True) +class IndexGeneration: + corpus_id: str + index: V8StdIndex + resources: dict[str, str] + +class SnapshotIndex: + # Same public search/page/related/explain_snippet/explain_diagnostics, + # read_resource_text/status/max_snippet_chars boundary as V8StdIndex. + def start(self) -> None: ... + def close(self) -> None: ... + +def build_generation(snapshot: VerifiedSnapshot, *, max_snippet_chars: int) -> IndexGeneration: ... +def present_result(value, *, canonical_site_url: str, site_url: str, + page_paths: dict): ... +``` + +Construct one V8StdIndex from already validated bytes without network. Add a +focused factory for this to existing index; retain legacy direct file entrypoints +for current tests/developer use. Never mutate this index after construction. +Facade captures coordinator.current() once per top-level call, invokes that +generation including nested snippet/search/related operations, then transforms +only presentation links on the returned copy. + +- [ ] **RED:** Wire tests initialize/list tools before source readiness; valid + call before ready is error `INDEX_NOT_READY`; `/healthz` is 503 and `/livez` + 200. Add full snippet compatibility and generation-swap tests; public source + fetch is forbidden in callbacks. A local-prefix fixture must retain literal + code containing a public URL while its Markdown/HTML links become local: + +```python +self.assertIn("`https://v8std.ru/std/437/`", result["page"]["body_markdown"]) +self.assertEqual(result["page"]["url"], "http://localhost:8080/kb/std/437/") +self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/content/437/hdoc"]) +``` + +- [ ] **GREEN presentation:** Parse Markdown link/image/reference/autolink and + HTML attributes outside code; rewrite only known internal paths, preserve + external provenance/query/fragment. Local URLs accepted as page lookup inputs + resolve to canonical keys without affecting ranking. All nested result URLs + and three legacy Resource bodies use the same policy; no global string replace. +- [ ] **GREEN runtime:** Startup chooses snapshot mode from SITE_URL/default, + supports stdio and HTTP from same build_server. Legacy explicit files remain + usable; ambiguous legacy URLs plus site setting fail before network. Default + direct Python transport stays compatible; container supplies stdio explicitly. + Initialize/schema immediate, background bootstrap, bounded EOF/SIGTERM cleanup, + clean stdout, health/version compact additive metadata, no raw errors/data. +- [ ] **Verify:** Task 1–3 tests plus all MCP server/index/snippet/combined tests; + real stdio subprocess and loopback HTTP smoke; before/after search benchmark + on same corpus, slow-source requests and CPU/RAM refresh measurements. Commit + only reviewed implementation; no changed scores or widened snippet/query limits. + +### Task 4: Runtime/static images, local profile and Compose + +**Files:** create `Dockerfile.mcp`, `Dockerfile.site`, `.dockerignore`, +`requirements-mcp.lock`, `compose.yaml`, `deploy/container/site.conf`, +`scripts/build_local_site.py`, `tests/test_v8std_mcp_distribution.py`, +`scripts/check_mcp_container.py`, `deploy/docker-catalog/server.yaml`; +modify `overrides/main.html`, docs and build wrapper only where profile needs it. + +**Consumes:** Task 1 producer CLI and Task 3 runtime CLI. Runtime container +default CMD selects stdio; HTTP command selects host 0.0.0.0/port 8000. +Static image consumes already built local `site` and included snapshot, not +the runtime image. Keep existing source-bind Compose explicitly dev. + +- [ ] **RED:** Tests execute the local-profile build into a temporary directory, + then assert HTTP pages+manifest/archive work without external network. Container + harness initializes stdio twice with shared volume and verifies a tool call, + generation ID/cache reuse, EOF exit and runtime readiness: + +```python +self.assertEqual(reply["result"]["serverInfo"]["name"], "v8std") +self.assertFalse(tool_reply.get("isError", False)) +self.assertEqual(container_inspect["Config"]["User"], "10001:10001") +``` + + Use actual current serverInfo name from build_server if it differs; this is + a named compatibility check, not permission to rename the server. +- [ ] **GREEN images/profile:** Resolve pinned base digests and full dependency + lock; copy only runtime modules/rules/licenses. Non-root UID/GID 10001, read-only + root, persistent writable cache, tmpfs, cap-drop and init. Local profile disables + analytics/recorder and external fonts/assets, preserving public profile. Same + archive bytes are used for public/local manifests. Build on arm64 and exercise + amd64 in available Docker emulation, noting native-CI gate separately. +- [ ] **GREEN launch/catalog:** Compose references published image coordinates + with explicit version/digest override for local test images, loopback published + ports, optional MCP profile, named cache and one common routable SITE_URL. + Document/test desktop host access and Linux host-gateway. Catalog points to + self-published image, declares site/snippet/cache and long-lived stdio behavior; + validate against current Docker schema without submitting an external PR. +- [ ] **Verify:** Run distribution unit/integration harness, real Docker stdio/HTTP, + non-root read-only startup, persistent cache offline restart, local site request + graph without public egress and multi-session Gateway where locally available. + Verify licenses/SBOM inputs. Record unavailable external catalog acceptance + explicitly, not as a passed test. Commit task and perform review. + +### Task 5: Restricted release transaction and independent index store + +**Files:** create `scripts/v8std_mcp_release.py`, `tests/test_v8std_mcp_release.py`, +`deploy/container/release.schema.json`, controller/service/nginx configurations +under `deploy/container/`, `spec/operations/mcp-container-activation.md`. + +**Interfaces produced:** CLI `validate-envelope`, `deploy`, `recover`, `status`, +`publish-index`; inputs are typed bounded JSON or fixed paths under configured +state/store roots. Release controller effects go through a narrow adapter to +Docker/nginx/systemd; tests use disposable processes/filesystems and record exact +calls at this external boundary, not pretend mocked return values prove health. + +- [ ] **RED:** Test unknown schema, invalid digest/namespace/config path, stale or + mutated duplicate ID, concurrent releases, failed pull/ready/switch/smoke, + rollback failure and restart reconciliation. Example visible invariant: + +```python +self.assertEqual(controller.status()["state"], "ROLLED_BACK") +self.assertEqual(edge.serving_digest(), predecessor_digest) +self.assertTrue(predecessor_snapshot_path.is_file()) +``` + +- [ ] **GREEN controller:** Verify envelope/attestation/trusted config before + effects, durable journal+lock, exact image digest, capacity preflight and + predecessor pins. Prepare candidate side port, real readiness and MCP smoke, + nginx test then atomic switch/reload, public smoke then commit/drain. Enforce + 5min transaction, 90s readiness, 30s smoke/drain, 45s final stop; loss of caller + does not kill host recovery. Stale retry cannot supersede current sequence. +- [ ] **GREEN static store/operations:** Independent read-only nginx alias for + `/indexes/v1/` with GET/HEAD, hash cache headers and bounded download admission; + publisher stages/verifies/renames objects and tracks references/pins before GC. + Activation runbook names exact backup/cleanup/TLS/default-vhost, secrets, + protection, native capacity and initial Python rollback prerequisites. No live + host changes occur from tests or writing the runbook. +- [ ] **Verify:** Fault matrix for each state transition with real subprocess + cancellation/crash recovery where possible; nginx syntax and static download + while runtime stopped in disposable containers; unprivileged controller input + rejects shell injection. Record capacity test settings and results without + claiming 100k. Commit and review the host code as security-sensitive code, + not an authorization to install it on production. + +### Task 6: Fail-closed publication CI, policy and final integration + +**Files:** workflows under `.github/workflows/`, `scripts/publish_mcp_artifacts.py`, +`tests/test_mcp_publication.py`, process plan +`spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md`, `AGENTS.md`, +`spec/README.md`, repo skill references, architecture loader/policy tests, +`spec/operations/mcp-container-verification.md`, public installation docs. + +**Consumes:** producer, image harness and typed release controller CLIs. +Publisher first places and externally verifies immutable corpus, then emits +Pages manifest. Every runtime deployment references published exact digest. + +- [ ] **RED:** Test publication sequencing and early failure leaves current + manifest unchanged; main/PR/fork/tag/stale eligibility matrix produces no host + effects for disallowed inputs. Process loader resolves v2 with identical schema; + frozen v1 and all accepted structured documents remain byte-identical. +- [ ] **GREEN pipeline:** PR validation without secrets; pin action SHAs, image + bases and runtime dependencies; architecture/build/tests/bench before publish; + buildx multiarch image, SBOM/provenance/attestation and digest smoke. Detect + content vs runtime inputs to avoid article-triggered MCP restart. Production + job requires verified main, restricted environment and explicit activation + variable; `cancel-in-progress: false`, idempotent/stale host validation. + Before activation retain working Pages delivery without publishing an ai + manifest that points to an unavailable object. Build candidate artifacts locally; + enabled publication either verifies the object or fails closed. +- [ ] **GREEN process:** Write separate process plan, synchronize process v2 + pointer/instructions/tests. Preserve initial manual activation and explicit + permission for push. Align all current policy references; historic v1 and old + structured plans stay frozen. Write reproducible commands/evidence and separate + external gates for registry/Catalog/target-host. Do not publish internal specs. +- [ ] **Final gates:** Run semantic impact on actual paths, CLI `impact`, + `validate --merge-ready`, all applicable fitness; strict build, then full suite; + container smoke and shared-host mixed load on disposable local stack, review + whole branch and repair concrete findings. Record exact SHA/results and + unperformed external operations. No push, merge of incomplete plan, PR closure + or production deployment inferred from these green tests. + +## Evidence + +Each task's implementation/review evidence is recorded during execution in its +SDD report and completion checkbox. Final durable summary goes in +`spec/operations/mcp-container-verification.md`, including commands, environment, +results and the boundary between local evidence and production acceptance. From 1c8d364538031383ec582a0897e6127cbd45e06f Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:35:40 +0300 Subject: [PATCH 03/88] feat: add deterministic MCP corpus snapshots --- scripts/generate_mcp_snapshot.py | 179 ++++++++ scripts/v8std_mcp_snapshot_format.py | 483 +++++++++++++++++++++ tests/mcp_snapshot_fixtures.py | 135 ++++++ tests/test_v8std_mcp_snapshot_format.py | 543 ++++++++++++++++++++++++ 4 files changed, 1340 insertions(+) create mode 100644 scripts/generate_mcp_snapshot.py create mode 100644 scripts/v8std_mcp_snapshot_format.py create mode 100644 tests/mcp_snapshot_fixtures.py create mode 100644 tests/test_v8std_mcp_snapshot_format.py diff --git a/scripts/generate_mcp_snapshot.py b/scripts/generate_mcp_snapshot.py new file mode 100644 index 0000000..6bcc32c --- /dev/null +++ b/scripts/generate_mcp_snapshot.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Build and atomically publish an immutable MCP corpus from existing docs outputs.""" + +from __future__ import annotations + +import argparse +import gzip +import io +import json +import os +from pathlib import Path +import tempfile + +from v8std_mcp_snapshot_format import ( + DEFAULT_SITE_URL, JSONL_MEMBERS, MAX_ARCHIVE_BYTES, MAX_MANIFEST_BYTES, + MAX_UNPACKED_BYTES, MEMBER_LIMITS, MEMBERS, PUBLIC_DELIVERY_URL, + SnapshotError, VECTOR_DIM, VECTOR_MODEL, canonical_json, jsonl_rows, + normalize_site_url, portable_page, sha256, tar_header, validate_manifest, verify_archive, +) + + +def _read(path: Path, maximum: int) -> bytes: + try: + with path.open("rb") as stream: + payload = stream.read(maximum + 1) + except OSError: + raise SnapshotError("source_io") from None + if len(payload) > maximum: + raise SnapshotError("member_size") + return payload + + +def build_snapshot(docs_dir: Path, source_sha: str, canonical_site_url: str) -> tuple[bytes, dict]: + site_url = normalize_site_url(canonical_site_url) + docs_dir = Path(docs_dir) + files = { + name: _read(docs_dir / ("ai" if name in JSONL_MEMBERS else "") / name, MEMBER_LIMITS[name]) + for name in MEMBERS[1:] + } + pages = bytearray() + page_count = 0 + for page in jsonl_rows(files["pages.jsonl"]): + row = portable_page(page, site_url) + pages.extend(json.dumps(row, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8") + b"\n") + if len(pages) > MEMBER_LIMITS["pages.jsonl"]: + raise SnapshotError("member_size") + page_count += 1 + files["pages.jsonl"] = bytes(pages) + vector_count = sum(1 for _ in jsonl_rows(files["search-vectors.jsonl"])) + counts = dict(zip(JSONL_MEMBERS, (page_count, vector_count))) + descriptor = { + "schema_version": 1, "source_sha": source_sha, "canonical_site_url": site_url, + "vector_model": VECTOR_MODEL, "vector_dim": VECTOR_DIM, + "files": { + name: {"sha256": sha256(payload), "bytes": len(payload), + **({"rows": counts[name]} if name in counts else {})} + for name, payload in files.items() + }, + } + metadata = {**descriptor, "corpus_id": sha256(canonical_json(descriptor))} + files = {"metadata.json": canonical_json(metadata), **files} + if len(files["metadata.json"]) > MEMBER_LIMITS["metadata.json"]: + raise SnapshotError("member_size") + raw = bytearray() + for name in MEMBERS: + payload = files[name] + raw.extend(tar_header(name, len(payload))) + raw.extend(payload) + raw.extend(b"\0" * (-len(payload) % 512)) + raw.extend(b"\0" * 1024) + raw.extend(b"\0" * (-len(raw) % 10240)) + if len(raw) > MAX_UNPACKED_BYTES: + raise SnapshotError("archive_unpacked_size") + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb", filename="", mtime=0, compresslevel=9) as stream: + stream.write(raw) + archive = output.getvalue() + digest = sha256(archive) + manifest = { + "schema_version": 1, "source_sha": source_sha, "corpus_id": metadata["corpus_id"], + "vector_model": VECTOR_MODEL, "vector_dim": VECTOR_DIM, + "archive": {"path": digest + "/snapshot.tar.gz", "sha256": digest, + "bytes": len(archive), "unpacked_bytes": sum(map(len, files.values()))}, + } + validate_manifest(canonical_json(manifest)) + verify_archive(archive, manifest) + return archive, manifest + + +def _fsync_directory(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _atomic_file(path: Path, payload: bytes, *, immutable: bool = False) -> None: + """Install durable complete bytes. link() is an atomic no-clobber publish.""" + temporary_path = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".snapshot-", delete=False) as stream: + temporary_path = Path(stream.name) + stream.write(payload) + stream.flush() + os.fchmod(stream.fileno(), 0o644) + os.fsync(stream.fileno()) + if immutable: + os.link(temporary_path, path) + else: + os.replace(temporary_path, path) + _fsync_directory(path.parent) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def publish_snapshot(docs_dir: Path, output_dir: Path, source_sha: str, + canonical_site_url: str, *, public_delivery: bool = False) -> Path: + site_url = normalize_site_url(canonical_site_url) + if public_delivery and site_url != DEFAULT_SITE_URL: + raise SnapshotError("archive_path") + archive, manifest = build_snapshot(docs_dir, source_sha, site_url) + output_dir = Path(output_dir) + archive_dir = output_dir / manifest["archive"]["sha256"] + archive_path = archive_dir / "snapshot.tar.gz" + try: + output_dir.mkdir(parents=True, exist_ok=True) + if archive_dir.is_symlink(): + raise SnapshotError("immutable_conflict") + archive_dir.mkdir(exist_ok=True) + _fsync_directory(output_dir.parent) + try: + _atomic_file(archive_path, archive, immutable=True) + except FileExistsError: + if archive_path.is_symlink() or not archive_path.is_file(): + raise SnapshotError("immutable_conflict") from None + existing = _read(archive_path, MAX_ARCHIVE_BYTES) + if existing != archive: + raise SnapshotError("immutable_conflict") from None + verify_archive(existing, manifest) + # Persist the directory entry before publishing a reference to it. + _fsync_directory(archive_dir) + _fsync_directory(output_dir) + if public_delivery: + manifest["archive"]["path"] = PUBLIC_DELIVERY_URL + manifest["archive"]["path"] + encoded_manifest = canonical_json(manifest) + validate_manifest(encoded_manifest) + if len(encoded_manifest) > MAX_MANIFEST_BYTES: + raise SnapshotError("manifest_size") + manifest_path = output_dir / "manifest.json" + _atomic_file(manifest_path, encoded_manifest + b"\n") + return manifest_path + except OSError: + raise SnapshotError("publish_io") from None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docs", type=Path, default=Path("docs")) + parser.add_argument("--output", type=Path, required=True, + help="Manifest directory; immutable hash directories are created below it.") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--site-url", default=None) + parser.add_argument("--public-delivery", action="store_true") + args = parser.parse_args() + site_url = args.site_url if args.site_url is not None else os.environ.get("V8STD_MCP_SITE_URL", DEFAULT_SITE_URL) + try: + path = publish_snapshot(args.docs, args.output, args.source_sha, site_url, + public_delivery=args.public_delivery) + except SnapshotError as error: + parser.exit(1, f"{error.code}\n") + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/v8std_mcp_snapshot_format.py b/scripts/v8std_mcp_snapshot_format.py new file mode 100644 index 0000000..90365d0 --- /dev/null +++ b/scripts/v8std_mcp_snapshot_format.py @@ -0,0 +1,483 @@ +"""Bounded, stdlib-only validation for MCP_CORPUS_SNAPSHOT@1.0. + +No filesystem extraction or network access occurs here. The cache loader owns +private staging and the trust decision for the selected site's archive URL. +""" + +from __future__ import annotations + +import base64 +import binascii +from dataclasses import dataclass +import hashlib +import io +import ipaddress +import json +import math +import re +import struct +import tarfile +from urllib.parse import quote, unquote, urlsplit +import zlib + + +DEFAULT_SITE_URL = "https://v8std.ru/" +PUBLIC_DELIVERY_URL = "https://ai.v8std.ru/indexes/v1/" +VECTOR_MODEL = "v8std-hash-embeddings-v1" +VECTOR_DIM = 256 +MAX_MANIFEST_BYTES = 64 * 1024 +MAX_ARCHIVE_BYTES = 16 * 1024 * 1024 +MAX_UNPACKED_BYTES = 64 * 1024 * 1024 +MAX_JSONL_LINE_BYTES = 1024 * 1024 +MAX_JSONL_ROWS = 100_000 +MAX_JSON_DEPTH = 32 +MEMBER_LIMITS = { + "metadata.json": 64 * 1024, + "pages.jsonl": 16 * 1024 * 1024, + "search-vectors.jsonl": 32 * 1024 * 1024, + "llms.txt": 4 * 1024 * 1024, + "llms-full.txt": 16 * 1024 * 1024, +} +MEMBERS = tuple(MEMBER_LIMITS) +JSONL_MEMBERS = ("pages.jsonl", "search-vectors.jsonl") +_READ_BYTES = 64 * 1024 +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_SOURCE_SHA = re.compile(r"[0-9a-f]{40}\Z") +_ERROR_CODES = frozenset({ + "snapshot_invalid", "site_url", "manifest_schema", "manifest_size", "archive_path", + "archive_size", "archive_hash", "archive_unpacked_size", "archive_gzip", + "archive_member", "archive_header", "archive_padding", "member_size", "member_hash", + "metadata_schema", "metadata_hash", "metadata_mismatch", "json_encoding", "json_syntax", + "json_duplicate_key", "json_depth", "json_type", "json_number", "jsonl_line_size", + "jsonl_rows", "page_schema", "page_id", "page_path", "vector_schema", "vector_identity", + "vector_data", "vector_text_hash", "corpus_empty", "source_io", "publish_io", + "immutable_conflict", +}) + + +class SnapshotError(ValueError): + """An error safe to put in status/logs; never embeds input data or paths.""" + + def __init__(self, code: str): + self.code = code if code in _ERROR_CODES else "snapshot_invalid" + super().__init__(self.code) + + +@dataclass(frozen=True) +class VerifiedSnapshot: + metadata: dict + files: dict[str, bytes] + archive_sha256: str + + +def sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise SnapshotError(code) + + +def _path(value: str, *, absolute: bool, code: str) -> str: + _require(isinstance(value, str), code) + _require(not re.search(r"[\x00-\x20\x7f\\?#:]", value), code) + _require(not re.search(r"%(?![0-9a-fA-F]{2})|%(?:2f|5c|25)", value, re.I), code) + try: + decoded = unquote(value, encoding="utf-8", errors="strict") + decoded.encode("utf-8") + except UnicodeError: + raise SnapshotError(code) from None + _require(not re.search(r"[\x00-\x20\x7f\\?#:]", decoded), code) + _require("//" not in decoded and not any(p in {".", ".."} for p in decoded.split("/")), code) + _require(not value or value.startswith("/") == absolute, code) + # Decode/encode once to give percent-encoded Unicode and unreserved bytes a + # single spelling, without allowing encoded separators or double decoding. + return quote(decoded, safe="/!$&'()*+,;=@-._~") + + +def _url(value: str, code: str) -> tuple[str, str, str]: + _require(isinstance(value, str) and bool(value), code) + _require(not re.search(r"[\x00-\x20\x7f\\?#]", value), code) + try: + parts = urlsplit(value) + _require(parts.scheme.lower() in {"https", "http"}, code) + _require(bool(parts.netloc) and "@" not in parts.netloc, code) + host = parts.hostname + _require(bool(host) and "%" not in host, code) + if ":" in host: + host = "[" + ipaddress.IPv6Address(host).compressed + "]" + else: + host = host.encode("idna").decode("ascii").lower() + _require(bool(re.fullmatch(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", host)), code) + _require(all(label and len(label) <= 63 and not label.startswith("-") + and not label.endswith("-") for label in host.split(".")), code) + port = parts.port + _require(port is None or 0 < port <= 65535, code) + _require(not parts.netloc.endswith(":"), code) + scheme = parts.scheme.lower() + authority = host + if port is not None and port != {"http": 80, "https": 443}[scheme]: + authority += f":{port}" + return scheme, authority, _path(parts.path or "/", absolute=True, code=code) + except (ValueError, UnicodeError): + raise SnapshotError(code) from None + + +def normalize_site_url(value: str) -> str: + _require(isinstance(value, str), "site_url") + scheme, host, path = _url(value.strip(), "site_url") + return f"{scheme}://{host}{path.rstrip('/')}/" + + +def canonical_page_path(value: str, canonical_site_url: str) -> str: + """Return a validated relative path, retaining the canonical site's prefix.""" + scheme, host, path = _url(value, "page_path") + base_scheme, base_host, base_path = _url(canonical_site_url, "page_path") + _require((scheme, host) == (base_scheme, base_host) and path.startswith(base_path), "page_path") + return _path(path[len(base_path):], absolute=False, code="page_path") + + +def _check_tree(value, *, no_floats: bool = False, level: int = 0) -> None: + if isinstance(value, (dict, list)): + _require(level < MAX_JSON_DEPTH, "json_depth") + if isinstance(value, list): + _require(len(value) <= MAX_JSONL_ROWS, "json_type") + children = value + else: + _require(all(isinstance(key, str) for key in value), "json_type") + children = [*value.keys(), *value.values()] + for child in children: + _check_tree(child, no_floats=no_floats, level=level + 1) + elif isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeError: + raise SnapshotError("json_encoding") from None + elif type(value) is float: + _require(not no_floats and math.isfinite(value), "json_number") + else: + _require(value is None or type(value) in {int, bool}, "json_type") + + +def canonical_json(value: dict) -> bytes: + """Descriptor encoding; floats, including floats in unknown fields, fail.""" + _check_tree(value, no_floats=True) + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8") + except (ValueError, TypeError, RecursionError): + raise SnapshotError("json_type") from None + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "json_duplicate_key") + result[key] = value + return result + + +def _invalid_constant(value): + raise SnapshotError("json_number") + + +def strict_json(payload: bytes) -> dict: + """Check nesting before json.loads can construct a deeply nested tree.""" + _require(isinstance(payload, bytes), "json_type") + try: + text = payload.decode("utf-8") + except UnicodeError: + raise SnapshotError("json_encoding") from None + depth, quoted, escaped = 0, False, False + for char in text: + if quoted: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + quoted = False + elif char == '"': + quoted = True + elif char in "[{": + depth += 1 + _require(depth <= MAX_JSON_DEPTH, "json_depth") + elif char in "]}": + depth -= 1 + try: + value = json.loads(text, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + except SnapshotError: + raise + except (ValueError, RecursionError): + raise SnapshotError("json_syntax") from None + _require(isinstance(value, dict), "json_type") + _check_tree(value) + return value + + +def _integer(value, minimum: int, maximum: int, code: str) -> None: + _require(type(value) is int and minimum <= value <= maximum, code) + + +def _hash(value, code: str) -> None: + _require(isinstance(value, str) and bool(_SHA256.fullmatch(value)), code) + + +def _schema(value: dict, code: str) -> None: + _integer(value.get("schema_version"), 1, 1, code) + _integer(value.get("vector_dim"), VECTOR_DIM, VECTOR_DIM, code) + _require(value.get("vector_model") == VECTOR_MODEL, code) + source = value.get("source_sha") + _require(isinstance(source, str) and bool(_SOURCE_SHA.fullmatch(source)), code) + _hash(value.get("corpus_id"), code) + + +def _manifest(manifest: dict) -> dict: + _require(isinstance(manifest, dict), "manifest_schema") + _check_tree(manifest) + # verify_archive is also a public entry point: a dict passed directly must + # not bypass the byte limit enforced when reading manifest JSON from bytes. + try: + encoded = json.dumps(manifest, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8") + except (ValueError, TypeError): + raise SnapshotError("json_number") from None + _require(len(encoded) <= MAX_MANIFEST_BYTES, "manifest_size") + _schema(manifest, "manifest_schema") + archive = manifest.get("archive") + _require(isinstance(archive, dict), "manifest_schema") + _hash(archive.get("sha256"), "manifest_schema") + _integer(archive.get("bytes"), 1, MAX_ARCHIVE_BYTES, "archive_size") + _integer(archive.get("unpacked_bytes"), 1, MAX_UNPACKED_BYTES, "archive_unpacked_size") + relative = archive["sha256"] + "/snapshot.tar.gz" + _require(archive.get("path") in (relative, PUBLIC_DELIVERY_URL + relative), "archive_path") + return manifest + + +def validate_manifest(payload: bytes) -> dict: + _require(isinstance(payload, bytes), "manifest_schema") + _require(len(payload) <= MAX_MANIFEST_BYTES, "manifest_size") + return _manifest(strict_json(payload)) + + +def jsonl_rows(payload: bytes): + """Yield validated objects, checking every line (including whitespace).""" + count = 0 + # BytesIO.readline avoids allocating a list containing every raw line. + stream = io.BytesIO(payload) + while line := stream.readline(MAX_JSONL_LINE_BYTES + 2): + content = line.removesuffix(b"\n") + _require(len(content) <= MAX_JSONL_LINE_BYTES, "jsonl_line_size") + if not content.strip(b" \t\r"): + continue + count += 1 + _require(count <= MAX_JSONL_ROWS, "jsonl_rows") + yield strict_json(content) + + +def _gunzip(payload: bytes) -> bytearray: + _require(payload[:8] == b"\x1f\x8b\x08\x00\x00\x00\x00\x00", "archive_gzip") + decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) + unpacked = bytearray() + try: + for offset in range(0, len(payload), _READ_BYTES): + chunk = payload[offset:offset + _READ_BYTES] + while chunk: + block = decoder.decompress(chunk, min(_READ_BYTES, MAX_UNPACKED_BYTES - len(unpacked) + 1)) + unpacked.extend(block) + _require(len(unpacked) <= MAX_UNPACKED_BYTES, "archive_unpacked_size") + chunk = decoder.unconsumed_tail + if decoder.eof: + _require(not decoder.unused_data and offset + _READ_BYTES >= len(payload), "archive_gzip") + break + _require(decoder.eof, "archive_gzip") + except zlib.error: + raise SnapshotError("archive_gzip") from None + return unpacked + + +def tar_header(name: str, size: int) -> bytes: + """The producer and verifier share one exact USTAR header definition.""" + info = tarfile.TarInfo(name) + info.size = size + info.mode = 0o644 + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = "" + return info.tobuf(format=tarfile.USTAR_FORMAT, encoding="ascii", errors="strict") + + +def _tar_files(raw: bytearray) -> dict[str, bytes]: + files = {} + offset = 0 + for name in MEMBERS: + header = bytes(raw[offset:offset + 512]) + _require(len(header) == 512 and any(header), "archive_member") + # Check raw fields before tarfile can hide extensions or combine prefix/name. + _require(header[:100].split(b"\0", 1)[0] == name.encode("ascii") + and header[156:157] == tarfile.REGTYPE, "archive_member") + try: + info = tarfile.TarInfo.frombuf(header, encoding="ascii", errors="strict") + except (tarfile.HeaderError, ValueError, UnicodeError): + raise SnapshotError("archive_header") from None + _integer(info.size, 0, MEMBER_LIMITS[name], "member_size") + _require(header == tar_header(name, info.size), "archive_header") + start = offset + 512 + end = start + info.size + offset = end + (-info.size % 512) + _require(offset <= len(raw), "archive_member") + _require(not any(raw[end:offset]), "archive_padding") + files[name] = bytes(raw[start:end]) + _require(len(raw) - offset >= 1024 and len(raw) % 512 == 0, "archive_member") + _require(not any(raw[offset:]), "archive_member") + return files + + +def _string(value, code: str, *, nonempty: bool = False) -> None: + _require(isinstance(value, str) and (not nonempty or bool(value.strip())), code) + + +def _page_schema(page: dict) -> None: + _string(page.get("id"), "page_id", nonempty=True) + for key in ("title", "description", "body_markdown", "type", "source_path"): + if key in page: + _string(page[key], "page_schema") + for key in ("aliases", "source_urls"): + values = page.get(key, []) + _require(isinstance(values, list), "page_schema") + for value in values: + _string(value, "page_schema") + related = page.get("related", []) + _require(isinstance(related, list), "page_schema") + for entry in related: + _require(isinstance(entry, dict), "page_schema") + _string(entry.get("id"), "page_schema", nonempty=True) + for key in ("title", "type", "relation", "url", "markdown_url", "source_path"): + if key in entry: + _string(entry[key], "page_schema") + if "source_path" in page: + _path(page["source_path"], absolute=False, code="page_path") + + +def portable_page(page: dict, canonical_site_url: str) -> dict: + """Add only presentation paths; retain the exact canonical retrieval fields.""" + _page_schema(page) + paths = { + "site_path": canonical_page_path(page.get("url"), canonical_site_url), + "markdown_path": canonical_page_path(page.get("markdown_url"), canonical_site_url), + } + for key, expected in paths.items(): + if key in page: + _require(page[key] == expected, "page_path") + return {**page, **paths} + + +def _page_chunks(page: dict): + # Kept byte-for-byte equivalent in behavior to generate_search_vectors.page_chunks + # (MAX_CHUNK_CHARS=2200). Importing that module also imports PyYAML. This tiny + # pure rule stays here until it can be extracted within shared-file ownership. + # Tests verify both boundary cases and all current generator chunk hashes. + metadata = " ".join([ + page.get("id", ""), page.get("title", ""), page.get("description", ""), + " ".join(page.get("aliases", [])), + ]).strip() + if metadata: + yield "metadata", 0, metadata + body = page.get("body_markdown") or "" + paragraphs = [item.strip() for item in re.split(r"\n{2,}", body) if item.strip()] + current, current_len, chunk_index = [], 0, 0 + for paragraph in paragraphs: + next_len = current_len + len(paragraph) + 2 + if current and next_len > 2200: + yield "body", chunk_index, "\n\n".join(current) + chunk_index += 1 + current, current_len = [], 0 + current.append(paragraph) + current_len += len(paragraph) + 2 + if current: + yield "body", chunk_index, "\n\n".join(current) + + +def _semantics(files: dict[str, bytes], site_url: str) -> dict[str, int]: + ids = set() + expected = {} + for page in jsonl_rows(files["pages.jsonl"]): + portable = portable_page(page, site_url) + _require(all(key in page and page[key] == portable[key] + for key in ("site_path", "markdown_path")), "page_path") + _require(page["id"] not in ids, "page_id") + ids.add(page["id"]) + for field, index, text in _page_chunks(page): + expected[page["id"], field, index] = sha256(text.encode("utf-8")) + _require(len(expected) <= MAX_JSONL_ROWS, "jsonl_rows") + _require(bool(ids), "corpus_empty") + count = 0 + for row in jsonl_rows(files["search-vectors.jsonl"]): + _string(row.get("id"), "vector_schema", nonempty=True) + _string(row.get("field"), "vector_schema") + _integer(row.get("chunk_index"), 0, MAX_JSONL_ROWS - 1, "vector_schema") + _integer(row.get("dim"), VECTOR_DIM, VECTOR_DIM, "vector_schema") + _require(row.get("model") == VECTOR_MODEL, "vector_schema") + identity = row["id"], row["field"], row["chunk_index"] + # Removing matched identities detects duplicates, missing chunks, and + # dangling rows with the same rule; no partially valid vectors survive. + _require(identity in expected, "vector_identity") + _hash(row.get("text_sha256"), "vector_text_hash") + _require(row["text_sha256"] == expected.pop(identity), "vector_text_hash") + encoded = row.get("vector_base64") + _require(isinstance(encoded, str) and len(encoded) == 1368, "vector_data") + try: + decoded = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError): + raise SnapshotError("vector_data") from None + _require(len(decoded) == 4 * VECTOR_DIM + and base64.b64encode(decoded).decode("ascii") == encoded, "vector_data") + _require(all(math.isfinite(value) for value in struct.unpack("<256f", decoded)), "vector_data") + count += 1 + _require(count > 0, "corpus_empty") + _require(not expected, "vector_identity") + return {"pages.jsonl": len(ids), "search-vectors.jsonl": count} + + +def _metadata(files: dict[str, bytes], manifest: dict) -> dict: + metadata = strict_json(files["metadata.json"]) + _schema(metadata, "metadata_schema") + descriptor = {key: value for key, value in metadata.items() if key != "corpus_id"} + _require(sha256(canonical_json(descriptor)) == metadata["corpus_id"], "metadata_hash") + for key in ("schema_version", "source_sha", "corpus_id", "vector_model", "vector_dim"): + _require(metadata[key] == manifest[key], "metadata_mismatch") + site_url = metadata.get("canonical_site_url") + _require(normalize_site_url(site_url) == site_url, "metadata_schema") + entries = metadata.get("files") + _require(isinstance(entries, dict) and set(entries) == set(MEMBERS[1:]), "metadata_schema") + for name, entry in entries.items(): + _require(isinstance(entry, dict), "metadata_schema") + _integer(entry.get("bytes"), 0, MEMBER_LIMITS[name], "member_size") + _hash(entry.get("sha256"), "metadata_schema") + _require(len(files[name]) == entry["bytes"], "member_size") + _require(sha256(files[name]) == entry["sha256"], "member_hash") + if name in JSONL_MEMBERS: + _integer(entry.get("rows"), 1, MAX_JSONL_ROWS, "jsonl_rows") + else: + try: + files[name].decode("utf-8") + except UnicodeError: + raise SnapshotError("json_encoding") from None + counts = _semantics(files, site_url) + for name, count in counts.items(): + _require(count == entries[name]["rows"], "jsonl_rows") + return metadata + + +def verify_archive(payload: bytes, manifest: dict) -> VerifiedSnapshot: + """Verify exact compressed bytes, framing, descriptors and complete semantics.""" + _manifest(manifest) + _require(isinstance(payload, bytes), "archive_size") + _require(len(payload) <= MAX_ARCHIVE_BYTES + and len(payload) == manifest["archive"]["bytes"], "archive_size") + digest = sha256(payload) + _require(digest == manifest["archive"]["sha256"], "archive_hash") + files = _tar_files(_gunzip(payload)) + _require(sum(map(len, files.values())) == manifest["archive"]["unpacked_bytes"], "archive_unpacked_size") + metadata = _metadata(files, manifest) + return VerifiedSnapshot(metadata=metadata, files=files, archive_sha256=digest) diff --git a/tests/mcp_snapshot_fixtures.py b/tests/mcp_snapshot_fixtures.py new file mode 100644 index 0000000..4d83dd5 --- /dev/null +++ b/tests/mcp_snapshot_fixtures.py @@ -0,0 +1,135 @@ +"""Independent, hand-authored snapshot fixtures; no production format imports.""" + +import base64 +import gzip +import hashlib +import io +import json +import struct +import tarfile +from pathlib import Path + + +SOURCE_SHA = "1" * 40 +SITE_URL = "https://v8std.ru/" +MODEL = "v8std-hash-embeddings-v1" +MEMBERS = ( + "metadata.json", "pages.jsonl", "search-vectors.jsonl", "llms.txt", "llms-full.txt", +) +BODY = ( + "# Запросы\n\n[Стандарт](https://v8std.ru/std/437/?view=full#query)\n\n" + '```bsl\nАдрес = "https://v8std.ru/std/437/";\n```' +) + + +def json_bytes(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def jsonl_bytes(rows): + return b"".join(json_bytes(row) + b"\n" for row in rows) + + +def sha256(payload): + return hashlib.sha256(payload).hexdigest() + + +def page_fixture(*, portable=True, site_url=SITE_URL): + page = { + "id": "std437", "type": "standard", "title": "Запросы", + "description": "Параметры", "aliases": ["#std437"], + "url": site_url + "std/437/", "markdown_url": site_url + "std/437.md", + "source_path": "std/437.md", "source_urls": ["https://its.1c.ru/db/v8std#437"], + "related": [], "body_markdown": BODY, + } + if portable: + page.update(site_path="std/437/", markdown_path="std/437.md") + return page + + +def vector_fixtures(): + # A valid finite one-hot vector, intentionally not generated by the producer. + vector = base64.b64encode(struct.pack("<256f", 1.0, *([0.0] * 255))).decode("ascii") + return [ + {"id": "std437", "field": field, "chunk_index": 0, + "text_sha256": sha256(text.encode("utf-8")), "model": MODEL, + "dim": 256, "vector_base64": vector} + for field, text in [("metadata", "std437 Запросы Параметры #std437"), ("body", BODY)] + ] + + +def corpus_files(*, pages=None, vectors=None): + return { + "pages.jsonl": jsonl_bytes([page_fixture()] if pages is None else pages), + "search-vectors.jsonl": jsonl_bytes(vector_fixtures() if vectors is None else vectors), + "llms.txt": "# Стандарты\n\n[Запросы](https://v8std.ru/std/437/)\n".encode(), + "llms-full.txt": (BODY + "\n").encode(), + } + + +def with_metadata(files, *, site_url=SITE_URL, mutate=None): + descriptor = { + "schema_version": 1, "source_sha": SOURCE_SHA, "canonical_site_url": site_url, + "vector_model": MODEL, "vector_dim": 256, + "files": { + name: {"sha256": sha256(payload), "bytes": len(payload), + **({"rows": sum(bool(line.strip()) for line in payload.split(b"\n"))} + if name.endswith(".jsonl") else {})} + for name, payload in files.items() + }, + } + if mutate: + mutate(descriptor) + metadata = {**descriptor, "corpus_id": sha256(json_bytes(descriptor))} + return {"metadata.json": json_bytes(metadata), **files} + + +def tar_bytes(files, *, members=None, tar_format=tarfile.USTAR_FORMAT): + """members may contain hand-crafted TarInfo entries, including hostile ones.""" + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w", format=tar_format) as archive: + for member in MEMBERS if members is None else members: + if isinstance(member, str): + info = tarfile.TarInfo(member) + content = files[member] + info.size = len(content) + info.mode = 0o644 + else: + info, content = member + archive.addfile(info, io.BytesIO(content)) + return output.getvalue() + + +def gzip_bytes(payload, *, filename="", mtime=0): + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb", filename=filename, mtime=mtime, + compresslevel=9) as archive: + archive.write(payload) + return output.getvalue() + + +def manifest_for(archive, files): + metadata = json.loads(files["metadata.json"]) + digest = sha256(archive) + return { + "schema_version": 1, "source_sha": metadata["source_sha"], + "corpus_id": metadata["corpus_id"], "vector_model": MODEL, "vector_dim": 256, + "archive": {"path": digest + "/snapshot.tar.gz", "sha256": digest, + "bytes": len(archive), "unpacked_bytes": sum(map(len, files.values()))}, + } + + +def snapshot_fixture(*, files=None, members=None, tar_format=tarfile.USTAR_FORMAT): + files = with_metadata(corpus_files()) if files is None else files + archive = gzip_bytes(tar_bytes(files, members=members, tar_format=tar_format)) + return archive, manifest_for(archive, files) + + +def write_docs(root: Path, *, site_url=SITE_URL): + files = corpus_files(pages=[page_fixture(portable=False, site_url=site_url)]) + for name, payload in files.items(): + path = root / ("ai" if name.endswith(".jsonl") else "") / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return files diff --git a/tests/test_v8std_mcp_snapshot_format.py b/tests/test_v8std_mcp_snapshot_format.py new file mode 100644 index 0000000..62a3826 --- /dev/null +++ b/tests/test_v8std_mcp_snapshot_format.py @@ -0,0 +1,543 @@ +import base64 +import copy +import gzip +import hashlib +import importlib +import importlib.util +import io +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from tests import mcp_snapshot_fixtures as fixture + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + + +class SnapshotFormatTests(unittest.TestCase): + def modules(self): + self.assertIsNotNone(importlib.util.find_spec("generate_mcp_snapshot")) + return (importlib.import_module("v8std_mcp_snapshot_format"), + importlib.import_module("generate_mcp_snapshot")) + + def verify_rejected(self, archive, manifest, code=None): + fmt, _ = self.modules() + with self.assertRaises(fmt.SnapshotError) as caught: + fmt.verify_archive(archive, manifest) + self.assertRegex(caught.exception.code, r"^[a-z_]{1,64}$") + self.assertEqual(str(caught.exception), caught.exception.code) + if code: + self.assertEqual(caught.exception.code, code) + + def test_00_producer_exists_and_rebuilds_identical_archive(self): + # This assertion must run before any production module import in RED. + self.assertIsNotNone(importlib.util.find_spec("generate_mcp_snapshot")) + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + docs = Path(directory) + original = fixture.write_docs(docs) + first, manifest = producer.build_snapshot(docs, fixture.SOURCE_SHA, fixture.SITE_URL) + second, again = producer.build_snapshot(docs, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(first, second) + self.assertEqual(manifest, again) + verified = fmt.verify_archive(first, fmt.validate_manifest(fixture.json_bytes(manifest))) + page = json.loads(verified.files["pages.jsonl"]) + self.assertEqual(page["site_path"], "std/437/") + self.assertEqual(page["markdown_path"], "std/437.md") + self.assertEqual(verified.metadata["vector_dim"], 256) + self.assertEqual(page["body_markdown"], fixture.BODY) + for name in ("search-vectors.jsonl", "llms.txt", "llms-full.txt"): + self.assertEqual(verified.files[name], original[name]) + self.assertEqual((docs / "ai/pages.jsonl").read_bytes(), original["pages.jsonl"]) + + def test_independent_fixture_and_descriptor_hash(self): + fmt, _ = self.modules() + archive, manifest = fixture.snapshot_fixture() + verified = fmt.verify_archive(archive, manifest) + descriptor = dict(verified.metadata) + corpus_id = descriptor.pop("corpus_id") + self.assertEqual(corpus_id, fixture.sha256(fixture.json_bytes(descriptor))) + self.assertEqual(verified.archive_sha256, fixture.sha256(archive)) + self.assertNotEqual(corpus_id, verified.archive_sha256) + self.assertEqual(tuple(verified.files), fixture.MEMBERS) + + def test_tar_gzip_metadata_is_exact_and_deterministic(self): + _, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + docs = Path(directory) + fixture.write_docs(docs) + archive, manifest = producer.build_snapshot(docs, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(archive[:10], b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff") + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + members = tar.getmembers() + self.assertEqual([m.name for m in members], list(fixture.MEMBERS)) + for member in members: + self.assertEqual((member.uid, member.gid, member.mode, member.mtime), (0, 0, 0o644, 0)) + self.assertEqual((member.uname, member.gname), ("", "")) + self.assertEqual(member.type, tarfile.REGTYPE) + self.assertEqual(member.pax_headers, {}) + self.assertEqual(sum(m.size for m in members), manifest["archive"]["unpacked_bytes"]) + + def test_normalized_site_url_and_prefix(self): + fmt, producer = self.modules() + for given, expected in [ + (" HTTPS://V8STD.RU:443/knowledge ", "https://v8std.ru/knowledge/"), + ("http://LOCALHOST:80/base/", "http://localhost/base/"), + ("http://[::1]:8080/base", "http://[::1]:8080/base/"), + ("https://example.org", "https://example.org/"), + ]: + with self.subTest(given=given): + self.assertEqual(fmt.normalize_site_url(given), expected) + self.assertEqual(fmt.normalize_site_url(expected), expected) + with tempfile.TemporaryDirectory() as directory: + docs = Path(directory) + fixture.write_docs(docs, site_url="https://example.org/knowledge/") + archive, manifest = producer.build_snapshot(docs, fixture.SOURCE_SHA, + "HTTPS://EXAMPLE.ORG:443/knowledge") + page = json.loads(fmt.verify_archive(archive, manifest).files["pages.jsonl"]) + self.assertEqual(page["site_path"], "std/437/") + self.assertEqual(page["markdown_path"], "std/437.md") + + def test_site_url_rejects_ambiguous_or_credentialed_inputs(self): + fmt, _ = self.modules() + for value in ["", " ", None, 1, "ftp://host/", "//host/base", "https://user:secret@host/", + "https://host/?", "https://host/#", "https://host/a/../b", "https://host/./", + "https://host/%2e%2e/", "https://host/a%2fb/", "https://host/%5c/", + "https://host/%252e%252e/", "https://host/a\\b", "https://host//base/", + "https://ho\nst/", "https://host:abc/", "https://host:65536/", + "https://host/%", "https://host/%00/", "https://host/a b/"]: + with self.subTest(value=value), self.assertRaisesRegex(fmt.SnapshotError, "site_url"): + fmt.normalize_site_url(value) + + def test_manifest_schema_types_hashes_and_path(self): + fmt, _ = self.modules() + _, manifest = fixture.snapshot_fixture() + variants = [] + for key, value in [("schema_version", True), ("schema_version", 2), ("source_sha", "abc"), + ("corpus_id", "A" * 64), ("vector_dim", 128), ("vector_dim", 256.0), + ("vector_model", "other")]: + variants.append({**manifest, key: value}) + for key, value in [("bytes", True), ("bytes", -1), ("unpacked_bytes", "32"), + ("sha256", "g" * 64), ("path", "../snapshot.tar.gz"), + ("path", "//evil.test/snapshot.tar.gz"), ("path", "https://evil.test/a"), + ("path", "0" * 64 + "/snapshot.tar.gz")]: + variants.append({**manifest, "archive": {**manifest["archive"], key: value}}) + for value in variants + [[], None, {k: v for k, v in manifest.items() if k != "archive"}]: + with self.subTest(value=value), self.assertRaises(fmt.SnapshotError): + fmt.validate_manifest(fixture.json_bytes(value)) + optional = {**manifest, "future_optional": {"enabled": True, "weight": 0.5}} + self.assertEqual(fmt.validate_manifest(fixture.json_bytes(optional))["corpus_id"], manifest["corpus_id"]) + + def test_direct_verifier_cannot_bypass_manifest_byte_budget(self): + archive, manifest = fixture.snapshot_fixture() + self.verify_rejected(archive, {**manifest, "future_optional": "x" * (64 * 1024)}, "manifest_size") + + def test_strict_json_duplicate_keys_depth_utf8_and_numbers(self): + fmt, _ = self.modules() + _, manifest = fixture.snapshot_fixture() + raw = fixture.json_bytes(manifest) + invalid = [b"\xff", raw[:-1] + b',"schema_version":1}', + raw[:-1] + b',"extra":{"x":1,"x":2}}', + raw[:-1] + b',"extra":NaN}', raw[:-1] + b',"extra":1e9999}', + raw[:-1] + b',"extra":"\\ud800"}', + raw[:-1] + b',"extra":' + b"[" * 32 + b"0" + b"]" * 32 + b"}"] + for value in invalid: + with self.subTest(value=value[:40]), self.assertRaises(fmt.SnapshotError): + fmt.validate_manifest(value) + valid = raw[:-1] + b',"extra":' + b"[" * 31 + b"0" + b"]" * 31 + b"}" + fmt.validate_manifest(valid) + + def test_independent_hostile_tar_members(self): + files = fixture.with_metadata(fixture.corpus_files()) + hostile = [] + for kind in [tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.DIRTYPE, + tarfile.CHRTYPE, tarfile.BLKTYPE, tarfile.FIFOTYPE, + tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.GNUTYPE_LONGNAME, + tarfile.GNUTYPE_SPARSE]: + info = tarfile.TarInfo("metadata.json") + info.type = kind + info.linkname = "secret" + hostile.append([(info, b""), *fixture.MEMBERS[1:]]) + for name in ["../metadata.json", "/metadata.json", "./metadata.json", "a/../metadata.json"]: + info = tarfile.TarInfo(name) + info.size = len(files["metadata.json"]) + hostile.append([(info, files["metadata.json"]), *fixture.MEMBERS[1:]]) + hostile += [list(reversed(fixture.MEMBERS)), list(fixture.MEMBERS[:-1]), + [*fixture.MEMBERS, "llms.txt"], ["metadata.json", *fixture.MEMBERS]] + for members in hostile: + with self.subTest(members=members): + archive, manifest = fixture.snapshot_fixture(files=files, members=members) + self.verify_rejected(archive, manifest, "archive_member") + + def test_gzip_trailing_members_truncation_and_headers(self): + files = fixture.with_metadata(fixture.corpus_files()) + archive, _ = fixture.snapshot_fixture(files=files) + raw = fixture.tar_bytes(files) + for bad in [archive + b"secret", archive + gzip.compress(b""), archive[:-1], + fixture.gzip_bytes(raw, filename="untrusted"), fixture.gzip_bytes(raw, mtime=1), + archive[:-8] + bytes([archive[-8] ^ 1]) + archive[-7:]]: + with self.subTest(size=len(bad)): + self.verify_rejected(bad, fixture.manifest_for(bad, files)) + + def test_tar_checksum_padding_and_hidden_trailing_payload(self): + files = fixture.with_metadata(fixture.corpus_files()) + raw = fixture.tar_bytes(files) + corrupted = bytearray(raw) + corrupted[148] ^= 1 + padded = bytearray(raw) + padded[512 + len(files["metadata.json"])] = 1 + for bad in [bytes(corrupted), bytes(padded), raw + b"secret", raw + raw, + raw[:512], raw[:512 + len(files["metadata.json"])]]: + archive = fixture.gzip_bytes(bad) + self.verify_rejected(archive, fixture.manifest_for(archive, files)) + + def test_extensions_and_noncanonical_tar_header_fields(self): + files = fixture.with_metadata(fixture.corpus_files()) + for field, value in [("uid", 12), ("gid", 12), ("mode", 0o777), ("mtime", 1), + ("uname", "secret"), ("gname", "secret")]: + info = tarfile.TarInfo("metadata.json") + info.mode, info.size = 0o644, len(files["metadata.json"]) + setattr(info, field, value) + self.verify_rejected(*fixture.snapshot_fixture( + files=files, members=[(info, files["metadata.json"]), *fixture.MEMBERS[1:]])) + info = tarfile.TarInfo("metadata.json") + info.mode, info.size = 0o644, len(files["metadata.json"]) + info.pax_headers = {"comment": "hidden extension"} + self.verify_rejected(*fixture.snapshot_fixture( + files=files, members=[(info, files["metadata.json"]), *fixture.MEMBERS[1:]], + tar_format=tarfile.PAX_FORMAT), "archive_member") + self.verify_rejected(*fixture.snapshot_fixture(files=files, tar_format=tarfile.GNU_FORMAT)) + + def test_member_headers_enforce_every_real_limit_before_reading(self): + fmt, _ = self.modules() + files = fixture.with_metadata(fixture.corpus_files()) + original = fixture.tar_bytes(files) + offset = 0 + for name, maximum in fmt.MEMBER_LIMITS.items(): + info = tarfile.TarInfo(name) + info.mode, info.size = 0o644, maximum + 1 + raw = original[:offset] + info.tobuf(format=tarfile.USTAR_FORMAT) + original[offset + 512:] + archive = fixture.gzip_bytes(raw) + self.verify_rejected(archive, fixture.manifest_for(archive, files), "member_size") + offset += 512 + len(files[name]) + (-len(files[name]) % 512) + + def test_archive_hash_size_and_unpacked_count_are_independent(self): + archive, manifest = fixture.snapshot_fixture() + for key, value in [("sha256", "0" * 64), ("bytes", len(archive) + 1), + ("unpacked_bytes", manifest["archive"]["unpacked_bytes"] + 1)]: + bad = copy.deepcopy(manifest) + bad["archive"][key] = value + if key == "sha256": + bad["archive"]["path"] = value + "/snapshot.tar.gz" + self.verify_rejected(archive, bad) + + def test_metadata_descriptors_counts_and_manifest_agreement(self): + def bad_size(meta): + meta["files"]["pages.jsonl"]["bytes"] += 1 + def bad_rows(meta): + meta["files"]["search-vectors.jsonl"]["rows"] += 1 + def bad_hash(meta): + meta["files"]["llms.txt"]["sha256"] = "0" * 64 + def missing_file(meta): + del meta["files"]["llms.txt"] + for mutate in [bad_size, bad_rows, bad_hash, missing_file, + lambda m: m.update(vector_dim=128), + lambda m: m.update(schema_version=True), + lambda m: m.update(extra=1.5), + lambda m: m.update(canonical_site_url="https://v8std.ru/../")]: + with self.subTest(mutate=mutate): + files = fixture.with_metadata(fixture.corpus_files(), mutate=mutate) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + files = fixture.with_metadata(fixture.corpus_files()) + metadata = json.loads(files["metadata.json"]) + metadata["corpus_id"] = "0" * 64 + files["metadata.json"] = fixture.json_bytes(metadata) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + archive, manifest = fixture.snapshot_fixture() + self.verify_rejected(archive, {**manifest, "source_sha": "2" * 40}) + + def test_page_schema_identity_and_portable_path_validation(self): + page = fixture.page_fixture() + for key, value in [("id", []), ("title", 4), ("body_markdown", []), + ("aliases", "std437"), ("aliases", [None]), ("related", ["bad"]), + ("source_urls", [1]), ("site_path", "../437/"), + ("site_path", "https://evil.test/"), ("site_path", "std%2f437/"), + ("site_path", "std/other/"), ("markdown_path", "/std/437.md"), + ("url", "https://evil.test/std/437/")]: + with self.subTest(key=key, value=value): + files = fixture.with_metadata(fixture.corpus_files(pages=[{**page, key: value}])) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + for pages in [[], [page, page]]: + files = fixture.with_metadata(fixture.corpus_files(pages=pages)) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + + def test_vector_semantics_include_every_original_chunk(self): + rows = fixture.vector_fixtures() + corruptions = [("id", "missing"), ("field", "unknown"), ("chunk_index", True), + ("chunk_index", -1), ("chunk_index", 1), ("text_sha256", "0" * 64), + ("model", "unsupported"), ("dim", 128), ("dim", 256.0), + ("vector_base64", "not base64!"), ("vector_base64", "AA==")] + for number in [float("nan"), float("inf"), -float("inf")]: + corruptions.append(("vector_base64", base64.b64encode( + struct.pack("<256f", number, *([0.0] * 255))).decode())) + for key, value in corruptions: + with self.subTest(key=key, value=str(value)[:24]): + vectors = [{**rows[0], key: value}, rows[1]] + files = fixture.with_metadata(fixture.corpus_files(vectors=vectors)) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + for vectors in [[], rows[:1], rows + [rows[0]]]: + files = fixture.with_metadata(fixture.corpus_files(vectors=vectors)) + self.verify_rejected(*fixture.snapshot_fixture(files=files)) + + def test_jsonl_duplicate_keys_invalid_utf8_and_nonobject_rows(self): + for name in ["pages.jsonl", "search-vectors.jsonl"]: + for payload in [b"[]\n", b"null\n", b"\xff\n", b'{"id":"a","id":"b"}\n']: + files = fixture.corpus_files() + files[name] = payload + self.verify_rejected(*fixture.snapshot_fixture(files=fixture.with_metadata(files))) + + def test_real_line_row_manifest_and_compressed_limits(self): + fmt, _ = self.modules() + archive, manifest = fixture.snapshot_fixture() + with self.assertRaisesRegex(fmt.SnapshotError, "manifest_size"): + fmt.validate_manifest(fixture.json_bytes(manifest) + b" " * (64 * 1024)) + too_large = b"x" * (16 * 1024 * 1024 + 1) + oversized = copy.deepcopy(manifest) + digest = fixture.sha256(too_large) + oversized["archive"].update(bytes=len(too_large), sha256=digest, path=digest + "/snapshot.tar.gz") + self.verify_rejected(too_large, oversized, "archive_size") + for name in ("pages.jsonl", "search-vectors.jsonl"): + files = fixture.corpus_files() + files[name] = b" " * (1024 * 1024 + 1) + b"\n" + files[name] + self.verify_rejected(*fixture.snapshot_fixture(files=fixture.with_metadata(files)), "jsonl_line_size") + files = fixture.corpus_files() + files["search-vectors.jsonl"] = b"{}\n" * 100001 + self.verify_rejected(*fixture.snapshot_fixture(files=fixture.with_metadata(files)), "jsonl_rows") + + def test_chunk_boundaries_match_existing_generator_without_reembedding(self): + fmt, _ = self.modules() + generator = importlib.import_module("generate_search_vectors") + for length in [2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 4400]: + with self.subTest(length=length): + page = fixture.page_fixture() + page["body_markdown"] = " " + "x" * length + "\n\nя\n\n尾 \n" + rows = [] + for field, index, text in generator.page_chunks(page): + row = dict(fixture.vector_fixtures()[0]) + row.update(field=field, chunk_index=index, text_sha256=fixture.sha256(text.encode())) + rows.append(row) + archive, manifest = fixture.snapshot_fixture(files=fixture.with_metadata( + fixture.corpus_files(pages=[page], vectors=rows))) + fmt.verify_archive(archive, manifest) + + def test_contract_budget_constants_and_boundary_enforcement(self): + fmt, _ = self.modules() + self.assertEqual(fmt.MAX_MANIFEST_BYTES, 64 * 1024) + self.assertEqual(fmt.MAX_ARCHIVE_BYTES, 16 * 1024 * 1024) + self.assertEqual(fmt.MAX_UNPACKED_BYTES, 64 * 1024 * 1024) + self.assertEqual(fmt.MAX_JSONL_LINE_BYTES, 1024 * 1024) + self.assertEqual(fmt.MAX_JSONL_ROWS, 100000) + self.assertEqual(fmt.MAX_JSON_DEPTH, 32) + self.assertEqual(fmt.MEMBER_LIMITS, dict(zip(fixture.MEMBERS, + [64 * 1024, 16 * 1024**2, 32 * 1024**2, 4 * 1024**2, 16 * 1024**2]))) + archive, manifest = fixture.snapshot_fixture() + raw_manifest = fixture.json_bytes(manifest) + with patch.object(fmt, "MAX_MANIFEST_BYTES", len(raw_manifest)): + fmt.validate_manifest(raw_manifest) + with patch.object(fmt, "MAX_MANIFEST_BYTES", len(raw_manifest) - 1): + with self.assertRaises(fmt.SnapshotError): + fmt.validate_manifest(raw_manifest) + with patch.object(fmt, "MAX_ARCHIVE_BYTES", len(archive) - 1): + self.verify_rejected(archive, manifest) + files = fixture.with_metadata(fixture.corpus_files()) + for name in fixture.MEMBERS: + with self.subTest(member=name), patch.dict(fmt.MEMBER_LIMITS, {name: len(files[name]) - 1}): + self.verify_rejected(archive, manifest) + with patch.object(fmt, "MAX_JSONL_ROWS", 1): + self.verify_rejected(archive, manifest) + with patch.object(fmt, "MAX_JSONL_LINE_BYTES", 20): + self.verify_rejected(archive, manifest) + # Files fit, framing does not: the decompression counter must include tar bytes. + with patch.object(fmt, "MAX_UNPACKED_BYTES", manifest["archive"]["unpacked_bytes"] + 1): + self.verify_rejected(archive, manifest) + + def test_real_decompressed_budget_includes_zero_tar_framing(self): + files = fixture.with_metadata(fixture.corpus_files()) + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb", filename="", mtime=0) as stream: + stream.write(fixture.tar_bytes(files)) + for _ in range(64): + stream.write(b"\0" * (1024 * 1024)) + archive = output.getvalue() + self.verify_rejected(archive, fixture.manifest_for(archive, files), "archive_unpacked_size") + + def test_publish_local_public_atomic_order_and_immutable_reuse(self): + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + docs, output = root / "docs", root / "output" + fixture.write_docs(docs) + manifest_path = producer.publish_snapshot(docs, output, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(manifest_path, output / "manifest.json") + manifest = fmt.validate_manifest(manifest_path.read_bytes()) + archive_path = output / manifest["archive"]["path"] + before = archive_path.stat() + local_bytes = archive_path.read_bytes() + producer.publish_snapshot(docs, output, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(archive_path.stat().st_mtime_ns, before.st_mtime_ns) + public = producer.publish_snapshot(docs, root / "public", fixture.SOURCE_SHA, + fixture.SITE_URL, public_delivery=True) + public_manifest = fmt.validate_manifest(public.read_bytes()) + self.assertEqual(public_manifest["archive"]["path"], + "https://ai.v8std.ru/indexes/v1/" + manifest["archive"]["path"]) + self.assertEqual((public.parent / manifest["archive"]["path"]).read_bytes(), local_bytes) + previous_manifest = manifest_path.read_bytes() + archive_path.write_bytes(b"corrupt") + with self.assertRaises(fmt.SnapshotError): + producer.publish_snapshot(docs, output, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(archive_path.read_bytes(), b"corrupt") + self.assertEqual(manifest_path.read_bytes(), previous_manifest) + + def test_manifest_is_replaced_only_after_verified_archive_exists(self): + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + original_replace = os.replace + observed = [] + def inspect_replace(source, target): + if Path(target).name == "manifest.json": + manifest = fmt.validate_manifest(Path(source).read_bytes()) + archive = (Path(target).parent / manifest["archive"]["path"]).read_bytes() + fmt.verify_archive(archive, manifest) + observed.append(True) + return original_replace(source, target) + with patch.object(producer.os, "replace", side_effect=inspect_replace): + producer.publish_snapshot(root / "docs", root / "output", fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(observed, [True]) + + def test_publish_failure_retains_previous_manifest_and_removes_temporary_files(self): + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + manifest_path = producer.publish_snapshot(root / "docs", root / "output", + fixture.SOURCE_SHA, fixture.SITE_URL) + before = manifest_path.read_bytes() + for target in ("link", "replace", "fsync"): + with self.subTest(target=target), patch.object(producer.os, target, side_effect=OSError("secret")): + with self.assertRaisesRegex(fmt.SnapshotError, "publish_io") as caught: + producer.publish_snapshot(root / "docs", root / "output", "2" * 40, fixture.SITE_URL) + self.assertNotIn("secret", str(caught.exception)) + self.assertEqual(manifest_path.read_bytes(), before) + self.assertEqual(list((root / "output").rglob(".snapshot-*")), []) + old = fmt.validate_manifest(before) + fmt.verify_archive((root / "output" / old["archive"]["path"]).read_bytes(), old) + + def test_existing_immutable_symlinks_are_rejected(self): + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + archive, manifest = producer.build_snapshot(root / "docs", fixture.SOURCE_SHA, fixture.SITE_URL) + output = root / "output" + output.mkdir() + foreign = root / "foreign" + foreign.mkdir() + (foreign / "snapshot.tar.gz").write_bytes(archive) + target = output / manifest["archive"]["sha256"] + target.symlink_to(foreign, target_is_directory=True) + with self.assertRaisesRegex(fmt.SnapshotError, "immutable_conflict"): + producer.publish_snapshot(root / "docs", output, fixture.SOURCE_SHA, fixture.SITE_URL) + target.unlink() + target.mkdir() + (target / "snapshot.tar.gz").symlink_to(foreign / "snapshot.tar.gz") + with self.assertRaisesRegex(fmt.SnapshotError, "immutable_conflict"): + producer.publish_snapshot(root / "docs", output, fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual((foreign / "snapshot.tar.gz").read_bytes(), archive) + self.assertFalse((output / "manifest.json").exists()) + + def test_invalid_source_corpus_cannot_publish_manifest(self): + fmt, producer = self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + (root / "docs/ai/search-vectors.jsonl").write_bytes(fixture.jsonl_bytes(fixture.vector_fixtures()[:1])) + with self.assertRaises(fmt.SnapshotError): + producer.publish_snapshot(root / "docs", root / "output", fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertFalse((root / "output/manifest.json").exists()) + fixture.write_docs(root / "docs", site_url="https://example.test/base/") + with self.assertRaisesRegex(fmt.SnapshotError, "archive_path"): + producer.publish_snapshot(root / "docs", root / "output", fixture.SOURCE_SHA, + "https://example.test/base/", public_delivery=True) + + def test_cli_site_precedence_public_delivery_and_explicit_empty_env(self): + self.modules() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + command = [sys.executable, "-S", str(ROOT / "scripts/generate_mcp_snapshot.py"), + "--docs", str(root / "docs"), "--output", str(root / "output"), + "--source-sha", fixture.SOURCE_SHA] + environment = {**os.environ, "V8STD_MCP_SITE_URL": ""} + result = subprocess.run(command, env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stderr.strip(), "site_url") + result = subprocess.run(command + ["--site-url", fixture.SITE_URL, "--public-delivery"], + env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((root / "output/manifest.json").read_bytes()) + self.assertTrue(manifest["archive"]["path"].startswith("https://ai.v8std.ru/indexes/v1/")) + + def test_cli_and_stdlib_only_import(self): + self.modules() + environment = {**os.environ, "PYTHONPATH": str(ROOT / "scripts")} + result = subprocess.run([sys.executable, "-S", "-c", + "import generate_mcp_snapshot, v8std_mcp_snapshot_format"], + env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root / "docs") + command = [sys.executable, "-S", str(ROOT / "scripts/generate_mcp_snapshot.py"), + "--docs", str(root / "docs"), "--output", str(root / "output"), + "--source-sha", fixture.SOURCE_SHA, "--site-url", fixture.SITE_URL] + result = subprocess.run(command, env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / "output/manifest.json").is_file()) + result = subprocess.run(command[:-1] + [""], env=environment, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + + def test_current_corpus_round_trip_and_chunk_rule_parity(self): + fmt, producer = self.modules() + original_vectors = (ROOT / "docs/ai/search-vectors.jsonl").read_bytes() + archive, manifest = producer.build_snapshot(ROOT / "docs", fixture.SOURCE_SHA, fixture.SITE_URL) + second, _ = producer.build_snapshot(ROOT / "docs", fixture.SOURCE_SHA, fixture.SITE_URL) + self.assertEqual(archive, second) + verified = fmt.verify_archive(archive, manifest) + self.assertEqual(verified.files["search-vectors.jsonl"], original_vectors) + original_pages = [json.loads(line) for line in (ROOT / "docs/ai/pages.jsonl").read_bytes().splitlines()] + pages = [json.loads(line) for line in verified.files["pages.jsonl"].splitlines()] + self.assertEqual(len(pages), len(original_pages)) + generator = importlib.import_module("generate_search_vectors") + expected_hashes = {(p["id"], field, index): hashlib.sha256(text.encode()).hexdigest() + for p in original_pages for field, index, text in generator.page_chunks(p)} + for page, original in zip(pages, original_pages): + self.assertEqual({k: v for k, v in page.items() if k not in {"site_path", "markdown_path"}}, original) + for line in verified.files["search-vectors.jsonl"].splitlines(): + row = json.loads(line) + self.assertEqual(row["text_sha256"], expected_hashes.pop((row["id"], row["field"], row["chunk_index"]))) + self.assertEqual(expected_hashes, {}) + + +if __name__ == "__main__": + unittest.main() From 6a40b4c622fcf46805ac636f5e969aba8ef366cb Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:37:47 +0300 Subject: [PATCH 04/88] docs: clarify cache activation and plan CI policy migration --- ...026-09-10-mcp-ci-deployment-policy-plan.md | 89 +++++++++++++++++++ ...6-09-10-mcp-container-distribution-plan.md | 5 +- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md diff --git a/spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md b/spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md new file mode 100644 index 0000000..9811312 --- /dev/null +++ b/spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md @@ -0,0 +1,89 @@ +--- +schema_version: 1 +kind: plan +id: mcp-ci-deployment-policy +design: design:mcp-ci-deployment-policy +implements: + - design:mcp-ci-deployment-policy + - process:architecture-artifacts@2 +requirements: + - MCP_SERVER_DEPLOYS_AUTOMATICALLY_FROM_VERIFIED_MAIN + - MCP_AUTODEPLOY_ACTIVATION_IS_CONTROLLED +--- + +# MCP CI Deployment Policy Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Согласованно включить process v2 в инструменты репозитория и код CI, сохраняя отдельный gate первичной активации production. + +**Architecture:** Структура архитектурного графа не меняется; его текущая нормативная версия становится v2. Инструкции, проверки и CI различают разрешённый main release, ещё не включённую автоматизацию и неавторизованные источники запуска. + +**Tech Stack:** Существующий Python architecture CLI/unittest, GitHub Actions YAML, Markdown process instructions. + +**Spec:** [Deployment policy design](../designs/2026-09-10-mcp-ci-deployment-policy-design.md), [process v2](../process/architecture-artifacts-v2.md). + +## Global Constraints + +- Structured documents из main не меняются: process v1 и прежние plans остаются историческими свидетельствами. +- Push локального main требует явного запроса. До первичной активации MCP deploy требует отдельного запроса. +- После активации только проверенный SHA main и опубликованный digest поступают в обычный автоматический rollout. +- PR/fork/tag/устаревший run не получают право менять production. +- Не создавать worktree, не менять repository permissions/secrets и не запускать live deploy при реализации. +- Этот plan исполняется в Task 6 продуктового implementation plan; второго независимого исполнителя для тех же файлов не создавать. + +### Task 1: Normative pointer and current instructions + +**Files:** modify `scripts/v8std_architecture_model.py`, `AGENTS.md`, +`spec/README.md`, `.agents/skills/v8std-architecture/SKILL.md` and its relevant +references; update current policy assertions in `tests/test_v8std_architecture_repository.py`, +`tests/test_v8std_architecture_process.py`, and CLI fixtures when needed. + +**Interface:** `load_process_schema(root)` loads v2; an explicitly constructed +test repository must supply its declared current schema rather than depend on +accidentally present checkout files. Graph reference semantics stay unchanged. + +- [ ] Write/run RED: a temporary repository with process v2 validates via CLI; + frozen v1 remains protected, and schema fields equal the prior schema. Use + actual CLI behavior and graph validation, not only a string-presence assertion. +- [ ] Implement the current schema pointer and synchronize policy prose. Replace + only the current manual-every-release rule with conditional automatic delivery + after activation; preserve branch, approval, plan, freeze, merge, push and + external-mutation boundaries. + +```python +PROCESS_SCHEMA_PATH = Path("spec/process/architecture-artifacts-v2.md") +``` + +- [ ] Run `.venv/bin/python -m unittest tests.test_v8std_architecture_cli tests.test_v8std_architecture_model tests.test_v8std_architecture_process tests.test_v8std_architecture_repository tests.test_v8std_architecture_validation -v`; prove no frozen document from main changed with architecture `validate --base-ref main`. + +### Task 2: Executable publication eligibility and fail-closed activation + +**Files:** publication workflows, `scripts/publish_mcp_artifacts.py`, +`tests/test_mcp_publication.py`, activation/verification operations documents. +These files are owned by product Task 6, not edited by a concurrent worker. + +**Interface:** release eligibility consumes event/ref/repository, successful +gate results and explicit activation state; outputs an allowed action set. +Missing or malformed fields deny production mutation. Host validates exact +envelope sequence/digests independently of workflow eligibility. + +- [ ] Write/run RED table tests for push/main with activation, push/main before + activation, PR, fork, tag, failed gates and stale sequence. Controlled publisher + adapter records actual attempted object/manifest/host operations: + +```python +self.assertEqual(result.production_actions, []) +self.assertEqual(host.accepted_releases, []) +self.assertTrue(previous_manifest_path.is_file()) +``` + +- [ ] Implement/pin workflows, bounded typed publication helper and disabled-by-default + activation. Preserve Pages-only publishing before activation; do not publish + a public manifest for a missing ai object or promote a public-default MCP + release without a ready source. Describe required main protection/environment + settings as explicit external prerequisites, not already configured facts. +- [ ] Run publication and architecture tests, then strict build before full suite. + Record gate results and explicit absence of live activation in operations + evidence; mark this plan complete only with its Task 6 review. External setup, + push and rollout remain integration operations outside these checkboxes. diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 014e321..256eb19 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -153,7 +153,7 @@ with self.assertRaisesRegex(SnapshotError, "archive_member"): class SnapshotStore: def __init__(self, site_url: str, cache_dir: Path): ... def cached(self) -> VerifiedSnapshot | None: ... - def refresh(self) -> VerifiedSnapshot: ... + def refresh(self, *, prepare=None): ... class SnapshotCoordinator: def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600): ... @@ -164,6 +164,9 @@ class SnapshotCoordinator: ``` `build: Callable[[VerifiedSnapshot], Any]` constructs an immutable generation. +`SnapshotStore.refresh(prepare=build)` invokes build before the durable cache +commit and returns its result; without prepare it returns VerifiedSnapshot. +Preparation failure preserves the former disk pointer as well as process state. The coordinator owns the active reference and stores network/parse work outside the request path. Prepare in a bounded worker with a lifecycle that is stopped on close; no network/CPU build on the ASGI event loop. Test/store internals may From 70e8bd5308d9e07c8b771bfc5485b2e3614b28e9 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:50:47 +0300 Subject: [PATCH 05/88] fix: share MCP chunk rules and verify fsync failure stages --- scripts/generate_search_vectors.py | 37 +------ scripts/v8std_mcp_chunks.py | 42 ++++++++ scripts/v8std_mcp_snapshot_format.py | 30 +----- tests/test_v8std_mcp_snapshot_format.py | 135 +++++++++++++++++++++++- 4 files changed, 179 insertions(+), 65 deletions(-) create mode 100644 scripts/v8std_mcp_chunks.py diff --git a/scripts/generate_search_vectors.py b/scripts/generate_search_vectors.py index c0dd479..3dfba6a 100755 --- a/scripts/generate_search_vectors.py +++ b/scripts/generate_search_vectors.py @@ -7,18 +7,16 @@ import hashlib import json import math -import re import struct from pathlib import Path -from typing import Any +from v8std_mcp_chunks import MAX_CHUNK_CHARS, page_chunks from v8std_retrieval_rules import tokenize from atomic_files import atomic_write_text DEFAULT_DIM = 256 DEFAULT_MODEL = "v8std-hash-embeddings-v1" -MAX_CHUNK_CHARS = 2200 def signed_hash(value: str) -> tuple[int, float]: @@ -56,39 +54,6 @@ def encode_vector(vector: list[float]) -> str: return base64.b64encode(struct.pack(f"<{len(vector)}f", *vector)).decode("ascii") -def page_chunks(page: dict[str, Any]) -> list[tuple[str, int, str]]: - metadata = " ".join( - [ - page.get("id", ""), - page.get("title", ""), - page.get("description", ""), - " ".join(page.get("aliases", [])), - ] - ).strip() - chunks: list[tuple[str, int, str]] = [] - if metadata: - chunks.append(("metadata", 0, metadata)) - - body = page.get("body_markdown") or "" - paragraphs = [item.strip() for item in re.split(r"\n{2,}", body) if item.strip()] - current: list[str] = [] - current_len = 0 - chunk_index = 0 - for paragraph in paragraphs: - next_len = current_len + len(paragraph) + 2 - if current and next_len > MAX_CHUNK_CHARS: - chunks.append(("body", chunk_index, "\n\n".join(current))) - chunk_index += 1 - current = [] - current_len = 0 - current.append(paragraph) - current_len += len(paragraph) + 2 - if current: - chunks.append(("body", chunk_index, "\n\n".join(current))) - - return chunks - - def generate_rows(pages_path: Path, *, dim: int = DEFAULT_DIM, model: str = DEFAULT_MODEL) -> list[str]: rows: list[str] = [] with pages_path.open(encoding="utf-8") as file: diff --git a/scripts/v8std_mcp_chunks.py b/scripts/v8std_mcp_chunks.py new file mode 100644 index 0000000..a6a01db --- /dev/null +++ b/scripts/v8std_mcp_chunks.py @@ -0,0 +1,42 @@ +"""Shared canonical chunk rules for vector generation and snapshot verification.""" + +from __future__ import annotations + +import re +from typing import Any + + +MAX_CHUNK_CHARS = 2200 + + +def page_chunks(page: dict[str, Any]) -> list[tuple[str, int, str]]: + metadata = " ".join( + [ + page.get("id", ""), + page.get("title", ""), + page.get("description", ""), + " ".join(page.get("aliases", [])), + ] + ).strip() + chunks: list[tuple[str, int, str]] = [] + if metadata: + chunks.append(("metadata", 0, metadata)) + + body = page.get("body_markdown") or "" + paragraphs = [item.strip() for item in re.split(r"\n{2,}", body) if item.strip()] + current: list[str] = [] + current_len = 0 + chunk_index = 0 + for paragraph in paragraphs: + next_len = current_len + len(paragraph) + 2 + if current and next_len > MAX_CHUNK_CHARS: + chunks.append(("body", chunk_index, "\n\n".join(current))) + chunk_index += 1 + current = [] + current_len = 0 + current.append(paragraph) + current_len += len(paragraph) + 2 + if current: + chunks.append(("body", chunk_index, "\n\n".join(current))) + + return chunks diff --git a/scripts/v8std_mcp_snapshot_format.py b/scripts/v8std_mcp_snapshot_format.py index 90365d0..bc2806a 100644 --- a/scripts/v8std_mcp_snapshot_format.py +++ b/scripts/v8std_mcp_snapshot_format.py @@ -20,6 +20,8 @@ from urllib.parse import quote, unquote, urlsplit import zlib +from v8std_mcp_chunks import page_chunks + DEFAULT_SITE_URL = "https://v8std.ru/" PUBLIC_DELIVERY_URL = "https://ai.v8std.ru/indexes/v1/" @@ -372,32 +374,6 @@ def portable_page(page: dict, canonical_site_url: str) -> dict: return {**page, **paths} -def _page_chunks(page: dict): - # Kept byte-for-byte equivalent in behavior to generate_search_vectors.page_chunks - # (MAX_CHUNK_CHARS=2200). Importing that module also imports PyYAML. This tiny - # pure rule stays here until it can be extracted within shared-file ownership. - # Tests verify both boundary cases and all current generator chunk hashes. - metadata = " ".join([ - page.get("id", ""), page.get("title", ""), page.get("description", ""), - " ".join(page.get("aliases", [])), - ]).strip() - if metadata: - yield "metadata", 0, metadata - body = page.get("body_markdown") or "" - paragraphs = [item.strip() for item in re.split(r"\n{2,}", body) if item.strip()] - current, current_len, chunk_index = [], 0, 0 - for paragraph in paragraphs: - next_len = current_len + len(paragraph) + 2 - if current and next_len > 2200: - yield "body", chunk_index, "\n\n".join(current) - chunk_index += 1 - current, current_len = [], 0 - current.append(paragraph) - current_len += len(paragraph) + 2 - if current: - yield "body", chunk_index, "\n\n".join(current) - - def _semantics(files: dict[str, bytes], site_url: str) -> dict[str, int]: ids = set() expected = {} @@ -407,7 +383,7 @@ def _semantics(files: dict[str, bytes], site_url: str) -> dict[str, int]: for key in ("site_path", "markdown_path")), "page_path") _require(page["id"] not in ids, "page_id") ids.add(page["id"]) - for field, index, text in _page_chunks(page): + for field, index, text in page_chunks(page): expected[page["id"], field, index] = sha256(text.encode("utf-8")) _require(len(expected) <= MAX_JSONL_ROWS, "jsonl_rows") _require(bool(ids), "corpus_empty") diff --git a/tests/test_v8std_mcp_snapshot_format.py b/tests/test_v8std_mcp_snapshot_format.py index 62a3826..87b2baa 100644 --- a/tests/test_v8std_mcp_snapshot_format.py +++ b/tests/test_v8std_mcp_snapshot_format.py @@ -8,6 +8,7 @@ import json import os from pathlib import Path +import stat import struct import subprocess import sys @@ -339,6 +340,53 @@ def test_chunk_boundaries_match_existing_generator_without_reembedding(self): fixture.corpus_files(pages=[page], vectors=rows))) fmt.verify_archive(archive, manifest) + def test_chunk_rule_is_shared_and_generator_exports_remain_available(self): + self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_chunks")) + chunks = importlib.import_module("v8std_mcp_chunks") + generator = importlib.import_module("generate_search_vectors") + fmt, _ = self.modules() + self.assertIs(generator.page_chunks, chunks.page_chunks) + self.assertIs(fmt.page_chunks, chunks.page_chunks) + self.assertEqual(generator.MAX_CHUNK_CHARS, chunks.MAX_CHUNK_CHARS) + self.assertEqual((generator.DEFAULT_DIM, generator.DEFAULT_MODEL, generator.MAX_CHUNK_CHARS), + (256, "v8std-hash-embeddings-v1", 2200)) + self.assertIsInstance(generator.page_chunks(fixture.page_fixture()), list) + + def test_chunk_rule_preserves_independent_boundary_examples(self): + generator = importlib.import_module("generate_search_vectors") + self.assertEqual(generator.page_chunks({}), []) + cases = [ + (" \n\n \n", []), + (" a \n\n b ", [("body", 0, "a\n\nb")]), + (" a\n \nb ", [("body", 0, "a\n \nb")]), + ("x" * 2195 + "\n\nя", [("body", 0, "x" * 2195 + "\n\nя")]), + ("x" * 2196 + "\n\nя", [("body", 0, "x" * 2196), ("body", 1, "я")]), + ("x" * 4400, [("body", 0, "x" * 4400)]), + ] + for body, expected in cases: + with self.subTest(length=len(body)): + page = {**fixture.page_fixture(), "body_markdown": body} + self.assertEqual(generator.page_chunks(page), + [("metadata", 0, "std437 Запросы Параметры #std437"), *expected]) + + def test_vector_regeneration_preserves_exact_current_corpus_bytes(self): + generator = importlib.import_module("generate_search_vectors") + rows = generator.generate_rows(ROOT / "docs/ai/pages.jsonl") + regenerated = ("\n".join(rows) + "\n").encode("utf-8") + self.assertEqual(regenerated, (ROOT / "docs/ai/search-vectors.jsonl").read_bytes()) + + def test_shared_chunks_and_reader_load_without_generator_or_external_packages(self): + result = subprocess.run( + [sys.executable, "-S", "-c", + "import sys; import v8std_mcp_chunks as chunks; " + "import v8std_mcp_snapshot_format as fmt; " + "assert fmt.page_chunks is chunks.page_chunks; " + "assert chunks.page_chunks({}) == []; " + "assert not {'yaml', 'PIL', 'generate_search_vectors', 'generate_ai_artifacts'} & sys.modules.keys()"], + env={**os.environ, "PYTHONPATH": str(ROOT / "scripts")}, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + def test_contract_budget_constants_and_boundary_enforcement(self): fmt, _ = self.modules() self.assertEqual(fmt.MAX_MANIFEST_BYTES, 64 * 1024) @@ -425,7 +473,7 @@ def inspect_replace(source, target): producer.publish_snapshot(root / "docs", root / "output", fixture.SOURCE_SHA, fixture.SITE_URL) self.assertEqual(observed, [True]) - def test_publish_failure_retains_previous_manifest_and_removes_temporary_files(self): + def test_link_or_rename_failure_retains_previous_manifest_and_removes_temporary_files(self): fmt, producer = self.modules() with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -433,7 +481,7 @@ def test_publish_failure_retains_previous_manifest_and_removes_temporary_files(s manifest_path = producer.publish_snapshot(root / "docs", root / "output", fixture.SOURCE_SHA, fixture.SITE_URL) before = manifest_path.read_bytes() - for target in ("link", "replace", "fsync"): + for target in ("link", "replace"): with self.subTest(target=target), patch.object(producer.os, target, side_effect=OSError("secret")): with self.assertRaisesRegex(fmt.SnapshotError, "publish_io") as caught: producer.publish_snapshot(root / "docs", root / "output", "2" * 40, fixture.SITE_URL) @@ -443,6 +491,89 @@ def test_publish_failure_retains_previous_manifest_and_removes_temporary_files(s old = fmt.validate_manifest(before) fmt.verify_archive((root / "output" / old["archive"]["path"]).read_bytes(), old) + def test_fsync_failures_at_each_publication_stage(self): + fmt, producer = self.modules() + stages = ["output_parent", "archive_file", "archive_install_directory", + "archive_before_manifest", "output_before_manifest", "manifest_file", + "output_after_manifest_rename"] + original_fsync = os.fsync + original_directory = producer._fsync_directory + original_replace = os.replace + for failed_stage in stages: + with self.subTest(stage=failed_stage), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + docs, output = root / "docs", root / "output" + fixture.write_docs(docs) + manifest_path = producer.publish_snapshot(docs, output, fixture.SOURCE_SHA, fixture.SITE_URL) + before = manifest_path.read_bytes() + new_archive, new_manifest = producer.build_snapshot(docs, "2" * 40, fixture.SITE_URL) + archive_dir = output / new_manifest["archive"]["sha256"] + archive_path = archive_dir / "snapshot.tar.gz" + observed = [] + directory_stage = None + archive_directory_syncs = 0 + renamed = False + + def sync_directory(path): + nonlocal directory_stage, archive_directory_syncs + if path == output.parent: + directory_stage = "output_parent" + elif path == archive_dir: + archive_directory_syncs += 1 + directory_stage = ("archive_install_directory" if archive_directory_syncs == 1 + else "archive_before_manifest") + else: + self.assertEqual(path, output) + directory_stage = ("output_after_manifest_rename" if renamed + else "output_before_manifest") + try: + original_directory(path) + finally: + directory_stage = None + + def sync_file_descriptor(fd): + if directory_stage is not None: + self.assertTrue(stat.S_ISDIR(os.fstat(fd).st_mode)) + stage = directory_stage + else: + self.assertTrue(stat.S_ISREG(os.fstat(fd).st_mode)) + stage = "manifest_file" if archive_path.exists() else "archive_file" + observed.append(stage) + if stage == failed_stage: + raise OSError("secret") + return original_fsync(fd) + + def replace_manifest(source, target): + nonlocal renamed + self.assertEqual(Path(target), manifest_path) + result = original_replace(source, target) + renamed = True + return result + + with (patch.object(producer, "_fsync_directory", side_effect=sync_directory), + patch.object(producer.os, "fsync", side_effect=sync_file_descriptor), + patch.object(producer.os, "replace", side_effect=replace_manifest) as replace_mock): + with self.assertRaisesRegex(fmt.SnapshotError, "^publish_io$"): + producer.publish_snapshot(docs, output, "2" * 40, fixture.SITE_URL) + self.assertEqual(observed, stages[:stages.index(failed_stage) + 1]) + after_rename = failed_stage == "output_after_manifest_rename" + self.assertEqual(replace_mock.call_count, int(after_rename)) + self.assertEqual(renamed, after_rename) + visible = manifest_path.read_bytes() + active = fmt.validate_manifest(visible) + verified = fmt.verify_archive((output / active["archive"]["path"]).read_bytes(), active) + if after_rename: + self.assertNotEqual(visible, before) + self.assertEqual(active, new_manifest) + self.assertEqual(archive_path.read_bytes(), new_archive) + self.assertEqual(verified.metadata["source_sha"], "2" * 40) + else: + self.assertEqual(visible, before) + self.assertEqual(verified.metadata["source_sha"], fixture.SOURCE_SHA) + old = fmt.validate_manifest(before) + fmt.verify_archive((output / old["archive"]["path"]).read_bytes(), old) + self.assertEqual(list(output.rglob(".snapshot-*")), []) + def test_existing_immutable_symlinks_are_rejected(self): fmt, producer = self.modules() with tempfile.TemporaryDirectory() as directory: From 6a25c9c6a54202bb07e39660df3c423d0ab29b31 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:53:39 +0300 Subject: [PATCH 06/88] docs: specify bounded snapshot worker lifecycle --- ...26-09-10-mcp-container-distribution-plan.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 256eb19..dd34240 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -69,7 +69,7 @@ corpus/runtime (1–3), container distribution (4), delivery (5–6). Их inter | Task | Write scope | Responsibility | |---|---|---| -| 1 | `scripts/v8std_mcp_snapshot_format.py`, `scripts/generate_mcp_snapshot.py`, `tests/test_v8std_mcp_snapshot_format.py`, `tests/mcp_snapshot_fixtures.py` | Pure format validation, deterministic producer; no network/index refresh. | +| 1 | `scripts/v8std_mcp_snapshot_format.py`, `scripts/generate_mcp_snapshot.py`, `scripts/v8std_mcp_chunks.py`, `scripts/generate_search_vectors.py`, `tests/test_v8std_mcp_snapshot_format.py`, `tests/mcp_snapshot_fixtures.py` | Pure format validation, deterministic producer and shared unchanged chunk rules; no network/index refresh. | | 2 | `scripts/v8std_mcp_snapshots.py`, `tests/test_v8std_mcp_snapshots.py` | URL trust boundary, HTTP/cache transaction, background coordinator. | | 3 | `scripts/v8std_mcp_runtime.py`, `scripts/v8std_mcp_presentation.py`, `scripts/v8std_mcp_index.py`, `scripts/v8std_mcp_server.py`, runtime tests | Frozen generation construction, request facade, stdio/HTTP lifecycle. | | 4 | Dockerfiles/Compose/lock, local-profile script, tests, docs | Build and exercise the two images and local site. | @@ -168,8 +168,16 @@ class SnapshotCoordinator: commit and returns its result; without prepare it returns VerifiedSnapshot. Preparation failure preserves the former disk pointer as well as process state. The coordinator owns the active reference and stores network/parse work outside -the request path. Prepare in a bounded worker with a lifecycle that is stopped -on close; no network/CPU build on the ASGI event loop. Test/store internals may +the request path. A supervised `multiprocessing` worker using the `spawn` start +method owns blocking source I/O, validation, generation preparation and cache +commit. The parent enforces the whole-attempt deadline and terminates/reaps the +worker on timeout or close; a daemon thread alone is not cancellation. Internal +builder callables and their results must support this trusted process boundary. +Only IPC from the application's own worker may carry serialized Python objects; +never load pickle from downloaded data or a persistent/shared cache. Task 3 adds +the minimal frozen-index serialization hook to reconstruct its process-local +lock. Measure transfer/startup/staging RSS and query latency during integration. +No network/CPU build runs on the ASGI event loop. Test/store internals may inject monotonic clock/transport at their actual dependency boundary, never test-only methods on production classes. @@ -232,7 +240,9 @@ def present_result(value, *, canonical_site_url: str, site_url: str, ``` Construct one V8StdIndex from already validated bytes without network. Add a -focused factory for this to existing index; retain legacy direct file entrypoints +focused factory and a trusted-IPC serialization hook (exclude/recreate the +process-local lock, never deserialize a persistent pickle cache) to existing +index; retain legacy direct file entrypoints for current tests/developer use. Never mutate this index after construction. Facade captures coordinator.current() once per top-level call, invokes that generation including nested snippet/search/related operations, then transforms From ecb28b09905b4b5280564406fca59b2e9fde19c4 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:54:33 +0300 Subject: [PATCH 07/88] docs: record reviewed snapshot format completion --- spec/plans/2026-09-10-mcp-container-distribution-plan.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index dd34240..24f643f 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -109,7 +109,7 @@ Producer converts canonical page URLs into additional `site_path`/`markdown_path without rewriting canonical body text; generated vectors and text hashes remain valid. Site-level generator outputs currently consumed elsewhere stay intact. -- [ ] **RED:** Add behavior tests with hand-authored one-page docs and independently +- [x] **RED:** Add behavior tests with hand-authored one-page docs and independently constructed hostile tar members. The first test asserts the producer module exists using `importlib.util.find_spec`, then execute the producer twice and require exact archive byte equality. Example expected path is literal: @@ -124,7 +124,7 @@ with self.assertRaisesRegex(SnapshotError, "archive_member"): Run `.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format -v`; missing implementation must fail before production code is written. -- [ ] **GREEN format:** Implement strict JSON with duplicate-key/depth/type +- [x] **GREEN format:** Implement strict JSON with duplicate-key/depth/type guards; exact five regular members; deterministic gzip/tar; independent descriptor/archive hashes; compressed/decompressed/member/row/line bounds; no symlink, trailing gzip data, PAX, duplicate or traversal member. Count all @@ -132,12 +132,12 @@ with self.assertRaisesRegex(SnapshotError, "archive_member"): Validate page/vector IDs, finite components, model/dim and chunk text hashes against the existing chunk rules. Reuse those pure rules without importing docs/Pillow. Normalize site URL and preserve base prefix. -- [ ] **GREEN producer:** CLI accepts `--docs`, `--output`, `--source-sha`, +- [x] **GREEN producer:** CLI accepts `--docs`, `--output`, `--source-sha`, `--site-url`, `--public-delivery`; publishes immutable hash directory first, manifest last with atomic write. In local mode archive path is relative; public mode uses the fixed ai delivery origin. Existing same-hash bytes are verified, never overwritten with different contents. -- [ ] **Verify:** Real current docs corpus round-trip, deterministic rebuild, +- [x] **Verify:** Real current docs corpus round-trip, deterministic rebuild, bad vectors/schema/count/hash and all archive budget cases pass. Record exact focused command and results, run existing vector/index tests, self-review, then commit only this task's files and submit for spec+quality review. From a0f2128c845b7b657e81a2a0c4e47950eb50d3b9 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 14:58:10 +0300 Subject: [PATCH 08/88] docs: connect publisher link validation and local license delivery --- .../2026-09-10-mcp-container-distribution-plan.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 24f643f..873474a 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -218,6 +218,10 @@ self.assertEqual(coordinator.current().corpus_id, "fixture-generation-a") **Files:** create `scripts/v8std_mcp_runtime.py`, `scripts/v8std_mcp_presentation.py`, `tests/test_v8std_mcp_runtime.py`, `tests/test_v8std_mcp_presentation.py`; modify index/server and focused tests. +Integrate publisher-side link validation in `generate_mcp_snapshot.py` and its +focused tests, reusing the presentation parser/catalog rather than a second +link grammar. Treat the three Resources and published license paths as explicit +auxiliary catalog entries, not corpus pages or arbitrary allowed paths. **Interfaces produced:** @@ -265,6 +269,9 @@ self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/con external provenance/query/fragment. Local URLs accepted as page lookup inputs resolve to canonical keys without affecting ranking. All nested result URLs and three legacy Resource bodies use the same policy; no global string replace. + Validate publisher links against the same catalog. Preserve generated bare + internal `URL:`/`HTML:` fields and resolve relative links in their page context; + ordinary code literals and external source records are not link nodes. - [ ] **GREEN runtime:** Startup chooses snapshot mode from SITE_URL/default, supports stdio and HTTP from same build_server. Legacy explicit files remain usable; ambiguous legacy URLs plus site setting fail before network. Default @@ -283,6 +290,10 @@ self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/con `scripts/build_local_site.py`, `tests/test_v8std_mcp_distribution.py`, `scripts/check_mcp_container.py`, `deploy/docker-catalog/server.yaml`; modify `overrides/main.html`, docs and build wrapper only where profile needs it. +Add an explicit index for the three already published license text files in +`scripts/publish_license_texts.py` and cover it in its existing tests: the current +attribution page links `/LICENSES/`, which otherwise has no page when directory +listing is disabled. Both public and local builds must resolve this same path. **Consumes:** Task 1 producer CLI and Task 3 runtime CLI. Runtime container default CMD selects stdio; HTTP command selects host 0.0.0.0/port 8000. From 315a119aa98d2147caab7bcba3ae94b67c3fe018 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:00:01 +0300 Subject: [PATCH 09/88] docs: record snapshot proof and container preflight evidence --- spec/operations/mcp-container-verification.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 spec/operations/mcp-container-verification.md diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md new file mode 100644 index 0000000..f008853 --- /dev/null +++ b/spec/operations/mcp-container-verification.md @@ -0,0 +1,90 @@ +# MCP container delivery — verification record + +This is implementation evidence, not authorization to publish or activate a host. +The accepted design and contracts remain authoritative. Unfinished or external +checks below must not be reported as passed. + +## Scope and baseline + +- Main baseline: `b7bef11e145a188b30e7a7b17df2be4cb1acbd0c`. +- Approved design package: `537fe79`; implementation plan: `a3b2474`, with later + recorded interface refinements on the same feature branch. +- Working branch: `codex/mcp-container-distribution-design`, primary checkout. +- No live host changes, image publication, Catalog submission, PR closure or + production activation are established by this record. +- Capacity of 100,000 coding agents is a target, **not a measured result**. + +## Completed local evidence + +### Deterministic corpus format and producer + +Implementation: `7255a93b818005a5290eb941ac74c5cccb579afa`. +Reviewed correction: `821d1700786e36d2f719b67a9d1ac323fa1266f0`. +Independent task review and scoped re-review completed without open findings. + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format tests.test_v8std_mcp_index tests.test_v8std_search_features -v +``` + +Result after correction: **65 tests passed** (35 snapshot tests and 30 existing +index/search tests). Initial implementation additionally passed the then-current +399-test full suite; this is not a substitute for the final suite after all +runtime, container and CI changes. + +The real corpus contained 1,423 pages and 3,281 vectors. Repeated snapshot builds +produced identical compressed bytes, original page fields were preserved, and +full vector regeneration matched the existing vector file byte for byte. +The generator and reader now share the unchanged dependency-free chunk helper. +The standalone producer also ran with Python `-S`. + +Hostile archive cases include invalid members, framing, paths, gzip payload, +JSON structure, vector identity/hash/data, size limits and missing corpus rows. +Publication fault injection covers seven distinct fsync stages. Failure after +manifest rename leaves the **new valid manifest visible**, while its durability +is unconfirmed; it does not prove the previous manifest remains selected. +The old immutable archive remains intact. These tests are fault injection, +not a real machine power-loss experiment. + +### Local build environment + +Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; +buildx v0.36.1. The installed application SDK is `mcp==1.27.0`. + +Pinned multi-platform base indices: + +- `python:3.12-slim@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea` +- `nginx:stable-alpine@sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c` + +Both amd64 bases executed locally under emulation with read-only root, network +disabled, all capabilities dropped and no-new-privileges. Python reported +`x86_64`, `3.12.14`, zlib `1.3.1`; nginx reported `1.30.4`. This establishes +base-image emulation availability, not runtime-image or native amd64 acceptance. + +### Search baseline + +The pre-runtime-change desktop benchmark reported MRR 0.994 and p95 46.1 ms +on the existing benchmark corpus. Timing is environment-dependent and not a +service capacity measurement. The final comparison must use the same corpus +and query set on the implemented snapshot runtime. + +## Remaining implementation and integration gates + +| Area | Current evidence | +| --- | --- | +| Bounded refresh/cache, crashes, shared volume, offline recovery | In progress; not yet accepted. | +| Frozen generations, URL presentation, stdio/HTTP lifecycle | Pending. | +| Runtime/static-site images, local request graph, Gateway sessions | Pending. | +| Restricted host controller, rollback and independent index delivery | Pending. | +| Fail-closed publication, process v2 synchronization | Pending. | +| Final semantic impact, merge-ready, fitness, strict build and full suite | Pending after all changes; strict build precedes the suite. | +| Final image smoke, refresh RSS/CPU, disposable mixed load | Pending; cannot establish production 100k capacity. | + +## External acceptance boundary + +Registry release digests, provenance verification against those releases, +Docker Catalog acceptance, GitHub protection/environment/secrets, target-host +prerequisites and initial controlled activation require separate evidence. +The existing Python deployment remains the initial rollback path until that +activation is explicitly performed and verified. An automatically enabled +release path must not be inferred from a locally passing test or a written +workflow. From 3d235c4aa2afaee8492e90631029bc4f5b57f458 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:06:43 +0300 Subject: [PATCH 10/88] docs: prepare controlled container activation gates --- spec/operations/mcp-container-activation.md | 89 +++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 spec/operations/mcp-container-activation.md diff --git a/spec/operations/mcp-container-activation.md b/spec/operations/mcp-container-activation.md new file mode 100644 index 0000000..e0b815c --- /dev/null +++ b/spec/operations/mcp-container-activation.md @@ -0,0 +1,89 @@ +# Controlled initial activation of container delivery + +**Status:** prerequisite checklist; host-controller commands and rehearsal +evidence are completed by the release implementation task. This document does +not authorize a live operation, enable CI, or establish current host capacity. + +## Authority and stop conditions + +Initial activation requires an explicit operational request for the exact +verified main SHA and published image digest. Until then, leave production, +secrets, branch rules, environment settings and unrelated hosted services alone. +The ordinary automatic path starts only after initial rollback has been proven. + +Stop before any switch if the backup cannot be restored, the source manifest +is not reachable, the image/provenance/configuration is unverified, capacity is +insufficient, the predecessor is missing, or the restricted controller cannot +reconcile a crash. A healthy systemd wrapper is not application readiness. + +## Host inventory and backup gate + +- Refresh the actual target host identity, OS, CPU/RAM/swap, available disk, + file descriptors, network budget, running services, containers and listeners. + Prior incident measurements are historical, not current capacity evidence. +- Identify the precise nginx virtual hosts, includes, default server, certificate + paths and renewal hooks. Ensure the ai.v8std.ru TLS endpoint and renewal no + longer depend on an old site's configuration before any cleanup. +- Preserve SSH access, monitoring, fail2ban and certificate renewal. Removing an + old website is not permission to remove unrelated operating services. +- Back up the existing Python runtime, environment/dependencies, unit/drop-ins, + nginx configuration, working corpus/cache and certificate configuration to + protected off-host storage. Keep secrets out of repository and public logs. +- Restore into a disposable environment and exercise the old endpoint. Record + exact cleanup targets and recovery instructions; obtain the operational + approval before removing or disabling them. Prefer recoverable moves. + +## CI and restricted-host gate + +- Protect main and require the actual validation checks used by this workflow. + Audit bypass permissions; a branch name alone is not an authorization check. +- Restrict the production environment to verified main releases. PRs, forks, + tags, untrusted inputs and stale runs must not obtain host credentials. +- Establish a restricted release identity and a separately constrained static + artifact publisher. Neither credential grants arbitrary shell commands, + arbitrary paths/environment variables, root login or Docker group access. +- Install and verify the trusted host controller, its fixed configuration, + attestation verifier, bounded job supervision and durable journal/recovery. + Ordinary release envelopes cannot replace their own trust policy. +- Keep automatic runtime switching disabled until the initial published-image + cutover and predecessor recovery are demonstrated. Test the kill switch: it + blocks new releases without interrupting an in-flight rollback. + +## Artifact and bootstrap ordering gate + +Use the same published runtime digest later supplied to local users and the +Catalog. Do not rebuild a special production image on the server. + +1. Verify the main source SHA, published multi-platform index digest, + host-platform child membership, publisher identity and configuration digest. +2. Install the independent nginx index store and publish the verified immutable + archive under its hash before publishing a manifest that references it. +3. Publish the site's manifest and verify its actual public source URL and + archive bytes. A successful Pages job alone is insufficient to prove freshness. +4. Verify the public-default thin image against that manifest before stable + promotion or the first container switch. Before this bootstrap, candidate + builds are not a usable public-default release. +5. Exercise the candidate on its private loopback port, with exact runtime SHA + and corpus ID, real MCP calls and static downloads while the runtime stops. +6. Perform the initial controlled cutover while retaining the tested Python + predecessor. Do not manufacture a Docker predecessor or claim rollback from + an empty release history. Establish the first verified container predecessor + before enabling ordinary automated transactions. + +## Capacity and acceptance gate + +Measure old runtime + candidate + snapshot preparation together, including +Docker/OS overhead and static index traffic. Verify memory, disk staging and +pins, descriptors, CPU and bandwidth before switching. Raise capacity or change +the accepted rollout design if the measured host cannot accommodate overlap; +do not silently kill the predecessor to make room. + +Exercise initialized idle agents, normal POST tool calls, reconnects, shared NAT, +snapshot refresh and concurrent archive downloads. Record the actual mix, +duration, error rate and latency. Neither worker_connections nor idle TCP count +proves support for 100,000 coding agents. + +Record the initial release journal, exact SHA/digests/corpus/configuration, +public MCP/TLS and static delivery results, failure/rollback rehearsal and +monitoring checks in the verification record. External Catalog acceptance and +closure of the alternative PR remain separate delivery outcomes. From 3b56ab799b59f4d81e81a72b925d9e38d0c39367 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:18:48 +0300 Subject: [PATCH 11/88] feat: add bounded MCP snapshot store and coordinator --- scripts/v8std_mcp_snapshots.py | 679 ++++++++++++++++++++++++ tests/test_v8std_mcp_snapshots.py | 839 ++++++++++++++++++++++++++++++ 2 files changed, 1518 insertions(+) create mode 100644 scripts/v8std_mcp_snapshots.py create mode 100644 tests/test_v8std_mcp_snapshots.py diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py new file mode 100644 index 0000000..c3ab093 --- /dev/null +++ b/scripts/v8std_mcp_snapshots.py @@ -0,0 +1,679 @@ +"""Bounded snapshot loading and immutable-generation coordination. + +Only the application's own spawned worker uses pickle, over a private socketpair. +Disk and HTTP inputs are always bytes/strict JSON verified by the format module. +Builders and their results must be spawn-serializable (including bounded local +result reconstruction). Network, verification, build and commit run in the child; +the supervisor can terminate and reap it even inside DNS or a blocking builder. + +Cache layout: /v1-/{state.json,rollback.json, +pins.json,generations//...}. Host-owned pins.json is an atomic +JSON object {"archives": [, ...]}. Invalid pins fail GC closed. +Python references returned by current() retain old in-memory generations for +in-flight requests; their lifetime is independent of disk-generation retention. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import fcntl +import http.client +import multiprocessing +import os +from pathlib import Path +import pickle +import random +import re +import select +import shutil +import socket +import ssl +import stat +import struct +import tempfile +import threading +import time +from urllib.parse import urlsplit + +from v8std_mcp_snapshot_format import ( + DEFAULT_SITE_URL, PUBLIC_DELIVERY_URL, MAX_ARCHIVE_BYTES, MAX_MANIFEST_BYTES, + MEMBER_LIMITS, SnapshotError, VerifiedSnapshot, canonical_json, + canonical_page_path, normalize_site_url, sha256, strict_json, + validate_manifest, verify_archive, +) + +ATTEMPT_SECONDS = 60 +READ_SECONDS = 20 +CACHE_BYTES = 256 * 1024 * 1024 +_CHUNK = 64 * 1024 +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") +_TEMP = re.compile(r"\.(?:stage|pointer)-[a-z0-9_]+\Z") +_CODES = frozenset({ + "loader_failed", "INDEX_NOT_READY", "url_policy", "redirect_limit", + "http_status", "http_headers", "http_encoding", "http_size", "network", + "deadline", "closed", "cache_io", "cache_budget", "lock_timeout", + "prepare_failed", "worker_failed", "configuration", +}) + + +class LoaderError(ValueError): + """Bounded loader diagnostics; never include URLs, input data or raw errors.""" + + def __init__(self, code: str): + self.code = code if code in _CODES else "loader_failed" + super().__init__(self.code) + + +def _remaining(deadline): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LoaderError("deadline") + return remaining + + +def _allowed_url(reference: str, current: str, boundary: str) -> str: + # Do not urljoin first: it removes dot segments before they can be rejected. + try: + if not isinstance(reference, str) or not reference or reference.startswith("//"): + raise LoaderError("url_policy") + if not urlsplit(reference).scheme: + if reference.startswith("/"): + parts = urlsplit(current) + reference = f"{parts.scheme}://{parts.netloc}" + reference + else: + reference = current.rsplit("/", 1)[0] + "/" + reference + return boundary + canonical_page_path(reference, boundary) + except (SnapshotError, ValueError): + raise LoaderError("url_policy") from None + + +def _archive_url(manifest, manifest_url, site_url): + path = manifest["archive"]["path"] + boundary = site_url + if site_url == DEFAULT_SITE_URL and path.startswith(PUBLIC_DELIVERY_URL): + boundary = PUBLIC_DELIVERY_URL + return _allowed_url(path, manifest_url, boundary), boundary + + +def _download(url, boundary, headers, limit, deadline, read_seconds, *, archive=False): + """One bounded HTTP exchange, validating each redirect before connecting.""" + for redirects in range(4): + url = _allowed_url(url, url, boundary) + parts = urlsplit(url) + timeout = min(read_seconds, _remaining(deadline)) + if parts.scheme == "https": + connection = http.client.HTTPSConnection( + parts.hostname, parts.port, timeout=timeout, context=ssl.create_default_context()) + else: + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=timeout) + try: + # Explicit connect retains the socket even when getresponse detaches + # a Connection: close response. DNS is bounded by the parent process. + connection.connect() + stream_socket = connection.sock + stream_socket.settimeout(min(read_seconds, _remaining(deadline))) + connection.request("GET", parts.path, headers={ + "Accept-Encoding": "identity", "User-Agent": "v8std-snapshot/1", **headers}) + stream_socket.settimeout(min(read_seconds, _remaining(deadline))) + response = connection.getresponse() + with response: + if response.status in {301, 302, 303, 307, 308}: + if redirects == 3: + raise LoaderError("redirect_limit") + locations = response.headers.get_all("Location", []) + if len(locations) != 1: + raise LoaderError("http_headers") + url = _allowed_url(locations[0], url, boundary) + continue + if response.status == 304: + return 304, b"", {}, url + if response.status != 200: + raise LoaderError("http_status") + encodings = response.headers.get_all("Content-Encoding", []) + if encodings and encodings != ["identity"]: + raise LoaderError("http_encoding") + if archive and response.headers.get_content_type() != "application/gzip": + raise LoaderError("http_headers") + lengths = response.headers.get_all("Content-Length", []) + length = None + if lengths: + if len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,10}", lengths[0]): + raise LoaderError("http_headers") + length = int(lengths[0]) + if length > limit: + raise LoaderError("http_size") + transfers = response.headers.get_all("Transfer-Encoding", []) + if transfers and (transfers != ["chunked"] or lengths): + raise LoaderError("http_headers") + payload = bytearray() + while not response.isclosed(): + stream_socket.settimeout(min(read_seconds, _remaining(deadline))) + chunk = response.read1(min(_CHUNK, limit - len(payload) + 1)) + if not chunk: + break + payload.extend(chunk) + if len(payload) > limit: + raise LoaderError("http_size") + if length is not None and length != len(payload): + raise LoaderError("http_size") + _remaining(deadline) + validators = {} + for name in ("ETag", "Last-Modified"): + values = response.headers.get_all(name, []) + if len(values) == 1 and _safe_header(values[0]): + validators[name] = values[0] + return 200, bytes(payload), validators, url + except (TimeoutError, OSError, http.client.HTTPException): + _remaining(deadline) + raise LoaderError("network") from None + finally: + connection.close() + raise LoaderError("redirect_limit") + + +def _safe_header(value): + return (isinstance(value, str) and 0 < len(value) <= 1024 + and all(32 <= ord(char) < 127 for char in value)) + + +def _directory(path, *, create=False): + if create and not path.exists(): + _directory(path.parent, create=True) + path.mkdir(mode=0o700, exist_ok=True) + # Persist each newly created directory entry, including intermediate + # parents. An existing mounted cache needs no writes to the container root. + _fsync_directory(path.parent) + if not stat.S_ISDIR(path.lstat().st_mode): + raise LoaderError("cache_io") + + +def _read_file(path, limit): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > limit: + raise LoaderError("cache_io") + payload = stream.read(limit + 1) + if len(payload) > limit: + raise LoaderError("cache_io") + return payload + + +def _fsync_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +@contextmanager +def _file_lock(path, deadline): + fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + waited = False + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise LoaderError("cache_io") + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + waited = True + if deadline - time.monotonic() <= 0: + raise LoaderError("lock_timeout") + time.sleep(max(0, min(.025, deadline - time.monotonic()))) + yield waited + finally: + os.close(fd) # OS releases flock even on process termination/crash. + + +def _prepare(snapshot, prepare): + try: + result = snapshot if prepare is None else prepare(snapshot) + # Serialization is also preparation: an unpickleable lock must not + # advance the disk pointer. These bytes go ONLY to our private IPC socket. + return pickle.dumps(("ok", result, { + "corpus_id": snapshot.metadata["corpus_id"], + "archive_sha256": snapshot.archive_sha256, + }), protocol=pickle.HIGHEST_PROTOCOL) + except Exception: + raise LoaderError("prepare_failed") from None + + +class SnapshotStore: + def __init__(self, site_url: str, cache_dir: Path): + self.site_url = normalize_site_url(site_url) + self.cache_dir = Path(cache_dir).absolute() + self.namespace = self.cache_dir / ("v1-" + sha256(self.site_url.encode("utf-8"))) + self._attempt_seconds = ATTEMPT_SECONDS + self._read_seconds = READ_SECONDS + self._transport = _download + + def _states(self): + for name in ("state.json", "rollback.json"): + try: + state = strict_json(_read_file(self.namespace / name, MAX_MANIFEST_BYTES)) + if (state.get("schema_version") != 1 or state.get("site_url") != self.site_url + or not self._digest(state.get("active")) + or (state.get("previous") is not None + and not self._digest(state["previous"]))): + continue + yield state + except (OSError, SnapshotError, LoaderError): + continue + + @staticmethod + def _digest(value): + return isinstance(value, str) and bool(_DIGEST.fullmatch(value)) + + def _generation(self, digest, manifest=None): + directory = self.namespace / "generations" / digest + _directory(directory) + stored_manifest = validate_manifest(_read_file(directory / "manifest.json", MAX_MANIFEST_BYTES)) + if stored_manifest["archive"]["sha256"] != digest: + raise LoaderError("cache_io") + archive = _read_file(directory / "snapshot.tar.gz", MAX_ARCHIVE_BYTES) + snapshot = verify_archive(archive, stored_manifest) + if manifest is not None and manifest != stored_manifest: + snapshot = verify_archive(archive, manifest) + for name, payload in snapshot.files.items(): + if _read_file(directory / name, MEMBER_LIMITS[name]) != payload: + raise LoaderError("cache_io") + return snapshot, stored_manifest + + def _cached_entry(self): + try: + _directory(self.cache_dir) + _directory(self.namespace) + _directory(self.namespace / "generations") + seen = set() + for state in self._states(): + for digest in (state["active"], state.get("previous")): + if digest is None or digest in seen: + continue + seen.add(digest) + try: + snapshot, manifest = self._generation(digest) + validators = state.get("validators", {}) if digest == state["active"] else {} + if not isinstance(validators, dict): + validators = {} + recovered = {**state, "active": digest, "validators": { + k: v for k, v in validators.items() + if k in {"ETag", "Last-Modified"} and _safe_header(v)}} + return snapshot, manifest, recovered + except (OSError, SnapshotError, LoaderError): + continue + except (OSError, LoaderError): + pass + return None + + def cached(self) -> VerifiedSnapshot | None: + """Verify local bytes, without network; coordinator calls this in a child.""" + entry = self._cached_entry() + return entry[0] if entry else None + + def refresh(self, *, prepare=None): + """Supervise a complete attempt; return preparation only after commit.""" + return self._run("refresh", prepare)[0] + + def _usage(self): + total = 0 + # Every namespace, pin and staging file in the shared cache volume + # counts. Never follow a symlink into another tree. + for directory, dirs, files in os.walk(self.cache_dir, followlinks=False, + onerror=lambda error: _cache_walk_error()): + for name in [*dirs, *files]: + info = (Path(directory) / name).lstat() + if not stat.S_ISDIR(info.st_mode): + total += info.st_size + return total + + def _space(self, needed): + if self._usage() + needed > CACHE_BYTES: + raise LoaderError("cache_budget") + + def _write(self, path, payload, deadline): + _remaining(deadline) + self._space(len(payload)) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as stream: + for offset in range(0, len(payload), _CHUNK): + _remaining(deadline) + stream.write(payload[offset:offset + _CHUNK]) + stream.flush() + os.fsync(stream.fileno()) + + def _atomic_file(self, path, payload, deadline): + temporary = self.namespace / (".pointer-" + os.urandom(12).hex()) + try: + self._write(temporary, payload, deadline) + os.replace(temporary, path) + _fsync_directory(self.namespace) + finally: + temporary.unlink(missing_ok=True) + + def _commit_state(self, state, old, deadline): + old_bytes = canonical_json(old) if old else None + if old_bytes is not None: + self._atomic_file(self.namespace / "rollback.json", old_bytes, deadline) + pointer = self.namespace / "state.json" + temporary = self.namespace / (".pointer-" + os.urandom(12).hex()) + replaced = False + try: + self._write(temporary, canonical_json(state), deadline) + _remaining(deadline) + os.replace(temporary, pointer) + replaced = True + _fsync_directory(self.namespace) + except OSError: + if replaced: + if old_bytes is not None: + # rollback.json was fsynced before activation. Its atomic + # rename restores the old pointer even if fsync keeps failing. + os.replace(self.namespace / "rollback.json", pointer) + else: + pointer.unlink(missing_ok=True) + try: + _fsync_directory(self.namespace) + except OSError: + pass + raise + finally: + temporary.unlink(missing_ok=True) + + def _gc(self, retained=()): + states = list(self._states()) + protected = set(retained) + if states: + protected.update(d for d in (states[0]["active"], states[0].get("previous")) if d) + try: + pins = strict_json(_read_file(self.namespace / "pins.json", MAX_MANIFEST_BYTES)) + except FileNotFoundError: + pins = {"archives": []} + archives = pins.get("archives") + if not isinstance(archives, list) or not all(self._digest(d) for d in archives): + raise LoaderError("cache_io") + protected.update(archives) + for entry in self.namespace.iterdir(): + if _TEMP.fullmatch(entry.name): + self._remove_owned(entry) + generations = self.namespace / "generations" + for entry in generations.iterdir(): + if self._digest(entry.name) and entry.name not in protected: + self._remove_owned(entry) + + @staticmethod + def _remove_owned(path): + mode = path.lstat().st_mode + if stat.S_ISDIR(mode): + shutil.rmtree(path) # fd-based, symlink-attack-resistant on supported POSIX. + elif stat.S_ISREG(mode): + path.unlink() + # Unknown/symlink paths are not owned cleanup targets. + + def _refresh(self, prepare, deadline): + _directory(self.cache_dir, create=True) + _directory(self.namespace, create=True) + _directory(self.namespace / "generations", create=True) + with _file_lock(self.namespace / ".lock", deadline) as waited: + with _file_lock(self.cache_dir / ".volume.lock", deadline): + entry = self._cached_entry() + if waited and entry: + return _prepare(entry[0], prepare) + self._gc((entry[0].archive_sha256,) if entry else ()) + headers = {} + if entry: + for field, header in (("ETag", "If-None-Match"), ("Last-Modified", "If-Modified-Since")): + if value := entry[2]["validators"].get(field): + headers[header] = value + bootstrap = self.site_url + "ai/mcp/v1/manifest.json" + status, raw, validators, final_url = self._transport( + bootstrap, self.site_url, headers, MAX_MANIFEST_BYTES, + deadline, self._read_seconds) + if status == 304 and entry is None: + status, raw, validators, final_url = self._transport( + bootstrap, self.site_url, {}, MAX_MANIFEST_BYTES, deadline, self._read_seconds) + if status == 304: + if entry is None: + raise LoaderError("http_status") + return _prepare(entry[0], prepare) + manifest = validate_manifest(raw) + archive_url, boundary = _archive_url(manifest, final_url, self.site_url) + digest = manifest["archive"]["sha256"] + old = entry[2] if entry else None + state = { + "schema_version": 1, "site_url": self.site_url, "active": digest, + "previous": (entry[2].get("previous") if digest == entry[0].archive_sha256 + else entry[0].archive_sha256) if entry else None, + "validators": validators, + } + try: + reusable, _ = self._generation(digest, manifest) + except (OSError, LoaderError): + reusable = None + except SnapshotError: + # A new manifest disagreeing with a verified current archive + # is invalid, rather than a reason to fetch that archive again. + if entry and entry[0].archive_sha256 == digest: + raise + reusable = None + if reusable is not None: + result = _prepare(reusable, prepare) + self._commit_state(state, old, deadline) + return result + self._space(manifest["archive"]["bytes"] + manifest["archive"]["unpacked_bytes"] + + len(raw) + 2 * MAX_MANIFEST_BYTES) + stage = Path(tempfile.mkdtemp(prefix=".stage-", dir=self.namespace)) + try: + self._write(stage / "manifest.json", raw, deadline) + status, archive, _, _ = self._transport( + archive_url, boundary, {}, manifest["archive"]["bytes"], + deadline, self._read_seconds, archive=True) + if status != 200: + raise LoaderError("http_status") + self._write(stage / "snapshot.tar.gz", archive, deadline) + snapshot = verify_archive(archive, manifest) + for name, payload in snapshot.files.items(): + self._write(stage / name, payload, deadline) + result = _prepare(snapshot, prepare) + _remaining(deadline) + _fsync_directory(stage) + target = self.namespace / "generations" / snapshot.archive_sha256 + if target.exists() or target.is_symlink(): + _directory(target) # Never replace a symlink/foreign path. + # A verified download repairs corrupt bytes under the same + # immutable identity. Keep the old directory until commit; + # a crash at either rename still permits previous fallback. + damaged = self.namespace / (".stage-" + os.urandom(12).hex()) + os.rename(target, damaged) + os.rename(stage, target) + _fsync_directory(target.parent) + self._commit_state(state, old, deadline) + try: + self._gc() + except (OSError, LoaderError, SnapshotError): + # Activation already succeeded. Leave garbage accounted + # in the volume budget for the next pre-attempt cleanup. + pass + return result + finally: + if stage.exists(): + self._remove_owned(stage) + + def _run(self, mode, prepare, stop=None): + deadline = time.monotonic() + self._attempt_seconds + stop = stop if stop is not None else threading.Event() + parent, child = socket.socketpair() + process = multiprocessing.get_context("spawn").Process( + target=_worker, args=(self, mode, prepare, deadline, child), + name="v8std-snapshot-worker", daemon=True) + started = False + try: + if stop.is_set(): + raise LoaderError("closed") + try: + process.start() + started = True + except Exception: + raise LoaderError("prepare_failed") from None + child.close() + parent.setblocking(False) + payload = bytearray() + length = None + while length is None or len(payload) < length: + if stop.is_set(): + raise LoaderError("closed") + remaining = _remaining(deadline) + if not select.select([parent], [], [], min(.025, remaining))[0]: + continue + chunk = parent.recv(_CHUNK if length is not None else 8 - len(payload)) + if not chunk: + raise LoaderError("worker_failed") + payload.extend(chunk) + if length is None and len(payload) == 8: + length = struct.unpack("!Q", payload)[0] + payload.clear() + process.join(min(.1, _remaining(deadline))) + # Local trusted IPC only. No filesystem or HTTP bytes reach loads. + kind, result, metadata = pickle.loads(payload) + _remaining(deadline) + if kind == "format_error": + raise SnapshotError(result) + if kind == "loader_error": + raise LoaderError(result) + return result, metadata + finally: + parent.close() + child.close() + if started: + if process.is_alive(): + process.terminate() + process.join(.3) + if process.is_alive(): + process.kill() + process.join(.3) + if not process.is_alive(): + process.join() + process.close() + + +def _cache_walk_error(): + raise LoaderError("cache_io") + + +def _worker(store, mode, prepare, deadline, channel): + try: + if mode == "cached": + snapshot = store.cached() + payload = _prepare(snapshot, prepare) if snapshot else pickle.dumps(("ok", None, None)) + else: + payload = store._refresh(prepare, deadline) + _remaining(deadline) + except SnapshotError as error: + payload = pickle.dumps(("format_error", error.code, None)) + except LoaderError as error: + payload = pickle.dumps(("loader_error", error.code, None)) + except OSError: + payload = pickle.dumps(("loader_error", "cache_io", None)) + except Exception: + payload = pickle.dumps(("loader_error", "worker_failed", None)) + try: + channel.settimeout(max(.001, deadline - time.monotonic())) + channel.sendall(struct.pack("!Q", len(payload))) + channel.sendall(payload) + except OSError: + pass + finally: + channel.close() + + +class SnapshotCoordinator: + def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600): + if type(refresh_seconds) is not int or refresh_seconds < 0: + raise LoaderError("configuration") + self.store = store + self.build = build + self.refresh_seconds = refresh_seconds + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread = None + self._current = None + self._archive_sha256 = None + self._state = {"ready": False, "corpus_id": None, "loaded_at": None, + "last_checked_at": None, "last_success_at": None, + "refresh_error_code": None} + + def start(self) -> None: + with self._lock: + if self._stop.is_set(): + raise LoaderError("closed") + if self._thread is None: + self._thread = threading.Thread(target=self._loop, name="v8std-snapshot-supervisor", daemon=True) + self._thread.start() + + def current(self): + with self._lock: + if not self._state["ready"]: + raise LoaderError("INDEX_NOT_READY") + return self._current + + def status(self) -> dict: + with self._lock: + return dict(self._state) + + def _accept(self, result, metadata, *, checked): + now = time.time() + retired = None + with self._lock: + if metadata["archive_sha256"] != self._archive_sha256: + retired = self._current + self._current = result + self._archive_sha256 = metadata["archive_sha256"] + self._state.update(ready=True, corpus_id=metadata["corpus_id"], loaded_at=now) + if checked: + self._state.update(last_checked_at=now, last_success_at=now, refresh_error_code=None) + # Dropping a large generation's final reference can release thousands of + # objects. Even that CPU work belongs outside the query state lock. + del retired + + def _delay(self, failures): + if failures: + base = min(3600, 30 * 2 ** min(failures - 1, 7)) + return min(3600, max(30, base * random.uniform(.8, 1.2))) + return self.refresh_seconds * random.uniform(.8, 1.2) + + def _loop(self): + try: + result, metadata = self.store._run("cached", self.build, self._stop) + if metadata: + self._accept(result, metadata, checked=False) + except (LoaderError, SnapshotError): + pass + failures = 0 + while not self._stop.is_set(): + try: + result, metadata = self.store._run("refresh", self.build, self._stop) + if self._stop.is_set(): + return + self._accept(result, metadata, checked=True) + failures = 0 + except (LoaderError, SnapshotError) as error: + if self._stop.is_set(): + return + failures += 1 + with self._lock: + self._state.update(last_checked_at=time.time(), refresh_error_code=error.code) + if self.refresh_seconds == 0 and self.status()["ready"]: + return + if self._stop.wait(self._delay(failures)): + return + + def close(self) -> None: + self._stop.set() + with self._lock: + thread = self._thread + if thread is not None: + thread.join(1.5) + if thread.is_alive(): + raise LoaderError("worker_failed") diff --git a/tests/test_v8std_mcp_snapshots.py b/tests/test_v8std_mcp_snapshots.py new file mode 100644 index 0000000..53fd2d8 --- /dev/null +++ b/tests/test_v8std_mcp_snapshots.py @@ -0,0 +1,839 @@ +"""Loader conformance using real HTTP, independent archives and spawned builders.""" + +from dataclasses import dataclass +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib +import importlib.util +import errno +import fcntl +import json +import multiprocessing +import os +from pathlib import Path +import socket +import signal +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from tests import mcp_snapshot_fixtures as fixture + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + + +@dataclass(frozen=True) +class Generation: + corpus_id: str + pid: int + start_method: str + + +def build(snapshot): + return Generation(snapshot.metadata["corpus_id"], os.getpid(), + multiprocessing.get_start_method()) + + +def fail_build(snapshot): + raise RuntimeError("https://secret:password@example.invalid/private-corpus") + + +def blocking_build(snapshot): + time.sleep(120) + return build(snapshot) + + +def ignores_termination_build(snapshot): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + return blocking_build(snapshot) + + +def unpickleable_build(snapshot): + return threading.RLock() + + +def large_build(snapshot): + return (build(snapshot), b"x" * (8 * 1024 * 1024)) + + +def blocking_ipc_build(snapshot, *, corpus_id): + if snapshot.metadata["corpus_id"] == corpus_id: + original = socket.socket.sendall + + def sendall(channel, payload, *args): + if len(payload) > 8: + original(channel, payload[:8], *args) + time.sleep(120) + else: + original(channel, payload, *args) + + socket.socket.sendall = sendall + return build(snapshot) + + +@dataclass +class CleanupFault: + namespace: Path + + def __call__(self, snapshot): + original = Path.iterdir + active = snapshot.archive_sha256 + namespace = self.namespace + + def iterdir(path): + if path == namespace and json.loads((path / "state.json").read_bytes())["active"] == active: + raise OSError(errno.EACCES, "injected postcommit cleanup failure") + return original(path) + + Path.iterdir = iterdir + return build(snapshot) + + +class RecordingStop: + """Replace the actual scheduling wait, not refresh/store behavior.""" + + def __init__(self, count): + self.event = threading.Event() + self.delays = [] + self.count = count + + def is_set(self): + return self.event.is_set() + + def set(self): + self.event.set() + + def wait(self, timeout): + self.delays.append(timeout) + if len(self.delays) >= self.count: + self.set() + return self.is_set() + + +class SlowRetirement: + def __init__(self, started, release): + self.started = started + self.release = release + + def __del__(self): + self.started.set() + self.release.wait(3) + + +@dataclass +class CommitFault: + number: int + crash: bool = False + + def __call__(self, snapshot): + # Inject at the actual OS fsync boundary, after successful preparation. + original = os.fsync + calls = 0 + + def fsync(fd): + nonlocal calls + calls += 1 + if calls == self.number: + if self.crash: + os._exit(91) + raise OSError(errno.ENOSPC, "injected disk full") + return original(fd) + + os.fsync = fsync + return build(snapshot) + + +def blocking_dns_transport(*args, **kwargs): + import v8std_mcp_snapshots as loader + + def blocked(*args, **kwargs): + time.sleep(120) + + with patch("socket.getaddrinfo", blocked): + return loader._download(*args, **kwargs) + + +def store_in_process(site_url, cache, output): + import v8std_mcp_snapshots as loader + try: + output.send(loader.SnapshotStore(site_url, cache).refresh(prepare=build)) + except (loader.LoaderError, loader.SnapshotError) as error: + output.send(error.code) + finally: + output.close() + + +class Source: + """HTTP dependency: handlers can delay headers/body and count actual GETs.""" + + def __init__(self): + self.archive, self.manifest = fixture.snapshot_fixture() + self.requests = [] + self.fault = None + self.headers = {} + self.redirect = None + self.redirects = {} + self.requested = threading.Event() + self.release = threading.Event() + source = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + source.requests.append((self.path, dict(self.headers))) + source.requested.set() + is_manifest = self.path.endswith("manifest.json") + if source.fault == "headers": + source.release.wait(10) + location = source.redirects.get(self.path, source.redirect) + if location: + self.send_response(302) + self.send_header("Location", location) + self.end_headers() + return + if source.fault == "missing": + self.send_error(404) + return + if is_manifest and (source.fault == "304" or ( + source.fault == "conditional" and self.headers.get("If-None-Match"))): + self.send_response(304) + self.end_headers() + return + payload = fixture.json_bytes(source.manifest) if is_manifest else source.archive + if not is_manifest and source.fault == "corrupt": + payload = bytes([payload[0] ^ 1]) + payload[1:] + self.send_response(200) + self.send_header("Content-Type", "application/json" if is_manifest else "application/gzip") + self.send_header("ETag", '"fixture-v1"') + self.send_header("Last-Modified", "Thu, 10 Sep 2026 00:00:00 GMT") + if "Content-Length" not in source.headers: + self.send_header("Content-Length", str(len(payload))) + for key, value in source.headers.items(): + if value is not None: + self.send_header(key, value) + self.end_headers() + try: + if source.fault == "body": + self.wfile.write(payload[:1]) + self.wfile.flush() + source.release.wait(10) + self.wfile.write(payload[1:]) + elif source.fault == "drip": + for byte in payload: + self.wfile.write(bytes([byte])) + self.wfile.flush() + if source.release.wait(.08): + break + else: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.url = f"http://127.0.0.1:{self.server.server_port}/knowledge/" + + def next_generation(self, label="Next release"): + files = fixture.corpus_files() + files["llms.txt"] += ("\n" + label + "\n").encode() + self.archive, self.manifest = fixture.snapshot_fixture(files=fixture.with_metadata(files)) + + def close(self): + self.release.set() + self.server.shutdown() + self.server.server_close() + self.thread.join(2) + + +class SnapshotTestCase(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_snapshots"), + "Task2 snapshot loader is not implemented") + self.loader = importlib.import_module("v8std_mcp_snapshots") + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.cache = Path(self.temp.name) / "cache" + self.source = Source() + self.addCleanup(self.source.close) + self.store = self.loader.SnapshotStore(self.source.url, self.cache) + + +class SnapshotStoreTests(SnapshotTestCase): + def test_prefix_real_archive_and_spawn_builder_survive_offline_restart(self): + result = self.store.refresh(prepare=build) + self.assertEqual(result.corpus_id, self.source.manifest["corpus_id"]) + self.assertNotEqual(result.pid, os.getpid()) + self.assertEqual(result.start_method, "spawn") + self.assertEqual([p for p, _ in self.source.requests], [ + "/knowledge/ai/mcp/v1/manifest.json", + "/knowledge/ai/mcp/v1/" + self.source.manifest["archive"]["path"], + ]) + self.source.fault = "missing" + restarted = self.loader.SnapshotStore(self.source.url, self.cache) + cached = restarted.cached() + self.assertEqual(cached.metadata["corpus_id"], result.corpus_id) + self.assertIn(b"std437", cached.files["pages.jsonl"]) + + def test_normalized_source_reuses_cache_and_other_sources_never_do(self): + first = self.store.refresh() + alias = self.loader.SnapshotStore(" " + self.source.url.rstrip("/") + " ", self.cache) + self.assertEqual(alias.cached().archive_sha256, first.archive_sha256) + other = self.loader.SnapshotStore(self.source.url + "other/", self.cache) + self.assertIsNone(other.cached()) + + def test_different_sources_on_shared_volume_download_and_cache_independently(self): + first = self.store.refresh() + other = self.loader.SnapshotStore(self.source.url + "other/", self.cache) + second = other.refresh() + self.assertEqual(first.archive_sha256, second.archive_sha256) + self.assertNotEqual(self.store.namespace, other.namespace) + self.assertEqual(len([p for p, _ in self.source.requests if p.endswith(".tar.gz")]), 2) + self.assertEqual(other.cached().archive_sha256, second.archive_sha256) + + def test_prepare_failure_does_not_activate_disk_generation(self): + before = self.store.refresh() + self.source.next_generation() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=fail_build) + self.assertEqual(caught.exception.code, "prepare_failed") + self.assertEqual(str(caught.exception), "prepare_failed") + self.assertEqual(self.store.cached().archive_sha256, before.archive_sha256) + + def test_invalid_selected_archive_preserves_cache_without_public_fallback(self): + before = self.store.refresh() + self.source.next_generation() + self.source.fault = "corrupt" + with self.assertRaises(self.loader.SnapshotError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "archive_hash") + self.assertEqual(self.store.cached().archive_sha256, before.archive_sha256) + self.assertTrue(all(p.startswith("/knowledge/") for p, _ in self.source.requests)) + + def test_local_manifest_cannot_select_public_archive(self): + self.source.manifest["archive"]["path"] = ( + "https://ai.v8std.ru/indexes/v1/" + self.source.manifest["archive"]["path"]) + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "url_policy") + self.assertEqual(len(self.source.requests), 1) + self.assertIsNone(self.store.cached()) + + def test_redirects_reject_foreign_traversal_credentials_and_outside_prefix(self): + for location in ("//example.invalid/", "../escape", "/elsewhere/", "%2e%2e/x", + "http://user:secret@127.0.0.1/x", "http://example.invalid/x"): + with self.subTest(location=location): + self.source.redirect = location + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "url_policy") + self.assertEqual(len(self.source.requests), 6) + + def test_redirect_loop_stops_after_three_hops(self): + self.source.redirect = "/knowledge/ai/mcp/v1/manifest.json" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "redirect_limit") + self.assertEqual(len(self.source.requests), 4) + + def test_conditional_get_and_same_hash_never_redownload_archive(self): + before = self.store.refresh() + self.source.fault = "conditional" + self.assertEqual(self.store.refresh().archive_sha256, before.archive_sha256) + self.source.fault = None + self.assertEqual(self.store.refresh().archive_sha256, before.archive_sha256) + manifests = [h for p, h in self.source.requests if p.endswith("manifest.json")] + archives = [p for p, h in self.source.requests if p.endswith(".tar.gz")] + self.assertEqual(len(archives), 1) + self.assertNotIn("If-None-Match", manifests[0]) + self.assertEqual(manifests[1]["If-None-Match"], '"fixture-v1"') + self.assertEqual(manifests[1]["If-Modified-Since"], "Thu, 10 Sep 2026 00:00:00 GMT") + + def test_304_without_valid_cache_retries_unconditionally_once(self): + self.source.fault = "304" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "http_status") + self.assertEqual(len(self.source.requests), 2) + self.assertTrue(all("If-None-Match" not in h for _, h in self.source.requests)) + + def test_encoding_length_and_stream_byte_caps(self): + for headers in ({"Content-Encoding": "gzip"}, {"Content-Length": "99999999"}, + {"Content-Length": "-1"}, {"Content-Length": "not-a-number"}): + with self.subTest(headers=headers): + self.source.headers = headers + with self.assertRaises(self.loader.LoaderError): + self.store.refresh() + self.assertIsNone(self.store.cached()) + self.source.headers = {"Content-Length": None} + self.source.manifest["extra"] = "x" * (64 * 1024) + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "http_size") + + def test_whole_deadline_reaps_slow_headers_body_and_drip(self): + for fault in ("headers", "body", "drip"): + with self.subTest(fault=fault): + self.source.fault = fault + self.store._attempt_seconds = .65 + start = time.monotonic() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "deadline") + self.assertLess(time.monotonic() - start, 2) + self.assertFalse(multiprocessing.active_children()) + + def test_current_corruption_falls_back_to_previous_same_source(self): + first = self.store.refresh() + self.source.next_generation() + second = self.store.refresh() + for path in self.cache.rglob("snapshot.tar.gz"): + if fixture.sha256(path.read_bytes()) == second.archive_sha256: + path.write_bytes(b"corrupt") + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + + def test_verified_redownload_repairs_corrupt_current_directory(self): + self.store.refresh() + self.source.next_generation() + second = self.store.refresh() + directory = self.store.namespace / "generations" / second.archive_sha256 + (directory / "pages.jsonl").write_bytes(b"corrupt expanded member") + self.assertNotEqual(self.store.cached().archive_sha256, second.archive_sha256) + self.assertEqual(self.store.refresh().archive_sha256, second.archive_sha256) + self.assertEqual(self.store.cached().archive_sha256, second.archive_sha256) + + def test_manifest_reactivates_previous_verified_archive_without_download(self): + original_archive, original_manifest = self.source.archive, self.source.manifest + first = self.store.refresh() + self.source.next_generation() + second = self.store.refresh() + self.source.archive, self.source.manifest = original_archive, original_manifest + self.assertEqual(self.store.refresh().archive_sha256, first.archive_sha256) + state = json.loads((self.store.namespace / "state.json").read_bytes()) + self.assertEqual(state["previous"], second.archive_sha256) + self.assertEqual(len([p for p, _ in self.source.requests if p.endswith(".tar.gz")]), 2) + + def test_new_cache_namespace_directory_entries_are_durable_before_activation(self): + synced = set() + observed = [] + original_sync, original_replace = os.fsync, os.replace + + def fsync(fd): + info = os.fstat(fd) + synced.add((info.st_dev, info.st_ino)) + return original_sync(fd) + + def replace(source, target): + if Path(target).name == "state.json": + for directory in (self.cache.parent, self.cache, self.store.namespace): + info = directory.stat() + observed.append((info.st_dev, info.st_ino) in synced) + return original_replace(source, target) + + with patch("os.fsync", fsync), patch("os.replace", replace): + self.store._refresh(build, time.monotonic() + 60) + self.assertEqual(self.store.cached().metadata["corpus_id"], self.source.manifest["corpus_id"]) + self.assertEqual(observed, [True, True, True]) + + def test_existing_cache_volume_does_not_require_writable_parent_filesystem(self): + before = self.store.refresh() + parent = self.cache.parent.stat() + original = os.fsync + + def fsync(fd): + info = os.fstat(fd) + if (info.st_dev, info.st_ino) == (parent.st_dev, parent.st_ino): + raise OSError(errno.EROFS, "read-only container root") + return original(fd) + + with patch("os.fsync", fsync): + self.store._refresh(build, time.monotonic() + 60) + self.assertEqual(self.store.cached().archive_sha256, before.archive_sha256) + + def test_postcommit_cleanup_failure_does_not_report_failed_activation(self): + self.store.refresh() + self.source.next_generation() + result = self.store.refresh(prepare=CleanupFault(self.store.namespace)) + self.assertEqual(result.corpus_id, self.source.manifest["corpus_id"]) + self.assertEqual(self.store.cached().metadata["corpus_id"], result.corpus_id) + + def test_unpickleable_result_is_preparation_failure_before_disk_activation(self): + first = self.store.refresh() + pointer = (self.store.namespace / "state.json").read_bytes() + self.source.next_generation() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=unpickleable_build) + self.assertEqual(caught.exception.code, "prepare_failed") + self.assertEqual((self.store.namespace / "state.json").read_bytes(), pointer) + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + + def test_large_prepared_result_crosses_real_ipc_and_returns_after_commit(self): + generation, payload = self.store.refresh(prepare=large_build) + self.assertEqual(payload, b"x" * (8 * 1024 * 1024)) + self.assertNotEqual(generation.pid, os.getpid()) + self.assertEqual(self.store.cached().metadata["corpus_id"], generation.corpus_id) + self.assertFalse(multiprocessing.active_children()) + + def test_prepare_and_dns_are_terminated_at_whole_attempt_deadline(self): + self.store._attempt_seconds = .65 + for prepare, transport in ((blocking_build, self.loader._download), + (build, blocking_dns_transport)): + with self.subTest(prepare=prepare.__name__): + self.store._transport = transport + start = time.monotonic() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=prepare) + self.assertEqual(caught.exception.code, "deadline") + self.assertLess(time.monotonic() - start, 2) + self.assertFalse(multiprocessing.active_children()) + self.assertIsNone(self.store.cached()) + + def test_uncooperative_worker_is_killed_and_reaped_after_terminate_grace(self): + self.store._attempt_seconds = .65 + start = time.monotonic() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=ignores_termination_build) + self.assertEqual(caught.exception.code, "deadline") + self.assertLess(time.monotonic() - start, 2) + self.assertFalse(multiprocessing.active_children()) + + def test_each_commit_fsync_failure_keeps_old_disk_pointer(self): + before = self.store.refresh() + pointer = (self.store.namespace / "state.json").read_bytes() + self.source.next_generation() + for number in range(1, 7): + with self.subTest(fsync=number): + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=CommitFault(number)) + self.assertEqual(caught.exception.code, "cache_io") + self.assertEqual((self.store.namespace / "state.json").read_bytes(), pointer) + self.assertEqual(self.store.cached().archive_sha256, before.archive_sha256) + + def test_crash_at_commit_stages_recovers_complete_old_or_new_generation(self): + first = self.store.refresh() + self.source.next_generation() + for number in range(1, 7): + with self.subTest(fsync=number): + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=CommitFault(number, crash=True)) + self.assertEqual(caught.exception.code, "worker_failed") + recovered = self.store.cached() + expected = first.archive_sha256 if number < 6 else self.source.manifest["archive"]["sha256"] + self.assertEqual(recovered.archive_sha256, expected) + self.assertFalse(multiprocessing.active_children()) + self.assertEqual(self.store.refresh().archive_sha256, self.source.manifest["archive"]["sha256"]) + + def test_corrupt_pointer_uses_durable_rollback_record(self): + first = self.store.refresh() + self.source.next_generation() + self.store.refresh() + (self.store.namespace / "state.json").write_bytes(b"broken JSON") + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + + def test_gc_preserves_last_verified_rollback_when_newer_pointer_targets_are_bad(self): + first = self.store.refresh() + self.source.next_generation() + second = self.store.refresh() + namespace = self.store.namespace + state = json.loads((namespace / "state.json").read_bytes()) + # Damaged newest pointer still has a valid schema; rollback.json retains + # the only usable generation. A failed refresh must not GC that fallback. + state["active"], state["previous"] = "a" * 64, second.archive_sha256 + (namespace / "state.json").write_bytes(fixture.json_bytes(state)) + (namespace / "generations" / second.archive_sha256 / "pages.jsonl").write_bytes(b"bad") + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + self.source.fault = "missing" + with self.assertRaises(self.loader.LoaderError): + self.store.refresh() + self.assertIsNotNone(self.store.cached(), "GC removed the only verified rollback") + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + + def test_deadline_covers_partial_trusted_ipc_result_and_reaps_sender(self): + self.store._attempt_seconds = .65 + start = time.monotonic() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh(prepare=partial(blocking_ipc_build, corpus_id=self.source.manifest["corpus_id"])) + self.assertEqual(caught.exception.code, "deadline") + self.assertLess(time.monotonic() - start, 2) + self.assertFalse(multiprocessing.active_children()) + # Commit completed before this injected partial-send crash window. + self.assertEqual(self.store.cached().metadata["corpus_id"], self.source.manifest["corpus_id"]) + + def test_pins_retain_predecessor_until_unpinned_and_gc_keeps_other_namespace(self): + first = self.store.refresh() + pins = self.store.namespace / "pins.json" + pins.write_bytes(fixture.json_bytes({"archives": [first.archive_sha256]})) + foreign = self.cache / "v1-other-source" / ".stage-owned-by-another-process" + foreign.mkdir(parents=True) + (foreign / "keep").write_bytes(b"foreign source") + self.source.next_generation("second") + second = self.store.refresh() + self.source.next_generation("third") + third = self.store.refresh() + generations = self.store.namespace / "generations" + self.assertEqual({p.name for p in generations.iterdir()}, + {first.archive_sha256, second.archive_sha256, third.archive_sha256}) + pins.write_bytes(fixture.json_bytes({"archives": []})) + self.store.refresh() + self.assertEqual({p.name for p in generations.iterdir()}, + {second.archive_sha256, third.archive_sha256}) + self.assertEqual((foreign / "keep").read_bytes(), b"foreign source") + + def test_volume_budget_counts_foreign_staging_pins_and_never_evicts_them(self): + first = self.store.refresh() + pins = self.store.namespace / "pins.json" + pins.write_bytes(fixture.json_bytes({"archives": [first.archive_sha256]})) + other = self.cache / "v1-other-source" / ".stage-foreign" + other.mkdir(parents=True) + blob = other / "reserved" + with blob.open("wb") as stream: + stream.truncate(256 * 1024 * 1024 - 32 * 1024) + before = sum(p.stat().st_size for p in self.cache.rglob("*") if p.is_file()) + self.assertLess(before, 256 * 1024 * 1024) + self.source.next_generation() + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "cache_budget") + self.assertEqual(blob.stat().st_size, 256 * 1024 * 1024 - 32 * 1024) + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + self.assertEqual(before, sum(p.stat().st_size for p in self.cache.rglob("*") if p.is_file())) + + def test_symlinks_and_invalid_pins_fail_closed_without_deleting_targets(self): + first = self.store.refresh() + outside = Path(self.temp.name) / "outside" + outside.mkdir() + (outside / "keep").write_bytes(b"external") + (self.store.namespace / ".stage-link").symlink_to(outside, target_is_directory=True) + (self.store.namespace / "pins.json").write_bytes(b'{"archives":["../outside"]}') + self.source.next_generation() + with self.assertRaises(self.loader.LoaderError): + self.store.refresh() + self.assertEqual((outside / "keep").read_bytes(), b"external") + self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + + def test_two_processes_share_download_and_each_builds_own_generation(self): + self.source.fault = "headers" + context = multiprocessing.get_context("spawn") + parents = [] + processes = [] + for _ in range(2): + parent, child = context.Pipe(duplex=False) + process = context.Process(target=store_in_process, args=(self.source.url, self.cache, child)) + parents.append(parent) + processes.append(process) + process.start() + child.close() + try: + self.assertTrue(self.source.requested.wait(4)) + time.sleep(.35) # second process reaches the contended namespace flock + self.source.release.set() + results = [] + for parent in parents: + self.assertTrue(parent.poll(5)) + results.append(parent.recv()) + self.assertTrue(all(isinstance(result, Generation) for result in results), results) + self.assertNotEqual(results[0].pid, results[1].pid) + self.assertEqual(results[0].corpus_id, results[1].corpus_id) + self.assertEqual(len(self.source.requests), 2) + finally: + self.source.release.set() + for process in processes: + process.join(2) + if process.is_alive(): + process.kill() + process.join(2) + process.close() + for parent in parents: + parent.close() + + def test_lock_wait_is_bounded_and_does_not_touch_network(self): + self.store.refresh() + self.source.requests.clear() + self.store._attempt_seconds = .65 + with (self.store.namespace / ".lock").open("rb") as stream: + fcntl.flock(stream, fcntl.LOCK_EX) + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertIn(caught.exception.code, {"lock_timeout", "deadline"}) + self.assertFalse(self.source.requests) + self.assertFalse(multiprocessing.active_children()) + + def test_read_timeout_is_distinct_from_whole_attempt_and_empty_length_is_validated(self): + self.store._read_seconds = .12 + self.source.fault = "body" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "network") + self.source.fault = None + self.source.headers = {"Content-Length": "0"} + with self.assertRaises(self.loader.SnapshotError): + self.store.refresh() + + def test_allowed_redirect_preserves_selected_base_and_limits_archive_redirects(self): + manifest_path = "/knowledge/ai/mcp/v1/manifest.json" + self.source.redirects[manifest_path] = "/knowledge/mirror/manifest.json" + self.store.refresh() + self.assertEqual(self.source.requests[-1][0], + "/knowledge/mirror/" + self.source.manifest["archive"]["path"]) + self.source.next_generation() + archive_path = "/knowledge/mirror/" + self.source.manifest["archive"]["path"] + self.source.redirects[archive_path] = "/outside/snapshot.tar.gz" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store.refresh() + self.assertEqual(caught.exception.code, "url_policy") + + def test_public_delivery_exception_and_https_downgrade_policy(self): + _, manifest = fixture.snapshot_fixture() + manifest["archive"]["path"] = "https://ai.v8std.ru/indexes/v1/" + manifest["archive"]["path"] + url, boundary = self.loader._archive_url(manifest, "https://v8std.ru/ai/mcp/v1/manifest.json", "https://v8std.ru/") + self.assertEqual(url, manifest["archive"]["path"]) + self.assertEqual(boundary, "https://ai.v8std.ru/indexes/v1/") + for location in (url.replace("https:", "http:"), "https://v8std.ru/ai/mcp/v1/x", + "https://ai.v8std.ru/indexes/v2/x"): + with self.assertRaises(self.loader.LoaderError) as caught: + self.loader._allowed_url(location, url, boundary) + self.assertEqual(caught.exception.code, "url_policy") + + +class SnapshotCoordinatorTests(SnapshotTestCase): + def coordinator(self, **kwargs): + coordinator = self.loader.SnapshotCoordinator(self.store, build, **kwargs) + self.addCleanup(coordinator.close) + return coordinator + + def wait_until(self, predicate, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(.01) + self.fail("background coordinator did not reach expected state") + + def test_cold_failure_is_explicit_and_status_does_not_expose_raw_errors(self): + self.source.fault = "missing" + coordinator = self.coordinator(refresh_seconds=0) + coordinator.start() + self.wait_until(lambda: coordinator.status()["refresh_error_code"] is not None) + with self.assertRaises(self.loader.LoaderError) as caught: + coordinator.current() + self.assertEqual(caught.exception.code, "INDEX_NOT_READY") + self.assertEqual(coordinator.status()["refresh_error_code"], "http_status") + self.assertNotIn(self.source.url, json.dumps(coordinator.status())) + + def test_warm_current_and_status_are_fast_while_network_blocks_then_close_reaps(self): + self.store.refresh() + self.source.requested.clear() + self.source.fault = "headers" + coordinator = self.coordinator(refresh_seconds=0) + coordinator.start() + self.wait_until(lambda: coordinator.status()["ready"]) + before = coordinator.current() + self.assertTrue(self.source.requested.wait(3)) + start = time.monotonic() + for _ in range(1000): + self.assertIs(coordinator.current(), before) + self.assertTrue(coordinator.status()["ready"]) + self.assertLess(time.monotonic() - start, .25) + start = time.monotonic() + coordinator.close() + self.assertLess(time.monotonic() - start, 2) + self.assertFalse(multiprocessing.active_children()) + + def test_close_reaps_partial_ipc_sender_and_preserves_process_generation(self): + original = self.store.refresh() + self.source.next_generation() + target = self.source.manifest["corpus_id"] + coordinator = self.loader.SnapshotCoordinator( + self.store, partial(blocking_ipc_build, corpus_id=target), refresh_seconds=0) + self.addCleanup(coordinator.close) + coordinator.start() + self.wait_until(lambda: coordinator.status()["ready"]) + before = coordinator.current() + self.assertEqual(before.corpus_id, original.metadata["corpus_id"]) + self.wait_until(lambda: self.store.cached().metadata["corpus_id"] == target) + start = time.monotonic() + coordinator.close() + self.assertLess(time.monotonic() - start, 2) + self.assertIs(coordinator.current(), before) + self.assertFalse(multiprocessing.active_children()) + + def test_zero_refresh_stops_after_successful_bootstrap(self): + coordinator = self.coordinator(refresh_seconds=0) + coordinator.start() + coordinator.start() + self.wait_until(lambda: coordinator.status()["last_success_at"] is not None) + self.source.next_generation() + time.sleep(.15) + self.assertEqual(len(self.source.requests), 2) + self.assertEqual(coordinator.current().start_method, "spawn") + self.assertIsNotNone(coordinator.status()["loaded_at"]) + self.assertIsNotNone(coordinator.status()["last_checked_at"]) + + def test_generation_retirement_does_not_hold_query_state_lock(self): + coordinator = self.coordinator(refresh_seconds=0) + retiring, release, queried = threading.Event(), threading.Event(), threading.Event() + coordinator._accept(SlowRetirement(retiring, release), + {"archive_sha256": "a", "corpus_id": "old"}, checked=True) + swap = threading.Thread(target=coordinator._accept, + args=("new", {"archive_sha256": "b", "corpus_id": "new"}), + kwargs={"checked": True}) + swap.start() + values = [] + + def query(): + values.append(coordinator.current()) + queried.set() + + reader = threading.Thread(target=query) + try: + self.assertTrue(retiring.wait(1)) + reader.start() + self.assertTrue(queried.wait(.25), "retiring generation holds query lock") + self.assertEqual(values, ["new"]) + finally: + release.set() + swap.join(2) + if reader.ident: + reader.join(2) + + def test_repeated_cold_faults_back_off_without_becoming_ready(self): + self.source.fault = "missing" + coordinator = self.coordinator(refresh_seconds=0) + schedule = RecordingStop(4) + coordinator._stop = schedule + with patch("v8std_mcp_snapshots.random.uniform", return_value=1.2): + coordinator.start() + self.wait_until(schedule.is_set) + self.assertEqual(schedule.delays, [36, 72, 144, 288]) + self.assertEqual(len(self.source.requests), 4) + self.assertFalse(coordinator.status()["ready"]) + + def test_same_hash_preserves_generation_and_success_interval_uses_jitter(self): + coordinator = self.coordinator() + schedule = RecordingStop(2) + coordinator._stop = schedule + with patch("v8std_mcp_snapshots.random.uniform", side_effect=[.8, 1.2]): + coordinator.start() + self.wait_until(schedule.is_set) + self.assertEqual(schedule.delays, [2880, 4320]) + self.assertEqual(len([p for p, _ in self.source.requests if p.endswith(".tar.gz")]), 1) + state = coordinator.status() + self.assertLess(state["loaded_at"], state["last_success_at"]) + self.assertEqual(state["last_checked_at"], state["last_success_at"]) + + def test_backoff_extremes_stay_within_contract(self): + coordinator = self.coordinator() + with patch("v8std_mcp_snapshots.random.uniform", return_value=.8): + self.assertEqual(coordinator._delay(1), 30) + with patch("v8std_mcp_snapshots.random.uniform", return_value=1.2): + self.assertEqual(coordinator._delay(10000), 3600) + + +if __name__ == "__main__": + unittest.main() From c257684ac5e048311cd0f5cd0bd9d7945ac1bf7d Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:26:16 +0300 Subject: [PATCH 12/88] fix: stream verified snapshots into private staging --- scripts/v8std_mcp_snapshots.py | 32 ++++++++++++++++++++++++++----- tests/test_v8std_mcp_snapshots.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py index c3ab093..dc09854 100644 --- a/scripts/v8std_mcp_snapshots.py +++ b/scripts/v8std_mcp_snapshots.py @@ -18,6 +18,7 @@ from contextlib import contextmanager import fcntl import http.client +import io import multiprocessing import os from pathlib import Path @@ -30,6 +31,7 @@ import ssl import stat import struct +import tarfile import tempfile import threading import time @@ -334,16 +336,37 @@ def _space(self, needed): raise LoaderError("cache_budget") def _write(self, path, payload, deadline): + self._write_stream(path, io.BytesIO(payload), len(payload), deadline) + + def _write_stream(self, path, source, size, deadline): _remaining(deadline) - self._space(len(payload)) + self._space(size) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "wb") as stream: - for offset in range(0, len(payload), _CHUNK): + remaining = size + while remaining: _remaining(deadline) - stream.write(payload[offset:offset + _CHUNK]) + chunk = source.read(min(remaining, _CHUNK)) + if not chunk or len(chunk) > remaining: + raise LoaderError("cache_io") + stream.write(chunk) + remaining -= len(chunk) stream.flush() os.fsync(stream.fileno()) + def _extract_verified(self, source, snapshot, stage, deadline): + """Stream the already-verified immutable archive into private staging. + + Task1 alone owns format/semantic rules. It verifies bytes before this + filesystem step; its buffers are not a substitute for streaming disk + extraction. Destinations and sizes come only from that verified result, + never from unverified tar paths. No extract/extractall filesystem API. + """ + with tarfile.open(fileobj=source, mode="r|gz") as reader: + for name, payload in snapshot.files.items(): + with reader.extractfile(reader.next()) as member: + self._write_stream(stage / name, member, len(payload), deadline) + def _atomic_file(self, path, payload, deadline): temporary = self.namespace / (".pointer-" + os.urandom(12).hex()) try: @@ -474,8 +497,7 @@ def _refresh(self, prepare, deadline): raise LoaderError("http_status") self._write(stage / "snapshot.tar.gz", archive, deadline) snapshot = verify_archive(archive, manifest) - for name, payload in snapshot.files.items(): - self._write(stage / name, payload, deadline) + self._extract_verified(io.BytesIO(archive), snapshot, stage, deadline) result = _prepare(snapshot, prepare) _remaining(deadline) _fsync_directory(stage) diff --git a/tests/test_v8std_mcp_snapshots.py b/tests/test_v8std_mcp_snapshots.py index 53fd2d8..7d30da2 100644 --- a/tests/test_v8std_mcp_snapshots.py +++ b/tests/test_v8std_mcp_snapshots.py @@ -5,11 +5,13 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import importlib import importlib.util +import io import errno import fcntl import json import multiprocessing import os +import random from pathlib import Path import socket import signal @@ -266,6 +268,35 @@ def setUp(self): class SnapshotStoreTests(SnapshotTestCase): + def test_verified_archive_is_streamed_into_staging_before_compressed_input_ends(self): + extract = getattr(self.store, "_extract_verified", None) + self.assertIsNotNone(extract, "Task2 must stream extraction into private staging") + files = fixture.corpus_files() + files["llms-full.txt"] = random.Random(37).randbytes(256 * 1024).hex().encode() + archive, manifest = fixture.snapshot_fixture(files=fixture.with_metadata(files)) + fmt = importlib.import_module("v8std_mcp_snapshot_format") + verified = fmt.verify_archive(archive, manifest) + self.store.namespace.mkdir(parents=True) + with tempfile.TemporaryDirectory(prefix=".stage-", dir=self.store.namespace) as directory: + stage = Path(directory) + + class ObservedStream(io.BytesIO): + output_before_eof = False + + def read(self, size=-1): + if not 0 < size <= 64 * 1024: + raise AssertionError("compressed input must be read in bounded chunks") + output = stage / "llms-full.txt" + if output.exists() and output.stat().st_size > 0 and self.tell() < len(archive): + self.output_before_eof = True + return super().read(size) + + source = ObservedStream(archive) + extract(source, verified, stage, time.monotonic() + 60) + self.assertTrue(source.output_before_eof, + "expanded bytes must reach staging before all compressed input is consumed") + self.assertEqual({p.name: p.read_bytes() for p in stage.iterdir()}, verified.files) + def test_prefix_real_archive_and_spawn_builder_survive_offline_restart(self): result = self.store.refresh(prepare=build) self.assertEqual(result.corpus_id, self.source.manifest["corpus_id"]) From d47c588efd26b40e256df8ce7815d4775094252a Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:44:49 +0300 Subject: [PATCH 13/88] fix: validate snapshot pointers and release idle generations --- scripts/v8std_mcp_snapshots.py | 11 +++- tests/test_v8std_mcp_snapshots.py | 100 ++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py index dc09854..0be3de6 100644 --- a/scripts/v8std_mcp_snapshots.py +++ b/scripts/v8std_mcp_snapshots.py @@ -256,11 +256,16 @@ def _states(self): for name in ("state.json", "rollback.json"): try: state = strict_json(_read_file(self.namespace / name, MAX_MANIFEST_BYTES)) - if (state.get("schema_version") != 1 or state.get("site_url") != self.site_url + if (type(state.get("schema_version")) is not int or state["schema_version"] != 1 + or state.get("site_url") != self.site_url or not self._digest(state.get("active")) or (state.get("previous") is not None and not self._digest(state["previous"]))): continue + # Recovery retains this record for the next rollback commit, + # including unknown fields. Reject it before selecting a corpus + # unless that same commit serializer can represent every field. + canonical_json(state) yield state except (OSError, SnapshotError, LoaderError): continue @@ -670,6 +675,7 @@ def _loop(self): result, metadata = self.store._run("cached", self.build, self._stop) if metadata: self._accept(result, metadata, checked=False) + del result # The active reference owns the accepted bootstrap result. except (LoaderError, SnapshotError): pass failures = 0 @@ -679,6 +685,9 @@ def _loop(self): if self._stop.is_set(): return self._accept(result, metadata, checked=True) + # Same-hash candidates are not adopted. Drop the loop's reference + # outside the query lock, before sleeping for a refresh interval. + del result failures = 0 except (LoaderError, SnapshotError) as error: if self._stop.is_set(): diff --git a/tests/test_v8std_mcp_snapshots.py b/tests/test_v8std_mcp_snapshots.py index 7d30da2..8e19afd 100644 --- a/tests/test_v8std_mcp_snapshots.py +++ b/tests/test_v8std_mcp_snapshots.py @@ -21,6 +21,7 @@ import time import unittest from unittest.mock import patch +import weakref from tests import mcp_snapshot_fixtures as fixture @@ -567,6 +568,68 @@ def test_corrupt_pointer_uses_durable_rollback_record(self): (self.store.namespace / "state.json").write_bytes(b"broken JSON") self.assertEqual(self.store.cached().archive_sha256, first.archive_sha256) + def pointer_recovery_pair(self, case): + self.source.archive, self.source.manifest = fixture.snapshot_fixture() + store = self.loader.SnapshotStore(self.source.url, self.cache / case) + previous = store.refresh() + self.source.next_generation("pointer-current") + current = store.refresh() + return store, previous, current + + def test_noninteger_pointer_versions_use_rollback_and_allow_refresh(self): + for case, version in (("float-version", 1.0), ("bool-version", True)): + with self.subTest(version=version): + store, previous, _ = self.pointer_recovery_pair(case) + pointer = store.namespace / "state.json" + state = json.loads(pointer.read_bytes()) + state["schema_version"] = version + pointer.write_bytes(fixture.json_bytes(state)) + self.assertEqual(store.cached().archive_sha256, previous.archive_sha256, + "noninteger version must not select the current pointer") + self.source.next_generation("pointer-recovered") + refreshed = store.refresh() + committed = json.loads(pointer.read_bytes()) + self.assertEqual(refreshed.metadata["corpus_id"], self.source.manifest["corpus_id"]) + self.assertEqual(committed["previous"], previous.archive_sha256) + self.assertIs(type(committed["schema_version"]), int) + self.assertEqual(store.refresh().archive_sha256, refreshed.archive_sha256) + + def test_noncanonical_pointer_extensions_fall_back_and_do_not_block_refresh(self): + for case, extra in (("float-field", .5), ("nested-float", {"values": [1, .5]})): + with self.subTest(extension=extra): + store, previous, _ = self.pointer_recovery_pair(case) + pointer = store.namespace / "state.json" + state = json.loads(pointer.read_bytes()) + state["extension"] = extra + pointer.write_bytes(fixture.json_bytes(state)) + self.source.next_generation("pointer-recovered") + try: + refreshed = store.refresh() + except (self.loader.LoaderError, self.loader.SnapshotError) as error: + self.fail("malformed current pointer blocked rollback refresh: " + error.code) + committed = json.loads(pointer.read_bytes()) + backup = json.loads((store.namespace / "rollback.json").read_bytes()) + self.assertEqual(refreshed.metadata["corpus_id"], self.source.manifest["corpus_id"]) + self.assertEqual(committed["previous"], previous.archive_sha256) + self.assertEqual(backup["active"], previous.archive_sha256) + self.assertNotIn("extension", backup) + self.assertEqual(store.refresh().archive_sha256, refreshed.archive_sha256) + + def test_canonical_pointer_extensions_remain_usable_and_survive_rollback_commit(self): + store, _, current = self.pointer_recovery_pair("canonical-field") + pointer = store.namespace / "state.json" + state = json.loads(pointer.read_bytes()) + extension = {"values": [1, True, None, "release-note"]} + state["extension"] = extension + pointer.write_bytes(fixture.json_bytes(state)) + self.assertEqual(store.cached().archive_sha256, current.archive_sha256) + self.source.next_generation("pointer-recovered") + refreshed = store.refresh() + backup = json.loads((store.namespace / "rollback.json").read_bytes()) + self.assertEqual(refreshed.metadata["corpus_id"], self.source.manifest["corpus_id"]) + self.assertEqual(backup["active"], current.archive_sha256) + self.assertEqual(backup["extension"], extension) + def test_gc_preserves_last_verified_rollback_when_newer_pointer_targets_are_bad(self): first = self.store.refresh() self.source.next_generation() @@ -866,5 +929,42 @@ def test_backoff_extremes_stay_within_contract(self): self.assertEqual(coordinator._delay(10000), 3600) +class SnapshotLifetimeTests(unittest.TestCase): + def test_same_hash_refresh_releases_unused_generation_before_idle_wait(self): + loader = importlib.import_module("v8std_mcp_snapshots") + references = [] + + class CompletedStore: + # The spawn/IPC boundary is covered by the real store tests. Here + # weakrefs isolate ownership after a completed result is delivered. + def _run(self, mode, prepare, stop): + generation = Generation("same-corpus", os.getpid(), "completed-ipc") + references.append(weakref.ref(generation)) + return generation, {"corpus_id": "same-corpus", "archive_sha256": "a" * 64} + + class ObservedStop(threading.Event): + def __init__(self): + super().__init__() + self.idle = threading.Event() + + def wait(self, timeout=None): + self.idle.set() + return super().wait(timeout) + + coordinator = loader.SnapshotCoordinator(CompletedStore(), build) + stop = ObservedStop() + coordinator._stop = stop + try: + coordinator.start() + self.assertTrue(stop.idle.wait(2), "coordinator never entered its refresh interval") + self.assertEqual(len(references), 2) + self.assertIs(coordinator.current(), references[0]()) + self.assertIsNone(references[1](), "unused same-hash result remains alive during idle") + self.assertTrue(coordinator._thread.is_alive()) + self.assertFalse(stop.is_set()) + finally: + coordinator.close() + + if __name__ == "__main__": unittest.main() From 37fdb98aadf8d03ed8015a63b93aaf0c41b13a21 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 15:54:11 +0300 Subject: [PATCH 14/88] docs: record reviewed snapshot cache completion --- spec/operations/mcp-container-verification.md | 30 ++++++++++++++++++- ...6-09-10-mcp-container-distribution-plan.md | 8 ++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index f008853..4e77920 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -45,6 +45,34 @@ is unconfirmed; it does not prove the previous manifest remains selected. The old immutable archive remains intact. These tests are fault injection, not a real machine power-loss experiment. +### Bounded source, cache and background coordinator + +Implementation: `ecc946c`, streaming staging correction: `005ba5d`, reviewed +pointer/lifetime fixes: `427aaf7cde5f288f980b335442c368dd1a28f081`. +Independent task review and scoped re-review completed without open findings. + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshots -v +``` + +Final result: **48 tests passed in 35.869 seconds**. The unchanged 35 format +tests passed during the combined pre-fix run. Coverage includes real local HTTP +sources, redirects, 304, corrupt data, slow DNS/headers/body, partial IPC, +process termination, private streaming staging, shared-volume locks/budget, +pins, fsync/crash recovery and responsive current-generation access. + +Review corrections reject pointer records that cannot later be canonically +committed, and release unused same-hash generation results before idle waits. +Parent-side production-index reconstruction, real refresh memory/latency and +the host release hold remain integration gates. Cache pins retain disk archives; +they do not freeze the running generation or prove release rollback behavior. + +An isolated pre-runtime-change probe of the real index, excluding only its +process-local lock, measured 21,714,441 serialized bytes and 318,029,824 bytes +peak RSS while retaining original, serialized and reconstructed state. Encoding +took 0.102 s and decoding 0.0772 s on this Mac. This excludes new runtime +resources, spawn/staging and host overlap; it is not the final RAM budget. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; @@ -71,7 +99,7 @@ and query set on the implemented snapshot runtime. | Area | Current evidence | | --- | --- | -| Bounded refresh/cache, crashes, shared volume, offline recovery | In progress; not yet accepted. | +| Bounded refresh/cache, crashes, shared volume, offline recovery | Task review accepted locally; Linux/runtime integration remains. | | Frozen generations, URL presentation, stdio/HTTP lifecycle | Pending. | | Runtime/static-site images, local request graph, Gateway sessions | Pending. | | Restricted host controller, rollback and independent index delivery | Pending. | diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 873474a..40c94fe 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -181,7 +181,7 @@ No network/CPU build runs on the ASGI event loop. Test/store internals may inject monotonic clock/transport at their actual dependency boundary, never test-only methods on production classes. -- [ ] **RED:** ThreadingHTTPServer fixtures count GETs, return delayed chunks, +- [x] **RED:** ThreadingHTTPServer fixtures count GETs, return delayed chunks, corrupt archives, foreign redirects and 304. Cold failure must produce explicit not-ready; warm store must keep its previous corpus ID. Example contract test: @@ -195,12 +195,12 @@ self.assertEqual(coordinator.current().corpus_id, "fixture-generation-a") Fixture generations in this test are small builder results independent of the format hash; real-format tests use actual corpus IDs. Run focused tests to RED. -- [ ] **GREEN URL/network:** Resolve manifest below selected base prefix; +- [x] **GREEN URL/network:** Resolve manifest below selected base prefix; permit fixed ai archive origin only for default public site; validate every redirect, no downgrade, no credentials and three redirects maximum. Stream reads enforce byte caps plus 60s whole attempt/20s read timeout. Validate Content-Encoding and length; reuse ETag/Last-Modified only with a valid cache. -- [ ] **GREEN cache/lifecycle:** Namespace by normalized source/schema; verify +- [x] **GREEN cache/lifecycle:** Namespace by normalized source/schema; verify cache before use; retain active+previous and pins. File lock serializes download/commit between processes, query never takes it. Atomic durable pointer update precedes in-process swap; crash, disk-full, corrupt current fall back to @@ -208,7 +208,7 @@ self.assertEqual(coordinator.current().corpus_id, "fixture-generation-a") at 256 MiB. One background updater, 3600s refresh ±20%, 30…3600s error backoff, zero disables periodic refresh after bootstrap. Close interrupts workers with a bounded deadline, without orphan processes/threads holding interpreter exit. -- [ ] **Verify:** Test same/different sources, prefix, zero interval, 304 without +- [x] **Verify:** Test same/different sources, prefix, zero interval, 304 without cache, delayed read, repeated faults, crash stages, two processes/shared volume, cache budget and no query blocking. Run Task 1+2 tests, record RED/GREEN and commit task files; request spec+quality review before integration. From 67a65b69c90e1a7f56507ebbf440542a335a43d1 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:05:16 +0300 Subject: [PATCH 15/88] docs: specify release generation hold integration --- .../2026-09-10-mcp-container-distribution-plan.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 40c94fe..8e6698a 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -73,7 +73,7 @@ corpus/runtime (1–3), container distribution (4), delivery (5–6). Их inter | 2 | `scripts/v8std_mcp_snapshots.py`, `tests/test_v8std_mcp_snapshots.py` | URL trust boundary, HTTP/cache transaction, background coordinator. | | 3 | `scripts/v8std_mcp_runtime.py`, `scripts/v8std_mcp_presentation.py`, `scripts/v8std_mcp_index.py`, `scripts/v8std_mcp_server.py`, runtime tests | Frozen generation construction, request facade, stdio/HTTP lifecycle. | | 4 | Dockerfiles/Compose/lock, local-profile script, tests, docs | Build and exercise the two images and local site. | -| 5 | `deploy/container/`, `scripts/v8std_mcp_release.py`, `tests/test_v8std_mcp_release.py` | Typed host transaction, nginx index store and recovery. | +| 5 | `deploy/container/`, `scripts/v8std_mcp_release.py`, release tests; snapshot/runtime integration and focused tests | Typed host transaction, nginx index store, fixed release generation and recovery. | | 6 | workflows, publication scripts, architecture policy/test references, docs/operations | Fail-closed CI delivery, process v2 synchronization, integration evidence. | ### Task 1: Deterministic corpus format and producer @@ -336,6 +336,8 @@ self.assertEqual(container_inspect["Config"]["User"], "10001:10001") **Files:** create `scripts/v8std_mcp_release.py`, `tests/test_v8std_mcp_release.py`, `deploy/container/release.schema.json`, controller/service/nginx configurations under `deploy/container/`, `spec/operations/mcp-container-activation.md`. +Extend snapshot/runtime modules and their focused tests only for the release +contract's generation hold, selected rollback corpus and refresh resumption. **Interfaces produced:** CLI `validate-envelope`, `deploy`, `recover`, `status`, `publish-index`; inputs are typed bounded JSON or fixed paths under configured @@ -359,6 +361,16 @@ self.assertTrue(predecessor_snapshot_path.is_file()) nginx test then atomic switch/reload, public smoke then commit/drain. Enforce 5min transaction, 90s readiness, 30s smoke/drain, 45s final stop; loss of caller does not kill host recovery. Stale retry cannot supersede current sequence. + Integrate a bounded trusted host-control path that holds the selected verified + generation during readiness/switch/rollback and resumes normal refresh after + commit. Cache pins retain bytes; they do not select or freeze a process's + generation. Likewise, zero refresh interval alone is not a release hold: the + current coordinator performs a warm-start network refresh. Exercise a changing + source manifest during the transaction, a postcommit worker failure, restart + from the selected predecessor and refresh resumption. Neither a retained + archive nor a disk pointer alone proves the endpoint serves the required + corpus ID. Preserve ordinary local stdio/HTTP behavior and the single public + SITE_URL setting; release control must not be exposed as an MCP tool. - [ ] **GREEN static store/operations:** Independent read-only nginx alias for `/indexes/v1/` with GET/HEAD, hash cache headers and bounded download admission; publisher stages/verifies/renames objects and tracks references/pins before GC. From f432b9c504e9be11e41a43d52ad19ad14997a477 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:19:55 +0300 Subject: [PATCH 16/88] feat: add frozen MCP generations and snapshot lifecycle --- scripts/generate_mcp_snapshot.py | 12 +- scripts/v8std_mcp_index.py | 39 ++- scripts/v8std_mcp_presentation.py | 261 +++++++++++++++++ scripts/v8std_mcp_runtime.py | 147 ++++++++++ scripts/v8std_mcp_server.py | 137 +++++++-- tests/mcp_runtime_benchmark.py | 207 +++++++++++++ tests/test_v8std_mcp_presentation.py | 125 ++++++++ tests/test_v8std_mcp_runtime.py | 420 +++++++++++++++++++++++++++ 8 files changed, 1326 insertions(+), 22 deletions(-) create mode 100644 scripts/v8std_mcp_presentation.py create mode 100644 scripts/v8std_mcp_runtime.py create mode 100644 tests/mcp_runtime_benchmark.py create mode 100644 tests/test_v8std_mcp_presentation.py create mode 100644 tests/test_v8std_mcp_runtime.py diff --git a/scripts/generate_mcp_snapshot.py b/scripts/generate_mcp_snapshot.py index 6bcc32c..7196536 100644 --- a/scripts/generate_mcp_snapshot.py +++ b/scripts/generate_mcp_snapshot.py @@ -11,6 +11,8 @@ from pathlib import Path import tempfile +from v8std_mcp_presentation import PresentationError, validate_links + from v8std_mcp_snapshot_format import ( DEFAULT_SITE_URL, JSONL_MEMBERS, MAX_ARCHIVE_BYTES, MAX_MANIFEST_BYTES, MAX_UNPACKED_BYTES, MEMBER_LIMITS, MEMBERS, PUBLIC_DELIVERY_URL, @@ -47,6 +49,14 @@ def build_snapshot(docs_dir: Path, source_sha: str, canonical_site_url: str) -> raise SnapshotError("member_size") page_count += 1 files["pages.jsonl"] = bytes(pages) + rows = list(jsonl_rows(files["pages.jsonl"])) + page_paths = {row["id"]: row for row in rows} + for row in rows: + validate_links(row.get("body_markdown", ""), canonical_site_url=site_url, + page_paths=page_paths, context=site_url + row["site_path"]) + for name in ("llms.txt", "llms-full.txt"): + validate_links(files[name].decode("utf-8"), canonical_site_url=site_url, + page_paths=page_paths, generated_fields=True) vector_count = sum(1 for _ in jsonl_rows(files["search-vectors.jsonl"])) counts = dict(zip(JSONL_MEMBERS, (page_count, vector_count))) descriptor = { @@ -169,7 +179,7 @@ def main() -> int: try: path = publish_snapshot(args.docs, args.output, args.source_sha, site_url, public_delivery=args.public_delivery) - except SnapshotError as error: + except (SnapshotError, PresentationError) as error: parser.exit(1, f"{error.code}\n") print(f"wrote {path}") return 0 diff --git a/scripts/v8std_mcp_index.py b/scripts/v8std_mcp_index.py index 5aa8c73..a11ae3d 100644 --- a/scripts/v8std_mcp_index.py +++ b/scripts/v8std_mcp_index.py @@ -406,6 +406,27 @@ def __init__( self._bm25_body = BM25Corpus({}) self._metadata_terms_by_id: dict[str, set[str]] = {} self._missing_rule_targets: list[dict[str, str]] = [] + self._frozen = False + + @classmethod + def from_validated_bytes(cls, pages: bytes, vectors: bytes, *, max_snippet_chars: int = MAX_SNIPPET_CHARS): + """Build from a VerifiedSnapshot's canonical bytes without corpus I/O.""" + index = cls(max_snippet_chars=max_snippet_chars) + payload = pages.decode("utf-8") + entries, metadata = index._parse_vectors(vectors.decode("utf-8"), "snapshot") + index._replace_index(index._parse_pages(payload), "snapshot", payload, entries, metadata) + index._frozen = True + return index + + def __getstate__(self): + # Trusted spawn IPC only; this is not a persistent pickle cache format. + state = self.__dict__.copy() + del state["_lock"] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._lock = threading.RLock() @property def max_snippet_chars(self) -> int: @@ -420,6 +441,8 @@ def vector_metadata(self) -> VectorMetadata | None: return self._vector_metadata def load(self, *, force_refresh: bool = False) -> None: + if self._frozen: + raise RuntimeError("index is frozen") with self._lock: payload, source = self._load_payload(force_refresh=force_refresh) pages = self._parse_pages(payload) @@ -427,7 +450,7 @@ def load(self, *, force_refresh: bool = False) -> None: self._replace_index(pages, source, payload, vectors, vector_metadata) def refresh_if_needed(self) -> None: - if self.pages_path is not None: + if self._frozen or self.pages_path is not None: return metadata = self._metadata if metadata is None or time.time() - metadata.loaded_at >= self.refresh_seconds: @@ -755,6 +778,8 @@ def resolve(self, id_or_alias_or_url: str) -> dict[str, Any] | None: return self._pages_by_key.get(key) def read_resource_text(self, resource_name: str) -> str: + if self._frozen: + raise RuntimeError("snapshot resources belong to the generation") self.refresh_if_needed() if resource_name == "pages.jsonl" and self.pages_path is not None: return self.pages_path.read_text(encoding="utf-8") @@ -779,7 +804,8 @@ def read_resource_text(self, resource_name: str) -> str: payload, _source = self._load_remote_resource(resource_name, remote_path) return payload - def _validate_types(self, types: list[str] | None) -> set[str] | None: + @staticmethod + def _validate_types(types: list[str] | None) -> set[str] | None: values = require_string_list(types, "types", MAX_ENUM_CHARS) if values is None: return None @@ -787,13 +813,15 @@ def _validate_types(self, types: list[str] | None) -> set[str] | None: raise ValueError("invalid page type") return set(values) - def _validate_mode(self, mode: str) -> str: + @staticmethod + def _validate_mode(mode: str) -> str: mode = require_text(mode, "mode", MAX_ENUM_CHARS) if mode not in VALID_MODES: raise ValueError("invalid search mode") return mode - def _validate_relations(self, relations: list[str] | None) -> set[str] | None: + @staticmethod + def _validate_relations(relations: list[str] | None) -> set[str] | None: values = require_string_list(relations, "relations", MAX_ENUM_CHARS) if values is None: return None @@ -1113,6 +1141,9 @@ def _load_vectors(self, *, force_refresh: bool) -> tuple[list[VectorEntry], Vect except IndexLoadError: return [], None + return self._parse_vectors(payload, source) + + def _parse_vectors(self, payload: str, source: str) -> tuple[list[VectorEntry], VectorMetadata | None]: vectors: list[VectorEntry] = [] model = "" dim = 0 diff --git a/scripts/v8std_mcp_presentation.py b/scripts/v8std_mcp_presentation.py new file mode 100644 index 0000000..0691b8d --- /dev/null +++ b/scripts/v8std_mcp_presentation.py @@ -0,0 +1,261 @@ +"""Source-preserving link-node presentation shared by runtime and publisher. + +No renderer or optional parser dependencies: a lexical scanner identifies code, +Markdown destinations and HTML attribute spans, and edits only those spans. +Canonical corpus bytes never pass through this module before retrieval. +""" +from __future__ import annotations + +import html +from html.parser import HTMLParser +import re +from urllib.parse import unquote, urljoin, urlsplit, urlunsplit + + +class PresentationError(ValueError): + """Bounded publisher error, with no source URL or document payload.""" + code = "unresolved_internal_link" + + def __init__(self): + super().__init__(self.code) + + +# Explicit published auxiliaries, not a prefix allowlist or searchable pages. +AUXILIARY_PATHS = frozenset({"llms.txt", "llms-full.txt", "ai/pages.jsonl", + "LICENSES/", "LICENSES/LGPL-3.0.txt", + "LICENSES/GPL-3.0.txt", "LICENSES/EPL-2.0.txt"}) +_ATTR = re.compile(r'''([^\s=<>/]+)(\s*=\s*)(?:"([^"]*)"|'([^']*)'|([^\s>]+))''') +_TAG = re.compile(r''']*?(?:"[^"]*"|'[^']*'|[^'"<>])*?>''') +_AUTOLINK = re.compile(r"\s]+>", re.I) +_FIELD = re.compile(r"(?:Markdown URL|URL|HTML):[ \t]+(https?://[^\s<>]+)") +_FENCE = re.compile(r" {0,3}(`{3,}|~{3,})[^\n]*\n?") +_REF = re.compile(r" {0,3}\[(?:\\.|[^\]\\\n])+\]:[ \t]*(?:\n[ \t]*)?") +_BACKTICKS = re.compile(r"`+") + + +class LinkCatalog: + def __init__(self, canonical_site_url, site_url, page_paths): + self.canonical = canonical_site_url + self.site = site_url + self.pages = page_paths + self.paths = set(AUXILIARY_PATHS) + for page in page_paths.values(): + self.paths.update((page["site_path"], page["markdown_path"])) + + def link(self, value, *, context=None, validate=False): + # Decode Markdown escapes/entities only for URL interpretation; preserve + # the original lexical spelling if this is not an internal link. + decoded = html.unescape(re.sub(r"\\([!\"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])", r"\1", value)) + if not decoded or decoded.startswith("#"): + return value + base = urlsplit(self.canonical) + target = urlsplit(urljoin(context or self.canonical, decoded)) + if (target.scheme, target.netloc) != (base.scheme, base.netloc): + return value + path = target.path + # Encoded separators/dot traversal must not turn a known path into an + # alternate route. Relative ../ links resolve normally inside the base. + unsafe = re.search(r"%(?:2f|5c|25|2e)", path, re.I) or "\\" in path + relative = unquote(path[len(base.path):]) if path.startswith(base.path) else None + if unsafe or relative not in self.paths: + if validate: + raise PresentationError() + return value + result = urlsplit(urljoin(self.site, relative)) + return urlunsplit((result.scheme, result.netloc, result.path, target.query, target.fragment)) + + def lookup(self, value): + """Translate configured local page URLs to the original lookup key.""" + target, base = urlsplit(value), urlsplit(self.site) + if (target.scheme, target.netloc) == (base.scheme, base.netloc) and target.path.startswith(base.path): + path = target.path[len(base.path):] + if path in self.paths - AUXILIARY_PATHS: + return urljoin(self.canonical, path) + return value + + +class _HTMLTag(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=False) + self.attributes = set() + + def handle_starttag(self, tag, attrs): + self.attributes = {name for name, _ in attrs if name in {"href", "src", "poster", "action", "cite"}} + + +def _destination(text, start): + """Return destination span, respecting escaped/balanced parentheses.""" + if start >= len(text): + return None + if text[start] == "<": + end = start + 1 + while end < len(text): + if text[end] == "\\": + end += 2 + continue + if text[end] == ">": + return start + 1, end + if text[end] == "\n": + return None + end += 1 + return None + end, depth = start, 0 + while end < len(text): + char = text[end] + if char == "\\": + end += 2 + continue + if char.isspace() or (char == ")" and depth == 0): + break + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + end += 1 + return (start, end) if end > start and depth == 0 else None + + +def _markdown(text, catalog, *, context=None, generated_fields=False, validate=False): + edits = [] + i, bracket_depth = 0, 0 + + def link(start, end, *, attribute=False): + value = text[start:end] + replacement = catalog.link(value, context=context, validate=validate) + if replacement != value: + edits.append((start, end, html.escape(replacement, quote=True) if attribute else replacement)) + + while i < len(text): + line_start = i == 0 or text[i - 1] == "\n" + if line_start: + if generated_fields and text.startswith("External sources:", i): + end = text.find("\n", i) + i = end + 1 if end >= 0 else len(text) + while text.startswith("- ", i): + end = text.find("\n", i) + i = end + 1 if end >= 0 else len(text) + continue + fence = _FENCE.match(text, i) + if fence: + marker = fence[1] + closing = re.compile(r"^ {0,3}" + re.escape(marker[0]) + "{" + str(len(marker)) + r",}[ \t]*$", re.M).search(text, fence.end()) + i = closing.end() if closing else len(text) + continue + if text.startswith((" ", "\t"), i): + end = text.find("\n", i) + i = end + 1 if end >= 0 else len(text) + continue + reference = _REF.match(text, i) + if reference: + span = _destination(text, reference.end()) + if span: + link(*span) + i = span[1] + (text[reference.end()] == "<") + continue + if text[i] == "\\": + i += 2 + continue + if text[i] == "`": + marker = _BACKTICKS.match(text, i)[0] + closing = re.compile(r"(?", i + 4) + i = end + 3 if end >= 0 else len(text) + continue + if text[i] == "<": + auto = _AUTOLINK.match(text, i) + if auto: + link(i + 1, auto.end() - 1) + i = auto.end() + continue + tag = _TAG.match(text, i) + if tag: + raw = tag[0] + protected = re.match(r"<(code|pre|script|style)(?:\s|>)", raw, re.I) + if protected: + closing = re.compile(r"", re.I).search(text, tag.end()) + i = closing.end() if closing else len(text) + continue + parser = _HTMLTag() + parser.feed(raw) + for attr in _ATTR.finditer(raw): + if attr[1].lower() in parser.attributes: + group = next(n for n in (3, 4, 5) if attr[n] is not None) + link(i + attr.start(group), i + attr.end(group), attribute=True) + i = tag.end() + continue + if generated_fields: + field = _FIELD.match(text, i) + line_prefix = text[text.rfind("\n", 0, i) + 1:i] if field else "" + if field and (line_start or (text.startswith("HTML:", i) and line_prefix.startswith("- ["))): + end = field.end(1) + # Generated list prose appends a sentence period to HTML URL. + if text[end - 1] == ".": + end -= 1 + if line_start and text.startswith("URL:", i): + context = text[field.start(1):end] + link(field.start(1), end) + i = field.end() + continue + if text[i] == "[": + bracket_depth += 1 + elif text[i] == "]" and bracket_depth: + bracket_depth -= 1 + if text.startswith("](", i): + start = i + 2 + while start < len(text) and text[start].isspace(): + start += 1 + span = _destination(text, start) + if span: + link(*span) + i = span[1] + (text[start] == "<") + continue + i += 1 + parts, previous = [], 0 + for start, end, replacement in edits: + parts.extend((text[previous:start], replacement)) + previous = end + parts.append(text[previous:]) + return "".join(parts) + + +def present_markdown(text, *, canonical_site_url, site_url, page_paths, + context=None, generated_fields=False): + return _markdown(text, LinkCatalog(canonical_site_url, site_url, page_paths), + context=context, generated_fields=generated_fields) + + +def validate_links(text, *, canonical_site_url, page_paths, context=None, generated_fields=False): + _markdown(text, LinkCatalog(canonical_site_url, canonical_site_url, page_paths), + context=context, generated_fields=generated_fields, validate=True) + + +def present_result(value, *, canonical_site_url, site_url, page_paths): + catalog = LinkCatalog(canonical_site_url, site_url, page_paths) + + def visit(item, context=None): + if isinstance(item, list): + return [visit(child, context) for child in item] + if not isinstance(item, dict): + return item + page = page_paths.get(item.get("id")) + if page: + context = urljoin(canonical_site_url, page["site_path"]) + result = {} + for key, child in item.items(): + if key in {"url", "markdown_url"} and isinstance(child, str): + path_key = "site_path" if key == "url" else "markdown_path" + if page: + base, suffix = urlsplit(urljoin(site_url, page[path_key])), urlsplit(child) + result[key] = urlunsplit((base.scheme, base.netloc, base.path, suffix.query, suffix.fragment)) + else: + result[key] = catalog.link(child, context=context) + elif key == "body_markdown" and isinstance(child, str): + result[key] = _markdown(child, catalog, context=context) + else: + result[key] = visit(child, context) + return result + + return visit(value) diff --git a/scripts/v8std_mcp_runtime.py b/scripts/v8std_mcp_runtime.py new file mode 100644 index 0000000..bd2770d --- /dev/null +++ b/scripts/v8std_mcp_runtime.py @@ -0,0 +1,147 @@ +"""One immutable index generation per complete data call.""" +from __future__ import annotations + +from dataclasses import dataclass +from functools import partial +import json +from pathlib import Path + +from v8std_mcp_index import ( + V8StdIndex, DEFAULT_CACHE_DIR, MAX_BODY_CHARS, MAX_QUERY_CHARS, MAX_ID_OR_ALIAS_CHARS, + MAX_SNIPPET_CHARS, MAX_ENUM_CHARS, MAX_DIAGNOSTIC_CODES, MAX_DIAGNOSTIC_CODE_CHARS, + clamp_body_limit, clamp_limit, require_text, trim_body, validate_max_snippet_chars, +) +from v8std_mcp_presentation import LinkCatalog, present_markdown, present_result +from v8std_mcp_snapshot_format import DEFAULT_SITE_URL, VerifiedSnapshot, normalize_site_url +from v8std_mcp_snapshots import LoaderError, SnapshotCoordinator, SnapshotStore + + +@dataclass(frozen=True) +class IndexGeneration: + corpus_id: str + index: V8StdIndex + resources: dict[str, str] + canonical_site_url: str + page_paths: dict + + +def build_generation(snapshot: VerifiedSnapshot, *, max_snippet_chars: int, + site_url: str | None = None) -> IndexGeneration: + index = V8StdIndex.from_validated_bytes(snapshot.files["pages.jsonl"], + snapshot.files["search-vectors.jsonl"], max_snippet_chars=max_snippet_chars) + paths = {page["id"]: {key: page[key] for key in ("site_path", "markdown_path")} + for page in index._pages} + canonical = snapshot.metadata["canonical_site_url"] + presentation = dict(canonical_site_url=canonical, site_url=site_url or canonical, page_paths=paths) + # Prepare bulk presentation in the supervised builder, before activation. + # Use source rows so the Resource does not acquire index-only aliases/graphs. + rows = [json.loads(line) for line in snapshot.files["pages.jsonl"].splitlines() if line.strip()] + resources = {"pages.jsonl": "".join(json.dumps(page, ensure_ascii=False) + "\n" + for page in present_result(rows, **presentation))} + for name in ("llms.txt", "llms-full.txt"): + resources[name] = present_markdown(snapshot.files[name].decode("utf-8"), + **presentation, generated_fields=True) + return IndexGeneration(snapshot.metadata["corpus_id"], index, resources, canonical, paths) + + +class SnapshotIndex: + def __init__(self, *, site_url: str = DEFAULT_SITE_URL, cache_dir: Path = DEFAULT_CACHE_DIR, + refresh_seconds: int = 3600, max_snippet_chars: int = MAX_SNIPPET_CHARS, + runtime_sha: str | None = None): + self.site_url = normalize_site_url(site_url) + self._max_snippet_chars = validate_max_snippet_chars(max_snippet_chars) + self.runtime_sha = runtime_sha if runtime_sha and len(runtime_sha) == 40 and all( + char in "0123456789abcdef" for char in runtime_sha) else None + self.coordinator = SnapshotCoordinator(SnapshotStore(self.site_url, cache_dir), + partial(build_generation, max_snippet_chars=max_snippet_chars, site_url=self.site_url), + refresh_seconds=refresh_seconds) + + @property + def max_snippet_chars(self): + return self._max_snippet_chars + + def start(self): + self.coordinator.start() + + def close(self): + self.coordinator.close() + + def status(self): + state = self.coordinator.status() + counts = {"row_count": 0, "semantic_enabled": False} + if state["ready"]: + generation = self.coordinator.current() + if generation.corpus_id == state["corpus_id"]: + counts = {"row_count": generation.index.metadata.row_count, + "semantic_enabled": generation.index.vector_metadata is not None} + # No source paths or unbounded missing-target list. + return {"ok": state["ready"], **counts, "runtime_sha": self.runtime_sha, **state} + + def _current(self): + try: + return self.coordinator.current() + except LoaderError as error: + if error.code == "INDEX_NOT_READY": + raise ValueError("INDEX_NOT_READY: retry later") from None + raise + + def _present(self, generation, result): + return present_result(result, canonical_site_url=generation.canonical_site_url, + site_url=self.site_url, page_paths=generation.page_paths) + + def _lookup(self, generation, value): + return LinkCatalog(generation.canonical_site_url, self.site_url, generation.page_paths).lookup(value) + + def search(self, query, *, types=None, mode="hybrid", limit=None): + require_text(query, "query", MAX_QUERY_CHARS) + clamp_limit(limit) + V8StdIndex._validate_types(types) + V8StdIndex._validate_mode(mode) + generation = self._current() + return self._present(generation, generation.index.search(query, types=types, mode=mode, limit=limit)) + + def page(self, id_or_alias_or_url, *, body_limit=MAX_BODY_CHARS): + require_text(id_or_alias_or_url, "id_or_alias_or_url", MAX_ID_OR_ALIAS_CHARS) + body_limit = clamp_body_limit(body_limit) + generation = self._current() + lookup = self._lookup(generation, id_or_alias_or_url) + page = generation.index.resolve(lookup) + if page is not None: + # Parse the full link before trimming. A longer local prefix cannot + # enlarge the inherited body budget or leave a cut public link. + presented = self._present(generation, page) + return {"found": True, "page": trim_body(presented, body_limit), "candidates": []} + return self._present(generation, generation.index.page(lookup, body_limit=body_limit)) + + def related(self, id_or_alias_or_url, *, relations=None, limit=None): + require_text(id_or_alias_or_url, "id_or_alias_or_url", MAX_ID_OR_ALIAS_CHARS) + clamp_limit(limit) + V8StdIndex._validate_relations(relations) + generation = self._current() + return self._present(generation, generation.index.related(self._lookup(generation, id_or_alias_or_url), + relations=relations, limit=limit)) + + def explain_snippet(self, snippet, *, language="auto", limit=None): + require_text(snippet, "snippet", self.max_snippet_chars) + require_text(language, "language", MAX_ENUM_CHARS) + if language not in {"auto", "bsl", "sdbl"}: + raise ValueError("language must be one of: auto, bsl, sdbl") + clamp_limit(limit) + generation = self._current() + return self._present(generation, generation.index.explain_snippet(snippet, language=language, limit=limit)) + + def explain_diagnostics(self, codes): + if not isinstance(codes, list): + raise ValueError("codes must be a list") + if len(codes) > MAX_DIAGNOSTIC_CODES: + raise ValueError(f"codes list is too long: max {MAX_DIAGNOSTIC_CODES}") + for code in codes: + require_text(code, "diagnostic code", MAX_DIAGNOSTIC_CODE_CHARS) + generation = self._current() + return self._present(generation, generation.index.explain_diagnostics(codes)) + + def read_resource_text(self, resource_name): + if resource_name not in {"pages.jsonl", "llms.txt", "llms-full.txt"}: + raise ValueError("unknown resource") + generation = self._current() + return generation.resources[resource_name] diff --git a/scripts/v8std_mcp_server.py b/scripts/v8std_mcp_server.py index 058aa53..ac7761e 100644 --- a/scripts/v8std_mcp_server.py +++ b/scripts/v8std_mcp_server.py @@ -7,13 +7,18 @@ import json import logging import os +import signal import sys +from contextlib import asynccontextmanager from contextvars import ContextVar from datetime import datetime, timezone from pathlib import Path from typing import Annotated, Any +import anyio + from mcp.server.fastmcp import FastMCP +from mcp.server.stdio import stdio_server from mcp.server.transport_security import TransportSecuritySettings from pydantic import Field from starlette.requests import Request @@ -30,6 +35,8 @@ V8StdIndex, validate_max_snippet_chars, ) +from v8std_mcp_runtime import SnapshotIndex +from v8std_mcp_snapshot_format import DEFAULT_SITE_URL, SnapshotError, normalize_site_url MCP_SELF_DOC_MESSAGE = "This is a MCP Streamable HTTP endpoint" @@ -490,17 +497,89 @@ async def _send_response( await send({"type": "http.response.body", "body": body}) -def install_self_documenting_mcp_app(server: FastMCP, *, mcp_path: str) -> None: +def install_self_documenting_mcp_app(server: FastMCP, *, mcp_path: str, index=None) -> None: original_streamable_http_app = server.streamable_http_app def streamable_http_app_with_self_documentation(): - return SelfDocumentingMcpApp(original_streamable_http_app(), mcp_path=mcp_path) + app = original_streamable_http_app() + if isinstance(index, SnapshotIndex): + original_lifespan = app.router.lifespan_context + + @asynccontextmanager + async def lifespan(app): + index.start() + try: + async with original_lifespan(app) as state: + yield state + finally: + with anyio.CancelScope(shield=True): + await anyio.to_thread.run_sync(index.close) + + app.router.lifespan_context = lifespan + return SelfDocumentingMcpApp(app, mcp_path=mcp_path) server.streamable_http_app = streamable_http_app_with_self_documentation # type: ignore[method-assign] +class _StdioLines: + """Cancellable POSIX pipe input for the SDK's stdio transport. + + AsyncFile.readline delegates a blocking pipe read to a shielded worker + thread. Waiting on readiness first lets SIGTERM cancel without requiring + the client to close its write end. No JSON/protocol handling lives here. + """ + def __init__(self, fd): + self.fd = fd + self.buffer = bytearray() + self.eof = False + + def __aiter__(self): + return self + + async def __anext__(self): + while True: + newline = self.buffer.find(b"\n") + if newline >= 0 or self.eof: + if not self.buffer: + raise StopAsyncIteration + end = newline + 1 if newline >= 0 else len(self.buffer) + line = bytes(self.buffer[:end]) + del self.buffer[:end] + return line.decode("utf-8", errors="replace") + await anyio.wait_readable(self.fd) + block = os.read(self.fd, 65536) + self.buffer.extend(block) + self.eof = not block + + +def install_stdio_lifecycle(server: FastMCP, index) -> None: + + async def run_stdio(): + if isinstance(index, SnapshotIndex): + index.start() + try: + async with anyio.create_task_group() as group: + async def terminate_on_signal(): + with anyio.open_signal_receiver(signal.SIGTERM) as signals: + async for _ in signals: + group.cancel_scope.cancel() + break + + group.start_soon(terminate_on_signal) + async with stdio_server(stdin=_StdioLines(sys.stdin.fileno())) as (read_stream, write_stream): + await server._mcp_server.run(read_stream, write_stream, + server._mcp_server.create_initialization_options()) + group.cancel_scope.cancel() + finally: + if isinstance(index, SnapshotIndex): + with anyio.CancelScope(shield=True): + await anyio.to_thread.run_sync(index.close) + + server.run_stdio_async = run_stdio # type: ignore[method-assign] + + def build_server( - index: V8StdIndex, + index: V8StdIndex | SnapshotIndex, *, host: str, port: int, @@ -675,13 +754,18 @@ async def healthz(_: Request) -> Response: status_code = 200 if status.get("ok") else 503 return JSONResponse(status, status_code=status_code) + @server.custom_route("/livez", methods=["GET"], include_in_schema=False) + async def livez(_: Request) -> Response: + return JSONResponse({"ok": True}) + @server.custom_route("/version", methods=["GET"], include_in_schema=False) async def version(_: Request) -> Response: return JSONResponse( {"service": "v8std-mcp", "api": "v2", "api_profiles": MCP_API_PROFILES, **index.status()} ) - install_self_documenting_mcp_app(server, mcp_path=mcp_path) + install_self_documenting_mcp_app(server, mcp_path=mcp_path, index=index) + install_stdio_lifecycle(server, index) return server @@ -690,8 +774,10 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run the v8std.ru read-only MCP server.") parser.add_argument("--pages", type=Path, help="Read pages JSONL from a local file.") parser.add_argument("--vectors", type=Path, help="Read search vectors JSONL from a local file.") - parser.add_argument("--index-url", default=DEFAULT_INDEX_URL, help="Remote pages JSONL URL.") - parser.add_argument("--vectors-url", default=DEFAULT_VECTORS_URL, help="Remote vectors JSONL URL.") + parser.add_argument("--index-url", default=None, help="Explicit legacy remote pages JSONL URL.") + parser.add_argument("--vectors-url", default=None, help="Explicit legacy remote vectors JSONL URL.") + parser.add_argument("--site-url", default=None, help="Snapshot source and presentation site URL.") + parser.add_argument("--transport", choices=["stdio", "streamable-http"], default="streamable-http") parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR) parser.add_argument("--refresh-seconds", type=int, default=DEFAULT_REFRESH_SECONDS) parser.add_argument("--host", default="127.0.0.1") @@ -732,6 +818,18 @@ def parse_args(argv: list[str]) -> argparse.Namespace: help="Allowed Origin header for MCP transport security. Can be repeated.", ) args = parser.parse_args(argv) + site = args.site_url if args.site_url is not None else os.environ.get("V8STD_MCP_SITE_URL") + legacy = any(value is not None for value in (args.pages, args.vectors, args.index_url, args.vectors_url)) + if legacy and site is not None: + parser.error("explicit legacy sources cannot be combined with SITE_URL") + if args.refresh_seconds < 0: + parser.error("refresh-seconds must be nonnegative") + try: + args.site_url = None if legacy else normalize_site_url(site if site is not None else DEFAULT_SITE_URL) + except SnapshotError: + parser.error("invalid SITE_URL") + args.index_url = args.index_url if args.index_url is not None else DEFAULT_INDEX_URL + args.vectors_url = args.vectors_url if args.vectors_url is not None else DEFAULT_VECTORS_URL raw_limit = args.max_snippet_chars if raw_limit is None: raw_limit = os.environ.get("V8STD_MCP_MAX_SNIPPET_CHARS", str(MAX_SNIPPET_CHARS)) @@ -751,16 +849,21 @@ def parse_args(argv: list[str]) -> argparse.Namespace: def main(argv: list[str] | None = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) configure_runtime_logging(args.log_level) - index = V8StdIndex( - pages_path=args.pages, - vectors_path=args.vectors, - index_url=args.index_url, - vectors_url=args.vectors_url, - cache_dir=args.cache_dir, - refresh_seconds=args.refresh_seconds, - max_snippet_chars=args.max_snippet_chars, - ) - index.load() + if args.site_url is not None: + index = SnapshotIndex(site_url=args.site_url, cache_dir=args.cache_dir, + refresh_seconds=args.refresh_seconds, max_snippet_chars=args.max_snippet_chars, + runtime_sha=os.environ.get("V8STD_MCP_RUNTIME_SHA")) + else: + index = V8StdIndex( + pages_path=args.pages, + vectors_path=args.vectors, + index_url=args.index_url, + vectors_url=args.vectors_url, + cache_dir=args.cache_dir, + refresh_seconds=args.refresh_seconds, + max_snippet_chars=args.max_snippet_chars, + ) + index.load() server = build_server( index, @@ -772,7 +875,7 @@ def main(argv: list[str] | None = None) -> int: log_level=args.log_level, usage_logger=McpToolUsageLogger(args.usage_log), ) - server.run(transport="streamable-http") + server.run(transport=args.transport) return 0 diff --git a/tests/mcp_runtime_benchmark.py b/tests/mcp_runtime_benchmark.py new file mode 100644 index 0000000..0456d83 --- /dev/null +++ b/tests/mcp_runtime_benchmark.py @@ -0,0 +1,207 @@ +"""Opt-in Task3 benchmark: real local corpus, supervised IPC, local HTTP only. + +Run: .venv/bin/python -m tests.mcp_runtime_benchmark +Instrumentation exists only here; temporary observations are removed on exit. +""" +from functools import partial +import gc +import json +import os +from pathlib import Path +import pickle +import resource +import statistics +import subprocess +import sys +import tempfile +import threading +import time +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from generate_mcp_snapshot import build_snapshot +from search_benchmark import collect_case_ids, read_case_payloads, run_ranked_case, run_diagnostics_case, percentile +from v8std_mcp_index import V8StdIndex +from v8std_mcp_runtime import SnapshotIndex, build_generation +from v8std_mcp_snapshots import SnapshotStore +from tests.test_v8std_mcp_runtime import Current +from tests.test_v8std_mcp_snapshots import Source +from tests import mcp_snapshot_fixtures as fixture + + +def record(path, phase, started, cpu, **fields): + with Path(path).open("a") as stream: + stream.write(json.dumps({"pid": os.getpid(), "phase": phase, + "seconds": time.perf_counter() - started, "cpu_seconds": time.process_time() - cpu, + "peak_rss_bytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * (1 if sys.platform == "darwin" else 1024), + **fields}) + "\n") + + +def observed_build(snapshot, *, profile, site_url): + start, cpu = time.perf_counter(), time.process_time() + generation = build_generation(snapshot, max_snippet_chars=4000, site_url=site_url) + record(profile, "build", start, cpu) + original = pickle.dumps + + def encode(*args, **kwargs): + start, cpu = time.perf_counter(), time.process_time() + value = original(*args, **kwargs) + record(profile, "encode", start, cpu, bytes=len(value)) + return value + + pickle.dumps = encode + return generation + + +class ObservedStore(SnapshotStore): + def __init__(self, site, cache, profile): + super().__init__(site, cache) + self.profile = profile + + def _generation(self, *args, **kwargs): + start, cpu = time.perf_counter(), time.process_time() + try: + return super()._generation(*args, **kwargs) + finally: + record(self.profile, "verify_cached", start, cpu) + + def _extract_verified(self, *args, **kwargs): + start, cpu = time.perf_counter(), time.process_time() + try: + return super()._extract_verified(*args, **kwargs) + finally: + record(self.profile, "verify_stage", start, cpu) + + +def tree_rss(): + rows = subprocess.check_output(["ps", "-axo", "pid=,ppid=,rss="], text=True) + rows = [tuple(map(int, row.split())) for row in rows.splitlines()] + included = {os.getpid()} + while True: + more = {pid for pid, parent, _ in rows if parent in included} + if more <= included: + break + included |= more + return sum(rss * 1024 for pid, _, rss in rows if pid in included) + + +def ranked(index): + cases, _ = read_case_payloads(ROOT / "tests/search_benchmark_cases.yml") + records, ranks, times = [], [], [] + for case in cases: + start = time.perf_counter() + if case.get("tool") == "diagnostics": + failures, ids = run_diagnostics_case(index, case) + assert not failures, failures + rank = None + elif "expected_absent" in case: + ids = collect_case_ids(index, case) + assert case["expected_absent"] not in ids + rank = None + else: + rank, ids = run_ranked_case(index, case) + assert rank is not None and rank <= int(case.get("required_top", 3)) + ranks.append(0 if rank is None else 1 / rank) + times.append((time.perf_counter() - start) * 1000) + records.append((rank, ids)) + return {"cases": len(cases), "mrr": statistics.mean(ranks), "p95_ms": percentile(times, 95)}, records + + +def main(): + legacy = V8StdIndex(pages_path=ROOT / "docs/ai/pages.jsonl", vectors_path=ROOT / "docs/ai/search-vectors.jsonl") + legacy.load() + before, ranks_before = ranked(legacy) + # Scores as well as rank order must survive the factory/presentation change. + scores_before = {query: legacy.search(query) for query in ("std437", "модальные окна", "параметры запроса")} + del legacy + gc.collect() + archive, manifest = build_snapshot(ROOT / "docs", "e07e1393c184a90968814af20fee0f7224c6d843", fixture.SITE_URL) + source = Source() + source.archive, source.manifest = archive, manifest + print(json.dumps({"before": before, "archive_bytes": len(archive), "corpus_id": manifest["corpus_id"]}), flush=True) + try: + with tempfile.TemporaryDirectory() as directory: + profile = Path(directory) / "profile.jsonl" + store = ObservedStore(source.url, Path(directory) / "cache", profile) + facade = SnapshotIndex(site_url=source.url, cache_dir=Path(directory) / "cache") + active = None + for phase in ("cold", "warm", "same_hash", "new_generation", "slow_source"): + if phase == "new_generation": + from v8std_mcp_snapshot_format import verify_archive + files = dict(verify_archive(archive, manifest).files) + del files["metadata.json"] + files["llms.txt"] += b"\nBenchmark next generation\n" + source.archive, source.manifest = fixture.snapshot_fixture(files=fixture.with_metadata(files)) + del files + if phase == "slow_source": + source.fault = "headers" + source.release.clear() + timer = threading.Timer(2, source.release.set) + timer.start() + samples, queries = [], [] + done = threading.Event() + + def sample(): + while not done.is_set(): + samples.append(tree_rss()) + done.wait(.04) + + def query(): + while not done.is_set(): + start = time.perf_counter() + result = facade.search("параметры запроса") + assert result["results"] + queries.append((time.perf_counter() - start) * 1000) + done.wait(.01) + + sampler = threading.Thread(target=sample) + reader = threading.Thread(target=query) if active else None + sampler.start() + if reader: + reader.start() + original = pickle.loads + def decode(*args, **kwargs): + start, cpu = time.perf_counter(), time.process_time() + value = original(*args, **kwargs) + record(profile, "parent_decode", start, cpu) + return value + start, cpu = time.perf_counter(), time.process_time() + try: + with patch("pickle.loads", side_effect=decode): + result, metadata = store._run("cached" if phase == "warm" else "refresh", + partial(observed_build, profile=profile, site_url=source.url)) + elapsed = time.perf_counter() - start + parent_cpu = time.process_time() - cpu + samples.append(tree_rss()) # Both old and reconstructed new still retained. + finally: + done.set() + sampler.join() + if reader: + reader.join() + observations = [json.loads(line) for line in profile.read_text().splitlines()] + profile.unlink() + print(json.dumps({"phase": phase, "seconds": elapsed, "parent_cpu_seconds": parent_cpu, + "tree_peak_rss_bytes": max(samples), "query_count": len(queries), + "query_p95_ms": percentile(queries, 95), "query_max_ms": max(queries, default=0), + "observations": observations}), flush=True) + active = result + del result + facade.coordinator = Current(active) + gc.collect() + after, ranks_after = ranked(facade) + assert ranks_after == ranks_before + for query, expected in scores_before.items(): + assert active.index.search(query) == expected + print(json.dumps({"after": after, "identical_ranks": True, "identical_score_samples": True}), flush=True) + for name in ("pages.jsonl", "llms.txt", "llms-full.txt"): + start = time.perf_counter() + body = facade.read_resource_text(name) + print(json.dumps({"resource": name, "seconds": time.perf_counter() - start, + "chars": len(body)}), flush=True) + finally: + source.close() + + +if __name__ == "__main__": + main() diff --git a/tests/test_v8std_mcp_presentation.py b/tests/test_v8std_mcp_presentation.py new file mode 100644 index 0000000..ae51bfc --- /dev/null +++ b/tests/test_v8std_mcp_presentation.py @@ -0,0 +1,125 @@ +"""Presentation changes link nodes, never retrieval input or code literals.""" +import importlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + +from tests import mcp_snapshot_fixtures as fixture + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +PUBLIC = "https://v8std.ru/" +LOCAL = "http://localhost:8080/kb/" +PATHS = {"std437": {"site_path": "std/437/", "markdown_path": "std/437.md"}, + "third": {"site_path": "THIRD_PARTY_DIAGNOSTIC_ARTICLES/", + "markdown_path": "THIRD_PARTY_DIAGNOSTIC_ARTICLES.md"}} + + +class PresentationTests(unittest.TestCase): + def module(self): + self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_presentation")) + return importlib.import_module("v8std_mcp_presentation") + + def present(self, value): + return self.module().present_result(value, canonical_site_url=PUBLIC, + site_url=LOCAL, page_paths=PATHS) + + def test_link_nodes_and_nested_urls_preserve_literals_and_provenance(self): + body = ('`https://v8std.ru/std/437/`\n' + '[link](https://v8std.ru/std/437/?x=1#anchor "title")\n' + '![image]()\n' + '[reference][ref]\n[ref]: https://v8std.ru/std/437/ "title"\n' + '\n' + 'a\n' + '\n' + '```bsl\nx = "https://v8std.ru/std/437/";\n' + '[code](https://v8std.ru/std/437/)\n```\n' + ' [indented](https://v8std.ru/std/437/)\n' + '[literal](https://v8std.ru/std/437/)\n' + '
literal
\n' + 'ordinary https://v8std.ru/std/437/\n') + original = {"page": {**fixture.page_fixture(), "body_markdown": body, + "source_urls": ["https://its.1c.ru/db/v8std/content/437/hdoc"]}, + "related": [{"url": PUBLIC + "std/437/"}]} + result = self.present(original) + self.assertEqual(original["page"]["body_markdown"], body) + self.assertEqual(result["page"]["url"], LOCAL + "std/437/") + self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/content/437/hdoc"]) + rendered = result["page"]["body_markdown"] + for literal in ('`https://v8std.ru/std/437/`', '[code](https://v8std.ru/std/437/)', + ' [indented](https://v8std.ru/std/437/)', + '[literal](https://v8std.ru/std/437/)', + '
literal
', + 'ordinary https://v8std.ru/std/437/'): + self.assertIn(literal, rendered) + self.assertIn('[link](' + LOCAL + 'std/437/?x=1#anchor "title")', rendered) + self.assertIn('![image](<' + LOCAL + 'std/437.md>)', rendered) + self.assertIn('[ref]: ' + LOCAL + 'std/437/ "title"', rendered) + self.assertIn('<' + LOCAL + 'std/437/>', rendered) + self.assertIn('href="' + LOCAL + 'std/437/?x=1&y=2#z"', rendered) + self.assertIn('src=' + LOCAL + 'std/437.md', rendered) + self.assertEqual(result["related"][0]["url"], LOCAL + "std/437/") + + def test_relative_license_catalog_and_unknown_internal_validation(self): + module = self.module() + page = {"id": "third", "url": PUBLIC + PATHS["third"]["site_path"], + "body_markdown": '[L](../LICENSES/LGPL-3.0.txt) [G](../LICENSES/GPL-3.0.txt) ' + '[E](../LICENSES/EPL-2.0.txt) [D](../LICENSES/)'} + rendered = self.present(page)["body_markdown"] + for path in ("LGPL-3.0.txt", "GPL-3.0.txt", "EPL-2.0.txt", ""): + self.assertIn(LOCAL + "LICENSES/" + path, rendered) + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + module.validate_links('[bad](https://v8std.ru/not-published/)', + canonical_site_url=PUBLIC, page_paths=PATHS) + for text in ('`[code](https://v8std.ru/not-published/)`', '[source](https://its.1c.ru/x)', + '[R](https://v8std.ru/llms.txt)', '[R](https://v8std.ru/llms-full.txt)', + '[R](https://v8std.ru/ai/pages.jsonl)'): + module.validate_links(text, canonical_site_url=PUBLIC, page_paths=PATHS) + + def test_generated_fields_and_page_context_only_in_resource_mode(self): + text = ('URL: https://v8std.ru/THIRD_PARTY_DIAGNOSTIC_ARTICLES/\n' + 'Markdown URL: https://v8std.ru/THIRD_PARTY_DIAGNOSTIC_ARTICLES.md\n' + '[license](../LICENSES/EPL-2.0.txt)\n' + '- [std](https://v8std.ru/std/437.md): HTML: https://v8std.ru/std/437/. Title.\n' + '`HTML: https://v8std.ru/std/437/`\n' + 'value = "URL: https://v8std.ru/std/437/ ";\n' + 'External sources: \n\n' + 'External sources: https://its.1c.ru/example\n') + rendered = self.module().present_markdown(text, canonical_site_url=PUBLIC, + site_url=LOCAL, page_paths=PATHS, generated_fields=True) + self.assertIn('URL: ' + LOCAL + 'THIRD_PARTY_DIAGNOSTIC_ARTICLES/', rendered) + self.assertIn('HTML: ' + LOCAL + 'std/437/. Title.', rendered) + self.assertIn('[license](' + LOCAL + 'LICENSES/EPL-2.0.txt)', rendered) + self.assertIn('`HTML: https://v8std.ru/std/437/`', rendered) + self.assertIn('value = "URL: https://v8std.ru/std/437/ ";', rendered) + self.assertIn('External sources: ', rendered) + self.assertIn('External sources: https://its.1c.ru/example', rendered) + + def test_publisher_rejects_unresolved_links_using_shared_catalog(self): + from generate_mcp_snapshot import build_snapshot + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root) + (root / "llms.txt").write_text('[bad](https://v8std.ru/not-published/)', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + build_snapshot(root, fixture.SOURCE_SHA, PUBLIC) + + def test_structured_url_suffixes_and_escaped_balanced_destinations(self): + result = self.present({"id": "std437", "url": PUBLIC + "std/437/?view=full#section"}) + self.assertEqual(result["url"], LOCAL + "std/437/?view=full#section") + body = '[a `label`][ref]\n[ref]:\n "title"\n' + self.assertIn('<' + LOCAL + 'std/437/>', self.present({"body_markdown": body})["body_markdown"]) + + def test_local_lookup_does_not_escape_prefix(self): + catalog = self.module().LinkCatalog(PUBLIC, LOCAL, PATHS) + self.assertEqual(catalog.lookup(LOCAL + "std/437/?view=full#x"), PUBLIC + "std/437/") + for value in ("http://localhost:8080/other/std/437/", LOCAL + "../std/437/", + LOCAL + "std%2f437/", "https://external.invalid/std/437/"): + self.assertEqual(catalog.lookup(value), value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v8std_mcp_runtime.py b/tests/test_v8std_mcp_runtime.py new file mode 100644 index 0000000..a0bc35e --- /dev/null +++ b/tests/test_v8std_mcp_runtime.py @@ -0,0 +1,420 @@ +"""Frozen generations and the official SDK lifecycle, with local sources only.""" +from functools import partial +import importlib +import importlib.util +import json +import contextlib +import io +import os +from pathlib import Path +import pickle +import queue +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from tests import mcp_snapshot_fixtures as fixture +from tests import test_v8std_mcp_snippet as snippet_compat + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +LOCAL = "http://localhost:8080/kb/" + + +def verified(): + from v8std_mcp_snapshot_format import verify_archive + return verify_archive(*fixture.snapshot_fixture()) + + +class Current: + """A deterministic pointer swap exactly when nested search starts.""" + def __init__(self, generation): + self.generation = generation + self.calls = 0 + + def current(self): + self.calls += 1 + return self.generation + + +class RuntimeTests(unittest.TestCase): + def module(self): + self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_runtime")) + return importlib.import_module("v8std_mcp_runtime") + + def test_frozen_factory_and_ipc_reconstruction_never_load_sources(self): + runtime = self.module() + from v8std_mcp_index import V8StdIndex + with patch.object(V8StdIndex, "_fetch_url", side_effect=AssertionError("network")): + generation = runtime.build_generation(verified(), max_snippet_chars=4000) + encoded = pickle.dumps(generation) + with patch.object(V8StdIndex, "_parse_pages", side_effect=AssertionError("reparse")), \ + patch.object(V8StdIndex, "_replace_index", side_effect=AssertionError("rebuild")): + decoded = pickle.loads(encoded) + self.assertEqual(decoded.index.search("std437"), generation.index.search("std437")) + self.assertEqual(decoded.index.page("std437")["page"]["id"], "std437") + with self.assertRaisesRegex(RuntimeError, "frozen"): + decoded.index.load() + with self.assertRaisesRegex(RuntimeError, "generation"): + decoded.index.read_resource_text("llms.txt") + self.assertEqual(set(decoded.resources), {"pages.jsonl", "llms.txt", "llms-full.txt"}) + + def test_generation_capture_pins_nested_calls_and_returns_independent_copy(self): + runtime = self.module() + from v8std_mcp_index import V8StdIndex + old = runtime.build_generation(verified(), max_snippet_chars=4000, site_url=LOCAL) + new = runtime.build_generation(verified(), max_snippet_chars=4000, site_url=LOCAL) + current = Current(old) + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=LOCAL, cache_dir=Path(directory)) + facade.coordinator = current + original = old.index.search + + def swap(*args, **kwargs): + current.generation = new + return original(*args, **kwargs) + + with patch.object(old.index, "search", side_effect=swap), \ + patch.object(new.index, "search", side_effect=AssertionError("mixed generation")), \ + patch.object(V8StdIndex, "_fetch_url", side_effect=AssertionError("network")): + result = facade.explain_snippet("std437") + self.assertEqual(current.calls, 1) + self.assertEqual(result["standards"][0]["url"], LOCAL + "std/437/") + page = facade.page(LOCAL + "std/437/") + self.assertTrue(page["found"]) + page["page"]["aliases"].append("mutation") + self.assertNotIn("mutation", facade.page("std437")["page"]["aliases"]) + self.assertIn('`', facade.read_resource_text("llms-full.txt")) + resource = json.loads(facade.read_resource_text("pages.jsonl")) + self.assertEqual(resource["url"], LOCAL + "std/437/") + self.assertEqual(resource["source_urls"], fixture.page_fixture()["source_urls"]) + + def test_validation_before_readiness_for_all_data_boundaries(self): + runtime = self.module() + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=LOCAL, cache_dir=Path(directory)) + for call in (lambda: facade.search("x" * 501), lambda: facade.search("x", mode="invalid"), + lambda: facade.page("x" * 1001), lambda: facade.related("std437", relations=["bad"]), + lambda: facade.explain_snippet("x" * 4001), + lambda: facade.explain_diagnostics([1])): + with self.assertRaises(ValueError) as caught: + call() + self.assertNotIn("INDEX_NOT_READY", str(caught.exception)) + for call in (lambda: facade.search("std437"), lambda: facade.page("std437"), + lambda: facade.related("std437"), lambda: facade.explain_snippet(""), + lambda: facade.explain_diagnostics([]), + lambda: facade.read_resource_text("llms.txt")): + with self.assertRaisesRegex(ValueError, "INDEX_NOT_READY"): + call() + + def test_full_link_is_rebased_before_existing_body_budget_is_applied(self): + runtime = self.module() + from v8std_mcp_snapshot_format import verify_archive + body = "x" * 965 + " [boundary](https://v8std.ru/std/437/?query=yes#anchor) end" + page = fixture.page_fixture() + page["body_markdown"] = body + vectors = fixture.vector_fixtures() + vectors[1]["text_sha256"] = fixture.sha256(body.encode()) + files = fixture.with_metadata(fixture.corpus_files(pages=[page], vectors=vectors)) + snapshot = verify_archive(*fixture.snapshot_fixture(files=files)) + generation = runtime.build_generation(snapshot, max_snippet_chars=4000) + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=LOCAL, cache_dir=Path(directory)) + facade.coordinator = Current(generation) + result = facade.page("std437", body_limit=1000)["page"] + expected = ("x" * 965 + " [boundary](http://localhost:8080/kb/std/437/?query=yes#anchor) end")[:1000] + "\n\n..." + self.assertEqual(result["body_markdown"], expected) + self.assertTrue(result["body_truncated"]) + self.assertEqual(len(result["body_markdown"]), 1005) # Existing five-character marker. + self.assertEqual(generation.index.resolve("std437")["body_markdown"], body) + + def test_official_http_discovery_before_ready_and_session_end_does_not_close_index(self): + runtime = self.module() + from v8std_mcp_server import build_server + from starlette.testclient import TestClient + from tests.test_v8std_mcp_snapshots import Source + source = Source() + try: + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=source.url, cache_dir=Path(directory), refresh_seconds=0) + server = build_server(facade, host="127.0.0.1", port=8765, mcp_path="/mcp", + allowed_hosts=["testserver"], allowed_origins=[]) + with TestClient(server.streamable_http_app()) as client: + self.assertEqual(client.get("/livez").status_code, 200) + for number in range(2): + response = client.post("/mcp", headers={"Accept": "application/json, text/event-stream"}, + json={"jsonrpc": "2.0", "id": number, "method": "tools/list", "params": {}}) + self.assertEqual(len(response.json()["result"]["tools"]), 5) + deadline = time.monotonic() + 8 + while not facade.status()["ok"] and time.monotonic() < deadline: + time.sleep(.02) + self.assertTrue(facade.status()["ok"], facade.status()) + self.assertEqual(facade.status().get("row_count"), 1) + self.assertTrue(facade.status().get("semantic_enabled")) + self.assertEqual(client.get("/healthz").status_code, 200) + response = client.post("/mcp", headers={"Accept": "application/json, text/event-stream"}, + json={"jsonrpc": "2.0", "id": 3, "method": "tools/call", + "params": {"name": "v8std_get_page", "arguments": {"id_or_alias_or_url": "std437"}}}) + self.assertFalse(response.json()["result"].get("isError", False)) + self.assertFalse(facade.coordinator._thread.is_alive()) + finally: + source.close() + + def test_resources_are_presented_during_build_not_in_callbacks(self): + runtime = self.module() + from v8std_mcp_snapshots import SnapshotStore + from tests.test_v8std_mcp_snapshots import Source + source = Source() + try: + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=source.url, cache_dir=Path(directory)) + generation = SnapshotStore(source.url, Path(directory)).refresh(prepare=facade.coordinator.build) + facade.coordinator = Current(generation) + with patch("v8std_mcp_runtime.present_markdown", side_effect=AssertionError("callback parsing")), \ + patch("v8std_mcp_runtime.present_result", side_effect=AssertionError("callback parsing")): + for name in ("pages.jsonl", "llms.txt", "llms-full.txt"): + result = facade.read_resource_text(name) + self.assertIn(source.url + "std/437/", result) + finally: + source.close() + + def test_verified_warm_generation_becomes_ready_with_source_offline(self): + runtime = self.module() + from v8std_mcp_snapshots import SnapshotStore + from tests.test_v8std_mcp_snapshots import Source + source = Source() + try: + with tempfile.TemporaryDirectory() as directory: + SnapshotStore(source.url, Path(directory)).refresh() + # A refused loopback connection cannot accidentally hit public data. + source.server.shutdown() + source.server.server_close() + facade = runtime.SnapshotIndex(site_url=source.url, cache_dir=Path(directory), refresh_seconds=0) + facade.start() + try: + deadline = time.monotonic() + 5 + while not facade.status()["ok"]: + self.assertLess(time.monotonic(), deadline) + time.sleep(.02) + self.assertEqual(facade.page("std437")["page"]["url"], source.url + "std/437/") + finally: + facade.close() + finally: + source.close() + + def test_facade_preserves_preview_budgets_and_captures_each_data_call_once(self): + runtime = self.module() + generation = runtime.build_generation(verified(), max_snippet_chars=32000, site_url=LOCAL) + with tempfile.TemporaryDirectory() as directory: + facade = runtime.SnapshotIndex(site_url=LOCAL, cache_dir=Path(directory), max_snippet_chars=32000) + current = Current(generation) + facade.coordinator = current + snippet = ('Адрес = "https://v8std.ru/std/437/";\n' + "слово " * 6000)[:32000] + canonical = generation.index.explain_snippet(snippet, limit=1) + result = facade.explain_snippet(snippet, limit=1) + self.assertEqual(result["normalized_text"], canonical["normalized_text"]) + self.assertEqual(result["tokens"], canonical["tokens"]) + self.assertLessEqual(len(result["normalized_text"]), 1000) + self.assertLessEqual(len(result["tokens"]), 80) + self.assertLessEqual(sum(map(len, result["tokens"])), 4000) + self.assertLessEqual(len(result["diagnostics"]) + len(result["standards"]), 1) + self.assertEqual(current.calls, 1) + for call in (lambda: facade.search("std437"), lambda: facade.page("std437"), + lambda: facade.related("std437"), lambda: facade.explain_diagnostics(["missing"]), + lambda: facade.read_resource_text("pages.jsonl")): + before = current.calls + call() + self.assertEqual(current.calls, before + 1) + + +class ConfigurationTests(unittest.TestCase): + def test_site_default_precedence_and_explicit_legacy_mode(self): + from v8std_mcp_server import parse_args + with patch.dict(os.environ, {}, clear=True): + args = parse_args([]) + self.assertEqual(getattr(args, "site_url", None), "https://v8std.ru/") + self.assertEqual(args.transport, "streamable-http") + self.assertEqual((args.host, args.port), ("127.0.0.1", 8765)) + self.assertIsNone(parse_args(["--pages", "/fixture/pages.jsonl"]).site_url) + self.assertIsNone(parse_args(["--index-url", "http://localhost/pages.jsonl"]).site_url) + with patch.dict(os.environ, {"V8STD_MCP_SITE_URL": LOCAL}): + self.assertEqual(parse_args([]).site_url, LOCAL) + self.assertEqual(parse_args(["--site-url", "https://example.org/kb"]).site_url, + "https://example.org/kb/") + + def test_ambiguous_or_empty_configuration_fails_without_loading(self): + from v8std_mcp_server import main + for argv in (["--site-url", ""], ["--site-url", LOCAL, "--index-url", "http://secret.invalid/x"], + ["--site-url", LOCAL, "--pages", "private-path"], ["--refresh-seconds", "-1"]): + with self.subTest(argv=argv), patch.dict(os.environ, {}, clear=True), \ + contextlib.redirect_stderr(io.StringIO()) as stderr, \ + patch("v8std_mcp_server.V8StdIndex.load", side_effect=AssertionError("source load")): + with self.assertRaises(SystemExit): + main(argv) + self.assertNotIn("private-path", stderr.getvalue()) + self.assertNotIn("secret.invalid", stderr.getvalue()) + + +def frozen_real_index(maximum=4000): + from v8std_mcp_index import V8StdIndex + return V8StdIndex.from_validated_bytes((ROOT / "docs/ai/pages.jsonl").read_bytes(), + (ROOT / "docs/ai/search-vectors.jsonl").read_bytes(), max_snippet_chars=maximum) + + +class FrozenSnippetCompatibility(snippet_compat.SnippetRetrievalTests): + @classmethod + def setUpClass(cls): + cls.index = frozen_real_index() + + +class FrozenLargeSnippetCompatibility(snippet_compat.LargeSnippetTests): + @classmethod + def setUpClass(cls): + cls.default = frozen_real_index() + cls.large = frozen_real_index(32000) + + +class StdioProcess: + def __init__(self, site, cache, *extra): + self.process = subprocess.Popen([sys.executable, str(ROOT / "scripts/v8std_mcp_server.py"), + "--transport", "stdio", "--site-url", site, "--cache-dir", str(cache), + "--refresh-seconds", "0", *extra], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, env={**os.environ, "NO_PROXY": "127.0.0.1,localhost"}) + self.lines = queue.Queue() + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + self.number = 0 + + def _read(self): + for line in self.process.stdout: + self.lines.put(line) + + def call(self, method, params=None): + self.number += 1 + self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "id": self.number, + "method": method, "params": params or {}}) + "\n") + self.process.stdin.flush() + result = json.loads(self.lines.get(timeout=5)) + if result.get("id") != self.number: + raise AssertionError(result) + return result + + def close(self): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(3) + raise + self.thread.join(1) + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + stream.close() + + +class WireTests(unittest.TestCase): + def test_stdio_initialize_not_ready_eof_and_sigterm_clean_workers(self): + from tests.test_v8std_mcp_snapshots import Source + source = Source() + source.fault = "headers" + try: + for shutdown in ("eof", "sigterm"): + with self.subTest(shutdown=shutdown), tempfile.TemporaryDirectory() as directory: + client = StdioProcess(source.url, Path(directory)) + try: + started = time.monotonic() + init = client.call("initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "task3-fixture", "version": "1"}}) + self.assertEqual(init["result"]["serverInfo"]["name"], "v8std") + self.assertLess(time.monotonic() - started, 3) + self.assertEqual(len(client.call("tools/list")["result"]["tools"]), 5) + self.assertEqual({r["uri"] for r in client.call("resources/list")["result"]["resources"]}, + {"v8std://llms.txt", "v8std://llms-full.txt", "v8std://ai/pages.jsonl"}) + result = client.call("tools/call", {"name": "v8std_search", "arguments": {"query": "std437"}}) + self.assertTrue(result["result"]["isError"]) + self.assertIn("INDEX_NOT_READY", str(result)) + self.assertIn("INDEX_NOT_READY", str(client.call("resources/read", {"uri": "v8std://llms.txt"}))) + self.assertTrue(source.requested.wait(3)) + child_rows = subprocess.check_output(["ps", "-axo", "pid=,ppid=,command="], text=True) + children = [int(row.split(None, 2)[0]) for row in child_rows.splitlines() + if row.split(None, 2)[1] == str(client.process.pid) + and "spawn_main" in row] + self.assertTrue(children) + started = time.monotonic() + if shutdown == "eof": + client.process.stdin.close() + else: + client.process.send_signal(signal.SIGTERM) + self.assertEqual(client.process.wait(4), 0) + self.assertLess(time.monotonic() - started, 3) + for pid in children: + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + self.assertTrue(client.lines.empty(), "stdout contains non-protocol output") + finally: + client.close() + finally: + source.close() + + def test_loopback_http_slow_bootstrap_then_two_agents_and_resources(self): + import httpx + from tests.test_v8std_mcp_snapshots import Source + source = Source() + source.fault = "headers" + try: + with tempfile.TemporaryDirectory() as directory, socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + reservation.close() + process = subprocess.Popen([sys.executable, str(ROOT / "scripts/v8std_mcp_server.py"), + "--site-url", source.url, "--cache-dir", directory, "--port", str(port), + "--refresh-seconds", "0"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=3, trust_env=False) as client: + deadline = time.monotonic() + 5 + while True: + try: + live = client.get("/livez") + break + except httpx.ConnectError: + if time.monotonic() > deadline: + self.fail("HTTP startup timeout") + time.sleep(.02) + self.assertEqual(live.status_code, 200) + self.assertEqual(client.get("/healthz").status_code, 503) + headers = {"Accept": "application/json, text/event-stream"} + def rpc(method, params): + return client.post("/mcp", headers=headers, json={"jsonrpc": "2.0", "id": 1, + "method": method, "params": params}).json() + for name in ("agent-one", "agent-two"): + self.assertIn("serverInfo", rpc("initialize", {"protocolVersion": "2025-03-26", + "capabilities": {}, "clientInfo": {"name": name, "version": "1"}})["result"]) + self.assertIn("INDEX_NOT_READY", str(rpc("tools/call", {"name": "v8std_search", + "arguments": {"query": "std437"}}))) + source.release.set() + deadline = time.monotonic() + 8 + while client.get("/healthz").status_code != 200: + self.assertLess(time.monotonic(), deadline) + time.sleep(.025) + for _ in range(2): + self.assertFalse(rpc("tools/call", {"name": "v8std_get_page", + "arguments": {"id_or_alias_or_url": "std437"}})["result"].get("isError", False)) + for uri in ("v8std://llms.txt", "v8std://llms-full.txt", "v8std://ai/pages.jsonl"): + self.assertIn("contents", rpc("resources/read", {"uri": uri})["result"]) + self.assertEqual(client.get("/version").json()["api"], "v2") + finally: + process.terminate() + process.communicate(timeout=5) + finally: + source.close() + + +if __name__ == "__main__": + unittest.main() From c34b07e35b1c56280dfda51070daa23b5ea2bade Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:25:34 +0300 Subject: [PATCH 17/88] docs: record frozen runtime and refresh measurements --- spec/operations/mcp-container-verification.md | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 4e77920..2c43ce3 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -73,6 +73,48 @@ peak RSS while retaining original, serialized and reconstructed state. Encoding took 0.102 s and decoding 0.0772 s on this Mac. This excludes new runtime resources, spawn/staging and host overlap; it is not the final RAM budget. +### Frozen runtime — implementation evidence, review pending + +Implementation: `b8656c1c238969d996e24f323333a925df9c5b87`. + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_presentation tests.test_v8std_mcp_runtime tests.test_v8std_mcp_index tests.test_v8std_mcp_snippet tests.test_v8std_mcp_server tests.test_v8std_mcp_combined tests.test_v8std_mcp_capacity tests.test_v8std_mcp_monitoring +.venv/bin/python -m tests.mcp_runtime_benchmark +``` + +Combined result: **189 tests passed in 63.620 seconds**. Actual subprocess tests +exercise clean stdio, discovery before readiness, EOF/SIGTERM worker cleanup, +and loopback HTTP with two agents and the existing Resources. The existing +Starlette/httpx deprecation warning remains recorded, not suppressed. + +The 256-case comparison kept all ranks/IDs identical and MRR 0.9939759036; +sampled score dictionaries were also identical. Desktop p95 was 46.570 ms before +and 49.018 ms after. These timings are not a stable production latency estimate. + +The first implementation's real-corpus benchmark included spawn/import, source +download and verification, streaming staging, Resource presentation, trusted +IPC and parent reconstruction. A 40 ms sampler included the fixture server, +old generation, worker, transfer/reconstruction buffers and allocator retention: + +| Operation | Wall time | Sampled process-tree peak RSS | Query p95 during operation | +| --- | --- | --- | --- | +| Cold download/build | 2.903 s | 487,636,992 bytes | Not measured | +| Warm build while old data serves | 3.084 s | 567,066,624 bytes | 56.05 ms | +| Unchanged refresh, before optimization | 3.537 s | 665,911,296 bytes | 65.21 ms | +| New generation while old data serves | 3.606 s | 610,746,368 bytes | 71.05 ms | +| Source headers delayed two seconds | 4.919 s | 647,069,696 bytes | 64.21 ms | + +Observed parent reconstruction was 93–179 ms; transferred prepared state was +32,663,778 bytes. Peaks are samples, not upper bounds or cgroup limits. Warm +build here deliberately retained a serving generation, not an empty-process +offline startup. Runtime overlap on the target host has not been measured. + +This exposed unnecessary reconstruction of an unchanged generation, including +duplicate archive verification. A scoped loader correction and refreshed +measurements are pending; these baseline numbers must not be reported as the +final optimized release. Resource presentation was moved into background build; +wire encoding of bulk responses still has a cost. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; From 3c54de9612557b94710ecd04400101ff187e4b17 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:36:07 +0300 Subject: [PATCH 18/88] perf: skip rebuilding verified unchanged MCP snapshots --- scripts/v8std_mcp_snapshots.py | 61 +++++--- tests/mcp_runtime_benchmark.py | 35 +++-- tests/test_v8std_mcp_snapshots.py | 229 +++++++++++++++++++++++++++++- 3 files changed, 296 insertions(+), 29 deletions(-) diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py index 0be3de6..33621ba 100644 --- a/scripts/v8std_mcp_snapshots.py +++ b/scripts/v8std_mcp_snapshots.py @@ -230,15 +230,21 @@ def _file_lock(path, deadline): os.close(fd) # OS releases flock even on process termination/crash. -def _prepare(snapshot, prepare): +def _prepare(snapshot, prepare, *, current_archive=None): try: - result = snapshot if prepare is None else prepare(snapshot) + metadata = {"corpus_id": snapshot.metadata["corpus_id"], + "source_sha": snapshot.metadata["source_sha"], + "archive_sha256": snapshot.archive_sha256} + if current_archive == snapshot.archive_sha256: + # Only a verified reusable entry may take this private path. The + # caller's identity belongs to its already accepted ready generation. + result = None + metadata["unchanged"] = True + else: + result = snapshot if prepare is None else prepare(snapshot) # Serialization is also preparation: an unpickleable lock must not # advance the disk pointer. These bytes go ONLY to our private IPC socket. - return pickle.dumps(("ok", result, { - "corpus_id": snapshot.metadata["corpus_id"], - "archive_sha256": snapshot.archive_sha256, - }), protocol=pickle.HIGHEST_PROTOCOL) + return pickle.dumps(("ok", result, metadata), protocol=pickle.HIGHEST_PROTOCOL) except Exception: raise LoaderError("prepare_failed") from None @@ -440,7 +446,15 @@ def _remove_owned(path): path.unlink() # Unknown/symlink paths are not owned cleanup targets. - def _refresh(self, prepare, deadline): + def _reuse_checked_entry(self, entry, prepare, deadline, current_archive): + result = _prepare(entry[0], prepare, current_archive=current_archive) + if current_archive == entry[0].archive_sha256: + # A rollback recovery can leave the on-disk current pointer damaged. + # Metadata-only success must also leave a durable consistent pointer. + self._commit_state(entry[2], entry[2], deadline) + return result + + def _refresh(self, prepare, deadline, *, current_archive=None): _directory(self.cache_dir, create=True) _directory(self.namespace, create=True) _directory(self.namespace / "generations", create=True) @@ -448,7 +462,7 @@ def _refresh(self, prepare, deadline): with _file_lock(self.cache_dir / ".volume.lock", deadline): entry = self._cached_entry() if waited and entry: - return _prepare(entry[0], prepare) + return self._reuse_checked_entry(entry, prepare, deadline, current_archive) self._gc((entry[0].archive_sha256,) if entry else ()) headers = {} if entry: @@ -465,7 +479,7 @@ def _refresh(self, prepare, deadline): if status == 304: if entry is None: raise LoaderError("http_status") - return _prepare(entry[0], prepare) + return self._reuse_checked_entry(entry, prepare, deadline, current_archive) manifest = validate_manifest(raw) archive_url, boundary = _archive_url(manifest, final_url, self.site_url) digest = manifest["archive"]["sha256"] @@ -477,7 +491,14 @@ def _refresh(self, prepare, deadline): "validators": validators, } try: - reusable, _ = self._generation(digest, manifest) + if entry and manifest == entry[1]: + # The complete validated manifest equals the one just + # verified against archive AND expanded cache bytes. + # Any changed field takes the existing strict path below; + # no format/metadata consistency rules are duplicated here. + reusable = entry[0] + else: + reusable, _ = self._generation(digest, manifest) except (OSError, LoaderError): reusable = None except SnapshotError: @@ -487,7 +508,8 @@ def _refresh(self, prepare, deadline): raise reusable = None if reusable is not None: - result = _prepare(reusable, prepare) + result = _prepare(reusable, prepare, current_archive=( + current_archive if entry and entry[0].archive_sha256 == digest else None)) self._commit_state(state, old, deadline) return result self._space(manifest["archive"]["bytes"] + manifest["archive"]["unpacked_bytes"] @@ -528,12 +550,12 @@ def _refresh(self, prepare, deadline): if stage.exists(): self._remove_owned(stage) - def _run(self, mode, prepare, stop=None): + def _run(self, mode, prepare, stop=None, *, current_archive=None): deadline = time.monotonic() + self._attempt_seconds stop = stop if stop is not None else threading.Event() parent, child = socket.socketpair() process = multiprocessing.get_context("spawn").Process( - target=_worker, args=(self, mode, prepare, deadline, child), + target=_worker, args=(self, mode, prepare, deadline, child, current_archive), name="v8std-snapshot-worker", daemon=True) started = False try: @@ -589,13 +611,13 @@ def _cache_walk_error(): raise LoaderError("cache_io") -def _worker(store, mode, prepare, deadline, channel): +def _worker(store, mode, prepare, deadline, channel, current_archive): try: if mode == "cached": snapshot = store.cached() payload = _prepare(snapshot, prepare) if snapshot else pickle.dumps(("ok", None, None)) else: - payload = store._refresh(prepare, deadline) + payload = store._refresh(prepare, deadline, current_archive=current_archive) _remaining(deadline) except SnapshotError as error: payload = pickle.dumps(("format_error", error.code, None)) @@ -653,6 +675,10 @@ def _accept(self, result, metadata, *, checked): now = time.time() retired = None with self._lock: + if metadata.get("unchanged") and (not self._state["ready"] + or metadata["archive_sha256"] != self._archive_sha256 + or metadata["corpus_id"] != self._state["corpus_id"]): + raise LoaderError("worker_failed") if metadata["archive_sha256"] != self._archive_sha256: retired = self._current self._current = result @@ -681,7 +707,10 @@ def _loop(self): failures = 0 while not self._stop.is_set(): try: - result, metadata = self.store._run("refresh", self.build, self._stop) + with self._lock: + current_archive = self._archive_sha256 if self._state["ready"] else None + result, metadata = self.store._run("refresh", self.build, self._stop, + current_archive=current_archive) if self._stop.is_set(): return self._accept(result, metadata, checked=True) diff --git a/tests/mcp_runtime_benchmark.py b/tests/mcp_runtime_benchmark.py index 0456d83..5696677 100644 --- a/tests/mcp_runtime_benchmark.py +++ b/tests/mcp_runtime_benchmark.py @@ -24,7 +24,7 @@ from search_benchmark import collect_case_ids, read_case_payloads, run_ranked_case, run_diagnostics_case, percentile from v8std_mcp_index import V8StdIndex from v8std_mcp_runtime import SnapshotIndex, build_generation -from v8std_mcp_snapshots import SnapshotStore +from v8std_mcp_snapshots import SnapshotCoordinator, SnapshotStore from tests.test_v8std_mcp_runtime import Current from tests.test_v8std_mcp_snapshots import Source from tests import mcp_snapshot_fixtures as fixture @@ -125,6 +125,8 @@ def main(): profile = Path(directory) / "profile.jsonl" store = ObservedStore(source.url, Path(directory) / "cache", profile) facade = SnapshotIndex(site_url=source.url, cache_dir=Path(directory) / "cache") + coordinator = SnapshotCoordinator(store, + partial(observed_build, profile=profile, site_url=source.url)) active = None for phase in ("cold", "warm", "same_hash", "new_generation", "slow_source"): if phase == "new_generation": @@ -164,16 +166,17 @@ def query(): def decode(*args, **kwargs): start, cpu = time.perf_counter(), time.process_time() value = original(*args, **kwargs) - record(profile, "parent_decode", start, cpu) + record(profile, "parent_decode", start, cpu, bytes=len(args[0])) return value start, cpu = time.perf_counter(), time.process_time() try: with patch("pickle.loads", side_effect=decode): result, metadata = store._run("cached" if phase == "warm" else "refresh", - partial(observed_build, profile=profile, site_url=source.url)) + coordinator.build, current_archive=( + coordinator._archive_sha256 if active is not None else None)) elapsed = time.perf_counter() - start parent_cpu = time.process_time() - cpu - samples.append(tree_rss()) # Both old and reconstructed new still retained. + samples.append(tree_rss()) # Before acceptance: includes any reconstructed candidate. finally: done.set() sampler.join() @@ -184,16 +187,26 @@ def decode(*args, **kwargs): print(json.dumps({"phase": phase, "seconds": elapsed, "parent_cpu_seconds": parent_cpu, "tree_peak_rss_bytes": max(samples), "query_count": len(queries), "query_p95_ms": percentile(queries, 95), "query_max_ms": max(queries, default=0), - "observations": observations}), flush=True) - active = result + "metadata": metadata, "observations": observations}), flush=True) + previous = active + coordinator._accept(result, metadata, checked=phase != "warm") + active = coordinator.current() + if metadata.get("unchanged"): + assert result is None and active is previous del result + del previous facade.coordinator = Current(active) gc.collect() - after, ranks_after = ranked(facade) - assert ranks_after == ranks_before - for query, expected in scores_before.items(): - assert active.index.search(query) == expected - print(json.dumps({"after": after, "identical_ranks": True, "identical_score_samples": True}), flush=True) + if phase == "cold": + # Compare the exact initial corpus before the refresh fixture + # changes llms/metadata (even though retrieval rows stay equal). + assert active.corpus_id == manifest["corpus_id"] + after, ranks_after = ranked(facade) + assert ranks_after == ranks_before + for query, expected in scores_before.items(): + assert active.index.search(query) == expected + print(json.dumps({"after": after, "corpus_id": active.corpus_id, + "identical_ranks": True, "identical_score_samples": True}), flush=True) for name in ("pages.jsonl", "llms.txt", "llms-full.txt"): start = time.perf_counter() body = facade.read_resource_text(name) diff --git a/tests/test_v8std_mcp_snapshots.py b/tests/test_v8std_mcp_snapshots.py index 8e19afd..f8e1573 100644 --- a/tests/test_v8std_mcp_snapshots.py +++ b/tests/test_v8std_mcp_snapshots.py @@ -28,6 +28,8 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) +from v8std_mcp_snapshots import SnapshotStore + @dataclass(frozen=True) class Generation: @@ -41,6 +43,23 @@ def build(snapshot): multiprocessing.get_start_method()) +@dataclass +class RecordingBuild: + log: Path + + def __call__(self, snapshot): + with self.log.open("a") as stream: + stream.write(snapshot.archive_sha256 + "\n") + return build(snapshot) + + +class RecordingStore(SnapshotStore): + def _generation(self, *args, **kwargs): + with self.verifications.open("a") as stream: + stream.write(args[0] + "\n") + return super()._generation(*args, **kwargs) + + def fail_build(snapshot): raise RuntimeError("https://secret:password@example.invalid/private-corpus") @@ -178,6 +197,7 @@ def __init__(self): self.requests = [] self.fault = None self.headers = {} + self.etag = '"fixture-v1"' self.redirect = None self.redirects = {} self.requested = threading.Event() @@ -213,7 +233,7 @@ def do_GET(self): payload = bytes([payload[0] ^ 1]) + payload[1:] self.send_response(200) self.send_header("Content-Type", "application/json" if is_manifest else "application/gzip") - self.send_header("ETag", '"fixture-v1"') + self.send_header("ETag", source.etag) self.send_header("Last-Modified", "Thu, 10 Sep 2026 00:00:00 GMT") if "Content-Length" not in source.headers: self.send_header("Content-Length", str(len(payload))) @@ -929,6 +949,211 @@ def test_backoff_extremes_stay_within_contract(self): self.assertEqual(coordinator._delay(10000), 3600) +class IdentityRefreshTests(SnapshotTestCase): + coordinator = SnapshotCoordinatorTests.coordinator + wait_until = SnapshotCoordinatorTests.wait_until + + def test_ready_200_and_304_build_only_at_bootstrap_and_verify_once_per_attempt(self): + self.store.refresh() + for fault in (None, "conditional"): + with self.subTest(fault=fault): + self.source.fault = fault + log = Path(self.temp.name) / f"build-{fault}" + store = RecordingStore(self.source.url, self.cache) + store.verifications = Path(self.temp.name) / f"verify-{fault}" + coordinator = self.loader.SnapshotCoordinator(store, RecordingBuild(log)) + self.addCleanup(coordinator.close) + schedule = RecordingStop(2) + coordinator._stop = schedule + coordinator.start() + self.wait_until(schedule.is_set) + self.assertEqual(log.read_text().splitlines(), [self.source.manifest["archive"]["sha256"]]) + self.assertEqual(len(store.verifications.read_text().splitlines()), 3) + self.assertEqual(coordinator.current().start_method, "spawn") + status = coordinator.status() + self.assertLess(status["loaded_at"], status["last_success_at"]) + self.assertIsNone(status["refresh_error_code"]) + self.assertEqual(sum(p.endswith(".tar.gz") for p, _ in self.source.requests), 1) + + def test_unchanged_ipc_is_metadata_only_after_validator_commit(self): + current = self.store.refresh() + self.source.etag = '"fixture-revalidated"' + result, metadata = self.store._run("refresh", fail_build, + current_archive=current.archive_sha256) + self.assertIsNone(result) + self.assertEqual(metadata, {"corpus_id": current.metadata["corpus_id"], + "source_sha": fixture.SOURCE_SHA, "archive_sha256": current.archive_sha256, + "unchanged": True}) + state = json.loads((self.store.namespace / "state.json").read_bytes()) + self.assertEqual(state["active"], metadata["archive_sha256"]) + self.assertEqual(state["validators"]["ETag"], self.source.etag) + self.source.fault = "conditional" + self.assertIsNone(self.store._run("refresh", fail_build, + current_archive=current.archive_sha256)[0]) + self.assertEqual(self.source.requests[-1][1]["If-None-Match"], self.source.etag) + + def test_cold_warm_and_public_refresh_without_ready_identity_always_build(self): + log = Path(self.temp.name) / "builds" + prepare = RecordingBuild(log) + cold, metadata = self.store._run("refresh", prepare) + self.source.fault = "missing" + warm, _ = self.store._run("cached", prepare, current_archive=metadata["archive_sha256"]) + self.source.fault = "conditional" + refreshed = self.store.refresh(prepare=prepare) + self.assertEqual([cold.corpus_id, warm.corpus_id, refreshed.corpus_id], + [self.source.manifest["corpus_id"]] * 3) + self.assertEqual(len(log.read_text().splitlines()), 3) + + def test_changed_archive_builds_even_when_corpus_id_matches(self): + current = self.store.refresh() + # Gzip OS byte is outside the corpus descriptor; the format accepts it. + archive = bytearray(self.source.archive) + archive[9] ^= 1 + self.source.archive = bytes(archive) + self.source.manifest = fixture.manifest_for(self.source.archive, current.files) + result, metadata = self.store._run("refresh", build, current_archive=current.archive_sha256) + self.assertIsInstance(result, Generation) + self.assertEqual(result.corpus_id, current.metadata["corpus_id"]) + self.assertNotEqual(metadata["archive_sha256"], current.archive_sha256) + self.assertFalse(metadata.get("unchanged", False)) + self.assertEqual(self.store.cached().archive_sha256, metadata["archive_sha256"]) + + def test_changed_source_metadata_builds_and_invalid_same_archive_claims_fail(self): + current = self.store.refresh() + pointer = self.store.namespace / "state.json" + original = pointer.read_bytes() + for field, value in (("source_sha", "2" * 40), ("corpus_id", "0" * 64), + ("vector_dim", 128), ("schema_version", 2)): + with self.subTest(field=field): + old = self.source.manifest[field] + self.source.manifest[field] = value + with self.assertRaises(self.loader.SnapshotError): + self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertEqual(pointer.read_bytes(), original) + self.source.manifest[field] = old + for field in ("bytes", "unpacked_bytes"): + with self.subTest(field=field): + self.source.manifest["archive"][field] += 1 + with self.assertRaises(self.loader.SnapshotError): + self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertEqual(pointer.read_bytes(), original) + self.source.manifest["archive"][field] -= 1 + files = fixture.with_metadata(fixture.corpus_files(), mutate=lambda m: m.update(source_sha="2" * 40)) + self.source.archive, self.source.manifest = fixture.snapshot_fixture(files=files) + result, metadata = self.store._run("refresh", build, current_archive=current.archive_sha256) + self.assertIsInstance(result, Generation) + self.assertEqual(metadata["source_sha"], "2" * 40) + self.assertNotEqual(result.corpus_id, current.metadata["corpus_id"]) + + def test_corrupt_cache_never_shortcuts_prepare_on_redownload(self): + for name in ("snapshot.tar.gz", "pages.jsonl", "manifest.json"): + with self.subTest(name=name): + current = self.store.refresh() + path = self.store.namespace / "generations" / current.archive_sha256 / name + path.write_bytes(b"corrupt") + self.source.fault = "conditional" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertEqual(caught.exception.code, "prepare_failed") + self.assertNotIn("If-None-Match", self.source.requests[-2][1]) + self.assertEqual(path.read_bytes(), b"corrupt") + self.source.fault = None + + def test_304_without_cache_does_not_trust_parent_identity(self): + self.source.fault = "304" + with self.assertRaises(self.loader.LoaderError) as caught: + self.store._run("refresh", fail_build, + current_archive=self.source.manifest["archive"]["sha256"]) + self.assertEqual(caught.exception.code, "http_status") + self.assertEqual(len(self.source.requests), 2) + + def test_recovered_304_repairs_pointer_before_unchanged_success(self): + current = self.store.refresh() + self.store.refresh() # Durable rollback of the same verified generation. + pointer = self.store.namespace / "state.json" + pointer.write_bytes(b"invalid") + self.source.fault = "conditional" + result, metadata = self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertIsNone(result) + self.assertEqual(json.loads(pointer.read_bytes())["active"], metadata["archive_sha256"]) + + def test_unchanged_commit_failure_preserves_pointer_for_200_and_304(self): + current = self.store.refresh() + pointer = self.store.namespace / "state.json" + before = pointer.read_bytes() + for fault in (None, "conditional"): + with self.subTest(fault=fault): + self.source.fault = fault + with patch("v8std_mcp_snapshots.os.fsync", side_effect=OSError(errno.ENOSPC, "full")): + with self.assertRaises(OSError): + self.store._refresh(fail_build, time.monotonic() + 5, + current_archive=current.archive_sha256) + self.assertEqual(pointer.read_bytes(), before) + + def test_lock_waiter_reuses_verified_current_without_network_or_build(self): + current = self.store.refresh() + self.source.requests.clear() + fd = os.open(self.store.namespace / ".lock", os.O_RDWR) + fcntl.flock(fd, fcntl.LOCK_EX) + release = threading.Timer(.5, lambda: fcntl.flock(fd, fcntl.LOCK_UN)) + release.start() + try: + result, metadata = self.store._run("refresh", fail_build, + current_archive=current.archive_sha256) + self.assertIsNone(result) + self.assertTrue(metadata["unchanged"]) + self.assertEqual(self.source.requests, []) + finally: + release.join() + os.close(fd) + + def test_other_process_committed_archive_still_builds_for_older_parent(self): + current = self.store.refresh() + self.source.next_generation() + committed = self.store.refresh() + result, metadata = self.store._run("refresh", build, current_archive=current.archive_sha256) + self.assertIsInstance(result, Generation) + self.assertEqual(metadata["archive_sha256"], committed.archive_sha256) + self.assertFalse(metadata.get("unchanged", False)) + + def test_changed_manifest_optional_fields_are_verified_without_rebuilding_current(self): + current = self.store.refresh() + self.source.manifest["optional"] = {"weight": .5} + result, metadata = self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertIsNone(result) + self.assertTrue(metadata["unchanged"]) + # Valid format path is nevertheless forbidden for this selected local site. + self.source.manifest["archive"]["path"] = ( + "https://ai.v8std.ru/indexes/v1/" + self.source.manifest["archive"]["path"]) + with self.assertRaises(self.loader.LoaderError) as caught: + self.store._run("refresh", fail_build, current_archive=current.archive_sha256) + self.assertEqual(caught.exception.code, "url_policy") + + def test_unready_coordinator_rejects_metadata_only_acceptance(self): + coordinator = self.coordinator(refresh_seconds=0) + with self.assertRaises(self.loader.LoaderError): + coordinator._accept(None, {"corpus_id": "untrusted", "archive_sha256": "a" * 64, + "unchanged": True}, checked=True) + self.assertFalse(coordinator.status()["ready"]) + + def test_unchanged_marker_must_match_accepted_archive_and_corpus(self): + coordinator = self.coordinator(refresh_seconds=0) + generation = Generation("original", os.getpid(), "accepted") + metadata = {"corpus_id": "original", "archive_sha256": "a" * 64} + coordinator._accept(generation, metadata, checked=False) + before = coordinator.status() + for mismatch in ({"archive_sha256": "b" * 64}, {"corpus_id": "different"}): + with self.subTest(mismatch=mismatch): + with self.assertRaises(self.loader.LoaderError): + coordinator._accept(None, {**metadata, **mismatch, "unchanged": True}, checked=True) + self.assertIs(coordinator.current(), generation) + self.assertEqual(coordinator.status(), before) + coordinator._accept(None, {**metadata, "unchanged": True}, checked=True) + self.assertIs(coordinator.current(), generation) + self.assertEqual(coordinator.status()["loaded_at"], before["loaded_at"]) + self.assertIsNotNone(coordinator.status()["last_success_at"]) + + class SnapshotLifetimeTests(unittest.TestCase): def test_same_hash_refresh_releases_unused_generation_before_idle_wait(self): loader = importlib.import_module("v8std_mcp_snapshots") @@ -937,7 +1162,7 @@ def test_same_hash_refresh_releases_unused_generation_before_idle_wait(self): class CompletedStore: # The spawn/IPC boundary is covered by the real store tests. Here # weakrefs isolate ownership after a completed result is delivered. - def _run(self, mode, prepare, stop): + def _run(self, mode, prepare, stop, *, current_archive=None): generation = Generation("same-corpus", os.getpid(), "completed-ipc") references.append(weakref.ref(generation)) return generation, {"corpus_id": "same-corpus", "archive_sha256": "a" * 64} From 23c4f607e6fe998e253b94898ebcaa3055744be7 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:38:03 +0300 Subject: [PATCH 19/88] docs: clarify parser dependency boundary --- spec/plans/2026-09-10-mcp-container-distribution-plan.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 8e6698a..91265e3 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -222,6 +222,11 @@ Integrate publisher-side link validation in `generate_mcp_snapshot.py` and its focused tests, reusing the presentation parser/catalog rather than a second link grammar. Treat the three Resources and published license paths as explicit auxiliary catalog entries, not corpus pages or arbitrary allowed paths. +If reliable source-preserving parsing needs a small parser dependency, include +its pinned runtime/build requirement and focused dependency-boundary tests in +this task. The producer must not import a docs builder; standalone Python `-S` +was an implementation check, not an approved prohibition on parser dependencies. +Keep the pure snapshot format reader independent of docs tooling. **Interfaces produced:** From 81cdb396810c39125db22ba2944733b37afae21d Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:38:52 +0300 Subject: [PATCH 20/88] docs: record unchanged refresh improvement --- spec/operations/mcp-container-verification.md | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 2c43ce3..c646057 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -110,11 +110,44 @@ build here deliberately retained a serving generation, not an empty-process offline startup. Runtime overlap on the target host has not been measured. This exposed unnecessary reconstruction of an unchanged generation, including -duplicate archive verification. A scoped loader correction and refreshed -measurements are pending; these baseline numbers must not be reported as the -final optimized release. Resource presentation was moved into background build; +duplicate archive verification. These baseline numbers must not be reported as +the optimized release. Resource presentation was moved into background build; wire encoding of bulk responses still has a cost. +### Unchanged-refresh correction — implementation evidence, review pending + +Implementation: `68902446bbb69df8e50eadb7be74b5c5d07521c4`. The ready coordinator +supplies its accepted archive identity. The worker still validates same-source +cached bytes and manifest consistency, but an unchanged result carries only +verified metadata. Cold/warm bootstrap, a different or repaired corrupt archive +retain full construction. Pointer/validator durability precedes success. + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_runtime.RuntimeTests tests.test_v8std_mcp_runtime.WireTests +.venv/bin/python -m tests.mcp_runtime_benchmark +``` + +Result: **72 tests passed in 51.553 seconds**. This does not close the separate +Task3 review findings about stdout backpressure and Markdown syntax handling. + +Fresh paired unchanged-refresh measurements on the same desktop/corpus: + +| Observation | Before correction | After correction | +| --- | --- | --- | +| Whole attempt | 3.704876 s | 0.742084 s | +| Received IPC payload | 32,663,778 bytes | 256 bytes | +| Parent decode | 0.157151 s | 0.002127 s | +| Sampled tree peak RSS | 627,490,816 bytes | 403,554,304 bytes | +| Query p95 during attempt | 76.368 ms (62 samples) | 48.613 ms (14 samples) | + +Generation construction/encoding are skipped; strict cache verification runs +once rather than twice. Query sample counts differ because the attempt is +shorter, so no statistical significance claim is made. All 256 same-corpus +ranks/IDs and sampled score dictionaries remained equal, MRR 0.9939759036. +New generations still require full preparation; measured new-generation wall +time was 3.566641 s with sampled tree RSS 555,876,352 bytes. These are sampled +desktop observations, not memory ceilings, target-host acceptance or 100k evidence. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; From e908c5e458bdaa6894d932bdaee8dfcb344acc72 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 16:40:08 +0300 Subject: [PATCH 21/88] docs: record refresh optimization review --- spec/operations/mcp-container-verification.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index c646057..85d183c 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -114,9 +114,10 @@ duplicate archive verification. These baseline numbers must not be reported as the optimized release. Resource presentation was moved into background build; wire encoding of bulk responses still has a cost. -### Unchanged-refresh correction — implementation evidence, review pending +### Unchanged-refresh correction — reviewed local evidence -Implementation: `68902446bbb69df8e50eadb7be74b5c5d07521c4`. The ready coordinator +Implementation: `68902446bbb69df8e50eadb7be74b5c5d07521c4`; independent scoped +spec/quality review approved without findings. The ready coordinator supplies its accepted archive identity. The worker still validates same-source cached bytes and manifest consistency, but an unchanged result carries only verified metadata. Cold/warm bootstrap, a different or repaired corrupt archive From fafd3405381abc35962b05aacb0d3a95f97d5c52 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 17:04:44 +0300 Subject: [PATCH 22/88] fix: preserve parsed MCP links and bound stdio shutdown --- requirements-mcp.txt | 1 + requirements.txt | 1 + scripts/v8std_mcp_presentation.py | 494 +++++++++++++++++------- scripts/v8std_mcp_server.py | 59 ++- tests/test_v8std_mcp_presentation.py | 189 ++++++++- tests/test_v8std_mcp_runtime.py | 115 ++++++ tests/test_v8std_mcp_snapshot_format.py | 6 +- 7 files changed, 708 insertions(+), 157 deletions(-) diff --git a/requirements-mcp.txt b/requirements-mcp.txt index ce0e5f7..f97f404 100644 --- a/requirements-mcp.txt +++ b/requirements-mcp.txt @@ -1,2 +1,3 @@ mcp==1.27.0 +markdown-it-py==4.0.0 PyYAML diff --git a/requirements.txt b/requirements.txt index 8603312..309b4d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Pillow +markdown-it-py==4.0.0 PyYAML pygments-bsl diff --git a/scripts/v8std_mcp_presentation.py b/scripts/v8std_mcp_presentation.py index 0691b8d..5ffe77f 100644 --- a/scripts/v8std_mcp_presentation.py +++ b/scripts/v8std_mcp_presentation.py @@ -1,16 +1,23 @@ -"""Source-preserving link-node presentation shared by runtime and publisher. +"""Source-preserving CommonMark link presentation shared by runtime/publisher. -No renderer or optional parser dependencies: a lexical scanner identifies code, -Markdown destinations and HTML attribute spans, and edits only those spans. -Canonical corpus bytes never pass through this module before retrieval. +The pinned parser decides syntax; instrumentation retains source offsets through +normalization and container indentation. Only accepted destination spans change. +Neither canonical retrieval input nor non-link prose is rendered/reserialized. """ from __future__ import annotations - +from array import array +from bisect import bisect_right +from difflib import SequenceMatcher import html from html.parser import HTMLParser import re +from types import SimpleNamespace from urllib.parse import unquote, urljoin, urlsplit, urlunsplit +from markdown_it import MarkdownIt +from markdown_it.rules_block import StateBlock, reference +from markdown_it.rules_inline import link, image, autolink, html_inline, text as inline_text + class PresentationError(ValueError): """Bounded publisher error, with no source URL or document payload.""" @@ -24,13 +31,11 @@ def __init__(self): AUXILIARY_PATHS = frozenset({"llms.txt", "llms-full.txt", "ai/pages.jsonl", "LICENSES/", "LICENSES/LGPL-3.0.txt", "LICENSES/GPL-3.0.txt", "LICENSES/EPL-2.0.txt"}) + _ATTR = re.compile(r'''([^\s=<>/]+)(\s*=\s*)(?:"([^"]*)"|'([^']*)'|([^\s>]+))''') -_TAG = re.compile(r''']*?(?:"[^"]*"|'[^']*'|[^'"<>])*?>''') -_AUTOLINK = re.compile(r"\s]+>", re.I) _FIELD = re.compile(r"(?:Markdown URL|URL|HTML):[ \t]+(https?://[^\s<>]+)") -_FENCE = re.compile(r" {0,3}(`{3,}|~{3,})[^\n]*\n?") -_REF = re.compile(r" {0,3}\[(?:\\.|[^\]\\\n])+\]:[ \t]*(?:\n[ \t]*)?") -_BACKTICKS = re.compile(r"`+") +_PROVENANCE = re.compile(r"^External sources:[^\n]*(?:\n- [^\n]*)*", re.M) +_PROTECTED_TAGS = {"code", "pre", "script", "style"} class LinkCatalog: @@ -43,13 +48,12 @@ def __init__(self, canonical_site_url, site_url, page_paths): self.paths.update((page["site_path"], page["markdown_path"])) def link(self, value, *, context=None, validate=False): - # Decode Markdown escapes/entities only for URL interpretation; preserve - # the original lexical spelling if this is not an internal link. - decoded = html.unescape(re.sub(r"\\([!\"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])", r"\1", value)) - if not decoded or decoded.startswith("#"): + # Callers supply the value interpreted in its own Markdown/HTML/JSON + # context. Applying a second universal unescape corrupts URL suffixes. + if not value or value.startswith("#"): return value base = urlsplit(self.canonical) - target = urlsplit(urljoin(context or self.canonical, decoded)) + target = urlsplit(urljoin(context or self.canonical, value)) if (target.scheme, target.netloc) != (base.scheme, base.netloc): return value path = target.path @@ -74,149 +78,351 @@ def lookup(self, value): return value -class _HTMLTag(HTMLParser): - def __init__(self): +class _MappedText(str): + """Original offsets survive parser slicing and container indentation. + + Contiguous slices use ranges; arrays join lines across removed prefixes. + Virtual indentation has offset -1. This is source mapping, not a grammar. + """ + def __new__(cls, value, offsets=None): + obj = super().__new__(cls, value) + obj.offsets = range(len(value)) if offsets is None else offsets + return obj + + def __getitem__(self, key): + value = super().__getitem__(key) + return _MappedText(value, self.offsets[key]) if isinstance(key, slice) else value + + def __add__(self, other): + return self.combine((self, other)) + + def __radd__(self, other): + return self.combine((other, self)) + + @classmethod + def combine(cls, parts): + parts = list(parts) + if len(parts) == 1: + return parts[0] + offsets = array("i") + for part in parts: + offsets.extend(part.offsets if isinstance(part, cls) else [-1] * len(part)) + return cls("".join(parts), offsets) + + def strip(self, chars=None): + start = len(self) - len(str.lstrip(self, chars)) + end = len(str.rstrip(self, chars)) + return self[start:max(start, end)] + + def span(self, start=0, end=None): + end = len(self) if end is None else end + if end <= start: + return None + offsets = self.offsets[start:end] + # Never delete a removed container prefix with a destination edit. + if offsets[0] < 0 or offsets[-1] - offsets[0] != len(offsets) - 1: + return None + return offsets[0], offsets[-1] + 1 + + +def _normalize(state): + source = state.src + parts, previous = [], 0 + for match in re.finditer(r"\r\n?|\x00", source): + parts.append(_MappedText(source[previous:match.start()], range(previous, match.start()))) + parts.append(_MappedText("\ufffd" if match[0] == "\x00" else "\n", [match.start()])) + previous = match.end() + parts.append(_MappedText(source[previous:], range(previous, len(source)))) + state.src = _MappedText.combine(parts) + + +def _block(state): + block = StateBlock(state.src, state.md, state.env, state.tokens) + original = block.getLines + + def get_lines(begin, end, indent, keep_last): + parts = [] + for line in range(begin, end): + keep = line + 1 < end or keep_last + rendered = original(line, line + 1, indent, keep) + raw = block.src[block.bMarks[line]:block.eMarks[line] + int(keep)] + if raw.endswith(rendered): + parts.append(raw[len(raw) - len(rendered):]) + continue + common = 0 + while common < min(len(raw), len(rendered)) and raw[-common - 1] == rendered[-common - 1]: + common += 1 + parts.append(_MappedText.combine(( + rendered[:len(rendered) - common], + raw[len(raw) - common:] if common else _MappedText(""), + ))) + return _MappedText.combine(parts) + + block.getLines = get_lines + try: + state.md.block.tokenize(block, block.line, block.lineMax) + finally: + # The adapter closes over StateBlock; detach it even on parser failure + # so source maps and token trees do not wait for cyclic GC. + del block.getLines + + +class _HTMLNodes(HTMLParser): + """HTML decides attributes; edits retain their original quoting context.""" + def __init__(self, source, nodes, tags): super().__init__(convert_charrefs=False) - self.attributes = set() + self.source, self.nodes, self.tags = source, nodes, tags + self.lines = [0] + [match.end() for match in re.finditer("\n", source)] + + def source_offset(self): + line, column = self.getpos() + return self.lines[line - 1] + column def handle_starttag(self, tag, attrs): - self.attributes = {name for name, _ in attrs if name in {"href", "src", "poster", "action", "cite"}} - - -def _destination(text, start): - """Return destination span, respecting escaped/balanced parentheses.""" - if start >= len(text): - return None - if text[start] == "<": - end = start + 1 - while end < len(text): - if text[end] == "\\": - end += 2 + start = self.source_offset() + raw = self.get_starttag_text() + # Only the tag's start anchors protection. A multiline opening tag + # may cross removed quote/list prefixes without making its code live. + span = self.source.span(start, start + 1) + if span: + self.tags.append((*span, tag, False)) + allowed = {key for key, _ in attrs if key in {"href", "src", "poster", "action", "cite"}} + for match in _ATTR.finditer(raw): + if match[1].lower() not in allowed: continue - if text[end] == ">": - return start + 1, end - if text[end] == "\n": - return None - end += 1 - return None - end, depth = start, 0 - while end < len(text): - char = text[end] - if char == "\\": - end += 2 - continue - if char.isspace() or (char == ")" and depth == 0): - break - if char == "(": - depth += 1 - elif char == ")": - depth -= 1 - end += 1 - return (start, end) if end > start and depth == 0 else None + group = next(n for n in (3, 4, 5) if match[n] is not None) + raw_value = self.source[start + match.start(group):start + match.end(group)] + span = raw_value.span() + value = html.unescape(match[group]) + if not span and raw_value and raw_value.offsets[0] >= 0 and raw_value.offsets[-1] >= 0: + # Multiline quoted destinations can cross container prefixes. + # Retain their source map for edit projection, not a broad cut. + span = raw_value.offsets[0], raw_value.offsets[-1] + 1 + value = _HTMLValue(value, raw_value) + if span: + self.nodes.append((*span, value, "html_unquoted" if group == 5 else "html")) + def handle_endtag(self, tag): + start = self.source_offset() + span = self.source.span(start, start + 1) + if span: + self.tags.append((*span, tag, True)) -def _markdown(text, catalog, *, context=None, generated_fields=False, validate=False): - edits = [] - i, bracket_depth = 0, 0 - def link(start, end, *, attribute=False): - value = text[start:end] - replacement = catalog.link(value, context=context, validate=validate) - if replacement != value: - edits.append((start, end, html.escape(replacement, quote=True) if attribute else replacement)) - - while i < len(text): - line_start = i == 0 or text[i - 1] == "\n" - if line_start: - if generated_fields and text.startswith("External sources:", i): - end = text.find("\n", i) - i = end + 1 if end >= 0 else len(text) - while text.startswith("- ", i): - end = text.find("\n", i) - i = end + 1 if end >= 0 else len(text) - continue - fence = _FENCE.match(text, i) - if fence: - marker = fence[1] - closing = re.compile(r"^ {0,3}" + re.escape(marker[0]) + "{" + str(len(marker)) + r",}[ \t]*$", re.M).search(text, fence.end()) - i = closing.end() if closing else len(text) +class _HTMLValue(str): + def __new__(cls, value, raw): + obj = super().__new__(cls, value) + obj.raw = raw + return obj + + def edits(self, replacement): + """Project destination-only edits around structural source gaps. + + URL parsing ignores literal CR/LF. Keep those physical line breaks and + removed container markers so rebasing never joins Markdown containers. + Entity escaping still follows the attribute's actual quoting context. + """ + for operation, start, end, new_start, new_end in SequenceMatcher( + None, str(self.raw), replacement).get_opcodes(): + if operation == "equal": continue - if text.startswith((" ", "\t"), i): - end = text.find("\n", i) - i = end + 1 if end >= 0 else len(text) + value = replacement[new_start:new_end] + if start == end: + offset = self.raw.offsets[start] if start < len(self.raw) else self.raw.offsets[-1] + 1 + if offset >= 0: + yield offset, offset, value continue - reference = _REF.match(text, i) - if reference: - span = _destination(text, reference.end()) - if span: - link(*span) - i = span[1] + (text[reference.end()] == "<") + spans = [] + for position in range(start, end): + offset = self.raw.offsets[position] + if offset < 0 or self.raw[position] == "\n": continue - if text[i] == "\\": - i += 2 - continue - if text[i] == "`": - marker = _BACKTICKS.match(text, i)[0] - closing = re.compile(r"(?", i + 4) - i = end + 3 if end >= 0 else len(text) + if spans and spans[-1][1] == offset: + spans[-1] = (spans[-1][0], offset + 1) + else: + spans.append((offset, offset + 1)) + for number, (begin, finish) in enumerate(spans): + yield begin, finish, value if number == 0 else "" + + +class _FirstHTMLTag(HTMLParser): + """Recognize HTML attributes also accepted by HTML outside CommonMark's + stricter inline-tag grammar (notably unquoted query '=' characters). + """ + def handle_starttag(self, tag, attrs): + if self.getpos() == (1, 0): + self.first = self.get_starttag_text() + raise _TagFinished + + def handle_data(self, data): + raise _TagFinished + + +class _TagFinished(Exception): + pass + + +def _merged_spans(spans): + merged = [] + for start, end in sorted(spans): + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(end, merged[-1][1])) + else: + merged.append((start, end)) + return merged + + +def _covered(offset, spans): + position = bisect_right(spans, (offset, float("inf"))) - 1 + return position >= 0 and offset < spans[position][1] + + +def _html_inline(state, silent): + if html_inline(state, silent): + return True + if not re.match(r"<[A-Za-z]", state.src[state.pos:state.pos + 2]): + return False + parser = _FirstHTMLTag() + parser.first = None + try: + parser.feed(str(state.src[state.pos:])) + except _TagFinished: + pass + if parser.first is None: + return False + end = state.pos + len(parser.first) + if not silent: + token = state.push("html_inline", "", 0) + token.content = state.src[state.pos:end] + state.pos = end + return True + + +def _link_nodes(source, generated_fields): + """Instrument successful parser rules, never error-recovery guesses.""" + md = MarkdownIt("commonmark") + nodes, tags, text_spans, frames = [], [], [], [] + md.core.ruler.at("normalize", _normalize) + md.core.ruler.at("block", _block) + helpers = md.helpers + md.helpers = SimpleNamespace(parseLinkLabel=helpers.parseLinkLabel, + parseLinkTitle=helpers.parseLinkTitle) + + def destination(src, pos, maximum): + result = helpers.parseLinkDestination(src, pos, maximum) + if result.ok and frames and isinstance(src, _MappedText): + angle = src[pos:pos + 1] == "<" + span = src.span(pos + int(angle), result.pos - int(angle)) + if span: + frames[-1].append((*span, result.str, "angle" if angle else "markdown")) + return result + + md.helpers.parseLinkDestination = destination + + def ref_rule(state, begin, end, silent): + frames.append([]) + accepted = reference(state, begin, end, silent) + candidates = frames.pop() + if accepted and not silent: + nodes.extend(candidates) + return accepted + + md.block.ruler.at("reference", ref_rule) + + def wrap(rule, kind): + def run(state, silent): + start, count = state.pos, len(state.tokens) + frames.append([]) + accepted = rule(state, silent) + candidates = frames.pop() + if not accepted or silent: + return accepted + src = state.src + emitted = state.tokens[count:] + if kind in {"link", "image"}: + urls = {token.attrGet("href") or token.attrGet("src") for token in emitted + if token.type in {"link_open", "image"}} + end_offset = src.offsets[state.pos - 1] + 1 + nodes.extend(candidate for candidate in candidates + if candidate[1] <= end_offset and state.md.normalizeLink(candidate[2]) in urls) + elif kind == "autolink": + span = src.span(start + 1, state.pos - 1) + if span: + nodes.append((*span, str(src[start + 1:state.pos - 1]), "autolink")) + elif kind == "html_inline": + _HTMLNodes(src[start:state.pos], nodes, tags).feed(str(src[start:state.pos])) + elif generated_fields: + span = src.span(start, state.pos) + if span: + text_spans.append(span) + return accepted + return run + + for name, rule in (("link", link), ("image", image), ("autolink", autolink), + ("html_inline", _html_inline), ("text", inline_text)): + md.inline.ruler.at(name, wrap(rule, name)) + tokens = md.parse(source) + for token in tokens: + if token.type == "html_block": + _HTMLNodes(token.content, nodes, tags).feed(str(token.content)) + protected, opened = [], [] + for start, end, tag, closing in sorted(tags): + if tag not in _PROTECTED_TAGS: continue - if text[i] == "<": - auto = _AUTOLINK.match(text, i) - if auto: - link(i + 1, auto.end() - 1) - i = auto.end() - continue - tag = _TAG.match(text, i) - if tag: - raw = tag[0] - protected = re.match(r"<(code|pre|script|style)(?:\s|>)", raw, re.I) - if protected: - closing = re.compile(r"", re.I).search(text, tag.end()) - i = closing.end() if closing else len(text) - continue - parser = _HTMLTag() - parser.feed(raw) - for attr in _ATTR.finditer(raw): - if attr[1].lower() in parser.attributes: - group = next(n for n in (3, 4, 5) if attr[n] is not None) - link(i + attr.start(group), i + attr.end(group), attribute=True) - i = tag.end() + if not closing: + opened.append(start) + elif opened: + protected.append((opened.pop(), end)) + protected.extend((start, len(source)) for start in opened) + if generated_fields: + protected.extend(match.span() for match in _PROVENANCE.finditer(source)) + text_spans = _merged_spans(text_spans) + for match in _FIELD.finditer(source): + prefix = source[source.rfind("\n", 0, match.start()) + 1:match.start()] + if prefix and not (match[0].startswith("HTML:") and prefix.startswith("- [")): continue - if generated_fields: - field = _FIELD.match(text, i) - line_prefix = text[text.rfind("\n", 0, i) + 1:i] if field else "" - if field and (line_start or (text.startswith("HTML:", i) and line_prefix.startswith("- ["))): - end = field.end(1) - # Generated list prose appends a sentence period to HTML URL. - if text[end - 1] == ".": - end -= 1 - if line_start and text.startswith("URL:", i): - context = text[field.start(1):end] - link(field.start(1), end) - i = field.end() + if not _covered(match.start(), text_spans): continue - if text[i] == "[": - bracket_depth += 1 - elif text[i] == "]" and bracket_depth: - bracket_depth -= 1 - if text.startswith("](", i): - start = i + 2 - while start < len(text) and text[start].isspace(): - start += 1 - span = _destination(text, start) - if span: - link(*span) - i = span[1] + (text[start] == "<") - continue - i += 1 + start, end = match.span(1) + if source[end - 1] == ".": + end -= 1 + nodes.append((start, end, source[start:end], "context" if match[0].startswith("URL:") else "field")) + protected = _merged_spans(protected) + return [node for node in sorted(set(nodes)) if not _covered(node[0], protected)], md + + +def _escape_destination(value, kind, md): + if kind.startswith("html"): + value = html.escape(value, quote=True) + if kind == "html_unquoted": + value = re.sub(r"[\s=\x60]", lambda m: "&#" + str(ord(m[0])) + ";", value) + return value + if kind in {"field", "context"}: + return value + value = md.normalizeLink(value) + if kind != "autolink": + value = value.replace("&", "&") + if kind == "markdown": + value = re.sub(r"[\\()]", lambda m: "\\" + m[0], value) + return value + + +def _markdown(text, catalog, *, context=None, generated_fields=False, validate=False): + nodes, md = _link_nodes(text, generated_fields) parts, previous = [], 0 - for start, end, replacement in edits: - parts.extend((text[previous:start], replacement)) - previous = end + for start, end, value, kind in nodes: + if kind == "context": + context = value + replacement = catalog.link(value, context=context, validate=validate) + if replacement == value: + continue + if start < previous: + continue + escaped = _escape_destination(replacement, kind, md) + edits = value.edits(escaped) if isinstance(value, _HTMLValue) else [(start, end, escaped)] + for begin, finish, content in edits: + parts.extend((text[previous:begin], content)) + previous = finish parts.append(text[previous:]) return "".join(parts) diff --git a/scripts/v8std_mcp_server.py b/scripts/v8std_mcp_server.py index ac7761e..da0db7e 100644 --- a/scripts/v8std_mcp_server.py +++ b/scripts/v8std_mcp_server.py @@ -532,6 +532,7 @@ def __init__(self, fd): self.fd = fd self.buffer = bytearray() self.eof = False + self.closed = anyio.Event() def __aiter__(self): return self @@ -541,6 +542,7 @@ async def __anext__(self): newline = self.buffer.find(b"\n") if newline >= 0 or self.eof: if not self.buffer: + self.closed.set() raise StopAsyncIteration end = newline + 1 if newline >= 0 else len(self.buffer) line = bytes(self.buffer[:end]) @@ -552,12 +554,44 @@ async def __anext__(self): self.eof = not block +class _StdioOutput: + """Cancellable byte writes, without the SDK AsyncFile's shielded thread. + + A ready pipe may accept only part of a frame. Nonblocking writes and + readiness waits preserve backpressure while permitting process shutdown. + The SDK still owns JSON serialization, framing and protocol handling. + """ + def __init__(self, fd): + self.fd = fd + self.blocking = os.get_blocking(fd) + os.set_blocking(fd, False) + + async def write(self, text): + pending = memoryview(text.encode("utf-8")) + while pending: + await anyio.lowlevel.checkpoint() + try: + count = os.write(self.fd, pending) + except BlockingIOError: + await anyio.wait_writable(self.fd) + else: + pending = pending[count:] + + async def flush(self): + await anyio.lowlevel.checkpoint() + + def close(self): + os.set_blocking(self.fd, self.blocking) + + def install_stdio_lifecycle(server: FastMCP, index) -> None: async def run_stdio(): - if isinstance(index, SnapshotIndex): - index.start() + output = _StdioOutput(sys.stdout.fileno()) + lines = _StdioLines(sys.stdin.fileno()) try: + if isinstance(index, SnapshotIndex): + index.start() async with anyio.create_task_group() as group: async def terminate_on_signal(): with anyio.open_signal_receiver(signal.SIGTERM) as signals: @@ -566,11 +600,21 @@ async def terminate_on_signal(): break group.start_soon(terminate_on_signal) - async with stdio_server(stdin=_StdioLines(sys.stdin.fileno())) as (read_stream, write_stream): + + async def terminate_on_eof(): + await lines.closed.wait() + # Permit queued responses to drain, but never let a departed + # client hold the process/refresh workers via a full pipe. + await anyio.sleep(.25) + group.cancel_scope.cancel() + + group.start_soon(terminate_on_eof) + async with stdio_server(stdin=lines, stdout=output) as (read_stream, write_stream): await server._mcp_server.run(read_stream, write_stream, server._mcp_server.create_initialization_options()) group.cancel_scope.cancel() finally: + output.close() if isinstance(index, SnapshotIndex): with anyio.CancelScope(shield=True): await anyio.to_thread.run_sync(index.close) @@ -778,7 +822,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--vectors-url", default=None, help="Explicit legacy remote vectors JSONL URL.") parser.add_argument("--site-url", default=None, help="Snapshot source and presentation site URL.") parser.add_argument("--transport", choices=["stdio", "streamable-http"], default="streamable-http") - parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR) + parser.add_argument("--cache-dir", default=None, help="Overrides V8STD_MCP_CACHE_DIR.") parser.add_argument("--refresh-seconds", type=int, default=DEFAULT_REFRESH_SECONDS) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8765) @@ -818,6 +862,13 @@ def parse_args(argv: list[str]) -> argparse.Namespace: help="Allowed Origin header for MCP transport security. Can be repeated.", ) args = parser.parse_args(argv) + cache = (args.cache_dir if args.cache_dir is not None + else os.environ.get("V8STD_MCP_CACHE_DIR", str(DEFAULT_CACHE_DIR))) + if not cache.strip(): + parser.error("--cache-dir / V8STD_MCP_CACHE_DIR must not be empty") + args.cache_dir = Path(cache) + if not 0 <= args.port <= 65535: + parser.error("port must be from 0 to 65535") site = args.site_url if args.site_url is not None else os.environ.get("V8STD_MCP_SITE_URL") legacy = any(value is not None for value in (args.pages, args.vectors, args.index_url, args.vectors_url)) if legacy and site is not None: diff --git a/tests/test_v8std_mcp_presentation.py b/tests/test_v8std_mcp_presentation.py index ae51bfc..27e566b 100644 --- a/tests/test_v8std_mcp_presentation.py +++ b/tests/test_v8std_mcp_presentation.py @@ -6,6 +6,8 @@ import sys import tempfile import unittest +from html.parser import HTMLParser +from markdown_it import MarkdownIt from tests import mcp_snapshot_fixtures as fixture @@ -18,7 +20,7 @@ "markdown_path": "THIRD_PARTY_DIAGNOSTIC_ARTICLES.md"}} -class PresentationTests(unittest.TestCase): +class PresentationHelpers: def module(self): self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_presentation")) return importlib.import_module("v8std_mcp_presentation") @@ -27,17 +29,19 @@ def present(self, value): return self.module().present_result(value, canonical_site_url=PUBLIC, site_url=LOCAL, page_paths=PATHS) + +class PresentationTests(PresentationHelpers, unittest.TestCase): def test_link_nodes_and_nested_urls_preserve_literals_and_provenance(self): body = ('`https://v8std.ru/std/437/`\n' '[link](https://v8std.ru/std/437/?x=1#anchor "title")\n' '![image]()\n' - '[reference][ref]\n[ref]: https://v8std.ru/std/437/ "title"\n' + '[reference][ref]\n\n[ref]: https://v8std.ru/std/437/ "title"\n' '\n' 'a\n' - '\n' + '\n\n' '```bsl\nx = "https://v8std.ru/std/437/";\n' - '[code](https://v8std.ru/std/437/)\n```\n' - ' [indented](https://v8std.ru/std/437/)\n' + '[code](https://v8std.ru/std/437/)\n```\n\n' + ' [indented](https://v8std.ru/std/437/)\n\n' '[literal](https://v8std.ru/std/437/)\n' '
literal
\n' 'ordinary https://v8std.ru/std/437/\n') @@ -110,7 +114,7 @@ def test_publisher_rejects_unresolved_links_using_shared_catalog(self): def test_structured_url_suffixes_and_escaped_balanced_destinations(self): result = self.present({"id": "std437", "url": PUBLIC + "std/437/?view=full#section"}) self.assertEqual(result["url"], LOCAL + "std/437/?view=full#section") - body = '[a `label`][ref]\n[ref]:\n "title"\n' + body = '[a `label`][ref]\n\n[ref]:\n "title"\n' self.assertIn('<' + LOCAL + 'std/437/>', self.present({"body_markdown": body})["body_markdown"]) def test_local_lookup_does_not_escape_prefix(self): @@ -121,5 +125,178 @@ def test_local_lookup_does_not_escape_prefix(self): self.assertEqual(catalog.lookup(value), value) +class SemanticPresentationTests(PresentationHelpers, unittest.TestCase): + def markdown(self, text): + return self.module().present_markdown(text, canonical_site_url=PUBLIC, site_url=LOCAL, page_paths=PATHS) + + def validate(self, text): + self.module().validate_links(text, canonical_site_url=PUBLIC, page_paths=PATHS) + + def test_pinned_parser_dependency_is_shared_but_pure_format_stays_independent(self): + from importlib.metadata import version + self.assertEqual(version("markdown-it-py"), "4.0.0") + self.assertEqual(version("mcp"), "1.27.0") + for name in ("requirements.txt", "requirements-mcp.txt"): + self.assertIn("markdown-it-py==4.0.0", (ROOT / name).read_text().splitlines()) + + def test_parser_source_maps_are_released_without_waiting_for_cyclic_gc(self): + import gc + import weakref + from unittest.mock import patch + references = [] + def parser(*args, **kwargs): + instance = MarkdownIt(*args, **kwargs) + references.append(weakref.ref(instance)) + return instance + enabled = gc.isenabled() + gc.disable() + try: + with patch.object(self.module(), "MarkdownIt", side_effect=parser): + self.markdown("> [x](https://v8std.ru/std/437/)\n") + self.assertTrue(references) + self.assertTrue(all(reference() is None for reference in references), + "parser closure retains the document tree/source map") + finally: + if enabled: + gc.enable() + gc.collect() + + def test_publisher_uses_semantics_for_nested_code_definitions_and_literal_fragments(self): + from generate_mcp_snapshot import build_snapshot + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture.write_docs(root) + for body, rejected in ( + ('> [r]: https://v8std.ru/missing/\n>\n> [x][r]', True), + ('- [r]: https://v8std.ru/missing/\n\n [x][r]', True), + ('> - ~~~bsl\n> [x](https://v8std.ru/missing/)', False), + ('[x](https://v8std.ru/missing/ "unfinished)', False), + ('[r]: https://v8std.ru/missing/ "title" extra', False), + ): + with self.subTest(body=body): + (root / "llms.txt").write_text(body) + if rejected: + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + build_snapshot(root, fixture.SOURCE_SHA, PUBLIC) + else: + build_snapshot(root, fixture.SOURCE_SHA, PUBLIC) + + def test_valid_autolink_inside_incomplete_outer_link_and_literal_entity_suffix(self): + text = '[x]( "unfinished)' + result = self.markdown(text) + self.assertEqual(result, text.replace(PUBLIC, LOCAL)) + links = [child for token in MarkdownIt("commonmark").parse(result) + for child in (token.children or []) if child.type == "link_open"] + self.assertEqual([token.attrGet("href") for token in links], [LOCAL + "std/437/?a=&"]) + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + self.validate(text.replace("std/437/", "missing/")) + + def test_multiline_titles_duplicate_references_headings_and_literal_source_bytes(self): + text = ('# Заголовок\x00 [link](https://v8std.ru/std/437/ "title")\r\n\r\n' + '[link](https://v8std.ru/std/437/ "a\r\nb")\r\n\r\n' + '> - [r]:\r\n> "title"\r\n>\r\n> [x][r]\r\n\r\n' + '[r]: https://v8std.ru/std/437.md\r\n\r\n' + '[same `https://v8std.ru/std/437/`](https://v8std.ru/std/437/)\r\n') + expected = text.replace('https://v8std.ru/std/437/', LOCAL + 'std/437/') + expected = expected.replace('`' + LOCAL + 'std/437/`', '`https://v8std.ru/std/437/`') + expected = expected.replace('https://v8std.ru/std/437.md', LOCAL + 'std/437.md') + self.assertEqual(self.markdown(text), expected) + value = {"related": [{"url": PUBLIC + "std/437/?a=&\\)"}]} + self.assertEqual(self.present(value)["related"][0]["url"], LOCAL + "std/437/?a=&\\)") + + def test_nested_containers_use_commonmark_code_and_reference_semantics(self): + for prefix, continuation in (("> ", "> "), ("- ", " "), ("> - ", "> "), + ("1. > ", " > "), ("> > ", "> > ")): + for fence in ("~~~", "```"): + for closed in (False, True): + text = prefix + fence + "bsl\n" + continuation + '[code](https://v8std.ru/missing/)\n' + if closed: + text += continuation + fence + "\n" + with self.subTest(prefix=prefix, fence=fence, closed=closed): + self.assertEqual(self.markdown(text), text) + self.validate(text) + text = prefix + '[r]: https://v8std.ru/std/437/ "title"\n\n' + prefix + '[use][r]\n' + with self.subTest(reference=prefix): + result = self.markdown(text) + self.assertEqual(result, text.replace(PUBLIC, LOCAL)) + self.assertIn('href="' + LOCAL + 'std/437/"', MarkdownIt("commonmark").render(result)) + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + self.validate(text.replace("std/437/", "missing/")) + + def test_only_complete_links_and_titles_are_rewritten_or_validated(self): + literals = ('[x]({url}', '[x]({url} "unfinished)', '[x]({url} "title" extra)', + '![x]({url}', '[r]: {url} "bad" trailing\n') + for template in literals: + for path in ("std/437/", "missing/"): + text = template.format(url=PUBLIC + path) + with self.subTest(text=text): + self.assertEqual(self.markdown(text), text) + self.validate(text) + # Valid fallback reference: only its definition is a destination. + text = '[x](https://v8std.ru/missing/\n\n[x]: https://v8std.ru/std/437/\n' + result = self.markdown(text) + self.assertEqual(result, '[x](https://v8std.ru/missing/\n\n[x]: ' + LOCAL + 'std/437/\n') + self.validate(text) + + def test_reparsed_markdown_preserves_interpreted_destinations_and_titles(self): + samples = ( + ('[x](https://v8std.ru/std/437/?q=foo\\)#end "title")', LOCAL + 'std/437/?q=foo)#end'), + ('[x](https://v8std.ru/std/437/?q=a&b=2)', LOCAL + 'std/437/?q=a&b=2'), + ('![x]()', LOCAL + 'std/437/?q=hello%20world'), + ('> [r]: https://v8std.ru/std/437/?q=foo\\) "title"\n>\n> [x][r]', LOCAL + 'std/437/?q=foo)'), + ) + for text, expected in samples: + with self.subTest(text=text): + rendered = MarkdownIt("commonmark").parse(self.markdown(text)) + links = [child for block in rendered for child in (block.children or []) + if child.type in {"link_open", "image"}] + self.assertEqual(len(links), 1) + self.assertEqual(links[0].attrGet("href") or links[0].attrGet("src"), expected) + if '"title"' in text: + self.assertEqual(links[0].attrGet("title"), "title") + + def test_html_escaping_uses_actual_attribute_context_and_does_not_decode_backslashes(self): + class Attributes(HTMLParser): + def __init__(self): + super().__init__() + self.tags = [] + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + for quote in ('', '"', "'"): + for suffix, expected in (("?q=hello world", "?q=hello world"), + ("?q=foo\\)&x=2", "?q=foo\\)&x=2"), + ("?q="hi"'", '?q="hi"\'')): + text = 'x' + with self.subTest(text=text): + parser = Attributes() + parser.feed(self.markdown(text)) + self.assertEqual(parser.tags, [("a", {"href": LOCAL + "std/437/" + expected, "title": "keep"})]) + + def test_source_offsets_survive_containers_crlf_tabs_and_repeated_literals(self): + text = ('> - `https://v8std.ru/std/437/`\r\n>\r\n' + '>\t[r]: \r\n>\r\n' + '> - [label `literal`][r]\r\n') + expected = text.replace('[r]: <' + PUBLIC, '[r]: <' + LOCAL) + self.assertEqual(self.markdown(text), expected) + + def test_multiline_html_code_tags_inside_containers_protect_their_contents(self): + for tag in ("pre", "code", "script", "style"): + text = ('> <' + tag + '\r\n> class="sample">\r\n' + '> literal\r\n' + '> \r\n') + with self.subTest(tag=tag): + self.assertEqual(self.markdown(text), text) + self.validate(text.replace("std/437/", "missing/")) + + def test_multiline_html_destination_retains_container_and_line_endings(self): + text = '> x\r\n' + result = self.markdown(text) + self.assertEqual(result, text.replace(PUBLIC, LOCAL)) + self.assertIn('href="' + LOCAL + 'std/437/?q=hello\nworld"', + MarkdownIt("commonmark").render(result)) + with self.assertRaisesRegex(ValueError, "unresolved_internal_link"): + self.validate(text.replace("std/437/", "missing/")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_v8std_mcp_runtime.py b/tests/test_v8std_mcp_runtime.py index a0bc35e..2440fd9 100644 --- a/tests/test_v8std_mcp_runtime.py +++ b/tests/test_v8std_mcp_runtime.py @@ -10,6 +10,7 @@ import pickle import queue import signal +import select import socket import subprocess import sys @@ -234,6 +235,31 @@ def test_facade_preserves_preview_budgets_and_captures_each_data_call_once(self) class ConfigurationTests(unittest.TestCase): + def test_cache_cli_env_default_precedence_and_early_invalid_inputs(self): + from v8std_mcp_server import parse_args, main + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(parse_args([]).cache_dir, Path("/var/lib/v8std-mcp")) + with patch.dict(os.environ, {"V8STD_MCP_CACHE_DIR": "/tmp/env-cache"}, clear=True): + self.assertEqual(parse_args([]).cache_dir, Path("/tmp/env-cache")) + self.assertEqual(parse_args(["--cache-dir", "/tmp/cli-cache"]).cache_dir, Path("/tmp/cli-cache")) + with patch.dict(os.environ, {"V8STD_MCP_CACHE_DIR": ""}, clear=True): + self.assertEqual(parse_args(["--cache-dir", "/tmp/cli-cache"]).cache_dir, Path("/tmp/cli-cache")) + for argv, environment in (([], {"V8STD_MCP_CACHE_DIR": ""}), + ([], {"V8STD_MCP_CACHE_DIR": " \t"}), + (["--cache-dir", ""], {}), + (["--cache-dir", " "], {}), + (["--port", "-1"], {}), (["--port", "65536"], {})): + with self.subTest(argv=argv, environment=environment), \ + patch.dict(os.environ, environment, clear=True), \ + contextlib.redirect_stderr(io.StringIO()), \ + patch("v8std_mcp_server.SnapshotIndex", side_effect=AssertionError("source construction")), \ + patch("v8std_mcp_server.V8StdIndex", side_effect=AssertionError("source construction")): + with self.assertRaises(SystemExit): + main(argv) + with patch.dict(os.environ, {}, clear=True): + for port in (0, 1, 65535): + self.assertEqual(parse_args(["--port", str(port)]).port, port) + def test_site_default_precedence_and_explicit_legacy_mode(self): from v8std_mcp_server import parse_args with patch.dict(os.environ, {}, clear=True): @@ -320,6 +346,95 @@ def close(self): class WireTests(unittest.TestCase): + def test_stdio_large_response_drained_and_backpressured_shutdown_cleans_workers(self): + from tests.test_v8std_mcp_snapshots import Source + from v8std_mcp_snapshots import SnapshotStore + source = Source() + payload = "x" * (2 * 1024 * 1024) + files = fixture.corpus_files() + files["llms-full.txt"] = payload.encode() + source.archive, source.manifest = fixture.snapshot_fixture(files=fixture.with_metadata(files)) + try: + for shutdown in ("drained-eof", "eof", "sigterm"): + with self.subTest(shutdown=shutdown), tempfile.TemporaryDirectory() as directory: + source.fault = None + SnapshotStore(source.url, Path(directory)).refresh() + source.requested.clear() + source.release.clear() + source.fault = "headers" + process = subprocess.Popen([sys.executable, str(ROOT / "scripts/v8std_mcp_server.py"), + "--transport", "stdio", "--site-url", source.url, "--cache-dir", directory, + "--refresh-seconds", "0"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0, start_new_session=True) + number, buffer = 0, bytearray() + + def send(method, params): + nonlocal number + number += 1 + process.stdin.write((json.dumps({"jsonrpc": "2.0", "id": number, + "method": method, "params": params}) + "\n").encode()) + + def response(): + deadline = time.monotonic() + 8 + while b"\n" not in buffer: + remaining = deadline - time.monotonic() + self.assertGreater(remaining, 0, "protocol response timeout") + self.assertTrue(select.select([process.stdout], [], [], remaining)[0]) + block = os.read(process.stdout.fileno(), 65536) + self.assertTrue(block, "unexpected protocol EOF") + buffer.extend(block) + end = buffer.index(b"\n") + 1 + line = bytes(buffer[:end]) + del buffer[:end] + result = json.loads(line) + self.assertEqual(result["id"], number, "stdout must contain only SDK frames") + return result + + try: + send("initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "backpressure", "version": "1"}}) + self.assertIn("serverInfo", response()["result"]) + self.assertTrue(source.requested.wait(5)) + # Warm-cache generation is ready before the blocked refresh. + send("tools/call", {"name": "v8std_search", "arguments": {"query": "std437"}}) + self.assertFalse(response()["result"].get("isError", False)) + children = [int(row.split(None, 2)[0]) + for row in subprocess.check_output(["ps", "-axo", "pid=,ppid=,command="], text=True).splitlines() + if row.split(None, 2)[1] == str(process.pid) and "spawn_main" in row] + self.assertTrue(children, "exercise shutdown with an active spawn worker") + send("resources/read", {"uri": "v8std://llms-full.txt"}) + if shutdown == "drained-eof": + self.assertEqual(response()["result"]["contents"][0]["text"], payload) + else: + self.assertTrue(select.select([process.stdout], [], [], 5)[0]) + # At most 64 bytes consumed: a 2MB frame cannot fit in the pipe. + self.assertTrue(os.read(process.stdout.fileno(), 64).startswith(b'{"jsonrpc"')) + time.sleep(.1) + started = time.monotonic() + if shutdown == "sigterm": + process.send_signal(signal.SIGTERM) + else: + process.stdin.close() + try: + process.wait(3) + except subprocess.TimeoutExpired: + self.fail("2MB stdio response prevented bounded " + shutdown + " shutdown") + self.assertEqual(process.returncode, 0) + self.assertLess(time.monotonic() - started, 3) + for pid in children: + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + finally: + source.release.set() + if process.poll() is None: + # Only this test's new session; no worker orphan on RED. + os.killpg(process.pid, signal.SIGKILL) + process.wait(3) + for stream in (process.stdin, process.stdout, process.stderr): + stream.close() + finally: + source.close() + def test_stdio_initialize_not_ready_eof_and_sigterm_clean_workers(self): from tests.test_v8std_mcp_snapshots import Source source = Source() diff --git a/tests/test_v8std_mcp_snapshot_format.py b/tests/test_v8std_mcp_snapshot_format.py index 87b2baa..a75d97a 100644 --- a/tests/test_v8std_mcp_snapshot_format.py +++ b/tests/test_v8std_mcp_snapshot_format.py @@ -616,7 +616,7 @@ def test_cli_site_precedence_public_delivery_and_explicit_empty_env(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) fixture.write_docs(root / "docs") - command = [sys.executable, "-S", str(ROOT / "scripts/generate_mcp_snapshot.py"), + command = [sys.executable, str(ROOT / "scripts/generate_mcp_snapshot.py"), "--docs", str(root / "docs"), "--output", str(root / "output"), "--source-sha", fixture.SOURCE_SHA] environment = {**os.environ, "V8STD_MCP_SITE_URL": ""} @@ -633,13 +633,13 @@ def test_cli_and_stdlib_only_import(self): self.modules() environment = {**os.environ, "PYTHONPATH": str(ROOT / "scripts")} result = subprocess.run([sys.executable, "-S", "-c", - "import generate_mcp_snapshot, v8std_mcp_snapshot_format"], + "import v8std_mcp_snapshot_format"], env=environment, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) with tempfile.TemporaryDirectory() as directory: root = Path(directory) fixture.write_docs(root / "docs") - command = [sys.executable, "-S", str(ROOT / "scripts/generate_mcp_snapshot.py"), + command = [sys.executable, str(ROOT / "scripts/generate_mcp_snapshot.py"), "--docs", str(root / "docs"), "--output", str(root / "output"), "--source-sha", fixture.SOURCE_SHA, "--site-url", fixture.SITE_URL] result = subprocess.run(command, env=environment, capture_output=True, text=True) From 60d093c3753ece420dd1448117dbc0cfdfcad812 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 17:14:36 +0300 Subject: [PATCH 23/88] docs: complete frozen runtime task and review evidence --- spec/operations/mcp-container-verification.md | 30 ++++++++++++++++++- ...6-09-10-mcp-container-distribution-plan.md | 8 ++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 85d183c..a26022c 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -73,9 +73,14 @@ peak RSS while retaining original, serialized and reconstructed state. Encoding took 0.102 s and decoding 0.0772 s on this Mac. This excludes new runtime resources, spawn/staging and host overlap; it is not the final RAM budget. -### Frozen runtime — implementation evidence, review pending +### Frozen runtime — reviewed task evidence Implementation: `b8656c1c238969d996e24f323333a925df9c5b87`. +Reviewed correction: `49e8b525c370a7f69f9681b3e1d695ee3e3b150f`. +Initial findings covered stdout backpressure, Markdown classification/escaping +and cache/port configuration. Scoped re-review confirmed all five addressed. +One pre-existing image-alt HTML protection edge case remains assigned to final +integration review; task acceptance is not final release acceptance. ```sh .venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_presentation tests.test_v8std_mcp_runtime tests.test_v8std_mcp_index tests.test_v8std_mcp_snippet tests.test_v8std_mcp_server tests.test_v8std_mcp_combined tests.test_v8std_mcp_capacity tests.test_v8std_mcp_monitoring @@ -149,6 +154,29 @@ New generations still require full preparation; measured new-generation wall time was 3.566641 s with sampled tree RSS 555,876,352 bytes. These are sampled desktop observations, not memory ceilings, target-host acceptance or 100k evidence. +### Runtime correction and semantic-parser cost + +The same combined MCP command at `49e8b52` passed **217 tests in 82.938 seconds**. +Regressions include drained 2 MiB frames, undrained stdout at EOF/SIGTERM, +worker cleanup, CLI/environment/default precedence, nested containers/fences, +incomplete links and titles, destination escaping, CRLF and source-map lifetime. +Runtime/build requirements pin `markdown-it-py==4.0.0`; pure format verification +remains independent of docs tooling. Canonical data/ranks were not rewritten. + +The real-corpus benchmark retained all 256 ranks/IDs and sampled score +dictionaries, MRR 0.9939759036. Full-generation preparation now includes the +semantic parser/source maps: cold 5.432 s, warm-with-old 5.594 s, new-with-old +6.071 s. Corresponding sampled tree RSS was 653,115,392 / 636,731,392 / +699,695,104 bytes. Unchanged refresh remained 0.731 s with 256-byte metadata IPC; +it does not rebuild or parse Resources. New-generation concurrent query p95 +was 62.55 ms across 106 samples. These desktop samples do not establish a +target-host memory ceiling or production capacity. + +Final review must resolve the known case where `` in image alt text can +incorrectly protect subsequent visible links from rebasing/validation. It was +reproduced in both the old scanner and the semantic-parser fix, so scoped +re-review did not extend its loop to it. This is explicitly not waived for release. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 91265e3..071cd7c 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -257,7 +257,7 @@ Facade captures coordinator.current() once per top-level call, invokes that generation including nested snippet/search/related operations, then transforms only presentation links on the returned copy. -- [ ] **RED:** Wire tests initialize/list tools before source readiness; valid +- [x] **RED:** Wire tests initialize/list tools before source readiness; valid call before ready is error `INDEX_NOT_READY`; `/healthz` is 503 and `/livez` 200. Add full snippet compatibility and generation-swap tests; public source fetch is forbidden in callbacks. A local-prefix fixture must retain literal @@ -269,7 +269,7 @@ self.assertEqual(result["page"]["url"], "http://localhost:8080/kb/std/437/") self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/content/437/hdoc"]) ``` -- [ ] **GREEN presentation:** Parse Markdown link/image/reference/autolink and +- [x] **GREEN presentation:** Parse Markdown link/image/reference/autolink and HTML attributes outside code; rewrite only known internal paths, preserve external provenance/query/fragment. Local URLs accepted as page lookup inputs resolve to canonical keys without affecting ranking. All nested result URLs @@ -277,13 +277,13 @@ self.assertEqual(result["page"]["source_urls"], ["https://its.1c.ru/db/v8std/con Validate publisher links against the same catalog. Preserve generated bare internal `URL:`/`HTML:` fields and resolve relative links in their page context; ordinary code literals and external source records are not link nodes. -- [ ] **GREEN runtime:** Startup chooses snapshot mode from SITE_URL/default, +- [x] **GREEN runtime:** Startup chooses snapshot mode from SITE_URL/default, supports stdio and HTTP from same build_server. Legacy explicit files remain usable; ambiguous legacy URLs plus site setting fail before network. Default direct Python transport stays compatible; container supplies stdio explicitly. Initialize/schema immediate, background bootstrap, bounded EOF/SIGTERM cleanup, clean stdout, health/version compact additive metadata, no raw errors/data. -- [ ] **Verify:** Task 1–3 tests plus all MCP server/index/snippet/combined tests; +- [x] **Verify:** Task 1–3 tests plus all MCP server/index/snippet/combined tests; real stdio subprocess and loopback HTTP smoke; before/after search benchmark on same corpus, slow-source requests and CPU/RAM refresh measurements. Commit only reviewed implementation; no changed scores or widened snippet/query limits. From 12b38add919599cbe1e0ccdcc7e57574207c9dd1 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 17:56:56 +0300 Subject: [PATCH 24/88] feat(distribution): add thin MCP and local-site container harness --- .dockerignore | 34 ++ Dockerfile.mcp | 24 + Dockerfile.site | 35 ++ compose.yaml | 57 +++ deploy/container/site.conf | 50 ++ deploy/docker-catalog/server.yaml | 41 ++ docker-compose/docker-compose.yml | 2 + docker-compose/docker/Dockerfile | 2 +- docs/container-installation.md | 178 +++++++ docs/mcp.md | 3 + docs/support.md | 6 +- overrides/main.html | 2 + requirements-build.lock | 393 +++++++++++++++ requirements-mcp.lock | 685 ++++++++++++++++++++++++++ scripts/build_local_site.py | 94 ++++ scripts/check_mcp_container.py | 509 +++++++++++++++++++ scripts/publish_license_texts.py | 8 + tests/test_published_license_links.py | 15 +- tests/test_v8std_mcp_distribution.py | 99 ++++ 19 files changed, 2234 insertions(+), 3 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile.mcp create mode 100644 Dockerfile.site create mode 100644 compose.yaml create mode 100644 deploy/container/site.conf create mode 100644 deploy/docker-catalog/server.yaml create mode 100644 docs/container-installation.md create mode 100644 requirements-build.lock create mode 100644 requirements-mcp.lock create mode 100644 scripts/build_local_site.py create mode 100644 scripts/check_mcp_container.py create mode 100644 tests/test_v8std_mcp_distribution.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8143d50 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,34 @@ +** +!Dockerfile.mcp +!Dockerfile.site +!requirements-mcp.lock +!requirements-build.lock +!zensical.toml +!docs/ +!docs/** +!overrides/ +!overrides/** +!retrieval-rules.yml +!LICENSE +!LICENSES/ +!LICENSES/*.txt +!scripts/ +!scripts/v8std_mcp_server.py +!scripts/v8std_mcp_runtime.py +!scripts/v8std_mcp_index.py +!scripts/v8std_mcp_snapshots.py +!scripts/v8std_mcp_snapshot_format.py +!scripts/v8std_mcp_presentation.py +!scripts/v8std_mcp_chunks.py +!scripts/v8std_retrieval_rules.py +!scripts/v8std_search_features.py +!scripts/build_local_site.py +!scripts/generate_mcp_snapshot.py +!scripts/generate_ai_artifacts.py +!scripts/generate_social_cards.py +!scripts/v8std_markdown.py +!scripts/atomic_files.py +!scripts/publish_license_texts.py +!deploy/ +!deploy/container/ +!deploy/container/site.conf diff --git a/Dockerfile.mcp b/Dockerfile.mcp new file mode 100644 index 0000000..1eacfbf --- /dev/null +++ b/Dockerfile.mcp @@ -0,0 +1,24 @@ +FROM python:3.12-slim@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea +ARG SOURCE_SHA +LABEL org.opencontainers.image.title="v8std MCP" \ + org.opencontainers.image.source="https://github.com/zeegin/v8std" \ + org.opencontainers.image.revision="${SOURCE_SHA}" \ + org.opencontainers.image.licenses="CC0-1.0" +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \ + V8STD_MCP_CACHE_DIR=/var/lib/v8std-mcp V8STD_MCP_RUNTIME_SHA=${SOURCE_SHA} +WORKDIR /opt/v8std +COPY requirements-mcp.lock ./ +RUN python -c 'import os,re; assert re.fullmatch("[0-9a-f]{40}",os.environ["V8STD_MCP_RUNTIME_SHA"])' \ + && python -m pip install --no-cache-dir --require-hashes -r requirements-mcp.lock \ + && python -m pip check \ + && mkdir -p /var/lib/v8std-mcp \ + && chown 10001:10001 /var/lib/v8std-mcp +COPY scripts/v8std_mcp_server.py scripts/v8std_mcp_runtime.py scripts/v8std_mcp_index.py \ + scripts/v8std_mcp_snapshots.py scripts/v8std_mcp_snapshot_format.py \ + scripts/v8std_mcp_presentation.py scripts/v8std_mcp_chunks.py \ + scripts/v8std_retrieval_rules.py scripts/v8std_search_features.py ./scripts/ +COPY retrieval-rules.yml LICENSE ./ +COPY LICENSES ./LICENSES/ +USER 10001:10001 +ENTRYPOINT ["python", "/opt/v8std/scripts/v8std_mcp_server.py"] +CMD ["--transport", "stdio"] diff --git a/Dockerfile.site b/Dockerfile.site new file mode 100644 index 0000000..023afca --- /dev/null +++ b/Dockerfile.site @@ -0,0 +1,35 @@ +# The named build context is a prepared local-profile output, never the public site. +FROM python:3.12-slim@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea AS local-builder +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /build +COPY requirements-build.lock ./ +RUN python -m pip install --no-cache-dir --require-hashes -r requirements-build.lock \ + && python -m pip check +COPY docs ./docs/ +COPY overrides ./overrides/ +COPY scripts/build_local_site.py scripts/generate_mcp_snapshot.py \ + scripts/v8std_mcp_snapshot_format.py scripts/v8std_mcp_chunks.py \ + scripts/v8std_mcp_presentation.py scripts/generate_ai_artifacts.py \ + scripts/generate_social_cards.py scripts/v8std_retrieval_rules.py \ + scripts/v8std_search_features.py scripts/v8std_markdown.py \ + scripts/atomic_files.py scripts/publish_license_texts.py ./scripts/ +COPY LICENSES ./LICENSES/ +COPY zensical.toml retrieval-rules.yml LICENSE ./ +RUN chmod -R a+rX /build +USER 10001:10001 +ENTRYPOINT ["python", "scripts/build_local_site.py"] + +FROM nginx:stable-alpine@sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c +ARG SOURCE_SHA +ARG SITE_PREFIX= +LABEL org.opencontainers.image.title="v8std local site" \ + org.opencontainers.image.source="https://github.com/zeegin/v8std" \ + org.opencontainers.image.revision="${SOURCE_SHA}" \ + org.opencontainers.image.licenses="CC0-1.0" +COPY deploy/container/site.conf /etc/nginx/nginx.conf +COPY --from=local-site --chown=10001:10001 / /srv/site/${SITE_PREFIX}/ +COPY LICENSE /usr/share/licenses/v8std/LICENSE +USER 10001:10001 +EXPOSE 8000 +ENTRYPOINT ["nginx"] +CMD ["-g", "daemon off;"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ddbaaed --- /dev/null +++ b/compose.yaml @@ -0,0 +1,57 @@ +# Release distribution. Source-bind development lives in docker-compose/. +# Both overrides must identify an actually published release or a local test image. +services: + site: + image: ${V8STD_SITE_IMAGE:?Set the verified ghcr.io/zeegin/v8std-site tag or digest} + user: "10001:10001" + read_only: true + init: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + tmpfs: ["/tmp:rw,noexec,nosuid,size=32m,uid=10001,gid=10001"] + mem_limit: 128m + cpus: 0.5 + # The image listens on 8000 by default; changing the common port needs + # only a temporary nginx config, leaving the image/root read-only. + entrypoint: [/bin/sh, -ec] + command: + - 'sed "s/listen 8000;/listen ${V8STD_SITE_PORT:-18765};/" /etc/nginx/nginx.conf > /tmp/site.conf; exec nginx -c /tmp/site.conf -g "daemon off;"' + ports: + - "127.0.0.1:${V8STD_SITE_PORT:-18765}:${V8STD_SITE_PORT:-18765}" + - "127.0.0.1:${V8STD_MCP_PORT:-18766}:8001" + networks: + corpus: + aliases: [v8std.localhost] + publish: {} + mcp: + image: ${V8STD_MCP_IMAGE:?Set the verified ghcr.io/zeegin/v8std-mcp tag or digest} + profiles: [mcp] + user: "10001:10001" + read_only: true + init: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + tmpfs: ["/tmp:rw,noexec,nosuid,size=64m,uid=10001,gid=10001"] + mem_limit: 1536m + cpus: 2 + pids_limit: 128 + volumes: [mcp-cache:/var/lib/v8std-mcp] + environment: + V8STD_MCP_SITE_URL: "http://v8std.localhost:${V8STD_SITE_PORT:-18765}${V8STD_SITE_PREFIX:-/}" + V8STD_MCP_MAX_SNIPPET_CHARS: "${V8STD_MCP_MAX_SNIPPET_CHARS:-4000}" + command: [--transport, streamable-http, --host, 0.0.0.0, --port, "8000"] + # Host HTTP goes through the site's loopback proxy; MCP has no egress route. + networks: [corpus] + depends_on: [site] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"] + interval: 10s + timeout: 3s + start_period: 90s + retries: 3 +volumes: + mcp-cache: +networks: + corpus: + internal: true + publish: {} diff --git a/deploy/container/site.conf b/deploy/container/site.conf new file mode 100644 index 0000000..07eb206 --- /dev/null +++ b/deploy/container/site.conf @@ -0,0 +1,50 @@ +worker_processes 1; +pid /tmp/nginx.pid; +error_log /dev/stderr warn; +events { worker_connections 512; } +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /dev/stdout; + client_body_temp_path /tmp/client; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + # Optional MCP HTTP profile, reached only through the published loopback port. + # Deferred DNS permits a site-only launch with no MCP container. + server { + listen 8001; + resolver 127.0.0.11 valid=10s ipv6=off; + location / { + set $mcp_backend mcp:8000; + proxy_pass http://$mcp_backend; + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_read_timeout 65s; + client_max_body_size 512k; + } + } + server { + listen 8000; + server_name _; + root /srv/site; + index index.html; + autoindex off; + # Match under any operator-selected site prefix. Never cache a pointer + # based on normalized mtimes, or a 404 for an immutable object. + location ~ /ai/mcp/v1/manifest\.json$ { + etag off; + if_modified_since off; + add_header Cache-Control "no-store" always; + try_files $uri =404; + } + location ~ "/ai/mcp/v1/[0-9a-f]{64}/snapshot\.tar\.gz$" { + types { application/gzip gz; } + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + location / { try_files $uri $uri/ =404; } + } +} diff --git a/deploy/docker-catalog/server.yaml b/deploy/docker-catalog/server.yaml new file mode 100644 index 0000000..f768c18 --- /dev/null +++ b/deploy/docker-catalog/server.yaml @@ -0,0 +1,41 @@ +# Docker mcp-registry source schema (pkg/servers/types.go), NOT a submitted entry. +# Release publication replaces release-pending and RELEASE_SOURCE_SHA with the +# verified multi-platform index digest and its matching source commit. +name: v8std +image: ghcr.io/zeegin/v8std-mcp:release-pending +type: server +longLived: true +meta: + category: developer-tools + tags: [1c, standards] +about: + title: V8std + description: Read-only 1C standards, diagnostics and snippet guidance. + icon: https://v8std.ru/assets/images/icon-192.png +source: + project: https://github.com/zeegin/v8std + commit: RELEASE_SOURCE_SHA + dockerfile: Dockerfile.mcp +run: + user: "10001:10001" + command: [--transport, stdio] + env: + V8STD_MCP_SITE_URL: '{{v8std.site_url}}' + V8STD_MCP_MAX_SNIPPET_CHARS: '{{v8std.max_snippet_chars}}' + volumes: ['{{v8std.cache_volume}}:/var/lib/v8std-mcp'] +config: + description: Snapshot source, response links and persistent verified cache. + parameters: + type: object + properties: + site_url: + type: string + default: https://v8std.ru/ + max_snippet_chars: + type: integer + description: Inclusive range 4000 to 32000 characters; validated by runtime. + default: 4000 + cache_volume: + type: string + default: v8std-mcp-cache + required: [site_url, max_snippet_chars, cache_volume] diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 9a1d1af..d5471e2 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,3 +1,5 @@ +# DEVELOPMENT ONLY: source bind mounts and build-on-start; use ../compose.yaml +# for the release images and verified snapshot cache. services: zensical: container_name: zensical diff --git a/docker-compose/docker/Dockerfile b/docker-compose/docker/Dockerfile index 083162a..245355d 100644 --- a/docker-compose/docker/Dockerfile +++ b/docker-compose/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea WORKDIR /opt/v8std diff --git a/docs/container-installation.md b/docs/container-installation.md new file mode 100644 index 0000000..f977466 --- /dev/null +++ b/docs/container-installation.md @@ -0,0 +1,178 @@ +--- +title: Локальная установка в контейнерах +llms: + ignore: true +--- + +# Локальная установка в контейнерах + +Поставка состоит из двух независимых образов: `ghcr.io/zeegin/v8std-mcp` +с MCP runtime и `ghcr.io/zeegin/v8std-site` с готовым локальным сайтом. +Поддерживаемые архитектуры — `linux/arm64` и `linux/amd64`. + +**Первый выпуск ещё не опубликован.** Команды ниже требуют действительных +image references из проверенного выпуска. Наличие Dockerfile или шаблона +Catalog не означает публикацию в GHCR или принятие Docker team. Не используйте +`release-pending` как установленную версию. Для локальной проверки разработчик +может явно передать теги своих тестовых образов вместо release references. + +## Сайт и MCP HTTP + +Получите `compose.yaml` из той же версии исходников и укажите проверенные +references вида `ghcr.io/zeegin/v8std-site@sha256:…` и +`ghcr.io/zeegin/v8std-mcp@sha256:…`. Значение digest относится к опубликованному +multi-platform index; локальный Docker image ID не заменяет этот digest. + +```bash +export V8STD_SITE_IMAGE='ghcr.io/zeegin/v8std-site@sha256:RELEASE_INDEX_DIGEST' +export V8STD_MCP_IMAGE='ghcr.io/zeegin/v8std-mcp@sha256:RELEASE_INDEX_DIGEST' +docker compose -p v8std-local -f compose.yaml up -d site +docker compose -p v8std-local -f compose.yaml --profile mcp up -d +curl --fail http://v8std.localhost:18765/ai/mcp/v1/manifest.json +curl --fail http://127.0.0.1:18766/healthz +``` + +Сайт открывается по адресу `http://v8std.localhost:18765/`, MCP — по адресу +`http://127.0.0.1:18766/mcp`. До подготовки проверенного поколения `/healthz` +отвечает `503`; `/livez` показывает только жизнь процесса. В `tools/list` +доступны пять инструментов, в `resources/list` — три Resources. + +`v8std.localhost` разрешается браузером в loopback, а внутри сети Compose +служит DNS alias сайта. Сайт подключён к внутренней сети `corpus` и обычной +сети `publish`, чтобы Docker Desktop мог опубликовать loopback ports. MCP +подключён только к `corpus`; его HTTP доступен через proxy статического сервера +на отдельном loopback port. Docker socket внутри MCP отсутствует. + +Можно изменить `V8STD_SITE_PORT` и `V8STD_MCP_PORT`, выбрав свободные порты. +Site port одинаков снаружи и внутри контейнера. Для образа сайта, подготовленного +под `/kb/`, задайте `V8STD_SITE_PREFIX=/kb/`. Один полный site URL одновременно +определяет источник manifest/archive и ссылки в ответах MCP. У него нет +скрытой публичной альтернативы. Префикс образа и настройка Compose должны совпадать. + +Перед использованием на другой машине проверьте URL из браузера и контейнера: + +```bash +curl --fail http://v8std.localhost:18765/ai/mcp/v1/manifest.json +docker run --rm --network v8std-local_corpus --read-only --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint python "$V8STD_MCP_IMAGE" \ + -c "import urllib.request; print(urllib.request.urlopen('http://v8std.localhost:18765/ai/mcp/v1/manifest.json', timeout=5).status)" +``` + +На Linux не полагайтесь на автоматическое существование `host.docker.internal`. +Если используете собственный доступный контейнерам host/LAN-сайт, явно проверьте +host-gateway и HTTP; успешное разрешение имени само по себе недостаточно: + +```bash +docker run --rm --add-host host.docker.internal:host-gateway \ + --read-only --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint python "$V8STD_MCP_IMAGE" \ + -c "import socket,urllib.request; print(socket.gethostbyname('host.docker.internal')); print(urllib.request.urlopen('http://host.docker.internal:18765/ai/mcp/v1/manifest.json',timeout=5).status)" +``` + +Для сайта, опубликованного **только** на `127.0.0.1`, последний HTTP probe +на Linux может не пройти: gateway IP не является host loopback. Не меняйте +bind/DNS автоматически. Основная схема выше использует общий `.localhost` +адрес и внутренний DNS alias; native Linux acceptance проверяется отдельно +от amd64-эмуляции Docker Desktop. + +## Только MCP через stdio + +Runtime image не содержит corpus или генератор документации. Для первого +запуска нужен доступный сайт с v1 snapshot. Public source по умолчанию — +`https://v8std.ru/`; первый выпуск может работать с ним только после публикации +`ai/mcp/v1/manifest.json` и указанного archive. + +```bash +docker volume create v8std-mcp-cache +docker run --rm -i --init --read-only --cap-drop ALL \ + --security-opt no-new-privileges --memory 1536m --cpus 2 --pids-limit 128 \ + --tmpfs /tmp:rw,noexec,nosuid,size=64m,uid=10001,gid=10001 \ + -v v8std-mcp-cache:/var/lib/v8std-mcp "$V8STD_MCP_IMAGE" +``` + +Для локального сайта добавьте `--network v8std-local_corpus` и +`-e V8STD_MCP_SITE_URL=http://v8std.localhost:18765/` **перед** именем образа. +Default CMD выбирает stdio; `stdout` предназначен только для MCP JSON-RPC. +EOF завершает runtime. Для HTTP передайте после имени образа +`--transport streamable-http --host 0.0.0.0 --port 8000` и опубликуйте порт +только на loopback либо используйте Compose. + +UID/GID runtime — `10001:10001`, cache — `/var/lib/v8std-mcp`. +Новый named volume наследует владельца каталога из образа. Для существующего +bind mount оператор должен заранее обеспечить права этого UID/GID; +контейнер не выполняет `chown` при старте. Не удаляйте cache для обычного restart. +`docker compose down -v` удаляет проверенный cache и лишает следующий запуск +возможности работать offline. + +С сохранённым cache и тем же SITE_URL warm restart работает без сети. +Cold-offline запуск без cache остаётся неготовым; это не самодостаточный образ +со встроенным корпусом. Другой SITE_URL создаёт другую cache namespace. +Общий volume экономит загрузки, но процессы разных агентов сохраняют отдельные +поколения индекса в памяти. Указанные memory/CPU limits — параметры проверок, +не обещание пропускной способности. + +`--site-url`, `--cache-dir` и `--max-snippet-chars` имеют приоритет над +`V8STD_MCP_SITE_URL`, `V8STD_MCP_CACHE_DIR` и `V8STD_MCP_MAX_SNIPPET_CHARS`. +Snippet default — 4000, максимум — 32000 символов. `--refresh-seconds 0` +отключает периодический refresh, но не первую попытку обновления после старта. +Старые `--index-url`/`--vectors-url` не являются настройками нового образа; +не смешивайте их с `--site-url`. Прямой Python CLI для разработки по-прежнему +по умолчанию выбирает HTTP `127.0.0.1:8765`. + +## Docker MCP Catalog и Gateway + +`deploy/docker-catalog/server.yaml` — шаблон в формате исходников Docker +mcp-registry. Перед выпуском его image reference и `source.commit` должны +быть заменены подтверждёнными digest/SHA. Это образ проекта v8std, не Docker +Official Image. CC0 проекта не заменяет лицензии зависимостей и корпуса: +три опубликованных текста доступны по `/LICENSES/`, атрибуция — на странице +[сторонних материалов](THIRD_PARTY_DIAGNOSTIC_ARTICLES.md). + +Catalog объявляет site URL, snippet limit, shared cache volume и `longLived`. +Gateway может запускать отдельный контейнер для каждой сессии. Host CLI +Gateway v0.43.3 не имеет флага подключения к дополнительной сети. Для локального +источника требуется отдельно проверенная маршрутизация containerized Gateway; +его доступ к Docker API требует полномочий оператора. Не монтируйте socket в MCP. + +В v0.43.3 Gateway задаёт `init` и `no-new-privileges`, но не поддерживает +read-only rootfs, cap-drop и tmpfs через эту схему Catalog. Поэтому прохождение +его lifecycle-теста не подтверждает весь hardening-контракт. Используйте direct +Docker/Compose, когда нужны перечисленные ограничения; внешняя приёмка Catalog, +registry publication и native CI остаются отдельными проверками. + +## Проверка разработчиком + +Старый `docker-compose/docker-compose.yml` — dev-only: он монтирует исходники +и может генерировать индекс при старте. Для проверки release path используйте +`Dockerfile.mcp`, `Dockerfile.site` и два hash-lock файла. `local-builder` в +`Dockerfile.site` содержит pinned Python/Zensical и зависимости без apt install. +Он потребляет подготовленные canonical `docs/ai/*` и `docs/llms*.txt`. + +`scripts/build_local_site.py --output NEW_DIRECTORY --site-url URL --source-sha SHA` +работает во временной копии входов. Он не вызывает public wrapper и не меняет +canonical docs/site. Local HTML отключает аналитику, recorder, внешние шрифты +и запросы статистики GitHub. Snapshot использует неизменённый canonical producer; +его archive bytes совпадают с public producer для тех же входов и Python/zlib. +Build-time установка зависимостей требует сети; подготовленный builder может +построить local profile с `--network none`. + +Статический образ собирается с именованным context +`--build-context local-site=NEW_DIRECTORY`; для base prefix используйте +`--build-arg SITE_PREFIX=kb`. Запускайте focused проверки: + +```bash +V8STD_TEST_LOCAL_BUILD=1 .venv/bin/python -m unittest \ + tests.test_v8std_mcp_distribution tests.test_published_license_links -v +.venv/bin/python scripts/check_mcp_container.py \ + --mcp-image LOCAL_MCP_IMAGE --site-image LOCAL_SITE_IMAGE \ + --platform linux/arm64 --prefix /kb/ \ + --chrome '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' \ + --node /opt/homebrew/bin/node --host-gateway +``` + +Harness проверяет реальные HTTP/stdio, холодный и тёплый cache, владельца volume, +сигналы завершения, snippet rule, ссылки с prefix, cache headers и запросы Chrome. +Опция `--host-gateway` использует два изолированных Gateway и warm cache без +сети; она не подменяет проверку сети containerized Gateway. Harness выбирает +уникальные имена ресурсов, проверяет свободные порты и удаляет только свои +контейнеры, сети, cache и временный Catalog. Образы остаются для анализа. diff --git a/docs/mcp.md b/docs/mcp.md index 22200c5..edd3728 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -115,6 +115,9 @@ Resources являются additive-профилем для клиентов, к Не отправляйте в публичный сервис закрытый код. Для такого сценария используйте локальный запуск MCP. +Подготовка локального сайта, stdio/HTTP, persistent cache и ограничения Gateway +описаны в [инструкции установки в контейнерах](container-installation.md). + ### `v8std_explain_diagnostics` Объясняет список диагностик АПК, BSL Language Server и EDT. diff --git a/docs/support.md b/docs/support.md index 2f97de5..d37ec00 100644 --- a/docs/support.md +++ b/docs/support.md @@ -91,7 +91,11 @@ hide: публичный сервис, работаете без доступа к интернету или проверяете изменения сайта до публикации. -Самый простой запуск — через Docker: +Поставка с готовыми runtime/static images описана в +[инструкции установки в контейнерах](container-installation.md). +Её первый выпуск и external Catalog acceptance требуют отдельных проверок. + +Для **разработки из исходников** сохранён Docker Compose с bind mount: ```bash git clone https://github.com/zeegin/v8std.git diff --git a/overrides/main.html b/overrides/main.html index 664f04f..086e1e3 100644 --- a/overrides/main.html +++ b/overrides/main.html @@ -58,7 +58,9 @@ {% block extrahead %} {{ super() }} + {% if not config.extra.local_publication %} + {% endif %} {% include "partials/social_meta.html" ignore missing %} {% endblock %} diff --git a/requirements-build.lock b/requirements-build.lock new file mode 100644 index 0000000..e5292f2 --- /dev/null +++ b/requirements-build.lock @@ -0,0 +1,393 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements-build.lock --constraint /dev/fd/11 --generate-hashes --universal --python-version 3.12 -o requirements-build.lock +click==8.4.2 \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via + # -c /dev/fd/11 + # zensical +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via click +deepmerge==2.1.0 \ + --hash=sha256:07ca7a7b8935df596c512fa8161877c0487ac61f691c07766e7d71d2b23bdd2f \ + --hash=sha256:8f148339a91d680a75ecb74ade235d9e759a93df373a0b04e9d31c8666cfeb75 + # via + # -c /dev/fd/11 + # zensical +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -c /dev/fd/11 + # zensical +markdown==3.10.2 \ + --hash=sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950 \ + --hash=sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36 + # via + # -c /dev/fd/11 + # pymdown-extensions + # zensical +markdown-it-py==4.0.0 \ + --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \ + --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 + # via + # -c /dev/fd/11 + # -r requirements-build.lock +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via + # -c /dev/fd/11 + # jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c /dev/fd/11 + # markdown-it-py +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via + # -c /dev/fd/11 + # -r requirements-build.lock +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via + # -c /dev/fd/11 + # pygments-bsl + # zensical +pygments-bsl==1.1.0 \ + --hash=sha256:7c736ceff465a40704c94b76374746a34001d9755ac93d3b0cbe37e2d583acb5 \ + --hash=sha256:871114e9cbcac74e6583e367eac847bdb8c814267571b38336ca4accbd71539f + # via + # -c /dev/fd/11 + # -r requirements-build.lock +pymdown-extensions==11.0.1 \ + --hash=sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2 \ + --hash=sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0 + # via + # -c /dev/fd/11 + # zensical +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c /dev/fd/11 + # -r requirements-build.lock + # pymdown-extensions + # zensical +tomli==2.4.1 \ + --hash=sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853 \ + --hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe \ + --hash=sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5 \ + --hash=sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d \ + --hash=sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd \ + --hash=sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26 \ + --hash=sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54 \ + --hash=sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6 \ + --hash=sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c \ + --hash=sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a \ + --hash=sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd \ + --hash=sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f \ + --hash=sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5 \ + --hash=sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9 \ + --hash=sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662 \ + --hash=sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9 \ + --hash=sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1 \ + --hash=sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585 \ + --hash=sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e \ + --hash=sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c \ + --hash=sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41 \ + --hash=sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f \ + --hash=sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085 \ + --hash=sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15 \ + --hash=sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7 \ + --hash=sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c \ + --hash=sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36 \ + --hash=sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076 \ + --hash=sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac \ + --hash=sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8 \ + --hash=sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232 \ + --hash=sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece \ + --hash=sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a \ + --hash=sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897 \ + --hash=sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d \ + --hash=sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4 \ + --hash=sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917 \ + --hash=sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396 \ + --hash=sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a \ + --hash=sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc \ + --hash=sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba \ + --hash=sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f \ + --hash=sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257 \ + --hash=sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30 \ + --hash=sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf \ + --hash=sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9 \ + --hash=sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049 + # via + # -c /dev/fd/11 + # zensical +zensical==0.0.47 \ + --hash=sha256:1bd94937c48a2e42b5b65b32c5075849937f23cebccaf250d249efa27266e0be \ + --hash=sha256:2ee29ff819372eaab02ca0f14ac82e804332d898c57c56ffd4d89674f3e5ff71 \ + --hash=sha256:319cf370ecc6d87da69c935c5acd6e1959dd48954766e9f954319bdb0bec5d36 \ + --hash=sha256:324f783b22cd0deed0d0f3b69e28d5380b47238a1ef0f25913b88014af2bade2 \ + --hash=sha256:4162fb8b62f38e6d9b75688c1fb87e18a1cffef5eee51d2c18b79334b548e973 \ + --hash=sha256:4702605b991bece11494a9bb318d5ba7229f00e5adcc9e6010acd58e9719686b \ + --hash=sha256:59656bf604a8b03eede4ce1a847640bab1129ab86dec2e39e5bd4b808b1802d2 \ + --hash=sha256:6f6b2de477c45284201e92301f997415f45f78493bd20f479ea9a1c86bbbbcca \ + --hash=sha256:77f08ffcc3da9ca2f972330e501927aa7e8e445bfa7b758107edc645acda266e \ + --hash=sha256:81a13a8bacadedada4847eed4aaa4a3f4ef0e5b78dbb2a2022bc6fb7e7dc9464 \ + --hash=sha256:944a309be69b11daa8bba46c61fb74f32a98b637f3e7c11135e7cc2a5eccbe32 \ + --hash=sha256:97ed2b21aba5f788fc39d1597d00938d602b4d2724ed599a1dab7548fe4f0025 \ + --hash=sha256:cdc2d84f38da809a28402eda5f2b6dbb150e14427f296c5b521e0dfc6a2e8a39 + # via + # -c /dev/fd/11 + # -r requirements-build.lock diff --git a/requirements-mcp.lock b/requirements-mcp.lock new file mode 100644 index 0000000..b791b90 --- /dev/null +++ b/requirements-mcp.lock @@ -0,0 +1,685 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements-mcp.txt --constraint /dev/fd/11 --generate-hashes --universal --python-version 3.12 -o requirements-mcp.lock +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ + --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 + # via + # -c /dev/fd/11 + # pydantic +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via + # -c /dev/fd/11 + # httpx + # mcp + # sse-starlette + # starlette +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via + # -c /dev/fd/11 + # jsonschema + # referencing +certifi==2026.6.17 \ + --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ + --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db + # via + # -c /dev/fd/11 + # httpcore + # httpx +cffi==2.1.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc \ + --hash=sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd \ + --hash=sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d \ + --hash=sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5 \ + --hash=sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f \ + --hash=sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6 \ + --hash=sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c \ + --hash=sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda \ + --hash=sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd \ + --hash=sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a \ + --hash=sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd \ + --hash=sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd \ + --hash=sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3 \ + --hash=sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb \ + --hash=sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66 \ + --hash=sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d \ + --hash=sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f \ + --hash=sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6 \ + --hash=sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0 \ + --hash=sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c \ + --hash=sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93 \ + --hash=sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d \ + --hash=sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d \ + --hash=sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8 \ + --hash=sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b \ + --hash=sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001 \ + --hash=sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d \ + --hash=sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43 \ + --hash=sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b \ + --hash=sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0 \ + --hash=sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0 \ + --hash=sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458 \ + --hash=sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8 \ + --hash=sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d \ + --hash=sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94 \ + --hash=sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022 \ + --hash=sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db \ + --hash=sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479 \ + --hash=sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376 \ + --hash=sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d \ + --hash=sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6 \ + --hash=sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3 \ + --hash=sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea \ + --hash=sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd \ + --hash=sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02 \ + --hash=sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde \ + --hash=sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224 \ + --hash=sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76 \ + --hash=sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804 \ + --hash=sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1 \ + --hash=sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913 \ + --hash=sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714 \ + --hash=sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e \ + --hash=sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda \ + --hash=sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512 \ + --hash=sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28 \ + --hash=sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699 \ + --hash=sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3 \ + --hash=sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c \ + --hash=sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe \ + --hash=sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a \ + --hash=sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f \ + --hash=sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c \ + --hash=sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2 \ + --hash=sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f \ + --hash=sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 \ + --hash=sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056 \ + --hash=sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629 \ + --hash=sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7 \ + --hash=sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0 \ + --hash=sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9 \ + --hash=sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853 \ + --hash=sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13 \ + --hash=sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a \ + --hash=sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4 \ + --hash=sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce \ + --hash=sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac \ + --hash=sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c \ + --hash=sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46 \ + --hash=sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384 \ + --hash=sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b \ + --hash=sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210 \ + --hash=sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc \ + --hash=sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a \ + --hash=sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5 \ + --hash=sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7 \ + --hash=sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2 \ + --hash=sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326 \ + --hash=sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f \ + --hash=sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca \ + --hash=sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98 \ + --hash=sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9 \ + --hash=sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5 \ + --hash=sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7 \ + --hash=sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc \ + --hash=sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da \ + --hash=sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f + # via + # -c /dev/fd/11 + # cryptography +click==8.4.2 ; sys_platform != 'emscripten' \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via + # -c /dev/fd/11 + # uvicorn +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via click +cryptography==49.0.0 \ + --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ + --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ + --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ + --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ + --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ + --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ + --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ + --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ + --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ + --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ + --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ + --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ + --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ + --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ + --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ + --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ + --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ + --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ + --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ + --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ + --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ + --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ + --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ + --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ + --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ + --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ + --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ + --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ + --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ + --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ + --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ + --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ + --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ + --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ + --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ + --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ + --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ + --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ + --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ + --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ + --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ + --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ + --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ + --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ + --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ + --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b + # via + # -c /dev/fd/11 + # pyjwt +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c /dev/fd/11 + # httpcore + # uvicorn +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via + # -c /dev/fd/11 + # httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via + # -c /dev/fd/11 + # mcp +httpx-sse==0.4.3 \ + --hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \ + --hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d + # via + # -c /dev/fd/11 + # mcp +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # -c /dev/fd/11 + # anyio + # httpx +jsonschema==4.26.0 \ + --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce + # via + # -c /dev/fd/11 + # mcp +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ + --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d + # via + # -c /dev/fd/11 + # jsonschema +markdown-it-py==4.0.0 \ + --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \ + --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 + # via + # -c /dev/fd/11 + # -r requirements-mcp.txt +mcp==1.27.0 \ + --hash=sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741 \ + --hash=sha256:d3dc35a7eec0d458c1da4976a48f982097ddaab87e278c5511d5a4a56e852b83 + # via + # -c /dev/fd/11 + # -r requirements-mcp.txt +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c /dev/fd/11 + # markdown-it-py +pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via + # -c /dev/fd/11 + # cffi +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 + # via + # -c /dev/fd/11 + # mcp + # pydantic-settings +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae + # via + # -c /dev/fd/11 + # pydantic +pydantic-settings==2.14.2 \ + --hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \ + --hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f + # via + # -c /dev/fd/11 + # mcp +pyjwt==2.13.0 \ + --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 + # via + # -c /dev/fd/11 + # mcp +python-dotenv==1.2.2 \ + --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \ + --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3 + # via + # -c /dev/fd/11 + # pydantic-settings +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via + # -c /dev/fd/11 + # mcp +pywin32==312 ; sys_platform == 'win32' \ + --hash=sha256:02ebca0f0242b75292e218065004310d6a477407c09fa449bfe4f6022bc0c0fc \ + --hash=sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c \ + --hash=sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd \ + --hash=sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831 \ + --hash=sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed \ + --hash=sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db \ + --hash=sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950 \ + --hash=sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e \ + --hash=sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c \ + --hash=sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa \ + --hash=sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e \ + --hash=sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b \ + --hash=sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9 \ + --hash=sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47 \ + --hash=sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc \ + --hash=sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5 \ + --hash=sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9 \ + --hash=sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a \ + --hash=sha256:d620900033cc7531e50727c3c8333091df5dd3ffe6d68cdca38c03f5821408d5 \ + --hash=sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b \ + --hash=sha256:dc90147579a905b8635e1b0ec6514967dcb07e6e0d9c42f1477feef14cac23bb + # via mcp +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c /dev/fd/11 + # -r requirements-mcp.txt +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ + --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 + # via + # -c /dev/fd/11 + # jsonschema + # jsonschema-specifications +rpds-py==2026.6.3 \ + --hash=sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5 \ + --hash=sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680 \ + --hash=sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9 \ + --hash=sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538 \ + --hash=sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804 \ + --hash=sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf \ + --hash=sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4 \ + --hash=sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97 \ + --hash=sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6 \ + --hash=sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96 \ + --hash=sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a \ + --hash=sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187 \ + --hash=sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975 \ + --hash=sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f \ + --hash=sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703 \ + --hash=sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9 \ + --hash=sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127 \ + --hash=sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f \ + --hash=sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa \ + --hash=sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05 \ + --hash=sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171 \ + --hash=sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba \ + --hash=sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c \ + --hash=sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223 \ + --hash=sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4 \ + --hash=sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885 \ + --hash=sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698 \ + --hash=sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f \ + --hash=sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7 \ + --hash=sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed \ + --hash=sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f \ + --hash=sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf \ + --hash=sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e \ + --hash=sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f \ + --hash=sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24 \ + --hash=sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a \ + --hash=sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41 \ + --hash=sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc \ + --hash=sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d \ + --hash=sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146 \ + --hash=sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e \ + --hash=sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e \ + --hash=sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4 \ + --hash=sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12 \ + --hash=sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7 \ + --hash=sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261 \ + --hash=sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6 \ + --hash=sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5 \ + --hash=sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93 \ + --hash=sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7 \ + --hash=sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda \ + --hash=sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8 \ + --hash=sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342 \ + --hash=sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c \ + --hash=sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb \ + --hash=sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0 \ + --hash=sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77 \ + --hash=sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3 \ + --hash=sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885 \ + --hash=sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826 \ + --hash=sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617 \ + --hash=sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb \ + --hash=sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577 \ + --hash=sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80 \ + --hash=sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e \ + --hash=sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945 \ + --hash=sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90 \ + --hash=sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7 \ + --hash=sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0 \ + --hash=sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140 \ + --hash=sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822 \ + --hash=sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba \ + --hash=sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9 \ + --hash=sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4 \ + --hash=sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a \ + --hash=sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8 \ + --hash=sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf \ + --hash=sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4 \ + --hash=sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324 \ + --hash=sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53 \ + --hash=sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b \ + --hash=sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41 \ + --hash=sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9 \ + --hash=sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca \ + --hash=sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1 \ + --hash=sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d \ + --hash=sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690 \ + --hash=sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107 \ + --hash=sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2 \ + --hash=sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76 \ + --hash=sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d \ + --hash=sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af \ + --hash=sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6 \ + --hash=sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db \ + --hash=sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369 \ + --hash=sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd \ + --hash=sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911 \ + --hash=sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504 \ + --hash=sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a \ + --hash=sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9 \ + --hash=sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13 \ + --hash=sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc \ + --hash=sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278 \ + --hash=sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868 \ + --hash=sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2 \ + --hash=sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd \ + --hash=sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4 \ + --hash=sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6 \ + --hash=sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9 \ + --hash=sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00 \ + --hash=sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f \ + --hash=sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e \ + --hash=sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442 \ + --hash=sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da \ + --hash=sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90 \ + --hash=sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef + # via + # -c /dev/fd/11 + # jsonschema + # referencing +sse-starlette==3.4.6 \ + --hash=sha256:56217ab4c9a9f9c5db7b21e08732d3e7c2b807f45231ad23de0551a24c4a41f6 \ + --hash=sha256:725f8a1bd6d26ae1b2c9610c0ef5065dfdd496f3988d28adcf8c4b49dc25c627 + # via + # -c /dev/fd/11 + # mcp +starlette==1.3.1 \ + --hash=sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0 \ + --hash=sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6 + # via + # -c /dev/fd/11 + # mcp + # sse-starlette +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -c /dev/fd/11 + # anyio + # mcp + # pydantic + # pydantic-core + # referencing + # starlette + # typing-inspection +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 + # via + # -c /dev/fd/11 + # mcp + # pydantic + # pydantic-settings +uvicorn==0.51.0 ; sys_platform != 'emscripten' \ + --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ + --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 + # via + # -c /dev/fd/11 + # mcp diff --git a/scripts/build_local_site.py b/scripts/build_local_site.py new file mode 100644 index 0000000..0ebf221 --- /dev/null +++ b/scripts/build_local_site.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Build local HTML from prepared canonical inputs in an isolated staging tree. + +Does not call zensical_docs.sh: that wrapper regenerates the canonical corpus. +The release builder must prepare docs/ai and llms files before this step. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib + +from generate_mcp_snapshot import publish_snapshot +from v8std_mcp_snapshot_format import normalize_site_url + + +def local_config(text: str, site_url: str) -> str: + text = re.sub(r'^(repo_url|repo_name|edit_uri) = .*\n', '', text, flags=re.M) + text = re.sub(r'^site_url = .*$', 'site_url = ' + json.dumps(site_url), text, count=1, flags=re.M) + text = text.replace('[project.theme]\n', '[project.theme]\nfont = false\n') + text = re.sub(r'\[project.theme.font\]\n.*?(?=\n\[|\Z)', '', text, flags=re.S) + text = re.sub(r'\[project.extra.consent(?:\.[^\]]+)?\]\n.*?(?=\n\[|\Z)', '', text, flags=re.S) + text = text.replace('[project.extra]\n', '[project.extra]\nlocal_publication = true\n') + text = text.replace('[project.plugins.social]\nenabled = true', + '[project.plugins.social]\nenabled = false') + config = tomllib.loads(text)["project"] + assert config["theme"]["font"] is False + assert config["extra"]["local_publication"] is True + return text + + +def build_local_site(root: Path, output: Path, site_url: str, source_sha: str) -> Path: + root, output = root.resolve(), output.resolve() + # Never replace a checkout input, public output, ancestor, or existing result. + if output == root or output in root.parents or any( + output == root / name or root / name in output.parents + for name in ("docs", "scripts", "overrides", "site", "spec", "LICENSES")): + raise ValueError("local output overlaps source or canonical site") + if output.exists(): + raise ValueError("local output must not exist") + site_url = normalize_site_url(site_url) + if not re.fullmatch(r"[0-9a-f]{40}", source_sha): + raise ValueError("source SHA must be 40 lowercase hexadecimal characters") + # All producers/build hooks run against this copy; no symlink back to inputs. + with tempfile.TemporaryDirectory(prefix="v8std-local-build-") as temporary: + stage = Path(temporary) + for name in ("docs", "scripts", "overrides", "LICENSES"): + shutil.copytree(root / name, stage / name, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + for name in ("zensical.toml", "retrieval-rules.yml", "LICENSE"): + shutil.copy2(root / name, stage / name) + env = {**os.environ, "V8STD_REPO_ROOT": str(stage), "PYTHONPATH": str(stage)} + def run(*args: str): + subprocess.run([sys.executable, *args], cwd=stage, env=env, check=True) + + # Generate Markdown sidecars against canonical config without overwriting + # any AI inputs. Snapshot bytes are the unchanged canonical producer's. + run("-c", "from pathlib import Path; from scripts.generate_ai_artifacts import " + "build_site_ai_index, write_site_markdown_pages; " + "write_site_markdown_pages(build_site_ai_index(Path.cwd()), Path('sidecars'))") + canonical = tomllib.loads((stage / "zensical.toml").read_text())["project"]["site_url"] + publish_snapshot(stage / "docs", stage / "snapshot", source_sha, canonical) + (stage / "zensical.toml").write_text(local_config( + (stage / "zensical.toml").read_text(), site_url), encoding="utf-8") + run("-m", "zensical", "build", "--strict") + # Do not run the unrelated article-HTML protection gate here; the final + # public strict wrapper retains that gate. This task checks local egress. + shutil.copytree(stage / "sidecars", stage / "site", dirs_exist_ok=True) + shutil.copytree(stage / "snapshot", stage / "site/ai/mcp/v1", dirs_exist_ok=True) + run("scripts/publish_license_texts.py", "--root", str(stage), "--site", str(stage / "site")) + output.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(stage / "site", output) + return output + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--site-url", required=True) + parser.add_argument("--source-sha", required=True) + args = parser.parse_args() + print(build_local_site(args.root, args.output, args.site_url, args.source_sha)) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_mcp_container.py b/scripts/check_mcp_container.py new file mode 100644 index 0000000..24a0c29 --- /dev/null +++ b/scripts/check_mcp_container.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Disposable real-image acceptance; never edits an active Docker catalog. + +Build images first. Requires Docker, Python runtime dependencies, and (with +--chrome) Chrome plus Node 22+. Each run removes only its own labeled resources. +""" +from __future__ import annotations + +import argparse +import contextlib +from functools import cache +import hashlib +import json +import os +from pathlib import Path +import queue +import socket +import subprocess +import sys +import tempfile +import threading +import time +from urllib.error import HTTPError +from urllib.request import Request, urlopen +import uuid + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +INIT = {"protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "v8std-container-check", "version": "1"}} +SIGNAL = 'Предупреждение("Текст");' +TOOLS = {"v8std_search", "v8std_get_page", "v8std_get_related", + "v8std_explain_snippet", "v8std_explain_diagnostics"} + + +@cache +def canonical_ranking(): + sys.path.insert(0, str(ROOT / "scripts")) + from v8std_mcp_index import V8StdIndex + index = V8StdIndex(pages_path=ROOT / "docs/ai/pages.jsonl", + vectors_path=ROOT / "docs/ai/search-vectors.jsonl") + index.load() + return [row["id"] for row in index.search("модальные окна", limit=5)["results"]] + + +def run(*args, timeout=180, **kwargs): + return subprocess.check_output(list(map(str, args)), text=True, timeout=timeout, **kwargs).strip() + + +def eventually(check, seconds=120): + end = time.monotonic() + seconds + last = None + while time.monotonic() < end: + try: + result = check() + if result: + return result + except (OSError, ValueError, AssertionError, subprocess.CalledProcessError) as error: + last = error + time.sleep(1) + raise AssertionError(f"readiness deadline: {last}") + + +def http(url, message=None, headers=None): + req = Request(url, data=json.dumps(message).encode() if message else None, + headers={"Accept": "application/json, text/event-stream", + "Content-Type": "application/json", **(headers or {})}) + try: + response = urlopen(req, timeout=10) + except HTTPError as error: + response = error + with response: + data = response.read() + if data.startswith(b"event:"): + data = next(line[6:] for line in data.splitlines() if line.startswith(b"data: ")) + return response.status, dict(response.headers), data + + +class Stdio: + def __init__(self, command, stderr, env=None): + self.process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=stderr, text=True, bufsize=1, env=env) + self.lines = queue.Queue() + self.seq = 0 + def read(): + for line in self.process.stdout: + self.lines.put(line) + self.lines.put(None) + self.reader = threading.Thread(target=read, daemon=True) + self.reader.start() + + def request(self, method, params=None): + self.seq += 1 + self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "id": self.seq, + "method": method, "params": params or {}}) + "\n") + self.process.stdin.flush() + while True: + line = self.lines.get(timeout=120) + assert line is not None, "premature stdout EOF" + message = json.loads(line) # Every stdout line must be protocol JSON. + if message.get("id") == self.seq: + assert "error" not in message, message + return message["result"] + + def initialize(self, name="v8std"): + reply = self.request("initialize", INIT) + if name: + assert reply["serverInfo"]["name"] == name, reply + self.process.stdin.write('{"jsonrpc":"2.0","method":"notifications/initialized"}\n') + self.process.stdin.flush() + return reply + + def close(self): + self.process.stdin.close() + code = self.process.wait(timeout=20) + self.reader.join(timeout=2) + self.process.stdout.close() + assert code == 0, f"stdio EOF exit={code}" + + +def content(reply): + assert not reply.get("isError"), str(reply)[:400] + return reply.get("structuredContent") or json.loads(reply["content"][0]["text"]) + + +def check_tools(request, site_url, *, resources=True): + listed = request("tools/list") + names = {tool["name"] for tool in listed["tools"]} + assert TOOLS <= names, names + if resources: + assert names == TOOLS + assert len(request("resources/list")["resources"]) == 3 + snippet = next(t for t in listed["tools"] if t["name"] == "v8std_explain_snippet") + assert snippet["inputSchema"]["properties"]["snippet"]["maxLength"] == 4000 + def call(name, args): + return content(request("tools/call", {"name": name, "arguments": args})) + search = eventually(lambda: call("v8std_search", {"query": "std437", "limit": 3})) + assert search["results"][0]["id"] == "std437", search + assert search["results"][0]["url"] == site_url + "std/437/", search + ranking = call("v8std_search", {"query": "модальные окна", "limit": 5}) + assert [row["id"] for row in ranking["results"]] == canonical_ranking(), ranking + result = call("v8std_explain_snippet", {"snippet": SIGNAL, "limit": 1}) + ids = [row["id"] for field in ("diagnostics", "standards") for row in result[field]] + assert ids == ["bslls:UsingModalWindows"], ids + row = (result["diagnostics"] + result["standards"])[0] + assert any(reason.startswith("snippet_signal:") for reason in row["match_reasons"]), row + page = call("v8std_get_page", {"id_or_alias_or_url": "std437"}) + assert page["page"]["url"] == site_url + "std/437/" + return [row["id"] for row in search["results"]] + + +# Browser target + worker events are captured before navigation. Request +# interception blocks every foreign origin and records attempts as failures. +CHROME_CHECK = r""" +const [endpoint, base] = process.argv.slice(1); +const socket = new WebSocket(endpoint); +await new Promise(r => socket.addEventListener('open', r, {once:true})); +let seq=0; const pending=new Map(), attached=new Map(), requests=[], failures=[], statuses=[]; +function send(method,params={},sessionId) { + const id=++seq; + return new Promise((resolve,reject)=>{ + pending.set(id,{resolve,reject}); + socket.send(JSON.stringify({id,method,params,...(sessionId?{sessionId}:{})})); + }); +} +socket.addEventListener('message',async ({data})=>{ + const m=JSON.parse(data); + if(m.id) { const p=pending.get(m.id); pending.delete(m.id); + if(m.error)p.reject(Error(JSON.stringify(m.error)));else p.resolve(m.result);return; } + const p=m.params; + if(m.method==='Target.attachedToTarget') { + await send('Network.enable',{},p.sessionId); + if(p.targetInfo.type==='page') { + await send('Fetch.enable',{patterns:[{urlPattern:'*'}]},p.sessionId); + await send('Target.setAutoAttach', + {autoAttach:true,waitForDebuggerOnStart:true,flatten:true},p.sessionId); + } + attached.set(p.targetInfo.targetId,p.sessionId); + await send('Runtime.runIfWaitingForDebugger',{},p.sessionId); + } + if(m.method==='Network.requestWillBeSent')requests.push(p.request.url); + if(m.method==='Network.responseReceived')statuses.push({url:p.response.url,status:p.response.status}); + if(m.method==='Fetch.requestPaused') { + const u=p.request.url; + if(/^https?:/.test(u) && new URL(u).origin!==new URL(base).origin) { + failures.push(u);await send('Fetch.failRequest',{requestId:p.requestId,errorReason:'BlockedByClient'},m.sessionId); + }else await send('Fetch.continueRequest',{requestId:p.requestId},m.sessionId); + } +}); +await send('Target.setAutoAttach',{autoAttach:true,waitForDebuggerOnStart:true,flatten:true}); +const {targetId}=await send('Target.createTarget',{url:'about:blank'}); +while(!attached.has(targetId))await new Promise(r=>setTimeout(r,10)); +const sessionId=attached.get(targetId); +await send('Page.enable',{},sessionId); +for (const path of ['', 'std/437/', 'diagnostics/bslls/', 'diagnostics/bslls/UsingModalWindows/', 'LICENSES/']) { + const navigation=await send('Page.navigate',{url:base+path},sessionId); + if(navigation.errorText)throw Error(navigation.errorText); + await new Promise(r=>setTimeout(r,2500)); + await send('Runtime.evaluate',{expression:`document.querySelector('input[data-md-component="search-query"]')?.focus()`},sessionId); + await new Promise(r=>setTimeout(r,1500)); +} +const unique=[...new Set(requests.filter(u=>/^https?:/.test(u)))]; +failures.push(...unique.filter(u=>new URL(u).origin!==new URL(base).origin)); +const bad=statuses.filter(x=>x.status>=400); +console.log(JSON.stringify({requests:unique,blocked:failures,badResponses:bad})); +await send('Target.closeTarget',{targetId});socket.close(); +if(failures.length||bad.length||unique.length<10)process.exitCode=1; +""" + + +def browser_graph(chrome, node, site_url, directory): + with (directory / "chrome.log").open("w") as log: + process = subprocess.Popen([chrome, "--headless=new", "--no-first-run", + "--no-default-browser-check", "--disable-background-networking", "--disable-component-update", + "--password-store=basic", "--remote-debugging-port=0", + "--disable-sync", "--disable-default-apps", "--disable-domain-reliability", + "--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE v8std.localhost, EXCLUDE localhost", + f"--user-data-dir={directory / 'chrome-profile'}", "about:blank"], + stdout=log, stderr=log) + try: + active = directory / "chrome-profile/DevToolsActivePort" + eventually(lambda: active.is_file(), seconds=15) + port, endpoint = active.read_text().splitlines()[:2] + try: + result = run(node, "--input-type=module", "-e", CHROME_CHECK, + f"ws://127.0.0.1:{port}{endpoint}", site_url, timeout=55) + except subprocess.CalledProcessError as error: + raise AssertionError("browser graph: " + error.output[-6000:]) from None + return json.loads(result) + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def host_gateway_check(project, image, volume, site_url, directory): + """Two isolated host Gateways consume the already verified cache offline. + + Containerized Gateway's site-network routing is a separate acceptance gate. + No HOME override, active catalog modification, or extra Docker privileges. + """ + catalog_root = Path.home() / ".docker/mcp/catalogs" + catalog_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=project + "-", dir=catalog_root) as catalog_dir: + catalog_path = Path(catalog_dir) / "catalog.yaml" + source = yaml.safe_load((ROOT / "deploy/docker-catalog/server.yaml").read_text()) + spec = {"name": project, "type": source["type"], "image": image, + "title": source["about"]["title"], "longLived": source["longLived"], + "user": source["run"]["user"], "command": source["run"]["command"], + "disableNetwork": True, "volumes": [volume + ":/var/lib/v8std-mcp"], + "env": [{"name": "V8STD_MCP_SITE_URL", "value": site_url}, + {"name": "V8STD_MCP_MAX_SNIPPET_CHARS", "value": "4000"}]} + catalog_path.write_text(yaml.safe_dump({"registry": {project: spec}})) + for name in ("config.yaml", "registry.yaml", "tools.yaml"): + (directory / name).write_text("{}\n") + (directory / "secrets.env").write_text("") + command = ["docker", "mcp", "gateway", "run", "--catalog", str(catalog_path), + "--config", str(directory / "config.yaml"), "--registry", str(directory / "registry.yaml"), + "--tools-config", str(directory / "tools.yaml"), "--secrets", str(directory / "secrets.env"), + "--servers", project, "--verify-signatures=false", "--watch=false", + "--cpus", "2", "--memory", "1536Mb", "--long-lived", "--transport", "stdio"] + sessions = [] + try: + with contextlib.ExitStack() as stack: + for number in range(2): + log = stack.enter_context((directory / f"gateway-{number}.log").open("w")) + session = Stdio(command, log) + sessions.append(session) + session.initialize(name=None) + check_tools(session.request, site_url, resources=False) + ids = run("docker", "ps", "-q", "--filter", "label=docker-mcp-name=" + project).splitlines() + assert len(ids) == 2, f"expected one long-lived server per session, got {ids}" + time.sleep(3) + for session in sessions: + check_tools(session.request, site_url, resources=False) + assert set(ids) == set(run("docker", "ps", "-q", "--filter", "label=docker-mcp-name=" + project).splitlines()) + states = json.loads(run("docker", "inspect", *ids)) + for state in states: + assert state["Config"]["User"] == "10001:10001" + assert state["Config"]["WorkingDir"] == "/opt/v8std" + assert state["HostConfig"]["NetworkMode"] == "none" + for session in sessions: + session.close() + return {"sessions": 2, "servers": len(ids), "network": "none", + "warm_cache": True, "long_lived_same_ids": True, + "read_only": states[0]["HostConfig"]["ReadonlyRootfs"], + "cap_drop": states[0]["HostConfig"]["CapDrop"]} + finally: + for session in sessions: + if session.process.poll() is None: + session.process.terminate() + try: + session.process.wait(timeout=15) + except subprocess.TimeoutExpired: + session.process.kill() + session.process.wait(timeout=5) + own = run("docker", "ps", "-aq", "--filter", "label=docker-mcp-name=" + project).splitlines() + if own: + run("docker", "rm", "-f", *own) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mcp-image", required=True) + parser.add_argument("--site-image", required=True) + parser.add_argument("--platform", choices=["linux/arm64", "linux/amd64"], required=True) + parser.add_argument("--site-port", type=int, default=18765) + parser.add_argument("--mcp-port", type=int, default=18766) + parser.add_argument("--prefix", default="/") + parser.add_argument("--chrome") + parser.add_argument("--node", default="node") + parser.add_argument("--host-gateway", action="store_true", + help="Two Gateway sessions using warm cache and network none") + args = parser.parse_args() + canonical_ranking() + for port in (args.site_port, args.mcp_port): + with socket.socket() as probe: + # Permit TIME_WAIT from our preceding run, but never an active listener. + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind(("127.0.0.1", port)) + assert args.prefix.startswith("/") and args.prefix.endswith("/") + site_url = f"http://v8std.localhost:{args.site_port}{args.prefix}" + project = "v8std-task4-" + uuid.uuid4().hex[:10] + env = {**os.environ, "V8STD_SITE_IMAGE": args.site_image, "V8STD_MCP_IMAGE": args.mcp_image, + "V8STD_SITE_PORT": str(args.site_port), "V8STD_MCP_PORT": str(args.mcp_port), + "V8STD_SITE_PREFIX": args.prefix, "DOCKER_DEFAULT_PLATFORM": args.platform} + compose = ["docker", "compose", "-p", project, "-f", str(ROOT / "compose.yaml")] + names = [] + report = {"platform": args.platform, "site_url": site_url} + with tempfile.TemporaryDirectory(prefix=project) as directory: + directory = Path(directory) + try: + run(*compose, "up", "-d", "site", env=env) + site = run(*compose, "ps", "-q", "site", env=env) + eventually(lambda: http(site_url)[0] == 200) + manifest_url = site_url + "ai/mcp/v1/manifest.json" + status, headers, payload = http(manifest_url) + assert status == 200 and headers.get("Cache-Control") == "no-store" + manifest = json.loads(payload) + assert not manifest["archive"]["path"].startswith(("/", "http")) + archive_url = site_url + "ai/mcp/v1/" + manifest["archive"]["path"] + status, headers, archive = http(archive_url) + assert status == 200 and headers["Content-Type"] == "application/gzip" + assert "immutable" in headers["Cache-Control"] + assert hashlib.sha256(archive).hexdigest() == manifest["archive"]["sha256"] + status, headers, _ = http(site_url + "ai/mcp/v1/" + "0" * 64 + "/snapshot.tar.gz") + assert status == 404 and "immutable" not in headers.get("Cache-Control", "") + status, _, _ = http(manifest_url, headers={"If-Modified-Since": "Wed, 31 Dec 2099 23:59:59 GMT"}) + assert status == 200, "manifest must not return stale 304" + assert http(site_url + "LICENSES/")[0] == 200 + for license_name in ("LGPL-3.0", "GPL-3.0", "EPL-2.0"): + assert http(site_url + f"LICENSES/{license_name}.txt")[2] == (ROOT / f"LICENSES/{license_name}.txt").read_bytes() + report["corpus_id"] = manifest["corpus_id"] + if args.chrome: + report["browser"] = browser_graph(args.chrome, args.node, site_url, directory) + network = project + "_corpus" + volume = project + "-stdio-cache" + run("docker", "volume", "create", "--label", f"v8std-task4={project}", volume) + def container(name, network_name, transport="stdio", cache=volume): + names.append(name) + return ["docker", "run", "--name", name, "--platform", args.platform, + "--label", f"v8std-task4={project}", "-i", "--init", "--read-only", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "2", + "--memory", "1536m", "--pids-limit", "128", "--network", network_name, + "--tmpfs", "/tmp:rw,noexec,nosuid,size=64m,uid=10001,gid=10001", + "-v", cache + ":/var/lib/v8std-mcp", "-e", "V8STD_MCP_SITE_URL=" + site_url, + args.mcp_image, "--transport", transport, "--refresh-seconds", "0"] + + def inspect(name): + state = json.loads(run("docker", "inspect", name))[0] + assert state["Config"]["User"] == "10001:10001" + assert state["Config"]["WorkingDir"] == "/opt/v8std" + assert state["HostConfig"]["ReadonlyRootfs"] + assert state["HostConfig"]["CapDrop"] == ["ALL"] + assert state["HostConfig"]["Init"] + assert all("docker.sock" not in m["Destination"] for m in state["Mounts"]) + return state + + def cache_state(name): + code = "from pathlib import Path; import json; p=Path('/var/lib/v8std-mcp'); " \ + "print(json.dumps({str(f.relative_to(p)): [f.stat().st_uid, f.stat().st_size, f.stat().st_mtime_ns] " \ + "for f in p.rglob('*') if f.is_file() and f.name in ('state.json','snapshot.tar.gz')}))" + return json.loads(run("docker", "exec", name, "python", "-c", code)) + + for iteration, network_name in enumerate((network, "none")): + name = project + f"-stdio-{iteration}" + with (directory / f"stdio-{iteration}.log").open("w") as log: + session = Stdio(container(name, network_name), log) + try: + session.initialize() + ranking = check_tools(session.request, site_url) + state = inspect(name) + if iteration == 0: + code = "import importlib.metadata as m,json; from pathlib import Path; " \ + "names={d.metadata['Name'].lower():d.version for d in m.distributions()}; " \ + "assert not {'pillow','zensical','markdown'} & names.keys(); " \ + "assert names['mcp']=='1.27.0' and names['markdown-it-py']=='4.0.0' and names['mdurl']=='0.1.2'; " \ + "assert not Path('/opt/v8std/docs').exists(); " \ + "assert not Path('/opt/v8std/zensical.toml').exists(); " \ + "assert Path('/opt/v8std/LICENSE').is_file(); " \ + "print(json.dumps(names,sort_keys=True))" + report["installed_runtime_graph"] = json.loads(run("docker", "exec", name, "python", "-c", code)) + current = cache_state(name) + assert current and all(value[0] == 10001 for value in current.values()) + if iteration == 0: + original, original_ranking = current, ranking + report["mcp_image_id"] = state["Image"] + else: + assert original == current, "offline warm restart rewrote cache" + assert original_ranking == ranking + session.close() + finally: + if session.process.poll() is None: + run("docker", "stop", "-t", "15", name) + session.process.wait(timeout=20) + assert not json.loads(run("docker", "inspect", name))[0]["State"]["OOMKilled"] + report["stdio"] = "cold-online + warm-network-none; EOF=0; cache UID10001 and bytes/mtime stable" + terminated = project + "-stdio-term" + with (directory / "stdio-term.log").open("w") as log: + session = Stdio(container(terminated, "none") + ["--max-snippet-chars", "32000"], log) + try: + session.initialize() + listed = session.request("tools/list")["tools"] + tool = next(t for t in listed if t["name"] == "v8std_explain_snippet") + assert tool["inputSchema"]["properties"]["snippet"]["maxLength"] == 32000 + body = " " * (32000 - len(SIGNAL)) + SIGNAL + result = eventually(lambda: content(session.request("tools/call", { + "name": "v8std_explain_snippet", "arguments": {"snippet": body, "limit": 1}}))) + assert result["diagnostics"][0]["id"] == "bslls:UsingModalWindows" + run("docker", "stop", "-t", "15", terminated) + report["stdio_sigterm_exit"] = session.process.wait(timeout=20) + assert report["stdio_sigterm_exit"] in (0, 143) + finally: + if session.process.poll() is None: + run("docker", "stop", "-t", "15", terminated) + session.process.wait(timeout=20) + session.process.stdin.close() + session.reader.join(timeout=2) + session.process.stdout.close() + if args.host_gateway: + report["gateway"] = host_gateway_check(project, args.mcp_image, volume, site_url, directory) + + # Ready/liveness boundary with no source and no cache. + cold = project + "-cold" + command = container(cold, "none", "streamable-http", cache=project + "-empty") + process = subprocess.Popen(command + ["--host", "0.0.0.0", "--port", "8000"], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + code = "import urllib.request,urllib.error; " \ + "r=urllib.request.urlopen('http://127.0.0.1:8000/livez'); print(r.status)" + eventually(lambda: run("docker", "exec", cold, "python", "-c", code, + stderr=subprocess.DEVNULL) == "200") + code = "import http.client; c=http.client.HTTPConnection('127.0.0.1',8000); " \ + "c.request('GET','/healthz'); r=c.getresponse(); print(r.status)" + assert run("docker", "exec", cold, "python", "-c", code) == "503" + run("docker", "stop", "-t", "15", cold) + report["cold_http_sigterm_exit"] = process.wait(timeout=20) + assert report["cold_http_sigterm_exit"] in (0, 143) + finally: + if process.poll() is None: + run("docker", "stop", "-t", "15", cold) + process.wait(timeout=20) + process.stdin.close() + + run(*compose, "--profile", "mcp", "up", "-d", env=env) + mcp = run(*compose, "ps", "-q", "mcp", env=env) + endpoint = f"http://127.0.0.1:{args.mcp_port}" + health = eventually(lambda: json.loads(http(endpoint + "/healthz")[2]) if http(endpoint + "/healthz")[0] == 200 else None) + assert health["corpus_id"] == manifest["corpus_id"], health + count = 0 + def request(method, params=None): + nonlocal count + count += 1 + status, _, body = http(endpoint + "/mcp", {"jsonrpc": "2.0", "id": count, + "method": method, "params": params or {}}) + assert status == 200, (status, body[:200]) + return json.loads(body)["result"] + assert request("initialize", INIT)["serverInfo"]["name"] == "v8std" + assert http(endpoint + "/mcp", {"jsonrpc":"2.0", "id":999, + "method":"initialize", "params":INIT}, headers={"Host":"untrusted.invalid"})[0] == 421 + check_tools(request, site_url) + inspect(mcp) + assert list(json.loads(run("docker", "inspect", mcp))[0]["NetworkSettings"]["Networks"]) == [network] + report["http"] = health + report["site_image_id"] = json.loads(run("docker", "inspect", site))[0]["Image"] + run(*compose, "stop", "-t", "15", "mcp", env=env) + report["warm_http_sigterm_exit"] = json.loads(run("docker", "inspect", mcp))[0]["State"]["ExitCode"] + assert report["warm_http_sigterm_exit"] in (0, 143) + report["limits"] = {"mcp_memory_bytes": 1536 * 1024**2, "mcp_cpus": 2, + "site_memory_bytes": 128 * 1024**2, "site_cpus": 0.5} + print(json.dumps(report, ensure_ascii=False, indent=2)) + except BaseException: + for log in directory.glob("*.log"): + print(f"{log.name}: {log.read_text(errors='replace')[-2500:]}", flush=True) + raise + finally: + for name in names: + subprocess.run(["docker", "rm", "-f", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + run(*compose, "--profile", "mcp", "down", "-v", env=env) + for volume_name in (project + "-stdio-cache", project + "-empty"): + subprocess.run(["docker", "volume", "rm", volume_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_license_texts.py b/scripts/publish_license_texts.py index 1780244..57b9819 100644 --- a/scripts/publish_license_texts.py +++ b/scripts/publish_license_texts.py @@ -49,6 +49,14 @@ def publish_license_texts(repo_root: Path, site_dir: Path) -> set[str]: raise FileNotFoundError(f"canonical license text is missing: {source}") shutil.copyfile(source, target_dir / filename) + links = "\n".join(f'
  • {name}
    • ' + for name in sorted(LICENSE_FILENAMES)) + (target_dir / "index.html").write_text( + '' + '' + '' + 'Third-party license texts

    Third-party license texts

    ' + f'
      {links}
    \n', encoding="utf-8") return set(LICENSE_FILENAMES) diff --git a/tests/test_published_license_links.py b/tests/test_published_license_links.py index bf33e20..03fae7b 100644 --- a/tests/test_published_license_links.py +++ b/tests/test_published_license_links.py @@ -1,14 +1,27 @@ import unittest +import tempfile from pathlib import Path from unittest.mock import patch -from scripts.publish_license_texts import check_published_license_links, default_repo_root +from scripts.publish_license_texts import (check_published_license_links, default_repo_root, + publish_license_texts, LinkParser) REPO_ROOT = Path(__file__).resolve().parents[1] class PublishedLicenseLinkTests(unittest.TestCase): + def test_directory_index_links_all_unchanged_license_texts(self): + with tempfile.TemporaryDirectory() as directory: + site = Path(directory) + names = publish_license_texts(REPO_ROOT, site) + parser = LinkParser() + parser.feed((site / "LICENSES/index.html").read_text()) + self.assertEqual(set(parser.hrefs), names) + for name in names: + self.assertEqual((site / "LICENSES" / name).read_bytes(), + (REPO_ROOT / "LICENSES" / name).read_bytes()) + def test_container_repo_root_comes_from_wrapper_environment(self): with patch.dict("os.environ", {"V8STD_REPO_ROOT": "/docs"}): self.assertEqual(default_repo_root(), Path("/docs")) diff --git a/tests/test_v8std_mcp_distribution.py b/tests/test_v8std_mcp_distribution.py new file mode 100644 index 0000000..f119cd4 --- /dev/null +++ b/tests/test_v8std_mcp_distribution.py @@ -0,0 +1,99 @@ +"""Focused distribution checks; real Docker/browser acceptance is opt-in.""" +import hashlib +import html as html_module +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +class DistributionTests(unittest.TestCase): + def test_images_are_thin_pinned_and_unprivileged(self): + runtime = (ROOT / "Dockerfile.mcp").read_text() + static = (ROOT / "Dockerfile.site").read_text() + for definition in (runtime, static): + self.assertRegex(definition, r"FROM [^\n]+@sha256:[0-9a-f]{64}") + self.assertIn("USER 10001:10001", definition) + self.assertIn('CMD ["--transport", "stdio"]', runtime) + self.assertIn("retrieval-rules.yml", runtime) + self.assertIn("v8std_search_features.py", runtime) + self.assertNotIn("v8std_mcp*.py", runtime) + self.assertNotIn("COPY docs", runtime) + self.assertIn("--require-hashes", runtime) + + def test_compose_has_common_address_and_internal_mcp(self): + compose = yaml.safe_load((ROOT / "compose.yaml").read_text()) + site, mcp = (compose["services"][name] for name in ("site", "mcp")) + for service in (site, mcp): + self.assertTrue(service["read_only"]) + self.assertTrue(service["init"]) + self.assertEqual(service["cap_drop"], ["ALL"]) + self.assertNotIn("build", service) + self.assertEqual(len(service["tmpfs"]), 1) + self.assertIn("uid=10001,gid=10001", service["tmpfs"][0]) + self.assertTrue(all(port.startswith("127.0.0.1:") for port in site["ports"])) + self.assertNotIn("ports", mcp) + self.assertEqual(mcp["profiles"], ["mcp"]) + self.assertEqual(mcp["networks"], ["corpus"]) + self.assertTrue(compose["networks"]["corpus"]["internal"]) + self.assertIn("v8std.localhost", site["networks"]["corpus"]["aliases"]) + self.assertIn("v8std.localhost", mcp["environment"]["V8STD_MCP_SITE_URL"]) + + def test_profile_rejects_source_output_overlap(self): + sys.path.insert(0, str(ROOT / "scripts")) + from build_local_site import build_local_site + for output in (ROOT, ROOT / "docs", ROOT / "site", ROOT / "scripts/out"): + with self.subTest(output=output), self.assertRaises(ValueError): + build_local_site(ROOT, output, "http://v8std.localhost:18765/kb/", "a" * 40) + + def test_catalog_has_long_lived_configurable_stdio(self): + spec = yaml.safe_load((ROOT / "deploy/docker-catalog/server.yaml").read_text()) + self.assertTrue(spec["longLived"]) + self.assertEqual(spec["run"]["user"], "10001:10001") + self.assertEqual(spec["run"]["command"], ["--transport", "stdio"]) + self.assertTrue(spec["image"].startswith("ghcr.io/zeegin/v8std-mcp")) + self.assertEqual(set(spec["run"]["env"]), + {"V8STD_MCP_SITE_URL", "V8STD_MCP_MAX_SNIPPET_CHARS"}) + + +@unittest.skipUnless(os.environ.get("V8STD_TEST_LOCAL_BUILD"), "explicit local build acceptance") +class LocalBuildTests(unittest.TestCase): + def test_actual_isolated_build_and_canonical_snapshot(self): + sys.path.insert(0, str(ROOT / "scripts")) + from build_local_site import build_local_site + from generate_mcp_snapshot import build_snapshot + from v8std_mcp_snapshot_format import verify_archive + + def hashes(): + return {str(p): hashlib.sha256(p.read_bytes()).hexdigest() + for folder in ("docs", "overrides", "site") + for p in (ROOT / folder).rglob("*") if p.is_file()} + + before = hashes() + source_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + with tempfile.TemporaryDirectory(prefix="v8std-local-test-") as temporary: + output = Path(temporary) / "site" + build_local_site(ROOT, output, "http://v8std.localhost:18765/kb/", source_sha) + manifest = json.loads((output / "ai/mcp/v1/manifest.json").read_bytes()) + self.assertFalse(manifest["archive"]["path"].startswith(("/", "http"))) + archive = (output / "ai/mcp/v1" / manifest["archive"]["path"]).read_bytes() + canonical, _ = build_snapshot(ROOT / "docs", source_sha, "https://v8std.ru/") + self.assertEqual(archive, canonical) + verify_archive(archive, manifest) + self.assertTrue((output / "LICENSES/index.html").is_file()) + html = (output / "std/437/index.html").read_text() + self.assertNotIn("u.ingvar.pro", html) + self.assertNotIn("fonts.googleapis.com", html) + self.assertTrue("http://v8std.localhost:18765/kb/" in html_module.unescape(html)) + self.assertEqual(before, hashes()) + + +if __name__ == "__main__": + unittest.main() From deb957afd78e214002dab7100357696b4c68e30e Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 18:06:07 +0300 Subject: [PATCH 25/88] docs: record partial container acceptance and platform gates --- spec/operations/mcp-container-verification.md | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index a26022c..f222d36 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -177,6 +177,60 @@ incorrectly protect subsequent visible links from rebasing/validation. It was reproduced in both the old scanner and the semantic-parser fix, so scoped re-review did not extend its loop to it. This is explicitly not waived for release. +### Container implementation — partial acceptance, review pending + +Scoped signed implementation: `9a3f483416484a0fe85ba55cc554df5b25e6f6c2`. +Independent task review is pending; this is not a completed multi-platform or +Catalog acceptance gate. Prototype images used base SHA `f5c45d2` labels while +packaging files were uncommitted, so they are test artifacts, not proof of exact +release provenance. Rebuild the final verified SHA before any release. + +```sh +V8STD_TEST_LOCAL_BUILD=1 .venv/bin/python -m unittest tests.test_v8std_mcp_distribution tests.test_published_license_links -v +.venv/bin/python scripts/check_mcp_container.py --mcp-image v8std-task4-mcp:arm64 --site-image v8std-task4-site:arm64 --platform linux/arm64 --prefix /kb/ --chrome '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --node /opt/homebrew/bin/node --host-gateway +``` + +Eight focused tests passed in 38.667 seconds, including isolated strict build. +Actual arm64 harness passed cold-online stdio, warm network-disabled restart, +HTTP readiness/tools, cold-offline 503, clean EOF and bounded SIGTERM. Direct +and Compose MCP used UID/GID10001, read-only root, cap-drop ALL, init, +no-new-privileges, persistent cache, 2 CPU and1536MiB memory limits. These are +test settings, not a throughput claim. Known modal-window snippet rules and +4000/32000-character configurations passed; five tools/three Resources retained. + +Local profile preserved canonical archive bytes and source inputs. A fresh +Chrome151 profile observed18 unique requests over five representative pages, +including search/worker/fonts/licenses, all within the local `/kb/` origin and +prefix; no blocked public requests or bad responses. This sampled browser +request graph is not exhaustive coverage of every site page. Chrome emitted +Keychain/encryption warnings even with the isolated basic-password profile. +Manifest freshness, immutable successful archives, uncached404s and licenses +passed. Native Linux routing remains separate from Docker Desktop testing. + +Two host Gateway0.43.3 sessions passed repeated warm-offline calls with separate +long-lived MCP containers and one cache. The actual Gateway-created containers +were **not read-only and did not drop capabilities**. Its Catalog configuration +does not expose the full direct/Compose hardening profile. The containerized +Gateway local-network test stopped at Docker socket permission denial under +UID/GID501:20; no root/group/permission retry was made. Cold local Gateway +routing and full hardening are not accepted. No socket was mounted into MCP. +Source Catalog schema decoding passed against mcp-registry8c773729; external +Catalog review/publication did not occur. + +Both architecture images built, and amd64 static delivery passed under QEMU. +However, supervised amd64 full-corpus startup failed with `deadline`, no OOM, +and continued503 readiness. A separate direct phase diagnostic measured +3.390s imports,0.167s download,7.465s verification and **86.912s generation**, +97.935s total. Generation alone exceeds the60s loader attempt limit. That +unsupervised diagnostic is not successful runtime acceptance. Preserve the +production cancellation bound; require native amd64 full-corpus proof before +publication, and fix preparation if the native run also fails. Warm amd64 and +full lifecycle acceptance remain incomplete. + +Owned test containers/networks/volumes and temporary Gateway/browser configs +were removed; test images and temporary build evidence remain. Unrelated +containers, listeners, active Docker configuration and host DNS were preserved. No registry, GitHub settings or target-host mutation occurred. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; @@ -204,8 +258,8 @@ and query set on the implemented snapshot runtime. | Area | Current evidence | | --- | --- | | Bounded refresh/cache, crashes, shared volume, offline recovery | Task review accepted locally; Linux/runtime integration remains. | -| Frozen generations, URL presentation, stdio/HTTP lifecycle | Pending. | -| Runtime/static-site images, local request graph, Gateway sessions | Pending. | +| Frozen generations, URL presentation, stdio/HTTP lifecycle | Task review accepted; known image-alt edge remains final release gate. | +| Runtime/static-site images, local request graph, Gateway sessions | Arm64/local profile and warm host Gateway pass; task review, native amd64 and Gateway cold routing/hardening remain open. | | Restricted host controller, rollback and independent index delivery | Pending. | | Fail-closed publication, process v2 synchronization | Pending. | | Final semantic impact, merge-ready, fitness, strict build and full suite | Pending after all changes; strict build precedes the suite. | From 7988f0698883121a2f37b077eb20f74eefa598d7 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 18:10:39 +0300 Subject: [PATCH 26/88] docs: record container review and design recovery gate --- spec/operations/mcp-container-verification.md | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index f222d36..2277243 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -180,8 +180,8 @@ re-review did not extend its loop to it. This is explicitly not waived for relea ### Container implementation — partial acceptance, review pending Scoped signed implementation: `9a3f483416484a0fe85ba55cc554df5b25e6f6c2`. -Independent task review is pending; this is not a completed multi-platform or -Catalog acceptance gate. Prototype images used base SHA `f5c45d2` labels while +Independent task review requires fixes; this is not a completed multi-platform +or Catalog acceptance gate. Prototype images used base SHA `f5c45d2` labels while packaging files were uncommitted, so they are test artifacts, not proof of exact release provenance. Rebuild the final verified SHA before any release. @@ -231,6 +231,22 @@ Owned test containers/networks/volumes and temporary Gateway/browser configs were removed; test images and temporary build evidence remain. Unrelated containers, listeners, active Docker configuration and host DNS were preserved. No registry, GitHub settings or target-host mutation occurred. +Review identified an implementation defect: Compose ignores an operator-set +`V8STD_MCP_SITE_URL`; the resumed fix must prove the same override controls +source and returned links. It also confirmed that the exercised Gateway launch +does not satisfy the accepted hardening profile. Upstream Gateway0.43.3 +[`baseArgs`/`argsAndEnv`](https://github.com/docker/mcp-gateway/blob/v0.43.3/pkg/gateway/clientpool.go#L304) +adds init/no-new-privileges and configurable resource/user/network arguments, +but not read-only root, cap-drop or tmpfs. Catalog approval alone cannot correct +that mismatch. No daemon changes or privileged retry were authorized. + +Implementation is paused at the architecture failure-recovery gate for revised +written design/scope approval. Task4 remains incomplete; release controller and +CI Tasks5/6 have not started. Keeping full direct/Compose/production hardening +and making Gateway a separately accepted channel is a proposal, not an approved +contract change. The60s runtime bound remains unchanged, and amd64 acceptance +is not claimed. Do not merge, publish or close PR33 from this partial result. + ### Local build environment Observed 2026-09-10: Docker Desktop, Engine 29.7.2, linux/arm64; Gateway v0.43.3; @@ -259,7 +275,7 @@ and query set on the implemented snapshot runtime. | --- | --- | | Bounded refresh/cache, crashes, shared volume, offline recovery | Task review accepted locally; Linux/runtime integration remains. | | Frozen generations, URL presentation, stdio/HTTP lifecycle | Task review accepted; known image-alt edge remains final release gate. | -| Runtime/static-site images, local request graph, Gateway sessions | Arm64/local profile and warm host Gateway pass; task review, native amd64 and Gateway cold routing/hardening remain open. | +| Runtime/static-site images, local request graph, Gateway sessions | Arm64/local profile and warm host Gateway pass; review requires Compose fix and explicit Gateway/gate design resolution; amd64 acceptance remains open. | | Restricted host controller, rollback and independent index delivery | Pending. | | Fail-closed publication, process v2 synchronization | Pending. | | Final semantic impact, merge-ready, fitness, strict build and full suite | Pending after all changes; strict build precedes the suite. | From 4c0d5a7fb0e1999c8de90c262cfa9d46dce02b6a Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 18:51:51 +0300 Subject: [PATCH 27/88] docs: approve 360-second snapshot preparation budget --- spec/contracts/mcp-corpus-snapshot-v1-r0.md | 10 ++++++- ...09-10-mcp-container-distribution-design.md | 7 +++++ ...6-09-10-mcp-container-distribution-plan.md | 30 ++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/spec/contracts/mcp-corpus-snapshot-v1-r0.md b/spec/contracts/mcp-corpus-snapshot-v1-r0.md index 761079f..4b63377 100644 --- a/spec/contracts/mcp-corpus-snapshot-v1-r0.md +++ b/spec/contracts/mcp-corpus-snapshot-v1-r0.md @@ -167,10 +167,18 @@ scores/IDs для одного запроса и corpus; различаются | llms / llms-full | 4 / 16 MiB | | Одна JSONL строка | 1 MiB UTF-8 | | Pages / vector rows | По 100 000 | -| Вся попытка обновления | 60 секунд monotonic deadline | +| Вся попытка обновления | 360 секунд monotonic deadline | | Блокирующий сетевой read | Не более 20 секунд и остатка общего deadline | | Cache на экземпляр volume | 256 MiB, включая staging и pinned generations | +Общий бюджет уточнён прямым решением пользователя «поставь 360 секунд и +продолжай» до первого выпуска этого контракта. Он включает download, +verification, generation construction и передачу результата координатору; +это не timeout tool call. Предел одного сетевого read остаётся 20 секунд. +Close/SIGTERM отменяет worker, не ожидая исчерпания всех 360 секунд. +Более короткие readiness/transaction budgets host-controller независимы: +контроллер вправе отменить ещё не готовый candidate раньше этого предела. + Content-Length проверяется, но не заменяет счётчик фактически прочитанных байтов. Число JSON nesting levels ограничено 32; строки, числа и arrays проверяются до построения index. Сжатый файл отдаётся как `application/gzip` diff --git a/spec/designs/2026-09-10-mcp-container-distribution-design.md b/spec/designs/2026-09-10-mcp-container-distribution-design.md index 8238abd..ec8e92c 100644 --- a/spec/designs/2026-09-10-mcp-container-distribution-design.md +++ b/spec/designs/2026-09-10-mcp-container-distribution-design.md @@ -170,6 +170,13 @@ Manifest, скачивание, распаковка, строки JSONL, кол включают gzip bomb, traversal, медленный поток, oversized record и повторные ошибки; последняя рабочая копия не повреждается. +До первого выпуска пользователь явно уточнил общий monotonic budget одной +попытки до 360 секунд. Это изменение количественного предела, а не смена +фонового executor, схемы snapshot или свойства atomic activation. Сетевой read +остаётся ограниченным 20 секундами; запросы не ждут refresh, а shutdown +прерывает подготовку раньше общего предела. Host readiness и rollout имеют +собственные меньшие бюджеты и должны отменять candidate при их исчерпании. + ### MCP_INDEX_DELIVERY_SURVIVES_RUNTIME_RESTART GET опубликованного immutable snapshot обслуживается nginx при остановленном diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 071cd7c..59f1a29 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -198,7 +198,7 @@ self.assertEqual(coordinator.current().corpus_id, "fixture-generation-a") - [x] **GREEN URL/network:** Resolve manifest below selected base prefix; permit fixed ai archive origin only for default public site; validate every redirect, no downgrade, no credentials and three redirects maximum. Stream - reads enforce byte caps plus 60s whole attempt/20s read timeout. Validate + reads enforce byte caps plus 360s whole attempt/20s read timeout. Validate Content-Encoding and length; reuse ETag/Last-Modified only with a valid cache. - [x] **GREEN cache/lifecycle:** Namespace by normalized source/schema; verify cache before use; retain active+previous and pins. File lock serializes @@ -336,6 +336,34 @@ self.assertEqual(container_inspect["Config"]["User"], "10001:10001") Verify licenses/SBOM inputs. Record unavailable external catalog acceptance explicitly, not as a passed test. Commit task and perform review. +#### Task 4 reviewed fixes — approved attempt-budget update + +User explicitly requested `360` seconds after reviewing the original timeout. +This supersedes the original numerical budget only. Gateway scope/security +decisions and external mutation authority are not inferred from that change. +Keep historical measurements labelled with their original 60-second build. + +- [ ] **RED:** In `tests/test_v8std_mcp_snapshots.py`, assert a default store + uses `360` attempt seconds and `20` read seconds; retain accelerated real + worker timeout/reaping, close and responsive-query tests. In distribution + tests resolve Compose with an explicit alternate SITE_URL and prove it is + passed intact instead of replaced by the default local URL. +- [ ] **GREEN:** Set `ATTEMPT_SECONDS = 360` in + `scripts/v8std_mcp_snapshots.py`. Compose consumes + `${V8STD_MCP_SITE_URL:-http://v8std.localhost:${V8STD_SITE_PORT:-18765}${V8STD_SITE_PREFIX:-/}}`. + Test actual Compose interpolation; do not assume nested defaults work without + executing its config resolver. Preserve local default, prefix and one setting. + Adapt only the integration startup wait to allow the accepted attempt plus + bounded startup margin; do not turn RPC/read/shutdown timeouts into360seconds. +- [ ] **VERIFY:** Run focused snapshot/distribution tests, commit the exact + changed runtime, build a new amd64 image from that clean source SHA and rerun + full-corpus supervised cold/warm stdio/HTTP acceptance under QEMU. Exercise an + explicit reachable SITE_URL override end-to-end, checking source and returned + links. No privileged Gateway retry, image publication or native-CI claim. +- [ ] **REVIEW:** Independent scoped review of the fix diff and evidence. + Gateway discrepancy remains separately open until an approved design decision; + passing these checks alone does not close Task4 or the full release plan. + ### Task 5: Restricted release transaction and independent index store **Files:** create `scripts/v8std_mcp_release.py`, `tests/test_v8std_mcp_release.py`, From 47cb04f30320a691d7b6604b9b06d86a250158d9 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 18:56:32 +0300 Subject: [PATCH 28/88] fix(distribution): honor site override and allow 360-second refresh --- compose.yaml | 2 +- docs/container-installation.md | 11 ++++ scripts/check_mcp_container.py | 82 +++++++++++++++++++++++++--- scripts/v8std_mcp_snapshots.py | 2 +- tests/test_v8std_mcp_distribution.py | 20 +++++++ tests/test_v8std_mcp_snapshots.py | 4 ++ 6 files changed, 112 insertions(+), 9 deletions(-) diff --git a/compose.yaml b/compose.yaml index ddbaaed..748fbd0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -37,7 +37,7 @@ services: pids_limit: 128 volumes: [mcp-cache:/var/lib/v8std-mcp] environment: - V8STD_MCP_SITE_URL: "http://v8std.localhost:${V8STD_SITE_PORT:-18765}${V8STD_SITE_PREFIX:-/}" + V8STD_MCP_SITE_URL: "${V8STD_MCP_SITE_URL:-http://v8std.localhost:${V8STD_SITE_PORT:-18765}${V8STD_SITE_PREFIX:-/}}" V8STD_MCP_MAX_SNIPPET_CHARS: "${V8STD_MCP_MAX_SNIPPET_CHARS:-4000}" command: [--transport, streamable-http, --host, 0.0.0.0, --port, "8000"] # Host HTTP goes through the site's loopback proxy; MCP has no egress route. diff --git a/docs/container-installation.md b/docs/container-installation.md index f977466..7506ea1 100644 --- a/docs/container-installation.md +++ b/docs/container-installation.md @@ -49,6 +49,13 @@ Site port одинаков снаружи и внутри контейнера. определяет источник manifest/archive и ссылки в ответах MCP. У него нет скрытой публичной альтернативы. Префикс образа и настройка Compose должны совпадать. +Явный `V8STD_MCP_SITE_URL` переопределяет вычисленный локальный URL в Compose: +например, `V8STD_MCP_SITE_URL=http://v8std.localhost:18765/kb/` для доступного +по этому адресу snapshot. Настройка одновременно меняет источник и ссылки, +но не перестраивает сайт и не добавляет MCP сетевой доступ: выбранный URL должен +быть доступен из существующей сети `corpus` и с компьютера пользователя. +Проверьте итоговое значение командой `docker compose --profile mcp config`. + Перед использованием на другой машине проверьте URL из браузера и контейнера: ```bash @@ -115,6 +122,10 @@ Cold-offline запуск без cache остаётся неготовым; эт `V8STD_MCP_SITE_URL`, `V8STD_MCP_CACHE_DIR` и `V8STD_MCP_MAX_SNIPPET_CHARS`. Snippet default — 4000, максимум — 32000 символов. `--refresh-seconds 0` отключает периодический refresh, но не первую попытку обновления после старта. +Общая попытка загрузки, проверки и построения поколения ограничена 360 секундами; +один сетевой read — 20 секундами и остатком общего бюджета. Это не timeout RPC: +до готовности tools возвращают INDEX_NOT_READY, не ожидая завершения построения. +Close/SIGTERM отменяет worker без ожидания всех 360 секунд. Старые `--index-url`/`--vectors-url` не являются настройками нового образа; не смешивайте их с `--site-url`. Прямой Python CLI для разработки по-прежнему по умолчанию выбирает HTTP `127.0.0.1:8765`. diff --git a/scripts/check_mcp_container.py b/scripts/check_mcp_container.py index 24a0c29..13be393 100644 --- a/scripts/check_mcp_container.py +++ b/scripts/check_mcp_container.py @@ -32,6 +32,9 @@ SIGNAL = 'Предупреждение("Текст");' TOOLS = {"v8std_search", "v8std_get_page", "v8std_get_related", "v8std_explain_snippet", "v8std_explain_diagnostics"} +# Whole-attempt budget plus bounded interpreter/container startup margin. +# This is only polling readiness; individual RPC/read/stop budgets stay shorter. +STARTUP_SECONDS = 360 + 30 @cache @@ -135,7 +138,8 @@ def check_tools(request, site_url, *, resources=True): assert snippet["inputSchema"]["properties"]["snippet"]["maxLength"] == 4000 def call(name, args): return content(request("tools/call", {"name": name, "arguments": args})) - search = eventually(lambda: call("v8std_search", {"query": "std437", "limit": 3})) + search = eventually(lambda: call("v8std_search", {"query": "std437", "limit": 3}), + seconds=STARTUP_SECONDS) assert search["results"][0]["id"] == "std437", search assert search["results"][0]["url"] == site_url + "std/437/", search ranking = call("v8std_search", {"query": "модальные окна", "limit": 5}) @@ -323,7 +327,8 @@ def main(): probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) probe.bind(("127.0.0.1", port)) assert args.prefix.startswith("/") and args.prefix.endswith("/") - site_url = f"http://v8std.localhost:{args.site_port}{args.prefix}" + local_default = f"http://v8std.localhost:{args.site_port}{args.prefix}" + site_url = os.environ.get("V8STD_MCP_SITE_URL") or local_default project = "v8std-task4-" + uuid.uuid4().hex[:10] env = {**os.environ, "V8STD_SITE_IMAGE": args.site_image, "V8STD_MCP_IMAGE": args.mcp_image, "V8STD_SITE_PORT": str(args.site_port), "V8STD_MCP_PORT": str(args.mcp_port), @@ -331,12 +336,19 @@ def main(): compose = ["docker", "compose", "-p", project, "-f", str(ROOT / "compose.yaml")] names = [] report = {"platform": args.platform, "site_url": site_url} + resolved = json.loads(run(*compose, "--profile", "mcp", "config", "--format", "json", env=env)) + assert resolved["services"]["mcp"]["environment"]["V8STD_MCP_SITE_URL"] == site_url + report["compose_site_url"] = site_url with tempfile.TemporaryDirectory(prefix=project) as directory: directory = Path(directory) try: run(*compose, "up", "-d", "site", env=env) site = run(*compose, "ps", "-q", "site", env=env) eventually(lambda: http(site_url)[0] == 200) + if site_url != local_default: + assert http(local_default + "ai/mcp/v1/manifest.json")[0] == 404, \ + "override regression requires an unavailable default source" + report["default_source_status"] = 404 manifest_url = site_url + "ai/mcp/v1/manifest.json" status, headers, payload = http(manifest_url) assert status == 200 and headers.get("Cache-Control") == "no-store" @@ -355,6 +367,8 @@ def main(): for license_name in ("LGPL-3.0", "GPL-3.0", "EPL-2.0"): assert http(site_url + f"LICENSES/{license_name}.txt")[2] == (ROOT / f"LICENSES/{license_name}.txt").read_bytes() report["corpus_id"] = manifest["corpus_id"] + report["corpus_source_sha"] = manifest["source_sha"] + report["archive_sha256"] = manifest["archive"]["sha256"] if args.chrome: report["browser"] = browser_graph(args.chrome, args.node, site_url, directory) network = project + "_corpus" @@ -387,12 +401,16 @@ def cache_state(name): return json.loads(run("docker", "exec", name, "python", "-c", code)) for iteration, network_name in enumerate((network, "none")): + started = time.monotonic() + print(f"stdio {iteration}: starting {network_name}", file=sys.stderr, flush=True) name = project + f"-stdio-{iteration}" with (directory / f"stdio-{iteration}.log").open("w") as log: session = Stdio(container(name, network_name), log) try: session.initialize() ranking = check_tools(session.request, site_url) + report[f"stdio_{iteration}_ready_seconds"] = round(time.monotonic() - started, 2) + print(f"stdio {iteration}: ready", file=sys.stderr, flush=True) state = inspect(name) if iteration == 0: code = "import importlib.metadata as m,json; from pathlib import Path; " \ @@ -406,6 +424,8 @@ def cache_state(name): report["installed_runtime_graph"] = json.loads(run("docker", "exec", name, "python", "-c", code)) current = cache_state(name) assert current and all(value[0] == 10001 for value in current.values()) + namespace = "v1-" + hashlib.sha256(site_url.encode()).hexdigest() + "/" + assert all(path.startswith(namespace) for path in current), current if iteration == 0: original, original_ranking = current, ranking report["mcp_image_id"] = state["Image"] @@ -429,7 +449,8 @@ def cache_state(name): assert tool["inputSchema"]["properties"]["snippet"]["maxLength"] == 32000 body = " " * (32000 - len(SIGNAL)) + SIGNAL result = eventually(lambda: content(session.request("tools/call", { - "name": "v8std_explain_snippet", "arguments": {"snippet": body, "limit": 1}}))) + "name": "v8std_explain_snippet", "arguments": {"snippet": body, "limit": 1}})), + seconds=STARTUP_SECONDS) assert result["diagnostics"][0]["id"] == "bslls:UsingModalWindows" run("docker", "stop", "-t", "15", terminated) report["stdio_sigterm_exit"] = session.process.wait(timeout=20) @@ -466,10 +487,15 @@ def cache_state(name): process.wait(timeout=20) process.stdin.close() + started = time.monotonic() + print("HTTP cold-online: starting", file=sys.stderr, flush=True) run(*compose, "--profile", "mcp", "up", "-d", env=env) mcp = run(*compose, "ps", "-q", "mcp", env=env) endpoint = f"http://127.0.0.1:{args.mcp_port}" - health = eventually(lambda: json.loads(http(endpoint + "/healthz")[2]) if http(endpoint + "/healthz")[0] == 200 else None) + health = eventually(lambda: json.loads(http(endpoint + "/healthz")[2]) if http(endpoint + "/healthz")[0] == 200 else None, + seconds=STARTUP_SECONDS) + report["http_cold_ready_seconds"] = round(time.monotonic() - started, 2) + print("HTTP cold-online: ready", file=sys.stderr, flush=True) assert health["corpus_id"] == manifest["corpus_id"], health count = 0 def request(method, params=None): @@ -483,13 +509,55 @@ def request(method, params=None): assert http(endpoint + "/mcp", {"jsonrpc":"2.0", "id":999, "method":"initialize", "params":INIT}, headers={"Host":"untrusted.invalid"})[0] == 421 check_tools(request, site_url) - inspect(mcp) + state = inspect(mcp) + expected_sha = json.loads(run("docker", "image", "inspect", args.mcp_image))[0]["Config"]["Labels"]["org.opencontainers.image.revision"] + assert health["runtime_sha"] == expected_sha, health + assert "V8STD_MCP_SITE_URL=" + site_url in state["Config"]["Env"] + http_cache = cache_state(mcp) + http_volume = next(m["Name"] for m in state["Mounts"] if m["Destination"] == "/var/lib/v8std-mcp") assert list(json.loads(run("docker", "inspect", mcp))[0]["NetworkSettings"]["Networks"]) == [network] report["http"] = health report["site_image_id"] = json.loads(run("docker", "inspect", site))[0]["Image"] run(*compose, "stop", "-t", "15", "mcp", env=env) - report["warm_http_sigterm_exit"] = json.loads(run("docker", "inspect", mcp))[0]["State"]["ExitCode"] - assert report["warm_http_sigterm_exit"] in (0, 143) + report["online_http_sigterm_exit"] = json.loads(run("docker", "inspect", mcp))[0]["State"]["ExitCode"] + assert report["online_http_sigterm_exit"] in (0, 143) + + # Same Compose cache, but no network: exercise real warm HTTP startup. + warm = project + "-warm-http" + started = time.monotonic() + print("HTTP warm-network-none: starting", file=sys.stderr, flush=True) + with (directory / "warm-http.log").open("w") as log: + process = subprocess.Popen(container(warm, "none", "streamable-http", cache=http_volume) + + ["--host", "0.0.0.0", "--port", "8000"], stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=log) + try: + code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8000/healthz',timeout=10).read().decode())" + warm_health = eventually(lambda: json.loads(run("docker", "exec", warm, "python", "-c", code, + stderr=subprocess.DEVNULL)), seconds=STARTUP_SECONDS) + assert warm_health["corpus_id"] == health["corpus_id"] + assert warm_health["runtime_sha"] == expected_sha + def warm_request(method, params=None): + message = {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}} + code = "import json,sys,urllib.request; r=urllib.request.Request('http://127.0.0.1:8000/mcp',data=sys.argv[1].encode(),headers={'Content-Type':'application/json','Accept':'application/json, text/event-stream'}); body=urllib.request.urlopen(r,timeout=10).read(); print(body.decode())" + body = run("docker", "exec", warm, "python", "-c", code, json.dumps(message)) + if body.startswith("event:"): + body = next(line[6:] for line in body.splitlines() if line.startswith("data: ")) + return json.loads(body)["result"] + assert warm_request("initialize", INIT)["serverInfo"]["name"] == "v8std" + check_tools(warm_request, site_url) + assert cache_state(warm) == http_cache, "offline warm HTTP rewrote cache" + assert inspect(warm)["HostConfig"]["NetworkMode"] == "none" + report["http_warm_ready_seconds"] = round(time.monotonic() - started, 2) + report["http_warm"] = warm_health + print("HTTP warm-network-none: ready", file=sys.stderr, flush=True) + run("docker", "stop", "-t", "15", warm) + report["warm_http_sigterm_exit"] = process.wait(timeout=20) + assert report["warm_http_sigterm_exit"] in (0, 143) + finally: + if process.poll() is None: + run("docker", "stop", "-t", "15", warm) + process.wait(timeout=20) + process.stdin.close() report["limits"] = {"mcp_memory_bytes": 1536 * 1024**2, "mcp_cpus": 2, "site_memory_bytes": 128 * 1024**2, "site_cpus": 0.5} print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py index 33621ba..2b2fcda 100644 --- a/scripts/v8std_mcp_snapshots.py +++ b/scripts/v8std_mcp_snapshots.py @@ -44,7 +44,7 @@ validate_manifest, verify_archive, ) -ATTEMPT_SECONDS = 60 +ATTEMPT_SECONDS = 360 READ_SECONDS = 20 CACHE_BYTES = 256 * 1024 * 1024 _CHUNK = 64 * 1024 diff --git a/tests/test_v8std_mcp_distribution.py b/tests/test_v8std_mcp_distribution.py index f119cd4..7b25b30 100644 --- a/tests/test_v8std_mcp_distribution.py +++ b/tests/test_v8std_mcp_distribution.py @@ -15,6 +15,26 @@ class DistributionTests(unittest.TestCase): + def test_actual_compose_resolution_preserves_site_override_and_local_defaults(self): + env = {key: value for key, value in os.environ.items() + if not key.startswith("V8STD_")} + env.update(V8STD_SITE_IMAGE="local-site:test", V8STD_MCP_IMAGE="local-mcp:test") + cases = [({}, "http://v8std.localhost:18765/"), + ({"V8STD_SITE_PORT": "19875", "V8STD_SITE_PREFIX": "/kb/"}, + "http://v8std.localhost:19875/kb/"), + ({"V8STD_SITE_PORT": "19875", "V8STD_SITE_PREFIX": "/unused/", + "V8STD_MCP_SITE_URL": "http://alternate.localhost:19876/knowledge/"}, + "http://alternate.localhost:19876/knowledge/")] + for settings, expected in cases: + with self.subTest(settings=settings): + resolved = json.loads(subprocess.check_output( + ["docker", "compose", "--env-file", os.devnull, "-f", str(ROOT / "compose.yaml"), + "--profile", "mcp", "config", "--format", "json"], + env={**env, **settings}, text=True, timeout=20)) + mcp = resolved["services"]["mcp"] + self.assertEqual(mcp["environment"]["V8STD_MCP_SITE_URL"], expected) + self.assertNotIn("--site-url", mcp["command"]) + def test_images_are_thin_pinned_and_unprivileged(self): runtime = (ROOT / "Dockerfile.mcp").read_text() static = (ROOT / "Dockerfile.site").read_text() diff --git a/tests/test_v8std_mcp_snapshots.py b/tests/test_v8std_mcp_snapshots.py index f8e1573..8b2d671 100644 --- a/tests/test_v8std_mcp_snapshots.py +++ b/tests/test_v8std_mcp_snapshots.py @@ -289,6 +289,10 @@ def setUp(self): class SnapshotStoreTests(SnapshotTestCase): + def test_default_attempt_and_read_budgets_are_independent(self): + self.assertEqual(self.store._attempt_seconds, 360) + self.assertEqual(self.store._read_seconds, 20) + def test_verified_archive_is_streamed_into_staging_before_compressed_input_ends(self): extract = getattr(self.store, "_extract_verified", None) self.assertIsNotNone(extract, "Task2 must stream extraction into private staging") From 55059b0bac9c65a44c10a399e87a22cfb5bdb922 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 21:44:53 +0300 Subject: [PATCH 29/88] fix(distribution): verify native Gateway isolation profile --- deploy/docker-catalog/server.yaml | 6 + docs/container-installation.md | 28 +++- scripts/check_mcp_container.py | 76 ++++++++-- ...26-09-10-mcp-published-combined-runtime.md | 6 + spec/contracts/mcp-distribution-v1-r0.md | 34 ++++- ...09-10-mcp-container-distribution-design.md | 11 ++ .../mcp-published-runtime-is-one-service.md | 4 + spec/operations/mcp-container-verification.md | 122 ++++++++++++++- ...6-09-10-mcp-container-distribution-plan.md | 56 +++++-- tests/test_v8std_mcp_distribution.py | 140 ++++++++++++++++++ 10 files changed, 443 insertions(+), 40 deletions(-) diff --git a/deploy/docker-catalog/server.yaml b/deploy/docker-catalog/server.yaml index f768c18..8812beb 100644 --- a/deploy/docker-catalog/server.yaml +++ b/deploy/docker-catalog/server.yaml @@ -1,6 +1,12 @@ # Docker mcp-registry source schema (pkg/servers/types.go), NOT a submitted entry. # Release publication replaces release-pending and RELEASE_SOURCE_SHA with the # verified multi-platform index digest and its matching source commit. +# Launcher-owned profiles: direct Docker/Compose retain read-only, cap-drop ALL +# and bounded tmpfs. Native Gateway v0.43.3 supplies init/no-new-privileges; +# inspect every MCP server for UID/GID 10001:10001, nonprivileged mode and only +# the expected cache volume (no Docker socket or aliased bind mounts). +# Native Catalog cannot request read-only/cap-drop/tmpfs; no invented fields. +# This distinction does not waive source routing, provenance or external review. name: v8std image: ghcr.io/zeegin/v8std-mcp:release-pending type: server diff --git a/docs/container-installation.md b/docs/container-installation.md index 7506ea1..5fa219e 100644 --- a/docs/container-installation.md +++ b/docs/container-installation.md @@ -145,11 +145,22 @@ Gateway v0.43.3 не имеет флага подключения к допол источника требуется отдельно проверенная маршрутизация containerized Gateway; его доступ к Docker API требует полномочий оператора. Не монтируйте socket в MCP. -В v0.43.3 Gateway задаёт `init` и `no-new-privileges`, но не поддерживает -read-only rootfs, cap-drop и tmpfs через эту схему Catalog. Поэтому прохождение -его lifecycle-теста не подтверждает весь hardening-контракт. Используйте direct -Docker/Compose, когда нужны перечисленные ограничения; внешняя приёмка Catalog, -registry publication и native CI остаются отдельными проверками. +Профиль изоляции задаёт launcher, а не образ. Production controller и наши +direct Docker/Compose используют строгий профиль: UID/GID `10001:10001`, +nonprivileged, без Docker socket, `init`, `no-new-privileges`, read-only rootfs, +`cap-drop ALL` и ограниченный tmpfs. Native Gateway v0.43.3 использует другой +согласованный профиль: тот же UID/GID, nonprivileged, без Docker socket, +`init` и `no-new-privileges`. Read-only rootfs, cap-drop и tmpfs через его схему +Catalog не задаются и для этого канала не обещаются. Их отсутствие само по себе +не блокирует Catalog; если эти ограничения нужны, используйте direct Docker/Compose. + +После обновления Gateway проверяйте фактические controls и mounts **каждого** +созданного MCP-сервера. Warm harness допускает только ожидаемый named cache +volume; неожиданный bind mount отклоняется независимо от имени socket alias. +До запуска harness отклоняет непустой `DOCKER_MCP_IN_DIND`, который может +заставить Gateway добавить `--privileged`; настройки пользователя не меняются. +Соответствие native-профилю не заменяет проверку cold source routing, +происхождения образа, внешнюю приёмку Catalog, registry publication или native CI. ## Проверка разработчиком @@ -187,3 +198,10 @@ Harness проверяет реальные HTTP/stdio, холодный и тё сети; она не подменяет проверку сети containerized Gateway. Harness выбирает уникальные имена ресурсов, проверяет свободные порты и удаляет только свои контейнеры, сети, cache и временный Catalog. Образы остаются для анализа. + +Для узкой повторной проверки native Gateway используйте `--gateway-warm-only` +вместо `--host-gateway`: harness подготовит один новый cache и проверит две +warm-сессии без сети, пропуская остальные HTTP/browser сценарии. Это не тест +cold routing Gateway. Обычный SITE_URL override допускает работающий default +source; только явный флаг `--require-default-source-404` дополнительно требует +404 от default manifest для специальной регрессии запрета fallback. diff --git a/scripts/check_mcp_container.py b/scripts/check_mcp_container.py index 13be393..f2305af 100644 --- a/scripts/check_mcp_container.py +++ b/scripts/check_mcp_container.py @@ -80,6 +80,52 @@ def http(url, message=None, headers=None): return response.status, dict(response.headers), data +def check_default_source(site_url, local_default, *, require_404=False): + """A working alternate source does not imply the default must be broken.""" + if not require_404: + return {} + assert site_url != local_default, "404 regression requires an explicit alternate source" + status = http(local_default + "ai/mcp/v1/manifest.json")[0] + assert status == 404, "override regression requires an unavailable default source" + return {"default_source_status": status} + + +def gateway_environment(environ): + """Reject Gateway's privileged DinD switch before any launch, never unset it.""" + if environ.get("DOCKER_MCP_IN_DIND"): + raise AssertionError("unsafe DOCKER_MCP_IN_DIND: Gateway may add --privileged") + return dict(environ) + + +def validate_gateway_profile(state, *, expected_cache): + """Validate this harness's native Gateway launch with its sole named cache. + + Match the volume identity/source/destination, not socket filenames: a bind + mounted socket (or its parent directory) may have an arbitrary alias. + Optional hardening is observed, not required by the native Gateway profile. + """ + config, host = state.get("Config", {}), state.get("HostConfig", {}) + assert config.get("User") == "10001:10001", "Gateway user must be 10001:10001" + assert host.get("Init") is True, "Gateway init is required" + assert host.get("Privileged") is False, "Gateway must not be privileged" + security = host.get("SecurityOpt") or [] + nnp = [option for option in security if option.startswith("no-new-privileges")] + assert nnp and all(option in ("no-new-privileges", "no-new-privileges:true", + "no-new-privileges=true") for option in nnp), "Gateway no-new-privileges is required" + mounts = state.get("Mounts") + assert isinstance(mounts, list) and len(mounts) == 1, "only the expected Gateway cache mount is allowed" + mount = mounts[0] + assert (mount.get("Type") == "volume" and mount.get("Name") == expected_cache["Name"] + and mount.get("Source") == expected_cache["Mountpoint"] + and mount.get("Destination") == "/var/lib/v8std-mcp" and mount.get("RW") is True), \ + "unexpected Gateway mount; bind mounts/socket aliases are forbidden" + return {"id": state["Id"], "image_id": state["Image"], "user": config["User"], + "init": host["Init"], "privileged": host["Privileged"], + "no_new_privileges": True, "security_opt": security, "mounts": mounts, + "network": host.get("NetworkMode"), "read_only": host.get("ReadonlyRootfs"), + "cap_drop": host.get("CapDrop"), "tmpfs": host.get("Tmpfs")} + + class Stdio: def __init__(self, command, stderr, env=None): self.process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -247,6 +293,8 @@ def host_gateway_check(project, image, volume, site_url, directory): Containerized Gateway's site-network routing is a separate acceptance gate. No HOME override, active catalog modification, or extra Docker privileges. """ + env = gateway_environment(os.environ) + expected_cache = json.loads(run("docker", "volume", "inspect", volume))[0] catalog_root = Path.home() / ".docker/mcp/catalogs" catalog_root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix=project + "-", dir=catalog_root) as catalog_dir: @@ -272,7 +320,7 @@ def host_gateway_check(project, image, volume, site_url, directory): with contextlib.ExitStack() as stack: for number in range(2): log = stack.enter_context((directory / f"gateway-{number}.log").open("w")) - session = Stdio(command, log) + session = Stdio(command, log, env=env) sessions.append(session) session.initialize(name=None) check_tools(session.request, site_url, resources=False) @@ -283,16 +331,15 @@ def host_gateway_check(project, image, volume, site_url, directory): check_tools(session.request, site_url, resources=False) assert set(ids) == set(run("docker", "ps", "-q", "--filter", "label=docker-mcp-name=" + project).splitlines()) states = json.loads(run("docker", "inspect", *ids)) + profiles = [validate_gateway_profile(state, expected_cache=expected_cache) for state in states] for state in states: - assert state["Config"]["User"] == "10001:10001" assert state["Config"]["WorkingDir"] == "/opt/v8std" assert state["HostConfig"]["NetworkMode"] == "none" for session in sessions: session.close() return {"sessions": 2, "servers": len(ids), "network": "none", "warm_cache": True, "long_lived_same_ids": True, - "read_only": states[0]["HostConfig"]["ReadonlyRootfs"], - "cap_drop": states[0]["HostConfig"]["CapDrop"]} + "profile": "native-gateway", "server_profiles": profiles} finally: for session in sessions: if session.process.poll() is None: @@ -319,7 +366,15 @@ def main(): parser.add_argument("--node", default="node") parser.add_argument("--host-gateway", action="store_true", help="Two Gateway sessions using warm cache and network none") + parser.add_argument("--gateway-warm-only", action="store_true", + help="Prepare one owned cache, then only check two warm host Gateway sessions") + parser.add_argument("--require-default-source-404", action="store_true", + help="Explicit override regression: additionally require the default manifest to be absent") args = parser.parse_args() + if args.host_gateway or args.gateway_warm_only: + gateway_environment(os.environ) + if args.gateway_warm_only and args.chrome: + parser.error("--gateway-warm-only excludes browser acceptance") canonical_ranking() for port in (args.site_port, args.mcp_port): with socket.socket() as probe: @@ -345,10 +400,8 @@ def main(): run(*compose, "up", "-d", "site", env=env) site = run(*compose, "ps", "-q", "site", env=env) eventually(lambda: http(site_url)[0] == 200) - if site_url != local_default: - assert http(local_default + "ai/mcp/v1/manifest.json")[0] == 404, \ - "override regression requires an unavailable default source" - report["default_source_status"] = 404 + report.update(check_default_source(site_url, local_default, + require_404=args.require_default_source_404)) manifest_url = site_url + "ai/mcp/v1/manifest.json" status, headers, payload = http(manifest_url) assert status == 200 and headers.get("Cache-Control") == "no-store" @@ -400,7 +453,7 @@ def cache_state(name): "for f in p.rglob('*') if f.is_file() and f.name in ('state.json','snapshot.tar.gz')}))" return json.loads(run("docker", "exec", name, "python", "-c", code)) - for iteration, network_name in enumerate((network, "none")): + for iteration, network_name in enumerate((network,) if args.gateway_warm_only else (network, "none")): started = time.monotonic() print(f"stdio {iteration}: starting {network_name}", file=sys.stderr, flush=True) name = project + f"-stdio-{iteration}" @@ -438,6 +491,11 @@ def cache_state(name): run("docker", "stop", "-t", "15", name) session.process.wait(timeout=20) assert not json.loads(run("docker", "inspect", name))[0]["State"]["OOMKilled"] + if args.gateway_warm_only: + report["stdio"] = "owned cold-online cache preparation only; EOF=0; cache UID10001" + report["gateway"] = host_gateway_check(project, args.mcp_image, volume, site_url, directory) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return report["stdio"] = "cold-online + warm-network-none; EOF=0; cache UID10001 and bytes/mtime stable" terminated = project + "-stdio-term" with (directory / "stdio-term.log").open("w") as log: diff --git a/spec/adr/2026-09-10-mcp-published-combined-runtime.md b/spec/adr/2026-09-10-mcp-published-combined-runtime.md index 5803f4e..718997c 100644 --- a/spec/adr/2026-09-10-mcp-published-combined-runtime.md +++ b/spec/adr/2026-09-10-mcp-published-combined-runtime.md @@ -51,6 +51,12 @@ Production, локальный запуск и каталог использую один активный runtime; максимум два однотипных экземпляра при переключении, без самостоятельного v3 endpoint. Внешняя граница остаётся `/mcp`. +Единство image/API не требует одинаковой конфигурации launchers. Production +и наши direct Docker/Compose сохраняют строгий профиль read-only/cap-drop; +Gateway использует документированный нативный профиль из distribution contract. +Этот согласованный выбор не создаёт fork, не откладывает Catalog лишь из-за +отсутствующих флагов и не разрешает privileged MCP либо Docker socket внутри него. + ## Влияние на инварианты Буквальная привязка к одному Python-процессу и старому systemd unit заменена diff --git a/spec/contracts/mcp-distribution-v1-r0.md b/spec/contracts/mcp-distribution-v1-r0.md index daf8b11..dec75b7 100644 --- a/spec/contracts/mcp-distribution-v1-r0.md +++ b/spec/contracts/mcp-distribution-v1-r0.md @@ -80,9 +80,14 @@ install на старте. Default transport — `stdio`, explicit `--cache-dir`/`V8STD_MCP_CACHE_DIR` и `--refresh-seconds`. Точное существующее имя snippet env и диапазон наследуются из design крупных процедур; не вводится параллельный alias. Cache path в образе — `/var/lib/v8std-mcp`, persistent volume -с документированным UID/GID. Образ работает non-root, read-only rootfs, -`cap-drop ALL`, `no-new-privileges`, без Docker socket, с writable cache и -ограниченным tmpfs. Stdio stdout чистый; healthcheck HTTP не применяется к stdio. +с документированным UID/GID. Во всех каналах MCP работает non-root, без +privileged mode и Docker socket. Writable cache сохраняется между запусками. +Профиль запуска принадлежит launcher, а не image: production controller и наши +direct Docker/Compose используют read-only rootfs, `cap-drop ALL`, +`no-new-privileges`, init и ограниченный tmpfs. Gateway использует свой нативный +профиль, описанный ниже; отсутствие у него отдельных флагов не меняет образ и +само по себе не блокирует Catalog. Stdio stdout чистый; healthcheck HTTP не +применяется к stdio. HTTP слушает внутренний port 8000 на `0.0.0.0`; production публикует его только на loopback для host nginx, local Compose по умолчанию только на loopback host. @@ -105,11 +110,12 @@ static-site image + опциональный MCP HTTP image + persistent cache. Production использует тот же MCP image; nginx/TLS/storage host — окружение, не другой MCP Dockerfile. Старый docs development Compose остаётся явно dev. -Оператор выбирает один site URL, достижимый с host и из контейнера. Для desktop -пример использует опубликованный host port и проверенное разрешение -`host.docker.internal` на выбранной платформе; для LAN — общий DNS/адрес host. +Оператор выбирает один site URL, достижимый с host и из контейнера. Desktop +пример использует общий `.localhost` адрес: loopback в браузере и DNS alias +в сети Compose, с одним портом и base-prefix; для LAN — общий DNS/адрес host. Именно этот адрес виден и в ссылках ответов. Linux example включает явную -проверку host-gateway, а не предполагает desktop DNS. Site base-prefix и +проверку маршрута; `host.docker.internal`/host-gateway допустимы только после +проверки доступа с обеих сторон, не по факту наличия имени внутри Docker. Site base-prefix и redirects проверяются end-to-end. Никакой скрытой подмены source на `site` с сохранением другого public URL не допускается. @@ -124,6 +130,17 @@ Catalog entry ссылается на наш опубликованный image агентов: Gateway может изолировать сессии. Shared volume экономит downloads, не объединяет автоматически Python heaps независимых процессов. +Согласованный профиль Gateway проверяет фактический non-root user, init, +`no-new-privileges`, отсутствие privileged mode и Docker socket внутри MCP. +Для Gateway v0.43.3 read-only rootfs, cap-drop и tmpfs недоступны через нативную +схему запуска Catalog; эти ограничения не обещаются для данного канала. +Оператору, которому они нужны, предназначен direct Docker/Compose. Обновление +Gateway требует повторной проверки создаваемых контейнеров; одного текста +Catalog недостаточно. Нельзя расширять права Gateway или менять настройки +Docker оператора ради прохождения проверки. Различие профилей не заменяет +остальные gates: lifecycle, доступность выбранного source, warm/cold сценарии, +проверенное происхождение образа и внешняя приёмка Catalog проверяются отдельно. + Каталог может временно указывать предыдущую проверенную версию, пока Docker рассматривает обновление entry. Для каждого release image digest одинаков во всех каналах этой версии; одновременное равенство версий каталога и production @@ -136,7 +153,8 @@ direct Docker и production путь и не даёт оснований соо ## Приёмка -Будущие container tests проверяют оба platform artifacts, non-root/read-only, +Будущие container tests проверяют оба platform artifacts и профиль каждого +launcher: non-root/read-only для owned launches и явно описанный профиль Gateway, чистый stdio, реальный HTTP POST, сохранение лимитов и cache, cold-offline неготовность, warm-offline работу и локальный сайт с запрещённым internet egress. Egress-проверка браузера включает fonts/analytics, а не только MCP socket trace. diff --git a/spec/designs/2026-09-10-mcp-container-distribution-design.md b/spec/designs/2026-09-10-mcp-container-distribution-design.md index ec8e92c..bf7c2c5 100644 --- a/spec/designs/2026-09-10-mcp-container-distribution-design.md +++ b/spec/designs/2026-09-10-mcp-container-distribution-design.md @@ -199,6 +199,17 @@ SIGTERM прекращают фоновые задачи. Warm cache переи Проверки включают cold/warm start, reconnect и несколько независимых сессий агента, в том числе через Docker Gateway; контейнер на каждый tool call не нужен. +Согласованное уточнение после проверки Gateway: единый артефакт не означает +идентичные флаги всех launchers. Production и наши direct Docker/Compose +сохраняют read-only rootfs, cap-drop ALL, no-new-privileges, init и ограниченный +tmpfs. Gateway использует нативную изоляцию с проверкой non-root, init, +no-new-privileges, отсутствия privileged mode и Docker socket внутри MCP. +Отсутствующие в Gateway v0.43.3 read-only/cap-drop/tmpfs явно описываются как +различие канала, а не требование отдельного образа или причина отложить Catalog. +Обязательства по источнику данных, lifecycle, отсутствию фонового public egress +для локального сайта и проверяемому происхождению остаются без ослабления. +Реальная cold-маршрутизация Gateway и внешняя приёмка не следуют из warm-теста. + ### MCP_RELEASE_SWITCH_IS_REVERSIBLE Ошибки pull, подготовки corpus, readiness, переключения nginx и smoke после diff --git a/spec/invariants/mcp-published-runtime-is-one-service.md b/spec/invariants/mcp-published-runtime-is-one-service.md index 63fa1b8..2c32a22 100644 --- a/spec/invariants/mcp-published-runtime-is-one-service.md +++ b/spec/invariants/mcp-published-runtime-is-one-service.md @@ -34,6 +34,10 @@ Local stdio/HTTP и catalog используют тот же multi-platform imag snippet-поведение совпадают. CPU architecture, транспорт и конфигурация сайта не выбирают другую кодовую реализацию MCP. +Равенство артефакта и API не означает равенства флагов изоляции launchers. +Production/direct Docker/Compose и Gateway применяют явно различённые профили +distribution contract; это не отдельные сборки или сервисы. + Fitness — будущие проверки digest/reference parity, surface parity, отсутствие v3 route, warm agent session и ограниченный old/new overlap. Наличие этого файла не означает, что контейнеры уже собраны или опубликованы. diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 2277243..a53cb76 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -240,12 +240,120 @@ adds init/no-new-privileges and configurable resource/user/network arguments, but not read-only root, cap-drop or tmpfs. Catalog approval alone cannot correct that mismatch. No daemon changes or privileged retry were authorized. -Implementation is paused at the architecture failure-recovery gate for revised -written design/scope approval. Task4 remains incomplete; release controller and -CI Tasks5/6 have not started. Keeping full direct/Compose/production hardening -and making Gateway a separately accepted channel is a proposal, not an approved -contract change. The60s runtime bound remains unchanged, and amd64 acceptance -is not claimed. Do not merge, publish or close PR33 from this partial result. +Initial review paused implementation at the architecture failure-recovery gate. +The subsequent user-approved360-second/Compose correction is recorded below; +Gateway profile/scope remains unresolved. Task4 remains incomplete; release +controller and CI Tasks5/6 have not started. Deferring Catalog or accepting +different launcher security profiles requires an explicit design choice, not +inference from the timeout instruction. Do not merge, publish or close PR33 +from this partial result. + +### Approved360-second attempt and Compose override — scoped fix + +User explicitly requested «поставь 360 секунд и продолжай». Candidate +design/contract/plan refinement: `068c127`; signed code correction: +`9c1f3a72ce370fc816c32a6e0eeb2f773f23fb03`. Independent scoped re-review accepted +the timeout/Compose fixes without new critical or important findings. Runtime +now uses360seconds per attempt and20seconds per network read. +RPC/shutdown budgets are unchanged; early cancellation does not wait360seconds. +This is a changed safety budget, not a retrieval-performance improvement. + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_distribution.DistributionTests tests.test_v8std_mcp_runtime.RuntimeTests tests.test_v8std_mcp_runtime.WireTests -q +V8STD_MCP_SITE_URL=http://v8std.localhost:18765/kb/ .venv/bin/python scripts/check_mcp_container.py --mcp-image v8std-task4-mcp:fix360-9c1f3a7-amd64 --site-image sha256:78e79de36ed45fc9d620d66b8e8c4d0b673c276a867ed4d5bfe4b16e72bdcbfd --platform linux/amd64 --prefix /deliberately-unused/ +``` + +Two focused tests first failed on60≠360 and the ignored explicit SITE_URL, +then passed. Combined **79tests passed in55.810seconds**, including accelerated +real worker timeout/reaping, close and wire lifecycle tests. The existing +Starlette warning remains recorded. Ordinary graph validation and diff checks +passed; this is not the final full suite/strict-build gate. + +The runtime image was built after the scoped commit from a clean checkout, +with exact source SHA9c1f3a7. Local Engine index ID: +`sha256:0c282c458e6e19092c8d4e0db92b87eb2f8b05fa83ec6e7d5765221424579b79`; +amd64 platform manifest: +`sha256:26f35e15fd61beb5feca4a392e4701ac9757debdd43e15f15f0b4f2f1a8e15f8`. +These are local artifact identities, not a published multi-platform release. +The unchanged site prototype/corpus source SHA remainsf5c45d2; runtime and +corpus provenance are deliberately independent, not relabelled to match. + +Actual supervised amd64/QEMU harness exited0: + +| Scenario | Startup/polling/check duration | Result | +| --- | --- | --- | +| Cold-online stdio |118.81s| Ready, tools/Resources/snippet/links pass; EOF0 | +| Warm-offline stdio |111.58s| Verified cache and ranking preserved; EOF0 | +| Cold-online Compose HTTP |122.61s| Ready200,1423rows, exact runtimeSHA | +| Warm-offline HTTP |135.80s| Same corpus/cache, tools/links pass | + +These durations are not isolated generation CPU measurements. The harness +allows390seconds only for readiness polling; individual RPC/read/stop limits +were not inflated. Cold-offline HTTP still returned live200/ready503; +HTTP SIGTERM exited143 within the bounded stop, stdio SIGTERM exited0. +An explicit `/kb/` SITE_URL worked while the computed default +`/deliberately-unused/` source returned404. Cache namespace, selected source +and returned links followed the override. No public fallback was used. + +Direct/Compose non-root/read-only/cap-drop/no-new-privileges/init constraints +remain; MCP2CPU/1536MiB, site0.5CPU/128MiB. Own containers/networks/cache volumes +were removed, unrelated resources preserved, and no tests remained +running at handoff. The new image and evidence log remain for inspection. +Gateway was not changed or rerun; native amd64, registry and production evidence +are still absent. The earlier60-second failure remains valid historical evidence +and must not be rewritten as a pass. + +Scoped review retains a minor harness limitation for final integration: any +differing SITE_URL override currently requires the computed default source to +return404, unnecessarily rejecting valid fixtures where both URLs exist. Keep +that assertion specific to the override regression scenario when repairing the +integration helper. This does not invalidate the saved run where default404 +was intentional, and is not a runtime routing defect. Gateway remains the only +open important finding from Task4; no profile change was approved yet. + +### Subsequent approved Gateway profile + +The user subsequently approved launcher-owned profiles: production and our +direct Docker/Compose retain read-only/cap-drop; Gateway uses its actual native +isolation with explicitly documented differences. Missing readonly/capdrop/tmpfs +alone no longer defer Catalog. Candidate design/ADR/invariant/distribution +contract and the scoped Task4 plan were amended together. Required non-root, +init/no-new-privileges, nonprivileged mode and absence of Docker socket inside +MCP will be checked on each created Gateway server. This decision does not +claim that verification already passed or that cold routing, native amd64, +registry/Catalog acceptance or target-host deployment happened. Historical findings +above describe the contract at the time of the corresponding review. + +Focused implementation now passes15tests (0.246s): required controls and exact +owned cache mount, rejection of root/privileged/missing controls/socket aliases, +DinD preflight before launch (also under Python optimization), and explicit +default404 regression mode. Generic overrides no longer require a broken +default. Compose/runtime/images were not modified by this fix. + +Narrow actual verification used Gateway0.43.3 and two simultaneous warm stdio +sessions on network none; both initialized, listed/called tools, retained the +same server IDs across repeated calls and exited0 on EOF. Each server had +user10001:10001, init=true, SecurityOpt=no-new-privileges, privileged=false and +exactly the owned named cache volume, with no bind/socket. Observed readonly=false, +capdrop=null and tmpfs=null match the approved native profile. Fresh cache +preparation through the runtime took8.02s. This is launcher-profile evidence, +not a repeated full arm64/QEMU suite or cold-routing Gateway proof. + +The run intentionally reused earlier local arm64 prototypes, not a new release: +MCP Engine image ID `sha256:bec25fa5b9f240225db206c5e21d35a8c28e2d4ae30b1b878d272eacd9df1031`, +site ID `sha256:ef6ceb9d711a1cd6830f3536d593fccb9c82bb0f6257a3da04c81460b785bb43`, +both labelled source`f5c45d23fc31285e96920719285b0a9253e4a6fe`. +No current-source provenance claim follows from these prototype labels. +Evidence log: `/tmp/v8std-task4-gateway.Fi3MeO/acceptance.log`. +Only owned project`v8std-task4-5137d494d1` containers/networks/cache and temporary +Gateway config were removed; unrelated containers were preserved. +Independent scoped review accepted both remaining findings: Gateway profile +conformance and the generic default404 helper defect are ADDRESSED; no new +Critical/Important breakage or out-of-scope findings. The reviewer inspected +the immutable diff, report and actual saved two-server log without rerunning +tests. Task4 local implementation/verification is complete, not published or +signed as a new commit. Commit signing was not completed at this stage. Tasks5/6 and +final release gates are still pending; signing was not bypassed. ### Local build environment @@ -275,7 +383,7 @@ and query set on the implemented snapshot runtime. | --- | --- | | Bounded refresh/cache, crashes, shared volume, offline recovery | Task review accepted locally; Linux/runtime integration remains. | | Frozen generations, URL presentation, stdio/HTTP lifecycle | Task review accepted; known image-alt edge remains final release gate. | -| Runtime/static-site images, local request graph, Gateway sessions | Arm64/local profile and warm host Gateway pass; review requires Compose fix and explicit Gateway/gate design resolution; amd64 acceptance remains open. | +| Runtime/static-site images, local request graph, Gateway sessions | Task4 local review accepted, including360-second QEMU/Compose fixes and native Gateway15focused tests/two warm sessions. Latest fix remains uncommitted pending signing permission. Cold Gateway routing/native amd64 remain external gates. | | Restricted host controller, rollback and independent index delivery | Pending. | | Fail-closed publication, process v2 synchronization | Pending. | | Final semantic impact, merge-ready, fitness, strict build and full suite | Pending after all changes; strict build precedes the suite. | diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 59f1a29..06e2f98 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -45,6 +45,7 @@ requirements: - Snippet: 4000 default, 32000 maximum; query 500, preview 1000, tokens 80 и суммарно 4000; один hybrid search, прежний ranking и отсутствие raw procedure в usage logs. - Tool/resource request не скачивает данные и не ждёт refresh; одновременно видит одно валидное поколение. - Один multi-platform image на версию, `linux/amd64` и `linux/arm64`; catalog review может отставать, но не создаёт другую сборку той же версии. +- Production/наш direct Docker/Compose сохраняют read-only rootfs, cap-drop ALL, no-new-privileges, init и bounded tmpfs. Gateway использует нативный профиль: non-root/init/no-new-privileges, без privileged MCP и Docker socket; отсутствующие readonly/capdrop/tmpfs документируются, не блокируя Catalog сами по себе. - Local site и MCP не делают background public egress. Cold-offline thin image без cache не готов; warm-offline с проверенным cache работает. - Основной checkout, существующая ветка `codex/mcp-container-distribution-design`; не создавать worktree, не менять main, не пушить и не деплоить во время реализации. - TDD и focused tests для каждого поведения. Strict build выполняется **до** полного suite: tests читают `site/LICENSES`, параллельная пересборка разрушает их вход. @@ -305,7 +306,7 @@ default CMD selects stdio; HTTP command selects host 0.0.0.0/port 8000. Static image consumes already built local `site` and included snapshot, not the runtime image. Keep existing source-bind Compose explicitly dev. -- [ ] **RED:** Tests execute the local-profile build into a temporary directory, +- [x] **RED:** Tests execute the local-profile build into a temporary directory, then assert HTTP pages+manifest/archive work without external network. Container harness initializes stdio twice with shared volume and verifies a tool call, generation ID/cache reuse, EOF exit and runtime readiness: @@ -318,19 +319,20 @@ self.assertEqual(container_inspect["Config"]["User"], "10001:10001") Use actual current serverInfo name from build_server if it differs; this is a named compatibility check, not permission to rename the server. -- [ ] **GREEN images/profile:** Resolve pinned base digests and full dependency +- [x] **GREEN images/profile:** Resolve pinned base digests and full dependency lock; copy only runtime modules/rules/licenses. Non-root UID/GID 10001, read-only - root, persistent writable cache, tmpfs, cap-drop and init. Local profile disables + root, persistent writable cache, tmpfs, cap-drop and init in our owned launches; + native Gateway uses the separately approved profile below. Local profile disables analytics/recorder and external fonts/assets, preserving public profile. Same archive bytes are used for public/local manifests. Build on arm64 and exercise amd64 in available Docker emulation, noting native-CI gate separately. -- [ ] **GREEN launch/catalog:** Compose references published image coordinates +- [x] **GREEN launch/catalog:** Compose references published image coordinates with explicit version/digest override for local test images, loopback published ports, optional MCP profile, named cache and one common routable SITE_URL. Document/test desktop host access and Linux host-gateway. Catalog points to self-published image, declares site/snippet/cache and long-lived stdio behavior; validate against current Docker schema without submitting an external PR. -- [ ] **Verify:** Run distribution unit/integration harness, real Docker stdio/HTTP, +- [x] **Verify:** Run distribution unit/integration harness, real Docker stdio/HTTP, non-root read-only startup, persistent cache offline restart, local site request graph without public egress and multi-session Gateway where locally available. Verify licenses/SBOM inputs. Record unavailable external catalog acceptance @@ -343,26 +345,58 @@ This supersedes the original numerical budget only. Gateway scope/security decisions and external mutation authority are not inferred from that change. Keep historical measurements labelled with their original 60-second build. -- [ ] **RED:** In `tests/test_v8std_mcp_snapshots.py`, assert a default store +- [x] **RED:** In `tests/test_v8std_mcp_snapshots.py`, assert a default store uses `360` attempt seconds and `20` read seconds; retain accelerated real worker timeout/reaping, close and responsive-query tests. In distribution tests resolve Compose with an explicit alternate SITE_URL and prove it is passed intact instead of replaced by the default local URL. -- [ ] **GREEN:** Set `ATTEMPT_SECONDS = 360` in +- [x] **GREEN:** Set `ATTEMPT_SECONDS = 360` in `scripts/v8std_mcp_snapshots.py`. Compose consumes `${V8STD_MCP_SITE_URL:-http://v8std.localhost:${V8STD_SITE_PORT:-18765}${V8STD_SITE_PREFIX:-/}}`. Test actual Compose interpolation; do not assume nested defaults work without executing its config resolver. Preserve local default, prefix and one setting. Adapt only the integration startup wait to allow the accepted attempt plus bounded startup margin; do not turn RPC/read/shutdown timeouts into360seconds. -- [ ] **VERIFY:** Run focused snapshot/distribution tests, commit the exact +- [x] **VERIFY:** Run focused snapshot/distribution tests, commit the exact changed runtime, build a new amd64 image from that clean source SHA and rerun full-corpus supervised cold/warm stdio/HTTP acceptance under QEMU. Exercise an explicit reachable SITE_URL override end-to-end, checking source and returned links. No privileged Gateway retry, image publication or native-CI claim. -- [ ] **REVIEW:** Independent scoped review of the fix diff and evidence. - Gateway discrepancy remains separately open until an approved design decision; - passing these checks alone does not close Task4 or the full release plan. +- [x] **REVIEW:** Independent scoped review of the fix diff and evidence. + These checks resolved Compose/QEMU findings only. The subsequent explicit + Gateway profile approval and its scoped verification are recorded below. + +#### Task 4 reviewed fixes — approved launcher-owned security profiles + +User approved retaining the strict owned-launch profile and using Gateway's +actual native isolation, without deferring Catalog solely for absent flags. +The candidate design/ADR/invariant/distribution contract now express that +distinction. No main structured document, public API, image identity or other +release acceptance boundary changes. This approval does not authorize daemon, +socket, host or registry changes. + +- [x] **RED:** Add focused inspection-validator tests in + `tests/test_v8std_mcp_distribution.py`: an actual-shaped native Gateway state + with `ReadonlyRootfs=False`/`CapDrop=None` is accepted only with user10001:10001, + init/no-new-privileges, nonprivileged mode and no Docker socket mount. Reject + each missing required control and socket aliases/mount destinations. Verify + the generic override helper accepts two valid URLs; require default404 only + when the explicit regression flag is enabled. +- [x] **GREEN:** In `scripts/check_mcp_container.py`, validate and report every + created Gateway server's required controls and observed optional controls. + Add `--require-default-source-404` for the intentional override regression; + keep source/link/namespace checks for every override. Update Catalog comments + and `docs/container-installation.md` to state both profiles and remaining + external gates. Do not invent unsupported Catalog fields or modify Compose + protections. Keep helper logic importable for focused tests. +- [x] **VERIFY:** Run focused distribution tests and actual two-session warm + Gateway check using isolated config, already verified cache and network none. + Inspect all session containers and retain actual profile evidence. Use only + task-owned Docker resources; no socket escalation, cold-network workaround, + public publication or unnecessary repeat of accepted QEMU/360 tests. +- [x] **REVIEW:** Scoped re-review of the Gateway finding against the approved + amended contract, helper regression and observed evidence before closing + Task4. Cold Gateway routing/native CI/publication remain external gates. ### Task 5: Restricted release transaction and independent index store diff --git a/tests/test_v8std_mcp_distribution.py b/tests/test_v8std_mcp_distribution.py index 7b25b30..8e5c8ff 100644 --- a/tests/test_v8std_mcp_distribution.py +++ b/tests/test_v8std_mcp_distribution.py @@ -8,10 +8,150 @@ import sys import tempfile import unittest +from unittest.mock import patch import yaml ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import check_mcp_container as harness + + +class GatewayProfileTests(unittest.TestCase): + def validate(self, state): + return harness.validate_gateway_profile(state, expected_cache={ + "Name": "owned-cache", "Mountpoint": "/var/lib/docker/volumes/owned-cache/_data"}) + + def test_unsafe_inherited_gateway_settings_are_rejected_without_mutation(self): + for value in ("true", "1", "false", "0"): + env = {"DOCKER_MCP_IN_DIND": value, "PATH": "/some/path"} + before = dict(env) + with self.subTest(value=value), self.assertRaises(AssertionError): + harness.gateway_environment(env) + self.assertEqual(env, before) + env = {"PATH": "/some/path"} + self.assertEqual(harness.gateway_environment(env), env) + + def state(self): + return {"Id": "session-server", "Image": "sha256:" + "a" * 64, + "Config": {"User": "10001:10001", "WorkingDir": "/opt/v8std"}, + "HostConfig": {"Init": True, "Privileged": False, + "SecurityOpt": ["no-new-privileges=true"], + "ReadonlyRootfs": False, "CapDrop": None, + "Tmpfs": None, "NetworkMode": "none"}, + "Mounts": [{"Type": "volume", "Name": "owned-cache", + "Source": "/var/lib/docker/volumes/owned-cache/_data", + "Destination": "/var/lib/v8std-mcp", "RW": True}]} + + def test_privileged_environment_is_rejected_before_any_gateway_or_docker_launch(self): + with patch.dict(os.environ, {"DOCKER_MCP_IN_DIND": "1"}), \ + patch.object(harness, "run") as docker, patch.object(harness, "Stdio") as launch: + with self.assertRaisesRegex(AssertionError, "unsafe DOCKER_MCP_IN_DIND"): + harness.host_gateway_check("not-launched", "test-image", "test-cache", + "http://v8std.localhost/", Path("/not-used")) + docker.assert_not_called() + launch.assert_not_called() + + def test_privileged_preflight_is_not_disabled_by_python_optimization(self): + code = "from check_mcp_container import gateway_environment; gateway_environment({'DOCKER_MCP_IN_DIND':'1'})" + result = subprocess.run([sys.executable, "-O", "-c", code], cwd=ROOT / "scripts", + capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe DOCKER_MCP_IN_DIND", result.stderr) + + def test_native_profile_reports_required_and_optional_controls(self): + for security_opt in ("no-new-privileges", "no-new-privileges:true", "no-new-privileges=true"): + state = self.state() + state["HostConfig"]["SecurityOpt"] = [security_opt] + result = self.validate(state) + self.assertEqual(result["id"], state["Id"]) + self.assertEqual(result["image_id"], state["Image"]) + self.assertEqual(result["user"], "10001:10001") + self.assertTrue(result["init"]) + self.assertTrue(result["no_new_privileges"]) + self.assertFalse(result["privileged"]) + self.assertFalse(result["read_only"]) + self.assertIsNone(result["cap_drop"]) + self.assertIsNone(result["tmpfs"]) + self.assertEqual(result["mounts"], state["Mounts"]) + + def test_native_profile_rejects_each_missing_or_disabled_required_control(self): + for section, key, value in (("Config", "User", "0:0"), + ("Config", "User", "10001:0"), + ("HostConfig", "Init", False), + ("HostConfig", "Privileged", True), + ("HostConfig", "SecurityOpt", []), + ("HostConfig", "SecurityOpt", ["no-new-privileges=false"]), + ("HostConfig", "SecurityOpt", ["no-new-privileges:true", "no-new-privileges:false"])): + for missing in (False, True): + with self.subTest(key=key, value=value, missing=missing): + state = self.state() + if missing: + del state[section][key] + else: + state[section][key] = value + with self.assertRaises(AssertionError): + self.validate(state) + state = self.state() + del state["Mounts"] + with self.assertRaises(AssertionError): + self.validate(state) + + def test_socket_sources_aliases_and_destinations_cannot_hide_in_mounts(self): + mounts = [ + {"Type": "bind", "Source": "/var/run/docker.sock", "Destination": "/socket-alias"}, + {"Type": "bind", "Source": "/Users/operator/.docker/run/docker.sock", "Destination": "/var/lib/v8std-mcp"}, + {"Type": "bind", "Source": "/tmp/opaque-daemon-alias", "Destination": "/var/lib/v8std-mcp"}, + {"Type": "bind", "Source": "/tmp/opaque-daemon-alias", "Destination": "/run/docker.sock"}, + {"Type": "volume", "Source": "/var/run/docker.raw.sock", "Destination": "/var/lib/v8std-mcp"}, + {"Type": "volume", "Source": "/cache", "Destination": "/var/run/docker.sock"}, + {"Type": "bind", "Source": "/run", "Destination": "/daemon-directory"}, + ] + for mount in mounts: + with self.subTest(mount=mount), self.assertRaises(AssertionError): + state = self.state() + state["Mounts"] = [mount] + self.validate(state) + + def test_only_exact_writable_owned_cache_mount_is_allowed(self): + for field, value in (("Name", "other-cache"), ("Source", "/unexpected/alias"), + ("Destination", "/unexpected"), ("RW", False)): + state = self.state() + state["Mounts"][0][field] = value + with self.subTest(field=field), self.assertRaises(AssertionError): + self.validate(state) + state = self.state() + state["Mounts"].append(dict(state["Mounts"][0])) + with self.assertRaises(AssertionError): + self.validate(state) + + def test_optional_hardening_is_reported_when_present(self): + state = self.state() + state["HostConfig"].update(ReadonlyRootfs=True, CapDrop=["ALL"], Tmpfs={"/tmp": "size=64m"}) + result = self.validate(state) + self.assertTrue(result["read_only"]) + self.assertEqual(result["cap_drop"], ["ALL"]) + self.assertEqual(result["tmpfs"], {"/tmp": "size=64m"}) + + +class SiteOverrideTests(unittest.TestCase): + def test_two_valid_urls_do_not_require_default_to_fail(self): + with patch.object(harness, "http", return_value=(200, {}, b"")) as request: + self.assertEqual(harness.check_default_source("http://selected.localhost/kb/", + "http://default.localhost/kb/"), {}) + request.assert_not_called() + + def test_explicit_regression_flag_requires_404_and_distinct_source(self): + selected, default = "http://selected.localhost/kb/", "http://default.localhost/kb/" + with patch.object(harness, "http", return_value=(200, {}, b"")): + with self.assertRaises(AssertionError): + harness.check_default_source(selected, default, require_404=True) + with patch.object(harness, "http", return_value=(404, {}, b"")) as request: + self.assertEqual(harness.check_default_source(selected, default, require_404=True), + {"default_source_status": 404}) + request.assert_called_once_with(default + "ai/mcp/v1/manifest.json") + with self.assertRaises(AssertionError): + harness.check_default_source(selected, selected, require_404=True) class DistributionTests(unittest.TestCase): From 428c43ae95ea8bd1f282ebf084b836c2a2845e2c Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Thu, 10 Sep 2026 22:42:48 +0300 Subject: [PATCH 30/88] docs: plan initial MCP container release and maintenance window --- ...26-09-10-mcp-published-combined-runtime.md | 4 +- ...09-10-mcp-recoverable-container-release.md | 9 + spec/contracts/mcp-release-runtime-v1-r0.md | 44 ++- ...09-10-mcp-container-distribution-design.md | 18 +- ...mcp-release-has-recoverable-predecessor.md | 10 +- spec/operations/mcp-container-verification.md | 98 ++++++- .../mcp-first-container-release-roadmap.md | 257 ++++++++++++++++++ ...6-09-10-mcp-container-distribution-plan.md | 104 ++++++- 8 files changed, 534 insertions(+), 10 deletions(-) create mode 100644 spec/operations/mcp-first-container-release-roadmap.md diff --git a/spec/adr/2026-09-10-mcp-published-combined-runtime.md b/spec/adr/2026-09-10-mcp-published-combined-runtime.md index 718997c..f795afb 100644 --- a/spec/adr/2026-09-10-mcp-published-combined-runtime.md +++ b/spec/adr/2026-09-10-mcp-published-combined-runtime.md @@ -75,5 +75,7 @@ runtime. Ранее отклонённый API 3.0 не возобновляет - Раздельные production/local/catalog образы: разные пути выпуска и проверки. - Отдельный сервис v3: отсутствует необходимость в изоляции пользователей v3. -- Остановка старого до проверки нового: увеличивает риск аварии при rollout. +- Остановка старого до проверки нового в обычном автоматическом rollout. + Для первой ручной миграции пользователь отдельно разрешил резервное окно + с простоем; правила возврата определены release contract. - Snapshot внутри runtime image: связывает выпуск статей с рестартом сервера. diff --git a/spec/adr/2026-09-10-mcp-recoverable-container-release.md b/spec/adr/2026-09-10-mcp-recoverable-container-release.md index f877577..87d04cc 100644 --- a/spec/adr/2026-09-10-mcp-recoverable-container-release.md +++ b/spec/adr/2026-09-10-mcp-recoverable-container-release.md @@ -46,11 +46,20 @@ Host выполняет ограниченную по времени идемп switch приводит к rollback. SSH служит каналом передачи задания, не владельцем жизни транзакции; host восстанавливает состояние после потери соединения. +Первая миграция Python → container имеет отдельную операторскую процедуру. +При недостатке памяти для overlap пользователь допускает stop/start в заранее +назначенном окне до двух часов. Сохраняются проверка артефакта, локальный +predecessor, host-owned recovery и ограниченные попытки. CI credential не может +выбрать эту процедуру. Первый container predecessor регистрируется только после +реального public smoke, а не создаётся искусственно для обхода обычного deploy. + ## Влияние на инварианты Добавляется проверяемый recoverable predecessor; единый endpoint и атомарность snapshot сохраняются. Capacity gate включает overlap двух runtime и staging индекса: нельзя обещать бесшовность, если они не помещаются в память host. +Исключение первой ручной миграции допускает простой, но не потерю predecessor. +Оно не отменяет capacity gate последующих автоматических обновлений. ## Влияние на контракты diff --git a/spec/contracts/mcp-release-runtime-v1-r0.md b/spec/contracts/mcp-release-runtime-v1-r0.md index a2784fe..bada917 100644 --- a/spec/contracts/mcp-release-runtime-v1-r0.md +++ b/spec/contracts/mcp-release-runtime-v1-r0.md @@ -55,6 +55,10 @@ Release ID идемпотентен: повтор с теми же полями ## Состояния и переходы +Этот автоматический путь применяется к уже зарегистрированному container +predecessor. Первая ручная миграция описана отдельно ниже; отсутствие +`active.json` не разрешает подставить фиктивный predecessor. + ```text RECEIVED → VERIFIED → PREPARED → READY → SWITCHED → COMMITTED └──────── до SWITCHED: FAILED, старый runtime продолжает работать @@ -83,7 +87,41 @@ drain старого runtime до 30 секунд и окончательный проверяет старый endpoint; candidate прекращает admission и завершается. Rollback failure — отдельный terminal `RECOVERY_REQUIRED` с alert и сохранёнными артефактами; нельзя обозначать его как успешный rollback. Предыдущий runtime -не останавливается до успешного post-switch smoke нового. +не останавливается до успешного post-switch smoke нового в автоматическом пути. + +## Первая ручная миграция + +Оператор заранее фиксирует проверенный main SHA, image/platform/configuration +digests, corpus ID, начало и конец окна в UTC и сохранённый Python deployment. +Окно не длиннее двух часов; без назначенного окна или при уже принятом container +predecessor первоначальный stop/start запрещён. CI forced command не принимает +bootstrap и не может менять операторское разрешение. Вход не содержит shell. +Истечение окна запрещает новую попытку, но не восстановление уже начатой. + +Предпочтителен overlap, если его capacity проверена. При нехватке overlap RAM +допускается только в этом окне: сохранить прежние config/data → заранее получить +и проверить образ/corpus → запустить независимый от SSH recovery guard → +остановить старый MCP → запустить candidate → readiness и MCP smoke → +nginx switch/public smoke → зарегистрировать первый container predecessor. +nginx, TLS, мониторинг и static index store не останавливаются. + +Если подготовка, запуск или smoke неуспешны, owned candidate останавливается, +возвращаются прежний upstream и Python service, проверяются endpoint и прежние +данные. При невозможности восстановления результат — `RECOVERY_REQUIRED`, +не успешный rollback. Journal и recovery guard охватывают также промежуток +между stop старого и start нового и сбой при записи первого `active.json`. +Повтор не создаёт ещё один runtime; искусственная запись `COMMITTED` запрещена. + +Одна попытка сохраняет бюджеты 300 s transaction / 90 s readiness / 30 s smoke / +45 s stop и запас на rollback; 360 s loader и 20 s read не увеличиваются до двух +часов. Повтор возможен только после проверенного восстановления и с новым ID. +Не позднее чем за 30 минут до конца окна новые попытки прекращаются; если новый +сервис не принят, выполняется возврат и проверка старого. Если репетиция требует +больше времени на возврат, резерв увеличивается до начала окна. + +Первый успех не включает автоматический runtime deploy сам по себе. Для него +по-прежнему нужны отдельная активация и память для old/new overlap. Эта ручная +процедура не является скрытым stop/start fallback автоматического контроллера. ## Crash recovery и данные @@ -117,7 +155,9 @@ wrapper без готового контейнера не удовлетворя Перед переключением capacity check проверяет disk headroom для pull/staging, RAM для old+new runtime и index preparation, file descriptors и сетевой бюджет. -При дефиците switch не начинается. Admission ограничивает одновременно +При дефиците автоматический switch не начинается. Для первой ручной миграции +без overlap измеряются один новый runtime, preparation, nginx и системные службы; +само двухчасовое окно не компенсирует недостаток RAM. Admission ограничивает одновременно выполняемые MCP запросы, idle keep-alive и большие downloads раздельно; перегрузка отвечает retryable status по прежней edge policy. Конкретные настройки допускаются в production только после mixed-load evidence. diff --git a/spec/designs/2026-09-10-mcp-container-distribution-design.md b/spec/designs/2026-09-10-mcp-container-distribution-design.md index bf7c2c5..aefa28b 100644 --- a/spec/designs/2026-09-10-mcp-container-distribution-design.md +++ b/spec/designs/2026-09-10-mcp-container-distribution-design.md @@ -217,6 +217,13 @@ no-new-privileges, отсутствия privileged mode и Docker socket вну Потеря SSH и отмена CI не бросают host в промежуточном состоянии. Это подтверждается fault-injection по каждому переходу release contract. +Уточнение первой миграции, согласованное пользователем: если одновременный +запуск не помещается в память, допускается остановка прежнего Python MCP +в отдельно назначенном окне длительностью до двух часов. Это ручная начальная +миграция с сохранённым и проверенным возвратом, не режим автоматического CI +rollout. Без даты/времени окна и разрешения на конкретный проверенный SHA +остановка не начинается. Окно не увеличивает бюджеты загрузчика и транзакции. + ### MCP_SHARED_HOST_LOAD_IS_MEASURED На production-подобном стенде одновременно воспроизводятся короткие MCP POST, @@ -283,9 +290,14 @@ last-good snapshot. Runtime rollout не требует одновременно Docker migration требует отдельной подготовки целевого сервера и свежего замера ресурсов до начала работ. -Одновременные old/new runtime плюс staging нового index должны поместиться -с запасом; иначе до включения rollout нужен более ёмкий host. Увеличение лимита -nginx не устраняет дефицит памяти или полосы. +Для автоматического rollout одновременные old/new runtime плюс staging нового +index должны поместиться с запасом. Для первой ручной миграции разрешён +описанный выше stop/start: отдельно проверяется, что новый runtime, preparation, +nginx и системные службы помещаются без старого MCP. Если это не выполняется, +миграция не начинается. Если помещается только один runtime, первая миграция +возможна, но автоматическая смена runtime остаётся выключенной до решения +capacity gate. Публикация образов и доставка corpus могут работать независимо. +Увеличение лимита nginx не устраняет дефицит памяти или полосы. На выделенном host остаются ai.v8std.ru, SSH, TLS renewal, защита и мониторинг. Посторонний vhost нельзя удалить вслепую: сначала backup вне host, проверка diff --git a/spec/invariants/mcp-release-has-recoverable-predecessor.md b/spec/invariants/mcp-release-has-recoverable-predecessor.md index fd1db74..61a82c8 100644 --- a/spec/invariants/mcp-release-has-recoverable-predecessor.md +++ b/spec/invariants/mcp-release-has-recoverable-predecessor.md @@ -23,12 +23,20 @@ required_when: implemented transaction. Повтор и устаревшее задание не откатывают более новый успешный release. Потеря invoker не отменяет recovery на host. -Сбой до switch оставляет старый serving runtime. Сбой после switch приводит +В обычном автоматическом rollout сбой до switch оставляет старый serving runtime. +Сбой после switch приводит к проверяемому rollback либо явному `RECOVERY_REQUIRED`, но не ложному успеху. При нехватке памяти для old/new overlap или диска для pinned data переключение не начинается. Для первого container cutover predecessor — сохранённый и проверенный старый Python deployment; он не считается обычным container release. +Исключение для первой ручной миграции: в явно назначенном окне до двух часов +старый Python MCP разрешено остановить раньше старта нового при нехватке +overlap capacity. Его файлы/config/data сохраняются, а host recovery восстанавливает +старый endpoint при неуспехе; invoker loss не должен оставлять сервис выключенным. +Запуск вне окна и запрос такого режима через CI запрещены. Нехватка памяти +для одного нового runtime с preparation остаётся запретом миграции. + Fitness — будущая матрица faults на переходах release contract, с проверкой реального endpoint, exact digest и данных после восстановления. Unit mock успешного `docker restart` не доказывает этот инвариант. diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index a53cb76..9ebefdd 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -383,14 +383,67 @@ and query set on the implemented snapshot runtime. | --- | --- | | Bounded refresh/cache, crashes, shared volume, offline recovery | Task review accepted locally; Linux/runtime integration remains. | | Frozen generations, URL presentation, stdio/HTTP lifecycle | Task review accepted; known image-alt edge remains final release gate. | -| Runtime/static-site images, local request graph, Gateway sessions | Task4 local review accepted, including360-second QEMU/Compose fixes and native Gateway15focused tests/two warm sessions. Latest fix remains uncommitted pending signing permission. Cold Gateway routing/native amd64 remain external gates. | -| Restricted host controller, rollback and independent index delivery | Pending. | +| Runtime/static-site images, local request graph, Gateway sessions | Task4 local review accepted, including360-second QEMU/Compose fixes and native Gateway15focused tests/two warm sessions; signed handoff3165cc7 completed. Cold Gateway routing/native amd64 remain external gates. | +| Restricted host controller, rollback and independent index delivery | Task5 partial/uncommitted:29 release tests passed; regression reruns, review and first-bootstrap slice remain. | | Fail-closed publication, process v2 synchronization | Pending. | | Final semantic impact, merge-ready, fitness, strict build and full suite | Pending after all changes; strict build precedes the suite. | | Final image smoke, refresh RSS/CPU, disposable mixed load | Pending; cannot establish production 100k capacity. | ## External acceptance boundary +### Refreshed release-request preflight + +Normal signed commit`3165cc7e6ef2afcf12cdeca66c8070ab0737ba5b` now contains the +approved Gateway fix and candidate graph/evidence. The four reviewed packaging +files matched their immutable review snapshot before commit. Signing-pending +statements above are historical. Tasks5/6 are not completed by this commit. + +Read-only checks on2026-09-10 found the public default snapshot manifest still404; +legacy `https://ai.v8std.ru/healthz` is200 with1423pages/3281vectors. +nginx syntax succeeds. Private host and TLS inventory is retained outside Git; +these observations do not establish capacity or migration readiness. +No host mutation, cleanup, tariff change or MCP cutover was performed. + +GitHub main remains`b7bef11e145a188b30e7a7b17df2be4cb1acbd0c`. +Protection, environment and credential readiness require separate private checks. +An anonymous GHCR token request for`zeegin/v8std-mcp:pull` also returned403. +This establishes no public availability, not proof of package nonexistence or +of future workflow inability to publish with its own permitted GITHUB_TOKEN. + +### Catalog producer round-trip gap + +Docker registry remains at`8c773729f13f036da8c909be503fe433923a9aa2`. +Actual `catalog.ToTile` from that pinned upstream module was executed on the +current source entry, not a hand-built projection. It returned: + +```json +{"input_long_lived":true,"output_long_lived_present":false, + "input_defaults":{"cache_volume":"v8std-mcp-cache","max_snippet_chars":4000,"site_url":"https://v8std.ru/"}, + "output_defaults":{"cache_volume":null,"max_snippet_chars":null,"site_url":null}} +``` + +User/env/volume settings survive. Source `pkg/catalog/types.go` has no LongLived +field and `tile.go` cannot propagate it; the documented `cmd/catalog` uses this +conversion. Gateway0.43.3 defaults its global long-lived option to false, while +client reuse requires the server flag or global option. Therefore the earlier +explicit `--long-lived` warm harness does not prove the generated Catalog path. +No failed end-user session is claimed from this conversion-only probe; actual +generated-entry lifecycle acceptance must accompany the repair. + +Probe used pinned Go1.25.11 image, non-root/read-only/cap-drop/no-new-privileges, +2CPU/768MiB,180s outer bound and no Docker socket. Final run copied go.mod to a +private non-root temp subdirectory (Go ignores a go.mod at the temp root), then +ran the exact upstream module; exit0. Source/evidence inputs remain in +`/tmp/v8std-catalog-path.r0u2kk/`. Both disposable probe containers exited and +were automatically removed; no default Catalog or user Docker settings changed. +The local GitHub license label is Other/NOASSERTION; upstream's actual license +filter rejects gpl/agpl/npl prefixes, so Other is not an automatic validator +failure. No license text or licensing terms were changed. + +Limited static-index/CI setup and an upstream PR preserving LongLived require +separate authorization; local verification does not authorize those writes. Direct image publication and external Docker review +remain distinct outcomes; the server/runtime artifact contract is unchanged. + Registry release digests, provenance verification against those releases, Docker Catalog acceptance, GitHub protection/environment/secrets, target-host prerequisites and initial controlled activation require separate evidence. @@ -398,3 +451,44 @@ The existing Python deployment remains the initial rollback path until that activation is explicitly performed and verified. An automatically enabled release path must not be inferred from a locally passing test or a written workflow. + +### Correction: test Catalog versus Docker-published Catalog, 2026-09-10 + +The earlier `ToTile` result is real but establishes a defect in the **local +test-catalog generator**, not loss in Docker's published catalog. Upstream +[Taskfile](https://github.com/docker/mcp-registry/blob/8c773729f13f036da8c909be503fe433923a9aa2/Taskfile.yml) +explicitly labels that command as generating a test catalog. Live GET of both +official catalogs on2026-09-10 found `longLived: true` for desktop-commander, +playwright, apify-mcp-server, inspektor-gadget and schemacrawler-ai. + +| Published artifact | Entries | Remote entries | SHA256 | +|---|---:|---:|---| +| [v2](https://desktop.docker.com/mcp/catalog/v2/catalog.yaml) |270|30|`fc371f25332f1509983734c642c92b6319a0589e1c2b41edce2ad547675a9208`| +| [v3](https://desktop.docker.com/mcp/catalog/v3/catalog.yaml) |317|77|`274afe7ad34b083c3d3140664a3bc8762246f3887e840a9b4d4a3c44fce3b80a`| + +Support was merged in [Gateway PR26](https://github.com/docker/mcp-gateway/pull/26). +The exact production transformation was not traced, and no v8std published +entry/default end-user session was tested. Requiring an upstream generator fix +before submitting an entry or releasing the image was an unsupported inference. +The user chose image-only publication; no upstream PR/Catalog submission occurred. +The implementation plan now separates this future channel gate from release. + +### Planning handoff: first migration and remaining implementation + +The planned first-migration stop/start is bounded to two hours if old/new +cannot coexist. The actual window must be separately authorized. This does not extend +loader/transaction deadlines, authorize stopping production now, or prove enough +memory for the new runtime alone. Normal automatic rollout still requires overlap. + +Task5 owner paused safely on request for this planning turn; no new commits or +owned diagnostics remain. Last29 release tests passed in60.515s; earlier2 hold +tests passed before subsequent edits. The89-test snapshot/runtime run had3 failures; +the owner reports fixes, but verification has not been rerun. Security/ingress +review, Docker/nginx checks, regressions, activation runbook/report and independent +Task5 review remain. The working ordinary controller requires an existing +container `active.json`; initial Python-to-container bootstrap is a separate +unfinished slice, now explicit in the plan. No successful release is inferred. + +See [the remaining-work roadmap](mcp-first-container-release-roadmap.md) for +local gates, external source/image publication, initial window, rollback reserve +and conditional activation of subsequent automated runtime updates. diff --git a/spec/operations/mcp-first-container-release-roadmap.md b/spec/operations/mcp-first-container-release-roadmap.md new file mode 100644 index 0000000..581352d --- /dev/null +++ b/spec/operations/mcp-first-container-release-roadmap.md @@ -0,0 +1,257 @@ +# MCP: план первого выпуска Docker и перехода публичного сервера + +Дата: 2026-09-10. Это план дальнейших работ на согласование, не отчёт о выпуске. +Он связывает локальный implementation plan с последующими операциями публикации +и первой миграции. Внешние операции ниже намеренно не являются checkboxes +structured plan: их нельзя требовать до merge-ready и одновременно выполнять +только после проверенного main. + +**Goal:** опубликовать рабочий `ghcr.io/zeegin/v8std-mcp`, обеспечить публичный +источник индексов и перевести `ai.v8std.ru/mcp` на тот же опубликованный образ. + +**Architecture:** один runtime image для локального stdio/HTTP и production; +независимые выпуски corpus; nginx раздаёт immutable архивы вне MCP. Первый +переход — отдельная операторская операция с возвратом к Python deployment; +обычные автоматические обновления используют проверенный container predecessor. + +**Tech Stack:** текущие Python/MCP, Docker/Compose, nginx/systemd, GitHub Actions +и GHCR. Новый поисковый движок или новый формат быстрого дискового индекса +в первый выпуск не добавляются. + +**Spec:** [design](../designs/2026-09-10-mcp-container-distribution-design.md), +[release contract](../contracts/mcp-release-runtime-v1-r0.md), +[implementation plan](../plans/2026-09-10-mcp-container-distribution-plan.md), +[CI policy plan](../plans/2026-09-10-mcp-ci-deployment-policy-plan.md). + +## Global Constraints + +- Один `/mcp`, один опубликованный runtime image на версию; без отдельного v3. +- `V8STD_MCP_SITE_URL` выбирает и данные, и адреса ответов; второй URL-setting нет. +- Persistent cache volume сохраняется при замене контейнера. Рабочие запросы + используют готовый индекс в памяти и не инициируют скачивание. +- Новый процесс читает/проверяет cache и готовит поисковые структуры. Это ещё + не быстрый старт из полностью материализованного индекса. Измерять отдельно + download, cache verification, preparation, readiness и query latency. +- Первая поставка — Docker image и публичный endpoint; Catalog пока отложен. + Потеря `longLived` в локальном тестовом генераторе Docker не блокирует выпуск. +- Нельзя объявлять 100000 подключений подтверждёнными без соответствующего + смешанного нагрузочного теста. Числа CPU/RAM/FD сами по себе такого права не дают. +- Основной checkout, ветка `codex/mcp-container-distribution-design`. + Structured files из main не изменяются. Код в работе не удаляется и не теряется. +- До live-операций нужны отдельные разрешения на точные host/settings targets; + до остановки MCP — назначенное окно и проверенный SHA main/digest. +- Окно первой миграции до120минут не увеличивает 360s loader /20s read, + 300s transaction /90s readiness /30s smoke /45s stop. + +## Зафиксированная исходная точка + +| Часть | Доказанное состояние на момент планирования | +|---|---| +| Формат snapshot, cache, runtime, образы | Tasks1–4 прошли локальные scoped reviews; повторять реализацию не нужно | +| Текущая ветка | Последний signed commit `3165cc7e6ef2afcf12cdeca66c8070ab0737ba5b`; изменения Task5 не закоммичены | +| Task5 release/controller | Приостановлен. Последние29 release-тестов GREEN; hold и snapshot/runtime после последних исправлений требуют повторного прогона и review | +| Первый переход | Обычный `deploy` уже требует `active.json` и container predecessor; bootstrap со старого Python service ещё нужно реализовать и проверить | +| CI | Task6 не реализован; существует прежний Pages workflow | +| Публичный bootstrap | `https://v8std.ru/ai/mcp/v1/manifest.json` в осмотре10сентября отвечал404 | +| Целевой сервер | Ресурсы и зависимости проверяются в закрытом preflight; прежние наблюдения не доказывают готовность к миграции | +| GitHub setup | Защита main, production environment и ограниченные credentials требуют отдельной проверки перед активацией | +| Окно | Stop/start ограничен двумя часами; конкретное окно требует отдельного согласования до остановки сервиса | + +Эти наблюдения не заменяют свежий preflight перед публикацией/миграцией. +Успешные тесты ранних задач не означают, что текущий dirty checkout готов к merge. + +## Этап 1. Завершить код поставки и первичной миграции + +Владелец: исполнитель Task5, затем независимый scoped reviewer. Сначала +продолжить сохранённую работу; после стабильного обычного контроллера выполнить +отдельный first-migration slice из обновлённого implementation plan. + +Файлы: `scripts/v8std_mcp_release.py`, `scripts/v8std_mcp_hold.py`, затронутые +runtime/snapshot modules, `deploy/container/`, release/hold tests и +`spec/operations/mcp-container-activation.md`. + +Порядок: + +1. Перепроверить исправленные3 сбоя snapshot/runtime; сверить hold, ingress, + recovery и завершающие операции после COMMITTED. Не считать прошлый GREEN + доказательством ещё не проверенных изменений. +2. Закончить static store и ограниченный upload ingress. Проверить реальным + nginx GET/HEAD, SHA/длину, cache headers, отсутствие immutable404 и доступность + файла при остановленном MCP. +3. Реализовать first-bootstrap без фиктивного `active.json`: операторское окно, + резервная копия legacy, host-owned recovery, smoke и запись первого accepted + container. CI не получает команду bootstrap или право выключить legacy. +4. Проверить stop/start и возврат на реальных одноразовых процессах, включая + потерю SSH, SIGKILL/reboot, сбой nginx, неготовый индекс и сбой сохранения state. +5. Завершить activation runbook, signed commit и scoped review каждого среза. + +Команда focused regression: + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_release tests.test_v8std_mcp_release_hold tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_runtime -v +``` + +**Выход:** повторяемый локально механизм обновления и первой миграции с +проверенным восстановлением; ни установки на целевом сервере, ни release в registry ещё нет. + +## Этап 2. Подключить CI и закрыть локальную приёмку + +Владелец: исполнитель Task6 после Task5; процессный plan исполняется внутри +этой же задачи, не вторым параллельным владельцем тех же файлов. + +Файлы: `.github/workflows/ci.yml`, `scripts/publish_mcp_artifacts.py`, +`tests/test_mcp_publication.py`, current architecture-policy instructions/tests, +public installation docs и verification record. + +1. Разделить разрешения на публикацию образа, upload corpus и runtime rollout. + Отключённый runtime deployment не должен блокировать первые два действия. +2. Реализовать цепочку archive upload → внешняя проверка → Pages manifest. + До готовности storage сохранить прежнюю рабочую Pages-публикацию без нового + manifest, указывающего на отсутствующий архив. +3. Собрать runtime `linux/amd64`/`linux/arm64`, SBOM/provenance и local-site image. + Различать runtime SHA, corpus SHA и trigger SHA; не перелицовывать старый образ + source SHA нового контентного commit. Классификацию делать относительно + последней успешно опубликованной версии, учитывая пропущенные/упавшие runs. +4. Проверить PR/fork/tag/stale/failed gates без host effects. До runtime activation + ни первый bootstrap, ни успешная публикация не включают автодеплой. +5. Исправить известный image-alt/`` дефект presentation перед выпуском; + отдельным тестом защитить переписывание следующих видимых ссылок. Текущие + предупреждения зависимостей классифицировать, не скрывать ради зелёного отчёта. +6. Выполнить native Linux smoke/cold/warm/offline и смешанную нагрузку на стенде: + MCP POST + idle/reconnect + общий NAT + refresh + static download. Записать + измеренную, а не предполагаемую границу производительности. +7. Semantic impact, CLI impact, fitness и merge-ready; strict build **до** + полного suite. Затем whole-branch review и исправление его конкретных findings. + +```sh +.venv/bin/python scripts/v8std_architecture.py impact --root . --base-ref main +.venv/bin/python scripts/v8std_architecture.py validate --root . --base-ref main --merge-ready +VIRTUAL_ENV="$PWD/.venv" ./scripts/zensical_docs.sh build --strict +.venv/bin/python -m unittest discover -s tests -v +``` + +**Выход:** проверенная release-ветка и точный SHA. Локально интегрировать после +всех gates; push main выполнять только с явным разрешением. CI до настройки +внешней активации не должен переключать MCP или публиковать битый bootstrap. + +## Этап 3. Подготовить сервер и внешние разрешения — до окна простоя + +Работа по согласованному activation runbook из проверенного main, не установка +скриптов из dirty feature checkout. Перед каждым изменением — свежий read-only +inventory и точный список targets. Этот документ сам по себе не выдаёт полномочия. + +1. Сохранить вне сервера backup legacy code/config/data, nginx/TLS/renewal и + мониторинга. Проверить чтение backup и путь запуска старого сервиса без сети. +2. Подготовить статическое хранилище `/indexes/v1/` и restricted upload identity. + nginx обслуживает его независимо от Docker/runtime. В первой фазе current + upstream MCP не менять. +3. Установить Docker и root-owned release/initial-recovery units по runbook, + проверить firewall и автозапуск. Любое потенциально прерывающее действие + перенести в согласованное окно; не останавливать MCP ради подготовки без него. +4. Настроить GitHub protections/environment/минимальные CI credentials для + нужных операций. Runtime activation держать выключенной. +5. Получить свежие значения memory/disk/FD и результаты native Linux теста. + Выбрать overlap только при доказанном запасе. Для stop/start доказать запас + под один новый runtime + preparation + nginx/службы. При нехватке не начинать + миграцию; согласовать оптимизацию или изменение ресурсов отдельно. + +Зависимости default TLS от сертификатов других vhosts требуют проверки. +Удаление сторонних vhosts/данных — отдельный согласованный cleanup, +не обязательный риск внутри первого cutover. Цель оставить на сервере только +ai сохраняется, но CI не получает права чистить посторонние сайты. + +**Выход:** storage и ограниченная доставка готовы, legacy продолжает обслуживать +запросы; разрешение на прекращение сервиса ещё не использовано. + +## Этап 4. Опубликовать рабочие артефакты + +Для прошедшего gates SHA main запустить разрешённый CI. Кандидатный образ можно +загрузить в registry раньше данных, но стабильный тег не рекламировать и не +продвигать до проверки default source. + +Порядок внешних действий: + +1. Опубликовать immutable archive на + `https://ai.v8std.ru/indexes/v1//snapshot.tar.gz`. +2. Извне проверить GET/HEAD, размер, SHA256 и независимость от runtime. +3. Опубликовать Pages manifest + `https://v8std.ru/ai/mcp/v1/manifest.json`, затем проверить реальную цепочку + manifest → archive. Идентификатор в URL берётся из проверенного архива. +4. Скачать runtime по digest без авторизации, с изолированной конфигурацией + credentials; не делать global docker logout. Проверить оба platform manifests, + подпись/provenance, default-source cold start и warm restart без сети. +5. Проверить local-site image/Compose: один выбранный SITE_URL определяет источник + и ссылки, локальный маршрут работает без скрытого выхода в публичный интернет. +6. Продвинуть стабильные теги на уже проверенные digests, записать команды запуска + с persistent volume и точные версии. Нельзя пересобирать отдельный prod image. + +**Выход:** пользователь может установить рабочий image; публичный MCP пока +остаётся прежним. Docker Catalog submission и принятие Docker team не требуются. + +## Этап 5. Провести первую миграцию в назначенное окно + +Входные условия: пользователь назначил дату/время по Москве; записаны UTC +границы, точный проверенный main SHA, image/platform/configuration digests, +corpus ID, backup hashes, способ восстановления и замер его длительности. +Никаких выдуманных дат или автоматического запуска «через два часа». + +До окна уже скачаны образ/данные и выполнена репетиция. План распределения +времени ниже — резерв, не ожидание длительности запуска: + +| От начала окна | Действия и условие продолжения | +|---|---| +| 0–10мин | Свежий preflight, подтвердить legacy health, артефакты, окно, backup и recovery guard | +| 10–25мин | Одна ограниченная попытка; если overlap не помещается — stop legacy/start candidate; при ошибке немедленный возврат, не ожидание конца окна | +| 25–60мин | Проверка результата/диагностика; повтор только после восстановленного legacy, понятной причины и с новым release ID | +| 60–90мин | Проверка стабильности принятого сервиса и bounded smoke; не экспериментальный stress test на production | +| 90–120мин | Резерв восстановления: если новый сервис не принят — возврат к старому и проверка; новых попыток нет | + +Одна попытка сохраняет90s readiness и300s transaction. Ошибка первого запуска +не оправдывает ожидание360s под неготовым публичным upstream. Если замер возврата +требует больше30минут, увеличить резерв до начала окна и сократить рабочую часть. + +Критерии успеха: + +- публичные TLS, `/mcp`, readiness и реальные initialize/tools/list/search/ + get_page/explain_snippet работают на точном image/corpus; +- новый runtime помещается в ресурсы, нет OOM/restart loop; +- индексы раздаются независимо; persistent cache сохранён; +- первый accepted container record записан, restart/reboot запускает его, + legacy unit не конкурирует с ним; backup legacy остаётся доступным; +- при неуспехе старый endpoint действительно отвечает с прежними данными. + +Полный mixed-load предел измеряется на стенде, не выжимается из production +во время миграции. Результат `RECOVERY_REQUIRED` требует действия оператора и +никогда не маскируется заявлением «всё развернуто». + +## Этап 6. Отдельно принять автоматические обновления + +1. Проверить content-only выпуск: image digest и процесс не меняются, corpus + обновляется фоном без сетевого I/O из запроса. +2. Проверить runtime-only выпуск и откат на том же опубликованном digest по + согласованной процедуре. До live-пробы соответствующая репетиция обязательна. +3. Включить automatic runtime rollout только при доказанной old/new capacity + и принятых protections/credentials/kill switch. Если помещается один процесс, + оставить runtime rollout выключенным: image/corpus publication работают, + бесшовный автодеплой на этом тарифе не объявляется готовым. +4. Зафиксировать public release, команды установки, точные SHA/digests и реальные + результаты производительности. PR33 закрывать с благодарностью только после + подтверждения локального и production пути; issue32 автоматически не закрывать. + +Следующим отдельным улучшением может стать быстрый старт из подготовленного +дискового индекса. Каталог Docker публикуется по отдельному решению с тем же +образом; проверяется per-entry `longLived`, а не обязательный глобальный флаг. + +## Готовность плана и следующая работа + +Сейчас выполнено только планирование, остановка/установка/публикация не проведены. +Следующее действие после согласования — продолжить сохранённый Task5, +проверить его незавершённые регрессии и закрыть первый bootstrap slice; +затем Task6. Существующий SDD-порядок с отдельным review сохраняется. + +Semantic impact: уточнены границы первой миграции и rollout conformance; +requirement/ADR IDs, публичный API и snapshot schema сохранены. Уточнения +синхронизированы в candidate design/ADR/invariant/release contract/plan, которых +нет в main. Обычные автоматические гарантии не ослаблены. Исправленный Catalog +вывод — корректировка доказательств и текущего scope, не объявление нового +канала принятым. Перед merge нужны все gates, перечисленные выше. diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 06e2f98..6f9976b 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -50,6 +50,11 @@ requirements: - Основной checkout, существующая ветка `codex/mcp-container-distribution-design`; не создавать worktree, не менять main, не пушить и не деплоить во время реализации. - TDD и focused tests для каждого поведения. Strict build выполняется **до** полного suite: tests читают `site/LICENSES`, параллельная пересборка разрушает их вход. - Утверждение о 100 000 подключений запрещено без production-like mixed-load evidence. +- Первый выпуск — опубликованный Docker image; Docker Catalog и upstream PR + отложены отдельным решением пользователя и не являются блокерами этого выпуска. +- Первичная ручная миграция может использовать назначенное пользователем окно + до двух часов без old/new overlap; автоматический rollout сохраняет overlap gate. + Дата окна ещё не назначена. Согласование plan не разрешает остановку production. ## Scope and acceptance boundaries @@ -412,6 +417,13 @@ state/store roots. Release controller effects go through a narrow adapter to Docker/nginx/systemd; tests use disposable processes/filesystems and record exact calls at this external boundary, not pretend mocked return values prove health. +**Resume boundary:** Task5 has uncommitted implementation from `3165cc7` and +is paused for the current planning turn. Resume the same owner; preserve the +existing files. The last29 release tests passed, but three failures in the +89-test snapshot/runtime run were only claimed repaired and need a fresh run. +Hold regressions, ingress/security review, Docker/nginx evidence, activation +runbook and independent task review remain. This is not a completed task. + - [ ] **RED:** Test unknown schema, invalid digest/namespace/config path, stale or mutated duplicate ID, concurrent releases, failed pull/ready/switch/smoke, rollback failure and restart reconciliation. Example visible invariant: @@ -451,13 +463,83 @@ self.assertTrue(predecessor_snapshot_path.is_file()) claiming 100k. Commit and review the host code as security-sensitive code, not an authorization to install it on production. +#### First-migration completion slice — before Task6 integration + +**Files:** extend `scripts/v8std_mcp_release.py`, +`tests/test_v8std_mcp_release.py`, `tests/mcp_release_fixture.py`, +`deploy/container/` and `spec/operations/mcp-container-activation.md`. +Use a separate scoped review after the ordinary controller is stable. Do not +add a second runtime, change snapshot format or enlarge the CI command allowlist. + +**Interface:** operator-only CLI `bootstrap`, `bootstrap-recover`, +`bootstrap-status`, dispatched in the existing release module; the restricted +CI entry must reject all three. Bootstrap consumes the existing validated +release envelope plus a root-owned, size-bounded window/legacy record from the +installed policy directory, not arbitrary CLI paths. That record contains UTC +start/end (at most7200seconds), exact envelope SHA256, the fixed legacy unit +`v8std-mcp.service` and hashes of saved config/data. Legacy startup/config paths +come from verified host inventory and root-owned policy, never from CI input. +It produces the existing bounded status shape and an initial accepted container +record only after real smoke. Reuse digest/attestation/hold/inspection/smoke code. +Prepare immutable artifacts/backup before the window; mint and authorize the +execution envelope just before each bounded attempt so its300second deadline +has not expired during preparation. Recovery of an already-started attempt +must remain allowed after window expiry; only new attempts are refused. + +- [ ] **RED initial boundary:** Execute CLI against a disposable fixture and + show rejection outside/missing window, wrong envelope hash, existing active + container, CI entry invocation and insufficient single-runtime capacity. + Add subprocess fault cases after legacy stop, after candidate start, after + switch and during initial active-record persistence. Assert the endpoint, + exact served data and owned process count, not just a successful exit. +- [ ] **GREEN initial transition:** Add a serialized initial journal and + independently scheduled host recovery before stopping legacy. Stop/start is + allowed only inside the operator window. Commit the first container record + after local/public smoke, or restore verified Python config/data/upstream. + A no-predecessor ordinary `deploy` remains rejected; do not fabricate its + required `active.json`. Failures after accepted commit reconcile persistence, + not blindly roll back an already accepted container. Before acceptance, + startup/reboot recovery restores the saved legacy service; after acceptance + it starts the exact accepted digest, without racing the still-enabled legacy unit. +- [ ] **VERIFY initial transition:** Run the real disposable process fixture + under restricted memory, including no-overlap, SIGKILL/lost SSH, crash at each + persistence boundary, duplicate request and rollback failure. Record absence + of any simultaneous legacy/candidate process in stop/start mode. Keep static + archive GET/HEAD available throughout. Before the host window, replay the + tested runbook on native Linux and measure return-to-legacy time. + +Required observable outcomes (the fixture's CLI returns JSON with these fields): + +```python +self.assertEqual(result["state"], "ROLLED_BACK") +self.assertEqual(served_runtime_sha, saved_legacy_sha) +self.assertEqual(served_data_sha, saved_legacy_data_sha) +self.assertEqual(candidate_process_count, 0) +self.assertEqual(static_archive_sha, published_archive_sha) +``` + +Here `result` is the parsed bootstrap CLI status. `served_runtime_sha` is the +restarted process's verified source identity, `served_data_sha` the legacy +health/data hash, and `candidate_process_count` the fixture-owned PID count; +the archive values come from an independent GET and the published manifest. +Do not compare legacy health fields against the new runtime's different schema. + +Run existing release/hold/snapshot/runtime tests explicitly before the scoped +review; do not mark the live migration complete from these fixtures: + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_release tests.test_v8std_mcp_release_hold tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_runtime -v +``` + ### Task 6: Fail-closed publication CI, policy and final integration **Files:** workflows under `.github/workflows/`, `scripts/publish_mcp_artifacts.py`, `tests/test_mcp_publication.py`, process plan `spec/plans/2026-09-10-mcp-ci-deployment-policy-plan.md`, `AGENTS.md`, `spec/README.md`, repo skill references, architecture loader/policy tests, -`spec/operations/mcp-container-verification.md`, public installation docs. +`spec/operations/mcp-container-verification.md`, public installation docs and +Catalog release metadata/harness under `deploy/docker-catalog/`, +`scripts/check_mcp_container.py`, `tests/test_v8std_mcp_distribution.py`. **Consumes:** producer, image harness and typed release controller CLIs. Publisher first places and externally verifies immutable corpus, then emits @@ -481,6 +563,13 @@ Pages manifest. Every runtime deployment references published exact digest. permission for push. Align all current policy references; historic v1 and old structured plans stay frozen. Write reproducible commands/evidence and separate external gates for registry/Catalog/target-host. Do not publish internal specs. +- [ ] **VERIFY release scope and independent activation:** Image publication + and corpus upload can run while runtime deployment is disabled. Test that + first-bootstrap success alone does not activate automatic runtime deployment; + failed overlap capacity leaves the running endpoint untouched. Document + image-only scope without claiming Docker Catalog acceptance. Preserve the + `longLived` source declaration and distinguish local test-catalog diagnostics + from the actual Docker-published catalog; no upstream PR is a release prerequisite. - [ ] **Final gates:** Run semantic impact on actual paths, CLI `impact`, `validate --merge-ready`, all applicable fitness; strict build, then full suite; container smoke and shared-host mixed load on disposable local stack, review @@ -488,6 +577,19 @@ Pages manifest. Every runtime deployment references published exact digest. unperformed external operations. No push, merge of incomplete plan, PR closure or production deployment inferred from these green tests. +### Deferred external Docker Catalog gate + +The user's current choice is image-only publication. A later Catalog submission +uses the same published digest and `longLived: true`, followed by real repeated +tool calls without a masking global `--long-lived` flag. The upstream +`task catalog`/`ToTile` diagnostic loses this field, but it is a **test** catalog +generator. On2026-09-10 actual Docker v2/v3 catalogs contain the field for +Playwright and four other servers. Therefore its repair is not a necessary +precondition for image publication or even submission. No Catalog publication +or default end-user lifecycle success for v8std is claimed by removing the +incorrect pre-merge gate. Keep the original diagnostic and its correction in +the verification record; do not repeat it simply to rediscover the known defect. + ## Evidence Each task's implementation/review evidence is recorded during execution in its From df90d0d0767a01014e0028f62571c5ad60b4518e Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 01:30:19 +0300 Subject: [PATCH 31/88] feat(mcp): implement bounded ordinary release controller --- Dockerfile.mcp | 2 +- deploy/container/edge-http.conf | 9 + deploy/container/edge-locations.conf | 57 + deploy/container/release-entry.py | 12 + deploy/container/release-policy.example.json | 17 + deploy/container/release.schema.json | 22 + deploy/container/release.sudoers | 8 + .../container/v8std-release-recover.service | 19 + deploy/container/v8std-release-recover.timer | 10 + scripts/v8std_mcp_hold.py | 123 ++ scripts/v8std_mcp_release.py | 1151 +++++++++++++++++ scripts/v8std_mcp_runtime.py | 6 +- scripts/v8std_mcp_snapshots.py | 49 +- spec/operations/mcp-container-activation.md | 262 +++- tests/mcp_release_fixture.py | 186 +++ tests/test_v8std_mcp_release.py | 619 +++++++++ tests/test_v8std_mcp_release_docker.py | 230 ++++ tests/test_v8std_mcp_release_hold.py | 112 ++ 18 files changed, 2876 insertions(+), 18 deletions(-) create mode 100644 deploy/container/edge-http.conf create mode 100644 deploy/container/edge-locations.conf create mode 100644 deploy/container/release-entry.py create mode 100644 deploy/container/release-policy.example.json create mode 100644 deploy/container/release.schema.json create mode 100644 deploy/container/release.sudoers create mode 100644 deploy/container/v8std-release-recover.service create mode 100644 deploy/container/v8std-release-recover.timer create mode 100644 scripts/v8std_mcp_hold.py create mode 100644 scripts/v8std_mcp_release.py create mode 100644 tests/mcp_release_fixture.py create mode 100644 tests/test_v8std_mcp_release.py create mode 100644 tests/test_v8std_mcp_release_docker.py create mode 100644 tests/test_v8std_mcp_release_hold.py diff --git a/Dockerfile.mcp b/Dockerfile.mcp index 1eacfbf..a146320 100644 --- a/Dockerfile.mcp +++ b/Dockerfile.mcp @@ -14,7 +14,7 @@ RUN python -c 'import os,re; assert re.fullmatch("[0-9a-f]{40}",os.environ["V8ST && mkdir -p /var/lib/v8std-mcp \ && chown 10001:10001 /var/lib/v8std-mcp COPY scripts/v8std_mcp_server.py scripts/v8std_mcp_runtime.py scripts/v8std_mcp_index.py \ - scripts/v8std_mcp_snapshots.py scripts/v8std_mcp_snapshot_format.py \ + scripts/v8std_mcp_snapshots.py scripts/v8std_mcp_snapshot_format.py scripts/v8std_mcp_hold.py \ scripts/v8std_mcp_presentation.py scripts/v8std_mcp_chunks.py \ scripts/v8std_retrieval_rules.py scripts/v8std_search_features.py ./scripts/ COPY retrieval-rules.yml LICENSE ./ diff --git a/deploy/container/edge-http.conf b/deploy/container/edge-http.conf new file mode 100644 index 0000000..9ccf397 --- /dev/null +++ b/deploy/container/edge-http.conf @@ -0,0 +1,9 @@ +# Include once in nginx http{}. Conservative LOCAL TEST starting values only; +# activation requires the recorded native shared-host mixed-load gate. +limit_conn_zone $server_name zone=v8std_active:1m; +limit_conn_zone $server_name zone=v8std_downloads:1m; +upstream v8std_release_runtime { + zone v8std_release_runtime 64k; + include /etc/nginx/v8std-release/upstream.conf; + keepalive 2; +} diff --git a/deploy/container/edge-locations.conf b/deploy/container/edge-locations.conf new file mode 100644 index 0000000..cbbd264 --- /dev/null +++ b/deploy/container/edge-locations.conf @@ -0,0 +1,57 @@ +# Include in the separately inventoried ai.v8std.ru TLS server; do not replace +# default vhosts, certificate configuration or renewal during release. +location = /mcp { + if ($request_method !~ ^(POST|HEAD)$) { return 405; } + limit_conn v8std_active 8; + limit_conn_status 429; + client_max_body_size 2m; + client_body_timeout 10s; + proxy_connect_timeout 2s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_buffering off; + proxy_pass http://v8std_release_runtime; + proxy_intercept_errors on; + error_page 429 = @v8std_busy; + error_page 502 503 504 = @v8std_unavailable; + add_header Allow "POST, HEAD" always; +} +location = /healthz { + proxy_connect_timeout 2s; + proxy_read_timeout 3s; + proxy_pass http://v8std_release_runtime; +} +location = /version { + proxy_connect_timeout 2s; + proxy_read_timeout 3s; + proxy_pass http://v8std_release_runtime; +} +location ~ "^/indexes/v1/([0-9a-f]{64})/snapshot[.]tar[.]gz$" { + alias /srv/v8std-indexes/v1/$1/snapshot.tar.gz; + limit_except GET { deny all; } + autoindex off; + types { } + default_type application/gzip; + gzip off; + etag on; + sendfile on; + limit_conn v8std_downloads 2; + limit_conn_status 429; + limit_rate 1m; + send_timeout 10s; + # Never use 'always': 403/404/429 must not acquire immutable caching. + add_header Cache-Control "public, max-age=31536000, immutable"; + error_page 429 = @v8std_busy; +} +location /indexes/ { return 404; } +location @v8std_busy { + add_header Retry-After 1 always; + return 429; +} +location @v8std_unavailable { + add_header Retry-After 1 always; + return 503; +} diff --git a/deploy/container/release-entry.py b/deploy/container/release-entry.py new file mode 100644 index 0000000..26d11b2 --- /dev/null +++ b/deploy/container/release-entry.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 -I +"""Root-owned SSH forced command; no shell, SCP, SFTP or argument passthrough.""" +import os +import sys + +PUBLIC = {"validate-envelope", "deploy", "recover", "status", "publish-index"} +command = os.environ.get("SSH_ORIGINAL_COMMAND", "") +if command not in PUBLIC or len(sys.argv) != 1: + raise SystemExit("restricted command") +os.execve("/usr/bin/sudo", ["sudo", "-n", "/usr/bin/python3", "-I", + "/opt/v8std-release/scripts/v8std_mcp_release.py", command], + {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}) diff --git a/deploy/container/release-policy.example.json b/deploy/container/release-policy.example.json new file mode 100644 index 0000000..db5e40f --- /dev/null +++ b/deploy/container/release-policy.example.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "enabled": false, + "runtime_enabled": false, + "platform": "linux/amd64", + "public_url": "https://ai.v8std.ru", + "ports": [18766, 18767], + "nginx_include": "/etc/nginx/v8std-release/upstream.conf", + "static_root": "/srv/v8std-indexes/v1", + "configs": {}, + "capacity": { + "disk_bytes": 3221225472, + "available_memory_bytes": 1073741824, + "file_descriptors": 4096, + "network_evidence": null + } +} diff --git a/deploy/container/release.schema.json b/deploy/container/release.schema.json new file mode 100644 index 0000000..1f8ddd7 --- /dev/null +++ b/deploy/container/release.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:v8std:release:1", + "title": "Restricted v8std runtime release envelope", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "release_id", "sequence", "runtime_source_sha", "trigger_sha", "image", "image_digest", "platform_digest", "configuration_digest", "corpus_id", "archive_sha256", "deadline"], + "properties": { + "schema_version": {"const": 1}, + "release_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"}, + "sequence": {"type": "integer", "minimum": 1, "maximum": 9007199254740991}, + "runtime_source_sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "trigger_sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "image": {"const": "ghcr.io/zeegin/v8std-mcp"}, + "image_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "platform_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "configuration_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "corpus_id": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "archive_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "deadline": {"type": "integer", "description": "UTC epoch seconds, future and at most 300 seconds from validation. Host additionally reserves recovery time."} + } +} diff --git a/deploy/container/release.sudoers b/deploy/container/release.sudoers new file mode 100644 index 0000000..feb88b9 --- /dev/null +++ b/deploy/container/release.sudoers @@ -0,0 +1,8 @@ +# Install with visudo -cf validation. No wildcards or editable interpreter paths. +Cmnd_Alias V8STD_RELEASE = /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py validate-envelope, \ + /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py deploy, \ + /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py recover, \ + /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py status, \ + /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py publish-index +v8std-publisher ALL=(root) NOPASSWD: V8STD_RELEASE +Defaults!V8STD_RELEASE !setenv diff --git a/deploy/container/v8std-release-recover.service b/deploy/container/v8std-release-recover.service new file mode 100644 index 0000000..d63403e --- /dev/null +++ b/deploy/container/v8std-release-recover.service @@ -0,0 +1,19 @@ +[Unit] +Description=Reconcile v8std release journal and durable inbox +After=docker.service nginx.service network-online.target +Wants=network-online.target +ConditionPathExists=/etc/v8std-release/policy.json + +[Service] +Type=exec +ExecStart=/usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py _recover +RuntimeMaxSec=300s +TimeoutStopSec=5s +KillMode=control-group +UMask=0077 +LimitNOFILE=4096 +PrivateTmp=yes +NoNewPrivileges=yes +ProtectHome=yes +# Host-owned job intentionally needs narrow Docker/nginx authority. The SSH +# identity has no Docker group/socket access and can invoke only fixed verbs. diff --git a/deploy/container/v8std-release-recover.timer b/deploy/container/v8std-release-recover.timer new file mode 100644 index 0000000..cea77a6 --- /dev/null +++ b/deploy/container/v8std-release-recover.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Recover interrupted v8std releases independently of SSH/CI + +[Timer] +OnBootSec=15s +OnUnitInactiveSec=30s +Unit=v8std-release-recover.service + +[Install] +WantedBy=timers.target diff --git a/scripts/v8std_mcp_hold.py b/scripts/v8std_mcp_hold.py new file mode 100644 index 0000000..6a2e1b3 --- /dev/null +++ b/scripts/v8std_mcp_hold.py @@ -0,0 +1,123 @@ +"""Private, read-only host control mount; never an MCP method or source setting. + +One control directory and cache per managed container. Host atomically replaces +control.json; the runtime can read it but cannot write it. Only the coordinator +acknowledges a command, after cancelling preparation and selecting actual bytes. +Missing/malformed control fails closed (keeps serving the last accepted data, +does not refresh or acknowledge). No persisted Python objects cross this path. +""" +from pathlib import Path +import re +import time + +from v8std_mcp_snapshot_format import canonical_json, strict_json, validate_manifest, SnapshotError + +CONTROL_PATH = Path("/run/v8std-release/control.json") + + +class ReleaseControl: + def __init__(self, coordinator, path): + self.coordinator = coordinator + self.path = path + + def read(self): + from v8std_mcp_snapshots import _read_file, LoaderError + request = strict_json(_read_file(self.path, 65536)) + if (set(request) != {"schema_version", "token", "mode", "manifest"} + or type(request["schema_version"]) is not int or request["schema_version"] != 1 + or not isinstance(request["token"], str) + or not re.fullmatch("[a-f0-9]{32}", request["token"]) + or request["mode"] not in {"hold", "resume"}): + raise LoaderError("configuration") + if request["manifest"] is not None: + validate_manifest(canonical_json(request["manifest"])) + if request["mode"] != "hold": + raise LoaderError("configuration") + return request + + def pin(self, archive): + from v8std_mcp_snapshots import _file_lock + store = self.coordinator.store + deadline = time.monotonic() + 1 + with _file_lock(store.namespace / ".lock", deadline): + with _file_lock(store.cache_dir / ".volume.lock", deadline): + store._atomic_file(store.namespace / "runtime-pin.json", + canonical_json({"archive": archive}), deadline) + + def run(self): + from v8std_mcp_snapshots import LoaderError + owner = self.coordinator + initial = True + last = None + seen = None + next_refresh = 0 + failures = 0 + while not owner._stop.is_set(): + try: + request = self.read() + changed = request != seen + if changed: + next_refresh = 0 + seen = request + if request["mode"] == "hold": + if request != last and time.monotonic() >= next_refresh: + manifest = request["manifest"] + if manifest is not None: + result, metadata = owner.store._run("refresh", owner.build, + CommandStop(self, request), selected_manifest=manifest) + if self.read() != request or owner._stop.is_set(): + continue + owner._accept(result, metadata, checked=False) + del result + elif not owner.status()["ready"]: + raise LoaderError("INDEX_NOT_READY") + # Capture uses the actual process identity, never state.json. + self.pin(owner.status()["archive_sha256"]) + with owner._lock: + owner._state["hold_token"] = request["token"] + owner._state["release_control_token"] = request["token"] + last = request + initial = False + else: + with owner._lock: + owner._state["hold_token"] = None + owner._state["release_control_token"] = request["token"] + last = request + if initial: + result, metadata = owner.store._run("cached", owner.build, CommandStop(self, request)) + if self.read() != request or owner._stop.is_set(): + continue + if metadata: + owner._accept(result, metadata, checked=False) + del result + initial = False + if time.monotonic() >= next_refresh: + result, metadata = owner.store._run("refresh", owner.build, + CommandStop(self, request), current_archive=owner._archive_sha256) + if self.read() != request or owner._stop.is_set(): + continue + owner._accept(result, metadata, checked=True) + del result + failures = 0 + next_refresh = (time.monotonic() + owner._delay(0) + if owner.refresh_seconds else float("inf")) + except (LoaderError, SnapshotError, OSError) as error: + failures += 1 + with owner._lock: + owner._state["refresh_error_code"] = getattr(error, "code", "configuration") + # A stale acknowledgment is never proof of a new hold. + owner._state["hold_token"] = None + owner._state["release_control_token"] = None + next_refresh = time.monotonic() + owner._delay(failures) + owner._stop.wait(.1) + + +class CommandStop: + def __init__(self, control, request): + self.control, self.request = control, request + + def is_set(self): + try: + return self.control.coordinator._stop.is_set() or self.control.read() != self.request + except (ValueError, OSError): + return True diff --git a/scripts/v8std_mcp_release.py b/scripts/v8std_mcp_release.py new file mode 100644 index 0000000..78d5017 --- /dev/null +++ b/scripts/v8std_mcp_release.py @@ -0,0 +1,1151 @@ +"""Restricted host release transaction. Install reviewed code as root-owned files. + +The public CLI has no path, environment, command, trust-policy or mount options. +Adapters own all external effects. Journals precede effects; deterministic object +names let recovery reconcile Docker operations whose CLI was killed mid-call. +This file is deliberately absent from the runtime image. +""" +from __future__ import annotations + +from contextlib import contextmanager +import fcntl +import hashlib +import json +import multiprocessing +import os +from pathlib import Path +import re +import select +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +# The installed, root-owned module directory is the only additional import root +# under python -I. Never read PYTHONPATH or code from a release envelope. +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from v8std_mcp_snapshot_format import ( + MAX_ARCHIVE_BYTES, canonical_json, strict_json, validate_manifest, verify_archive, +) + +IMAGE = "ghcr.io/zeegin/v8std-mcp" +REPO = "zeegin/v8std" +WORKFLOW = "zeegin/v8std/.github/workflows/ci.yml" +REF = "refs/heads/main" +ROOT = Path("/var/lib/v8std-release") +POLICY = Path("/etc/v8std-release/policy.json") +INSTALL = Path("/opt/v8std-release/scripts/v8std_mcp_release.py") +TRANSACTION = 300 +READINESS = 90 +SMOKE = DRAIN = 30 +STOP = 45 +# Rollback gets a live budget even if preparation uses its entire work allowance. +# 90 ready + 30 smoke + 45 stop + 15 nginx/control overhead. +RECOVERY_RESERVE = 180 +TERMINAL = {"FAILED", "ROLLED_BACK", "COMMITTED", "RECOVERY_REQUIRED"} +ID = re.compile(r"[a-z0-9][a-z0-9-]{0,63}\Z") +HEX = re.compile(r"[0-9a-f]{64}\Z") +SHA = re.compile(r"[0-9a-f]{40}\Z") +DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z") +INDEX_TYPES = {"application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json"} +MANIFEST_TYPES = {"application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json"} +CONFIG_TYPES = {"application/vnd.oci.image.config.v1+json", + "application/vnd.docker.container.image.v1+json"} + + +class ReleaseError(ValueError): + """Codes only; raw external errors and command output never enter status.""" + + def __init__(self, code): + self.code = code + super().__init__(code) + + +def require(condition, code): + if not condition: + raise ReleaseError(code) + + +def matches(pattern, value): + return isinstance(value, str) and bool(pattern.fullmatch(value)) + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def parse(raw, limit=8192): + require(len(raw) <= limit, "input_size") + try: + value = strict_json(raw) + require(isinstance(value, dict), "input_shape") + return value + except ValueError: + raise ReleaseError("input_shape") from None + + +def validate_envelope(raw, *, now=None, expired=False): + value = parse(raw) + require(set(value) == {"schema_version", "release_id", "sequence", "runtime_source_sha", + "trigger_sha", "image", "image_digest", "platform_digest", "configuration_digest", + "corpus_id", "archive_sha256", "deadline"}, "envelope_fields") + require(type(value["schema_version"]) is int and value["schema_version"] == 1, "schema") + require(matches(ID, value["release_id"]), "release_id") + require(type(value["sequence"]) is int and 0 < value["sequence"] <= 2**53 - 1, "sequence") + require(value["image"] == IMAGE, "namespace") + for key in ("runtime_source_sha", "trigger_sha"): + require(matches(SHA, value[key]), key) + for key in ("image_digest", "platform_digest"): + require(matches(DIGEST, value[key]), key) + for key in ("configuration_digest", "corpus_id", "archive_sha256"): + require(matches(HEX, value[key]), key) + require(type(value["deadline"]) is int, "deadline") + if not expired: + now = time.time() if now is None else now + require(now < value["deadline"] <= now + TRANSACTION, "deadline") + return value + + +def read_file(path, limit=65536): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_size <= limit, "file_shape") + raw = stream.read(limit + 1) + require(len(raw) <= limit, "file_size") + return raw + + +def sync_dir(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def ensure_directory(path, mode=0o700): + if not path.exists(): + ensure_directory(path.parent) + path.mkdir(mode=mode, exist_ok=True) + sync_dir(path.parent) + require(stat.S_ISDIR(path.lstat().st_mode), "directory_shape") + + +def atomic(path, raw): + ensure_directory(path.parent) + fd, name = tempfile.mkstemp(prefix=".write-", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + os.replace(name, path) + sync_dir(path.parent) + finally: + Path(name).unlink(missing_ok=True) + + +def write_json(path, value): + # Operational timestamps/health contain finite floats; snapshot descriptors + # and envelope hashes keep the separate float-free canonical encoding. + atomic(path, json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode()) + + +def read_record(path): + # The wire header remains <=64KiB; a durable receipt wraps it with state. + # This host-owned record limit must not accidentally reapply the 8KiB + # release-envelope limit to valid corpus manifests or publication receipts. + return parse(read_file(path, 128 * 1024), 128 * 1024) + + +@contextmanager +def locked(root): + ensure_directory(root) + fd = os.open(root / "release.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ReleaseError("busy") from None + yield + finally: + os.close(fd) + + +def remaining(deadline, cap=None): + value = deadline - time.monotonic() + require(value > 0, "deadline") + return min(value, cap) if cap is not None else value + + +def run(argv, deadline, *, limit=2 * 1024 * 1024): + """No shell. Kill/reap the CLI process group; daemon effects need reconciliation.""" + with tempfile.TemporaryFile() as output: + process = subprocess.Popen([str(x) for x in argv], stdin=subprocess.DEVNULL, + stdout=output, stderr=subprocess.DEVNULL, start_new_session=True, + env={"PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "HOME": "/var/lib/v8std-release"}) + try: + while process.poll() is None: + require(output.tell() <= limit, "command_output") + time.sleep(min(.025, remaining(deadline))) + require(process.returncode == 0, "command_failed") + require(output.tell() <= limit, "command_output") + output.seek(0) + return output.read(limit + 1) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def trusted_policy(path=POLICY): + # Reject symlinked/writable policy and all parents before interpreting paths. + for entry in (path, *path.parents): + info = entry.lstat() + require(not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022, + "policy_permissions") + policy = parse(read_file(path), 65536) + return validate_policy(policy) + + +def validate_policy(policy): + require(set(policy) == {"schema_version", "enabled", "runtime_enabled", "platform", "public_url", "configs", + "capacity", "ports", "nginx_include", "static_root"}, "policy_fields") + require(type(policy["schema_version"]) is int and policy["schema_version"] == 1 + and policy["enabled"] is True, "not_activated") + require(type(policy["runtime_enabled"]) is bool, "runtime_activation") + require(policy["platform"] in {"linux/amd64", "linux/arm64"}, "platform") + require(policy["public_url"] == "https://ai.v8std.ru", "public_url") + require(policy["nginx_include"] == "/etc/nginx/v8std-release/upstream.conf", "nginx_path") + require(policy["static_root"] == "/srv/v8std-indexes/v1", "static_path") + require(policy["ports"] == [18766, 18767], "ports") + require(isinstance(policy["configs"], dict) and len(policy["configs"]) <= 16 + and (bool(policy["configs"]) or not policy["runtime_enabled"]), "configs") + for key, config in policy["configs"].items(): + require(matches(HEX, key) and digest(canonical_json(config)) == key, "configuration_digest") + require(set(config) == {"site_url", "refresh_seconds", "max_snippet_chars", "memory_bytes", "cpus"}, "config_fields") + require(config["site_url"] == "https://v8std.ru/", "site_url") + for field, low, high in (("refresh_seconds", 0, 86400), ("max_snippet_chars", 4000, 32000), + ("memory_bytes", 268435456, 8589934592), ("cpus", 1, 64)): + require(type(config[field]) is int and low <= config[field] <= high, "config_value") + capacity = policy["capacity"] + require(set(capacity) == {"disk_bytes", "available_memory_bytes", "file_descriptors", "network_evidence"}, "capacity") + for name in ("disk_bytes", "available_memory_bytes", "file_descriptors"): + require(type(capacity[name]) is int and capacity[name] > 0, "capacity") + if policy["runtime_enabled"]: + require(matches(HEX, capacity["network_evidence"]), "capacity_evidence") + require(capacity["available_memory_bytes"] >= max(c["memory_bytes"] for c in policy["configs"].values()) + 128 * 1024 * 1024, + "capacity_reserve") + else: + require(capacity["network_evidence"] is None or matches(HEX, capacity["network_evidence"]), "capacity_evidence") + return policy + + +def verify_descriptors(index_raw, child_raw, envelope, platform): + def decoded(raw, expected): + # buildx adds a display newline on some versions; only discard it if + # the exact remaining bytes match the requested content address. + if "sha256:" + digest(raw) != expected and raw.endswith(b"\n"): + raw = raw[:-1] + require("sha256:" + digest(raw) == expected, "descriptor_digest") + return parse(raw, 2 * 1024 * 1024) + index = decoded(index_raw, envelope["image_digest"]) + child = decoded(child_raw, envelope["platform_digest"]) + require(index.get("mediaType") in INDEX_TYPES and index.get("schemaVersion") == 2, "index_media_type") + require(child.get("mediaType") in MANIFEST_TYPES and child.get("schemaVersion") == 2, "child_media_type") + os_name, architecture = platform.split("/") + members = [item for item in index.get("manifests", []) if item.get("digest") == envelope["platform_digest"] + and item.get("mediaType") in MANIFEST_TYPES + and item.get("platform", {}).get("os") == os_name + and item.get("platform", {}).get("architecture") == architecture + and item.get("platform", {}).get("variant", "") in ({"", "v8"} if architecture == "arm64" else {""})] + require(len(members) == 1, "platform_membership") + require(members[0].get("size") in {len(child_raw), len(child_raw.rstrip(b"\n"))}, "descriptor_size") + config = child.get("config", {}) + require(config.get("mediaType") in CONFIG_TYPES and matches(DIGEST, config.get("digest")), "config_descriptor") + return {envelope["image_digest"]: index["mediaType"], envelope["platform_digest"]: child["mediaType"], + config["digest"]: config["mediaType"]} + + +def attestation_command(subject, source_sha): + return ["gh", "attestation", "verify", subject, "--repo", REPO, "--signer-workflow", WORKFLOW, + "--source-ref", REF, "--source-digest", source_sha, "--deny-self-hosted-runners", + "--cert-oidc-issuer", "https://token.actions.githubusercontent.com", "--format", "json"] + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + raise ReleaseError("http_redirect") + + +def _http(url, deadline, body, limit): + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + request = urllib.request.Request(url, data=body, headers={ + "Content-Type": "application/json", "Accept": "application/json, text/event-stream", + "Accept-Encoding": "identity"}) + try: + with opener.open(request, timeout=remaining(deadline, 3)) as response: + require(response.status == 200, "http_status") + raw = response.read(limit + 1) + require(len(raw) <= limit, "http_size") + remaining(deadline) + return raw + except (OSError, urllib.error.URLError): + raise ReleaseError("http_failed") from None + + +def _http_worker(channel, url, deadline, body, limit): + try: + channel.send_bytes(b"1" + _http(url, deadline, body, limit)) + except Exception: + channel.send_bytes(b"0") + finally: + channel.close() + + +def http(url, deadline, *, body=None, limit=1024 * 1024): + # DNS and drip-fed bodies cannot spend rollback's reserved time. This process + # has no host effects and is killed/reaped at the caller's actual deadline. + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe(duplex=False) + process = context.Process(target=_http_worker, args=(child, url, deadline, body, limit), daemon=True) + process.start() + child.close() + try: + require(parent.poll(remaining(deadline)), "deadline") + raw = parent.recv_bytes(limit + 1) + remaining(deadline) + require(raw[:1] == b"1", "http_failed") + return raw[1:] + except (EOFError, OSError): + raise ReleaseError("http_failed") from None + finally: + parent.close() + if process.is_alive(): + process.kill() + process.join() + process.close() + + +def rpc(url, method, params, deadline, number): + raw = http(url + "/mcp", deadline, body=canonical_json({"jsonrpc": "2.0", "id": number, + "method": method, "params": params})) + if raw.startswith(b"event:") or raw.startswith(b"data:"): + messages = [line[6:] for line in raw.splitlines() if line.startswith(b"data: ")] + require(len(messages) == 1, "rpc_stream") + raw = messages[0] + reply = parse(raw, 1024 * 1024) + require(reply.get("id") == number and "error" not in reply and isinstance(reply.get("result"), dict), "rpc") + result = reply["result"] + require(not result.get("isError"), "rpc_tool") + return result + + +def smoke(url, record, deadline): + health = parse(http(url + "/healthz", deadline)) + require(health.get("ok") is True and health.get("runtime_sha") == record["runtime_source_sha"] + and health.get("corpus_id") == record["corpus_id"] + and health.get("archive_sha256") == record["archive_sha256"] + and health.get("hold_token") == record["hold_token"], "health_identity") + initialized = rpc(url, "initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "v8std-release", "version": "1"}}, deadline, 1) + require(initialized.get("serverInfo", {}).get("name") == "v8std", "server_identity") + listed = rpc(url, "tools/list", {}, deadline, 2) + expected = {"v8std_search", "v8std_get_page", "v8std_get_related", "v8std_explain_snippet", "v8std_explain_diagnostics"} + require({item["name"] for item in listed.get("tools", [])} == expected, "tool_surface") + result = rpc(url, "tools/call", {"name": "v8std_search", "arguments": {"query": "std437", "limit": 1}}, deadline, 3) + # Successful JSON-RPC alone cannot establish a useful search/page response. + def structured(value): + if "structuredContent" in value: + return value["structuredContent"] + texts = [item["text"] for item in value.get("content", []) if item.get("type") == "text"] + require(len(texts) == 1, "tool_content") + return parse(texts[0].encode(), 1024 * 1024) + search = structured(result) + require(bool(search.get("results")), "search_empty") + page_id = search["results"][0]["id"] + page = structured(rpc(url, "tools/call", {"name": "v8std_get_page", "arguments": + {"id_or_alias_or_url": page_id}}, deadline, 4)) + require(page.get("found") is True and page.get("page", {}).get("id") == page_id, "page_smoke") + structured(rpc(url, "tools/call", {"name": "v8std_explain_snippet", "arguments": + {"snippet": "Запрос = Новый Запрос;", "limit": 1}}, deadline, 5)) + resources = rpc(url, "resources/list", {}, deadline, 6) + require({item["uri"] for item in resources.get("resources", [])} == { + "v8std://llms.txt", "v8std://llms-full.txt", "v8std://ai/pages.jsonl"}, "resource_surface") + # Bracket tool calls with the held identity so a health-only mismatch cannot + # pass while the actual endpoint refreshes or nginx reload serves old workers. + after = parse(http(url + "/healthz", deadline)) + require(all(after.get(k) == health.get(k) for k in ( + "runtime_sha", "corpus_id", "archive_sha256", "hold_token")), "smoke_generation_changed") + return health + + +class HostAdapter: + def __init__(self, root, policy): + self.root, self.policy = Path(root), policy + + def config(self, envelope): + config = self.policy["configs"].get(envelope["configuration_digest"]) + require(config is not None and digest(canonical_json(config)) == envelope["configuration_digest"], "untrusted_configuration") + return config + + def verify(self, envelope, deadline): + self.config(envelope) + run(attestation_command("oci://" + IMAGE + "@" + envelope["image_digest"], + envelope["runtime_source_sha"]), deadline) + # Certificate source-ref binds main at build time. Current eligibility of + # both runtime and triggering commits additionally requires main ancestry. + for sha in {envelope["runtime_source_sha"], envelope["trigger_sha"]}: + result = parse(run(["gh", "api", f"repos/{REPO}/compare/{sha}...main"], deadline), 2 * 1024 * 1024) + require(result.get("status") in {"ahead", "identical"} + and result.get("merge_base_commit", {}).get("sha") == sha, "main_ancestry") + index = run(["docker", "buildx", "imagetools", "inspect", "--raw", IMAGE + "@" + envelope["image_digest"]], deadline) + child = run(["docker", "buildx", "imagetools", "inspect", "--raw", IMAGE + "@" + envelope["platform_digest"]], deadline) + return verify_descriptors(index, child, envelope, self.policy["platform"]) + + def capacity(self, deadline): + limits = self.policy["capacity"] + require(shutil.disk_usage(self.root).free >= limits["disk_bytes"], "disk_capacity") + values = dict(re.findall(r"^(\w+):\s+(\d+)", read_file(Path("/proc/meminfo")).decode(), re.M)) + require(int(values.get("MemAvailable", 0)) * 1024 >= limits["available_memory_bytes"], "memory_capacity") + import resource + require(resource.getrlimit(resource.RLIMIT_NOFILE)[0] >= limits["file_descriptors"], "fd_capacity") + evidence = read_file(self.root / "capacity" / (limits["network_evidence"] + ".json")) + require(digest(evidence) == limits["network_evidence"], "network_capacity") + remaining(deadline) + + def pull(self, record, deadline): + run(["docker", "pull", "--platform", self.policy["platform"], IMAGE + "@" + record["platform_digest"]], deadline) + + def inspect(self, record, deadline): + try: + result = json.loads(run(["docker", "inspect", "--type", "container", record["name"]], deadline)) + except ReleaseError as error: + if error.code != "command_failed": + raise + # Only confirmed absence can authorize creation, not inspect failure. + names = run(["docker", "ps", "-a", "--format", "{{.Names}}"], deadline).decode().splitlines() + require(record["name"] not in names, "inspect_failed") + return None + info = result[0] + labels = info["Config"].get("Labels") or {} + require(labels.get("pro.v8std.release") == record["release_id"] + and labels.get("pro.v8std.envelope") == record["envelope_hash"], "ownership") + require(info["Config"]["Image"] == IMAGE + "@" + record["platform_digest"], "container_image") + require(info["Image"] in record["descriptors"], "image_descriptor_identity") + require(labels.get("org.opencontainers.image.revision") == record["runtime_source_sha"], "runtime_revision") + return info + + def start(self, record, deadline): + info = self.inspect(record, deadline) + if info: + if not info["State"]["Running"]: + run(["docker", "start", record["name"]], deadline) + return + config = self.config(record) + directory = self.root / "slots" / record["release_id"] + cache = directory / "cache" + ensure_directory(cache) + os.chown(cache, 10001, 10001) + # Dedicated cache/control paths are derived solely from the validated ID. + run(["docker", "run", "-d", "--name", record["name"], "--pull", "never", + "--label", "pro.v8std.release=" + record["release_id"], + "--label", "pro.v8std.envelope=" + record["envelope_hash"], + "--platform", self.policy["platform"], "--read-only", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--init", "--user", "10001:10001", + "--memory", str(config["memory_bytes"]), "--memory-swap", str(config["memory_bytes"]), + "--cpus", str(config["cpus"]), "--pids-limit", "128", "--stop-timeout", "30", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=64m,mode=1777", + "--mount", f"type=bind,source={cache},target=/var/lib/v8std-mcp", + "--mount", f"type=bind,source={directory / 'control'},target=/run/v8std-release,readonly", + "-p", f"127.0.0.1:{record['port']}:8000", IMAGE + "@" + record["platform_digest"], + "--transport", "streamable-http", "--host", "0.0.0.0", "--port", "8000", + "--site-url", config["site_url"], "--refresh-seconds", str(config["refresh_seconds"]), + "--max-snippet-chars", str(config["max_snippet_chars"])], deadline) + self.inspect(record, deadline) + + def control(self, record, mode, token, manifest=None): + control_directory = self.root / "slots" / record["release_id"] / "control" + ensure_directory(control_directory, 0o755) + write_json(control_directory / "control.json", {"schema_version": 1, "token": token, + "mode": mode, "manifest": manifest}) + os.chmod(control_directory / "control.json", 0o644) + + def hold(self, record, token, deadline, manifest=None): + self.control(record, "hold", token, manifest) + if manifest is not None: + self.start(record, deadline) + while True: + try: + state = parse(http(self.url(record) + "/healthz", deadline)) + if state.get("ok") and state.get("hold_token") == token: + require(state.get("runtime_sha") == record["runtime_source_sha"], "runtime_identity") + require(matches(HEX, state.get("archive_sha256")) and matches(HEX, state.get("corpus_id")), "corpus_identity") + if manifest is not None: + require(state["archive_sha256"] == manifest["archive"]["sha256"] + and state["corpus_id"] == manifest["corpus_id"], "selection_identity") + return {**record, "corpus_id": state["corpus_id"], "archive_sha256": state["archive_sha256"], + "corpus_source_sha": state["corpus_source_sha"], "hold_token": token} + except ReleaseError as error: + if error.code not in {"http_failed", "http_status"}: + raise + time.sleep(min(.1, remaining(deadline))) + + @staticmethod + def url(record): + return f"http://127.0.0.1:{record['port']}" + + def check(self, record, deadline, *, public=False): + self.inspect(record, deadline) + state = smoke(self.policy["public_url"] if public else self.url(record), record, deadline) + if public: + raw = http(self.policy["public_url"] + "/indexes/v1/" + record["archive_sha256"] + "/snapshot.tar.gz", + deadline, limit=MAX_ARCHIVE_BYTES) + require(digest(raw) == record["archive_sha256"], "static_hash") + return state + + def switch(self, record, deadline): + path = Path(self.policy["nginx_include"]) + previous = read_file(path) + atomic(path, (f"server 127.0.0.1:{record['port']} max_conns=8;\n").encode()) + try: + run(["nginx", "-t"], deadline) + except BaseException: + atomic(path, previous) + raise + run(["nginx", "-s", "reload"], deadline) + + def stop(self, record, deadline): + if self.inspect(record, deadline) is not None: + # No automatic restart policy; deterministic named objects survive + # controller death and are reconciled, not replaced by unrelated IDs. + run(["docker", "stop", "--time", str(max(0, min(DRAIN, int(remaining(deadline)) - 5))), record["name"]], deadline) + info = self.inspect(record, deadline) + require(not info["State"]["Running"], "stop_failed") + + def resume(self, record, deadline): + token = digest((record["release_id"] + ":resume").encode())[:32] + self.control(record, "resume", token) + while True: + health = parse(http(self.url(record) + "/healthz", deadline)) + require(health.get("runtime_sha") == record["runtime_source_sha"] and health.get("ok"), "resume_identity") + if health.get("hold_token") is None and health.get("release_control_token") == token: + return + time.sleep(min(.1, remaining(deadline))) + + def manifest(self, record): + raw = read_file(self.root / "manifests" / (record["archive_sha256"] + ".json")) + manifest = validate_manifest(raw) + require(manifest["corpus_id"] == record["corpus_id"] + and manifest["archive"]["sha256"] == record["archive_sha256"], "manifest_identity") + return manifest + + +class Controller: + def __init__(self, root, adapter): + self.root, self.adapter = Path(root), adapter + + def journals(self): + directory = self.root / "releases" + return [parse(read_file(path), 65536) for path in directory.glob("*.json")] if directory.exists() else [] + + def status(self): + records = self.journals() + if not records: + return {"state": "EMPTY"} + journal = max(records, key=lambda item: item["envelope"]["sequence"]) + return self.result(journal) + + @staticmethod + def result(journal): + return {**journal["envelope"], "state": journal["state"], "intent": journal["intent"], + "error_code": journal.get("error_code"), "cleanup_complete": journal.get("cleanup_complete", False)} + + def save(self, journal, state=None, intent=None): + if state: + journal["state"] = state + if intent: + journal["intent"] = intent + journal["updated_at"] = time.time() + write_json(self.root / "releases" / (journal["envelope"]["release_id"] + ".json"), journal) + + def existing(self, envelope): + records = self.journals() + for item in records: + if item["envelope"]["release_id"] == envelope["release_id"]: + require(item["envelope"] == envelope, "mutated_duplicate") + return item + require(not records or envelope["sequence"] > max(x["envelope"]["sequence"] for x in records), "stale_sequence") + require(not any(x["state"] not in TERMINAL or x["state"] == "RECOVERY_REQUIRED" + or x["state"] == "COMMITTED" and not x.get("cleanup_complete") for x in records), "recovery_pending") + return None + + def deploy(self, raw): + envelope = validate_envelope(raw, expired=True) + with locked(self.root): + existing = self.existing(envelope) + if existing: + return self.result(existing) + require(self.adapter.policy.get("runtime_enabled") is True, "runtime_not_activated") + self.adapter.config(envelope) + validate_envelope(raw) + require(envelope["deadline"] - time.time() > RECOVERY_RESERVE, "insufficient_transaction_budget") + require((self.root / "active.json").is_file(), "predecessor_required") + previous = parse(read_file(self.root / "active.json"), 65536) + self.adapter.config(previous) + deadline = time.monotonic() + min(TRANSACTION, envelope["deadline"] - time.time()) + work = deadline - RECOVERY_RESERVE + journal = {"envelope": envelope, "state": "RECEIVED", "intent": "verify", "predecessor": previous, + "candidate": None, "cleanup_complete": False} + self.save(journal) + try: + descriptors = self.adapter.verify(envelope, work) + self.adapter.capacity(work) + manifest = self.adapter.manifest(envelope) + self.save(journal, "VERIFIED", "hold_predecessor") + token = digest(canonical_json(envelope))[:32] + previous = self.adapter.hold(previous, token, min(work, time.monotonic() + READINESS)) + journal["predecessor"] = previous + self.adapter.manifest(previous) # Published record of observed, not disk-pointer, corpus. + self.save(journal, intent="pin_predecessor") + self.pins(journal) + candidate = {**envelope, "name": "v8std-release-" + envelope["release_id"], + "envelope_hash": digest(canonical_json(envelope)), "descriptors": descriptors, + "port": next(p for p in self.adapter.policy["ports"] if p != previous["port"]), + "hold_token": token} + journal["candidate"] = candidate + self.save(journal, intent="pull_candidate") + self.adapter.pull(candidate, work) + self.save(journal, intent="start_candidate") + candidate = self.adapter.hold(candidate, token, min(work, time.monotonic() + READINESS), manifest) + journal["candidate"] = candidate + self.save(journal, "PREPARED", "candidate_smoke") + self.adapter.check(candidate, min(work, time.monotonic() + SMOKE)) + journal["switch_attempted"] = True + self.save(journal, "READY", "switch") + self.adapter.switch(candidate, work) + self.save(journal, "SWITCHED", "public_smoke") + self.adapter.check(candidate, min(work, time.monotonic() + SMOKE), public=True) + self.save(journal, "COMMITTED", "accept_pointer") + self.cleanup(journal, deadline) + except Exception as error: + journal["error_code"] = error.code if isinstance(error, ReleaseError) else "host_failure" + self.save(journal) + if journal["state"] == "RECEIVED": + # Rejected authority cannot trigger runtime mutation. + journal["cleanup_complete"] = True + self.save(journal, "FAILED", "complete") + elif journal["state"] == "COMMITTED": + # Accepted release is never undone by post-commit cleanup failure. + self.save(journal, intent="cleanup_pending") + else: + self.rollback(journal, deadline) + return self.status() + + + def pins(self, journal): + # Pins retain public objects independent of the mutable manifest pointer. + retained = {journal["envelope"]["archive_sha256"], journal["predecessor"]["archive_sha256"]} + previous = self.root / "predecessor.json" + if previous.exists(): + retained.add(parse(read_file(previous), 65536)["archive_sha256"]) + write_json(self.root / "pins.json", {"archives": sorted(retained)}) + + def cleanup(self, journal, deadline): + candidate = journal["candidate"] + self.save(journal, intent="ensure_accepted_candidate") + candidate = self.adapter.hold(candidate, candidate["hold_token"], + min(deadline - STOP - SMOKE - 5, time.monotonic() + READINESS), self.adapter.manifest(candidate)) + journal["candidate"] = candidate + self.adapter.switch(candidate, deadline - STOP - SMOKE) + self.adapter.check(candidate, min(deadline - STOP, time.monotonic() + SMOKE), public=True) + self.save(journal, intent="accept_pointer") + write_json(self.root / "active.json", candidate) + write_json(self.root / "predecessor.json", journal["predecessor"]) + self.pins(journal) + self.save(journal, intent="drain_predecessor") + self.adapter.stop(journal["predecessor"], min(deadline, time.monotonic() + STOP)) + self.save(journal, intent="resume_candidate") + self.adapter.resume(candidate, min(deadline, time.monotonic() + SMOKE)) + journal["cleanup_complete"] = True + journal.pop("error_code", None) + self.save(journal, intent="complete") + + def rollback(self, journal, deadline): + switched = journal.get("switch_attempted", False) + try: + previous = journal["predecessor"] + self.save(journal, intent="restore_predecessor") + # A crash during capture might leave no acknowledged identity. Select + # the persisted previously accepted generation, never a newer cache pointer. + token = previous.get("hold_token") or digest((previous["release_id"] + ":recover").encode())[:32] + previous = self.adapter.hold(previous, token, min(deadline - STOP - SMOKE - 5, + time.monotonic() + READINESS), self.adapter.manifest(previous)) + journal["predecessor"] = previous + self.save(journal, intent="restore_upstream") + self.adapter.switch(previous, deadline - STOP - SMOKE) + self.adapter.check(previous, min(deadline - STOP, time.monotonic() + SMOKE), public=True) + write_json(self.root / "active.json", previous) + if journal["candidate"]: + self.save(journal, intent="stop_candidate") + self.adapter.stop(journal["candidate"], min(deadline, time.monotonic() + STOP)) + self.adapter.resume(previous, deadline) + journal["cleanup_complete"] = True + self.save(journal, "ROLLED_BACK" if switched else "FAILED", "complete") + except Exception: + journal["error_code"] = "rollback_failed" + self.save(journal, "RECOVERY_REQUIRED", "operator_recovery") + + def recover(self): + with locked(self.root): + records = self.journals() + if not records: + return self.status() + journal = max(records, key=lambda item: item["envelope"]["sequence"]) + if journal.get("cleanup_complete"): + active_path = self.root / "active.json" + if active_path.exists(): + active = parse(read_file(active_path), 65536) + deadline = time.monotonic() + TRANSACTION + try: + info = self.adapter.inspect(active, deadline) + if info is None or not info["State"]["Running"]: + token = digest((active["release_id"] + ":restart").encode())[:32] + active = self.adapter.hold(active, token, deadline - STOP - SMOKE, + self.adapter.manifest(active)) + self.adapter.switch(active, deadline - SMOKE) + self.adapter.check(active, min(deadline, time.monotonic() + SMOKE), public=True) + write_json(active_path, active) + self.adapter.resume(active, deadline) + journal.pop("error_code", None) + self.save(journal) + except Exception: + journal["error_code"] = "active_recovery_failed" + self.save(journal) + return self.status() + if journal["state"] == "RECEIVED": + journal["cleanup_complete"] = True + journal["error_code"] = "interrupted_verification" + self.save(journal, "FAILED", "complete") + return self.status() + # A separate detached recovery job has its own bounded 300s budget. + # This never extends the candidate's original acceptance deadline. + deadline = time.monotonic() + TRANSACTION + if journal["state"] == "COMMITTED": + try: + self.cleanup(journal, deadline) + except Exception: + journal["error_code"] = "cleanup_failed" + self.save(journal, intent="cleanup_pending") + else: + self.rollback(journal, deadline) + return self.status() + + +def validate_upload(header): + require(set(header) == {"schema_version", "publication_id", "sequence", "trigger_sha", + "manifest", "deadline", "action"}, "upload_fields") + require(type(header["schema_version"]) is int and header["schema_version"] == 1, "schema") + require(matches(ID, header["publication_id"]) and matches(SHA, header["trigger_sha"]), "upload_identity") + require(type(header["sequence"]) is int and 0 < header["sequence"] <= 2**53 - 1, "sequence") + require(type(header["deadline"]) is int and header["action"] in {"publish", "reference"}, "upload_action") + manifest = validate_manifest(canonical_json(header["manifest"])) + expected = "https://ai.v8std.ru/indexes/v1/" + manifest["archive"]["sha256"] + "/snapshot.tar.gz" + require(manifest["archive"]["path"] == expected, "upload_archive_path") + return header + + +def read_stream(fd, amount, deadline): + """Actual fd reads, including pipes; a slow uploader cannot own the lock forever.""" + output = bytearray() + while len(output) < amount: + require(select.select([fd], [], [], remaining(deadline, 20))[0], "ingress_timeout") + data = os.read(fd, min(65536, amount - len(output))) + require(bool(data), "ingress_truncated") + output.extend(data) + return bytes(output) + + +def read_header(fd, deadline, limit=65536): + output = bytearray() + while len(output) <= limit: + char = read_stream(fd, 1, deadline) + if char == b"\n": + return parse(bytes(output), limit) + output.extend(char) + raise ReleaseError("input_size") + + +def eof(fd, deadline): + require(select.select([fd], [], [], remaining(deadline, 20))[0], "ingress_timeout") + require(os.read(fd, 1) == b"", "ingress_extra_bytes") + + +def remove_upload(path): + if not path.exists(): + return + require(not path.is_symlink() and path.is_dir(), "stage_shape") + require(all(item.name == "snapshot.tar.gz" and stat.S_ISREG(item.lstat().st_mode) + for item in path.iterdir()), "stage_shape") + shutil.rmtree(path) + sync_dir(path.parent) + + +def cleanup_uploads(root, static_root): + """Called only under release.lock, so no live ingress can own these files.""" + if not static_root.exists(): + return + for path in static_root.iterdir(): + if re.fullmatch(r"\.upload-[a-z0-9_]+", path.name): + remove_upload(path) + elif path.name.startswith(".stage-") and matches(ID, path.name[7:]): + receipt = root / "publications" / (path.name[7:] + ".json") + if not receipt.exists() or read_record(receipt)["state"] in {"COMMITTED", "FAILED"}: + remove_upload(path) + + +def publication_result(record): + header = record["header"] + manifest = header["manifest"] + return {"publication_id": header["publication_id"], "sequence": header["sequence"], + "action": header["action"], "state": record["state"], "trigger_sha": header["trigger_sha"], + "corpus_source_sha": manifest["source_sha"], "corpus_id": manifest["corpus_id"], + "archive_sha256": manifest["archive"]["sha256"], "error_code": record.get("error_code")} + + +def restore_index_inbox(root): + """Under release.lock: the fsynced receipt also is an enqueue intent.""" + pending = root / "pending-index.json" + if pending.exists(): + return + directory = root / "publications" + records = [read_record(path) for path in directory.glob("*.json")] + unfinished = [r for r in records if r["state"] not in {"COMMITTED", "FAILED"}] + if unfinished: + record = min(unfinished, key=lambda r: (r["header"]["sequence"], r["header"]["publication_id"])) + write_json(pending, record["header"]) + + +def query_status(root, adapter, query=None): + if query is None: + return Controller(root, adapter).status() + require(set(query) == {"schema_version", "kind", "id"} + and type(query["schema_version"]) is int and query["schema_version"] == 1 + and query["kind"] in {"release", "publication"} and matches(ID, query["id"]), "status_query") + directory = "releases" if query["kind"] == "release" else "publications" + path = Path(root) / directory / (query["id"] + ".json") + if path.exists(): + record = read_record(path) + return Controller.result(record) if query["kind"] == "release" else publication_result(record) + if query["kind"] == "release": + pending = Path(root) / "pending-deploy.json" + if pending.exists(): + envelope = parse(read_file(pending)) + if envelope["release_id"] == query["id"]: + return {**envelope, "state": "QUEUED", "cleanup_complete": False} + rejected = Path(root) / "rejected" / (query["id"] + ".json") + if rejected.exists(): + return parse(read_file(rejected)) + return {"state": "NOT_FOUND", "id": query["id"], "kind": query["kind"]} + + +def read_status_query(fd): + deadline = time.monotonic() + 20 + require(select.select([fd], [], [], remaining(deadline))[0], "ingress_timeout") + first = os.read(fd, 1) + if first == b"": + return None + raw = bytearray(first) + while not raw.endswith(b"\n") and len(raw) <= 1024: + raw.extend(read_stream(fd, 1, deadline)) + eof(fd, deadline) + return parse(bytes(raw), 1024) + + +def ingest(root, static_root, fd, *, seconds=30): + """JSON line + exactly archive.bytes raw bytes + EOF; no client-side paths.""" + root, static_root = Path(root), Path(static_root) + deadline = time.monotonic() + seconds + header = validate_upload(read_header(fd, deadline)) + with locked(root): + cleanup_uploads(root, static_root) + restore_index_inbox(root) + pending = root / "pending-index.json" + record_path = root / "publications" / (header["publication_id"] + ".json") + record = read_record(record_path) if record_path.exists() else None + if record: + require(record["header"] == header, "mutated_duplicate") + else: + if pending.exists(): + require(read_record(pending) == header, "publication_busy") + require(time.time() < header["deadline"] <= time.time() + TRANSACTION, "deadline") + manifest = header["manifest"] + size = manifest["archive"]["bytes"] if header["action"] == "publish" else 0 + require(0 <= size <= MAX_ARCHIVE_BYTES, "input_size") + ensure_directory(static_root, 0o755) + require(shutil.disk_usage(static_root).free > 2 * MAX_ARCHIVE_BYTES, "disk_capacity") + temporary = Path(tempfile.mkdtemp(prefix=".upload-", dir=static_root)) + try: + archive = temporary / "snapshot.tar.gz" + hashed = hashlib.sha256() + with archive.open("xb") as stream: + for start in range(0, size, 65536): + chunk = read_stream(fd, min(65536, size - start), deadline) + hashed.update(chunk) + stream.write(chunk) + stream.flush() + os.fsync(stream.fileno()) + eof(fd, deadline) + if size: + require(hashed.hexdigest() == manifest["archive"]["sha256"], "archive_hash") + if record: + return publication_result(record) + if size: + stage = static_root / (".stage-" + header["publication_id"]) + if stage.exists(): + require(read_file(stage / "snapshot.tar.gz", MAX_ARCHIVE_BYTES) == read_file(archive, MAX_ARCHIVE_BYTES), "stage_conflict") + else: + sync_dir(temporary) + os.rename(temporary, stage) + sync_dir(static_root) + write_json(record_path, {"header": header, "state": "RECEIVED"}) + write_json(pending, header) + return {"publication_id": header["publication_id"], "state": "QUEUED"} + finally: + if temporary.exists(): + shutil.rmtree(temporary) + + +class Publisher: + def __init__(self, root, static_root, verifier=None): + self.root, self.static_root = Path(root), Path(static_root) + self.verifier = verifier or self.verify + + @staticmethod + def verify(archive, header, deadline): + run(attestation_command(str(archive), header["manifest"]["source_sha"]), deadline) + for sha in {header["manifest"]["source_sha"], header["trigger_sha"]}: + comparison = parse(run(["gh", "api", f"repos/{REPO}/compare/{sha}...main"], deadline), 2 * 1024 * 1024) + require(comparison.get("status") in {"ahead", "identical"} + and comparison.get("merge_base_commit", {}).get("sha") == sha, "main_ancestry") + + def publish(self, header): + header = validate_upload(header) + with locked(self.root): + manifest = header["manifest"] + archive_hash = manifest["archive"]["sha256"] + record_path = self.root / "publications" / (header["publication_id"] + ".json") + record = read_record(record_path) + require(record["header"] == header, "mutated_duplicate") + if record["state"] == "COMMITTED": + self.clear_pending(header) + return record + verified = record["state"] in {"VERIFIED", "RECOVERY_REQUIRED"} + require(record["state"] != "FAILED", "publication_terminal") + deadline = time.monotonic() + (TRANSACTION if verified else min(TRANSACTION, header["deadline"] - time.time())) + remaining(deadline) + target = self.static_root / archive_hash + stage = self.static_root / (".stage-" + header["publication_id"]) + if header["action"] == "publish": + archive = (target if target.exists() else stage) / "snapshot.tar.gz" + verify_archive(read_file(archive, MAX_ARCHIVE_BYTES), manifest) + if not verified: + self.verifier(archive, header, deadline) + # Reference bookkeeping is durable before visibility/GC. Failed + # Pages publication retains the object for at least seven days. + reference = self.root / "references" / (archive_hash + ".json") + write_json(reference, {"last_reference": time.time()}) + write_json(self.root / "manifests" / (archive_hash + ".json"), manifest) + record["state"] = "VERIFIED" + write_json(record_path, record) + if not target.exists(): + os.chmod(archive, 0o644) + os.chmod(stage, 0o755) + sync_dir(stage) + os.rename(stage, target) + sync_dir(self.static_root) + elif stage.exists(): + shutil.rmtree(stage) # Exact validated owned staging directory only. + else: + # A reference acknowledgment follows successful Pages publication; + # it cannot introduce an object or change the verified manifest. + require(read_record(self.root / "manifests" / (archive_hash + ".json")) == manifest, "unpublished_manifest") + verify_archive(read_file(target / "snapshot.tar.gz", MAX_ARCHIVE_BYTES), manifest) + if not verified: + self.verifier(target / "snapshot.tar.gz", header, deadline) + current_path = self.root / "current-index.json" + if current_path.exists(): + current = read_record(current_path) + require(header["sequence"] > current["sequence"] or current == header, "stale_sequence") + old_hash = current["manifest"]["archive"]["sha256"] + write_json(self.root / "references" / (old_hash + ".json"), {"last_reference": time.time()}) + record["state"] = "VERIFIED" + write_json(record_path, record) + write_json(self.root / "references" / (archive_hash + ".json"), {"last_reference": time.time()}) + write_json(current_path, header) + record["state"] = "COMMITTED" + record.pop("error_code", None) + write_json(record_path, record) + self.clear_pending(header) + return record + + def clear_pending(self, header): + pending = self.root / "pending-index.json" + if pending.exists() and read_record(pending) == header: + pending.unlink() + sync_dir(self.root) + + def recover(self): + pending = self.root / "pending-index.json" + with locked(self.root): + cleanup_uploads(self.root, self.static_root) + restore_index_inbox(self.root) + header = read_record(pending) if pending.exists() else None + if header is None: + return {"state": "EMPTY"} + try: + return self.publish(header) + except Exception as error: + if isinstance(error, ReleaseError) and error.code == "busy": + raise # Another worker owns the journal; contention is not failure. + # A failed verifier/deadline cannot permanently occupy the ingress + # slot. Keep immutable ID outcome; a new attempt uses a new ID. + with locked(self.root): + record_path = self.root / "publications" / (header["publication_id"] + ".json") + record = read_record(record_path) + # Visibility may precede COMMITTED after a crash: VERIFIED receipts + # are reconciled on restart, never described as a committed job. + verified = record["state"] in {"VERIFIED", "RECOVERY_REQUIRED"} + record.update(state="RECOVERY_REQUIRED" if verified else "FAILED", + error_code=getattr(error, "code", "publication_failed")) + write_json(record_path, record) + if not verified: + self.clear_pending(header) + stage = self.static_root / (".stage-" + header["publication_id"]) + remove_upload(stage) + return record + + def gc(self, *, now=None): + """Internal operator maintenance: references and pins, never mtime alone.""" + now = time.time() if now is None else now + with locked(self.root): + pins = parse(read_file(self.root / "pins.json"))["archives"] + require(isinstance(pins, list) and all(matches(HEX, x) for x in pins), "pins_invalid") + current = read_record(self.root / "current-index.json") + retained = set(pins) | {current["manifest"]["archive"]["sha256"]} + pending = self.root / "pending-index.json" + if pending.exists(): + retained.add(read_record(pending)["manifest"]["archive"]["sha256"]) + removed = [] + for path in self.static_root.iterdir(): + if not matches(HEX, path.name) or path.name in retained or path.is_symlink(): + continue + reference = self.root / "references" / (path.name + ".json") + last = parse(read_file(reference))["last_reference"] + require(type(last) in {int, float} and 0 <= last <= now, "reference_invalid") + if now - last >= 7 * 86400: + # Only exact known immutable-layout objects are ours to remove. + require({p.name for p in path.iterdir()} == {"snapshot.tar.gz"}, "store_layout") + require(digest(read_file(path / "snapshot.tar.gz", MAX_ARCHIVE_BYTES)) == path.name, "store_corrupt") + shutil.rmtree(path) + sync_dir(self.static_root) + removed.append(path.name) + return removed + + +def schedule(kind): + require(kind in {"deploy", "index", "recover"}, "job_kind") + # Shared unit name and controller lock serialize all host effects. No --pipe, + # --wait or inherited SSH stdin; timer recovers a crash before enqueue. + return run(["systemd-run", "--unit=v8std-release-job", "--collect", "--no-block", + "--property=Type=exec", "--property=RuntimeMaxSec=300s", "--property=TimeoutStopSec=5s", + "--property=KillMode=control-group", "/usr/bin/python3", "-I", INSTALL, "_" + kind], + time.monotonic() + 10) + + +def submit(root, adapter, raw): + envelope = validate_envelope(raw, expired=True) + controller = Controller(root, adapter) + with locked(root): + existing = controller.existing(envelope) + if existing: + return controller.result(existing) + require(adapter.policy.get("runtime_enabled") is True, "runtime_not_activated") + adapter.config(envelope) + require((Path(root) / "active.json").is_file(), "predecessor_required") + validate_envelope(raw) + require(envelope["deadline"] - time.time() > RECOVERY_RESERVE, "insufficient_transaction_budget") + pending = Path(root) / "pending-deploy.json" + if pending.exists(): + previous = parse(read_file(pending)) + require(previous == envelope or any(item["envelope"] == previous and item.get("cleanup_complete") + for item in controller.journals()), "busy") + write_json(pending, envelope) + schedule("deploy") + return {"state": "QUEUED", "release_id": envelope["release_id"]} + + +def main(): + require(len(sys.argv) == 2, "command") + command = sys.argv[1] + require(command in {"validate-envelope", "deploy", "recover", "status", "publish-index", + "_deploy", "_index", "_recover"}, "command") + if command == "validate-envelope": + header = read_header(sys.stdin.fileno(), time.monotonic() + 20, 8192) + eof(sys.stdin.fileno(), time.monotonic() + 20) + return validate_envelope(canonical_json(header)) + require(os.geteuid() == 0, "host_privilege") + policy = trusted_policy() + adapter = HostAdapter(ROOT, policy) + controller = Controller(ROOT, adapter) + if command == "status": + return query_status(ROOT, adapter, read_status_query(sys.stdin.fileno())) + if command == "deploy": + envelope = read_header(sys.stdin.fileno(), time.monotonic() + 20, 8192) + eof(sys.stdin.fileno(), time.monotonic() + 20) + return submit(ROOT, adapter, canonical_json(envelope)) + if command == "publish-index": + result = ingest(ROOT, policy["static_root"], sys.stdin.fileno()) + if result["state"] not in {"COMMITTED", "FAILED"}: + schedule("index") + return result + if command == "recover": + schedule("recover") + return {"state": "RECOVERY_QUEUED"} + if command in {"_recover", "_deploy"}: + controller.recover() + pending = ROOT / "pending-deploy.json" + if pending.exists(): + envelope = parse(read_file(pending)) + try: + result = controller.deploy(canonical_json(envelope)) + except ReleaseError as error: + if error.code not in {"deadline", "insufficient_transaction_budget", "runtime_not_activated", "predecessor_required", "stale_sequence"}: + raise + with locked(ROOT): + write_json(ROOT / "rejected" / (envelope["release_id"] + ".json"), + {**envelope, "state": "REJECTED", "error_code": error.code}) + if parse(read_file(pending)) == envelope: + pending.unlink() + sync_dir(ROOT) + if command in {"_recover", "_index"}: + return Publisher(ROOT, policy["static_root"]).recover() + return controller.status() + + +if __name__ == "__main__": + try: + print(json.dumps(main(), sort_keys=True)) + except Exception as error: + print(json.dumps({"state": "REJECTED", "error_code": error.code if isinstance(error, ReleaseError) else "host_failure"})) + raise SystemExit(1) diff --git a/scripts/v8std_mcp_runtime.py b/scripts/v8std_mcp_runtime.py index bd2770d..7ede68a 100644 --- a/scripts/v8std_mcp_runtime.py +++ b/scripts/v8std_mcp_runtime.py @@ -14,6 +14,7 @@ from v8std_mcp_presentation import LinkCatalog, present_markdown, present_result from v8std_mcp_snapshot_format import DEFAULT_SITE_URL, VerifiedSnapshot, normalize_site_url from v8std_mcp_snapshots import LoaderError, SnapshotCoordinator, SnapshotStore +from v8std_mcp_hold import CONTROL_PATH @dataclass(frozen=True) @@ -47,14 +48,15 @@ def build_generation(snapshot: VerifiedSnapshot, *, max_snippet_chars: int, class SnapshotIndex: def __init__(self, *, site_url: str = DEFAULT_SITE_URL, cache_dir: Path = DEFAULT_CACHE_DIR, refresh_seconds: int = 3600, max_snippet_chars: int = MAX_SNIPPET_CHARS, - runtime_sha: str | None = None): + runtime_sha: str | None = None, release_control: Path | None = None): self.site_url = normalize_site_url(site_url) self._max_snippet_chars = validate_max_snippet_chars(max_snippet_chars) self.runtime_sha = runtime_sha if runtime_sha and len(runtime_sha) == 40 and all( char in "0123456789abcdef" for char in runtime_sha) else None self.coordinator = SnapshotCoordinator(SnapshotStore(self.site_url, cache_dir), partial(build_generation, max_snippet_chars=max_snippet_chars, site_url=self.site_url), - refresh_seconds=refresh_seconds) + refresh_seconds=refresh_seconds, release_control=(release_control if release_control is not None + else CONTROL_PATH if CONTROL_PATH.parent.exists() else None)) @property def max_snippet_chars(self): diff --git a/scripts/v8std_mcp_snapshots.py b/scripts/v8std_mcp_snapshots.py index 2b2fcda..7dea55f 100644 --- a/scripts/v8std_mcp_snapshots.py +++ b/scripts/v8std_mcp_snapshots.py @@ -429,6 +429,15 @@ def _gc(self, retained=()): if not isinstance(archives, list) or not all(self._digest(d) for d in archives): raise LoaderError("cache_io") protected.update(archives) + # A release-managed process may lag the durable pointer after worker IPC + # failure. Its last accepted bytes must survive subsequent refresh GC. + try: + runtime = strict_json(_read_file(self.namespace / "runtime-pin.json", MAX_MANIFEST_BYTES)) + if not self._digest(runtime.get("archive")): + raise LoaderError("cache_io") + protected.add(runtime["archive"]) + except FileNotFoundError: + pass for entry in self.namespace.iterdir(): if _TEMP.fullmatch(entry.name): self._remove_owned(entry) @@ -454,14 +463,14 @@ def _reuse_checked_entry(self, entry, prepare, deadline, current_archive): self._commit_state(entry[2], entry[2], deadline) return result - def _refresh(self, prepare, deadline, *, current_archive=None): + def _refresh(self, prepare, deadline, *, current_archive=None, selected_manifest=None): _directory(self.cache_dir, create=True) _directory(self.namespace, create=True) _directory(self.namespace / "generations", create=True) with _file_lock(self.namespace / ".lock", deadline) as waited: with _file_lock(self.cache_dir / ".volume.lock", deadline): entry = self._cached_entry() - if waited and entry: + if waited and entry and selected_manifest is None: return self._reuse_checked_entry(entry, prepare, deadline, current_archive) self._gc((entry[0].archive_sha256,) if entry else ()) headers = {} @@ -470,9 +479,12 @@ def _refresh(self, prepare, deadline, *, current_archive=None): if value := entry[2]["validators"].get(field): headers[header] = value bootstrap = self.site_url + "ai/mcp/v1/manifest.json" - status, raw, validators, final_url = self._transport( - bootstrap, self.site_url, headers, MAX_MANIFEST_BYTES, - deadline, self._read_seconds) + if selected_manifest is None: + status, raw, validators, final_url = self._transport( + bootstrap, self.site_url, headers, MAX_MANIFEST_BYTES, + deadline, self._read_seconds) + else: + status, raw, validators, final_url = 200, canonical_json(selected_manifest), {}, bootstrap if status == 304 and entry is None: status, raw, validators, final_url = self._transport( bootstrap, self.site_url, {}, MAX_MANIFEST_BYTES, deadline, self._read_seconds) @@ -550,12 +562,12 @@ def _refresh(self, prepare, deadline, *, current_archive=None): if stage.exists(): self._remove_owned(stage) - def _run(self, mode, prepare, stop=None, *, current_archive=None): + def _run(self, mode, prepare, stop=None, *, current_archive=None, selected_manifest=None): deadline = time.monotonic() + self._attempt_seconds stop = stop if stop is not None else threading.Event() parent, child = socket.socketpair() process = multiprocessing.get_context("spawn").Process( - target=_worker, args=(self, mode, prepare, deadline, child, current_archive), + target=_worker, args=(self, mode, prepare, deadline, child, current_archive, selected_manifest), name="v8std-snapshot-worker", daemon=True) started = False try: @@ -611,13 +623,14 @@ def _cache_walk_error(): raise LoaderError("cache_io") -def _worker(store, mode, prepare, deadline, channel, current_archive): +def _worker(store, mode, prepare, deadline, channel, current_archive, selected_manifest=None): try: if mode == "cached": snapshot = store.cached() payload = _prepare(snapshot, prepare) if snapshot else pickle.dumps(("ok", None, None)) else: - payload = store._refresh(prepare, deadline, current_archive=current_archive) + payload = store._refresh(prepare, deadline, current_archive=current_archive, + selected_manifest=selected_manifest) _remaining(deadline) except SnapshotError as error: payload = pickle.dumps(("format_error", error.code, None)) @@ -638,7 +651,8 @@ def _worker(store, mode, prepare, deadline, channel, current_archive): class SnapshotCoordinator: - def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600): + def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600, + release_control: Path | None = None): if type(refresh_seconds) is not int or refresh_seconds < 0: raise LoaderError("configuration") self.store = store @@ -649,7 +663,13 @@ def __init__(self, store: SnapshotStore, build, *, refresh_seconds: int = 3600): self._thread = None self._current = None self._archive_sha256 = None + self._release_control = None + if release_control is not None: + from v8std_mcp_hold import ReleaseControl + self._release_control = ReleaseControl(self, Path(release_control)) self._state = {"ready": False, "corpus_id": None, "loaded_at": None, + "archive_sha256": None, "corpus_source_sha": None, "hold_token": None, + "release_control_token": None, "last_checked_at": None, "last_success_at": None, "refresh_error_code": None} @@ -674,6 +694,8 @@ def status(self) -> dict: def _accept(self, result, metadata, *, checked): now = time.time() retired = None + if self._release_control is not None: + self._release_control.pin(metadata["archive_sha256"]) with self._lock: if metadata.get("unchanged") and (not self._state["ready"] or metadata["archive_sha256"] != self._archive_sha256 @@ -683,7 +705,9 @@ def _accept(self, result, metadata, *, checked): retired = self._current self._current = result self._archive_sha256 = metadata["archive_sha256"] - self._state.update(ready=True, corpus_id=metadata["corpus_id"], loaded_at=now) + self._state.update(ready=True, corpus_id=metadata["corpus_id"], loaded_at=now, + archive_sha256=metadata["archive_sha256"], + corpus_source_sha=metadata.get("source_sha")) if checked: self._state.update(last_checked_at=now, last_success_at=now, refresh_error_code=None) # Dropping a large generation's final reference can release thousands of @@ -697,6 +721,9 @@ def _delay(self, failures): return self.refresh_seconds * random.uniform(.8, 1.2) def _loop(self): + if self._release_control is not None: + self._release_control.run() + return try: result, metadata = self.store._run("cached", self.build, self._stop) if metadata: diff --git a/spec/operations/mcp-container-activation.md b/spec/operations/mcp-container-activation.md index e0b815c..0011296 100644 --- a/spec/operations/mcp-container-activation.md +++ b/spec/operations/mcp-container-activation.md @@ -1,8 +1,10 @@ # Controlled initial activation of container delivery -**Status:** prerequisite checklist; host-controller commands and rehearsal -evidence are completed by the release implementation task. This document does -not authorize a live operation, enable CI, or establish current host capacity. +**Status (2026-09-15):** ordinary controller implemented for scoped review; +first-bootstrap implementation and native rehearsal are still pending. +Local disposable evidence is not a live installation, CI activation, published +image proof, or target-host capacity guarantee. This runbook authorizes none of +those operations. The current external scope remains image-only. ## Authority and stop conditions @@ -72,12 +74,24 @@ Catalog. Do not rebuild a special production image on the server. ## Capacity and acceptance gate -Measure old runtime + candidate + snapshot preparation together, including +For ordinary deployment, measure old runtime + candidate + snapshot preparation together, including Docker/OS overhead and static index traffic. Verify memory, disk staging and pins, descriptors, CPU and bandwidth before switching. Raise capacity or change the accepted rollout design if the measured host cannot accommodate overlap; do not silently kill the predecessor to make room. +Only the separately authorized first migration may use stop-legacy/start-new +inside a scheduled window of at most two hours if overlap cannot fit. That +exception does not weaken the ordinary automatic gate, establish single-runtime +capacity, or authorize stopping production now. Prepare and verify the backup, +immutable artifacts and independent recovery guard before the window. Mint the +300-second execution envelope just before each attempt, not before lengthy +preparation. Stop new attempts at least 30 minutes before the window ends; +reserve more time if the measured legacy restoration needs it. Already-started +recovery remains necessary after the window expires. `bootstrap`, +`bootstrap-recover` and `bootstrap-status` are **not implemented in this slice**; +the restricted SSH entry rejects them. Do not manufacture `active.json`. + Exercise initialized idle agents, normal POST tool calls, reconnects, shared NAT, snapshot refresh and concurrent archive downloads. Record the actual mix, duration, error rate and latency. Neither worker_connections nor idle TCP count @@ -87,3 +101,243 @@ Record the initial release journal, exact SHA/digests/corpus/configuration, public MCP/TLS and static delivery results, failure/rollback rehearsal and monitoring checks in the verification record. External Catalog acceptance and closure of the alternative PR remain separate delivery outcomes. + +## Read-only legacy observations, not restoration proof + +The coordinating owner recorded the following on 2026-09-15. Refresh before an +authorized migration; corpus files can change. No local test configured this +host. Full private evidence is in the Task5 handoff's +`legacy-preflight-2026-09-15.md`. + +- `/etc/systemd/system/v8std-mcp.service` is enabled/active; no drop-ins or + EnvironmentFiles. Unit SHA256: + `072bd0e8ee1d2b24012085a4d7563f60ab5103fcef41de41a8ca9afb95df84ab`. +- WorkingDirectory `/opt/v8std-mcp`, User/Group `v8std-mcp`; + `/opt/v8std-mcp/venv/bin/python` resolves to `/usr/bin/python3.12`. + ExecStart runs `/opt/v8std-mcp/scripts/v8std_mcp_server.py` with + `--index-url https://v8std.ru/ai/pages.jsonl --vectors-url https://v8std.ru/ai/search-vectors.jsonl --cache-dir /var/lib/v8std-mcp --host 127.0.0.1 --port 8765 --mcp-path /mcp --max-snippet-chars 4000 --usage-log /var/lib/v8std-mcp/tool-usage.jsonl`. +- A protected backup must establish trusted ownership and complete source/venv + identity; do not infer these from a few matching source hashes. +- Preserve `/var/lib/v8std-mcp/{pages.jsonl,search-vectors.jsonl,llms.txt,llms-full.txt}`. + Separately preserve private usage logs; never print their raw contents or use + changing usage logs as corpus identity. +- Native legacy health is HTTP200 with 1423 pages/3281 vectors. Compare its + `sha256` = `4876122c8f2fa3c25c49afc0c986a72a976d4466e9b654041729ab42a634f4e4` + and `vectors.sha256` = `7713439da96757be4cf786d303e1cfb4b7e336ca8ec54a28a722701f6ef6c3c8`. + It does **not** expose the new runtime/corpus/hold identity fields. +- Refresh the private resource inventory before migration. Neither overlap + nor single-container preparation is proven to fit. +- Check the MCP certificate and any dependencies on other virtual hosts + before changing TLS. No unrelated service deletion is authorized. + Preserve SSH, HTTP/HTTPS, loopback8765 until cutover, renewal and + monitoring. Record the remaining private inventory outside Git. + +Before migration, inventory and hash the entire saved code, dependency lock and +installed venv/interpreter, unit/configuration and coherent data set; protect +the backup off-host, restore it in disposable Linux, and measure return time. +The ordinary controller does not restore this legacy Python deployment; that +is the pending bootstrap slice. Do not stop the still-enabled legacy unit until +that slice proves preaccept restoration and postaccept reboot ownership. + +## Reviewed host installation boundary (future operator action) + +Installation is separate from release envelopes. Install only an independently +reviewed, verified main revision; update trust/controller only under a separate +operator authorization. No release input can replace them. + +| Fixed host target | Owner / access / contents | +|---|---| +| `/opt/v8std-release/scripts/` | root, not group/world writable; `v8std_mcp_release.py`, `v8std_mcp_snapshot_format.py`, `v8std_mcp_chunks.py` from the same reviewed revision; stdlib-only host dependency chain | +| `/opt/v8std-release/release-entry.py` | root0755, installed from `deploy/container/release-entry.py`; parent chain root-owned | +| `/etc/v8std-release/policy.json` | root0600, parent root0755, installed from the disabled example and explicitly configured; no symlinks/writable parents | +| `/var/lib/v8std-release` | root0700; journals, durable inbox, receipts, trusted manifests, references, pins, slots and verifier configuration; never writable by CI | +| `/srv/v8std-indexes` and `/srv/v8std-indexes/v1` | precreate root0755 for nginx traversal; publisher alone writes, nginx only reads; objects directories0755/files0644; staging0700 | +| `/etc/nginx/v8std-release/upstream.conf` | root-owned managed include, initially created only by the approved bootstrap; not an arbitrary caller path | +| systemd recovery units / sudoers | exact reviewed files from `deploy/container/`; validate locally on native Linux before enabling | + +The installed controller runs `/usr/bin/python3 -I`; it adds only its own +root-owned module directory to imports. Provision the fixed trusted executable +PATH (`/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin`), `gh`, Docker/buildx and +nginx as operator-owned dependencies. Test actual `gh attestation verify` +capability/read access using the intended root service context: child commands +receive only that PATH and `HOME=/var/lib/v8std-release`, not CI environment +variables, user Docker config, proxy settings or a caller's token. Do not copy +the developer's credentials or change their Docker socket/privileges. + +The publisher account `v8std-publisher` has no Docker group, general sudo, root +SSH, SCP/SFTP or writable controller/policy/store paths. Install the exact +`release.sudoers` only after `visudo -cf` succeeds. The authorized key entry +must use `restrict,command="/opt/v8std-release/release-entry.py"` and an approved +public key, with no PTY/agent/port/X11 forwarding and no alternative shell/key +entry. Confirm this through an actual restricted-account native test before +issuing a CI secret. SSH host key pinning is mandatory; do not use accept-new or +disable StrictHostKeyChecking in CI. + +`release-policy.example.json` is intentionally unusable by default: + +- `enabled:false` disables all stateful public host commands, including status; + syntax-only `validate-envelope` remains available. Leave it false until the + restricted entry, durable recovery and store have passed their own gate. +- `enabled:true,runtime_enabled:false` permits static publish/reference and + structured status/recovery with empty runtime configs and no capacity proof. + This is the static-only setup: it does not require a Docker cutover. Image + publication in GitHub is independent of either host switch. +- Ordinary runtime deploy requires **both** host flags true, a real accepted + predecessor, trusted configuration and capacity evidence, plus independently + enabled/protected runtime deployment in future CI. Image/corpus publication + must not implicitly enable that CI job. Disabling `runtime_enabled` blocks + new runtime jobs, not cleanup/recovery already owed to accepted work. +- Do not set `enabled:false` as an in-flight kill switch: it also blocks the + recovery service. Keep recovery enabled until all accepted work is reconciled. + +Each allowed config digest is SHA256 of canonical JSON with exactly `site_url` +(`https://v8std.ru/`), `refresh_seconds`, `max_snippet_chars`, `memory_bytes`, +`cpus`. Keep the existing strict/direct runtime profile unchanged. No second +source URL, image tag, mount, arbitrary file, shell or env appears in an envelope. +Policy `platform` is explicitly `linux/amd64` or `linux/arm64`, not inferred from +Docker `.Id`. Verify the attested index, child mediaType/platform membership and +config descriptor: `.Id` can be index, child or config in the verified mapping. + +Runtime activation requires root-reviewed native mixed-load evidence at +`/var/lib/v8std-release/capacity/.json`; policy binds its byte hash. +The hash is an operator approval binding, not automatic interpretation or proof +of its measurement content. Set policy disk/MemAvailable/FD thresholds from the +rehearsal; memory reserve must be at least candidate memory cap +128MiB, measured +while the old service is running. Controller checks live disk, MemAvailable, +RLIMIT_NOFILE and the evidence hash before runtime effects. Example thresholds +are not target-host recommendations. + +Use `edge-http.conf` once in nginx `http{}`, and `edge-locations.conf` only in +the inventoried ai TLS server. Preserve existing TLS/default-vhost ownership. +Set native main-context `worker_shutdown_timeout 30s`; rehearse worker/FD, +client-header and idle keep-alive bounds with the existing configuration. +The sample has shared active8/download2 admission, upstream keepalive2, +download1MiB/s and retryable429/503+Retry-After1. These are local test settings, +not permission to apply them on the 961MiB host. On any nginx syntax error the +old include must be restored with no reload. Public TLS smoke must succeed +without insecure TLS options before acceptance. + +Install/enable recovery timer only after native `systemd-analyze verify` and +fault rehearsal. Services are `Type=exec` with `RuntimeMaxSec=300s`, +`TimeoutStopSec=5s`, `KillMode=control-group`. Do not substitute `Type=oneshot`: +RuntimeMaxSec does not bound its execution. The timer runs on boot+15s and +30s after completion; SSH scheduling uses fixed detached `v8std-release-job`, +not `--pipe`/live stdin. A queued receipt survives scheduling failure, SSH loss +and CI cancellation and is picked up by recovery. + +## Task6 restricted wire interface and exact acknowledgements + +The only public forced commands are `validate-envelope`, `deploy`, `recover`, +`status`, `publish-index`, with **no arguments**. The examples below describe +future CI invocations, not commands executed against production in Task5. +`release_host` must come from the separately approved, pinned SSH setup. + +`validate-envelope` and `deploy` consume one compact JSON line (at most8192 +bytes) followed by EOF. Schema is `deploy/container/release.schema.json`. +`deadline` is UTC epoch seconds, future and at most300s away; an ordinary deploy +also requires more than180s left for rollback reserve. Runtime source SHA, +corpus manifest source SHA and triggering workflow SHA are independent. Never +label a reused image with a new content SHA. Bind image/artifact attestations +to `zeegin/v8std/.github/workflows/ci.yml`, source-ref `refs/heads/main`, exact +source digest and GitHub issuer; self-hosted attestations are denied. Chosen +runtime/corpus and trigger commits must belong to current authorized main +history. A reusable/different signing workflow requires separately reviewed +host trust changes, not an envelope field. + +`publish-index` consumes a JSON header line (at most65536 bytes), then exactly +`manifest.archive.bytes` raw archive bytes, then EOF. The header has exactly: + +```json +{"schema_version":1,"publication_id":"content-run-123","sequence":123,"trigger_sha":"<40 lowercase hex>","manifest":{},"deadline":0,"action":"publish"} +``` + +Replace the placeholders with the validated snapshot manifest and fresh +deadline. `manifest.archive.path` must be exactly +`https://ai.v8std.ru/indexes/v1//snapshot.tar.gz`. +Compressed archive cap16MiB, unpacked cap64MiB; existing snapshot member/row +validation applies. Upload gets30s total, each fd wait at most20s. Extra/truncated +bytes, hash mismatch, arbitrary path/env/shell fields or missing EOF fail closed. +The host derives staging/final paths itself. CI never supplies a preexisting +host path and never needs unrestricted file transfer. + +With `upload-header.json` compact and `snapshot.tar.gz` from the CI artifact +workspace, one concrete invocation is: + +```sh +python3 -c 'import json,sys; h=json.load(open("upload-header.json")); sys.stdout.buffer.write(json.dumps(h,separators=(",",":")).encode()+b"\n"); sys.stdout.buffer.flush(); import shutil; shutil.copyfileobj(open("snapshot.tar.gz","rb"),sys.stdout.buffer)' | ssh -T -o BatchMode=yes -o StrictHostKeyChecking=yes "$release_host" publish-index +printf '%s\n' '{"schema_version":1,"kind":"publication","id":"content-run-123"}' | ssh -T -o BatchMode=yes -o StrictHostKeyChecking=yes "$release_host" status +``` + +Use pipeline failure propagation (`set -o pipefail`) in CI. Poll the exact typed +status query with bounded retry/backoff. Accept only the expected +`publication_id`, `sequence`, `action`, `trigger_sha`, `corpus_source_sha`, +`corpus_id`, `archive_sha256`, **state COMMITTED and no error_code**. `QUEUED`, +`RECEIVED`, `VERIFIED`, `RECOVERY_REQUIRED`, `FAILED`, `NOT_FOUND`, an SSH exit0, +or an already-existing immutable HTTP200 is not publication acknowledgement. +The receipt is durable before enqueue; recovery reconstructs a missing inbox. +COMMITTED means verification, retention bookkeeping and immutable visibility +have completed, not that Pages has published a new manifest. + +After that receipt, verify public archive hash/GET/HEAD, publish Pages, verify +the public manifest and its target bytes, then send a **new** publication ID +with `action:"reference"`, the same verified manifest, a fresh deadline and a +monotonically increasing reference sequence. It sends only the header line +and EOF, **no archive bytes**. For example: + +```sh +python3 -c 'import json; print(json.dumps(json.load(open("reference-header.json")),separators=(",",":")))' | ssh -T -o BatchMode=yes -o StrictHostKeyChecking=yes "$release_host" publish-index +printf '%s\n' '{"schema_version":1,"kind":"publication","id":"reference-run-123"}' | ssh -T -o BatchMode=yes -o StrictHostKeyChecking=yes "$release_host" status +``` + +Reference verification also binds corpus/trigger authority. Its exact COMMITTED +receipt proves `current-index.json` and old/new reference timestamps were +durably updated; it does not independently attest the Pages job. Failed Pages +publication must not send this acknowledgement. Failed/unreferenced objects +remain retained for at least seven days. Stale reference sequences cannot undo +a newer current reference. Immutable publication IDs and release IDs cannot be +mutated: retry the exact original header/envelope, including its deadline, or +create a new ID/sequence for a new attempt. A terminal FAILED ID stays failed. + +Runtime status query is the same bounded shape with `kind:"release"` and the +exact release ID. `status` with EOF and no JSON returns the latest release only; +use `ssh -n ... status` for that form, not for queries/uploads. Output is bounded +structured identity/status, not raw command output/secrets. Runtime acceptance +is COMMITTED; require `cleanup_complete:true`, no error and actual health for +an operationally complete deployment. COMMITTED with cleanup pending is already +accepted and recovery must finish it, never undo it. `recover` queues independent +reconciliation and does not mean rollback has already succeeded. + +## Runtime recovery and retention semantics + +Each attempt keeps300s total /90s readiness /30s smoke and drain /45s stop. +The forward phase conservatively reserves180s for rollback, so verification, +pull/readiness/switch share at most120s (less envelope transport time). No phase +borrows from rollback; an over-budget candidate is cancelled. The runtime loader +retains its360s attempt/20s read configuration. Detached recovery has a separate +bounded300s to restore owed state, not to extend candidate acceptance authority. + +Host pins retain archives; they do not select a runtime. The private read-only +`/run/v8std-release/control.json` mount commands hold/capture or selected manifest. +The coordinator cancels preparation, loads/verifies selected bytes and emits an +actual hold token with archive/corpus/runtime identity. Hold is not an MCP tool +or public source setting. Runtime cache `runtime-pin.json` retains its actual +accepted generation even if a worker changed the disk pointer but died before +the process accepted it. Source movement cannot silently change the selected +candidate or rollback corpus; resume explicitly acknowledges normal refresh. + +Before acceptance, recovery reconciles deterministic owned objects, restores +the selected predecessor/upstream, checks real public MCP/static results, then +stops the candidate. Failure is RECOVERY_REQUIRED, never successful rollback. +After COMMITTED, recovery starts/reselects the accepted candidate if necessary, +finishes active/predecessor persistence, drain and resume. A later stopped +accepted process is restarted from the accepted record. Monitor both receipt +errors and live readiness; historical COMMITTED alone is not current liveness. + +Retain at least one successfully served predecessor image/config/corpus and all +in-flight pins. Stopped containers and dedicated caches are intentionally kept; +there is no automatic Docker prune. Static GC is internal operator maintenance, +not a public CLI verb: it checks exact immutable layout/hash, current reference, +all pins/pending work and seven-day last-reference age before removing anything. +Without valid pin/current-reference evidence GC refuses to run. Review exact +owned targets before later cleanup; preserve recovery records. No legacy/old +vhost cleanup is bundled with a successful container release. diff --git a/tests/mcp_release_fixture.py b/tests/mcp_release_fixture.py new file mode 100644 index 0000000..28458ca --- /dev/null +++ b/tests/mcp_release_fixture.py @@ -0,0 +1,186 @@ +"""Disposable real runtime/proxy processes for host-controller fault injection. + +This adapter replaces Docker/gh/nginx command boundaries, NOT runtime health or +MCP results. It is not positive registry, signature or native systemd evidence. +""" +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import v8std_mcp_release as release + + +class ProcessAdapter(release.HostAdapter): + def __init__(self, root, policy, fault=""): + super().__init__(root, policy) + self.fault = fault + self.children = [] + + def record(self, operation, record=None): + with (self.root / "calls.jsonl").open("a") as stream: + stream.write(json.dumps({"operation": operation, "release_id": (record or {}).get("release_id")}) + "\n") + if operation in self.fault.split(","): + raise release.ReleaseError("injected_" + operation) + + def verify(self, envelope, deadline): + self.record("verify", envelope) + return {envelope["image_digest"]: next(iter(release.INDEX_TYPES)), + envelope["platform_digest"]: next(iter(release.MANIFEST_TYPES))} + + def capacity(self, deadline): + self.record("capacity") + + def pull(self, record, deadline): + self.record("pull", record) + + def inspect(self, record, deadline): + path = self.root / "processes" / (record["release_id"] + ".json") + if not path.exists(): + return None + state = json.loads(path.read_text()) + release.require(state["record"]["envelope_hash"] == record["envelope_hash"], "ownership") + try: + os.kill(state["pid"], 0) + # An orphan zombie on Linux is no longer a running endpoint. + proc = Path(f"/proc/{state['pid']}/stat") + if proc.exists(): + running = proc.read_text().split()[2] != "Z" + else: + status = subprocess.run(["ps", "-p", str(state["pid"]), "-o", "stat="], capture_output=True).stdout.strip() + running = bool(status) and not status.startswith(b"Z") + except ProcessLookupError: + running = False + return {"State": {"Running": running}, "pid": state["pid"]} + + def start(self, record, deadline): + self.record("start", record) + info = self.inspect(record, deadline) + if info and info["State"]["Running"]: + return + directory = self.root / "slots" / record["release_id"] + config = self.config(record) + arguments = [sys.executable, "-m", "tests.mcp_release_fixture", "runtime", str(directory), + str(record["port"]), config["site_url"], record["runtime_source_sha"], + "blocked" if self.fault == "ready" and record["release_id"] != "predecessor" else "normal"] + with (directory / "runtime.log").open("ab") as log: + process = subprocess.Popen(arguments, cwd=ROOT, stdin=subprocess.DEVNULL, + stdout=log, stderr=log, start_new_session=True) + self.children.append(process) + release.write_json(self.root / "processes" / (record["release_id"] + ".json"), + {"pid": process.pid, "record": record}) + if self.fault == "crash_after_start" and record["release_id"] != "predecessor": + os._exit(93) + + def hold(self, record, token, deadline, manifest=None): + return super().hold(record, token, min(deadline, time.monotonic() + 5), manifest) + + def switch(self, record, deadline): + self.record("switch_old" if record["release_id"] == "predecessor" else "switch", record) + release.write_json(self.root / "edge.json", {"port": record["port"]}) + if self.fault == "crash_after_switch" and record["release_id"] != "predecessor": + os._exit(92) + + def check(self, record, deadline, *, public=False): + operation = ("public_old" if record["release_id"] == "predecessor" else "public") if public else "smoke" + self.record(operation, record) + # For the public failure case, stop the actual candidate. The edge must + # fail a real HTTP request before rollback can be claimed. + if "public_dead" in self.fault.split(",") and public and record["release_id"] != "predecessor": + self.stop(record, deadline) + return super().check(record, min(deadline, time.monotonic() + 5), public=public) + + def stop(self, record, deadline): + self.record("stop", record) + info = self.inspect(record, deadline) + if info and info["State"]["Running"]: + os.kill(info["pid"], signal.SIGTERM) + while self.inspect(record, deadline)["State"]["Running"]: + release.remaining(deadline) + time.sleep(.025) + try: + os.waitpid(info["pid"], os.WNOHANG) + except (ChildProcessError, TypeError): + pass + + +class CrashController(release.Controller): + def save(self, journal, state=None, intent=None): + super().save(journal, state, intent) + if self.adapter.fault == "crash_" + journal["state"] or self.adapter.fault == "intent_" + journal["intent"]: + os._exit(91) + + +def serve_edge(root, port): + class Edge(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def dispatch(self): + if self.path.startswith("/indexes/v1/"): + parts = self.path.split("/") + if len(parts) != 6 or not release.matches(release.HEX, parts[3]) or parts[4] != "snapshot.tar.gz": + # split('/indexes/v1/hash/snapshot.tar.gz') has five entries. + if len(parts) != 5 or not release.matches(release.HEX, parts[3]) or parts[4] != "snapshot.tar.gz": + self.send_error(404) + return + try: + data = (root / "static" / parts[3] / "snapshot.tar.gz").read_bytes() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + except OSError: + self.send_error(404) + return + connection = None + try: + target = json.loads((root / "edge.json").read_text())["port"] + connection = http.client.HTTPConnection("127.0.0.1", target, timeout=3) + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + connection.request(self.command, self.path, body, dict(self.headers)) + response = connection.getresponse() + data = response.read() + self.send_response(response.status) + self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Type", response.getheader("Content-Type", "application/json")) + self.end_headers() + self.wfile.write(data) + except OSError: + self.send_error(503) + finally: + if connection: + connection.close() + + do_GET = do_POST = dispatch + ThreadingHTTPServer(("127.0.0.1", port), Edge).serve_forever() + + +if __name__ == "__main__": + mode, directory, *args = sys.argv[1:] + directory = Path(directory) + if mode == "runtime": + from v8std_mcp_runtime import SnapshotIndex + from v8std_mcp_server import build_server + port, site_url, sha, behavior = args + index = SnapshotIndex(site_url=site_url, cache_dir=directory / "cache", runtime_sha=sha, + refresh_seconds=1, release_control=directory / "control" / "control.json") + if behavior == "blocked": + from tests.test_v8std_mcp_snapshots import blocking_build + index.coordinator.build = blocking_build + build_server(index, host="127.0.0.1", port=int(port), mcp_path="/mcp", + allowed_hosts=["127.0.0.1:*"], allowed_origins=[]).run(transport="streamable-http") + elif mode == "edge": + serve_edge(directory, int(args[0])) + else: + adapter = ProcessAdapter(directory, json.loads((directory / "policy.json").read_text()), args[0] if args else "") + controller = CrashController(directory, adapter) + result = controller.deploy((directory / "envelope.json").read_bytes()) if mode == "deploy" else controller.recover() + print(json.dumps(result)) diff --git a/tests/test_v8std_mcp_release.py b/tests/test_v8std_mcp_release.py new file mode 100644 index 0000000..2537799 --- /dev/null +++ b/tests/test_v8std_mcp_release.py @@ -0,0 +1,619 @@ +"""Restricted release boundary and real process transaction conformance.""" +import importlib +import importlib.util +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import patch +from tests.test_v8std_mcp_snapshots import Source +from tests.test_v8std_mcp_release_hold import eventually +from tests.mcp_release_fixture import ProcessAdapter +from tests import mcp_snapshot_fixtures as fixture +import v8std_mcp_release as release + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + + +def envelope(**changes): + return dict(schema_version=1, release_id="release-1", sequence=1, + runtime_source_sha="a" * 40, trigger_sha="b" * 40, + image="ghcr.io/zeegin/v8std-mcp", image_digest="sha256:" + "1" * 64, + platform_digest="sha256:" + "2" * 64, configuration_digest="3" * 64, + corpus_id="4" * 64, archive_sha256="5" * 64, + deadline=int(time.time()) + 300, **changes) + + +class EnvelopeTests(unittest.TestCase): + def test_new_journal_parents_are_fsynced_and_symlink_file_rejected(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + synced = [] + original = release.sync_dir + def record(path): + synced.append(path) + original(path) + with patch.object(release, "sync_dir", record): + release.write_json(root / "state/releases/release.json", {"intent": "before_effect"}) + self.assertLess(synced.index(root), synced.index(root / "state")) + self.assertIn(root / "state/releases", synced) + (root / "alias").symlink_to(root / "state/releases/release.json") + with self.assertRaises(OSError): + release.read_file(root / "alias") + + def test_docker_stop_refuses_foreign_ownership_or_wrong_descriptor(self): + record = envelope() | {"name": "v8std-release-release-1", "envelope_hash": "a" * 64, + "descriptors": {"sha256:" + "e" * 64: next(iter(release.CONFIG_TYPES))}} + info = {"Config": {"Image": release.IMAGE + "@" + record["platform_digest"], "Labels": { + "pro.v8std.release": record["release_id"], "pro.v8std.envelope": record["envelope_hash"], + "org.opencontainers.image.revision": record["runtime_source_sha"]}}, "Image": "sha256:" + "e" * 64} + with tempfile.TemporaryDirectory() as temp: + adapter = release.HostAdapter(Path(temp), {}) + for mutation in ("ownership", "image"): + bad = json.loads(json.dumps(info)) + if mutation == "ownership": + bad["Config"]["Labels"]["pro.v8std.release"] = "foreign" + else: + bad["Image"] = "sha256:" + "f" * 64 + with patch.object(release, "run", return_value=json.dumps([bad]).encode()) as run: + with self.assertRaises(release.ReleaseError): + adapter.stop(record, time.monotonic() + 3) + self.assertEqual(run.call_count, 1) + self.assertEqual(run.call_args.args[0][1], "inspect") + def test_root_policy_static_only_activation_and_immutable_trust(self): + policy = json.loads((ROOT / "deploy/container/release-policy.example.json").read_text()) + with self.assertRaisesRegex(release.ReleaseError, "not_activated"): + release.validate_policy(policy) + policy["enabled"] = True + self.assertFalse(release.validate_policy(policy)["runtime_enabled"]) + for change in ({"runtime_enabled": True}, {"signer_workflow": "evil/repo/build.yml"}, + {"static_root": "/etc"}, {"schema_version": True}): + with self.assertRaises(release.ReleaseError): + release.validate_policy(policy | change) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "policy.json" + target.write_text(json.dumps(policy)) + target.chmod(0o666) + with self.assertRaisesRegex(release.ReleaseError, "policy_permissions"): + release.trusted_policy(target) + + def test_ci_entry_rejects_bootstrap_internal_commands_and_shell_syntax(self): + for command in ("bootstrap", "bootstrap-recover", "bootstrap-status", "_deploy", "_index", + "deploy --policy /tmp/x", "status; id", "scp -t /etc", "internal-sftp"): + result = subprocess.run([sys.executable, "-I", str(ROOT / "deploy/container/release-entry.py")], + env={"SSH_ORIGINAL_COMMAND": command}, capture_output=True, timeout=2) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"restricted command", result.stderr) + + def test_detached_systemd_job_uses_runtime_enforced_service_type(self): + with patch.object(release, "run") as run: + release.schedule("deploy") + argv = run.call_args.args[0] + self.assertIn("--property=Type=exec", argv) + self.assertIn("--property=RuntimeMaxSec=300s", argv) + self.assertNotIn("--pipe", argv) + self.assertNotIn("--wait", argv) + self.assertEqual(argv[-1], "_deploy") + unit = (ROOT / "deploy/container/v8std-release-recover.service").read_text() + self.assertIn("Type=exec\n", unit) + self.assertIn("RuntimeMaxSec=300s\n", unit) + def module(self): + self.assertIsNotNone(importlib.util.find_spec("v8std_mcp_release")) + return importlib.import_module("v8std_mcp_release") + + def test_strict_unprivileged_envelope_boundary(self): + release = self.module() + self.assertEqual(release.validate_envelope(json.dumps(envelope()).encode())["sequence"], 1) + for changes in ({"schema_version": 2}, {"sequence": True}, {"image": "evil/repo"}, + {"image_digest": "latest"}, {"release_id": "x; touch /tmp/owned"}, + {"configuration_digest": "../../policy.json"}, {"env": {}}, + {"mount": "/var/run/docker.sock"}, {"trigger_sha": "refs/heads/main"}, + {"deadline": int(time.time()) - 1}): + with self.subTest(changes=changes), self.assertRaises(release.ReleaseError): + release.validate_envelope(json.dumps(envelope() | changes).encode()) + for raw in (b"{}", b"x" * 8193, json.dumps(envelope()).replace( + '"sequence": 1', '"sequence": 1, "sequence": 2').encode()): + with self.assertRaises(release.ReleaseError): + release.validate_envelope(raw) + + def test_descriptor_kind_membership_and_docker_id_variants(self): + child = {"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:" + "c" * 64}} + raw_child = release.canonical_json(child) + child_hash = "sha256:" + release.digest(raw_child) + index = {"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [ + {"mediaType": child["mediaType"], "digest": child_hash, "size": len(raw_child), + "platform": {"os": "linux", "architecture": "arm64", "variant": "v8"}}]} + raw_index = release.canonical_json(index) + request = envelope() | {"image_digest": "sha256:" + release.digest(raw_index), "platform_digest": child_hash} + mapping = release.verify_descriptors(raw_index, raw_child, request, "linux/arm64") + self.assertIn(request["image_digest"], mapping) # containerd index .Id + self.assertIn(child["config"]["digest"], mapping) # conventional config .Id + for args in ((raw_index, raw_child, request, "linux/amd64"), + (raw_child, raw_child, request | {"image_digest": child_hash}, "linux/arm64"), + (raw_index + b"x", raw_child, request, "linux/arm64")): + with self.assertRaises(release.ReleaseError): + release.verify_descriptors(*args) + + def test_attestation_binds_certificate_workflow_main_source_and_namespace(self): + command = release.attestation_command("oci://" + release.IMAGE + "@sha256:" + "a" * 64, "b" * 40) + self.assertEqual(command[command.index("--repo") + 1], "zeegin/v8std") + self.assertEqual(command[command.index("--signer-workflow") + 1], "zeegin/v8std/.github/workflows/ci.yml") + self.assertEqual(command[command.index("--source-ref") + 1], "refs/heads/main") + self.assertEqual(command[command.index("--source-digest") + 1], "b" * 40) + self.assertIn("--deny-self-hosted-runners", command) + + def test_cli_unprivileged_input_has_no_shell_side_effect(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "injected" + request = envelope() | {"release_id": "$(touch " + str(target) + ")"} + result = subprocess.run([sys.executable, "-I", str(ROOT / "scripts/v8std_mcp_release.py"), "validate-envelope"], + input=release.canonical_json(request) + b"\n", capture_output=True, timeout=5) + self.assertEqual(result.returncode, 1) + self.assertFalse(target.exists()) + + def test_actual_subprocess_and_drip_http_cancel_at_deadline(self): + started = time.monotonic() + with self.assertRaisesRegex(release.ReleaseError, "deadline"): + release.run([sys.executable, "-c", "import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); time.sleep(120)"], + started + .15) + self.assertLess(time.monotonic() - started, .6) + class Drip(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "100") + self.end_headers() + try: + for _ in range(100): + self.wfile.write(b"x") + self.wfile.flush() + time.sleep(.03) + except OSError: + pass + server = ThreadingHTTPServer(("127.0.0.1", 0), Drip) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + started = time.monotonic() + with self.assertRaisesRegex(release.ReleaseError, "deadline"): + release.http(f"http://127.0.0.1:{server.server_port}/", started + .3) + self.assertLess(time.monotonic() - started, .7) + finally: + server.shutdown() + server.server_close() + thread.join() + + +class IngressTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.archive, manifest = fixture.snapshot_fixture() + manifest["archive"]["path"] = "https://ai.v8std.ru/indexes/v1/" + manifest["archive"]["sha256"] + "/snapshot.tar.gz" + self.header = {"schema_version": 1, "publication_id": "content-1", "sequence": 1, + "trigger_sha": "b" * 40, "manifest": manifest, "deadline": int(time.time()) + 300, "action": "publish"} + + def ingest(self, header=None, archive=None): + data = release.canonical_json(header or self.header) + b"\n" + (self.archive if archive is None else archive) + reader, writer = os.pipe() + def send(): + try: + with os.fdopen(writer, "wb") as stream: + stream.write(data) + except BrokenPipeError: + pass + worker = threading.Thread(target=send) + worker.start() + try: + return release.ingest(self.root, self.root / "static", reader, seconds=1) + finally: + os.close(reader) + worker.join(2) + + def test_ingress_process_dies_between_receipt_and_inbox_then_reconciles(self): + code = ''' +import os, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import v8std_mcp_release as r +root = Path(sys.argv[2]) +original = r.write_json +def crash(path, value): + original(path, value) + if path.parent.name == 'publications': + os._exit(73) +r.write_json = crash +r.ingest(root, root / 'static', sys.stdin.fileno()) +''' + result = subprocess.run([sys.executable, "-I", "-c", code, str(ROOT / "scripts"), str(self.root)], + input=release.canonical_json(self.header) + b"\n" + self.archive, capture_output=True, timeout=5) + self.assertEqual(result.returncode, 73, result.stderr) + self.assertFalse((self.root / "pending-index.json").exists()) + self.assertEqual(release.query_status(self.root, None, { + "schema_version": 1, "kind": "publication", "id": "content-1"})["state"], "RECEIVED") + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + self.assertEqual(publisher.recover()["state"], "COMMITTED") + target = self.root / "static" / self.header["manifest"]["archive"]["sha256"] / "snapshot.tar.gz" + self.assertEqual(target.read_bytes(), self.archive) + + def test_orphan_receipt_blocks_new_upload_until_reconciled(self): + self.ingest() + (self.root / "pending-index.json").unlink() + with self.assertRaisesRegex(release.ReleaseError, "publication_busy"): + self.ingest(self.header | {"publication_id": "content-2", "sequence": 2}) + + def test_reference_rechecks_trigger_authority_before_pointer_effects(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + publisher.publish(self.header) + reference = self.header | {"publication_id": "reference-1", "action": "reference"} + self.ingest(reference, b"") + def reject(*args): + raise release.ReleaseError("main_ancestry") + publisher.verifier = reject + self.assertEqual(publisher.recover()["state"], "FAILED") + self.assertFalse((self.root / "current-index.json").exists()) + + def test_reconciliation_busy_does_not_change_durable_publication(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + before = (self.root / "publications/content-1.json").read_bytes() + with patch.object(publisher, "publish", side_effect=release.ReleaseError("busy")): + with self.assertRaisesRegex(release.ReleaseError, "busy"): + publisher.recover() + self.assertEqual((self.root / "publications/content-1.json").read_bytes(), before) + + def test_maximum_header_survives_receipt_wrapper_status_and_reference(self): + header = json.loads(json.dumps(self.header)) + header["manifest"]["test_extension"] = "" + header["manifest"]["test_extension"] = "x" * (65536 - len(release.canonical_json(header))) + self.assertEqual(len(release.canonical_json(header)), 65536) + self.assertEqual(self.ingest(header)["state"], "QUEUED") + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + self.assertEqual(publisher.recover()["state"], "COMMITTED") + query = {"schema_version": 1, "kind": "publication", "id": header["publication_id"]} + self.assertEqual(release.query_status(self.root, None, query)["state"], "COMMITTED") + reference = header | {"publication_id": "ref-1", "action": "reference"} + self.assertEqual(self.ingest(reference, b"")["state"], "QUEUED") + self.assertEqual(publisher.recover()["state"], "COMMITTED") + self.assertEqual(self.ingest(header)["state"], "COMMITTED") + + def test_bounded_pipe_ingress_verification_rename_and_reference(self): + result = self.ingest() + self.assertEqual(result["state"], "QUEUED") + calls = [] + publisher = release.Publisher(self.root, self.root / "static", lambda path, header, deadline: calls.append(path)) + self.assertEqual(publisher.publish(self.header)["state"], "COMMITTED") + self.assertEqual(len(calls), 1) # Stub verifies invocation only, not attestation success. + archive = self.root / "static" / self.header["manifest"]["archive"]["sha256"] / "snapshot.tar.gz" + self.assertEqual(archive.read_bytes(), self.archive) + self.assertFalse(list((self.root / "static").glob(".upload-*"))) + reference = self.header | {"publication_id": "reference-1", "action": "reference"} + self.ingest(reference, b"") + self.assertEqual(publisher.publish(reference)["state"], "COMMITTED") + self.assertEqual(json.loads((self.root / "current-index.json").read_text()), reference) + + def test_truncated_excess_corrupt_and_path_injection_reject_without_visibility(self): + for data in (self.archive[:-1], self.archive + b"extra", b"x" * len(self.archive)): + with self.subTest(size=len(data)), self.assertRaises(release.ReleaseError): + self.ingest(archive=data) + for changes in ({"path": "/etc/v8std-release/policy.json"}, {"publication_id": "../../policy"}, + {"env": {"PATH": "/tmp"}}, {"action": "shell"}): + with self.subTest(changes=changes), self.assertRaises(release.ReleaseError): + self.ingest(self.header | changes) + self.assertFalse(list((self.root / "static").glob("*/snapshot.tar.gz"))) + self.assertFalse((self.root / "pending-index.json").exists()) + + def test_stalled_pipe_is_cancelled_and_own_staging_cleaned(self): + reader, writer = os.pipe() + os.write(writer, release.canonical_json(self.header) + b"\n") + started = time.monotonic() + try: + with self.assertRaises(release.ReleaseError): + release.ingest(self.root, self.root / "static", reader, seconds=.1) + finally: + os.close(reader) + os.close(writer) + self.assertLess(time.monotonic() - started, .5) + self.assertFalse(list((self.root / "static").glob(".upload-*"))) + + def test_failed_verification_never_publishes_and_retry_preserves_identity(self): + self.ingest() + def reject(*args): + raise release.ReleaseError("attestation") + publisher = release.Publisher(self.root, self.root / "static", reject) + with self.assertRaisesRegex(release.ReleaseError, "attestation"): + publisher.publish(self.header) + self.assertFalse((self.root / "static" / self.header["manifest"]["archive"]["sha256"]).exists()) + with self.assertRaisesRegex(release.ReleaseError, "publication_busy|mutated_duplicate"): + self.ingest(self.header | {"trigger_sha": "c" * 40}) + publisher.verifier = lambda *args: None + self.assertEqual(publisher.publish(self.header)["state"], "COMMITTED") + + def test_committed_upload_retry_does_not_reset_receipt_or_reference(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + committed = publisher.publish(self.header) + reference = self.root / "references" / (self.header["manifest"]["archive"]["sha256"] + ".json") + before = reference.read_bytes() + self.assertEqual(self.ingest()["state"], "COMMITTED") + self.assertEqual(json.loads((self.root / "publications/content-1.json").read_text()), committed) + self.assertEqual(reference.read_bytes(), before) + self.assertFalse((self.root / "pending-index.json").exists()) + + def test_committed_crash_before_pending_unlink_reconciles(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + publisher.publish(self.header) + release.write_json(self.root / "pending-index.json", self.header) + publisher.publish(self.header) + self.assertFalse((self.root / "pending-index.json").exists()) + + def test_status_acknowledges_exact_publication_and_reference_ids(self): + self.ingest() + query = {"schema_version": 1, "kind": "publication", "id": "content-1"} + self.assertEqual(release.query_status(self.root, None, query)["state"], "RECEIVED") + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + publisher.publish(self.header) + outcome = release.query_status(self.root, None, query) + self.assertEqual((outcome["publication_id"], outcome["sequence"], outcome["state"]), ("content-1", 1, "COMMITTED")) + reference = self.header | {"publication_id": "reference-1", "action": "reference"} + query["id"] = "reference-1" + self.assertEqual(release.query_status(self.root, None, query)["state"], "NOT_FOUND") + self.ingest(reference, b"") + self.assertEqual(release.query_status(self.root, None, query)["state"], "RECEIVED") + publisher.publish(reference) + self.assertEqual(release.query_status(self.root, None, query)["action"], "reference") + self.assertEqual(release.query_status(self.root, None, query)["state"], "COMMITTED") + with self.assertRaises(release.ReleaseError): + release.query_status(self.root, None, query | {"id": "../../policy"}) + + def test_reference_cannot_revert_current_sequence_and_gc_retains_pins(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + publisher.publish(self.header) + reference = self.header | {"publication_id": "current", "action": "reference", "sequence": 3} + self.ingest(reference, b"") + publisher.publish(reference) + stale = reference | {"publication_id": "stale", "sequence": 2} + self.ingest(stale, b"") + self.assertEqual(publisher.recover()["state"], "FAILED") + self.assertEqual(json.loads((self.root / "current-index.json").read_text()), reference) + release.write_json(self.root / "pins.json", {"archives": []}) + self.assertEqual(publisher.gc(now=time.time() + 8 * 86400), []) + # An unreferenced older immutable object is removable only after seven + # days and only with verified ownership/hash, while a pin retains it. + other = b"test-only transport blob" + key = release.digest(other) + release.atomic(self.root / "static" / key / "snapshot.tar.gz", other) + release.write_json(self.root / "references" / (key + ".json"), {"last_reference": time.time()}) + self.assertEqual(publisher.gc(now=time.time() + 6 * 86400), []) + release.write_json(self.root / "pins.json", {"archives": [key]}) + self.assertEqual(publisher.gc(now=time.time() + 8 * 86400), []) + release.write_json(self.root / "pins.json", {"archives": []}) + self.assertEqual(publisher.gc(now=time.time() + 8 * 86400), [key]) + + def test_failed_queued_publication_records_failure_and_allows_new_job(self): + self.ingest() + def reject(*args): + raise release.ReleaseError("attestation") + publisher = release.Publisher(self.root, self.root / "static", reject) + result = publisher.recover() + self.assertEqual(result["state"], "FAILED") + self.assertFalse((self.root / "pending-index.json").exists()) + self.assertEqual(self.ingest(self.header | {"publication_id": "retry-2", "sequence": 2})["state"], "QUEUED") + + +def port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class TransactionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.source = Source() + self.addCleanup(self.temporary.cleanup) + self.addCleanup(self.source.close) + self.addCleanup(self.stop_processes) + ports = [port(), port()] + edge_port = port() + config = {"site_url": self.source.url, "refresh_seconds": 1, "max_snippet_chars": 4000, + "memory_bytes": 536870912, "cpus": 1} + config_hash = release.digest(release.canonical_json(config)) + self.policy = {"runtime_enabled": True, "ports": ports, "public_url": f"http://127.0.0.1:{edge_port}", "configs": {config_hash: config}} + release.write_json(self.root / "policy.json", self.policy) + self.adapter = ProcessAdapter(self.root, self.policy) + self.env = envelope() | {"configuration_digest": config_hash, + "corpus_id": self.source.manifest["corpus_id"], "archive_sha256": self.source.manifest["archive"]["sha256"]} + self.previous = {**self.env, "release_id": "predecessor", "sequence": 0, "runtime_source_sha": "c" * 40, + "name": "v8std-release-predecessor", "envelope_hash": "d" * 64, "descriptors": {}, "port": ports[0], + "hold_token": "e" * 32} + self.install_snapshot() + self.adapter.hold(self.previous, "e" * 32, time.monotonic() + 5, self.source.manifest) + release.write_json(self.root / "active.json", self.previous) + release.write_json(self.root / "envelope.json", self.env) + release.write_json(self.root / "edge.json", {"port": ports[0]}) + with (self.root / "edge.log").open("wb") as log: + self.edge = subprocess.Popen([sys.executable, "-m", "tests.mcp_release_fixture", "edge", + str(self.root), str(edge_port)], cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=log) + eventually(self.health) + + def install_snapshot(self): + key = self.source.manifest["archive"]["sha256"] + release.write_json(self.root / "manifests" / (key + ".json"), self.source.manifest) + release.atomic(self.root / "static" / key / "snapshot.tar.gz", self.source.archive) + + def stop_processes(self): + for path in (self.root / "processes").glob("*.json"): + pid = json.loads(path.read_text())["pid"] + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + for child in self.adapter.children: + child.poll() + if hasattr(self, "edge"): + self.edge.terminate() + self.edge.wait(5) + + def health(self): + try: + return json.loads(release.http(self.policy["public_url"] + "/healthz", time.monotonic() + .3)) + except release.ReleaseError: + return None + + def invoke(self, fault="", mode="deploy"): + result = subprocess.run([sys.executable, "-m", "tests.mcp_release_fixture", mode, str(self.root), fault], + cwd=ROOT, capture_output=True, timeout=25) + if fault.startswith(("crash_", "intent_")): + self.assertIn(result.returncode, {91, 92, 93}, result.stderr.decode()) + else: + self.assertEqual(result.returncode, 0, result.stderr.decode()) + return release.Controller(self.root, self.adapter).status() + + def test_success_real_mcp_health_and_independent_runtime_corpus_shas(self): + result = self.invoke() + self.assertEqual(result["state"], "COMMITTED", result) + self.assertTrue(result["cleanup_complete"]) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + self.assertEqual(self.health()["corpus_source_sha"], self.source.manifest["source_sha"]) + self.assertFalse(self.adapter.inspect(self.previous, time.monotonic() + 1)["State"]["Running"]) + + def test_reboot_after_complete_commit_restores_accepted_container(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = json.loads((self.root / "active.json").read_text()) + self.adapter.stop(active, time.monotonic() + 5) + self.assertEqual(self.invoke(mode="recover")["state"], "COMMITTED") + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + + def test_ordinary_submit_rejects_missing_predecessor_before_scheduling(self): + (self.root / "active.json").unlink() + with patch.object(release, "schedule") as schedule: + with self.assertRaisesRegex(release.ReleaseError, "predecessor_required"): + release.submit(self.root, self.adapter, release.canonical_json(self.env)) + schedule.assert_not_called() + self.assertFalse((self.root / "pending-deploy.json").exists()) + + def test_runtime_activation_is_separate_from_publication(self): + self.adapter.policy["runtime_enabled"] = False + with patch.object(release, "schedule") as schedule: + with self.assertRaisesRegex(release.ReleaseError, "runtime_not_activated"): + release.submit(self.root, self.adapter, release.canonical_json(self.env)) + schedule.assert_not_called() + + def test_public_failure_restores_observed_predecessor_and_retains_snapshot(self): + result = self.invoke("public_dead") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + self.assertTrue((self.root / "static" / self.previous["archive_sha256"] / "snapshot.tar.gz").is_file()) + + def test_pre_switch_pull_failure(self): + result = self.invoke("pull") + self.assertEqual(result["state"], "FAILED", result) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + + def test_crash_after_daemon_switch_reconciles_real_edge(self): + self.invoke("crash_after_switch") + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + result = self.invoke(mode="recover") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + + def test_committed_crash_finishes_cleanup_without_undo(self): + self.invoke("crash_COMMITTED") + result = self.invoke(mode="recover") + self.assertEqual(result["state"], "COMMITTED", result) + self.assertTrue(result["cleanup_complete"]) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + + def test_committed_candidate_death_restarts_accepted_release(self): + self.invoke("crash_COMMITTED") + journal = json.loads((self.root / "releases/release-1.json").read_text()) + self.adapter.stop(journal["candidate"], time.monotonic() + 5) + result = self.invoke(mode="recover") + self.assertEqual(result["state"], "COMMITTED", result) + self.assertTrue(result["cleanup_complete"], result) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + + def test_failed_authority_has_no_runtime_effect(self): + before = len((self.root / "calls.jsonl").read_text().splitlines()) + result = self.invoke("verify") + self.assertEqual(result["state"], "FAILED", result) + operations = [json.loads(x)["operation"] for x in (self.root / "calls.jsonl").read_text().splitlines()[before:]] + self.assertEqual(operations, ["verify"]) + + def test_readiness_cancels_actual_blocked_worker_before_loader_360(self): + started = time.monotonic() + result = self.invoke("ready") + self.assertEqual(result["state"], "FAILED", result) + self.assertLess(time.monotonic() - started, 12) + journal = json.loads((self.root / "releases/release-1.json").read_text()) + self.assertFalse(self.adapter.inspect(journal["candidate"], time.monotonic() + 1)["State"]["Running"]) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + + def test_rollback_failure_is_recovery_required_and_retry_reconciles(self): + result = self.invoke("public_dead,switch_old") + self.assertEqual(result["state"], "RECOVERY_REQUIRED", result) + self.assertFalse(result["cleanup_complete"]) + self.assertEqual(self.invoke(mode="recover")["state"], "ROLLED_BACK") + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + + def test_manifest_change_between_envelope_and_start_fails_before_switch(self): + self.source.next_generation() + result = self.invoke() + self.assertEqual(result["state"], "FAILED", result) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + + def test_mutated_duplicate_stale_retry_and_concurrent_lock(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + controller = release.Controller(self.root, self.adapter) + count = len((self.root / "calls.jsonl").read_text().splitlines()) + controller.deploy(release.canonical_json(self.env)) + self.assertEqual(count, len((self.root / "calls.jsonl").read_text().splitlines())) + for changes, code in (({"runtime_source_sha": "f" * 40}, "mutated_duplicate"), + ({"release_id": "late-job"}, "stale_sequence")): + with self.assertRaisesRegex(release.ReleaseError, code): + controller.deploy(release.canonical_json(self.env | changes)) + with release.locked(self.root), self.assertRaisesRegex(release.ReleaseError, "busy"): + controller.deploy(release.canonical_json(self.env)) + + +def crash_case(fault): + def test(self): + self.invoke(fault) + result = self.invoke(mode="recover") + self.assertIn(result["state"], {"FAILED", "ROLLED_BACK"}, result) + self.assertTrue(result["cleanup_complete"], result) + self.assertEqual(self.health()["runtime_sha"], self.previous["runtime_source_sha"]) + count = len((self.root / "calls.jsonl").read_text().splitlines()) + self.invoke(mode="recover") + self.assertEqual(count, len((self.root / "calls.jsonl").read_text().splitlines())) + return test + + +for _fault in ("crash_RECEIVED", "crash_VERIFIED", "crash_PREPARED", "crash_READY", "crash_SWITCHED", + "crash_after_start", "intent_pin_predecessor", "intent_pull_candidate", "intent_start_candidate"): + setattr(TransactionTests, "test_recovery_" + _fault, crash_case(_fault)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v8std_mcp_release_docker.py b/tests/test_v8std_mcp_release_docker.py new file mode 100644 index 0000000..6808948 --- /dev/null +++ b/tests/test_v8std_mcp_release_docker.py @@ -0,0 +1,230 @@ +"""Explicit local Task5 evidence; no pull/build/registry or host configuration. + +Run V8STD_TASK5_DOCKER=1 .venv/bin/python -m unittest +tests.test_v8std_mcp_release_docker -v. The retained Task4 image supplies locked +dependencies; only Task5 runtime modules are mounted read-only. This is not a +claim that an image containing the new code has already been published. +""" +from concurrent.futures import ThreadPoolExecutor +import http.client +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import tarfile +import tempfile +import time +import unittest +import uuid +from unittest.mock import patch + +from tests import mcp_snapshot_fixtures as fixture +from tests.test_v8std_mcp_release import port +import v8std_mcp_release as release + +ROOT = Path(__file__).resolve().parents[1] +NGINX = "sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c" +RUNTIME = "sha256:bec25fa5b9f240225db206c5e21d35a8c28e2d4ae30b1b878d272eacd9df1031" + + +@unittest.skipUnless(os.environ.get("V8STD_TASK5_DOCKER") == "1", "explicit disposable Docker evidence") +class DockerReleaseTests(unittest.TestCase): + def docker(self, *args, check=True): + self.calls.append(["docker", *map(str, args)]) + result = subprocess.run(["docker", *map(str, args)], capture_output=True, timeout=60) + if check: + details = result.stderr.decode() + if result.returncode and args[0] == "exec": + details += subprocess.run(["docker", "logs", str(args[1])], capture_output=True).stderr.decode() + self.assertEqual(result.returncode, 0, details) + return result + + def test_native_nginx_static_independence_hold_and_admission(self): + self.calls = [] + name = "v8std-task5-" + uuid.uuid4().hex[:12] + network, nginx, runtime, volume = (name + "-" + suffix for suffix in ("net", "edge", "runtime", "cache")) + endpoint_port = port() + source_url = f"http://v8std-task5.localhost:{endpoint_port}/" + archive, manifest = fixture.snapshot_fixture() + archive_hash = manifest["archive"]["sha256"] + token = "a" * 32 + evidence = {"runtime_image": RUNTIME, "nginx_image": NGINX, "source_sha_fixture": manifest["source_sha"], + "limits": {"nginx_workers": 2, "mcp_active": 8, "downloads": 2, "download_rate": "1m", + "runtime_memory": "512m", "nginx_memory": "128m", "cpus_each": 1}} + with tempfile.TemporaryDirectory(prefix="v8std-task5-docker-") as directory: + directory = Path(directory) + config, control, static, source = [directory / x for x in ("nginx", "control", "static", "source")] + for path in (config, control, static, source): + path.mkdir(mode=0o755) + for filename in ("edge-http.conf", "edge-locations.conf"): + shutil.copyfile(ROOT / "deploy/container" / filename, config / filename) + (config / "v8std-release").mkdir() + upstream = config / "v8std-release/upstream.conf" + upstream.write_text("server 127.0.0.1:9 max_conns=8;\n") + (config / "nginx.conf").write_text( + "worker_processes 2; worker_shutdown_timeout 30s; pid /tmp/nginx.pid;\n" + "error_log /dev/stderr warn; events { worker_connections 128; }\n" + "http { access_log off; client_body_temp_path /tmp/client; proxy_temp_path /tmp/proxy;\n" + "fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi;\n" + "include /etc/nginx/edge-http.conf; server {\n" + f"listen {endpoint_port}; server_name v8std-task5.localhost;\n" + "include /etc/nginx/edge-locations.conf;\n" + "location = /ai/mcp/v1/manifest.json { alias /srv/source/manifest.json; etag off; if_modified_since off; add_header Cache-Control 'max-age=0, must-revalidate'; }\n" + 'location ~ "^/ai/mcp/v1/([0-9a-f]{64})/snapshot[.]tar[.]gz$" { alias /srv/v8std-indexes/v1/$1/snapshot.tar.gz; types {} default_type application/gzip; }\n' + "} }\n") + (static / archive_hash).mkdir() + (static / archive_hash / "snapshot.tar.gz").write_bytes(archive) + (source / "manifest.json").write_bytes(release.canonical_json(manifest)) + control_file = control / "control.json" + control_file.write_text(json.dumps({"schema_version": 1, "token": token, "mode": "hold", "manifest": manifest})) + + def request(path, method="GET", body=None): + client = http.client.HTTPConnection("127.0.0.1", endpoint_port, timeout=10) + try: + client.request(method, path, body) + result = client.getresponse() + return result.status, dict(result.getheaders()), result.read() + finally: + client.close() + + def ready(): + until = time.monotonic() + 15 + observed = None + while time.monotonic() < until: + try: + status, _, body = request("/healthz") + observed = (status, body) + if status == 200 and json.loads(body).get("hold_token") == token: + return json.loads(body) + except (OSError, ValueError): + pass + time.sleep(.1) + logs = [self.docker("logs", x, check=False) for x in (runtime, nginx)] + self.fail(repr(observed) + "\n" + "\n".join((x.stdout + x.stderr).decode() for x in logs)) + + self.docker("network", "create", "--label", "pro.v8std.test=task5", network) + self.docker("volume", "create", "--label", "pro.v8std.test=task5", volume) + try: + common = ["--pull=never", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--init", "--user=10001:10001", "--cpus=1", "--pids-limit=128", + "--network", network, "--label=pro.v8std.test=task5", "--tmpfs=/tmp:rw,noexec,nosuid,size=64m"] + self.docker("run", "-d", "--name", nginx, *common, "--memory=128m", "--memory-swap=128m", + "--network-alias=v8std-task5.localhost", "-p", f"127.0.0.1:{endpoint_port}:{endpoint_port}", + "--mount", f"type=bind,source={config},target=/etc/nginx,readonly", + "--mount", f"type=bind,source={static},target=/srv/v8std-indexes/v1,readonly", + "--mount", f"type=bind,source={source},target=/srv/source,readonly", + "--entrypoint=nginx", NGINX, "-g", "daemon off;") + evidence["nginx_t"] = self.docker("exec", nginx, "nginx", "-t").stderr.decode().strip() + modules = [] + for module in ("v8std_mcp_hold.py", "v8std_mcp_runtime.py", "v8std_mcp_snapshots.py"): + modules.extend(["--mount", f"type=bind,source={ROOT / 'scripts' / module},target=/opt/v8std/scripts/{module},readonly"]) + self.docker("run", "-d", "--name", runtime, *common, "--memory=512m", "--memory-swap=512m", + "--mount", f"type=volume,source={volume},target=/var/lib/v8std-mcp", + "--mount", f"type=bind,source={control},target=/run/v8std-release,readonly", *modules, + RUNTIME, "--transport", "streamable-http", "--host", "0.0.0.0", "--port", "8000", + "--site-url", source_url, "--refresh-seconds", "1", "--allowed-host", "127.0.0.1") + upstream.write_text(f"server {runtime}:8000 max_conns=8;\n") + self.docker("exec", nginx, "nginx", "-t") + self.docker("exec", nginx, "nginx", "-s", "reload") + health = ready() + evidence["health"] = health + record = {"runtime_source_sha": health["runtime_sha"], "corpus_id": manifest["corpus_id"], + "archive_sha256": archive_hash, "hold_token": token} + release.smoke(f"http://127.0.0.1:{endpoint_port}", record, time.monotonic() + 30) + # Exercise the production switch method against a real nginx -t + # rejection. Failed validation must restore include before reload. + saved_upstream = upstream.read_bytes() + main_config = config / "nginx.conf" + saved_config = main_config.read_bytes() + main_config.write_bytes(saved_config + b"invalid_task5_directive;\n") + nginx_calls = [] + def native_nginx(argv, deadline): + nginx_calls.append(argv) + result = self.docker("exec", nginx, *argv, check=False) + if result.returncode: + raise release.ReleaseError("command_failed") + return result.stdout + adapter = release.HostAdapter(directory, {"nginx_include": str(upstream)}) + with patch.object(release, "run", native_nginx), self.assertRaises(release.ReleaseError): + adapter.switch({"port": 9}, time.monotonic() + 5) + self.assertEqual(nginx_calls, [["nginx", "-t"]]) + self.assertEqual(upstream.read_bytes(), saved_upstream) + main_config.write_bytes(saved_config) + upstream.chmod(0o644) + self.docker("exec", nginx, "nginx", "-t") + self.assertEqual(request("/healthz")[0], 200) + evidence["invalid_nginx_switch"] = "rejected; previous include restored; no reload; endpoint 200" + info = json.loads(self.docker("inspect", runtime).stdout)[0] + self.assertTrue(info["HostConfig"]["ReadonlyRootfs"]) + self.assertEqual(info["Config"]["User"], "10001:10001") + self.assertTrue(info["HostConfig"]["Init"]) + self.assertIn("ALL", info["HostConfig"]["CapDrop"]) + evidence["container_image_id"] = info["Image"] + # Read local OCI export: descriptor type/membership are explicit, + # independent of Docker's ambiguous .Id representation. + saved = directory / "image.tar" + self.docker("image", "save", "-o", saved, RUNTIME) + with tarfile.open(saved) as tar: + index_raw = tar.extractfile("blobs/sha256/" + RUNTIME[7:]).read() + index = json.loads(index_raw) + member = next(x for x in index["manifests"] if x.get("platform", {}).get("architecture") == "arm64") + child_raw = tar.extractfile("blobs/sha256/" + member["digest"][7:]).read() + evidence["descriptor_types"] = release.verify_descriptors(index_raw, child_raw, + {"image_digest": RUNTIME, "platform_digest": member["digest"]}, "linux/arm64") + self.assertIn(info["Image"], evidence["descriptor_types"]) + path = f"/indexes/v1/{archive_hash}/snapshot.tar.gz" + before = request(path) + self.assertEqual(before[0], 200) + self.assertEqual(release.digest(before[2]), archive_hash) + self.assertIn("immutable", before[1]["Cache-Control"]) + self.assertNotIn("Content-Encoding", before[1]) + self.assertEqual(request(path, "HEAD")[1]["Content-Length"], str(len(archive))) + denied = request(path, "POST", b"x") + self.assertEqual(denied[0], 403) + self.assertNotIn("Cache-Control", denied[1]) + missing = request("/indexes/v1/" + "f" * 64 + "/snapshot.tar.gz") + self.assertEqual(missing[0], 404) + self.assertNotIn("Cache-Control", missing[1]) + self.docker("stop", "--time=5", runtime) + after = request(path) + self.assertEqual((after[0], after[2]), (before[0], before[2])) + for header in ("Content-Length", "ETag", "Cache-Control"): + self.assertEqual(after[1][header], before[1][header]) + evidence["runtime_stopped_static_sha256"] = release.digest(after[2]) + # 8 same-NAT downloads, 2 globally admitted across 2 workers. + # Synthetic hash-addressed bytes exercise transport capacity only. + blob = os.urandom(3 * 1024 * 1024) + key = release.digest(blob) + (static / key).mkdir() + (static / key / "snapshot.tar.gz").write_bytes(blob) + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda _: request(f"/indexes/v1/{key}/snapshot.tar.gz"), range(8))) + codes = [value[0] for value in responses] + self.assertIn(200, codes) + self.assertIn(429, codes) + self.assertLessEqual(codes.count(200), 2) + for code, headers, _ in responses: + if code == 429: + self.assertEqual(headers["Retry-After"], "1") + self.assertNotIn("Cache-Control", headers) + evidence["download_statuses"] = codes + self.assertEqual(request("/mcp", "GET")[0], 405) + unavailable = request("/mcp", "POST", b"{}") + self.assertEqual(unavailable[0], 503) + self.assertEqual(unavailable[1]["Retry-After"], "1") + evidence["commands"] = self.calls + print("TASK5_DOCKER_EVIDENCE=" + json.dumps(evidence, sort_keys=True)) + finally: + for container in (runtime, nginx): + info = self.docker("inspect", container, check=False) + if info.returncode == 0: + self.assertEqual(json.loads(info.stdout)[0]["Config"]["Labels"]["pro.v8std.test"], "task5") + self.docker("rm", "-f", container) + self.docker("volume", "rm", volume) + self.docker("network", "rm", network) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v8std_mcp_release_hold.py b/tests/test_v8std_mcp_release_hold.py new file mode 100644 index 0000000..245c163 --- /dev/null +++ b/tests/test_v8std_mcp_release_hold.py @@ -0,0 +1,112 @@ +"""Host-only hold uses real snapshot workers and a changing HTTP source.""" +from functools import partial +import importlib +import json +from pathlib import Path +import tempfile +import time +import unittest + +from tests.test_v8std_mcp_snapshots import Source, build, blocking_ipc_build +from tests import mcp_snapshot_fixtures as fixture +from v8std_mcp_snapshots import SnapshotStore, SnapshotCoordinator + + +def eventually(check, timeout=8): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = check() + if value: + return value + time.sleep(.025) + raise AssertionError("observed condition timed out") + + +class HoldTests(unittest.TestCase): + def test_hold_selects_delivery_identity_even_when_corpus_is_equal(self): + with tempfile.TemporaryDirectory() as directory: + source = Source() + self.addCleanup(source.close) + store = SnapshotStore(source.url, Path(directory) / "cache") + first = store.refresh() + control = Path(directory) / "control.json" + command = {"schema_version": 1, "token": "a" * 32, "mode": "hold", "manifest": source.manifest} + control.write_text(json.dumps(command)) + coordinator = SnapshotCoordinator(store, build, release_control=control) + self.addCleanup(coordinator.close) + coordinator.start() + eventually(lambda: coordinator.status().get("hold_token") == "a" * 32) + archive = bytearray(source.archive) + archive[9] ^= 1 + source.archive = bytes(archive) + source.manifest = fixture.manifest_for(source.archive, first.files) + self.assertEqual(source.manifest["corpus_id"], first.metadata["corpus_id"]) + command.update(token="b" * 32, manifest=source.manifest) + control.write_text(json.dumps(command)) + eventually(lambda: coordinator.status().get("hold_token") == "b" * 32) + self.assertEqual(coordinator.status()["archive_sha256"], source.manifest["archive"]["sha256"]) + self.assertNotEqual(coordinator.status()["archive_sha256"], first.archive_sha256) + + def test_selected_hold_ignores_advancing_manifest_then_resumes(self): + with tempfile.TemporaryDirectory() as directory: + source = Source() + self.addCleanup(source.close) + store = SnapshotStore(source.url, Path(directory) / "cache") + selected = dict(source.manifest) + store.refresh() + source.next_generation() + control = Path(directory) / "control.json" + command = {"schema_version": 1, "token": "a" * 32, + "mode": "hold", "manifest": selected} + control.write_text(json.dumps(command)) + coordinator = SnapshotCoordinator(store, build, refresh_seconds=1, release_control=control) + self.addCleanup(coordinator.close) + coordinator.start() + state = eventually(lambda: coordinator.status() if coordinator.status().get("hold_token") else None) + self.assertEqual(state["corpus_id"], selected["corpus_id"]) + self.assertEqual(state["archive_sha256"], selected["archive"]["sha256"]) + requests = len(source.requests) + time.sleep(.3) + self.assertEqual(len(source.requests), requests) + command.update(token="b" * 32, mode="resume", manifest=None) + control.write_text(json.dumps(command)) + eventually(lambda: coordinator.status()["corpus_id"] == source.manifest["corpus_id"]) + self.assertIsNone(coordinator.status()["hold_token"]) + + def test_capture_after_worker_commit_failure_and_restart_selected_archive(self): + with tempfile.TemporaryDirectory() as directory: + source = Source() + self.addCleanup(source.close) + store = SnapshotStore(source.url, Path(directory) / "cache") + first = dict(source.manifest) + store.refresh() + source.next_generation() + second = dict(source.manifest) + store._attempt_seconds = 1.2 + control = Path(directory) / "control.json" + command = {"schema_version": 1, "token": "a" * 32, "mode": "resume", "manifest": None} + control.write_text(json.dumps(command)) + coordinator = SnapshotCoordinator(store, partial(blocking_ipc_build, corpus_id=second["corpus_id"]), + refresh_seconds=1, release_control=control) + self.addCleanup(coordinator.close) + coordinator.start() + eventually(lambda: coordinator.status()["ready"]) + eventually(lambda: (store.namespace / "state.json").exists() and json.loads( + (store.namespace / "state.json").read_text())["active"] == second["archive"]["sha256"]) + command.update(token="b" * 32, mode="hold") + control.write_text(json.dumps(command)) + state = eventually(lambda: coordinator.status() if coordinator.status().get("hold_token") == "b" * 32 else None) + self.assertEqual(state["corpus_id"], first["corpus_id"]) + self.assertEqual(state["archive_sha256"], first["archive"]["sha256"]) + coordinator.close() + command.update(token="c" * 32, manifest=first) + control.write_text(json.dumps(command)) + restarted = SnapshotCoordinator(store, build, release_control=control) + self.addCleanup(restarted.close) + restarted.start() + eventually(lambda: restarted.status().get("hold_token") == "c" * 32) + self.assertEqual(restarted.current().corpus_id, first["corpus_id"]) + + +if __name__ == "__main__": + unittest.main() From f1d6b934342d714feb366497f188332f8f85ebbd Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 01:53:25 +0300 Subject: [PATCH 32/88] fix(mcp): preserve release recovery and terminal outcomes --- scripts/v8std_mcp_hold.py | 20 +++- scripts/v8std_mcp_release.py | 86 +++++++++++---- tests/mcp_release_fixture.py | 24 ++++- tests/test_v8std_mcp_release.py | 138 ++++++++++++++++++++++++- tests/test_v8std_mcp_release_docker.py | 52 ++++++++-- tests/test_v8std_mcp_release_hold.py | 29 ++++++ 6 files changed, 317 insertions(+), 32 deletions(-) diff --git a/scripts/v8std_mcp_hold.py b/scripts/v8std_mcp_hold.py index 6a2e1b3..47102a6 100644 --- a/scripts/v8std_mcp_hold.py +++ b/scripts/v8std_mcp_hold.py @@ -52,9 +52,20 @@ def run(self): seen = None next_refresh = 0 failures = 0 + + def read_request(): + nonlocal seen + try: + return self.read() + except (LoaderError, SnapshotError, OSError): + # Recovery of the command file starts a fresh validation, even + # for the same token. Do not bypass build-failure backoff. + seen = None + raise + while not owner._stop.is_set(): try: - request = self.read() + request = read_request() changed = request != seen if changed: next_refresh = 0 @@ -65,7 +76,7 @@ def run(self): if manifest is not None: result, metadata = owner.store._run("refresh", owner.build, CommandStop(self, request), selected_manifest=manifest) - if self.read() != request or owner._stop.is_set(): + if read_request() != request or owner._stop.is_set(): continue owner._accept(result, metadata, checked=False) del result @@ -85,7 +96,7 @@ def run(self): last = request if initial: result, metadata = owner.store._run("cached", owner.build, CommandStop(self, request)) - if self.read() != request or owner._stop.is_set(): + if read_request() != request or owner._stop.is_set(): continue if metadata: owner._accept(result, metadata, checked=False) @@ -94,7 +105,7 @@ def run(self): if time.monotonic() >= next_refresh: result, metadata = owner.store._run("refresh", owner.build, CommandStop(self, request), current_archive=owner._archive_sha256) - if self.read() != request or owner._stop.is_set(): + if read_request() != request or owner._stop.is_set(): continue owner._accept(result, metadata, checked=True) del result @@ -103,6 +114,7 @@ def run(self): if owner.refresh_seconds else float("inf")) except (LoaderError, SnapshotError, OSError) as error: failures += 1 + last = None # Revoked acknowledgement must be earned again. with owner._lock: owner._state["refresh_error_code"] = getattr(error, "code", "configuration") # A stale acknowledgment is never proof of a new hold. diff --git a/scripts/v8std_mcp_release.py b/scripts/v8std_mcp_release.py index 78d5017..228662f 100644 --- a/scripts/v8std_mcp_release.py +++ b/scripts/v8std_mcp_release.py @@ -140,13 +140,14 @@ def ensure_directory(path, mode=0o700): require(stat.S_ISDIR(path.lstat().st_mode), "directory_shape") -def atomic(path, raw): +def atomic(path, raw, *, mode=0o600): ensure_directory(path.parent) fd, name = tempfile.mkstemp(prefix=".write-", dir=path.parent) try: with os.fdopen(fd, "wb") as stream: stream.write(raw) stream.flush() + os.fchmod(stream.fileno(), mode) os.fsync(stream.fileno()) os.replace(name, path) sync_dir(path.parent) @@ -154,11 +155,11 @@ def atomic(path, raw): Path(name).unlink(missing_ok=True) -def write_json(path, value): +def write_json(path, value, *, mode=0o600): # Operational timestamps/health contain finite floats; snapshot descriptors # and envelope hashes keep the separate float-free canonical encoding. atomic(path, json.dumps(value, ensure_ascii=False, sort_keys=True, - separators=(",", ":"), allow_nan=False).encode()) + separators=(",", ":"), allow_nan=False).encode(), mode=mode) def read_record(path): @@ -477,9 +478,12 @@ def start(self, record, deadline): def control(self, record, mode, token, manifest=None): control_directory = self.root / "slots" / record["release_id"] / "control" ensure_directory(control_directory, 0o755) + # mkdir's mode is filtered by the recovery service's UMask=0077. + # Publish final traversal/read permissions before making a command visible. + os.chmod(control_directory, 0o755) + sync_dir(control_directory) write_json(control_directory / "control.json", {"schema_version": 1, "token": token, - "mode": mode, "manifest": manifest}) - os.chmod(control_directory / "control.json", 0o644) + "mode": mode, "manifest": manifest}, mode=0o644) def hold(self, record, token, deadline, manifest=None): self.control(record, "hold", token, manifest) @@ -568,8 +572,11 @@ def status(self): @staticmethod def result(journal): + if journal["state"] == "REJECTED": + return dict(journal) # Durable queued rejection is already a public outcome. return {**journal["envelope"], "state": journal["state"], "intent": journal["intent"], - "error_code": journal.get("error_code"), "cleanup_complete": journal.get("cleanup_complete", False)} + "error_code": journal.get("error_code"), "cleanup_complete": bool( + journal.get("cleanup_complete", False) and not journal.get("active_recovery"))} def save(self, journal, state=None, intent=None): if state: @@ -580,13 +587,18 @@ def save(self, journal, state=None, intent=None): write_json(self.root / "releases" / (journal["envelope"]["release_id"] + ".json"), journal) def existing(self, envelope): + rejected_path = self.root / "rejected" / (envelope["release_id"] + ".json") + if rejected_path.exists(): + rejected = read_record(rejected_path) + require({key: rejected.get(key) for key in envelope} == envelope, "mutated_duplicate") + return rejected records = self.journals() for item in records: if item["envelope"]["release_id"] == envelope["release_id"]: require(item["envelope"] == envelope, "mutated_duplicate") return item require(not records or envelope["sequence"] > max(x["envelope"]["sequence"] for x in records), "stale_sequence") - require(not any(x["state"] not in TERMINAL or x["state"] == "RECOVERY_REQUIRED" + require(not any(x.get("active_recovery") or x["state"] not in TERMINAL or x["state"] == "RECOVERY_REQUIRED" or x["state"] == "COMMITTED" and not x.get("cleanup_complete") for x in records), "recovery_pending") return None @@ -719,14 +731,28 @@ def recover(self): deadline = time.monotonic() + TRANSACTION try: info = self.adapter.inspect(active, deadline) - if info is None or not info["State"]["Running"]: + if journal.get("active_recovery") or info is None or not info["State"]["Running"]: + # This is a new recovery transaction, not the old + # release's completed drain. Persist before start; + # Running alone never discharges switch/smoke/resume. + active = journal.get("active_recovery", {}).get("record", active) + journal["active_recovery"] = {"record": active} + self.save(journal, intent="recover_active_hold") token = digest((active["release_id"] + ":restart").encode())[:32] - active = self.adapter.hold(active, token, deadline - STOP - SMOKE, + active = self.adapter.hold(active, token, + min(deadline - STOP - SMOKE, time.monotonic() + READINESS), self.adapter.manifest(active)) - self.adapter.switch(active, deadline - SMOKE) - self.adapter.check(active, min(deadline, time.monotonic() + SMOKE), public=True) + journal["active_recovery"]["record"] = active + self.save(journal, intent="recover_active_switch") + self.adapter.switch(active, deadline - 2 * SMOKE) + self.save(journal, intent="recover_active_smoke") + self.adapter.check(active, min(deadline - SMOKE, time.monotonic() + SMOKE), public=True) + self.save(journal, intent="recover_active_pointer") write_json(active_path, active) - self.adapter.resume(active, deadline) + self.save(journal, intent="recover_active_resume") + self.adapter.resume(active, min(deadline, time.monotonic() + SMOKE)) + journal.pop("active_recovery") + journal["intent"] = "complete" journal.pop("error_code", None) self.save(journal) except Exception: @@ -820,7 +846,8 @@ def publication_result(record): return {"publication_id": header["publication_id"], "sequence": header["sequence"], "action": header["action"], "state": record["state"], "trigger_sha": header["trigger_sha"], "corpus_source_sha": manifest["source_sha"], "corpus_id": manifest["corpus_id"], - "archive_sha256": manifest["archive"]["sha256"], "error_code": record.get("error_code")} + "archive_sha256": manifest["archive"]["sha256"], "error_code": record.get("error_code"), + "cleanup_complete": record["state"] == "COMMITTED" and not record.get("cleanup_pending", False)} def restore_index_inbox(root): @@ -830,7 +857,7 @@ def restore_index_inbox(root): return directory = root / "publications" records = [read_record(path) for path in directory.glob("*.json")] - unfinished = [r for r in records if r["state"] not in {"COMMITTED", "FAILED"}] + unfinished = [r for r in records if r["state"] not in {"COMMITTED", "FAILED"} or r.get("cleanup_pending")] if unfinished: record = min(unfinished, key=lambda r: (r["header"]["sequence"], r["header"]["publication_id"])) write_json(pending, record["header"]) @@ -948,8 +975,7 @@ def publish(self, header): record = read_record(record_path) require(record["header"] == header, "mutated_duplicate") if record["state"] == "COMMITTED": - self.clear_pending(header) - return record + return self.finish_cleanup(record_path, record) verified = record["state"] in {"VERIFIED", "RECOVERY_REQUIRED"} require(record["state"] != "FAILED", "publication_terminal") deadline = time.monotonic() + (TRANSACTION if verified else min(TRANSACTION, header["deadline"] - time.time())) @@ -994,10 +1020,18 @@ def publish(self, header): write_json(self.root / "references" / (archive_hash + ".json"), {"last_reference": time.time()}) write_json(current_path, header) record["state"] = "COMMITTED" + record["cleanup_pending"] = True + record.pop("error_code", None) + write_json(record_path, record) + return self.finish_cleanup(record_path, record) + + def finish_cleanup(self, record_path, record): + self.clear_pending(record["header"]) + if record.get("cleanup_pending") or record.get("error_code"): + record["cleanup_pending"] = False record.pop("error_code", None) write_json(record_path, record) - self.clear_pending(header) - return record + return record def clear_pending(self, header): pending = self.root / "pending-index.json" @@ -1023,6 +1057,12 @@ def recover(self): with locked(self.root): record_path = self.root / "publications" / (header["publication_id"] + ".json") record = read_record(record_path) + if record["state"] == "COMMITTED": + # Visibility/reference already accepted. Even unlink+fsync + # failure must only retry cleanup, never rewrite acceptance. + record.update(cleanup_pending=True, error_code="publication_cleanup_failed") + write_json(record_path, record) + return record # Visibility may precede COMMITTED after a crash: VERIFIED receipts # are reconciled on restart, never described as a committed job. verified = record["state"] in {"VERIFIED", "RECOVERY_REQUIRED"} @@ -1116,7 +1156,8 @@ def main(): return submit(ROOT, adapter, canonical_json(envelope)) if command == "publish-index": result = ingest(ROOT, policy["static_root"], sys.stdin.fileno()) - if result["state"] not in {"COMMITTED", "FAILED"}: + if result["state"] not in {"COMMITTED", "FAILED"} or ( + result["state"] == "COMMITTED" and not result.get("cleanup_complete", True)): schedule("index") return result if command == "recover": @@ -1129,6 +1170,13 @@ def main(): envelope = parse(read_file(pending)) try: result = controller.deploy(canonical_json(envelope)) + if result["state"] == "REJECTED": + # Reconcile a crash after the durable rejection but before + # inbox deletion, without reconsidering its immutable ID. + with locked(ROOT): + if pending.exists() and parse(read_file(pending)) == envelope: + pending.unlink() + sync_dir(ROOT) except ReleaseError as error: if error.code not in {"deadline", "insufficient_transaction_budget", "runtime_not_activated", "predecessor_required", "stale_sequence"}: raise diff --git a/tests/mcp_release_fixture.py b/tests/mcp_release_fixture.py index 28458ca..e8f91e7 100644 --- a/tests/mcp_release_fixture.py +++ b/tests/mcp_release_fixture.py @@ -78,6 +78,8 @@ def start(self, record, deadline): {"pid": process.pid, "record": record}) if self.fault == "crash_after_start" and record["release_id"] != "predecessor": os._exit(93) + if self.fault == "kill_active_after_start": + os.kill(os.getpid(), signal.SIGKILL) def hold(self, record, token, deadline, manifest=None): return super().hold(record, token, min(deadline, time.monotonic() + 5), manifest) @@ -95,7 +97,15 @@ def check(self, record, deadline, *, public=False): # fail a real HTTP request before rollback can be claimed. if "public_dead" in self.fault.split(",") and public and record["release_id"] != "predecessor": self.stop(record, deadline) - return super().check(record, min(deadline, time.monotonic() + 5), public=public) + result = super().check(record, min(deadline, time.monotonic() + 5), public=public) + if self.fault == "kill_active_after_smoke" and public: + os.kill(os.getpid(), signal.SIGKILL) + return result + + def resume(self, record, deadline): + if self.fault == "kill_active_before_resume": + os.kill(os.getpid(), signal.SIGKILL) + return super().resume(record, deadline) def stop(self, record, deadline): self.record("stop", record) @@ -182,5 +192,15 @@ def dispatch(self): else: adapter = ProcessAdapter(directory, json.loads((directory / "policy.json").read_text()), args[0] if args else "") controller = CrashController(directory, adapter) - result = controller.deploy((directory / "envelope.json").read_bytes()) if mode == "deploy" else controller.recover() + if mode == "queued_expired": + # Execute the actual internal CLI worker against owned local state; + # replace only host authorization/adapters and wall clock, not logic. + from unittest.mock import patch + now = time.time() + with patch.object(release, "ROOT", directory), patch.object(release, "trusted_policy", return_value=adapter.policy), \ + patch.object(release, "HostAdapter", return_value=adapter), patch.object(os, "geteuid", return_value=0), \ + patch.object(sys, "argv", ["fixture", "_deploy"]), patch.object(time, "time", return_value=now + 130): + result = release.main() + else: + result = controller.deploy((directory / "envelope.json").read_bytes()) if mode == "deploy" else controller.recover() print(json.dumps(result)) diff --git a/tests/test_v8std_mcp_release.py b/tests/test_v8std_mcp_release.py index 2537799..1f63cb8 100644 --- a/tests/test_v8std_mcp_release.py +++ b/tests/test_v8std_mcp_release.py @@ -34,6 +34,36 @@ def envelope(**changes): class EnvelopeTests(unittest.TestCase): + def test_control_directory_is_readable_under_recovery_umask(self): + with tempfile.TemporaryDirectory() as temp: + adapter = release.HostAdapter(Path(temp), {}) + old_umask = os.umask(0o077) + try: + adapter.control({"release_id": "held"}, "hold", "a" * 32) + directory = Path(temp) / "slots/held/control" + self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + directory.chmod(0o700) # Repair a directory left by old recovery. + adapter.control({"release_id": "held"}, "hold", "a" * 32) + self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + finally: + os.umask(old_umask) + + def test_control_file_has_final_permissions_at_atomic_publication(self): + with tempfile.TemporaryDirectory() as temp: + original = os.replace + published = [] + def replace(source, target): + published.append(Path(source).stat().st_mode & 0o777) + original(source, target) + self.assertEqual(Path(target).stat().st_mode & 0o777, 0o644) + old_umask = os.umask(0o077) + try: + with patch.object(release.os, "replace", replace): + release.HostAdapter(Path(temp), {}).control({"release_id": "held"}, "hold", "a" * 32) + self.assertEqual(published, [0o644]) + finally: + os.umask(old_umask) + def test_new_journal_parents_are_fsynced_and_symlink_file_rejected(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -196,6 +226,41 @@ def do_GET(self): class IngressTests(unittest.TestCase): + def test_committed_publication_survives_inbox_cleanup_exception(self): + self.ingest() + publisher = release.Publisher(self.root, self.root / "static", lambda *args: None) + original = publisher.clear_pending + for action in ("publish", "reference"): + for after_unlink in (False, True): + with self.subTest(action=action, after_unlink=after_unlink): + header = self.header | {"publication_id": f"{action}-{int(after_unlink)}", + "action": action, "sequence": 10 + int(after_unlink)} + if action == "publish" and not after_unlink: + header = self.header + else: + self.ingest(header, self.archive if action == "publish" else b"") + def fail_cleanup(value): + if after_unlink: + original(value) + raise OSError("injected inbox cleanup error") + with patch.object(publisher, "clear_pending", fail_cleanup): + result = publisher.recover() + self.assertEqual(result["state"], "COMMITTED") + self.assertEqual(result["error_code"], "publication_cleanup_failed") + query = {"schema_version": 1, "kind": "publication", "id": header["publication_id"]} + self.assertEqual(release.query_status(self.root, None, query)["state"], "COMMITTED") + archive_hash = header["manifest"]["archive"]["sha256"] + target = self.root / "static" / archive_hash / "snapshot.tar.gz" + self.assertEqual(target.read_bytes(), self.archive) + reference = (self.root / "references" / (archive_hash + ".json")).read_bytes() + if action == "reference": + self.assertEqual(release.read_record(self.root / "current-index.json"), header) + recovered = publisher.recover() + self.assertEqual(recovered["state"], "COMMITTED") + self.assertNotIn("error_code", recovered) + self.assertFalse((self.root / "pending-index.json").exists()) + self.assertEqual((self.root / "references" / (archive_hash + ".json")).read_bytes(), reference) + def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) @@ -484,7 +549,9 @@ def health(self): def invoke(self, fault="", mode="deploy"): result = subprocess.run([sys.executable, "-m", "tests.mcp_release_fixture", mode, str(self.root), fault], cwd=ROOT, capture_output=True, timeout=25) - if fault.startswith(("crash_", "intent_")): + if fault.startswith("kill_active_"): + self.assertEqual(result.returncode, -signal.SIGKILL, result.stderr.decode()) + elif fault.startswith(("crash_", "intent_")): self.assertIn(result.returncode, {91, 92, 93}, result.stderr.decode()) else: self.assertEqual(result.returncode, 0, result.stderr.decode()) @@ -505,6 +572,44 @@ def test_reboot_after_complete_commit_restores_accepted_container(self): self.assertEqual(self.invoke(mode="recover")["state"], "COMMITTED") self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + def test_accepted_recovery_readiness_is_capped_at_90_seconds(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = release.read_record(self.root / "active.json") + self.adapter.stop(active, time.monotonic() + 5) + allowances = [] + def blocked_hold(record, token, deadline, manifest=None): + allowances.append(deadline - time.monotonic()) + raise release.ReleaseError("deadline") + with patch.object(self.adapter, "hold", blocked_hold): + result = release.Controller(self.root, self.adapter).recover() + self.assertEqual(result["error_code"], "active_recovery_failed") + self.assertEqual(len(allowances), 1) + self.assertLessEqual(allowances[0], release.READINESS) + self.assertFalse(result["cleanup_complete"]) + + def test_rejected_queued_release_id_is_immutable_and_exactly_idempotent(self): + with patch.object(release, "schedule"): + self.assertEqual(release.submit(self.root, self.adapter, release.canonical_json(self.env))["state"], "QUEUED") + self.invoke(mode="queued_expired") # Actual worker, clock advanced past work allowance. + query = {"schema_version": 1, "kind": "release", "id": self.env["release_id"]} + rejected = release.query_status(self.root, self.adapter, query) + self.assertEqual(rejected["state"], "REJECTED") + self.assertEqual(rejected["error_code"], "insufficient_transaction_budget") + self.assertFalse((self.root / "pending-deploy.json").exists()) + with patch.object(release, "schedule") as schedule: + self.assertEqual(release.submit(self.root, self.adapter, release.canonical_json(self.env)), rejected) + self.assertEqual(release.Controller(self.root, self.adapter).deploy(release.canonical_json(self.env)), rejected) + for change in ({"deadline": self.env["deadline"] + 1}, {"runtime_source_sha": "f" * 40}): + with self.assertRaisesRegex(release.ReleaseError, "mutated_duplicate"): + release.submit(self.root, self.adapter, release.canonical_json(self.env | change)) + schedule.assert_not_called() + self.assertFalse((self.root / "pending-deploy.json").exists()) + # Crash after writing REJECTED but before deleting the durable inbox. + release.write_json(self.root / "pending-deploy.json", self.env) + self.invoke(mode="queued_expired") + self.assertFalse((self.root / "pending-deploy.json").exists()) + self.assertEqual(release.query_status(self.root, self.adapter, query), rejected) + def test_ordinary_submit_rejects_missing_predecessor_before_scheduling(self): (self.root / "active.json").unlink() with patch.object(release, "schedule") as schedule: @@ -610,6 +715,37 @@ def test(self): return test +def active_recovery_crash_case(fault): + def test(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = release.read_record(self.root / "active.json") + self.adapter.stop(active, time.monotonic() + 5) + # A restarted process is not proof the public upstream/smoke/resume ran. + release.write_json(self.root / "edge.json", {"port": self.previous["port"]}) + self.invoke(fault, mode="recover") + process = self.adapter.inspect(active, time.monotonic() + 2) + self.assertTrue(process["State"]["Running"]) + self.assertFalse(release.Controller(self.root, self.adapter).status()["cleanup_complete"]) + with self.assertRaisesRegex(release.ReleaseError, "recovery_pending"): + release.Controller(self.root, self.adapter).existing(self.env | {"release_id": "next", "sequence": 2}) + result = self.invoke(mode="recover") + self.assertEqual(result["state"], "COMMITTED") + self.assertTrue(result["cleanup_complete"]) + self.assertIsNone(result["error_code"]) + state = self.health() + self.assertEqual(state["runtime_sha"], active["runtime_source_sha"]) + self.assertEqual(state["archive_sha256"], active["archive_sha256"]) + self.assertIsNone(state["hold_token"]) + self.source.next_generation() + eventually(lambda: (self.health() or {}).get("corpus_id") == self.source.manifest["corpus_id"]) + self.assertFalse(self.adapter.inspect(self.previous, time.monotonic() + 1)["State"]["Running"]) + return test + + +for _fault in ("kill_active_after_start", "kill_active_after_smoke", "kill_active_before_resume"): + setattr(TransactionTests, "test_" + _fault, active_recovery_crash_case(_fault)) + + for _fault in ("crash_RECEIVED", "crash_VERIFIED", "crash_PREPARED", "crash_READY", "crash_SWITCHED", "crash_after_start", "intent_pin_predecessor", "intent_pull_candidate", "intent_start_candidate"): setattr(TransactionTests, "test_recovery_" + _fault, crash_case(_fault)) diff --git a/tests/test_v8std_mcp_release_docker.py b/tests/test_v8std_mcp_release_docker.py index 6808948..6c43f96 100644 --- a/tests/test_v8std_mcp_release_docker.py +++ b/tests/test_v8std_mcp_release_docker.py @@ -45,6 +45,7 @@ def test_native_nginx_static_independence_hold_and_admission(self): self.calls = [] name = "v8std-task5-" + uuid.uuid4().hex[:12] network, nginx, runtime, volume = (name + "-" + suffix for suffix in ("net", "edge", "runtime", "cache")) + control_volume, writer = name + "-control", name + "-writer" endpoint_port = port() source_url = f"http://v8std-task5.localhost:{endpoint_port}/" archive, manifest = fixture.snapshot_fixture() @@ -55,8 +56,8 @@ def test_native_nginx_static_independence_hold_and_admission(self): "runtime_memory": "512m", "nginx_memory": "128m", "cpus_each": 1}} with tempfile.TemporaryDirectory(prefix="v8std-task5-docker-") as directory: directory = Path(directory) - config, control, static, source = [directory / x for x in ("nginx", "control", "static", "source")] - for path in (config, control, static, source): + config, static, source = [directory / x for x in ("nginx", "static", "source")] + for path in (config, static, source): path.mkdir(mode=0o755) for filename in ("edge-http.conf", "edge-locations.conf"): shutil.copyfile(ROOT / "deploy/container" / filename, config / filename) @@ -77,8 +78,28 @@ def test_native_nginx_static_independence_hold_and_admission(self): (static / archive_hash).mkdir() (static / archive_hash / "snapshot.tar.gz").write_bytes(archive) (source / "manifest.json").write_bytes(release.canonical_json(manifest)) - control_file = control / "control.json" - control_file.write_text(json.dumps({"schema_version": 1, "token": token, "mode": "hold", "manifest": manifest})) + + def publish_control(*, unreadable=False): + # Native Linux root writer with the recovery service's umask. + # Only this disposable named volume is writable; no Docker socket. + code = ("import os,sys,json; from pathlib import Path; " + "sys.path.insert(0,'/opt/v8std/scripts'); import v8std_mcp_release as r; " + "os.umask(0o077); p=Path('/state/slots/runtime/control'); ") + if unreadable: + code += "os.chmod(p/'control.json',0); print('{}')" + else: + code += (f"r.HostAdapter(Path('/state'),{{}}).control({{'release_id':'runtime'}},'hold',{token!r},{manifest!r}); " + "print(json.dumps({'directory_mode':oct(p.stat().st_mode & 0o777)," + "'file_mode':oct((p/'control.json').stat().st_mode & 0o777)," + "'directory_uid':p.stat().st_uid,'file_uid':(p/'control.json').stat().st_uid}))") + result = self.docker("run", "--rm", "--name", writer, "--pull=never", "--network=none", + "--user=0:0", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--init", + "--memory=128m", "--memory-swap=128m", "--cpus=1", "--pids-limit=128", + "--label=pro.v8std.test=task5", "--tmpfs=/tmp:rw,noexec,nosuid,size=64m", + "--mount", f"type=volume,source={control_volume},target=/state", + "--mount", f"type=bind,source={ROOT / 'scripts/v8std_mcp_release.py'},target=/opt/v8std/scripts/v8std_mcp_release.py,readonly", + "--entrypoint=python", RUNTIME, "-I", "-c", code) + return json.loads(result.stdout) def request(path, method="GET", body=None): client = http.client.HTTPConnection("127.0.0.1", endpoint_port, timeout=10) @@ -106,7 +127,12 @@ def ready(): self.docker("network", "create", "--label", "pro.v8std.test=task5", network) self.docker("volume", "create", "--label", "pro.v8std.test=task5", volume) + self.docker("volume", "create", "--label", "pro.v8std.test=task5", control_volume) try: + permissions = publish_control() + self.assertEqual(permissions, {"directory_mode": "0o755", "file_mode": "0o644", + "directory_uid": 0, "file_uid": 0}) + evidence["root_control_umask_0077"] = permissions common = ["--pull=never", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--init", "--user=10001:10001", "--cpus=1", "--pids-limit=128", "--network", network, "--label=pro.v8std.test=task5", "--tmpfs=/tmp:rw,noexec,nosuid,size=64m"] @@ -122,7 +148,7 @@ def ready(): modules.extend(["--mount", f"type=bind,source={ROOT / 'scripts' / module},target=/opt/v8std/scripts/{module},readonly"]) self.docker("run", "-d", "--name", runtime, *common, "--memory=512m", "--memory-swap=512m", "--mount", f"type=volume,source={volume},target=/var/lib/v8std-mcp", - "--mount", f"type=bind,source={control},target=/run/v8std-release,readonly", *modules, + "--mount", f"type=volume,source={control_volume},target=/run/v8std-release,volume-subpath=slots/runtime/control,readonly", *modules, RUNTIME, "--transport", "streamable-http", "--host", "0.0.0.0", "--port", "8000", "--site-url", source_url, "--refresh-seconds", "1", "--allowed-host", "127.0.0.1") upstream.write_text(f"server {runtime}:8000 max_conns=8;\n") @@ -130,6 +156,19 @@ def ready(): self.docker("exec", nginx, "nginx", "-s", "reload") health = ready() evidence["health"] = health + publish_control(unreadable=True) + until = time.monotonic() + 5 + while True: + state = json.loads(request("/healthz")[2]) + if state.get("hold_token") is None: + break + self.assertLess(time.monotonic(), until) + time.sleep(.05) + self.assertTrue(state["ready"]) + self.assertEqual(state["archive_sha256"], archive_hash) + self.assertEqual(publish_control(), permissions) # Same exact token/manifest. + self.assertEqual(ready()["release_control_token"], token) + evidence["same_token_read_failure_recovery"] = "ready; ack revoked; same command reacknowledged" record = {"runtime_source_sha": health["runtime_sha"], "corpus_id": manifest["corpus_id"], "archive_sha256": archive_hash, "hold_token": token} release.smoke(f"http://127.0.0.1:{endpoint_port}", record, time.monotonic() + 30) @@ -217,12 +256,13 @@ def native_nginx(argv, deadline): evidence["commands"] = self.calls print("TASK5_DOCKER_EVIDENCE=" + json.dumps(evidence, sort_keys=True)) finally: - for container in (runtime, nginx): + for container in (runtime, nginx, writer): info = self.docker("inspect", container, check=False) if info.returncode == 0: self.assertEqual(json.loads(info.stdout)[0]["Config"]["Labels"]["pro.v8std.test"], "task5") self.docker("rm", "-f", container) self.docker("volume", "rm", volume) + self.docker("volume", "rm", control_volume) self.docker("network", "rm", network) diff --git a/tests/test_v8std_mcp_release_hold.py b/tests/test_v8std_mcp_release_hold.py index 245c163..c247550 100644 --- a/tests/test_v8std_mcp_release_hold.py +++ b/tests/test_v8std_mcp_release_hold.py @@ -23,6 +23,35 @@ def eventually(check, timeout=8): class HoldTests(unittest.TestCase): + def test_same_hold_reacknowledges_after_transient_control_read_failure(self): + for selected in (False, True): + with self.subTest(selected=selected), tempfile.TemporaryDirectory() as directory: + source = Source() + store = SnapshotStore(source.url, Path(directory) / "cache") + control = Path(directory) / "control.json" + command = {"schema_version": 1, "token": "a" * 32, "mode": "hold", "manifest": source.manifest} + control.write_text(json.dumps(command)) + coordinator = SnapshotCoordinator(store, build, release_control=control) + coordinator._delay = lambda failures: .15 # Production backoff is not changed. + coordinator.start() + try: + eventually(lambda: coordinator.status()["hold_token"] == command["token"]) + if not selected: + command.update(token="b" * 32, manifest=None) + control.write_text(json.dumps(command)) + eventually(lambda: coordinator.status()["hold_token"] == command["token"]) + old = coordinator.current() + control.unlink() # Actual transient unreadable command, not a forged ack. + eventually(lambda: coordinator.status()["hold_token"] is None) + self.assertTrue(coordinator.status()["ready"]) + control.write_text(json.dumps(command)) + eventually(lambda: coordinator.status()["hold_token"] == command["token"], timeout=2) + self.assertEqual(coordinator.status()["release_control_token"], command["token"]) + self.assertEqual(coordinator.current().corpus_id, old.corpus_id) + finally: + coordinator.close() + source.close() + def test_hold_selects_delivery_identity_even_when_corpus_is_equal(self): with tempfile.TemporaryDirectory() as directory: source = Source() From 1e5a0ac7b2b5941a2a3fa4e6ced2347fe4299b3b Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 01:58:52 +0300 Subject: [PATCH 33/88] docs: record reviewed MCP release controller evidence --- spec/operations/mcp-container-verification.md | 62 +++++++++++++++++++ ...6-09-10-mcp-container-distribution-plan.md | 23 +++---- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 9ebefdd..ab24741 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -492,3 +492,65 @@ unfinished slice, now explicit in the plan. No successful release is inferred. See [the remaining-work roadmap](mcp-first-container-release-roadmap.md) for local gates, external source/image publication, initial window, rollback reserve and conditional activation of subsequent automated runtime updates. + +### Ordinary release controller: reviewed local evidence, 2026-09-15 + +The user resumed the saved plan. Signed commits `1d04817a94eb087aacff2c46e43a14a93386946a` +and `5ae7588f5417deddc7af1ffc9f273d0b1dcba5af` implement the ordinary controller, +private generation hold and bounded independent index publication. A separate +review found five defects in control permissions, same-token acknowledgement, +interrupted accepted recovery, publication finalization and rejected-ID identity. +All five have RED/GREEN regressions and passed scoped re-review; no new blocking +finding remained. First-bootstrap is not yet implemented at this checkpoint. + +Final commands/results: + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_release tests.test_v8std_mcp_release_hold tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_runtime -v +V8STD_TASK5_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_release_docker -v +``` + +148 focused tests/154.042s and one Docker/nginx integration test/9.992s passed. +The latter uses actual root-owned control under umask0077, read-only access by +UID10001 runtime, revocation/reacknowledgement, nginx syntax/invalid-switch, +static GET/HEAD/hash/cache headers while runtime is stopped, and download admission. +Eight same-NAT transfers through two nginx workers produced two200 and six429 +with retry guidance. This small fixture is not a full mixed-load capacity proof. + +The Docker test reuses local runtime index +`sha256:bec25fa5b9f240225db206c5e21d35a8c28e2d4ae30b1b878d272eacd9df1031` +with three current modules overlaid read-only, and pinned nginx +`sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c`. +It does not claim a newly built or published release. Native local Linuxarm64 +host has14CPUs/8318976000bytesRAM; runtime limit512MiB and nginx128MiB,1CPU each. +The temporary root writer has networknone/read-only/cap-drop/no-new-privileges, +no socket and access only to its test-owned volume. All task-owned diagnostic +processes/containers/volumes/networks were removed; existing images and unrelated +containers were preserved. + +Publication status now acknowledges the exact publication/reference ID, sequence, +action and identities. Archive HTTP200 alone is insufficient. Static publication +and runtime activation have separate host flags; disabling runtime updates does +not cancel owed recovery. The activation runbook records exact bounded CLI forms. +Normal recovery retains300s total/90s readiness/30s smoke/45s stop; the loader's +360s/20s budgets are unchanged. Native SSH/sudo/systemd installation, positive +published-artifact provenance, external TLS/index delivery and target capacity +still require their separately authorized acceptance. + +The existing Starlette warning was traced without suppression: unchanged +`tests/test_v8std_mcp_snippet.py:23` imports `starlette.testclient`, whose +`httpx2` import falls back to `httpx` and warns. Runtime lock pins httpx0.28.1 +and starlette1.3.1; those files did not change in this controller slice. This +is test-dependency debt for final integration, not pristine-output evidence. + +Fresh read-only production preflight: legacy health and actual `/mcp` initialize +succeed; public snapshot manifest still404. Target-host prerequisites and +capacity remain unverified. No server operation or backup occurred. Local +actual legacy code from `b7bef11` proved that stale cache tries HTTP, whereas +restoring the same four verified cache files with fresh mtimes permits original- +configuration startup/search/three-resource reads without network. This informs +bootstrap tests; it is not an installed backup/restart or indefinite hold proof. + +Ordinary architecture validation/impact and whitespace checks passed. No whole +repository suite, strict build, merge-ready, main merge/push, registry publish, +Catalog submission, server setup or production migration is claimed here. diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 6f9976b..698b73d 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -417,14 +417,15 @@ state/store roots. Release controller effects go through a narrow adapter to Docker/nginx/systemd; tests use disposable processes/filesystems and record exact calls at this external boundary, not pretend mocked return values prove health. -**Resume boundary:** Task5 has uncommitted implementation from `3165cc7` and -is paused for the current planning turn. Resume the same owner; preserve the -existing files. The last29 release tests passed, but three failures in the -89-test snapshot/runtime run were only claimed repaired and need a fresh run. -Hold regressions, ingress/security review, Docker/nginx evidence, activation -runbook and independent task review remain. This is not a completed task. - -- [ ] **RED:** Test unknown schema, invalid digest/namespace/config path, stale or +**Execution boundary:** The ordinary slice is implemented in signed `1d04817` +and `5ae7588` and independently reviewed. Final148 release/hold/snapshot/runtime +tests and the real Docker/nginx check passed; five review findings were repaired +with RED/GREEN and approved in scoped re-review. This supersedes the earlier +paused29-test evidence. The first-migration slice below is still pending, so +expanded Task5 and the complete plan are not yet complete. Native host setup, +published-artifact evidence and production capacity remain external gates. + +- [x] **RED:** Test unknown schema, invalid digest/namespace/config path, stale or mutated duplicate ID, concurrent releases, failed pull/ready/switch/smoke, rollback failure and restart reconciliation. Example visible invariant: @@ -434,7 +435,7 @@ self.assertEqual(edge.serving_digest(), predecessor_digest) self.assertTrue(predecessor_snapshot_path.is_file()) ``` -- [ ] **GREEN controller:** Verify envelope/attestation/trusted config before +- [x] **GREEN controller:** Verify envelope/attestation/trusted config before effects, durable journal+lock, exact image digest, capacity preflight and predecessor pins. Prepare candidate side port, real readiness and MCP smoke, nginx test then atomic switch/reload, public smoke then commit/drain. Enforce @@ -450,13 +451,13 @@ self.assertTrue(predecessor_snapshot_path.is_file()) archive nor a disk pointer alone proves the endpoint serves the required corpus ID. Preserve ordinary local stdio/HTTP behavior and the single public SITE_URL setting; release control must not be exposed as an MCP tool. -- [ ] **GREEN static store/operations:** Independent read-only nginx alias for +- [x] **GREEN static store/operations:** Independent read-only nginx alias for `/indexes/v1/` with GET/HEAD, hash cache headers and bounded download admission; publisher stages/verifies/renames objects and tracks references/pins before GC. Activation runbook names exact backup/cleanup/TLS/default-vhost, secrets, protection, native capacity and initial Python rollback prerequisites. No live host changes occur from tests or writing the runbook. -- [ ] **Verify:** Fault matrix for each state transition with real subprocess +- [x] **Verify:** Fault matrix for each state transition with real subprocess cancellation/crash recovery where possible; nginx syntax and static download while runtime stopped in disposable containers; unprivileged controller input rejects shell injection. Record capacity test settings and results without From 62f7d3cff521a33c9e3f5bb408c12fe0dd8ef859 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 02:16:58 +0300 Subject: [PATCH 34/88] docs: retain monitoring compatibility in MCP integration gates --- ...026-09-10-mcp-container-distribution-plan.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 698b73d..80e0cb3 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -540,7 +540,10 @@ review; do not mark the live migration complete from these fixtures: `spec/README.md`, repo skill references, architecture loader/policy tests, `spec/operations/mcp-container-verification.md`, public installation docs and Catalog release metadata/harness under `deploy/docker-catalog/`, -`scripts/check_mcp_container.py`, `tests/test_v8std_mcp_distribution.py`. +`scripts/check_mcp_container.py`, `tests/test_v8std_mcp_distribution.py`; +existing monitoring integration in `scripts/v8std_mcp_monitoring.py`, +`scripts/v8std_mcp_usage.logrotate`, `tests/test_v8std_mcp_monitoring.py`, +release launch/host templates and their focused integration tests. **Consumes:** producer, image harness and typed release controller CLIs. Publisher first places and externally verifies immutable corpus, then emits @@ -571,6 +574,18 @@ Pages manifest. Every runtime deployment references published exact digest. image-only scope without claiming Docker Catalog acceptance. Preserve the `longLived` source declaration and distinguish local test-catalog diagnostics from the actual Docker-published catalog; no upstream PR is a release prerequisite. +- [ ] **VERIFY preserved monitoring:** First demonstrate the migration regression: + the existing timer reads the legacy unit and flat usage log, while the new + container emits no usage file. Preserve the existing public projection and + historical log readers, feed it actual accepted-runtime state and new tool + events, and verify rollback/restart/rotation without stale status or lost + history. Keep logs private, the existing unprivileged monitor and non-root + runtime; grant neither Docker-group membership nor a Docker socket to them. + Exercise the real launcher/logger/aggregator path, including failed or missing + status, rather than accepting a running controller as a healthy MCP runtime. + This implements the approved preservation of monitoring, not the separately + deferred dashboard/events-schema/OpenMetrics redesign. Any necessary public + schema or trust-boundary change must return to design before implementation. - [ ] **Final gates:** Run semantic impact on actual paths, CLI `impact`, `validate --merge-ready`, all applicable fitness; strict build, then full suite; container smoke and shared-host mixed load on disposable local stack, review From 0209fa712e8e421de386575e74da29e8ef631f4a Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 02:52:10 +0300 Subject: [PATCH 35/88] docs: make retained parser release regression explicit --- .../2026-09-10-mcp-container-distribution-plan.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 80e0cb3..44618f3 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -543,7 +543,9 @@ Catalog release metadata/harness under `deploy/docker-catalog/`, `scripts/check_mcp_container.py`, `tests/test_v8std_mcp_distribution.py`; existing monitoring integration in `scripts/v8std_mcp_monitoring.py`, `scripts/v8std_mcp_usage.logrotate`, `tests/test_v8std_mcp_monitoring.py`, -release launch/host templates and their focused integration tests. +release launch/host templates and their focused integration tests; +the retained final parser regression in `scripts/v8std_mcp_presentation.py` +and `tests/test_v8std_mcp_presentation.py`. **Consumes:** producer, image harness and typed release controller CLIs. Publisher first places and externally verifies immutable corpus, then emits @@ -586,6 +588,12 @@ Pages manifest. Every runtime deployment references published exact digest. This implements the approved preservation of monitoring, not the separately deferred dashboard/events-schema/OpenMetrics redesign. Any necessary public schema or trust-boundary change must return to design before implementation. +- [ ] **VERIFY retained parser finding:** Add RED/GREEN for `![](...)` + followed by a visible internal link. HTML-looking image-alt text must not + suppress rebasing or unknown-target validation of subsequent visible links. + Preserve actual code/literal content, source offsets and canonical hashes. + This is the concrete deferred Task3 review finding, not a new Markdown + interpretation contract or permission to waive a pre-existing release defect. - [ ] **Final gates:** Run semantic impact on actual paths, CLI `impact`, `validate --merge-ready`, all applicable fitness; strict build, then full suite; container smoke and shared-host mixed load on disposable local stack, review From cf94db4c21d3ccf00542b2ff525b57dfcfa3491d Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 02:56:22 +0300 Subject: [PATCH 36/88] feat(mcp): implement guarded first-migration bootstrap --- deploy/container/legacy-release-guard.conf | 6 + .../container/v8std-bootstrap-recover.service | 16 + .../container/v8std-bootstrap-recover.timer | 10 + scripts/v8std_mcp_release.py | 508 +++++++++++++++++- spec/operations/mcp-container-activation.md | 129 ++++- tests/mcp_release_fixture.py | 271 +++++++++- tests/test_v8std_mcp_release.py | 452 +++++++++++++++- tests/test_v8std_mcp_release_docker.py | 46 +- 8 files changed, 1414 insertions(+), 24 deletions(-) create mode 100644 deploy/container/legacy-release-guard.conf create mode 100644 deploy/container/v8std-bootstrap-recover.service create mode 100644 deploy/container/v8std-bootstrap-recover.timer diff --git a/deploy/container/legacy-release-guard.conf b/deploy/container/legacy-release-guard.conf new file mode 100644 index 0000000..940c371 --- /dev/null +++ b/deploy/container/legacy-release-guard.conf @@ -0,0 +1,6 @@ +[Unit] +Requires=v8std-bootstrap-recover.timer +After=v8std-bootstrap-recover.timer + +[Service] +ExecCondition=+/usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py _legacy-allowed diff --git a/deploy/container/v8std-bootstrap-recover.service b/deploy/container/v8std-bootstrap-recover.service new file mode 100644 index 0000000..98c2021 --- /dev/null +++ b/deploy/container/v8std-bootstrap-recover.service @@ -0,0 +1,16 @@ +[Unit] +Description=Recover operator v8std first migration independently of SSH +After=docker.service nginx.service network-online.target +Wants=network-online.target + +[Service] +Type=exec +ExecStart=/usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py bootstrap-recover +RuntimeMaxSec=300s +TimeoutStopSec=5s +KillMode=control-group +UMask=0077 +LimitNOFILE=4096 +PrivateTmp=yes +NoNewPrivileges=yes +ProtectHome=yes diff --git a/deploy/container/v8std-bootstrap-recover.timer b/deploy/container/v8std-bootstrap-recover.timer new file mode 100644 index 0000000..2995cb0 --- /dev/null +++ b/deploy/container/v8std-bootstrap-recover.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Independent first-migration recovery guard + +[Timer] +OnBootSec=5s +OnUnitInactiveSec=15s +Unit=v8std-bootstrap-recover.service + +[Install] +WantedBy=timers.target diff --git a/scripts/v8std_mcp_release.py b/scripts/v8std_mcp_release.py index 228662f..8a799a0 100644 --- a/scripts/v8std_mcp_release.py +++ b/scripts/v8std_mcp_release.py @@ -41,6 +41,24 @@ ROOT = Path("/var/lib/v8std-release") POLICY = Path("/etc/v8std-release/policy.json") INSTALL = Path("/opt/v8std-release/scripts/v8std_mcp_release.py") +LEGACY_UNIT = "v8std-mcp.service" +LEGACY_APP = Path("/opt/v8std-mcp") +LEGACY_DATA = Path("/var/lib/v8std-mcp") +LEGACY_CONFIG = Path("/etc/systemd/system/v8std-mcp.service") +LEGACY_PYTHON = Path("/usr/bin/python3.12") +LEGACY_CACHE = {"pages.jsonl", "search-vectors.jsonl", "llms.txt", "llms-full.txt"} +RESTORE_STAGE = ".v8std-release-restore-v1" +BOOTSTRAP_WINDOW = Path("/etc/v8std-release/bootstrap.json") +LEGACY_GUARD = ("[Unit]\nRequires=v8std-bootstrap-recover.timer\nAfter=v8std-bootstrap-recover.timer\n" + "\n[Service]\nExecCondition=+/usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py _legacy-allowed\n") +BOOTSTRAP_SERVICE = ("[Unit]\nDescription=Recover operator v8std first migration independently of SSH\n" + "After=docker.service nginx.service network-online.target\nWants=network-online.target\n\n" + "[Service]\nType=exec\nExecStart=/usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py bootstrap-recover\n" + "RuntimeMaxSec=300s\nTimeoutStopSec=5s\nKillMode=control-group\nUMask=0077\nLimitNOFILE=4096\n" + "PrivateTmp=yes\nNoNewPrivileges=yes\nProtectHome=yes\n") +BOOTSTRAP_TIMER = ("[Unit]\nDescription=Independent first-migration recovery guard\n\n[Timer]\n" + "OnBootSec=5s\nOnUnitInactiveSec=15s\nUnit=v8std-bootstrap-recover.service\n\n" + "[Install]\nWantedBy=timers.target\n") TRANSACTION = 300 READINESS = 90 SMOKE = DRAIN = 30 @@ -211,12 +229,157 @@ def run(argv, deadline, *, limit=2 * 1024 * 1024): def trusted_policy(path=POLICY): # Reject symlinked/writable policy and all parents before interpreting paths. + trusted_path(path) + policy = parse(read_file(path), 65536) + return validate_policy(policy) + + +def trusted_path(path): for entry in (path, *path.parents): info = entry.lstat() require(not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022, "policy_permissions") - policy = parse(read_file(path), 65536) - return validate_policy(policy) + + +def validate_bootstrap_window(window, envelope, *, recovery=False): + require(set(window) == {"schema_version", "start_utc", "end_utc", "return_reserve_seconds", + "envelope_sha256", "mode", "legacy_unit", "legacy_source_sha", "backup_manifest_sha256", + "capacity"}, "bootstrap_fields") + require(type(window["schema_version"]) is int and window["schema_version"] == 1, "schema") + for key in ("start_utc", "end_utc", "return_reserve_seconds"): + require(type(window[key]) is int, "bootstrap_window") + duration = window["end_utc"] - window["start_utc"] + require(0 < duration <= 7200 and 1800 <= window["return_reserve_seconds"] < duration, "bootstrap_window") + require(window["envelope_sha256"] == digest(canonical_json(envelope)), "bootstrap_envelope") + require(window["legacy_unit"] == LEGACY_UNIT and window["mode"] in {"overlap", "stop-start"}, "bootstrap_target") + require(matches(SHA, window["legacy_source_sha"]) and matches(HEX, window["backup_manifest_sha256"]), "bootstrap_identity") + capacity = window["capacity"] + require(isinstance(capacity, dict) and set(capacity) == { + "disk_bytes", "available_memory_bytes", "file_descriptors", "network_evidence"}, "capacity") + require(all(type(capacity[k]) is int and capacity[k] > 0 for k in ( + "disk_bytes", "available_memory_bytes", "file_descriptors")) + and matches(HEX, capacity["network_evidence"]), "capacity") + if not recovery: + now = time.time() + require(window["start_utc"] <= now < window["end_utc"] - window["return_reserve_seconds"], "bootstrap_window") + require(envelope["deadline"] <= window["end_utc"] - window["return_reserve_seconds"], "bootstrap_window") + return window + + +def legacy_start_allowed(root): + # ExecCondition is fail-closed even with corrupt journal or missing active.json. + # An enabled legacy unit must never race accepted Docker recovery at boot. + if (Path(root) / "active.json").exists(): + return False + records = Controller(root, None).journals() + if any(item["state"] == "COMMITTED" for item in records): + return False + if not records: + return True + latest = max(records, key=lambda item: item["envelope"]["sequence"]) + return latest.get("kind") != "bootstrap" or latest.get("legacy_start_allowed") is True + + +def file_hash(path, deadline): + hashed = hashlib.sha256() + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), "backup_file") + while chunk := stream.read(65536): + remaining(deadline) + hashed.update(chunk) + return hashed.hexdigest() + + +def restore_file(source, target, entry, deadline): + """Fresh destination mtime is essential for the original legacy cache TTL. + + Stream, hash, fchown/fchmod and fsync before rename. No copy2/stale timestamps. + Usage logs are never enumerated, copied, logged or replaced here. + """ + require(file_hash(source, deadline) == entry["sha256"], "backup_hash") + ensure_directory(target.parent) + # Same-filesystem private staging survives SIGKILL without being confused + # with unlisted application code. One deterministic owned slot per target; + # retry truncates only that protected regular file, never a link or path + # received from a caller. No wildcard removal or cleanup of legacy files. + staging = target.parent / RESTORE_STAGE + ensure_directory(staging) + info = staging.lstat() + require(info.st_uid == os.geteuid() and info.st_mode & 0o777 == 0o700, "restore_staging") + name = staging / digest(str(target).encode()) + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + with os.fdopen(fd, "wb") as output, source.open("rb") as input_file: + info = os.fstat(output.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1, "restore_staging") + output.truncate(0) + while chunk := input_file.read(65536): + remaining(deadline) + output.write(chunk) + output.flush() + os.fchown(output.fileno(), entry["uid"], entry["gid"]) + os.fchmod(output.fileno(), entry["mode"]) + os.fsync(output.fileno()) + require(file_hash(Path(name), deadline) == entry["sha256"], "restore_hash") + os.replace(name, target) + sync_dir(target.parent) + sync_dir(staging) + finally: + Path(name).unlink(missing_ok=True) + + +def backup_inventory(root, window, deadline): + """Root-owned, bounded full app manifest; roots are compiled, never supplied.""" + backup = root / "legacy" + path = backup / "manifest.json" + trusted_path(path) + raw = read_file(path, 16 * 1024 * 1024) + require(digest(raw) == window["backup_manifest_sha256"], "backup_manifest") + value = parse(raw, 16 * 1024 * 1024) + require(set(value) == {"schema_version", "app", "cache", "directories", "unit", "upstream", "interpreter_sha256"} + and type(value["schema_version"]) is int and value["schema_version"] == 1, "backup_fields") + require(matches(HEX, value["interpreter_sha256"]), "backup_interpreter") + require(set(value["cache"]) == LEGACY_CACHE and isinstance(value["app"], dict) + and 1 <= len(value["app"]) <= 20000, "backup_files") + require({"scripts/v8std_mcp_server.py", "scripts/v8std_mcp_index.py", "scripts/v8std_retrieval_rules.py", + "venv/pyvenv.cfg", "venv/bin/python"} <= set(value["app"]), "backup_incomplete") + for component in ("app", "cache"): + require(set(value["directories"]) == {"app", "cache"}, "backup_directories") + directories = value["directories"][component] + expected = {str(parent) for name in value[component] for parent in Path(name).parents} + require(set(directories) == expected, "backup_directories") + for entry in directories.values(): + require(set(entry) == {"mode", "uid", "gid"} and type(entry["mode"]) is int + and 0 <= entry["mode"] <= 0o777 and all(type(entry[k]) is int and + 0 <= entry[k] <= 2**31-1 for k in ("uid", "gid")), "backup_metadata") + for relative, entry in value[component].items(): + parts = relative.split("/") + require(len(relative) <= 512 and all(matches(re.compile(r"[A-Za-z0-9_.+@-]+\Z"), p) + and p not in {".", ".."} for p in parts), "backup_path") + require(isinstance(entry, dict), "backup_entry") + if "link" in entry: + require(component == "app" and set(entry) == {"link"}, "backup_link") + link = entry["link"] + require(isinstance(link, str) and len(link) <= 512, "backup_link") + destination = Path(os.path.normpath(str(LEGACY_APP / relative / ".." / link))) + require(destination == LEGACY_PYTHON or destination.is_relative_to(LEGACY_APP), "backup_link") + # Manifest links are data; protected backup contains no symlinks. + continue + require(set(entry) == {"sha256", "mode", "uid", "gid"}, "backup_entry") + require(matches(HEX, entry["sha256"]) and type(entry["mode"]) is int + and 0 <= entry["mode"] <= 0o777 and all(type(entry[k]) is int and + 0 <= entry[k] <= 2**31-1 for k in ("uid", "gid")), "backup_metadata") + source = backup / component / relative + trusted_path(source) + require(file_hash(source, deadline) == entry["sha256"], "backup_hash") + for name in ("unit", "upstream"): + require(set(value[name]) == {"sha256", "mode", "uid", "gid"} + and matches(HEX, value[name]["sha256"]) and value[name]["uid"] == 0 + and value[name]["gid"] == 0 and value[name]["mode"] in {0o600, 0o644}, "backup_config") + trusted_path(backup / name) + require(file_hash(backup / name, deadline) == value[name]["sha256"], "backup_hash") + return value def validate_policy(policy): @@ -338,14 +501,14 @@ def http(url, deadline, *, body=None, limit=1024 * 1024): process.close() -def rpc(url, method, params, deadline, number): +def rpc(url, method, params, deadline, number, *, limit=1024 * 1024): raw = http(url + "/mcp", deadline, body=canonical_json({"jsonrpc": "2.0", "id": number, - "method": method, "params": params})) + "method": method, "params": params}), limit=limit) if raw.startswith(b"event:") or raw.startswith(b"data:"): messages = [line[6:] for line in raw.splitlines() if line.startswith(b"data: ")] require(len(messages) == 1, "rpc_stream") raw = messages[0] - reply = parse(raw, 1024 * 1024) + reply = parse(raw, limit) require(reply.get("id") == number and "error" not in reply and isinstance(reply.get("result"), dict), "rpc") result = reply["result"] require(not result.get("isError"), "rpc_tool") @@ -414,11 +577,11 @@ def verify(self, envelope, deadline): child = run(["docker", "buildx", "imagetools", "inspect", "--raw", IMAGE + "@" + envelope["platform_digest"]], deadline) return verify_descriptors(index, child, envelope, self.policy["platform"]) - def capacity(self, deadline): + def capacity(self, deadline, *, reclaim_bytes=0): limits = self.policy["capacity"] require(shutil.disk_usage(self.root).free >= limits["disk_bytes"], "disk_capacity") values = dict(re.findall(r"^(\w+):\s+(\d+)", read_file(Path("/proc/meminfo")).decode(), re.M)) - require(int(values.get("MemAvailable", 0)) * 1024 >= limits["available_memory_bytes"], "memory_capacity") + require(int(values.get("MemAvailable", 0)) * 1024 + reclaim_bytes >= limits["available_memory_bytes"], "memory_capacity") import resource require(resource.getrlimit(resource.RLIMIT_NOFILE)[0] >= limits["file_descriptors"], "fd_capacity") evidence = read_file(self.root / "capacity" / (limits["network_evidence"] + ".json")) @@ -554,6 +717,166 @@ def manifest(self, record): and manifest["archive"]["sha256"] == record["archive_sha256"], "manifest_identity") return manifest + def bootstrap_window(self, envelope): + require(BOOTSTRAP_WINDOW.exists(), "bootstrap_window_required") + trusted_path(BOOTSTRAP_WINDOW) + return validate_bootstrap_window(parse(read_file(BOOTSTRAP_WINDOW)), envelope) + + def bootstrap_backup(self, window, deadline, *, current=False): + saved = backup_inventory(self.root, window, deadline) + require(file_hash(LEGACY_PYTHON, deadline) == saved["interpreter_sha256"], "legacy_interpreter_changed") + if current: + self.legacy_files(saved, deadline, restore=False) + require(file_hash(LEGACY_CONFIG, deadline) == saved["unit"]["sha256"] + and file_hash(Path(self.policy["nginx_include"]), deadline) == saved["upstream"]["sha256"], "legacy_config_changed") + # The only installed drop-in is our separately reviewed boot fence. + unit = dict(line.split("=", 1) for line in run(["systemctl", "show", LEGACY_UNIT, + "--property=MainPID", "--property=ActiveState", "--property=DropInPaths", + "--property=EnvironmentFiles"], deadline).decode().splitlines()) + require(unit.get("ActiveState") == "active" and unit.get("MainPID", "0").isdigit() + and int(unit["MainPID"]) > 0 and not unit.get("EnvironmentFiles") + and unit.get("DropInPaths") == "/etc/systemd/system/v8std-mcp.service.d/10-release-guard.conf", "legacy_unit_changed") + return saved + + def bootstrap_capacity(self, window, envelope, deadline, *, after_stop=False): + limits = window["capacity"] + require(limits["available_memory_bytes"] >= self.config(envelope)["memory_bytes"] + 128 * 1024 * 1024, + "capacity_reserve") + reclaim = 0 + if window["mode"] == "stop-start" and not after_stop: + value = run(["systemctl", "show", LEGACY_UNIT, "--property=MemoryCurrent", "--value"], deadline).strip() + require(value.isdigit(), "legacy_memory") + reclaim = int(value) + HostAdapter(self.root, self.policy | {"capacity": limits}).capacity(deadline, reclaim_bytes=reclaim) + + def bootstrap_prepared(self, candidate, deadline): + # Artifacts must already exist. No pull/build during the migration window. + info = json.loads(run(["docker", "image", "inspect", IMAGE + "@" + candidate["platform_digest"]], deadline))[0] + require(info["Id"] in candidate["descriptors"] and info["Os"] + "/" + info["Architecture"] == self.policy["platform"] + and info["Config"].get("Labels", {}).get("org.opencontainers.image.revision") == candidate["runtime_source_sha"], + "prepared_image") + manifest = self.manifest(candidate) + verify_archive(read_file(Path(self.policy["static_root"]) / candidate["archive_sha256"] / "snapshot.tar.gz", + MAX_ARCHIVE_BYTES), manifest) + + def arm_bootstrap_guard(self, deadline): + for path, expected in ( + ("/etc/systemd/system/v8std-mcp.service.d/10-release-guard.conf", LEGACY_GUARD), + ("/etc/systemd/system/v8std-bootstrap-recover.service", BOOTSTRAP_SERVICE), + ("/etc/systemd/system/v8std-bootstrap-recover.timer", BOOTSTRAP_TIMER)): + trusted_path(Path(path)) + require(read_file(Path(path)) == expected.encode(), "bootstrap_guard_config") + require(run(["systemctl", "is-enabled", "v8std-bootstrap-recover.timer"], deadline).strip() == b"enabled", "bootstrap_guard") + require(run(["systemctl", "is-active", "v8std-bootstrap-recover.timer"], deadline).strip() == b"active", "bootstrap_guard") + for unit in (LEGACY_UNIT, "v8std-bootstrap-recover.service", "v8std-bootstrap-recover.timer"): + properties = dict(line.split("=", 1) for line in run(["systemctl", "show", unit, + "--property=NeedDaemonReload", "--property=LoadState", "--property=DropInPaths"], deadline).decode().splitlines()) + require(properties.get("NeedDaemonReload") == "no" and properties.get("LoadState") == "loaded" + and properties.get("DropInPaths") == ( + "/etc/systemd/system/v8std-mcp.service.d/10-release-guard.conf" if unit == LEGACY_UNIT else ""), + "bootstrap_guard_loaded") + + def legacy_stop(self, deadline): + run(["systemctl", "stop", LEGACY_UNIT], deadline) + require(run(["systemctl", "show", LEGACY_UNIT, "--property=MainPID", "--value"], deadline).strip() == b"0", "legacy_stop") + + def legacy_files(self, saved, deadline, *, restore): + for component, target_root in (("app", LEGACY_APP), ("cache", LEGACY_DATA)): + require(not target_root.is_symlink(), "legacy_path") + if component == "app" and target_root.exists(): + actual = set() + for directory, dirs, files in os.walk(target_root, followlinks=False): + if RESTORE_STAGE in dirs: + info = (Path(directory) / RESTORE_STAGE).lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.geteuid() + and info.st_mode & 0o777 == 0o700, "restore_staging") + dirs.remove(RESTORE_STAGE) + dirs[:] = [d for d in dirs if d != "__pycache__"] + actual.update(str((Path(directory) / name).relative_to(target_root)) for name in + files + [d for d in dirs if (Path(directory) / d).is_symlink()]) + require(actual <= set(saved["app"]), "legacy_unlisted") + for relative, metadata in sorted(saved["directories"][component].items(), key=lambda item: len(item[0])): + directory = target_root / relative + if restore: + ensure_directory(directory) + os.chown(directory, metadata["uid"], metadata["gid"]) + os.chmod(directory, metadata["mode"]) + sync_dir(directory) + else: + info = directory.lstat() + require(stat.S_ISDIR(info.st_mode) and (info.st_mode & 0o777) == metadata["mode"] + and info.st_uid == metadata["uid"] and info.st_gid == metadata["gid"], "legacy_directory_changed") + for relative, entry in sorted(saved[component].items()): + target = target_root / relative + # Reject symlink ancestors before any destination write. + for parent in target.parents: + require(not parent.is_symlink(), "legacy_path") + if parent == target_root: + break + if "link" in entry: + if restore and not target.exists() and not target.is_symlink(): + ensure_directory(target.parent) + target.symlink_to(entry["link"]) + sync_dir(target.parent) + require(target.is_symlink() and os.readlink(target) == entry["link"], "legacy_link_changed") + elif restore: + restore_file(self.root / "legacy" / component / relative, target, entry, deadline) + else: + if "__pycache__" in target.parts: + continue # Derived bytecode changes on an exact-source restart. + require(file_hash(target, deadline) == entry["sha256"], "legacy_files_changed") + + def legacy_restore(self, window, deadline): + saved = self.bootstrap_backup(window, deadline) + self.legacy_files(saved, deadline, restore=True) + restore_file(self.root / "legacy/unit", LEGACY_CONFIG, saved["unit"], deadline) + restore_file(self.root / "legacy/upstream", Path(self.policy["nginx_include"]), saved["upstream"], deadline) + run(["systemctl", "daemon-reload"], deadline) + run(["nginx", "-t"], deadline) + run(["nginx", "-s", "reload"], deadline) + + def legacy_start(self, deadline): + run(["systemctl", "start", LEGACY_UNIT], deadline) + + def legacy_identity(self, deadline): + pid = run(["systemctl", "show", LEGACY_UNIT, "--property=MainPID", "--value"], deadline).strip() + require(pid.isdigit() and int(pid) > 0, "legacy_process") + arguments = read_file(Path("/proc") / pid.decode() / "cmdline").split(b"\0")[:-1] + expected = [str(LEGACY_APP / "venv/bin/python"), str(LEGACY_APP / "scripts/v8std_mcp_server.py"), + "--index-url", "https://v8std.ru/ai/pages.jsonl", "--vectors-url", "https://v8std.ru/ai/search-vectors.jsonl", + "--cache-dir", str(LEGACY_DATA), "--host", "127.0.0.1", "--port", "8765", "--mcp-path", "/mcp", + "--max-snippet-chars", "4000", "--usage-log", str(LEGACY_DATA / "tool-usage.jsonl")] + require(arguments == [arg.encode() for arg in expected] + and Path(os.readlink(Path("/proc") / pid.decode() / "exe")) == LEGACY_PYTHON, "legacy_process") + + def legacy_check(self, window, deadline, *, public=False): + saved = self.bootstrap_backup(window, deadline) + self.legacy_files(saved, deadline, restore=False) + url = self.policy["public_url"] if public else "http://127.0.0.1:8765" + while True: + try: + self.legacy_identity(deadline) + health = parse(http(url + "/healthz", deadline)) + require(health.get("ok") is True and health.get("sha256") == saved["cache"]["pages.jsonl"]["sha256"] + and health.get("vectors", {}).get("sha256") == saved["cache"]["search-vectors.jsonl"]["sha256"], "legacy_health_identity") + rpc(url, "initialize", {"protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "v8std-release", "version": "1"}}, deadline, 1) + result = rpc(url, "tools/call", {"name": "v8std_search", "arguments": {"query": "std437", "limit": 1}}, deadline, 2) + require(bool(result.get("content") or result.get("structuredContent")), "legacy_search") + for number, (uri, name) in enumerate((("v8std://llms.txt", "llms.txt"), + ("v8std://llms-full.txt", "llms-full.txt"), ("v8std://ai/pages.jsonl", "pages.jsonl")), 3): + contents = rpc(url, "resources/read", {"uri": uri}, deadline, number, limit=32 * 1024 * 1024).get("contents", []) + require(len(contents) == 1 and isinstance(contents[0].get("text"), str) + and digest(contents[0]["text"].encode()) == saved["cache"][name]["sha256"], "legacy_resource") + after = parse(http(url + "/healthz", deadline)) + require(after.get("sha256") == health["sha256"] and after.get("vectors", {}).get("sha256") + == health["vectors"]["sha256"], "legacy_generation_changed") + return health + except ReleaseError as error: + if error.code not in {"http_failed", "legacy_process"}: + raise + time.sleep(min(.1, remaining(deadline))) + class Controller: def __init__(self, root, adapter): @@ -724,6 +1047,8 @@ def recover(self): if not records: return self.status() journal = max(records, key=lambda item: item["envelope"]["sequence"]) + if journal.get("kind") == "bootstrap": + return BootstrapController(self.root, self.adapter).reconcile(journal) if journal.get("cleanup_complete"): active_path = self.root / "active.json" if active_path.exists(): @@ -778,6 +1103,157 @@ def recover(self): return self.status() +class BootstrapController(Controller): + """Initial acceptance only. It never activates policy or invents a predecessor.""" + + def submit(self, raw): + envelope = validate_envelope(raw, expired=True) + with locked(self.root): + existing = self.existing(envelope) + if existing: + return self.result(existing) + require(not (self.root / "active.json").exists(), "bootstrap_already_accepted") + require(not any(x["state"] == "COMMITTED" for x in self.journals()), "bootstrap_already_accepted") + validate_envelope(raw) + require(envelope["deadline"] - time.time() > RECOVERY_RESERVE, "insufficient_transaction_budget") + self.adapter.config(envelope) + window = self.adapter.bootstrap_window(envelope) + validate_bootstrap_window(window, envelope) + journal = {"kind": "bootstrap", "envelope": envelope, "window": window, "state": "RECEIVED", + "intent": "queued", "candidate": None, "cleanup_complete": False, "legacy_start_allowed": True} + self.save(journal) + schedule("bootstrap") + return self.result(journal) + + def execute(self): + with locked(self.root): + records = self.journals() + require(bool(records), "bootstrap_missing") + journal = max(records, key=lambda item: item["envelope"]["sequence"]) + require(journal.get("kind") == "bootstrap", "bootstrap_missing") + if journal["state"] != "RECEIVED": + return self.result(journal) + envelope, window = journal["envelope"], journal["window"] + deadline = time.monotonic() + min(TRANSACTION, envelope["deadline"] - time.time()) + work = deadline - RECOVERY_RESERVE + try: + validate_bootstrap_window(window, envelope) + validate_envelope(canonical_json(envelope)) + remaining(work) + require(not (self.root / "active.json").exists(), "bootstrap_already_accepted") + descriptors = self.adapter.verify(envelope, work) + self.adapter.bootstrap_backup(window, work, current=True) + self.adapter.bootstrap_capacity(window, envelope, work) + token = digest(canonical_json(envelope))[:32] + candidate = {**envelope, "name": "v8std-release-" + envelope["release_id"], + "envelope_hash": digest(canonical_json(envelope)), "descriptors": descriptors, + "port": self.adapter.policy["ports"][0], "hold_token": token} + self.adapter.bootstrap_prepared(candidate, work) + self.adapter.legacy_check(window, min(work, time.monotonic() + SMOKE)) + self.adapter.arm_bootstrap_guard(work) + validate_bootstrap_window(window, envelope) + journal.update(candidate=candidate, legacy_start_allowed=False) + self.save(journal, "VERIFIED", "guard_armed") + write_json(self.root / "pins.json", {"archives": [envelope["archive_sha256"]]}) + if window["mode"] == "stop-start": + self.save(journal, intent="stop_legacy") + self.adapter.legacy_stop(min(work, time.monotonic() + STOP)) + self.adapter.bootstrap_capacity(window, envelope, work, after_stop=True) + self.save(journal, intent="start_candidate") + candidate = self.adapter.hold(candidate, token, min(work, time.monotonic() + READINESS), + self.adapter.manifest(candidate)) + journal["candidate"] = candidate + self.save(journal, "PREPARED", "candidate_smoke") + self.adapter.check(candidate, min(work, time.monotonic() + SMOKE)) + self.save(journal, "READY", "switch") + self.adapter.switch(candidate, work) + self.save(journal, "SWITCHED", "public_smoke") + self.adapter.check(candidate, min(work, time.monotonic() + SMOKE), public=True) + remaining(work) + # Durable COMMITTED is the acceptance point, never active.json. + self.save(journal, "COMMITTED", "accept_pointer") + self.finish(journal, deadline) + except Exception as error: + # A failed fsync/rename has an uncertain result: re-read the + # durable journal instead of trusting a mutated in-memory state. + journal = read_record(self.root / "releases" / (envelope["release_id"] + ".json")) + journal["error_code"] = getattr(error, "code", "host_failure") + if journal["state"] == "COMMITTED": + self.save(journal, intent="cleanup_pending") + elif journal["state"] == "RECEIVED": + journal["cleanup_complete"] = True + self.save(journal, "FAILED", "complete") + else: + self.rollback(journal, deadline) + return self.result(journal) + + def finish(self, journal, deadline): + # Also used after accepted crash/reboot. Fence before any candidate start. + journal["cleanup_complete"] = False + journal["legacy_start_allowed"] = False + self.save(journal, intent="ensure_accepted_candidate") + self.adapter.legacy_stop(min(deadline - READINESS - 2 * SMOKE, time.monotonic() + STOP)) + candidate = journal["candidate"] + candidate = self.adapter.hold(candidate, candidate["hold_token"], + min(deadline - 2 * SMOKE, time.monotonic() + READINESS), self.adapter.manifest(candidate)) + journal["candidate"] = candidate + self.save(journal, intent="accepted_switch") + self.adapter.switch(candidate, deadline - 2 * SMOKE) + self.save(journal, intent="accepted_smoke") + self.adapter.check(candidate, min(deadline - SMOKE, time.monotonic() + SMOKE), public=True) + self.save(journal, intent="accept_pointer") + write_json(self.root / "active.json", candidate) + write_json(self.root / "pins.json", {"archives": [candidate["archive_sha256"]]}) + self.save(journal, intent="resume_candidate") + self.adapter.resume(candidate, min(deadline, time.monotonic() + SMOKE)) + journal["cleanup_complete"] = True + journal.pop("error_code", None) + self.save(journal, intent="complete") + + def rollback(self, journal, deadline): + try: + journal["legacy_start_allowed"] = False + self.save(journal, intent="stop_candidate") + if journal["candidate"]: + self.adapter.stop(journal["candidate"], min(deadline - READINESS - SMOKE - 5, time.monotonic() + STOP)) + # Stop legacy too if overlap was chosen; restore coherent bytes only + # while the original process is stopped, then restart its exact unit. + self.save(journal, intent="restore_legacy") + self.adapter.legacy_stop(min(deadline - READINESS - SMOKE, time.monotonic() + STOP)) + self.adapter.legacy_restore(journal["window"], deadline - READINESS - SMOKE) + journal["legacy_start_allowed"] = True + self.save(journal, intent="start_legacy") + self.adapter.legacy_start(min(deadline - SMOKE, time.monotonic() + READINESS)) + self.save(journal, intent="legacy_smoke") + self.adapter.legacy_check(journal["window"], min(deadline, time.monotonic() + SMOKE), public=True) + journal["cleanup_complete"] = True + self.save(journal, "ROLLED_BACK", "complete") + except Exception: + journal["cleanup_complete"] = False + journal["error_code"] = "legacy_recovery_failed" + self.save(journal, "RECOVERY_REQUIRED", "restore_legacy") + + def reconcile(self, journal): + # Caller holds the same global release/publication lock. No window gate: + # restoring owed work remains required after expiry or policy disablement. + if journal["state"] == "RECEIVED": + journal["cleanup_complete"] = True + self.save(journal, "FAILED", "interrupted_preparation") + elif journal["state"] == "COMMITTED": + candidate = journal["candidate"] + deadline = time.monotonic() + TRANSACTION + info = self.adapter.inspect(candidate, deadline) + if not journal.get("cleanup_complete") or info is None or not info["State"]["Running"]: + try: + self.finish(journal, deadline) + except Exception: + journal.update(cleanup_complete=False, error_code="bootstrap_cleanup_failed") + self.save(journal, intent="cleanup_pending") + elif not journal.get("cleanup_complete"): + self.rollback(journal, time.monotonic() + TRANSACTION) + return self.result(journal) + + def validate_upload(header): require(set(header) == {"schema_version", "publication_id", "sequence", "trigger_sha", "manifest", "deadline", "action"}, "upload_fields") @@ -1104,7 +1580,7 @@ def gc(self, *, now=None): def schedule(kind): - require(kind in {"deploy", "index", "recover"}, "job_kind") + require(kind in {"deploy", "index", "recover", "bootstrap"}, "job_kind") # Shared unit name and controller lock serialize all host effects. No --pipe, # --wait or inherited SSH stdin; timer recovers a crash before enqueue. return run(["systemd-run", "--unit=v8std-release-job", "--collect", "--no-block", @@ -1139,21 +1615,33 @@ def main(): require(len(sys.argv) == 2, "command") command = sys.argv[1] require(command in {"validate-envelope", "deploy", "recover", "status", "publish-index", - "_deploy", "_index", "_recover"}, "command") + "_deploy", "_index", "_recover", "bootstrap", "bootstrap-recover", "bootstrap-status", + "_bootstrap", "_legacy-allowed"}, "command") if command == "validate-envelope": header = read_header(sys.stdin.fileno(), time.monotonic() + 20, 8192) eof(sys.stdin.fileno(), time.monotonic() + 20) return validate_envelope(canonical_json(header)) require(os.geteuid() == 0, "host_privilege") + if command == "_legacy-allowed": + require(legacy_start_allowed(ROOT), "legacy_fenced") + return {"state": "LEGACY_ALLOWED"} policy = trusted_policy() adapter = HostAdapter(ROOT, policy) controller = Controller(ROOT, adapter) - if command == "status": + if command in {"status", "bootstrap-status"}: return query_status(ROOT, adapter, read_status_query(sys.stdin.fileno())) if command == "deploy": envelope = read_header(sys.stdin.fileno(), time.monotonic() + 20, 8192) eof(sys.stdin.fileno(), time.monotonic() + 20) return submit(ROOT, adapter, canonical_json(envelope)) + if command == "bootstrap": + envelope = read_header(sys.stdin.fileno(), time.monotonic() + 20, 8192) + eof(sys.stdin.fileno(), time.monotonic() + 20) + return BootstrapController(ROOT, adapter).submit(canonical_json(envelope)) + if command == "_bootstrap": + return BootstrapController(ROOT, adapter).execute() + if command == "bootstrap-recover": + return controller.recover() if command == "publish-index": result = ingest(ROOT, policy["static_root"], sys.stdin.fileno()) if result["state"] not in {"COMMITTED", "FAILED"} or ( diff --git a/spec/operations/mcp-container-activation.md b/spec/operations/mcp-container-activation.md index 0011296..dbb3273 100644 --- a/spec/operations/mcp-container-activation.md +++ b/spec/operations/mcp-container-activation.md @@ -1,7 +1,8 @@ # Controlled initial activation of container delivery -**Status (2026-09-15):** ordinary controller implemented for scoped review; -first-bootstrap implementation and native rehearsal are still pending. +**Status (2026-09-15):** ordinary controller independently approved; +operator first-bootstrap implementation is awaiting its separate scoped review. +Native systemd/complete-backup rehearsal and live activation remain pending. Local disposable evidence is not a live installation, CI activation, published image proof, or target-host capacity guarantee. This runbook authorizes none of those operations. The current external scope remains image-only. @@ -89,8 +90,8 @@ immutable artifacts and independent recovery guard before the window. Mint the preparation. Stop new attempts at least 30 minutes before the window ends; reserve more time if the measured legacy restoration needs it. Already-started recovery remains necessary after the window expires. `bootstrap`, -`bootstrap-recover` and `bootstrap-status` are **not implemented in this slice**; -the restricted SSH entry rejects them. Do not manufacture `active.json`. +`bootstrap-recover` and `bootstrap-status` are operator-only; the restricted SSH +entry rejects them. Do not manufacture `active.json`. Exercise initialized idle agents, normal POST tool calls, reconnects, shared NAT, snapshot refresh and concurrent archive downloads. Record the actual mix, @@ -135,9 +136,121 @@ host. Full private evidence is in the Task5 handoff's Before migration, inventory and hash the entire saved code, dependency lock and installed venv/interpreter, unit/configuration and coherent data set; protect the backup off-host, restore it in disposable Linux, and measure return time. -The ordinary controller does not restore this legacy Python deployment; that -is the pending bootstrap slice. Do not stop the still-enabled legacy unit until -that slice proves preaccept restoration and postaccept reboot ownership. +The ordinary rollout never substitutes a legacy stop/start fallback. The separate +bootstrap path restores the saved legacy deployment before acceptance. Do not +stop the still-enabled legacy unit until scoped review and native rehearsal prove +preaccept restoration and postaccept reboot ownership. + +## Operator first-bootstrap interface (future authorized window only) + +These are implementation interfaces, not authorization to install or run them on +the current host. No window, host backup or target capacity has been approved by +these local tests. Keep `enabled:true,runtime_enabled:false` after separately +approved static installation; add only the reviewed configuration digest. The +operator window has its own measured capacity record. Ordinary CI runtime deploy +remains independently disabled after first success. + +The installed root-owned `/etc/v8std-release/bootstrap.json` is at most8192bytes, +has no caller paths, and has exactly these fields: + +```json +{"schema_version":1,"start_utc":0,"end_utc":0,"return_reserve_seconds":1800,"envelope_sha256":"<64 lowercase hex>","mode":"stop-start","legacy_unit":"v8std-mcp.service","legacy_source_sha":"","backup_manifest_sha256":"<64 lowercase hex>","capacity":{"disk_bytes":1,"available_memory_bytes":1,"file_descriptors":4096,"network_evidence":"<64 lowercase hex>"}} +``` + +The placeholder values above deliberately do not authorize an attempt. UTC times +are integer epoch seconds; duration must be <=7200s. Reserve >=1800s at the end, +increased before the window if measured restoration needs more. `overlap` is +preferred when measured to fit; `stop-start` is the first-migration-only exception. +Memory threshold is at least candidate limit+128MiB, with real disk/FD/evidence +checks. Stop/start precheck may count the legacy cgroup's MemoryCurrent as +reclaimable; after stop it rechecks actual MemAvailable without that allowance. +An inability to fit even one candidate restores legacy; it never changes limits. + +Before the window, verify/pull the exact published image and publish/verify the +immutable corpus. Bootstrap rechecks authority/descriptors, requires the image +already local and verifies the existing static archive; it does not pull/build. +Mint the standard release envelope immediately before each bounded attempt and +bind SHA256 of its **canonical JSON** (UTF-8, sorted keys, compact separators) in +the root window record. Deadline stays <=300s, not the two-hour window; it must +also precede the window's reserved return interval. A new attempt needs a new ID +and sequence after the prior attempt has fully restored. Exact retries return +the durable outcome, mutated retries reject. Root policy/window changes are +separate operator actions; the command cannot install or update them. + +Protected backup layout is fixed at `/var/lib/v8std-release/legacy/`: + +- `manifest.json` (<=16MiB, <=20000 app entries), hash bound by the window; + exact keys `schema_version:1`, `app`, `cache`, `directories`, `unit`, + `upstream`, `interpreter_sha256`. +- `app/`: complete saved `/opt/v8std-mcp` source, rules, locks, + venv and configuration. Every regular entry has `sha256`, numeric `uid`, `gid`, + `mode` (no special permission bits). Symlink entries contain only `link`; + links stay within the fixed app root or point to `/usr/bin/python3.12`. + Backup symlinks themselves are not followed/stored; their descriptors are data. + Unknown live app files block restore, not broad deletion. Derived + `__pycache__` is excluded from live identity checks, not from backup hashing. +- `cache/{pages.jsonl,search-vectors.jsonl,llms.txt,llms-full.txt}` and corresponding + hash/uid/gid/mode entries. `directories.app` and `directories.cache` enumerate + every parent (including `.`) with numeric uid/gid/mode; directory traversal + permissions are explicitly restored under UMask0077. +- `unit` is the exact original `/etc/systemd/system/v8std-mcp.service` bytes; + `upstream` is the exact inventoried managed upstream include. Both descriptors + require root ownership and mode0600 or0644. Other nginx/TLS/monitoring units + are not rewritten. `interpreter_sha256` binds `/usr/bin/python3.12`; a changed + system interpreter fails closed and requires operator repair, not an automatic + write into `/usr/bin`. + +All backup files/parents must be root-owned, non-symlink and not group/world +writable. Establish this from the complete native backup, not the three matching +historical source hashes. Rehearse restore before granting the window. The +controller streams/hash-checks/fsyncs temporary copies and restores original +file owner/mode. A fixed private `.v8std-release-restore-v1` directory in each +destination parent holds one deterministic temporary slot per target. It stays +owned by the controller with mode0700; interrupted copies are retried there, +not mistaken for unknown app code. Successful copies leave empty private +directories; no broad cleanup is performed. Cache destination mtimes must be **fresh**: never `copy2` stale +timestamps. This lets the unchanged legacy unit/default remote URLs and3600s +refresh serve its four verified files without startup network fallback. It is +an immediate bounded rollback guarantee, not an indefinite old-runtime hold. +Changing `tool-usage.jsonl` and monitoring inputs are never copied or overwritten; +monitoring-output preservation after container cutover is a separate Task6 gate. + +Install the reviewed `legacy-release-guard.conf` only as +`/etc/systemd/system/v8std-mcp.service.d/10-release-guard.conf`, plus the exact +`v8std-bootstrap-recover.service` and `.timer`. The drop-in preserves original +ExecStart and existing Before=monitoring/multi-user ordering. Its privileged +ExecCondition fails closed while a bootstrap is in flight or any container is +accepted, even if active.json is missing. Recovery marks legacy start allowed +only after the candidate is stopped and config/data/upstream are restored. +The timer is required by legacy startup, runs at boot+5s and every15s after its +service completes, independently of SSH. Recovery is not ordered Before=legacy: +that would deadlock when it starts the original service. Validate this topology +with native systemd, including enabled-legacy reboot and monitoring ordering. +The guard checks exact root-owned unit/drop-in bytes, active/enabled timer and +loaded non-stale manager state before legacy stop. No automatic installation, +enablement, daemon reload of an unreviewed unit set, or fallback guard is supplied. + +After all external gates and explicit window authorization, the operator invokes +the fixed installed module (never the restricted CI credential): + +```sh +sudo -n /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py bootstrap < release-envelope.json +sudo -n /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py bootstrap-status < /dev/null +sudo -n /usr/bin/python3 -I /opt/v8std-release/scripts/v8std_mcp_release.py bootstrap-recover +``` + +The first command requires one compact JSON line + EOF, durably queues the +attempt, then schedules the same detached Type=exec bounded host job. It does +not wait on SSH stdin. Status also supports the exact release-ID query described +below. Require expected identity, COMMITTED, cleanup_complete and live readiness; +RECEIVED is not acceptance. A killed queued attempt fails without runtime effects. +Before durable COMMITTED, recovery stops only the owned candidate, restores the +exact legacy files/unit/upstream and verifies public old health hashes, search +and all three Resource byte hashes. A failed restoration is RECOVERY_REQUIRED, +never ROLLED_BACK. After public smoke and durable COMMITTED, active-pointer +failure/reboot finishes acceptance and resume; it must not undo the accepted +release. Keep the legacy boot fence and recovery installed thereafter. No command +automatically enables ordinary runtime deployment or removes legacy artifacts. ## Reviewed host installation boundary (future operator action) @@ -152,7 +265,7 @@ operator authorization. No release input can replace them. | `/etc/v8std-release/policy.json` | root0600, parent root0755, installed from the disabled example and explicitly configured; no symlinks/writable parents | | `/var/lib/v8std-release` | root0700; journals, durable inbox, receipts, trusted manifests, references, pins, slots and verifier configuration; never writable by CI | | `/srv/v8std-indexes` and `/srv/v8std-indexes/v1` | precreate root0755 for nginx traversal; publisher alone writes, nginx only reads; objects directories0755/files0644; staging0700 | -| `/etc/nginx/v8std-release/upstream.conf` | root-owned managed include, initially created only by the approved bootstrap; not an arbitrary caller path | +| `/etc/nginx/v8std-release/upstream.conf` | root-owned managed include, installed pointing at the verified legacy endpoint during separately approved host setup, then hash-bound in the backup before bootstrap; not an arbitrary caller path | | systemd recovery units / sudoers | exact reviewed files from `deploy/container/`; validate locally on native Linux before enabling | The installed controller runs `/usr/bin/python3 -I`; it adds only its own diff --git a/tests/mcp_release_fixture.py b/tests/mcp_release_fixture.py index e8f91e7..e97ed4c 100644 --- a/tests/mcp_release_fixture.py +++ b/tests/mcp_release_fixture.py @@ -8,6 +8,8 @@ import json import os from pathlib import Path +import resource +import shutil import signal import subprocess import sys @@ -17,6 +19,51 @@ sys.path.insert(0, str(ROOT / "scripts")) import v8std_mcp_release as release +LEGACY_SHA = "b7bef11e145a188b30e7a7b17df2be4cb1acbd0c" + + +def prepare_legacy(root, files): + """Pinned historical sources, never mutable main; tiny data in fault matrix.""" + app = root / "legacy/app" + names = ["scripts/v8std_mcp_server.py", "scripts/v8std_mcp_index.py", + "scripts/v8std_retrieval_rules.py", "retrieval-rules.yml"] + export = Path("/legacy-source") # Only the disposable Docker test mounts this. + if not export.is_dir(): + tracked = subprocess.check_output(["git", "ls-tree", "-r", "--name-only", LEGACY_SHA], cwd=ROOT).decode().splitlines() + names[-1] = next(x for x in tracked if x.endswith("retrieval-rules.yml")) + entries = {} + def entry(raw, mode=0o644): + return {"sha256": release.digest(raw), "mode": mode, "uid": os.getuid(), "gid": os.getgid()} + for name in names: + raw = (export / name).read_bytes() if export.is_dir() else subprocess.check_output(["git", "show", LEGACY_SHA + ":" + name], cwd=ROOT) + release.atomic(app / name, raw, mode=0o644) + entries[name] = entry(raw) + raw = b"disposable fixture uses the local locked dependencies, not a native backup proof\n" + release.atomic(app / "venv/pyvenv.cfg", raw) + entries["venv/pyvenv.cfg"] = entry(raw) + entries["venv/bin/python"] = {"link": str(release.LEGACY_PYTHON)} + cache = {} + for name, raw in files.items(): + release.atomic(root / "legacy/cache" / name, raw) + os.utime(root / "legacy/cache" / name, (1, 1)) + cache[name] = entry(raw) + unit = b"original-unit-with-default-remote-urls-and-3600-refresh\n" + upstream = release.canonical_json({"port": release.read_record(root / "legacy-port.json")["port"]}) + release.atomic(root / "legacy/unit", unit) + release.atomic(root / "legacy/upstream", upstream) + directories = {"app": {}, "cache": {".": {"mode": 0o755, "uid": os.getuid(), "gid": os.getgid()}}} + for name in entries: + for parent in Path(name).parents: + directories["app"][str(parent)] = {"mode": 0o755, "uid": os.getuid(), "gid": os.getgid()} + saved = {"schema_version": 1, "app": entries, "cache": cache, "directories": directories, "unit": entry(unit), + "upstream": entry(upstream), "interpreter_sha256": "f" * 64} + # Test temp files are owned by the test user. Host validation is independently + # exercised under Linux root by Docker; no sudo/host permissions changes. + saved["unit"].update(uid=0, gid=0) + saved["upstream"].update(uid=0, gid=0) + release.write_json(root / "legacy/manifest.json", saved) + return release.digest((root / "legacy/manifest.json").read_bytes()) + class ProcessAdapter(release.HostAdapter): def __init__(self, root, policy, fault=""): @@ -121,6 +168,154 @@ def stop(self, record, deadline): pass +class BootstrapAdapter(ProcessAdapter): + def observe(self): + live = [p.stem for p in (self.root / "processes").glob("*.json") + if self.inspect(release.read_record(p)["record"], time.monotonic() + 2)["State"]["Running"]] + records = self.root / "releases/release-1.json" + window = release.read_record(records)["window"] if records.exists() else self.bootstrap_window({}) + if window["mode"] == "stop-start": + release.require(len(live) <= 1, "fixture_overlap") + with (self.root / "observations.jsonl").open("a") as stream: + stream.write(json.dumps({"live_runtimes": live}) + "\n") + + def bootstrap_window(self, envelope): + return release.read_record(self.root / "window.json") + + def bootstrap_backup(self, window, deadline, *, current=False): + from unittest.mock import patch + with patch.object(release, "trusted_path"): + saved = release.backup_inventory(self.root, window, deadline) + if current: + self.legacy_files(saved, deadline, restore=False) + return saved + + def bootstrap_capacity(self, window, envelope, deadline, *, after_stop=False): + self.record("capacity_after_stop" if after_stop else "capacity") + + def bootstrap_prepared(self, candidate, deadline): + self.record("prepared") + self.manifest(candidate) + + def arm_bootstrap_guard(self, deadline): + self.record("guard") + # A separate session/process periodically acquires the actual controller + # lock. Survives killed caller/worker, not a fake successful adapter ack. + state = self.root / "guard.json" + if state.exists(): + return + with (self.root / "guard.log").open("ab") as log: + process = subprocess.Popen([sys.executable, "-m", "tests.mcp_release_fixture", "guard", + str(self.root)], stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True, cwd=ROOT) + release.write_json(state, {"pid": process.pid}) + + def legacy_record(self): + return {"release_id": "legacy", "envelope_hash": LEGACY_SHA, + "port": release.read_record(self.root / "legacy-port.json")["port"]} + + def legacy_stop(self, deadline): + self.record("legacy_stop") + self.stop(self.legacy_record(), deadline) + self.observe() + if self.fault == "kill_after_legacy_stop": + os.kill(os.getpid(), signal.SIGKILL) + + def legacy_restore(self, window, deadline): + self.record("legacy_restore") + saved = self.bootstrap_backup(window, deadline) + self.legacy_files(saved, deadline, restore=True) + for name, target in (("unit", self.root / "legacy-unit"), ("upstream", self.root / "edge.json")): + entry = saved[name] | {"uid": os.getuid(), "gid": os.getgid()} + release.restore_file(self.root / "legacy" / name, target, entry, deadline) + + def legacy_start(self, deadline): + self.record("legacy_start") + release.require(release.legacy_start_allowed(self.root), "legacy_fenced") + for path in (self.root / "processes").glob("*.json"): + if path.stem != "legacy": + release.require(not self.inspect(release.read_record(path)["record"], deadline)["State"]["Running"], "fixture_overlap") + record = self.legacy_record() + info = self.inspect(record, deadline) + if info and info["State"]["Running"]: + return + with (self.root / "legacy.log").open("ab") as log: + process = subprocess.Popen([sys.executable, "-m", "tests.mcp_release_fixture", "legacy", + str(self.root), str(record["port"])], cwd=ROOT, stdin=subprocess.DEVNULL, + stdout=log, stderr=log, start_new_session=True) + self.children.append(process) + release.write_json(self.root / "processes/legacy.json", {"pid": process.pid, "record": record}) + self.observe() + + def legacy_check(self, window, deadline, *, public=False): + self.record("legacy_public" if public else "legacy_local") + # Run the real production hash + MCP checks, only translate loopback port. + from unittest.mock import patch + original = release.http + def http(url, *args, **kwargs): + return original(url.replace("127.0.0.1:8765", "127.0.0.1:" + str(self.legacy_record()["port"])), *args, **kwargs) + with patch.object(release, "http", http): + return super().legacy_check(window, deadline, public=public) + + def legacy_identity(self, deadline): + record = release.read_record(self.root / "processes/legacy.json") + release.require(self.inspect(record["record"], deadline)["State"]["Running"], "legacy_process") + observed = release.read_record(self.root / "legacy-observed.json") + saved = release.read_record(self.root / "legacy/manifest.json") + release.require(observed["pid"] == record["pid"] and all(observed["sources"][name] == + saved["app"]["scripts/" + name]["sha256"] for name in observed["sources"]), "legacy_process") + + def start(self, record, deadline): + if self.bootstrap_window(record)["mode"] == "stop-start": + old = self.inspect(self.legacy_record(), deadline) + release.require(not old or not old["State"]["Running"], "fixture_overlap") + super().start(record, deadline) + self.observe() + if self.fault == "kill_after_candidate_start": + os.kill(os.getpid(), signal.SIGKILL) + + def switch(self, record, deadline): + super().switch(record, deadline) + if self.fault == "kill_after_switch": + os.kill(os.getpid(), signal.SIGKILL) + + +def bootstrap_environment(root): + from contextlib import ExitStack + from unittest.mock import patch + stack = ExitStack() + for name, value in (("LEGACY_APP", root / "restored-app"), ("LEGACY_DATA", root / "restored-cache")): + stack.enter_context(patch.object(release, name, value)) + return stack + + +def run_legacy(root, port): + # Actual immutable historical modules, default URLs/default refresh3600. + # RLIMIT_DATA applies to this disposable process only; native cgroup evidence + # is separate. Do not inherit a monkeypatch of the new server or index. + if sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_DATA, (384 * 1024 * 1024, 384 * 1024 * 1024)) + sys.dont_write_bytecode = True + sys.path.insert(0, str(root / "restored-app/scripts")) + for name in ("v8std_mcp_index", "v8std_retrieval_rules", "v8std_mcp_server"): + sys.modules.pop(name, None) + import v8std_mcp_index as old_index + from v8std_mcp_server import build_server + def trap(*args, **kwargs): + with (root / "legacy-network.jsonl").open("a") as stream: + stream.write('"forbidden-network-attempt"\n') + raise OSError("network forbidden in legacy regression") + old_index.urlopen = trap + index = old_index.V8StdIndex(cache_dir=root / "restored-cache") + index.load() + release.write_json(root / "legacy-observed.json", {"source_sha": LEGACY_SHA, "pid": os.getpid(), + "sources": {name: release.file_hash(root / "restored-app/scripts" / name, time.monotonic() + 2) + for name in ("v8std_mcp_server.py", "v8std_mcp_index.py", "v8std_retrieval_rules.py")}, + "server_sha256": release.file_hash(root / "restored-app/scripts/v8std_mcp_server.py", time.monotonic() + 2), + "refresh_seconds": index.refresh_seconds, "index_url": index.index_url, "vectors_url": index.vectors_url}) + build_server(index, host="127.0.0.1", port=port, mcp_path="/mcp", allowed_hosts=["127.0.0.1:*"], + allowed_origins=[]).run(transport="streamable-http") + + class CrashController(release.Controller): def save(self, journal, state=None, intent=None): super().save(journal, state, intent) @@ -146,7 +341,8 @@ def dispatch(self): self.send_response(200) self.send_header("Content-Length", str(len(data))) self.end_headers() - self.wfile.write(data) + if self.command != "HEAD": + self.wfile.write(data) except OSError: self.send_error(404) return @@ -169,14 +365,83 @@ def dispatch(self): if connection: connection.close() - do_GET = do_POST = dispatch + do_GET = do_POST = do_HEAD = dispatch ThreadingHTTPServer(("127.0.0.1", port), Edge).serve_forever() if __name__ == "__main__": mode, directory, *args = sys.argv[1:] directory = Path(directory) - if mode == "runtime": + if mode == "legacy": + run_legacy(directory, int(args[0])) + elif mode == "guard": + # Release the test gate only when assertions have inspected the crash + # boundary. Loss-of-caller test leaves gate open from the beginning. + with bootstrap_environment(directory): + adapter = BootstrapAdapter(directory, release.read_record(directory / "policy.json")) + while True: + if not (directory / "guard-paused").exists(): + try: + release.Controller(directory, adapter).recover() + except release.ReleaseError: + pass + time.sleep(.2) + elif mode.startswith("bootstrap"): + from unittest.mock import patch + fault = args[0] if args else "" + with bootstrap_environment(directory): + adapter = BootstrapAdapter(directory, release.read_record(directory / "policy.json"), fault) + class BootstrapCrash(release.BootstrapController): + def save(self, journal, state=None, intent=None): + if fault == "accept_persist_failure" and state == "COMMITTED": + raise OSError("injected acceptance persistence failure") + super().save(journal, state, intent) + if fault == "kill_" + journal["intent"] or fault == "kill_" + journal["state"]: + os.kill(os.getpid(), signal.SIGKILL) + controller = BootstrapCrash(directory, adapter) + original = release.write_json + require = release.require + def fixture_authority(condition, code): + return require(True if code == "host_privilege" else condition, code) + replace = os.replace + def replace_with_fault(source, target): + if "kill_during_restore" in fault.split(",") and Path(target) == directory / "restored-app/scripts/v8std_mcp_server.py": + os.kill(os.getpid(), signal.SIGKILL) + return replace(source, target) + def write(path, value, **kwargs): + if fault in {"active_persist_failure", "kill_after_active"} and path == directory / "active.json": + if fault == "active_persist_failure": + raise OSError("injected active pointer failure") + original(path, value, **kwargs) + os.kill(os.getpid(), signal.SIGKILL) + return original(path, value, **kwargs) + try: + with patch.object(release, "write_json", write), patch.object(release.os, "replace", replace_with_fault), \ + patch.object(release, "schedule"), \ + patch.object(release, "ROOT", directory), patch.object(release, "trusted_policy", return_value=adapter.policy), \ + patch.object(release, "HostAdapter", return_value=adapter), patch.object(release, "require", fixture_authority), \ + patch.object(release, "BootstrapController", BootstrapCrash): + if mode == "bootstrap": + read_fd, write_fd = os.pipe() + os.write(write_fd, (directory / "envelope.json").read_bytes() + b"\n") + os.close(write_fd) + with os.fdopen(read_fd) as input_stream, patch.object(sys, "stdin", input_stream), \ + patch.object(sys, "argv", ["fixture", "bootstrap"]): + submitted = release.main() + with patch.object(sys, "argv", ["fixture", "_bootstrap"]): + result = release.main() if submitted["state"] == "RECEIVED" else submitted + elif mode == "bootstrap-legacy-start": + with patch.object(sys, "argv", ["fixture", "_legacy-allowed"]): + release.main() + adapter.legacy_start(time.monotonic() + 5) + result = {"state": "LEGACY_STARTED"} + else: + with patch.object(sys, "argv", ["fixture", "bootstrap-recover"]): + result = release.main() + print(json.dumps(result)) + except Exception as error: + print(json.dumps({"state": "REJECTED", "error_code": getattr(error, "code", "host_failure")})) + elif mode == "runtime": from v8std_mcp_runtime import SnapshotIndex from v8std_mcp_server import build_server port, site_url, sha, behavior = args diff --git a/tests/test_v8std_mcp_release.py b/tests/test_v8std_mcp_release.py index 1f63cb8..611e9d2 100644 --- a/tests/test_v8std_mcp_release.py +++ b/tests/test_v8std_mcp_release.py @@ -16,7 +16,7 @@ from unittest.mock import patch from tests.test_v8std_mcp_snapshots import Source from tests.test_v8std_mcp_release_hold import eventually -from tests.mcp_release_fixture import ProcessAdapter +from tests.mcp_release_fixture import ProcessAdapter, BootstrapAdapter, bootstrap_environment, prepare_legacy, LEGACY_SHA from tests import mcp_snapshot_fixtures as fixture import v8std_mcp_release as release @@ -487,6 +487,80 @@ def port(): return sock.getsockname()[1] +def bootstrap_window(request): + now = int(time.time()) + return {"schema_version": 1, "start_utc": now - 60, "end_utc": now + 7000, + "return_reserve_seconds": 1800, "envelope_sha256": release.digest(release.canonical_json(request)), + "mode": "stop-start", "legacy_unit": "v8std-mcp.service", + "legacy_source_sha": "b7bef11e145a188b30e7a7b17df2be4cb1acbd0c", + "backup_manifest_sha256": "d" * 64, + "capacity": {"disk_bytes": 1, "available_memory_bytes": 671088640, + "file_descriptors": 4096, "network_evidence": "e" * 64}} + + +class BootstrapBoundaryTests(unittest.TestCase): + def test_guard_refuses_stale_manager_configuration(self): + with tempfile.TemporaryDirectory() as temp: + adapter = release.HostAdapter(Path(temp), {}) + def run(argv, deadline): + if argv[1] == "is-enabled": + return b"enabled\n" + if argv[1] == "is-active": + return b"active\n" + return b"NeedDaemonReload=yes\nDropInPaths=\nLoadState=loaded\n" + def read(path): + return {"10-release-guard.conf": release.LEGACY_GUARD, + "v8std-bootstrap-recover.service": release.BOOTSTRAP_SERVICE, + "v8std-bootstrap-recover.timer": release.BOOTSTRAP_TIMER}[path.name].encode() + with patch.object(release, "trusted_path"), patch.object(release, "read_file", read), patch.object(release, "run", run): + with self.assertRaisesRegex(release.ReleaseError, "bootstrap_guard_loaded"): + adapter.arm_bootstrap_guard(time.monotonic() + 2) + + def test_installed_boot_guard_templates_and_ci_exclusion(self): + for name, expected in (("legacy-release-guard.conf", release.LEGACY_GUARD), + ("v8std-bootstrap-recover.service", release.BOOTSTRAP_SERVICE), + ("v8std-bootstrap-recover.timer", release.BOOTSTRAP_TIMER)): + self.assertEqual((ROOT / "deploy/container" / name).read_text(), expected) + self.assertNotIn("Before=v8std-mcp.service", release.BOOTSTRAP_SERVICE) + self.assertNotIn("ExecStart=", release.LEGACY_GUARD) # Original legacy command preserved. + for command in ("bootstrap", "bootstrap-status", "bootstrap-recover", "_bootstrap", "_legacy-allowed"): + result = subprocess.run([sys.executable, "-I", str(ROOT / "deploy/container/release-entry.py")], + env={"SSH_ORIGINAL_COMMAND": command}, capture_output=True, timeout=2) + self.assertNotEqual(result.returncode, 0) + + def test_window_exact_identity_time_and_fixed_targets(self): + request = envelope() + window = bootstrap_window(request) + self.assertEqual(release.validate_bootstrap_window(window, request), window) + for change in ({"start_utc": int(time.time()) + 1}, {"end_utc": int(time.time()) + 1800}, + {"end_utc": window["start_utc"] + 7201}, {"return_reserve_seconds": 1799}, + {"envelope_sha256": "f" * 64}, {"legacy_unit": "sshd.service"}, + {"backup_path": "/etc"}, {"mode": "automatic"}, {"start_utc": True}): + with self.subTest(change=change), self.assertRaises(release.ReleaseError): + release.validate_bootstrap_window(window | change, request) + # Recovery is an owed duty, not new window authority. + expired = window | {"start_utc": 1, "end_utc": 7201} + self.assertEqual(release.validate_bootstrap_window(expired, request, recovery=True), expired) + + def test_boot_guard_denies_pending_and_accepted_legacy_start(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.assertTrue(release.legacy_start_allowed(root)) + journal = {"kind": "bootstrap", "envelope": envelope(), "state": "VERIFIED", + "intent": "stop_legacy", "legacy_start_allowed": False} + release.write_json(root / "releases/release-1.json", journal) + self.assertFalse(release.legacy_start_allowed(root)) + journal["legacy_start_allowed"] = True + release.write_json(root / "releases/release-1.json", journal) + self.assertTrue(release.legacy_start_allowed(root)) + journal["state"] = "COMMITTED" + release.write_json(root / "releases/release-1.json", journal) + self.assertFalse(release.legacy_start_allowed(root)) + (root / "releases/release-1.json").write_bytes(b"broken") + with self.assertRaises(release.ReleaseError): + release.legacy_start_allowed(root) + + class TransactionTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -751,5 +825,381 @@ def test(self): setattr(TransactionTests, "test_recovery_" + _fault, crash_case(_fault)) +class BootstrapProcessTests(unittest.TestCase): + install_snapshot = TransactionTests.install_snapshot + + def health(self): + # This independent observer includes spawning an HTTP worker. The former + # 300ms harness allowance is not a product readiness requirement and is + # below process startup jitter on the one-CPU Linux fixture. Stay below + # the controller's existing3s read cap; transaction/ready budgets do not + # change. Real identity/smoke gates are still executed by the controller. + try: + return json.loads(release.http(self.policy["public_url"] + "/healthz", time.monotonic() + 2)) + except release.ReleaseError as error: + self.last_probe_error = error.code + return None + + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="v8std-bootstrap-") + self.root = Path(self.temp.name) + self.addCleanup(self.temp.cleanup) + self.source = Source() + self.addCleanup(self.source.close) + self.environment = bootstrap_environment(self.root) + self.addCleanup(self.environment.close) + self.addCleanup(self.stop_owned) + config = {"site_url": self.source.url, "refresh_seconds": 1, "max_snippet_chars": 4000, + "memory_bytes": 536870912, "cpus": 1} + key = release.digest(release.canonical_json(config)) + self.policy = {"runtime_enabled": False, "configs": {key: config}, "ports": [port(), port()], + "public_url": f"http://127.0.0.1:{port()}"} + self.env = envelope() | {"configuration_digest": key, "corpus_id": self.source.manifest["corpus_id"], + "archive_sha256": self.source.manifest["archive"]["sha256"]} + self.window = bootstrap_window(self.env) + release.write_json(self.root / "legacy-port.json", {"port": port()}) + self.files = fixture.corpus_files() + self.window["backup_manifest_sha256"] = prepare_legacy(self.root, self.files) + for filename, value in (("policy.json", self.policy), ("envelope.json", self.env), ("window.json", self.window)): + release.write_json(self.root / filename, value) + self.install_snapshot() + self.adapter = BootstrapAdapter(self.root, self.policy) + self.adapter.legacy_restore(self.window, time.monotonic() + 5) + self.adapter.legacy_start(time.monotonic() + 5) + (self.root / "guard-paused").touch() + with (self.root / "edge.log").open("wb") as log: + self.edge = subprocess.Popen([sys.executable, "-m", "tests.mcp_release_fixture", "edge", + str(self.root), self.policy["public_url"].rsplit(":", 1)[1]], cwd=ROOT, stdin=subprocess.DEVNULL, + stdout=log, stderr=log) + try: + eventually(self.health) + except AssertionError: + self.fail((self.root / "legacy.log").read_text()) + self.static_errors = [] + self.static_samples = 0 + self.static_stop = threading.Event() + def sample(): + while not self.static_stop.is_set(): + try: + self.assert_static() + self.static_samples += 1 + except Exception as error: + self.static_errors.append(str(error)) + self.static_stop.wait(.1) + self.static_thread = threading.Thread(target=sample) + self.static_thread.start() + + def stop_owned(self): + if hasattr(self, "static_thread"): + self.static_stop.set() + self.static_thread.join(3) + guard = self.root / "guard.json" + if guard.exists(): + pid = release.read_record(guard)["pid"] + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + for path in (self.root / "processes").glob("*.json"): + record = release.read_record(path)["record"] + self.adapter.stop(record, time.monotonic() + 5) + for child in getattr(self, "adapter", ProcessAdapter(self.root, {})).children: + child.poll() + if hasattr(self, "edge"): + self.edge.terminate() + self.edge.wait(5) + + def invoke(self, fault="", mode="bootstrap"): + # Crash survivors (including multiprocessing's resource tracker) must + # not keep a PIPE's EOF open after the controller itself has exited. + with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as error: + result = subprocess.run([sys.executable, "-m", "tests.mcp_release_fixture", mode, str(self.root), fault], + cwd=ROOT, stdout=output, stderr=error, timeout=25) + output.seek(0) + error.seek(0) + result.stdout, result.stderr = output.read(), error.read() + if fault.startswith("kill_"): + self.assertEqual(result.returncode, -signal.SIGKILL, result.stderr.decode()) + outcome = release.Controller(self.root, self.adapter).status() + else: + self.assertEqual(result.returncode, 0, result.stderr.decode()) + outcome = json.loads(result.stdout) + self.assertEqual(self.static_errors, []) + self.assertGreater(self.static_samples, 0) + if self.window["mode"] == "stop-start": + observations = [json.loads(line) for line in (self.root / "observations.jsonl").read_text().splitlines()] + self.assertTrue(observations) + self.assertLessEqual(max(len(x["live_runtimes"]) for x in observations), 1) + return outcome + + def assert_legacy(self): + self.adapter.legacy_check(self.window, time.monotonic() + 8, public=True) + health = self.health() + self.assertEqual(health["sha256"], release.digest(self.files["pages.jsonl"])) + self.assertEqual(health["vectors"]["sha256"], release.digest(self.files["search-vectors.jsonl"])) + observed = release.read_record(self.root / "legacy-observed.json") + self.assertEqual(observed["source_sha"], LEGACY_SHA) + self.assertEqual(observed["server_sha256"], "be1e73a27ad2c2aea08a516ffeede6286feb92a70752e964a13bd8567139d713") + self.assertEqual(observed["refresh_seconds"], 3600) + self.assertEqual(observed["index_url"], "https://v8std.ru/ai/pages.jsonl") + self.assertFalse((self.root / "legacy-network.jsonl").exists()) + live_candidates = [p for p in (self.root / "processes").glob("*.json") if p.stem != "legacy" + and self.adapter.inspect(release.read_record(p)["record"], time.monotonic() + 1)["State"]["Running"]] + self.assertEqual(live_candidates, []) + self.assert_static() + + def assert_static(self): + import http.client + client = http.client.HTTPConnection("127.0.0.1", int(self.policy["public_url"].rsplit(":", 1)[1]), timeout=2) + try: + path = "/indexes/v1/" + self.env["archive_sha256"] + "/snapshot.tar.gz" + for method in ("GET", "HEAD"): + client.request(method, path) + response = client.getresponse() + data = response.read() + self.assertEqual(response.status, 200) + self.assertEqual(int(response.getheader("Content-Length")), len(self.source.archive)) + if method == "GET": + self.assertEqual(release.digest(data), self.env["archive_sha256"]) + finally: + client.close() + + def test_success_only_after_smoke_without_autoactivation(self): + result = self.invoke() + self.assertEqual(result["state"], "COMMITTED", result) + self.assertTrue(result["cleanup_complete"], result) + self.assertFalse(self.policy["runtime_enabled"]) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + self.assertFalse(release.legacy_start_allowed(self.root)) + self.assertFalse(self.adapter.inspect(self.adapter.legacy_record(), time.monotonic() + 1)["State"]["Running"]) + self.assert_static() + + def test_boundary_rejections_preserve_original_endpoint(self): + for change in ({"end_utc": int(time.time()) - 1}, {"envelope_sha256": "f" * 64}): + release.write_json(self.root / "window.json", self.window | change) + self.assertEqual(self.invoke()["state"], "REJECTED") + self.assertFalse((self.root / "releases/release-1.json").exists()) + self.assert_legacy() + (self.root / "window.json").unlink() + self.assertEqual(self.invoke()["state"], "REJECTED") + release.write_json(self.root / "window.json", self.window) + release.write_json(self.root / "active.json", {"existing": True}) + self.assertEqual(self.invoke()["error_code"], "bootstrap_already_accepted") + self.assert_legacy() + + def test_capacity_rejection_no_stop(self): + result = self.invoke("capacity") + self.assertEqual(result["state"], "FAILED", result) + self.assertFalse((self.root / "guard.json").exists()) + self.assert_legacy() + + def test_complete_backup_paths_and_directory_permissions(self): + self.assertEqual((self.root / "restored-app/scripts").stat().st_mode & 0o777, 0o755) + release.atomic(self.root / "restored-app/scripts/unlisted.py", b"untrusted code") + result = self.invoke() + self.assertEqual(result["state"], "FAILED", result) + self.assertEqual(result["error_code"], "legacy_unlisted") + self.assertFalse((self.root / "guard.json").exists()) + + def test_accept_persistence_failure_rolls_back(self): + result = self.invoke("accept_persist_failure") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assertFalse((self.root / "active.json").exists()) + self.assert_legacy() + + def test_active_persistence_failure_preserves_committed(self): + result = self.invoke("active_persist_failure") + self.assertEqual(result["state"], "COMMITTED", result) + self.assertFalse(result["cleanup_complete"]) + self.assertFalse((self.root / "active.json").exists()) + self.assertFalse(release.legacy_start_allowed(self.root)) + result = self.invoke(mode="bootstrap-recover") + self.assertTrue(result["cleanup_complete"], result) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + + def test_reboot_fence_prevents_enabled_legacy_racing_accepted_recovery(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = release.read_record(self.root / "active.json") + self.adapter.stop(active, time.monotonic() + 5) + self.assertEqual(self.invoke(mode="bootstrap-legacy-start")["error_code"], "legacy_fenced") + result = self.invoke(mode="bootstrap-recover") + self.assertTrue(result["cleanup_complete"], result) + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + self.assertFalse(self.adapter.inspect(self.adapter.legacy_record(), time.monotonic() + 1)["State"]["Running"]) + + def test_rollback_failure_retains_owed_recovery_and_usage_logs(self): + usage = self.root / "restored-cache/tool-usage.jsonl" + usage.write_bytes(b"test-private-usage-before\n") + result = self.invoke("public_dead,legacy_restore") + self.assertEqual(result["state"], "RECOVERY_REQUIRED", result) + self.assertFalse(result["cleanup_complete"]) + self.assert_static() + with usage.open("ab") as stream: + stream.write(b"test-private-usage-after\n") + result = self.invoke(mode="bootstrap-recover") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assertEqual(usage.read_bytes(), b"test-private-usage-before\ntest-private-usage-after\n") + self.assertNotIn("test-private", json.dumps(result)) + self.assert_legacy() + + def test_retry_rollback_closes_boot_fence_before_rewriting_files(self): + self.assertEqual(self.invoke("public_dead,legacy_public")["state"], "RECOVERY_REQUIRED") + self.assertTrue(release.legacy_start_allowed(self.root)) + self.invoke("kill_restore_legacy", mode="bootstrap-recover") + self.assertFalse(release.legacy_start_allowed(self.root)) + self.assertEqual(self.invoke(mode="bootstrap-recover")["state"], "ROLLED_BACK") + self.assert_legacy() + + def test_kill_during_file_restore_retries_without_unlisted_temp_blocker(self): + self.invoke("kill_during_restore,public_dead") + self.assertFalse(release.legacy_start_allowed(self.root)) + self.assertEqual(self.invoke(mode="bootstrap-recover")["state"], "ROLLED_BACK") + self.assert_legacy() + + def test_queued_caller_loss_and_expiry_have_no_stop_authority(self): + with patch.object(release, "schedule", side_effect=release.ReleaseError("command_failed")): + with self.assertRaises(release.ReleaseError): + release.BootstrapController(self.root, self.adapter).submit(release.canonical_json(self.env)) + self.assertEqual(release.Controller(self.root, self.adapter).status()["state"], "RECEIVED") + release.write_json(self.root / "window.json", self.window | {"start_utc": 1, "end_utc": 7201}) + self.assertEqual(self.invoke(mode="bootstrap-recover")["state"], "FAILED") + self.assertFalse((self.root / "guard.json").exists()) + self.assert_legacy() + + def test_backup_hash_failure_has_no_stop(self): + (self.root / "legacy/cache/pages.jsonl").write_bytes(b"changed backup") + result = self.invoke() + self.assertEqual(result["error_code"], "backup_hash", result) + self.assertEqual(result["state"], "FAILED") + self.assertFalse((self.root / "guard.json").exists()) + self.assertEqual(self.health()["sha256"], release.digest(self.files["pages.jsonl"])) + + def test_backup_rejects_traversal_and_external_link_before_restore(self): + path = self.root / "legacy/manifest.json" + original = release.read_record(path) + for attack in ("path", "link"): + bad = json.loads(json.dumps(original)) + if attack == "path": + bad["app"]["../outside"] = original["app"]["scripts/v8std_mcp_server.py"] + bad["directories"]["app"][".."] = original["directories"]["app"]["."] + else: + bad["app"]["venv/bin/python"] = {"link": "/etc/shadow"} + release.write_json(path, bad) + window = self.window | {"backup_manifest_sha256": release.digest(path.read_bytes())} + with self.assertRaisesRegex(release.ReleaseError, "backup_path|backup_link"): + self.adapter.bootstrap_backup(window, time.monotonic() + 3) + self.assertFalse((self.root / "outside").exists()) + + def test_readiness_cancels_actual_worker_and_restores_legacy(self): + result = self.invoke("ready") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assert_legacy() + + def test_post_stop_capacity_failure_restores_legacy(self): + result = self.invoke("capacity_after_stop") + self.assertEqual(result["state"], "ROLLED_BACK", result) + self.assert_legacy() + + def test_recovery_readiness_is_capped_and_durable(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = release.read_record(self.root / "active.json") + self.adapter.stop(active, time.monotonic() + 5) + allowances = [] + def blocked(record, token, deadline, manifest=None): + allowances.append(deadline - time.monotonic()) + raise release.ReleaseError("deadline") + with patch.object(self.adapter, "hold", blocked): + result = release.Controller(self.root, self.adapter).recover() + self.assertLessEqual(allowances[0], 90) + self.assertEqual(result["state"], "COMMITTED") + self.assertFalse(result["cleanup_complete"]) + self.assertFalse(release.legacy_start_allowed(self.root)) + self.assertTrue(self.invoke(mode="bootstrap-recover")["cleanup_complete"]) + + def test_overlap_keeps_old_until_public_smoke(self): + self.window["mode"] = "overlap" + release.write_json(self.root / "window.json", self.window) + self.assertEqual(self.invoke()["state"], "COMMITTED") + operations = [json.loads(line)["operation"] for line in (self.root / "calls.jsonl").read_text().splitlines()] + self.assertGreater(operations.index("legacy_stop"), operations.index("public")) + + def test_recovery_is_durable_after_accepted_restart_then_kill(self): + self.assertEqual(self.invoke()["state"], "COMMITTED") + active = release.read_record(self.root / "active.json") + self.adapter.stop(active, time.monotonic() + 5) + self.invoke("kill_after_candidate_start", mode="bootstrap-recover") + self.assertFalse(release.Controller(self.root, self.adapter).status()["cleanup_complete"]) + result = self.invoke(mode="bootstrap-recover") + self.assertTrue(result["cleanup_complete"], result) + self.assertEqual(self.health()["runtime_sha"], active["runtime_source_sha"]) + self.assertIsNone(self.health()["hold_token"]) + + def test_caller_loss_guard_recovers_after_window_expiry(self): + self.invoke("kill_after_legacy_stop") + self.assertFalse(self.health()) + self.assert_static() + release.write_json(self.root / "window.json", self.window | {"start_utc": 1, "end_utc": 7201}) + # The durable attempt, not newly revoked window input, drives recovery. + (self.root / "guard-paused").unlink() + eventually(lambda: release.Controller(self.root, self.adapter).status().get("state") == "ROLLED_BACK", timeout=12) + self.assert_legacy() + + def test_actual_old_full_cache_restart_default_urls_without_network(self): + import shutil + self.adapter.legacy_stop(time.monotonic() + 5) + # Generated cache is not tracked in Git. Its bytes become the protected + # fixture's hash-checked input; only executable legacy code uses history. + paths = {name: ROOT / "docs" / ("ai" if name.endswith(".jsonl") else "") / name for name in release.LEGACY_CACHE} + if not all(path.is_file() for path in paths.values()): + self.skipTest("generated full cache unavailable; tiny-cache regression remains mandatory") + self.files = {name: path.read_bytes() for name, path in paths.items()} + self.window["backup_manifest_sha256"] = prepare_legacy(self.root, self.files) + release.write_json(self.root / "window.json", self.window) + # Keep the negative control permanently: a preserved stale timestamp + # attempts remote HTTP even though fallback eventually serves the cache. + for name in release.LEGACY_CACHE: + shutil.copy2(self.root / "legacy/cache" / name, self.root / "restored-cache" / name) + self.adapter.legacy_start(time.monotonic() + 5) + eventually(lambda: (self.health() or {}).get("row_count") == len(self.files["pages.jsonl"].splitlines()), timeout=15) + self.assertTrue((self.root / "legacy-network.jsonl").exists()) + self.adapter.legacy_stop(time.monotonic() + 5) + (self.root / "legacy-network.jsonl").unlink() + self.adapter.legacy_restore(self.window, time.monotonic() + 10) + self.adapter.legacy_start(time.monotonic() + 5) + eventually(lambda: (self.health() or {}).get("row_count") == len(self.files["pages.jsonl"].splitlines()), timeout=15) + self.assert_legacy() + + +def bootstrap_crash_case(fault, accepted=False): + def test(self): + self.invoke(fault) + self.assert_static() + result = self.invoke(mode="bootstrap-recover") + self.assertEqual(result["state"], "COMMITTED" if accepted else "ROLLED_BACK", result) + self.assertTrue(result["cleanup_complete"], result) + if accepted: + self.assertEqual(self.health()["runtime_sha"], self.env["runtime_source_sha"]) + self.assertFalse(release.legacy_start_allowed(self.root)) + else: + self.assert_legacy() + before = (self.root / "calls.jsonl").read_bytes() + self.assertEqual(self.invoke(), result) + self.assertEqual((self.root / "calls.jsonl").read_bytes(), before) + release.write_json(self.root / "envelope.json", self.env | {"trigger_sha": "f" * 40}) + self.assertEqual(self.invoke()["error_code"], "mutated_duplicate") + return test + + +for _fault in ("kill_guard_armed", "kill_stop_legacy", "kill_after_legacy_stop", "kill_start_candidate", + "kill_after_candidate_start", "kill_PREPARED", "kill_READY", "kill_after_switch", "kill_SWITCHED"): + setattr(BootstrapProcessTests, "test_" + _fault, bootstrap_crash_case(_fault)) +for _fault in ("kill_COMMITTED", "kill_after_active", "kill_resume_candidate"): + setattr(BootstrapProcessTests, "test_" + _fault, bootstrap_crash_case(_fault, accepted=True)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_v8std_mcp_release_docker.py b/tests/test_v8std_mcp_release_docker.py index 6c43f96..c261d05 100644 --- a/tests/test_v8std_mcp_release_docker.py +++ b/tests/test_v8std_mcp_release_docker.py @@ -31,9 +31,13 @@ @unittest.skipUnless(os.environ.get("V8STD_TASK5_DOCKER") == "1", "explicit disposable Docker evidence") class DockerReleaseTests(unittest.TestCase): - def docker(self, *args, check=True): + def docker(self, *args, check=True, timeout=60): self.calls.append(["docker", *map(str, args)]) - result = subprocess.run(["docker", *map(str, args)], capture_output=True, timeout=60) + try: + result = subprocess.run(["docker", *map(str, args)], capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired as error: + print((error.stderr or b"")[-128 * 1024:].decode(errors="replace"), flush=True) + raise if check: details = result.stderr.decode() if result.returncode and args[0] == "exec": @@ -41,6 +45,44 @@ def docker(self, *args, check=True): self.assertEqual(result.returncode, 0, details) return result + def test_bootstrap_process_fault_matrix_in_restricted_linux(self): + from tests.mcp_release_fixture import LEGACY_SHA + from tests.test_v8std_mcp_release import BootstrapBoundaryTests, BootstrapProcessTests + self.calls = [] + prefix = "v8std-task5-bootstrap-" + uuid.uuid4().hex[:12] + cases = ["tests.test_v8std_mcp_release." + cls.__name__ + "." + method + for cls in (BootstrapBoundaryTests, BootstrapProcessTests) + for method in unittest.defaultTestLoader.getTestCaseNames(cls)] + with tempfile.TemporaryDirectory(prefix="v8std-task5-legacy-source-") as temp: + source = Path(temp) + for path in ("scripts/v8std_mcp_server.py", "scripts/v8std_mcp_index.py", + "scripts/v8std_retrieval_rules.py", "retrieval-rules.yml"): + target = source / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(subprocess.check_output(["git", "show", LEGACY_SHA + ":" + path], cwd=ROOT)) + # Keep the outer watchdog at240s; bounded batches distinguish total + # matrix duration from one stuck child/transaction on the one-CPU rig. + for start in range(0, len(cases), 8): + name = prefix + "-" + str(start // 8) + began = time.monotonic() + try: + result = self.docker("run", "--rm", "--name", name, "--pull=never", "--network=none", + "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--init", + "--user=10001:10001", "--memory=768m", "--memory-swap=768m", "--cpus=1", "--pids-limit=128", + "--label=pro.v8std.test=task5", "--tmpfs=/tmp:rw,nosuid,size=256m,mode=1777", + "--mount", f"type=bind,source={ROOT},target=/work,readonly", + "--mount", f"type=bind,source={source},target=/legacy-source,readonly", + "--workdir=/work", "--entrypoint=python", RUNTIME, "-m", "unittest", + *cases[start:start + 8], "-v", timeout=240) + print(result.stderr.decode(), flush=True) + print(json.dumps({"batch": start // 8, "tests": min(8, len(cases) - start), + "elapsed_seconds": round(time.monotonic() - began, 3)}), flush=True) + finally: + self.docker("rm", "-f", name, check=False) + print(json.dumps({"bootstrap_linux": "PASS", "runtime_image": RUNTIME, + "legacy_source": LEGACY_SHA, "memory_bytes": 768 * 1024 * 1024, + "cpus": 1, "network": "none", "read_only": True, "user": 10001}, sort_keys=True)) + def test_native_nginx_static_independence_hold_and_admission(self): self.calls = [] name = "v8std-task5-" + uuid.uuid4().hex[:12] From cfed37c69e7cf533bbb2292a7df774e16bd4a667 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 03:05:28 +0300 Subject: [PATCH 37/88] docs: record approved MCP first-migration evidence --- spec/operations/mcp-container-verification.md | 63 +++++++++++++++++++ ...6-09-10-mcp-container-distribution-plan.md | 19 ++++-- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index ab24741..7a97061 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -554,3 +554,66 @@ bootstrap tests; it is not an installed backup/restart or indefinite hold proof. Ordinary architecture validation/impact and whitespace checks passed. No whole repository suite, strict build, merge-ready, main merge/push, registry publish, Catalog submission, server setup or production migration is claimed here. + +### First migration: reviewed local implementation, 2026-09-15 + +Signed `fdb1d829ca9b9958ea2f3c716bafe68b7f33c58f` implements operator-only +bootstrap, a bounded hash-bound window, protected legacy restoration and an +independent recovery/boot guard. Separate review approved spec compliance and +scoped quality without Critical, Important or new Minor findings. Task5 is +complete locally, not the first live migration or whole-branch acceptance. + +Verification sequence (overlapping suites are not independent coverage): + +```sh +.venv/bin/python -m unittest tests.test_v8std_mcp_release tests.test_v8std_mcp_release_hold tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_runtime -v +.venv/bin/python -m unittest tests.test_v8std_mcp_release.BootstrapBoundaryTests tests.test_v8std_mcp_release.BootstrapProcessTests -v +V8STD_TASK5_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_release_docker.DockerReleaseTests.test_bootstrap_process_fault_matrix_in_restricted_linux -v +``` + +The first run passed182 tests in253.933s. Two subsequent bootstrap self-review +fixes close the legacy boot fence before restoration retry and make mid-copy +SIGKILL staging retryable. The amended bootstrap run passed36 tests in108.894s. +All36 cases then passed in restricted Linux containers: wrapper365.299s total, +five batches of8/8/8/8/4 cases, unittest times42.853/65.154/101.238/95.499/58.021s. +Each batch retained a240s test watchdog. Production300s transaction,90s readiness, +30s smoke,45s stop and loader360s/read20s are unchanged. + +The existing local runtime image above was used with a read-only source mount +and historical code pinned to `b7bef11`; this is not a new release image build. +Each Linux batch had1CPU,768MiB memory/no extra swap,128PIDs,256MiB tmpfs, +networknone, UID10001, read-only/cap-dropALL/init/no-new-privileges. Real old/new +MCP processes, HTTP proxy, served identities and static requests were tested; +Docker/systemd/GitHub adapter effects were substituted where necessary. + +Earlier failures are not erased: crash-surviving descendants held captured +stdout pipes open, so fixture capture now uses temporary files and controller +PID wait. Two unpartitioned Linux runs timed out at240s. The first batched run +then failed because a separate post-recovery health observer allowed only300ms +including Python startup; only that observer changed to2s, below production's +existing HTTP3s cap. The complete36-case Linux run covers the correction. +No production timeout was increased. The earlier combined two-test Docker run +remains failed; its nginx/hold/static test separately passed, repeating umask0077 +control, same-token reack, invalid-switch restoration, static GET/HEAD after +runtime stop and two200/six429 downloads. Exact owned test resources were cleaned; +existing images and unrelated containers/services were preserved. + +Acceptance is durable COMMITTED only after real local/public MCP smoke. Missing +active-pointer persistence then reconciles the accepted image; preacceptance +failure restores exact legacy source/config/cache. Tests cover SIGKILL around +side effects, caller loss, expired-window recovery, failed rollback/retry, +duplicates, capacity rejection and boot fencing. Full historical1423-page/ +3281-vector startup/search/three-Resource reads retain the stale-mtime negative +control and zero-network fresh-cache restoration. Usage logs are not overwritten +or treated as corpus identity. Existing Starlette warning remains unsuppressed. + +Task6 still must preserve monitoring: the current timer reads the old unit and +flat usage log, while container launch does not yet supply new usage events. +This is an integration defect, not the deferred dashboard/OpenMetrics redesign. +The retained image-alt parser defect and warning triage remain final obligations. + +Native protected backup/venv/interpreter restoration, boot ordering/return time, +single-runtime/overlap capacity, installed SSH/sudo policy, published-image +provenance and external TLS/index delivery remain prerequisites for the later +window. No window is scheduled; no host setup, main merge/push, registry publish, +CI activation,100k capacity or live migration is claimed. diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 44618f3..1fa5992 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -421,9 +421,12 @@ calls at this external boundary, not pretend mocked return values prove health. and `5ae7588` and independently reviewed. Final148 release/hold/snapshot/runtime tests and the real Docker/nginx check passed; five review findings were repaired with RED/GREEN and approved in scoped re-review. This supersedes the earlier -paused29-test evidence. The first-migration slice below is still pending, so -expanded Task5 and the complete plan are not yet complete. Native host setup, -published-artifact evidence and production capacity remain external gates. +paused29-test evidence. First migration is implemented in signed `fdb1d82` and +passed separate spec/quality review. The182-test focused run, amended36 bootstrap +tests and all36 restricted Linux cases passed; the verification record retains +the exact sequence and earlier failed diagnostics. Task5 is locally complete. +Native host setup/rehearsal, published-artifact evidence and production capacity +remain external gates. Task6 and final branch gates are still required. - [x] **RED:** Test unknown schema, invalid digest/namespace/config path, stale or mutated duplicate ID, concurrent releases, failed pull/ready/switch/smoke, @@ -487,13 +490,13 @@ execution envelope just before each bounded attempt so its300second deadline has not expired during preparation. Recovery of an already-started attempt must remain allowed after window expiry; only new attempts are refused. -- [ ] **RED initial boundary:** Execute CLI against a disposable fixture and +- [x] **RED initial boundary:** Execute CLI against a disposable fixture and show rejection outside/missing window, wrong envelope hash, existing active container, CI entry invocation and insufficient single-runtime capacity. Add subprocess fault cases after legacy stop, after candidate start, after switch and during initial active-record persistence. Assert the endpoint, exact served data and owned process count, not just a successful exit. -- [ ] **GREEN initial transition:** Add a serialized initial journal and +- [x] **GREEN initial transition:** Add a serialized initial journal and independently scheduled host recovery before stopping legacy. Stop/start is allowed only inside the operator window. Commit the first container record after local/public smoke, or restore verified Python config/data/upstream. @@ -502,13 +505,17 @@ must remain allowed after window expiry; only new attempts are refused. not blindly roll back an already accepted container. Before acceptance, startup/reboot recovery restores the saved legacy service; after acceptance it starts the exact accepted digest, without racing the still-enabled legacy unit. -- [ ] **VERIFY initial transition:** Run the real disposable process fixture +- [x] **VERIFY initial transition:** Run the real disposable process fixture under restricted memory, including no-overlap, SIGKILL/lost SSH, crash at each persistence boundary, duplicate request and rollback failure. Record absence of any simultaneous legacy/candidate process in stop/start mode. Keep static archive GET/HEAD available throughout. Before the host window, replay the tested runbook on native Linux and measure return-to-legacy time. +The completed checkbox records local fixture verification. Native host rehearsal +remains a prerequisite for the later operational window; it has not been +performed or inferred from Docker tests. + Required observable outcomes (the fixture's CLI returns JSON with these fields): ```python From a1eda7a3bbd6ac930f52c7c1d93d066d07828fc0 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 03:26:39 +0300 Subject: [PATCH 38/88] fix: restore dev build context and verify runtime COPY closure --- .dockerignore | 3 + tests/test_v8std_mcp_distribution.py | 112 +++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/.dockerignore b/.dockerignore index 8143d50..42a5f23 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,8 @@ !Dockerfile.site !requirements-mcp.lock !requirements-build.lock +!requirements.txt +!requirements-mcp.txt !zensical.toml !docs/ !docs/** @@ -18,6 +20,7 @@ !scripts/v8std_mcp_index.py !scripts/v8std_mcp_snapshots.py !scripts/v8std_mcp_snapshot_format.py +!scripts/v8std_mcp_hold.py !scripts/v8std_mcp_presentation.py !scripts/v8std_mcp_chunks.py !scripts/v8std_retrieval_rules.py diff --git a/tests/test_v8std_mcp_distribution.py b/tests/test_v8std_mcp_distribution.py index 8e5c8ff..bc707db 100644 --- a/tests/test_v8std_mcp_distribution.py +++ b/tests/test_v8std_mcp_distribution.py @@ -4,10 +4,13 @@ import json import os from pathlib import Path +import shlex +import signal import subprocess import sys import tempfile import unittest +import uuid from unittest.mock import patch import yaml @@ -17,6 +20,115 @@ import check_mcp_container as harness +@unittest.skipUnless(os.environ.get("V8STD_TEST_IMAGE_BUILD"), "explicit fresh image build acceptance") +class ImageContextClosureTests(unittest.TestCase): + def test_retained_dev_copy_context_includes_requirements_and_entrypoint_dependencies(self): + definition = (ROOT / "docker-compose/docker/Dockerfile").read_text().replace("\\\n", " ") + copies = [line for line in definition.splitlines() if line.startswith("COPY ")] + with tempfile.TemporaryDirectory(prefix="v8std-task6-dev-context-") as output: + result = subprocess.run( + ["docker", "build", "--progress=plain", "--file", "-", + "--output", "type=local,dest=" + output, "."], + cwd=ROOT, input="FROM scratch\n" + "\n".join(copies) + "\n", + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout) + required = ["requirements.txt", "requirements-mcp.txt"] + ["scripts/" + name for name in ( + "generate_social_cards.py", "generate_search_vectors.py", "generate_ai_artifacts.py", + "install_zensical.sh", "run_v8std_mcp.sh", "zensical_docs.sh", "zensical-version.sh", + "v8std_mcp_server.py", "check_article_html.py", "publish_diagnostic_sitemap.py", + "publish_license_texts.py")] + for relative in required: + self.assertEqual((Path(output) / "opt/v8std" / relative).read_bytes(), + (ROOT / relative).read_bytes(), relative) + + def test_every_runtime_copy_survives_actual_buildkit_context_filter(self): + definition = (ROOT / "Dockerfile.mcp").read_text().replace("\\\n", " ") + copies = [line for line in definition.splitlines() if line.startswith("COPY ")] + # Execute the real COPY closure through the real ignore file. Scratch + # isolates context failure from registry availability and dependency I/O. + with tempfile.TemporaryDirectory(prefix="v8std-task6-context-") as output: + result = subprocess.run( + ["docker", "build", "--progress=plain", "--file", "-", + "--output", "type=local,dest=" + output, "."], + cwd=ROOT, input="FROM scratch\nWORKDIR /opt/v8std\n" + "\n".join(copies) + "\n", + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout) + for line in copies: + *sources, destination = shlex.split(line)[1:] + for source in sources: + path = ROOT / source + files = path.rglob("*") if path.is_dir() else [path] + for item in files: + if item.is_file(): + relative = item.relative_to(path) if path.is_dir() else Path(item.name) + copied = Path(output) / "opt/v8std" / destination / relative + self.assertEqual(copied.read_bytes(), item.read_bytes(), str(item)) + + def test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime(self): + # Catch missing dockerignore entries using BuildKit's actual context, not + # a second implementation of ignore-pattern semantics. This is a fixture + # image; the all-zero revision deliberately makes no release claim. + expected = {} + definition = (ROOT / "Dockerfile.mcp").read_text().replace("\\\n", " ") + for line in definition.splitlines(): + if not line.startswith("COPY "): + continue + *sources, destination = shlex.split(line)[1:] + for source in sources: + path = ROOT / source + files = sorted(path.rglob("*")) if path.is_dir() else [path] + for item in files: + if item.is_file(): + relative = item.relative_to(path) if path.is_dir() else Path(item.name) + target = str(Path("/opt/v8std") / destination / relative) + expected[target] = hashlib.sha256(item.read_bytes()).hexdigest() + tag = "v8std-task6-context-" + uuid.uuid4().hex + ":fixture" + try: + # Desktop credential helpers can outlive Docker and retain stderr. + # A regular log file avoids waiting for an inherited pipe's EOF; + # the process group bounds only this build and its own helpers. + with tempfile.TemporaryFile(mode="w+") as log: + built = subprocess.Popen( + ["docker", "build", "--progress=plain", "--file", "Dockerfile.mcp", + "--build-arg", "SOURCE_SHA=" + "0" * 40, "--tag", tag, "."], + cwd=ROOT, text=True, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + try: + built.wait(timeout=600) + finally: + try: + os.killpg(built.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + built.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(built.pid, signal.SIGKILL) + built.wait(timeout=5) + log.seek(0) + self.assertEqual(built.returncode, 0, log.read()) + command = ["docker", "run", "--rm", "--network", "none", "--read-only", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--init", + "--memory", "256m", "--pids-limit", "64", "--tmpfs", "/tmp:size=16m", + "--entrypoint", "python", tag] + probe = ( + "import hashlib,json,os,pathlib,sys; " + "sys.path.insert(0,'/opt/v8std/scripts'); import v8std_mcp_server,v8std_mcp_hold; " + "assert os.getuid()==10001; " + "print(json.dumps({p:hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() " + "for p in json.loads(sys.argv[1])}))" + ) + result = subprocess.run(command + ["-c", probe, json.dumps(list(expected))], + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), expected) + checked = subprocess.run(command + ["-m", "pip", "check"], + capture_output=True, text=True, timeout=30) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + finally: + # Remove only our unique fixture tag, never prune shared Docker data. + subprocess.run(["docker", "image", "rm", tag], capture_output=True, timeout=30) + + class GatewayProfileTests(unittest.TestCase): def validate(self, state): return harness.validate_gateway_profile(state, expected_cache={ From 71b885360b1347f7cbbcd31cc0bfea7074792ea4 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 03:53:40 +0300 Subject: [PATCH 39/88] test: bound context fixtures and verify Docker cleanup --- tests/test_v8std_mcp_distribution.py | 379 ++++++++++++++++++++++++--- 1 file changed, 336 insertions(+), 43 deletions(-) diff --git a/tests/test_v8std_mcp_distribution.py b/tests/test_v8std_mcp_distribution.py index bc707db..f320544 100644 --- a/tests/test_v8std_mcp_distribution.py +++ b/tests/test_v8std_mcp_distribution.py @@ -1,5 +1,6 @@ """Focused distribution checks; real Docker/browser acceptance is opt-in.""" import hashlib +from contextlib import contextmanager import html as html_module import json import os @@ -9,6 +10,7 @@ import subprocess import sys import tempfile +import time import unittest import uuid from unittest.mock import patch @@ -20,17 +22,337 @@ import check_mcp_container as harness +def _context_group_running(pgid): + # Orphaned zombies cannot run or retain descriptors. Their reap belongs to + # init; waiting on killpg(0) alone can never finish under a non-reaping PID1. + states = subprocess.run(["ps", "-A", "-o", "pgid=,stat="], check=True, + capture_output=True, text=True, timeout=1).stdout + return any(fields[0] == str(pgid) and not fields[1].startswith("Z") + for line in states.splitlines() if len(fields := line.split()) == 2) + + +def _signal_context_group(pgid, sig): + try: + os.killpg(pgid, sig) + except (ProcessLookupError, PermissionError): + # macOS can report EPERM for an already vanished group. A live group + # still makes signal denial a real cleanup failure, never a success. + if _context_group_running(pgid): + raise + + +def _stop_context_group(process): + stopped = False + try: + for sig, grace in ((signal.SIGTERM, 1), (signal.SIGKILL, 2)): + _signal_context_group(process.pid, sig) + deadline = time.monotonic() + grace + while True: + process.poll() # Reap the leader, independently of descendants. + if not _context_group_running(process.pid): + stopped = True + return + if time.monotonic() >= deadline: + break + time.sleep(.02) + raise AssertionError(f"owned process group {process.pid} did not terminate") + finally: + try: + if not stopped: + _signal_context_group(process.pid, signal.SIGKILL) + finally: + process.wait(timeout=2) + + +def _context_command(args, *, input=None, timeout): + """Bound a CLI and its group; inherited output descriptors never delay EOF.""" + with tempfile.TemporaryFile(mode="w+") as source, tempfile.TemporaryFile(mode="w+") as log: + if input is not None: + source.write(input) + source.seek(0) + process = subprocess.Popen(args, cwd=ROOT, stdin=source, stdout=log, + stderr=subprocess.STDOUT, text=True, start_new_session=True) + primary = None + try: + process.wait(timeout=timeout) + except BaseException as error: + primary = error + raise + finally: + try: + _stop_context_group(process) + except BaseException as error: + if primary is None: + raise + primary.add_note(f"process cleanup failed: {error}") + log.seek(0) + output = log.read(2 * 1024 * 1024 + 1) + if len(output) > 2 * 1024 * 1024: + raise AssertionError("fixture command output exceeded 2MiB") + return subprocess.CompletedProcess(args, process.returncode, output, "") + + +class _ContextImage: + """Own only a fresh UUID tag and the named containers launched from it.""" + def __init__(self): + self.name = "v8std-task6-context-" + uuid.uuid4().hex + self.tag = self.name + ":fixture" + self.containers = [] + + def __enter__(self): + return self + + def run(self, args): + name = self.name + "-" + str(len(self.containers) + 1) + self.containers.append(name) # Record ownership before the daemon call. + return _context_command( + ["docker", "run", "--name", name, "--network", "none", "--read-only", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--init", + "--memory", "256m", "--pids-limit", "64", "--tmpfs", "/tmp:size=16m", + "--entrypoint", "python", self.tag, *args], timeout=30) + + @staticmethod + def present(kind, name): + selector = "name=^/" + name + "$" if kind == "container" else "reference=" + name + format = "{{.Names}}" if kind == "container" else "{{.Repository}}:{{.Tag}}" + result = _context_command(["docker", kind, "ls", "--all", "--filter", selector, + "--format", format], timeout=10) + if result.returncode: + raise AssertionError(f"cannot verify {kind} {name}: {result.stdout.strip()}") + names = result.stdout.splitlines() + if any(value != name for value in names): + raise AssertionError(f"unexpected inventory for exact {kind} {name}") + return bool(names) + + def __exit__(self, exception_type, primary, traceback): + failures = [] + for kind, name in [("container", name) for name in self.containers] + [("image", self.tag)]: + try: + if not self.present(kind, name): + continue + command = ["docker", "rm", "--force", name] if kind == "container" else ["docker", "image", "rm", name] + result = _context_command(command, timeout=30) + if result.returncode: + failures.append(f"{kind} {name}: {result.stdout.strip()}") + if self.present(kind, name): + failures.append(f"{kind} {name} still exists") + except Exception as error: + failures.append(f"{kind} {name}: {error}") + if failures: + error = AssertionError("fixture cleanup failed: " + "; ".join(failures)) + if primary is None: + raise error + primary.add_note(str(error)) + return False + + +_DOCKER_FAULT_CLI = r''' +import hashlib, json, os, pathlib, subprocess, sys, time +state_path = pathlib.Path(os.environ["CONTEXT_FAULT_STATE"]) +state = json.loads(state_path.read_text()) +args = sys.argv[1:] +state["calls"].append(args) +def save(): + state_path.write_text(json.dumps(state)) +def option(name): + return args[args.index(name) + 1] +if args[0] == "build": + if state.get("orphan"): + helper = subprocess.Popen([sys.executable, "-c", """ +import json, os, pathlib, signal, time +signal.signal(signal.SIGTERM, signal.SIG_IGN) +pathlib.Path(os.environ['CONTEXT_HELPER_PID']).write_text(json.dumps({'pid':os.getpid(),'pgid':os.getpgrp()})) +while True: time.sleep(1) +"""]) + deadline = time.monotonic() + 2 + while not pathlib.Path(os.environ["CONTEXT_HELPER_PID"]).exists(): + if time.monotonic() >= deadline: raise RuntimeError("helper failed to start") + time.sleep(.01) + print("fixture build refused", file=sys.stderr, flush=True) + sys.exit(17) + state["images"].append(option("--tag")) +elif args[0] == "run": + name = option("--name") if "--name" in args else "anonymous-fixture" + state["containers"].append(name) + if state.get("timeout"): + save() + time.sleep(60) + if "-c" in args: + root = pathlib.Path(os.environ["CONTEXT_FAULT_ROOT"]) + print(json.dumps({p:hashlib.sha256((root / pathlib.Path(p).relative_to('/opt/v8std')).read_bytes()).hexdigest() + for p in json.loads(args[-1])})) + else: + print("No broken requirements found.") + if "--rm" in args: state["containers"].remove(name) +elif args[:2] == ["container", "ls"]: + name = option("--filter").removeprefix("name=^/").removesuffix("$") + sys.stdout.write("".join(x + "\n" for x in state["containers"] if x == name)) +elif args[0] == "rm": + if state.get("container_rm_error"): + print("fixture container removal refused", file=sys.stderr) + save() + sys.exit(1) + state["containers"].remove(args[-1]) +elif args[:2] == ["image", "ls"]: + tag = option("--filter").removeprefix("reference=") + sys.stdout.write("".join(x + "\n" for x in state["images"] if x == tag)) +elif args[:2] == ["image", "rm"]: + if state.get("image_rm_error"): + print("fixture image removal refused", file=sys.stderr) + save() + sys.exit(1) + if not state.get("image_rm_lies") and args[-1] in state["images"]: + state["images"].remove(args[-1]) +else: + raise AssertionError("unexpected Docker operation: " + repr(args)) +save() +''' + + +@contextmanager +def _docker_fault_cli(**faults): + """External CLI seam; daemon state outlives a timed-out client process.""" + with tempfile.TemporaryDirectory(prefix="v8std-context-fault-") as temporary: + directory = Path(temporary) + cli = directory / "docker" + cli.write_text("#!" + sys.executable + "\n" + _DOCKER_FAULT_CLI) + cli.chmod(0o700) + state = directory / "daemon.json" + state.write_text(json.dumps({"images": ["foreign:image"], "containers": ["foreign-container"], + "calls": [], **faults})) + with patch.dict(os.environ, {"PATH": str(directory) + os.pathsep + os.environ["PATH"], + "CONTEXT_FAULT_STATE": str(state), + "CONTEXT_FAULT_ROOT": str(ROOT), + "CONTEXT_HELPER_PID": str(directory / "helper.json")}): + yield state, directory / "helper.json" + + +def _test_process_running(pid): + result = subprocess.run(["ps", "-p", str(pid), "-o", "stat="], + capture_output=True, text=True, timeout=2) + return bool(result.stdout.strip()) and not result.stdout.strip().startswith("Z") + + +class ContextHarnessLifecycleTests(unittest.TestCase): + def test_command_timeout_reaps_leader_without_cleanup_error(self): + with self.assertRaises(subprocess.TimeoutExpired) as raised: + _context_command([sys.executable, "-c", "import time; time.sleep(10)"], timeout=.2) + self.assertEqual(getattr(raised.exception, "__notes__", []), []) + + def test_all_build_paths_stop_term_ignoring_helper_after_leader_exit(self): + methods = ( + "test_retained_dev_copy_context_includes_requirements_and_entrypoint_dependencies", + "test_every_runtime_copy_survives_actual_buildkit_context_filter", + "test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime", + ) + for method in methods: + with self.subTest(method=method), _docker_fault_cli(orphan=True) as (_, pid_path), \ + tempfile.TemporaryFile(mode="w+") as output: + driver = ( + "from tests.test_v8std_mcp_distribution import ImageContextClosureTests\n" + "try: ImageContextClosureTests()." + method + "()\n" + "except AssertionError as error:\n" + " assert 'fixture build refused' in str(error), str(error)\n" + "else: raise AssertionError('missing build failure')\n" + ) + process = subprocess.Popen([sys.executable, "-c", driver], cwd=ROOT, + stdout=output, stderr=subprocess.STDOUT, start_new_session=True) + try: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.fail("build caller waited on a descendant's inherited descriptor") + output.seek(0) + self.assertEqual(process.returncode, 0, output.read()) + self.assertTrue(pid_path.exists(), "real helper did not start") + helper = json.loads(pid_path.read_text()) + self.assertFalse(_test_process_running(helper["pid"]), + "TERM-ignoring helper survived its build leader") + finally: + # RED must not leak the deliberately hostile fixture either. + if pid_path.exists(): + try: + os.kill(json.loads(pid_path.read_text())["pid"], signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + + @contextmanager + def short_runtime_timeout(self): + real_run, real_wait = subprocess.run, subprocess.Popen.wait + observed = [] + def run(args, **kwargs): + if args[:2] == ["docker", "run"]: + kwargs["timeout"] = .3 + try: + return real_run(args, **kwargs) + except subprocess.TimeoutExpired as error: + if args[:2] == ["docker", "run"]: + observed.append(error) + raise + def wait(process, timeout=None): + if process.args[:2] == ["docker", "run"] and timeout == 30: + timeout = .3 + try: + return real_wait(process, timeout=timeout) + except subprocess.TimeoutExpired as error: + if process.args[:2] == ["docker", "run"]: + observed.append(error) + raise + with patch.object(subprocess, "run", run), patch.object(subprocess.Popen, "wait", wait): + yield observed + + def test_runtime_timeout_removes_exact_daemon_container_then_image(self): + with _docker_fault_cli(timeout=True) as (path, _), self.short_runtime_timeout() as observed: + with self.assertRaises(subprocess.TimeoutExpired) as raised: + ImageContextClosureTests().test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime() + state = json.loads(path.read_text()) + self.assertIs(raised.exception, observed[0]) + self.assertEqual(getattr(raised.exception, "__notes__", []), []) + self.assertEqual(state["containers"], ["foreign-container"]) + self.assertEqual(state["images"], ["foreign:image"]) + removed = [x for x in state["calls"] if x[0] == "rm" or x[:2] == ["image", "rm"]] + self.assertEqual([x[0] for x in removed], ["rm", "image"]) + + def test_image_removal_failure_is_visible_after_successful_assertions(self): + with _docker_fault_cli(image_rm_error=True) as (path, _): + with self.assertRaisesRegex(AssertionError, "cleanup.*fixture image removal refused"): + ImageContextClosureTests().test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime() + self.assertEqual(json.loads(path.read_text())["containers"], ["foreign-container"]) + + def test_successful_remove_exit_cannot_hide_a_retained_fixture_tag(self): + with _docker_fault_cli(image_rm_lies=True): + with self.assertRaisesRegex(AssertionError, "cleanup.*still exists"): + ImageContextClosureTests().test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime() + + def test_cleanup_failures_preserve_primary_timeout_and_attempt_both_removals(self): + with _docker_fault_cli(timeout=True, container_rm_error=True, image_rm_error=True) as (path, _), \ + self.short_runtime_timeout() as observed: + with self.assertRaises(subprocess.TimeoutExpired) as raised: + ImageContextClosureTests().test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtime() + self.assertIs(raised.exception, observed[0]) + notes = " ".join(getattr(raised.exception, "__notes__", [])) + self.assertIn("fixture container removal refused", notes) + self.assertIn("fixture image removal refused", notes) + state = json.loads(path.read_text()) + self.assertIn("foreign-container", state["containers"]) + self.assertIn("foreign:image", state["images"]) + + @unittest.skipUnless(os.environ.get("V8STD_TEST_IMAGE_BUILD"), "explicit fresh image build acceptance") class ImageContextClosureTests(unittest.TestCase): def test_retained_dev_copy_context_includes_requirements_and_entrypoint_dependencies(self): definition = (ROOT / "docker-compose/docker/Dockerfile").read_text().replace("\\\n", " ") copies = [line for line in definition.splitlines() if line.startswith("COPY ")] with tempfile.TemporaryDirectory(prefix="v8std-task6-dev-context-") as output: - result = subprocess.run( + result = _context_command( ["docker", "build", "--progress=plain", "--file", "-", "--output", "type=local,dest=" + output, "."], - cwd=ROOT, input="FROM scratch\n" + "\n".join(copies) + "\n", - text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + input="FROM scratch\n" + "\n".join(copies) + "\n", timeout=60) self.assertEqual(result.returncode, 0, result.stdout) required = ["requirements.txt", "requirements-mcp.txt"] + ["scripts/" + name for name in ( "generate_social_cards.py", "generate_search_vectors.py", "generate_ai_artifacts.py", @@ -47,11 +369,10 @@ def test_every_runtime_copy_survives_actual_buildkit_context_filter(self): # Execute the real COPY closure through the real ignore file. Scratch # isolates context failure from registry availability and dependency I/O. with tempfile.TemporaryDirectory(prefix="v8std-task6-context-") as output: - result = subprocess.run( + result = _context_command( ["docker", "build", "--progress=plain", "--file", "-", "--output", "type=local,dest=" + output, "."], - cwd=ROOT, input="FROM scratch\nWORKDIR /opt/v8std\n" + "\n".join(copies) + "\n", - text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + input="FROM scratch\nWORKDIR /opt/v8std\n" + "\n".join(copies) + "\n", timeout=60) self.assertEqual(result.returncode, 0, result.stdout) for line in copies: *sources, destination = shlex.split(line)[1:] @@ -82,34 +403,11 @@ def test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtim relative = item.relative_to(path) if path.is_dir() else Path(item.name) target = str(Path("/opt/v8std") / destination / relative) expected[target] = hashlib.sha256(item.read_bytes()).hexdigest() - tag = "v8std-task6-context-" + uuid.uuid4().hex + ":fixture" - try: - # Desktop credential helpers can outlive Docker and retain stderr. - # A regular log file avoids waiting for an inherited pipe's EOF; - # the process group bounds only this build and its own helpers. - with tempfile.TemporaryFile(mode="w+") as log: - built = subprocess.Popen( - ["docker", "build", "--progress=plain", "--file", "Dockerfile.mcp", - "--build-arg", "SOURCE_SHA=" + "0" * 40, "--tag", tag, "."], - cwd=ROOT, text=True, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) - try: - built.wait(timeout=600) - finally: - try: - os.killpg(built.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - built.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(built.pid, signal.SIGKILL) - built.wait(timeout=5) - log.seek(0) - self.assertEqual(built.returncode, 0, log.read()) - command = ["docker", "run", "--rm", "--network", "none", "--read-only", - "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--init", - "--memory", "256m", "--pids-limit", "64", "--tmpfs", "/tmp:size=16m", - "--entrypoint", "python", tag] + with _ContextImage() as fixture: + built = _context_command( + ["docker", "build", "--progress=plain", "--file", "Dockerfile.mcp", + "--build-arg", "SOURCE_SHA=" + "0" * 40, "--tag", fixture.tag, "."], timeout=600) + self.assertEqual(built.returncode, 0, built.stdout) probe = ( "import hashlib,json,os,pathlib,sys; " "sys.path.insert(0,'/opt/v8std/scripts'); import v8std_mcp_server,v8std_mcp_hold; " @@ -117,16 +415,11 @@ def test_fresh_runtime_build_contains_every_copy_input_and_imports_locked_runtim "print(json.dumps({p:hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() " "for p in json.loads(sys.argv[1])}))" ) - result = subprocess.run(command + ["-c", probe, json.dumps(list(expected))], - capture_output=True, text=True, timeout=30) - self.assertEqual(result.returncode, 0, result.stderr) + result = fixture.run(["-c", probe, json.dumps(list(expected))]) + self.assertEqual(result.returncode, 0, result.stdout) self.assertEqual(json.loads(result.stdout), expected) - checked = subprocess.run(command + ["-m", "pip", "check"], - capture_output=True, text=True, timeout=30) - self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) - finally: - # Remove only our unique fixture tag, never prune shared Docker data. - subprocess.run(["docker", "image", "rm", tag], capture_output=True, timeout=30) + checked = fixture.run(["-m", "pip", "check"]) + self.assertEqual(checked.returncode, 0, checked.stdout) class GatewayProfileTests(unittest.TestCase): From 551e83a41d0cd2f67d38222abbe9843d84d8e825 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 03:57:46 +0300 Subject: [PATCH 40/88] docs: record Docker context verification and monitoring design gap --- spec/operations/mcp-container-verification.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 7a97061..0a3bb8d 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -617,3 +617,68 @@ single-runtime/overlap capacity, installed SSH/sudo policy, published-image provenance and external TLS/index delivery remain prerequisites for the later window. No window is scheduled; no host setup, main merge/push, registry publish, CI activation,100k capacity or live migration is claimed. + +### Task6 partial evidence and design pause, 2026-09-15 + +Task6 is **not complete**. At signed `7bf5291`, the confirmed retained dev-build +failure is repaired by explicitly including `requirements.txt` and +`requirements-mcp.txt` in the root Docker context. Actual BuildKit disproved an +earlier inspection-based claim that `hold.py` or dev scripts were excluded: +`!scripts/` already includes its descendants. Runtime COPY closure passed before +the change. Negative COPY probes for `.git/HEAD` and `spec/README.md` failed as +expected. This is not an exclusive per-script allowlist. + +Two actual scratch COPY/export byte-comparison tests passed in0.477s. A fresh +runtime build/import/package test passed in243.864s, verifying Dockerfile COPY +bytes, server/hold imports, UID10001 and `pip check` under the restricted runtime +profile. Its all-zero `SOURCE_SHA` deliberately identifies a local fixture, +**not** a committed-source release candidate or a published artifact. No corpus +load, default-source activation or capacity conclusion follows from this test. + +The local strict build passed with3282 vectors,1430 article checks and no article +HTML violations; all3 license texts were published locally. The subsequent full +suite passed628 tests in316.896s with6 opt-in Docker checks skipped. These results +predate the later test-harness-only review fixes; they do not establish Task6 or +whole-branch acceptance. Existing Starlette and ordinary pip build warnings were +not suppressed. No runtime dependencies were changed to hide them. + +Scoped review of `203cbc0..7bf5291` approved the context correction but found two +Important defects in the new harness: surviving process-group descendants after +the Docker leader exits, and unverified daemon-container/image cleanup after a +runtime-client timeout. Their focused repair and re-review are required before +the slice can be considered complete. + +Signed `08ab175c0c984929dc2bbb79fe031de936eaa47a` contains their test-only repair: +all three build paths use bounded group lifetime and file-backed capture; +named fixture containers and the exact image tag are removed and their absence +verified. Cleanup failure remains visible without replacing a primary timeout. +Six focused regressions passed in9.134s; the25-test module passed in9.184s with4 +explicit opt-in skips. All3 actual Docker context/runtime checks then passed +in245.553s, using ordinary BuildKit cache and the same explicit fixture revision. +Post-run queries confirmed exact fixture removal; unrelated resources remained. +An intermediate new EPERM/ResourceWarning was reproduced by an added failing +regression and corrected before these final runs. It was not suppressed. +Independent scoped re-review of `7bf5291..08ab175` approved both fixes, with no +new Critical, Important or Minor findings. The reviewer checked the scenarios +and evidence against the fix diff without duplicating the test runs. This +completes only the context/harness repair slice, not Task6 or release acceptance. + +The earlier monitoring classification above is refined: preserving the existing +dashboard is required, but the approved package does not define how its +unprivileged reader obtains fresh live state from the root-owned container +controller. A durable COMMITTED receipt is not runtime liveness; the controller +unit being active is not MCP uptime; snapshot `loaded_at` is not process start. +The real monitor reader and controller-status probes reproduce these semantic +counterexamples even while the existing monitoring tests pass. + +Under `v8std-architecture` failure recovery, implementation of this integration +and remaining Task6 work is paused for brainstorming and revised design/plan +approval. A bounded private atomic state file is a proposed approach, not an +implemented or accepted interface. Its producer/reader authority, identity, +freshness, error behavior and compatibility must be specified before code. +No Docker/sudo access was granted to the monitor and no public schema changed. + +CI publication/classification, process-v2 integration, the retained image-alt +parser repair, committed-source candidate, mixed-load proof and final gates +remain outstanding. Production, registry, GitHub settings, `main` and the +unscheduled first-migration window were not changed by this work. From 51348b12c27a859f69865cffd21abce927e5f86a Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:07:51 +0300 Subject: [PATCH 41/88] docs: specify private container monitoring input --- ...-15-mcp-private-container-monitor-input.md | 52 +++++ .../mcp-container-monitor-input-v1-r0.md | 194 +++++++++++++++++ ...9-15-mcp-container-monitor-input-design.md | 201 ++++++++++++++++++ .../mcp-monitor-state-trust-is-bounded.md | 32 +++ 4 files changed, 479 insertions(+) create mode 100644 spec/adr/2026-09-15-mcp-private-container-monitor-input.md create mode 100644 spec/contracts/mcp-container-monitor-input-v1-r0.md create mode 100644 spec/designs/2026-09-15-mcp-container-monitor-input-design.md create mode 100644 spec/invariants/mcp-monitor-state-trust-is-bounded.md diff --git a/spec/adr/2026-09-15-mcp-private-container-monitor-input.md b/spec/adr/2026-09-15-mcp-private-container-monitor-input.md new file mode 100644 index 0000000..2c7260f --- /dev/null +++ b/spec/adr/2026-09-15-mcp-private-container-monitor-input.md @@ -0,0 +1,52 @@ +--- +schema_version: 1 +kind: adr +id: MCP_PRIVATE_CONTAINER_MONITOR_INPUT +scope: product +design: design:mcp-container-monitor-input +requirements: + - MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + - MCP_MONITOR_OBSERVES_SERVING_RUNTIME + - MCP_MONITOR_RETAINS_UNPRIVILEGED_READER + - MCP_CONTAINER_USAGE_SURVIVES_RELEASE +aliases: [] +supersedes: [] +cancels: [] +invariants: + introduces: + - invariant:MCP_MONITOR_STATE_TRUST_IS_BOUNDED + preserves: + - invariant:OPERATOR_DATA_STAYS_OUTSIDE_WEB_ROOT + - invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR + replaces: {} + cancels: [] +contracts: + introduces: + - contract:MCP_CONTAINER_MONITOR_INPUT@1.0 + preserves: + - contract:MCP_MONITORING_PROJECTION@1.0 + - contract:MCP_USAGE_EVENTS@1.0 + - contract:MCP_RELEASE_RUNTIME@1.0 + replaces: {} + cancels: [] +--- + +# Закрытый вход непривилегированного мониторинга + +Root-owned sampler перед существующим batch job собирает действительное +состояние serving runtime и атомарно пишет маленький private JSON. Агрегатор +может только читать его; не получает полномочий Docker или release-controller. +История usage отдельно остаётся persistent private data, а не частью cache. + +Свежесть, boot identity и проверка serving identity ограничивают доверие к +сводке. Ошибка или переход означает unknown. Квитанция COMMITTED, active +controller и загрузка нового corpus не заменяют процессные наблюдения. + +Решение сохраняет старые readers и public fields. `restarts:null` для container +честно обозначает отсутствие сопоставимого общего счётчика. Private details не +расширяют публичную схему; общий редизайн/privacy-remediation сюда не включены. +Это дополнение к release ADR, не его замена и не изменение release budgets. + +Socket/group у reader отвергнуты из-за избыточной власти; фиксированный sudo RPC +избыточен для чтения batch-сводки; отдельный daemon не требуется. Цена файла — +явное истечение срока доверия и необходимость тестировать invalidation/races. diff --git a/spec/contracts/mcp-container-monitor-input-v1-r0.md b/spec/contracts/mcp-container-monitor-input-v1-r0.md new file mode 100644 index 0000000..98e70a1 --- /dev/null +++ b/spec/contracts/mcp-container-monitor-input-v1-r0.md @@ -0,0 +1,194 @@ +--- +schema_version: 1 +kind: contract +id: MCP_CONTAINER_MONITOR_INPUT +scope: product +version: 1 +revision: 0 +compatibility: backward-compatible +design: design:mcp-container-monitor-input +producer: root-owned host sampler and existing runtime usage writer +consumers: + - existing unprivileged monitoring aggregator + - local operator +requirements: + - MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + - MCP_MONITOR_OBSERVES_SERVING_RUNTIME + - MCP_MONITOR_RETAINS_UNPRIVILEGED_READER + - MCP_CONTAINER_USAGE_SURVIVES_RELEASE + - LEGACY_USAGE_EVENTS_REMAIN_READABLE +governs: + - scripts/v8std_mcp_monitor_state.py + - scripts/v8std_mcp_monitoring.py + - scripts/v8std_mcp_release.py + - scripts/v8std_mcp_server.py + - deploy/container +conformance: + module: tests.test_v8std_mcp_monitor_state + command: .venv/bin/python -m unittest tests.test_v8std_mcp_monitor_state tests.test_v8std_mcp_monitoring tests.test_v8std_mcp_release -v +required_when: implemented +supersedes: [] +deprecates: [] +--- + +# Private monitoring input 1.0 + +## State file и значения + +Фиксированный host path `/run/v8std-monitor/state.json`; UTF-8 JSON object, +не более8192 bytes, без duplicate/unknown keys, NaN или неявного coercion. +`schema_version` — integer1, bool и float вместо integer запрещены. +Неизвестная будущая схема не интерпретируется как текущая. + +Все перечисленные поля обязательны; nullable означает явный JSON null: + +| Поле | Тип и смысл | +|---|---| +| `schema_version` | integer1 | +| `boot_id` | canonical UUID текущей загрузки Linux host | +| `invocation_id` | 32 lowercase hex из systemd INVOCATION_ID текущего monitoring job; null только при invalidation | +| `observed_at` | RFC3339 UTC; timestamp наблюдения, не публикации страницы | +| `observed_boottime_ns` | integer >=0, CLOCK_BOOTTIME host при завершении проверки | +| `backend` | `legacy`, `container`, `unknown` | +| `release_id` | validated release ID для container, иначе null | +| `live` | bool/null; наблюдаемая жизнь выбранного процесса | +| `ready` | bool/null; готовность того же процесса обслуживать MCP | +| `active_since` | RFC3339 UTC/null, действительный старт процесса | +| `uptime_seconds` | integer >=0/null, возраст процесса при наблюдении | +| `restarts` | integer >=0/null; legacy NRestarts, container всегда null | +| `reason` | `ok`, `transition`, `absent`, `unready`, `probe_failed`, `identity_mismatch`, `invalid_state` | + +Ни paths/ports/container names, ни Docker inspect, exceptions, environment, +payloads, IP/query/User-Agent, ни вся release-квитанция в файл не копируются. +`release_id` private: в legacy public projection не переносится. + +`live=true` требует подтверждённого процесса и `/livez`200 для container; +`ready=true` дополнительно требует валидного `/healthz`200 с `ok=true`, +`ready=true` и ожидаемым runtime SHA. `ready=false` допустимо при валидной +503-неготовности того же runtime. Ошибка транспорта/парсинга не равна доказанной +остановке: live/ready неизвестны. Подтверждённо отсутствующий/остановленный +выбранный runtime даёт live=false, ready=false и reason=absent. +При unknown оба bool и все uptime/restart данные null. `ready=true` при +`live!=true` недопустимо. Corpus ID может обновляться без смены runtime. + +## Доверенная идентичность и race + +Sampler получает цели только из root-owned validated host state, не из +state.json, args/env агрегатора, сетевого запроса или названия unit контроллера. +Короткая nonblocking release-lock секция снимает fingerprint trusted active +record, terminal/recovery state и managed upstream. Незавершённая транзакция, +busy lock или несовпадающий upstream дают unknown. Lock не удерживается во +время Docker/HTTP/proc I/O и collector не запускает recovery. + +Container сверяется с approved descriptor: exact image/config revision и +ownership labels по прежней Adapter.inspect проверке. Проверяются Running, +host PID/start identity до и после HTTP-проб. Процессный возраст берётся из +наблюдения kernel process start в текущем boot; wall-clock timestamp Docker +нельзя использовать для неограниченного вычисления uptime после clock jump. +При недоступном надёжном process age `uptime_seconds` и `active_since` null, +а доказанная liveness может остаться известной. `loaded_at` не используется. + +До первого принятого container и после подтверждённого legacy rollback +разрешён только зафиксированный `v8std-mcp.service`, его MainPID/start identity +и неизменённый legacy health endpoint. Не любой inactive old unit обозначает +состояние нового MCP. Корректный legacy результат не требует Docker. + +Перед публикацией повторно берётся nonblocking release lock: fingerprint, +serving identity и окончание перехода должны совпасть. Иначе unknown. Под lock +происходит только сравнение trusted state и атомарная запись, без сетевых +операций. Старый collector не может вернуть healthy после начавшегося switch. + +Controller атомарно инвалидирует state перед первым serving-affecting stop, +start/restart, upstream switch/restore, bootstrap или rollback side effect. +При ошибке записи пробует безопасно удалить точный старый файл, не трогая +чужие пути. Неудача обоих способов диагностируется, но не запрещает и не +откладывает owed recovery; мониторинг не имеет права блокировать восстановление. +При отказе storage старая корректная сводка может оставаться допустимой до +60s от своего наблюдения; это ограниченная устарелость, не гарантия мгновенной +актуальности. Её нельзя использовать как управляющий вход release или readiness. +Post-success не копирует COMMITTED в live; следующее наблюдение проверяет факт. + +## Freshness, сохранение и reader + +Root-owned parent0750 и regular file0640 с группой reader проверяются по +descriptor; symlink, hardlink count!=1, чужой owner или group/world write +отклоняются. Parent не writable reader/runtime. Temporary file создаётся в +том же parent c O_EXCL/no-follow, полное содержимое flush/fsync, rename и fsync +parent; в случае незавершённой записи reader видит прежний целый файл либо +unknown. Boot directory создаётся заново: persistent last-good не нужен. + +Reader требует совпадения invocation_id с текущим systemd INVOCATION_ID, +boot_id с host, собственного CLOCK_BOOTTIME и возраста0..60s; +mtime и wall-clock не продлевают доверие. Проверка выполняется при чтении и +непосредственно перед записью готовой публичной проекции. Если данные успели +устареть/измениться, uptime пересчитывается из нового валидного наблюдения +или заменяется на unknown, без повтора aggregation и без Docker-вызова. +Обнаружение clock inconsistency не даёт отрицательного/ложного uptime. + +Public mapping: статическая прежняя `service`; `active=live`, +`active_since`, `seconds=uptime_seconds`, `restarts`; `human` вычисляет прежний +normalizer. Выдаётся возраст на момент наблюдения, не экстраполяция после +возможной остановки. Private schema не передаётся через `dict.update` целиком. +`ready`, `reason`, boot/time identity и release ID в public fields не добавляются. +При неизвестном state нет fallback на active controller или старый healthy JSON. +Без нового CLI `--runtime-state-file` прежний standalone режим --service +сохраняется; production unit явно включает новый reader. + +## Выполнение sampler + +Фиксированная команда root из защищённого host tooling — единственный +ExecStartPre существующего monitoring job. Она делает только ограниченные +read-only наблюдения и запись state. Timer не меняется. User/Group основного +ExecStart остаются прежними; расширенного sudo verb или setuid-бинарника нет. +Команда не принимает произвольный path/target из stdin, CLI или environment. +Единственное входное значение от systemd — валидированный INVOCATION_ID, +который не выбирает путь или команду. Его отсутствие/невалидность даёт unknown. +Перед probe записывается unknown текущего invocation; завершённый результат +заменяет его. Даже SIGKILL до этой записи не позволяет reader принять файл +предыдущего запуска. Controller invalidation может писать invocation_id=null. + +Предлагается общий monotonic watchdog8s с reap потомков; один probe <=2s, +HTTP body <=16KiB, CLI output <=256KiB. Нет retry-loop и DNS/public запросов: +только доверенный local Docker и фиксированный loopback target. Timeout/error +пишет unknown с code; raw command output не печатается. При аварии helper +существующий unprivileged ExecStart должен всё равно запускаться и обработать +missing/stale input. Ошибки обнаруживаемы по private диагностике, не скрываются +как успешная проверка. Native systemd test обязан проверить credentials, +watchdog, failure continuation и сохранение sandbox; один YAML/string test +этого не доказывает. Ни loader360s, ни release300s budgets не меняются. +Watchdog находится вне probe worker, а его8s включают kill/reap; поток с +невозможностью прервать I/O не подходит. Завершение watchdog — завершение +отдельной `-!` pre-команды, а не истечение TimeoutStartSec всего monitoring job: +последнее могло бы не запустить reader и потому не удовлетворяет контракту. + +## Private usage stream + +Отдельный новый host-файл `/var/log/v8std-mcp/tool-usage.jsonl` создаёт root +до запуска container, owner10001, группа существующего reader,0640. +Parent root:reader0750; runtime получает file bind в +`/var/log/v8std-mcp/tool-usage.jsonl` и явный `--usage-log` этого path. +Только этот файл writable, не directory/raw legacy history или control state. +Формат payload остаётся `MCP_USAGE_EVENTS@1.0`. Encoded line ограничена64KiB, +одна append-write на строку; oversize/write failure не влияет на MCP ответ и +не вызывает неограниченные повторы. Межпроцессные строки не склеиваются при +штатном append; partial I/O остаётся явно best-effort telemetry. + +Новый logrotate stanza повторяет daily/365/compress/copytruncate; root +сохраняет точные owner/mode и inode file bind. Старый stanza и старая история +не меняются. Reader получает оба источника и их прежние rotation variants; +один inode не читается дважды через алиасы. Cleanup slot, rollback restore и +публикация corpus не владеют этими логами. Они недоступны из nginx/static tree. +Rotation/read race и copytruncate не дают exactly-once гарантии; этот пакет +не меняет историческую retention policy и не называет числа unique users. + +## Conformance и статус + +Обязательны fault cases: stale/wrong boot/clock/oversize/schema/types/owner/ +symlink, write failure/SIGKILL, container absent/foreign/unready/restarted, +collector-vs-switch race, rollback legacy и refresh без сброса process age. +Тесты разделяют real process/socket/UID проверки и substituted Docker faults. +Проверяется отрицательный доступ reader к state write/Docker/release root и +отсутствие private полей в public output. Реальный container writer/overlap/ +rotation/reader прогон доказывает передачу новых событий вместе с историей. +Нативная host-интеграция и server deployment — последующие внешние gates, +не результат принятия этого spec. Future tests ещё не реализованы. diff --git a/spec/designs/2026-09-15-mcp-container-monitor-input-design.md b/spec/designs/2026-09-15-mcp-container-monitor-input-design.md new file mode 100644 index 0000000..c80e1bd --- /dev/null +++ b/spec/designs/2026-09-15-mcp-container-monitor-input-design.md @@ -0,0 +1,201 @@ +--- +schema_version: 1 +kind: design +id: mcp-container-monitor-input +scope: product +requirements: + introduces: + - MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + - MCP_MONITOR_OBSERVES_SERVING_RUNTIME + - MCP_MONITOR_RETAINS_UNPRIVILEGED_READER + - MCP_CONTAINER_USAGE_SURVIVES_RELEASE + uses: + - MONITORING_REMAINS_PUBLIC + - LEGACY_USAGE_EVENTS_REMAIN_READABLE + - OPERATOR_DETAILS_STAY_OUTSIDE_WEB_ROOT + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + - MCP_ONE_LOGICAL_SERVICE_FROM_PUBLISHED_IMAGE + replaces: {} + cancels: [] +decisions: + - adr:MCP_PRIVATE_CONTAINER_MONITOR_INPUT +invariants: + - invariant:MCP_MONITOR_STATE_TRUST_IS_BOUNDED + - invariant:OPERATOR_DATA_STAYS_OUTSIDE_WEB_ROOT + - invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR +contracts: + - contract:MCP_CONTAINER_MONITOR_INPUT@1.0 + - contract:MCP_MONITORING_PROJECTION@1.0 + - contract:MCP_USAGE_EVENTS@1.0 + - contract:MCP_RELEASE_RUNTIME@1.0 +supersedes: [] +cancels: [] +--- + +# Вход мониторинга после контейнерной миграции + +## Согласованное направление и статус пакета + +Пользователь согласовал закрытый файл свежего состояния без доступа агрегатора +к Docker. Этот пакет уточняет обнаруженную в Task6 границу producer/consumer; +числа и интеграционные детали ниже предлагаются на письменное рассмотрение. +База: `f78b81c6b49d20102735fb0847fb726a06551691`, feature-ветка +`codex/mcp-container-distribution-design`. Реализация ещё не начата. + +Нет второго MCP, нового daemon/timer, public endpoint, версии событий, +редизайна dashboard или изменений модели индекса. Схема публичных полей остаётся +legacy. Этот пакет дополняет container design, а не заменяет его или старый +design будущей аналитики. Права на публикацию и production не расширяются. + +## Причина + +Сейчас `read_service_uptime()` читает `v8std-mcp.service`, который после +миграции остановлен. Новый launcher не передаёт `--usage-log`. Замена имени +unit на release-controller не помогает: COMMITTED означает результат операции, +а active wrapper — процесс контроллера, не жизнь MCP. `loaded_at` меняется +при refresh индекса и также не является временем старта процесса. + +Прежнее требование сохранить мониторинг верно, но его доверенная граница была +недоописана. Зелёные старые тесты не закрывают этот пробел. + +## Требования + +### MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + +Отчёт принимает только завершённое наблюдение текущего запуска monitoring job +и текущей загрузки host не старше60 секунд на момент публикации. Отсутствие, +повреждение, чужой владелец, +неподдерживаемая схема или истёкший возраст дают unknown, не healthy и не down. +Wall-clock перевод не делает старую запись свежей. Collector ограничен +8 секундами, включая ожидания/очистку; это предлагаемые бюджеты новой границы, +не изменение 360-секундного loader или release transaction. + +### MCP_MONITOR_OBSERVES_SERVING_RUNTIME + +Наблюдается только подтверждённый serving predecessor/active runtime, +сопоставленный с managed upstream, реальным процессом и ответами этого процесса. +Живой candidate, чужой контейнер, квитанция или ложный upstream не дают healthy. +Во время switch/неоконченного rollback допустим unknown. Stop/reboot/refresh +проверяются отдельно; обновление corpus не сбрасывает process uptime. + +### MCP_MONITOR_RETAINS_UNPRIVILEGED_READER + +Root выполняет только фиксированный bounded sampler. Агрегатор сохраняет +`v8std-mcp:v8std-mcp`, доступ adm к nginx logs и запись только своего output. +Ни агрегатор, ни MCP не получают Docker socket/group, sudo, возможность выбрать +root-команду, её путь, target или environment. Private state не является +управляющим входом release-controller. + +### MCP_CONTAINER_USAGE_SURVIVES_RELEASE + +Новые экземпляры пишут прежние события в отдельный persistent private log, +независимый от slots/cache. Агрегатор читает прежнюю историю и новый log; +перезапуск, rollback и очистка slot их не удаляют и не восстанавливают из backup. +Сохраняется best-effort характер telemetry и существующая политика +daily/365/copytruncate, без обещания нулевых потерь при copytruncate. + +## Компоненты и данные + +1. Существующий `v8std-mcp-monitoring.service` получает короткий root + `ExecStartPre` sampler. Его `ExecStart` по-прежнему запускает непривилегированный + агрегатор. Существующий timer и его пятиминутный период не меняются. +2. Sampler из root-owned `/opt/v8std-release/scripts/` читает ограниченную + выборку доверенного release state и проверяет runtime. Пишет только + `/run/v8std-monitor/state.json`, атомарно и без пользовательских payload. +3. Reader читает этот файл, повторно проверяет свежесть непосредственно перед + публикацией и передаёт только прежние uptime-поля в legacy renderer. +4. Runtime получает writable bind только отдельного файла + `/var/log/v8std-mcp/tool-usage.jsonl`, не каталога state/cache/логов host. + Старый `/var/lib/v8std-mcp/tool-usage.jsonl` и его owner не меняются. + +Один общий новый log допустим для ограниченного old/new overlap, поскольку +события не являются управляющими данными. Запись одной сериализованной строки +делается одной ограниченной append-операцией; частичные строки остаются +ошибками telemetry, а не ошибками MCP. Нельзя обещать exactly-once или считать +internal smoke calls отдельными людьми. Межпроцессный append проверяется +реальными параллельными writers, а его стоимость входит в общий mixed-load. + +## Привилегии и установка + +Для единственной root-команды выбирается `ExecStartPre` с префиксом `-!`, а не +`+`: первый меняет credentials, не снимая остальные sandbox-настройки unit. +`-` позволяет основному reader запуститься после отказа sampler; несовпадающий +с текущим systemd invocation ID файл тогда не принимается даже внутри TTL. +Смысл проверен по [systemd v255](https://github.com/systemd/systemd/blob/v255/man/systemd.service.xml). +Нативная проверка установленной версии systemd остаётся gate перед активацией. +Root sampler не импортирует код из writable каталогов; использует isolated +Python, фиксированный PATH и endpoint Docker, без наследуемых Docker/Python +параметров. Обычный пользователь не может изменять unit или sampler. + +`/run/v8std-monitor` создаётся root при host setup/boot с mode0750 и группой +`v8std-mcp`; файл root:`v8std-mcp`0640. Добавление каталога в ReadWritePaths +не даёт агрегатору DAC write. Root release state остаётся0700. Raw log имеет +UID10001, группу `v8std-mcp` и0640, parent root:`v8std-mcp`0750. Числовой GID +группы определяется на host; runtime остаётся10001:10001 и пишет как owner. +nginx/CI не входят в эту группу. Проверяется отсутствие иных путей доступа. + +Новый monitor code устанавливается как host tooling вместе с controller, +не зависит от изменяемого container cache. Python runtime старого сервиса +остаётся доступным для rollback. Установка/boot directory rules, unit и +logrotate policy входят в будущий локальный plan; применение на host отдельно. + +## Совместимость и ошибки + +Точные поля, идентичность и invalidation определяет +[private input contract](../contracts/mcp-container-monitor-input-v1-r0.md). +`uptime.service` сохраняет логическую legacy-метку; image/port/path/journal +не добавляются в публичный JSON. `active` остаётся liveness, не readiness. +Readiness и failure reason доступны только в private state. + +Для container `restarts` возвращается `null`: надёжного общего счётчика +Docker-policy и ручных/controller перезапусков сейчас нет. `RestartCount` +нельзя выдавать за их сумму, а пропущенные между polls рестарты — считать +нулём. Legacy сохраняет `NRestarts`. Поле и допустимый тип сохраняются; потеря +числовой метрики при смене backend явно принята к рассмотрению, не спрятана. +Введение точного lifecycle counter требует отдельного решения и не входит сюда. + +Ошибка sampler даёт unknown и краткую диагностику без raw output. Агрегатор +продолжает строить статистику; невозможность его публикации сохраняет прошлый +артефакт с прежним `generated_at`. Batch-страница — наблюдение на указанное +время, не real-time alert: истечение TTL файла не изменяет уже отданный HTML. +State не разрешает и не запрещает recovery. Перед serving mutation controller +инвалидирует его, но ошибка invalidation не задерживает аварийный возврат: +остаётся явная диагностика и верхняя граница доверия60s. Принципиально нельзя +обещать мгновенную актуальность статического отчёта при отказе его storage. + +У текущего renderer есть отдельная известная публикация поисковых текстов +(`search_events`, HTML и stats.json). Private state этих данных не содержит; +этот пакет не объявляет реализованным общий privacy invariant будущего +dashboard и не отменяет его. Перед публичным выпуском нужна отдельная +диспозиция этого риска; сохранение схемы не является разрешением на утечку. + +## Проверки и включение в оставшуюся работу + +Будущая conformance проверяет fresh/stale/boot/schema/owner/symlink/oversize, +живой candidate вместо serving, missing/foreign/restarted контейнер, +раздельные live/ready, refresh без сброса uptime, collector во время switch и +SIGKILL между invalidation и side effect. Сбой чтения не вызывает Docker от +имени агрегатора. Реальный launcher/logger/rotation/aggregator проверяется +с UID10001 и прежней историей, включая overlap и rollback. + +После письменного рассмотрения пакета writing-plans создаёт отдельный +незавершённый plan этого среза, связанный с Task6 основного container plan: +контракт/reader → sampler/invalidation → logs/unit integration → совместимость. +Только после его реализации и review возобновляется оставшаяся Task6 CI. +Ранее пройденные тесты относятся к прежнему коду и не принимают этот design. +Ни весь container design, ни внешние release/capacity gates не становятся +IMPLEMENTED от завершения этого небольшого среза. + +## Отклонённые варианты + +- Docker socket/group или произвольный sudo у reader: новая избыточная власть. +- Фиксированный sudo RPC: возможен, но создаёт вызываемую привилегированную + поверхность; для batch-reader достаточно файла. +- Сбор только во время deploy/recover: связывает свежесть с ремонтом runtime, + а не генерацией мониторинга; требует отдельного расписания или budget coupling. +- Новый daemon/exporter: не требуется для сохранения текущего batch-dashboard. + +Semantic impact: добавлена private boundary и уточнены обязанности freshness/ +runtime identity; существующие требования/ADR/invariants сохраняются, ничего +не отменяется. В этом шаге создаются только новые spec-файлы; structured main, +runtime, units, permissions и production не меняются. diff --git a/spec/invariants/mcp-monitor-state-trust-is-bounded.md b/spec/invariants/mcp-monitor-state-trust-is-bounded.md new file mode 100644 index 0000000..cd0060a --- /dev/null +++ b/spec/invariants/mcp-monitor-state-trust-is-bounded.md @@ -0,0 +1,32 @@ +--- +schema_version: 1 +kind: invariant +id: MCP_MONITOR_STATE_TRUST_IS_BOUNDED +scope: product +introduced_by: adr:MCP_PRIVATE_CONTAINER_MONITOR_INPUT +requirements: + - MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + - MCP_MONITOR_OBSERVES_SERVING_RUNTIME + - MCP_MONITOR_RETAINS_UNPRIVILEGED_READER +owner: v8std maintainers +governs: + - scripts/v8std_mcp_monitor_state.py + - scripts/v8std_mcp_monitoring.py + - scripts/v8std_mcp_release.py + - deploy/container +check: + module: tests.test_v8std_mcp_monitor_state + command: .venv/bin/python -m unittest tests.test_v8std_mcp_monitor_state -v +required_when: implemented +--- + +# Состояние не переживает границу доверия + +Положительная liveness допустима только для свежего root-owned наблюдения +текущей загрузки host, идентичность процесса которого совпала с serving +runtime. Просрочка, race переключения или повреждение не превращаются в успех. +Reader не может подделать сводку, управлять Docker или публиковать её целиком. + +Fitness включает реального непривилегированного reader, race collector/switch, +boot/clock/freshness/identity faults и отсутствие private fields в projection. +Декларация будущего теста не является выполненным доказательством. From 947ae9ab2d9ce0a02cc92ea50a3cd8667a298068 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:27:35 +0300 Subject: [PATCH 42/88] docs: record approved public monitoring retirement --- ...2026-09-15-retire-public-mcp-monitoring.md | 48 ++++++++ .../mcp-monitoring-projection-v3-r0.md | 30 +++++ spec/contracts/mcp-usage-events-v1-r1.md | 31 +++++ ...mcp-public-monitoring-retirement-design.md | 112 ++++++++++++++++++ ...ivate-operational-data-is-not-published.md | 24 ++++ .../public-monitoring-routes-are-gone.md | 19 +++ ...5-mcp-public-monitoring-retirement-plan.md | 93 +++++++++++++++ 7 files changed, 357 insertions(+) create mode 100644 spec/adr/2026-09-15-retire-public-mcp-monitoring.md create mode 100644 spec/contracts/mcp-monitoring-projection-v3-r0.md create mode 100644 spec/contracts/mcp-usage-events-v1-r1.md create mode 100644 spec/designs/2026-09-15-mcp-public-monitoring-retirement-design.md create mode 100644 spec/invariants/private-operational-data-is-not-published.md create mode 100644 spec/invariants/public-monitoring-routes-are-gone.md create mode 100644 spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md diff --git a/spec/adr/2026-09-15-retire-public-mcp-monitoring.md b/spec/adr/2026-09-15-retire-public-mcp-monitoring.md new file mode 100644 index 0000000..3e6b326 --- /dev/null +++ b/spec/adr/2026-09-15-retire-public-mcp-monitoring.md @@ -0,0 +1,48 @@ +--- +schema_version: 1 +kind: adr +id: RETIRE_PUBLIC_MCP_MONITORING +scope: product +design: design:mcp-public-monitoring-retirement +requirements: + - PUBLIC_MONITORING_IS_RETIRED + - PRIVATE_USAGE_HISTORY_IS_PRESERVED + - MONITORING_RETIREMENT_PRESERVES_MCP + - OPERATOR_DETAILS_STAY_OUTSIDE_WEB_ROOT +aliases: [] +supersedes: [adr:PUBLIC_MCP_MONITORING] +cancels: [adr:MCP_PRIVATE_CONTAINER_MONITOR_INPUT] +invariants: + introduces: + - invariant:PUBLIC_MONITORING_ROUTES_ARE_GONE + - invariant:PRIVATE_OPERATIONAL_DATA_IS_NOT_PUBLISHED + preserves: [invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR] + replaces: + invariant:PUBLIC_MONITORING_EXCLUDES_SENSITIVE_DATA: invariant:PUBLIC_MONITORING_ROUTES_ARE_GONE + invariant:OPERATOR_DATA_STAYS_OUTSIDE_WEB_ROOT: invariant:PRIVATE_OPERATIONAL_DATA_IS_NOT_PUBLISHED + cancels: [invariant:MCP_MONITOR_STATE_TRUST_IS_BOUNDED] +contracts: + introduces: + - contract:MCP_MONITORING_PROJECTION@3.0 + - contract:MCP_USAGE_EVENTS@1.1 + preserves: [contract:MCP_RELEASE_RUNTIME@1.0] + replaces: + contract:MCP_MONITORING_PROJECTION@1.0: contract:MCP_MONITORING_PROJECTION@3.0 + contract:MCP_MONITORING_PROJECTION@2.0: contract:MCP_MONITORING_PROJECTION@3.0 + contract:MCP_USAGE_EVENTS@1.0: contract:MCP_USAGE_EVENTS@1.1 + cancels: + - contract:MCP_USAGE_EVENTS@2.0 + - contract:MCP_CONTAINER_MONITOR_INPUT@1.0 +--- + +# Прекратить публичную публикацию мониторинга + +Пользователь отменил публичный dashboard целиком. Останавливаем и маскируем +его job, удаляем publication code и alias, оставляем HTTP410 tombstones. +Root-only архив обеспечивает восстановимость без сохранения публичного доступа. +Ни аутентифицированный dashboard, ни новый sampler взамен не создаются. + +Существующие private logs, их формат/rotation и MCP readiness остаются. +Ненужное сохранение public projection исключается из container release; +постоянство закрытых usage events нового runtime остаётся release-обязательством. +Локальные OpenMetrics — независимое от этого решения принятое намерение. diff --git a/spec/contracts/mcp-monitoring-projection-v3-r0.md b/spec/contracts/mcp-monitoring-projection-v3-r0.md new file mode 100644 index 0000000..ea98a32 --- /dev/null +++ b/spec/contracts/mcp-monitoring-projection-v3-r0.md @@ -0,0 +1,30 @@ +--- +schema_version: 1 +kind: contract +id: MCP_MONITORING_PROJECTION +scope: product +version: 3 +revision: 0 +compatibility: breaking +design: design:mcp-public-monitoring-retirement +producer: nginx edge +consumers: [former public monitoring clients] +requirements: [PUBLIC_MONITORING_IS_RETIRED] +governs: [deploy/container/edge-locations.conf] +conformance: + module: tests.test_v8std_mcp_monitoring_retirement + command: V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement -v +required_when: implemented +supersedes: + - contract:MCP_MONITORING_PROJECTION@1.0 + - contract:MCP_MONITORING_PROJECTION@2.0 +deprecates: [] +--- + +# Retired monitoring HTTP boundary + +HTTPS GET/HEAD `/monitoring`, `/monitoring/` и любой дочерний path отвечают410 +с `Cache-Control: no-store`. Query parameters не меняют результат. +Тело не содержит dashboard data; HEAD без тела. Другие методы также не +возвращают старые данные. HTTP может пройти существующий TLS redirect. +Публичного JSON и заменяющего его отчёта больше нет. diff --git a/spec/contracts/mcp-usage-events-v1-r1.md b/spec/contracts/mcp-usage-events-v1-r1.md new file mode 100644 index 0000000..71fc4fe --- /dev/null +++ b/spec/contracts/mcp-usage-events-v1-r1.md @@ -0,0 +1,31 @@ +--- +schema_version: 1 +kind: contract +id: MCP_USAGE_EVENTS +scope: product +version: 1 +revision: 1 +compatibility: backward-compatible +design: design:mcp-public-monitoring-retirement +producer: current MCP logger and nginx +consumers: [private operator diagnostics] +requirements: [PRIVATE_USAGE_HISTORY_IS_PRESERVED] +governs: + - scripts/v8std_mcp_server.py + - scripts/v8std_mcp_usage.logrotate +conformance: + module: tests.test_v8std_mcp_server + command: .venv/bin/python -m unittest tests.test_v8std_mcp_server -v +required_when: accepted +supersedes: [contract:MCP_USAGE_EVENTS@1.0] +deprecates: [] +--- + +# Private legacy usage events + +Существующий JSONL формат с `ts`/`tool` и необязательными metadata не меняется. +Отсутствие `schema_version` и `api` остаётся допустимым. Существующие файлы, +rotation и текущие writer settings не переписываются при retirement. +Потребитель теперь только операторская диагностика; публичный aggregator +удалён. Эта совместимая ревизия переносит conformance на действующий logger, +не обещает новую схему, reader либо exactly-once запись. diff --git a/spec/designs/2026-09-15-mcp-public-monitoring-retirement-design.md b/spec/designs/2026-09-15-mcp-public-monitoring-retirement-design.md new file mode 100644 index 0000000..cc31e54 --- /dev/null +++ b/spec/designs/2026-09-15-mcp-public-monitoring-retirement-design.md @@ -0,0 +1,112 @@ +--- +schema_version: 1 +kind: design +id: mcp-public-monitoring-retirement +scope: product +requirements: + introduces: + - PUBLIC_MONITORING_IS_RETIRED + - PRIVATE_USAGE_HISTORY_IS_PRESERVED + - MONITORING_RETIREMENT_PRESERVES_MCP + uses: + - OPERATOR_DETAILS_STAY_OUTSIDE_WEB_ROOT + - MCP_RELEASE_SWITCH_IS_REVERSIBLE + replaces: + MONITORING_REMAINS_PUBLIC: PUBLIC_MONITORING_IS_RETIRED + PUBLIC_MONITORING_EXCLUDES_SENSITIVE_DATA: PUBLIC_MONITORING_IS_RETIRED + LEGACY_USAGE_EVENTS_REMAIN_READABLE: PRIVATE_USAGE_HISTORY_IS_PRESERVED + MCP_CONTAINER_USAGE_SURVIVES_RELEASE: PRIVATE_USAGE_HISTORY_IS_PRESERVED + cancels: + - MONITORING_SHOWS_AGENT_FAMILIES + - MONITORING_SHOWS_API_VERSIONS + - MONITORING_SHOWS_MCP_OPERATIONS + - MCP_MONITOR_STATE_HAS_BOUNDED_FRESHNESS + - MCP_MONITOR_OBSERVES_SERVING_RUNTIME + - MCP_MONITOR_RETAINS_UNPRIVILEGED_READER +decisions: [adr:RETIRE_PUBLIC_MCP_MONITORING] +invariants: + - invariant:PUBLIC_MONITORING_ROUTES_ARE_GONE + - invariant:PRIVATE_OPERATIONAL_DATA_IS_NOT_PUBLISHED + - invariant:MCP_RELEASE_HAS_RECOVERABLE_PREDECESSOR +contracts: + - contract:MCP_MONITORING_PROJECTION@3.0 + - contract:MCP_USAGE_EVENTS@1.1 + - contract:MCP_RELEASE_RUNTIME@1.0 +supersedes: + - design:mcp-monitoring-dashboard + - design:mcp-container-monitor-input +cancels: [] +--- + +# Полное отключение публичного мониторинга + +## Согласование и границы + +Пользователь запросил полное удаление публичного мониторинга и подтвердил +предложенный письменный scope словом «согл» 2026-09-15: отключить job/timer, +убрать HTML/JSON и alias, вернуть 410, удалить генератор и ссылки, отменить +старые требования и исключить сохранение dashboard из container release. +Подтверждение явно включает production. Существующие файлы сначала переносятся +в закрытый архив. Это фиксация согласованного решения, не расширение полномочий. + +Не входят: MCP deploy/restart, push, публикация Docker, удаление raw logs, +чужих vhosts, TLS/renewal, SSH или fail2ban. Health/readiness, закрытая +диагностика и recoverable release сохраняются. Отдельный design локальных +OpenMetrics не отменяется и не реализуется этой работой. + +## Требования + +### PUBLIC_MONITORING_IS_RETIRED + +На production и в следующей поставке `/monitoring` и всё `/monitoring/` +возвращают HTTP410 с `Cache-Control: no-store`, без прежнего отчёта или redirect +на него. Генератор, timer/service и deploy-ссылки больше не публикуют dashboard. +Прежние файлы не остаются ни в web root, ни за другим alias. + +### PRIVATE_USAGE_HISTORY_IS_PRESERVED + +Существующие access/usage logs и rotation сохраняются без переписывания истории +или изменения формата. Отмена aggregator не отменяет закрытые JSONL-события. +Container migration не может удалить прежние журналы; persistent private +usage logging для нового runtime остаётся отдельной проверкой выпуска. +Новый sampler, dashboard reader и его log-to-public pipeline не создаются. + +### MONITORING_RETIREMENT_PRESERVES_MCP + +Изменяются только мониторинговые unit/files и два nginx location. +После успешного `nginx -t` используется reload. PID и время старта MCP +не меняются; health и реальные initialize/tools/list остаются успешными. +Архив root-owned0700 вне web root содержит оригинальную конфигурацию, +unit/drop-in, генератор и опубликованные файлы для ручного восстановления. + +## Решение и альтернативы + +Выбрано полное прекращение публикации, а не скрытие ссылки, `noindex` или +пароль поверх существующего генератора. Удаление только HTML оставило бы +публичный JSON и timer, воссоздающий оба файла. Поэтому отключается вся цепочка. +Архив не обслуживается HTTP и не включается в Git или Docker context. +Уже скачанные сторонними клиентами копии отозвать невозможно; прежний origin +отдавал max-age60, после смены отдаёт no-store. + +Legacy projection и её не реализованный редизайн заменены tombstone-контрактом. +Dashboard-specific v2 events и private monitor-input contract отменены: +их producer/consumer ещё не реализованы и больше не нужны этому dashboard. +Это не удаляет нынешний logger и не запрещает отдельное будущее проектирование +закрытой телеметрии. Privacy-инвариант оператора заменён эквивалентным вне +зависимости от удалённого renderer; принятую историю в main не переписываем. + +## Проверки и восстановление + +До изменения фиксируются HTTP-коды, unit state, MCP PID/start и права журналов. +После остановки timer/service оригиналы перемещаются в root-only archive; +оба unit маскируются от случайного запуска. Nginx получает только два tombstone +location; при ошибке `nginx -t` исходный конфиг восстанавливается без reload. +Проверяются GET/HEAD, JSON/query/unknown descendant, отсутствие генератора и +публичного каталога, masked/inactive units, сохранение процесса и MCP ответов. +Локальный настоящий nginx проверяет shipped include, в том числе при лежащих +под ним старых файлах и при недоступном MCP upstream. + +Rollback конфигурации и файлов технически возможен из архива, но повторная +публикация мониторинга требует нового явного решения пользователя. Для ремонта +nginx сначала сохраняется закрытая граница, а не автоматически возвращается +утечка. Изменение не доказывает производительность на100000 подключений. diff --git a/spec/invariants/private-operational-data-is-not-published.md b/spec/invariants/private-operational-data-is-not-published.md new file mode 100644 index 0000000..cabc810 --- /dev/null +++ b/spec/invariants/private-operational-data-is-not-published.md @@ -0,0 +1,24 @@ +--- +schema_version: 1 +kind: invariant +id: PRIVATE_OPERATIONAL_DATA_IS_NOT_PUBLISHED +scope: product +introduced_by: adr:RETIRE_PUBLIC_MCP_MONITORING +requirements: + - OPERATOR_DETAILS_STAY_OUTSIDE_WEB_ROOT + - PRIVATE_USAGE_HISTORY_IS_PRESERVED +governs: + - deploy/container/edge-locations.conf + - scripts/v8std_mcp_server.py +check: + module: tests.test_v8std_mcp_monitoring_retirement + command: V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement tests.test_v8std_mcp_server -v +required_when: implemented +--- + +# Операционные данные остаются закрытыми без dashboard + +Raw logs и архив вне публикуемого дерева; ни root archive, ни usage log не +имеют HTTP alias. Retired routes не отдают существовавшие HTML/JSON даже при +случайном возврате старых файлов. Проверка host ACL и отсутствия alias +фиксируется в operations evidence; logger conformance остаётся отдельной. diff --git a/spec/invariants/public-monitoring-routes-are-gone.md b/spec/invariants/public-monitoring-routes-are-gone.md new file mode 100644 index 0000000..9461976 --- /dev/null +++ b/spec/invariants/public-monitoring-routes-are-gone.md @@ -0,0 +1,19 @@ +--- +schema_version: 1 +kind: invariant +id: PUBLIC_MONITORING_ROUTES_ARE_GONE +scope: product +introduced_by: adr:RETIRE_PUBLIC_MCP_MONITORING +requirements: [PUBLIC_MONITORING_IS_RETIRED] +governs: [deploy/container/edge-locations.conf] +check: + module: tests.test_v8std_mcp_monitoring_retirement + command: V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement -v +required_when: implemented +--- + +# Dashboard не возвращается при следующей поставке + +Точный `/monitoring` и prefix `/monitoring/` отвечают410/no-store независимо +от наличия старых файлов и доступности MCP upstream. Необходимы реальные +HTTP-проверки shipped nginx include, не поиск строки в конфигурации. diff --git a/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md new file mode 100644 index 0000000..4cce6fa --- /dev/null +++ b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md @@ -0,0 +1,93 @@ +--- +schema_version: 1 +kind: plan +id: mcp-public-monitoring-retirement +design: design:mcp-public-monitoring-retirement +implements: + - design:mcp-public-monitoring-retirement + - adr:RETIRE_PUBLIC_MCP_MONITORING + - invariant:PUBLIC_MONITORING_ROUTES_ARE_GONE + - invariant:PRIVATE_OPERATIONAL_DATA_IS_NOT_PUBLISHED + - contract:MCP_MONITORING_PROJECTION@3.0 + - contract:MCP_USAGE_EVENTS@1.1 +--- + +# Public monitoring retirement implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Remove public monitoring without disturbing MCP or private operational evidence. + +**Architecture:** Retire producer and public artifacts; nginx keeps explicit410 tombstones. Preserve private logging and the independent release controller. Record recoverable production retirement separately from MCP deployment. + +**Tech Stack:** Python unittest, nginx, systemd, SSH, local Docker fixture. + +**Spec:** `spec/designs/2026-09-15-mcp-public-monitoring-retirement-design.md` + +## Global Constraints + +- HTTPS `/monitoring` and all `/monitoring/` return410 with `Cache-Control: no-store`. +- Do not restart/deploy MCP, push, publish Docker, remove logs/rotation or alter other vhosts/TLS/SSH/fail2ban. +- Production removal is separately authorized by the user's2026-09-15 confirmation; preserve root-owned0700 archive outside web root. +- No new dashboard, sampler or monitoring privileges. Private logger and health/readiness remain. +- Main structured documents remain byte-identical; change their lifecycle through successors. +- Existing container plan remains incomplete; this retirement does not authorize integration of the whole branch. + +## Operational execution (controller, separate from implementation checkboxes) + +1. Capture public410-vs-current regression, MCP health/PID/start, exact nginx realpath and units. +2. Create root-only archive with `mktemp -d /root/v8std-monitoring-retired-XXXXXXXX`; copy original nginx config there. Disable/stop exact timer and service. +3. Patch only the two existing monitoring locations to the410 fragment below using a local candidate and `apply_patch`; transfer it into the archive. Validate exact original checksum before installing. Run `nginx -t`; if it fails restore original config without reload. Reload nginx only after successful validation. +4. Move the exact monitoring units/drop-in, generator and web directory into the archive; daemon-reload; mask both retired units. Do not remove the harmless legacy MCP Before= dependency or restart its process. +5. Verify GET/HEAD410/no-store for exact/slash/JSON/query/unknown child paths; masked inactive units, absent public directory and renderer; original MCP PID/start, health and initialize/tools/list. Check raw-log modes/rotation unchanged. Record archive path and commands in operations evidence without raw log or dashboard contents. + +### Task 1: Remove the publication path and lock the retired HTTP boundary + +**Files:** Delete `scripts/v8std_mcp_monitoring.py`, `tests/test_v8std_mcp_monitoring.py`; modify `deploy/container/edge-locations.conf`, `tests/test_v8std_architecture_repository.py`; create `tests/test_v8std_mcp_monitoring_retirement.py`. + +**Interfaces:** Consume the shipped `edge-locations.conf` using a real local nginx fixture (pinned local nginx image from `tests/test_v8std_mcp_release_docker.py`); produce opt-in HTTP conformance with `V8STD_MONITORING_RETIREMENT_DOCKER=1`. No host SSH or production work in the subagent. + +- [ ] **RED:** Start a disposable nginx with shipped include, bounded timeouts, dead upstream, legacy `monitoring/index.html` and `stats.json` containing a sentinel under fixture root. Probe GET/HEAD paths `/monitoring`, `/monitoring/`, `/monitoring/stats.json?x=1`, `/monitoring/unknown`. Assert literal410/no-store and absent sentinel. Before implementation existing config fails this boundary. Use loopback-only published port and exact UUID-owned container cleanup; never prune or pull. Preserve health/index boundaries in the fixture. Add architecture lifecycle expectations that old dashboard/input designs are superseded and OpenMetrics remains accepted. + +```python +self.assertEqual(status, 410) +self.assertEqual(headers.get('cache-control'), 'no-store') +self.assertNotIn(b'private-monitoring-sentinel', body) +``` + +- [ ] **GREEN:** Remove generator and its dedicated tests; leave logger/server/logrotate unchanged. Add these locations to shipped include. Update architecture repository expectations for the successor graph; preserve historical aliases. + +```nginx +location = /monitoring { + add_header Cache-Control "no-store" always; + return 410; +} +location ^~ /monitoring/ { + add_header Cache-Control "no-store" always; + return 410; +} +``` + +- [ ] **VERIFY:** Run real nginx conformance and existing private logger tests. Record RED/GREEN output and prove owned fixture cleanup. Search active publication surfaces for remaining renderer/monitoring links; no source-text-only substitute for HTTP behavior. + +```sh +V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement -v +.venv/bin/python -m unittest tests.test_v8std_mcp_server tests.test_v8std_architecture_repository -v +``` + +### Task 2: Reconcile release obligations and record evidence + +**Files:** Modify candidate-only `spec/plans/2026-09-10-mcp-container-distribution-plan.md`, `spec/designs/2026-09-10-mcp-container-distribution-design.md`, `spec/operations/mcp-container-activation.md`, `spec/operations/mcp-container-verification.md`, `spec/operations/mcp-first-container-release-roadmap.md`; create `spec/operations/2026-09-15-public-monitoring-retirement.md`. + +**Interfaces:** Consume Task1 tests and controller's actual production evidence. Produce unambiguous current runbooks, retaining dated historical observations as history. + +- [ ] Replace the container plan's public-monitor preservation step with retirement evidence plus persistent private logging/rotation checks for the new runtime (remaining incomplete until actually verified). Remove generator/tests from active test commands; replace with retirement conformance. Do not claim new container telemetry has been implemented. +- [ ] Clarify preserve-monitoring prose to mean private logs and health/readiness; link the retirement decision for old dated statements. Record exact production archive/PID/codes/unit states with no payloads. Scan `docs`, `.github`, `deploy`, scripts and current runbooks; historical frozen design references remain as evidence, not active instructions. +- [ ] Run semantic impact and ordinary architecture validation, strict build then full suite once. Record existing whole-branch release gates separately; no merge-ready or production-release claim from this scoped completion. + +```sh +.venv/bin/python scripts/v8std_architecture.py impact --root . --base-ref main +.venv/bin/python scripts/v8std_architecture.py validate --root . +VIRTUAL_ENV="$PWD/.venv" ./scripts/zensical_docs.sh build --strict +.venv/bin/python -m unittest discover -s tests -v +``` From 12ffc558ba7abc08120edef01bac443877019819 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:33:14 +0300 Subject: [PATCH 43/88] fix: retire public MCP monitoring publication --- deploy/container/edge-locations.conf | 8 + scripts/v8std_mcp_monitoring.py | 1075 ----------------- tests/test_v8std_architecture_repository.py | 5 +- tests/test_v8std_mcp_monitoring.py | 281 ----- tests/test_v8std_mcp_monitoring_retirement.py | 198 +++ 5 files changed, 209 insertions(+), 1358 deletions(-) delete mode 100644 scripts/v8std_mcp_monitoring.py delete mode 100644 tests/test_v8std_mcp_monitoring.py create mode 100644 tests/test_v8std_mcp_monitoring_retirement.py diff --git a/deploy/container/edge-locations.conf b/deploy/container/edge-locations.conf index cbbd264..334b60e 100644 --- a/deploy/container/edge-locations.conf +++ b/deploy/container/edge-locations.conf @@ -1,5 +1,13 @@ # Include in the separately inventoried ai.v8std.ru TLS server; do not replace # default vhosts, certificate configuration or renewal during release. +location = /monitoring { + add_header Cache-Control "no-store" always; + return 410; +} +location ^~ /monitoring/ { + add_header Cache-Control "no-store" always; + return 410; +} location = /mcp { if ($request_method !~ ^(POST|HEAD)$) { return 405; } limit_conn v8std_active 8; diff --git a/scripts/v8std_mcp_monitoring.py b/scripts/v8std_mcp_monitoring.py deleted file mode 100644 index 5bdf6ac..0000000 --- a/scripts/v8std_mcp_monitoring.py +++ /dev/null @@ -1,1075 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import gzip -import html -import json -import shlex -import subprocess -from collections import Counter -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Iterable, Iterator - - -DEFAULT_ACCESS_LOG = Path("/var/log/nginx/ai.v8std.ru.access.log") -DEFAULT_USAGE_LOG = Path("/var/lib/v8std-mcp/tool-usage.jsonl") -DEFAULT_OUTPUT_DIR = Path("/var/www/ai.v8std.ru-monitoring") -DEFAULT_SERVICE = "v8std-mcp.service" -DEFAULT_WINDOW_HOURS = 24 -TOP_RANKING_LIMIT = 50 -RECENT_SEARCH_LIMIT = 10 -MAX_SEARCH_RESULTS_PER_QUERY = 50 - -SYSTEM_LABELS = { - "codex": "Codex", - "claude": "Claude", - "cursor": "Cursor", - "jetbrains": "JetBrains", - "vscode": "VS Code", - "monitoring": "Monitoring", - "curl": "curl", - "browser": "Browser", - "node": "Node", - "opencode": "opencode", - "go": "Go", - "python_httpx": "Python httpx", - "kilo": "Kilo", - "java": "Java", - "unknown": "Unknown", - "other": "Other", -} -UNIDENTIFIED_CLIENT_SYSTEMS = {"unknown", "other"} - -TOOL_LABELS = { - "v8std_search": "v8std_search", - "v8std_get_page": "v8std_get_page", - "v8std_get_related": "v8std_get_related", - "v8std_explain_snippet": "v8std_explain_snippet", - "v8std_explain_diagnostics": "v8std_explain_diagnostics", -} - -IGNORED_NON_MCP_PATHS = {"/", "/healthz", "/version", "/monitoring"} - -def parse_log_line(line: str) -> dict[str, str] | None: - try: - parts = shlex.split(line) - except ValueError: - return None - - fields: dict[str, str] = {} - for part in parts: - if "=" not in part: - continue - key, value = part.split("=", 1) - fields[key] = value - return fields if "ts" in fields else None - - -def parse_iso_datetime(value: str) -> datetime | None: - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - if parsed.tzinfo is None: - return parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) - - -def is_mcp_request(uri: str) -> bool: - return uri.split("?", 1)[0] in {"/mcp", "/mcp/"} - - -def is_ignored_non_mcp_request(uri: str) -> bool: - path = uri.split("?", 1)[0] - return path in IGNORED_NON_MCP_PATHS or path.startswith("/monitoring/") - - -def classify_other_request(method: str, uri: str, status: int | None) -> str: - path = uri.split("?", 1)[0] or "/" - if len(path) > 80: - path = f"{path[:77]}..." - status_text = str(status) if status is not None else "unknown" - return f"{method.upper() or 'UNKNOWN'} {path} -> {status_text}" - - -def is_rate_limited(fields: dict[str, str], status: int | None) -> bool: - uri = fields.get("uri", "") - upstream_time = fields.get("upstream_time", "") - return status in {429, 503} and uri.split("?", 1)[0] in {"/mcp", "/mcp/"} and upstream_time in {"", "-"} - - -def parse_usage_line(line: str) -> dict[str, object] | None: - try: - payload = json.loads(line) - except json.JSONDecodeError: - return None - if not isinstance(payload, dict): - return None - ts = payload.get("ts") - tool = payload.get("tool") - if not isinstance(ts, str) or not isinstance(tool, str): - return None - return payload - - -def public_text(value: object, *, limit: int = 240) -> str | None: - if not isinstance(value, str): - return None - text = " ".join(value.split()) - if not text: - return None - return text[:limit] - - -def public_system(value: object) -> str: - system = public_text(value, limit=80) - if system is None: - return "unknown" - system = system.lower() - if system in SYSTEM_LABELS: - return system - return "unknown" - - -def public_url(value: object) -> str | None: - url = public_text(value, limit=500) - if url is None or not url.startswith("https://v8std.ru/"): - return None - return url - - -def public_result(value: object) -> dict[str, str] | None: - if not isinstance(value, dict): - return None - url = public_url(value.get("url")) - if url is None: - return None - result = {"url": url} - item_id = public_text(value.get("id"), limit=120) - title = public_text(value.get("title")) - if item_id: - result["id"] = item_id - if title: - result["title"] = title - return result - - -def public_ranked_item(value: object) -> dict[str, str] | None: - if not isinstance(value, dict): - return None - raw_url = public_text(value.get("url"), limit=500) - url = public_url(value.get("url")) - if raw_url and url is None: - return None - item_id = public_text(value.get("id"), limit=120) - code = public_text(value.get("code"), limit=120) - title = public_text(value.get("title")) - result: dict[str, str] = {} - if item_id: - result["id"] = item_id - elif code: - result["id"] = code - if title: - result["title"] = title - if url: - result["url"] = url - return result or None - - -def public_frequency(value: object) -> int: - if isinstance(value, int) and not isinstance(value, bool): - return max(1, min(value, TOP_RANKING_LIMIT * 10)) - return 1 - - -def add_diagnostic_ranking_item( - requests: Counter[str], - metadata: dict[str, dict[str, str]], - *, - item_id: str, - title: str, - frequency: int, - kind: str, - url: str = "", -) -> None: - key = f"{kind}:{url or item_id or title}" - requests[key] += frequency - metadata.setdefault( - key, - { - "key": key, - "id": item_id, - "title": title, - "url": url, - "kind": kind, - }, - ) - - -def human_duration(seconds: int | float | None) -> str: - if seconds is None: - return "unknown" - total_minutes = max(0, int(seconds) // 60) - days, day_minutes = divmod(total_minutes, 24 * 60) - hours, minutes = divmod(day_minutes, 60) - if days: - return f"{days}d {hours}h" - if hours: - return f"{hours}h {minutes}m" - return f"{minutes}m" - - -def normalize_uptime(uptime: dict[str, object] | None) -> dict[str, object]: - payload = { - "service": DEFAULT_SERVICE, - "active": None, - "active_since": None, - "seconds": None, - "human": "unknown", - "restarts": None, - } - if uptime: - payload.update(uptime) - if payload.get("active") is False: - payload["human"] = "down" - elif "human" not in payload or payload["human"] == "unknown": - payload["human"] = human_duration(payload.get("seconds")) # type: ignore[arg-type] - return payload - - -def counter_items(counter: Counter[str], labels: dict[str, str]) -> list[dict[str, object]]: - items = [ - { - "key": key, - "label": labels.get(key, key), - "count": count, - } - for key, count in counter.items() - if count > 0 - ] - return sorted(items, key=lambda item: (-int(item["count"]), str(item["label"]))) - - -def build_report( - log_lines: Iterable[str], - *, - usage_lines: Iterable[str] | None = None, - now: datetime | None = None, - window_hours: int = DEFAULT_WINDOW_HOURS, - uptime: dict[str, object] | None = None, -) -> dict[str, object]: - generated_at = now.astimezone(timezone.utc) if now else datetime.now(timezone.utc) - window_start = generated_at - timedelta(hours=window_hours) - future_cutoff = generated_at + timedelta(minutes=5) - - systems: Counter[str] = Counter() - tools: Counter[str] = Counter() - other_requests: Counter[str] = Counter() - page_requests: Counter[str] = Counter() - page_metadata: dict[str, dict[str, str]] = {} - search_events: list[tuple[datetime, dict[str, object]]] = [] - diagnostic_requests: Counter[str] = Counter() - diagnostic_metadata: dict[str, dict[str, str]] = {} - rate_limited = 0 - - for line in log_lines: - fields = parse_log_line(line) - if not fields: - continue - - ts = parse_iso_datetime(fields.get("ts", "")) - if ts is None or ts < window_start or ts > future_cutoff: - continue - - try: - status = int(fields.get("status", "")) - except ValueError: - status = None - - uri = fields.get("uri", "") - method = fields.get("method", "") - limited = is_rate_limited(fields, status) - if limited: - rate_limited += 1 - - if is_mcp_request(uri): - continue - - if not is_ignored_non_mcp_request(uri): - other_requests[classify_other_request(method, uri, status)] += 1 - - for line in usage_lines or []: - usage = parse_usage_line(line) - if not usage: - continue - ts = parse_iso_datetime(usage["ts"]) - if ts is None or ts < window_start or ts > future_cutoff: - continue - tool = usage["tool"] - tools[tool] += 1 - system = public_system(usage.get("system")) - if system not in UNIDENTIFIED_CLIENT_SYSTEMS: - systems[system] += 1 - if tool == "v8std_get_page": - url = public_url(usage.get("url")) - page_id = public_text(usage.get("page_id"), limit=120) - requested_page = public_text(usage.get("requested_page"), limit=120) - title = public_text(usage.get("title")) or page_id or requested_page or url - page_key = url or page_id or requested_page - if page_key: - page_requests[page_key] += 1 - page_metadata.setdefault( - page_key, - { - "key": page_key, - "title": title or page_key, - "url": url or "", - }, - ) - elif tool == "v8std_search": - query = public_text(usage.get("query")) - if query: - results = [] - seen = set() - raw_results = usage.get("results") - if isinstance(raw_results, list): - for raw_result in raw_results: - result = public_result(raw_result) - if result is None or result["url"] in seen: - continue - seen.add(result["url"]) - results.append(result) - if len(results) >= MAX_SEARCH_RESULTS_PER_QUERY: - break - search_events.append( - ( - ts, - { - "ts": ts.replace(microsecond=0).isoformat(), - "query": query, - "system": system, - "system_label": SYSTEM_LABELS[system], - "results": results, - }, - ) - ) - elif tool == "v8std_explain_diagnostics": - raw_diagnostics = usage.get("diagnostics") - if isinstance(raw_diagnostics, list): - for raw_diagnostic in raw_diagnostics: - diagnostic = public_ranked_item(raw_diagnostic) - if diagnostic is None: - continue - diagnostic_id = diagnostic.get("id") or diagnostic.get("url") or diagnostic.get("title") - if not diagnostic_id: - continue - frequency = public_frequency(raw_diagnostic.get("frequency") if isinstance(raw_diagnostic, dict) else None) - add_diagnostic_ranking_item( - diagnostic_requests, - diagnostic_metadata, - item_id=diagnostic_id, - title=diagnostic.get("title") or diagnostic_id, - url=diagnostic.get("url", ""), - frequency=frequency, - kind="diagnostic", - ) - raw_unknown_codes = usage.get("unknown_codes") - if isinstance(raw_unknown_codes, list): - for raw_unknown_code in raw_unknown_codes: - if not isinstance(raw_unknown_code, dict): - continue - code = public_text(raw_unknown_code.get("code"), limit=120) - if code is None: - continue - add_diagnostic_ranking_item( - diagnostic_requests, - diagnostic_metadata, - item_id=code, - title=f"Неизвестная диагностика: {code}", - frequency=public_frequency(raw_unknown_code.get("frequency")), - kind="unknown_code", - ) - raw_standards = usage.get("standards_without_page") - if isinstance(raw_standards, list): - for raw_standard in raw_standards: - standard = public_ranked_item(raw_standard) - if standard is None or standard.get("url"): - continue - standard_id = standard.get("id") or standard.get("title") - if not standard_id: - continue - standard_title = standard.get("title") or standard_id - add_diagnostic_ranking_item( - diagnostic_requests, - diagnostic_metadata, - item_id=standard_id, - title=f"Стандарт без страницы: {standard_title}", - frequency=public_frequency(raw_standard.get("frequency") if isinstance(raw_standard, dict) else None), - kind="standard_without_page", - ) - - tool_items = counter_items(tools, TOOL_LABELS) - tool_calls = sum(int(item["count"]) for item in tool_items) - top_pages = [ - {**page_metadata[key], "count": count} - for key, count in page_requests.most_common(TOP_RANKING_LIMIT) - ] - recent_searches = [ - event - for _ts, event in sorted(search_events, key=lambda item: item[0], reverse=True)[:RECENT_SEARCH_LIMIT] - ] - top_diagnostics = [ - {**diagnostic_metadata[key], "count": count} - for key, count in diagnostic_requests.most_common(TOP_RANKING_LIMIT) - ] - - return { - "generated_at": generated_at.replace(microsecond=0).isoformat(), - "window_hours": window_hours, - "window_start": window_start.replace(microsecond=0).isoformat(), - "totals": { - "mcp_requests": tool_calls, - "tool_calls": tool_calls, - "rate_limited": rate_limited, - }, - "tools": tool_items, - "top_pages": top_pages, - "recent_searches": recent_searches, - "top_diagnostics": top_diagnostics, - "systems": counter_items(systems, SYSTEM_LABELS), - "other_requests": counter_items(other_requests, {}), - "uptime": normalize_uptime(uptime), - } - - -def read_log_lines(paths: Iterable[Path]) -> Iterator[str]: - for path in paths: - if not path.exists(): - continue - opener = gzip.open if path.suffix == ".gz" else open - with opener(path, "rt", encoding="utf-8", errors="replace") as handle: - yield from handle - - -def expand_access_logs(path: Path) -> list[Path]: - candidates = [path, path.with_name(f"{path.name}.1"), path.with_name(f"{path.name}.1.gz")] - return [candidate for candidate in candidates if candidate.exists()] - - -def parse_systemd_timestamp(value: str) -> datetime | None: - if not value or value == "n/a": - return None - parts = value.split(maxsplit=1) - if len(parts) == 2 and "," not in parts[0]: - value = parts[1] - for fmt in ("%Y-%m-%d %H:%M:%S.%f %Z", "%Y-%m-%d %H:%M:%S %Z"): - try: - parsed = datetime.strptime(value, fmt) - except ValueError: - continue - return parsed.replace(tzinfo=timezone.utc) - return None - - -def read_service_uptime(service: str, *, now: datetime | None = None) -> dict[str, object]: - generated_at = now.astimezone(timezone.utc) if now else datetime.now(timezone.utc) - result = subprocess.run( - [ - "systemctl", - "show", - service, - "-p", - "ActiveState", - "-p", - "ActiveEnterTimestamp", - "-p", - "NRestarts", - "--no-pager", - ], - check=False, - capture_output=True, - text=True, - timeout=5, - ) - fields = {} - for line in result.stdout.splitlines(): - if "=" in line: - key, value = line.split("=", 1) - fields[key] = value - - active_since = parse_systemd_timestamp(fields.get("ActiveEnterTimestamp", "")) - active = fields.get("ActiveState") == "active" if fields else None - seconds = None - if active and active_since: - seconds = max(0, int((generated_at - active_since).total_seconds())) - try: - restarts: int | None = int(fields["NRestarts"]) - except (KeyError, ValueError): - restarts = None - - return { - "service": service, - "active": active, - "active_since": active_since.isoformat() if active_since else fields.get("ActiveEnterTimestamp") or None, - "seconds": seconds, - "restarts": restarts, - } - - -def metric_card(label: str, value: object, detail: str) -> str: - return ( - '
    ' - f'
    {html.escape(label)}
    ' - f'
    {html.escape(str(value))}
    ' - f'
    {html.escape(detail)}
    ' - "
    " - ) - - -def render_bar_list(items: list[dict[str, object]], empty_text: str) -> str: - if not items: - return f'

    {html.escape(empty_text)}

    ' - max_count = max(int(item["count"]) for item in items) or 1 - rows = [] - for item in items: - count = int(item["count"]) - width = max(4, round(count / max_count * 100)) - rows.append( - '
    ' - f'
    {html.escape(str(item["label"]))}{count}
    ' - '
    ' - f'
    ' - "
    " - "
    " - ) - return "\n".join(rows) - - -def render_link(url: object, label: object) -> str: - url_text = public_url(url) - label_text = public_text(label) or url_text or "" - if url_text is None: - return html.escape(label_text) - return f'{html.escape(label_text)}' - - -def render_page_ranking(items: list[dict[str, object]]) -> str: - if not items: - return '

    Данных по v8std_get_page пока нет.

    ' - rows = [] - limited_items = items[:TOP_RANKING_LIMIT] - for index, item in enumerate(limited_items, start=1): - title = item.get("title") or item.get("key") or item.get("url") - rows.append( - '
    ' - f'
    {index}
    ' - '
    ' - f'
    {render_link(item.get("url"), title)}
    ' - f'
    {html.escape(str(item.get("url") or item.get("key") or ""))}
    ' - "
    " - f'{int(item["count"])}' - "
    " - ) - split_index = (len(rows) + 1) // 2 - columns = [rows[:split_index], rows[split_index:]] - return "\n".join( - '
    ' + "\n".join(column_rows) + "
    " - for column_rows in columns - if column_rows - ) - - -def render_search_time(value: object) -> str: - if not isinstance(value, str): - return "" - ts = parse_iso_datetime(value) - if ts is None: - return "" - return ts.strftime("%H:%M UTC") - - -def render_search_ranking(items: list[dict[str, object]]) -> str: - if not items: - return '

    Данных по v8std_search пока нет.

    ' - rows = [] - for index, item in enumerate(items[:TOP_RANKING_LIMIT], start=1): - results = item.get("results") - result_rows = [] - if isinstance(results, list): - for result_index, result in enumerate(results[:MAX_SEARCH_RESULTS_PER_QUERY], start=1): - if isinstance(result, dict): - title = result.get("title") or result.get("id") or result.get("url") - meta = result.get("url") or result.get("id") or "" - result_rows.append( - '
    ' - f'
    {index}.{result_index}
    ' - '
    ' - f'
    {render_link(result.get("url"), title)}
    ' - f'
    {html.escape(str(meta))}
    ' - "
    " - "
    " - ) - rendered_results = ( - '
    ' + "\n".join(result_rows) + "
    " - if result_rows - else '
    Результатов в логе нет.
    ' - ) - meta_parts = [] - if item.get("system") not in UNIDENTIFIED_CLIENT_SYSTEMS: - meta_parts.append(str(item.get("system_label") or "")) - meta_parts.append("запрос v8std_search") - meta = " · ".join(part for part in meta_parts if part) - rows.append( - '
    ' - '
    ' - f'
    {index}
    ' - '
    ' - f'
    {html.escape(str(item["query"]))}
    ' - f'
    {html.escape(meta)}
    ' - "
    " - f'{html.escape(render_search_time(item.get("ts")))}' - "
    " - f"{rendered_results}" - "
    " - ) - return "\n".join(rows) - - -def render_diagnostic_ranking(items: list[dict[str, object]]) -> str: - if not items: - return '

    Данных по v8std_explain_diagnostics пока нет.

    ' - kind_labels = { - "unknown_code": "неизвестная диагностика", - "standard_without_page": "стандарт без страницы", - } - rows = [] - for index, item in enumerate(items[:TOP_RANKING_LIMIT], start=1): - title = item.get("title") or item.get("id") or item.get("key") or item.get("url") - diagnostic_id = public_text(item.get("id"), limit=120) - kind = public_text(item.get("kind"), limit=80) - meta_parts = [] - if kind in kind_labels: - meta_parts.append(kind_labels[kind]) - if diagnostic_id: - meta_parts.append(diagnostic_id) - meta = " · ".join(meta_parts) or item.get("url") or item.get("key") or "" - rows.append( - '
    ' - f'
    {index}
    ' - '
    ' - f'
    {render_link(item.get("url"), title)}
    ' - f'
    {html.escape(str(meta))}
    ' - "
    " - f'{int(item["count"])}' - "
    " - ) - return "\n".join(rows) - - -def render_html(report: dict[str, object]) -> str: - totals = report["totals"] # type: ignore[assignment] - uptime = report["uptime"] # type: ignore[assignment] - tools = report["tools"] # type: ignore[assignment] - top_pages = report["top_pages"] # type: ignore[assignment] - recent_searches = report["recent_searches"] # type: ignore[assignment] - top_diagnostics = report["top_diagnostics"] # type: ignore[assignment] - systems = report["systems"] # type: ignore[assignment] - other_requests = report["other_requests"] # type: ignore[assignment] - generated_at = str(report["generated_at"]) - window_hours = int(report["window_hours"]) - active = "active" if uptime.get("active") else "not active" if uptime.get("active") is False else "unknown" # type: ignore[attr-defined] - - metrics = "\n".join( - [ - metric_card("MCP запросы", totals["mcp_requests"], f"вызовы MCP tools за последние {window_hours} часа"), # type: ignore[index] - metric_card("Rate limit", totals["rate_limited"], "отклонено лимитом"), # type: ignore[index] - metric_card("Аптайм MCP", uptime.get("human", "unknown"), f'{uptime.get("service", DEFAULT_SERVICE)}: {active}'), # type: ignore[attr-defined] - ] - ) - - return f""" - - - - - - - Мониторинг v8std MCP - - - -
    -
    -
    -

    Мониторинг v8std MCP

    -
    -
    Обновлено
    {html.escape(generated_at)}
    -
    - -
    - {metrics} -
    - -
    -
    -

    MCP tools

    - {render_bar_list(tools, "Вызовов tools/call после включения счетчика пока нет.")} -
    - -
    -

    Системы

    - {render_bar_list(systems, "Клиенты MCP tools пока не накоплены.")} -
    -
    - -
    -
    -

    Последние 10 запросов search

    -
    - {render_search_ranking(recent_searches)} -
    -
    - -
    -

    Топ диагностик explain_diagnostics

    - {render_diagnostic_ranking(top_diagnostics)} -
    - -
    -

    Топ страниц get_page

    -
    - {render_page_ranking(top_pages)} -
    -
    -
    - -
    -

    Прочие запросы

    - {render_bar_list(other_requests[:10], "Прочих запросов нет.")} -
    - -
    - Данные агрегированы без IP-адресов. Системы считаются по MCP tool calls. - JSON: stats.json -
    -
    - - -""" - - -def write_dashboard(report: dict[str, object], output_dir: Path) -> None: - output_dir.mkdir(parents=True, exist_ok=True) - html_path = output_dir / "index.html" - json_path = output_dir / "stats.json" - html_tmp = output_dir / ".index.html.tmp" - json_tmp = output_dir / ".stats.json.tmp" - - html_tmp.write_text(render_html(report), encoding="utf-8") - json_tmp.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - html_tmp.replace(html_path) - json_tmp.replace(json_path) - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Render a static public dashboard for the v8std MCP endpoint.") - parser.add_argument("--access-log", action="append", type=Path, default=None) - parser.add_argument("--usage-log", action="append", type=Path, default=None) - parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) - parser.add_argument("--service", default=DEFAULT_SERVICE) - parser.add_argument("--window-hours", type=int, default=DEFAULT_WINDOW_HOURS) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv) - access_logs = args.access_log or [DEFAULT_ACCESS_LOG] - log_paths: list[Path] = [] - for access_log in access_logs: - expanded = expand_access_logs(access_log) - log_paths.extend(expanded or [access_log]) - usage_logs = args.usage_log or [DEFAULT_USAGE_LOG] - usage_paths: list[Path] = [] - for usage_log in usage_logs: - expanded = expand_access_logs(usage_log) - usage_paths.extend(expanded or [usage_log]) - - now = datetime.now(timezone.utc) - report = build_report( - read_log_lines(log_paths), - usage_lines=read_log_lines(usage_paths), - now=now, - window_hours=args.window_hours, - uptime=read_service_uptime(args.service, now=now), - ) - write_dashboard(report, args.output_dir) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_v8std_architecture_repository.py b/tests/test_v8std_architecture_repository.py index 374f3c7..f62ad70 100644 --- a/tests/test_v8std_architecture_repository.py +++ b/tests/test_v8std_architecture_repository.py @@ -36,17 +36,18 @@ def test_complete_corpus_uses_one_valid_model(self) -> None: ["README.md"], ) - def test_mcp_design_lifecycle_reflects_combined_endpoint_decision(self) -> None: + def test_mcp_design_lifecycle_reflects_monitoring_retirement(self) -> None: superseded = { "design:mcp-v3-resource-contract", "design:mcp-100k-agent-capacity", + "design:mcp-monitoring-dashboard", + "design:mcp-container-monitor-input", } for reference in superseded: self.assertIn("SUPERSEDED", self.states[reference]) planned = { - "design:mcp-monitoring-dashboard", "design:mcp-openmetrics-generation", } diff --git a/tests/test_v8std_mcp_monitoring.py b/tests/test_v8std_mcp_monitoring.py deleted file mode 100644 index 100138e..0000000 --- a/tests/test_v8std_mcp_monitoring.py +++ /dev/null @@ -1,281 +0,0 @@ -from __future__ import annotations - -import json -import sys -import tempfile -import unittest -from datetime import datetime, timezone -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS_PATH = REPO_ROOT / "scripts" -USAGE_LOGROTATE_PATH = SCRIPTS_PATH / "v8std_mcp_usage.logrotate" - -if str(SCRIPTS_PATH) not in sys.path: - sys.path.insert(0, str(SCRIPTS_PATH)) - -import v8std_mcp_monitoring as monitoring - - -NOW = datetime(2026, 4, 29, 12, 0, tzinfo=timezone.utc) - - -class V8StdMcpMonitoringTests(unittest.TestCase): - def test_usage_logrotate_retains_search_history_for_one_year(self): - config = USAGE_LOGROTATE_PATH.read_text(encoding="utf-8") - - self.assertIn("/var/lib/v8std-mcp/tool-usage.jsonl", config) - self.assertIn("daily", config) - self.assertIn("rotate 365", config) - self.assertIn("compress", config) - - def test_build_report_focuses_on_mcp_and_separates_rate_limit_from_real_errors(self): - report = monitoring.build_report( - [ - 'ts=2026-04-29T11:59:00+00:00 remote=203.0.113.10 method=POST uri=/mcp status=200 request_time=0.006 upstream_time=0.005 bytes=438 ua="codex-cli/0.8"', - 'ts=2026-04-29T11:58:00+00:00 remote=203.0.113.11 method=GET uri=/mcp status=200 request_time=0.001 upstream_time=0.001 bytes=972 ua="curl/8.7.1"', - 'ts=2026-04-29T11:57:00+00:00 remote=203.0.113.12 method=GET uri=/healthz status=200 request_time=0.002 upstream_time=0.002 bytes=2 ua="Uptime-Kuma/1.23"', - 'ts=2026-04-29T11:56:00+00:00 remote=203.0.113.13 method=POST uri=/mcp status=503 request_time=0.000 upstream_time=- bytes=197 ua="Claude Desktop"', - 'ts=2026-04-29T11:55:00+00:00 remote=203.0.113.14 method=POST uri=/mcp status=503 request_time=0.010 upstream_time=0.010 bytes=197 ua="Mozilla/5.0"', - 'ts=2026-04-29T11:54:50+00:00 remote=203.0.113.18 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="node"', - 'ts=2026-04-29T11:54:40+00:00 remote=203.0.113.19 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="opencode/1.2.27"', - 'ts=2026-04-29T11:54:30+00:00 remote=203.0.113.20 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="Go-http-client/1.1"', - 'ts=2026-04-29T11:54:20+00:00 remote=203.0.113.21 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="python-httpx/0.28.1"', - 'ts=2026-04-29T11:54:10+00:00 remote=203.0.113.22 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="kilo/7.2.25"', - 'ts=2026-04-29T11:54:05+00:00 remote=203.0.113.23 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="Java-http-client/17.0.16"', - 'ts=2026-04-29T11:54:04+00:00 remote=203.0.113.24 method=POST uri=/mcp status=200 request_time=0.004 upstream_time=0.004 bytes=438 ua="undici"', - 'ts=2026-04-29T11:54:00+00:00 remote=203.0.113.16 method=GET uri=/monitoring/stats.json status=200 request_time=0.000 upstream_time=- bytes=1200 ua="curl/8.7.1"', - 'ts=2026-04-29T11:53:00+00:00 remote=203.0.113.17 method=GET uri=/wp-login.php status=404 request_time=0.000 upstream_time=- bytes=120 ua="bot"', - 'ts=2026-04-28T11:59:00+00:00 remote=203.0.113.15 method=POST uri=/mcp status=200 request_time=0.002 upstream_time=0.002 bytes=48 ua="stale-client"', - ], - usage_lines=[ - '{"ts":"2026-04-29T11:59:30+00:00","tool":"v8std_search"}', - '{"ts":"2026-04-29T11:58:30+00:00","tool":"v8std_search","system":"cursor","query":"модальные окна","results":[{"id":"std404","title":"Модальные окна","url":"https://v8std.ru/std/404/"},{"id":"std519","title":"Оповещения","url":"https://v8std.ru/std/519/"}]}', - '{"ts":"2026-04-29T11:57:30+00:00","tool":"v8std_get_page","system":"node","requested_page":"std437","page_id":"std437","title":"Оформление текстов запросов","url":"https://v8std.ru/std/437/"}', - '{"ts":"2026-04-29T11:56:30+00:00","tool":"v8std_get_page","system":"node","requested_page":"https://v8std.ru/std/437/","page_id":"std437","title":"Оформление текстов запросов","url":"https://v8std.ru/std/437/"}', - '{"ts":"2026-04-29T11:55:30+00:00","tool":"v8std_explain_diagnostics","system":"opencode","diagnostics":[{"id":"bslls:UsingModalWindows","title":"Использование модальных окон","url":"https://v8std.ru/diagnostics/bslls/UsingModalWindows/","frequency":2},{"id":"acc:1245","title":"Нельзя использовать * в запросах","url":"https://v8std.ru/diagnostics/acc/1245/","frequency":1},{"id":"external","title":"External","url":"https://example.com/","frequency":99}],"unknown_codes":[{"code":"v8cs:NewUnknownDiagnostic","frequency":3}],"standards_without_page":[{"id":"std999","title":"Стандарт без страницы","frequency":2}]}', - '{"ts":"2026-04-29T11:55:00+00:00","tool":"v8std_explain_snippet","system":"other"}', - '{"ts":"2026-04-28T11:57:30+00:00","tool":"v8std_get_page"}', - ], - now=NOW, - window_hours=24, - uptime={ - "service": "v8std-mcp.service", - "active": True, - "active_since": "2026-04-28T16:22:56+00:00", - "seconds": 70424, - }, - ) - - self.assertEqual(report["totals"]["mcp_requests"], 6) - self.assertEqual(report["totals"]["tool_calls"], 6) - self.assertEqual(report["totals"]["rate_limited"], 1) - self.assertNotIn("real_5xx", report["totals"]) - self.assertEqual(report["uptime"]["human"], "19h 33m") - - self.assertNotIn("request_types", report) - - systems = {item["key"]: item["count"] for item in report["systems"]} - self.assertNotIn("codex", systems) - self.assertNotIn("curl", systems) - self.assertNotIn("claude", systems) - self.assertNotIn("browser", systems) - self.assertEqual(systems["node"], 2) - self.assertEqual(systems["opencode"], 1) - self.assertEqual(systems["cursor"], 1) - self.assertNotIn("unknown", systems) - self.assertNotIn("other", systems) - self.assertNotIn("go", systems) - self.assertNotIn("python_httpx", systems) - self.assertNotIn("kilo", systems) - self.assertNotIn("java", systems) - self.assertNotIn("monitoring", systems) - - tools = {item["key"]: item["count"] for item in report["tools"]} - self.assertEqual(tools["v8std_search"], 2) - self.assertEqual(tools["v8std_get_page"], 2) - self.assertEqual(tools["v8std_explain_diagnostics"], 1) - self.assertEqual(tools["v8std_explain_snippet"], 1) - - self.assertEqual(report["top_pages"][0]["count"], 2) - self.assertEqual(report["top_pages"][0]["title"], "Оформление текстов запросов") - self.assertEqual(report["top_pages"][0]["url"], "https://v8std.ru/std/437/") - - self.assertNotIn("top_searches", report) - self.assertEqual(report["recent_searches"][0]["query"], "модальные окна") - self.assertEqual(report["recent_searches"][0]["system"], "cursor") - self.assertEqual(report["recent_searches"][0]["system_label"], "Cursor") - self.assertEqual(report["recent_searches"][0]["results"][0]["url"], "https://v8std.ru/std/404/") - - diagnostics_by_id = {item["id"]: item for item in report["top_diagnostics"]} - self.assertEqual(diagnostics_by_id["v8cs:NewUnknownDiagnostic"]["count"], 3) - self.assertEqual(diagnostics_by_id["v8cs:NewUnknownDiagnostic"]["title"], "Неизвестная диагностика: v8cs:NewUnknownDiagnostic") - self.assertEqual(diagnostics_by_id["v8cs:NewUnknownDiagnostic"]["url"], "") - self.assertEqual(diagnostics_by_id["v8cs:NewUnknownDiagnostic"]["kind"], "unknown_code") - self.assertEqual(diagnostics_by_id["bslls:UsingModalWindows"]["count"], 2) - self.assertEqual(diagnostics_by_id["bslls:UsingModalWindows"]["title"], "Использование модальных окон") - self.assertEqual(diagnostics_by_id["bslls:UsingModalWindows"]["url"], "https://v8std.ru/diagnostics/bslls/UsingModalWindows/") - self.assertEqual(diagnostics_by_id["std999"]["count"], 2) - self.assertEqual(diagnostics_by_id["std999"]["title"], "Стандарт без страницы: Стандарт без страницы") - self.assertEqual(diagnostics_by_id["std999"]["url"], "") - self.assertEqual(diagnostics_by_id["std999"]["kind"], "standard_without_page") - self.assertEqual(diagnostics_by_id["acc:1245"]["count"], 1) - self.assertEqual(diagnostics_by_id["acc:1245"]["url"], "https://v8std.ru/diagnostics/acc/1245/") - self.assertNotIn("example.com", json.dumps(report["top_diagnostics"], ensure_ascii=False)) - - self.assertNotIn("status_classes", report) - - other = {item["key"]: item["count"] for item in report["other_requests"]} - self.assertEqual(other["GET /wp-login.php -> 404"], 1) - self.assertNotIn("GET /healthz -> 200", other) - self.assertNotIn("GET /monitoring/stats.json -> 200", other) - - def test_build_report_keeps_ten_latest_searches(self): - usage_lines = [ - json.dumps( - { - "ts": f"2026-04-29T11:{index:02d}:00+00:00", - "tool": "v8std_search", - "system": "claude" if index % 2 else "cursor", - "query": f"query-{index}", - "results": [ - { - "id": f"std{index}", - "title": f"Result {index}", - "url": f"https://v8std.ru/std/{index}/", - } - ], - } - ) - for index in range(11) - ] - - report = monitoring.build_report( - [], - usage_lines=usage_lines, - now=NOW, - window_hours=24, - uptime={"service": "v8std-mcp.service", "active": True, "seconds": 60}, - ) - - self.assertEqual(len(report["recent_searches"]), 10) - self.assertEqual([item["query"] for item in report["recent_searches"]], [f"query-{index}" for index in range(10, 0, -1)]) - self.assertEqual(report["recent_searches"][0]["system_label"], "Cursor") - self.assertEqual(report["recent_searches"][1]["system_label"], "Claude") - self.assertEqual(report["recent_searches"][0]["results"][0]["url"], "https://v8std.ru/std/10/") - self.assertNotIn("top_searches", report) - - def test_render_page_ranking_splits_rows_by_column_not_by_grid_rows(self): - items = [ - { - "key": f"std{index}", - "title": f"Page {index}", - "url": f"https://v8std.ru/std/{index}/", - "count": index, - } - for index in range(1, 5) - ] - - html = monitoring.render_page_ranking(items) - - self.assertEqual(html.count('class="page-ranking__column"'), 2) - first_column_start = html.index('class="page-ranking__column"') - second_column_start = html.index('class="page-ranking__column"', first_column_start + 1) - - self.assertLess(html.index('
    1
    '), second_column_start) - self.assertLess(html.index('
    2
    '), second_column_start) - self.assertGreater(html.index('
    3
    '), second_column_start) - self.assertGreater(html.index('
    4
    '), second_column_start) - - def test_render_html_is_mcp_focused_and_omits_raw_clients_and_nginx_details(self): - report = monitoring.build_report( - [ - 'ts=2026-04-29T11:59:00+00:00 remote=203.0.113.10 method=POST uri=/mcp status=200 request_time=0.006 upstream_time=0.005 bytes=438 ua="codex-cli/0.8"', - 'ts=2026-04-29T11:56:00+00:00 remote=203.0.113.13 method=POST uri=/mcp status=503 request_time=0.000 upstream_time=- bytes=197 ua="Claude Desktop"', - 'ts=2026-04-29T11:53:00+00:00 remote=203.0.113.17 method=GET uri=/wp-login.php status=404 request_time=0.000 upstream_time=- bytes=120 ua="bot"', - ], - usage_lines=[ - '{"ts":"2026-04-29T11:59:30+00:00","tool":"v8std_search","system":"cursor","query":"модальные окна","results":[{"id":"std404","title":"Модальные окна","url":"https://v8std.ru/std/404/"}]}', - '{"ts":"2026-04-29T11:59:00+00:00","tool":"v8std_search","query":"привилегированный режим"}', - '{"ts":"2026-04-29T11:58:45+00:00","tool":"v8std_search","system":"other","query":"старая методика проведения документов"}', - '{"ts":"2026-04-29T11:58:30+00:00","tool":"v8std_get_page","requested_page":"std437","page_id":"std437","title":"Оформление текстов запросов","url":"https://v8std.ru/std/437/"}', - '{"ts":"2026-04-29T11:57:30+00:00","tool":"v8std_explain_diagnostics","diagnostics":[{"id":"bslls:UsingModalWindows","title":"Использование модальных окон","url":"https://v8std.ru/diagnostics/bslls/UsingModalWindows/","frequency":2}],"unknown_codes":[{"code":"v8cs:NewUnknownDiagnostic","frequency":1}],"standards_without_page":[{"id":"std999","title":"Стандарт без страницы","frequency":1}]}', - ], - now=NOW, - window_hours=24, - uptime={"service": "v8std-mcp.service", "active": True, "seconds": 3600}, - ) - - html = monitoring.render_html(report) - - self.assertIn("Мониторинг v8std MCP", html) - self.assertIn("MCP запросы", html) - self.assertNotIn("Tool calls", html) - self.assertNotIn("Реальные 5xx", html) - self.assertIn("Прочие запросы", html) - self.assertIn("GET /wp-login.php -> 404", html) - self.assertIn("v8std_search", html) - self.assertIn("Топ страниц get_page", html) - self.assertIn("Последние 10 запросов search", html) - self.assertNotIn("Топ запросов search", html) - self.assertIn("Топ диагностик explain_diagnostics", html) - self.assertIn("https://v8std.ru/std/437/", html) - self.assertIn("https://v8std.ru/std/404/", html) - self.assertIn("https://v8std.ru/diagnostics/bslls/UsingModalWindows/", html) - self.assertIn("Использование модальных окон", html) - self.assertIn("Cursor", html) - self.assertNotIn("Unknown · запрос v8std_search", html) - self.assertNotIn("Other · запрос v8std_search", html) - self.assertIn("Неизвестная диагностика: v8cs:NewUnknownDiagnostic", html) - self.assertIn("Стандарт без страницы: Стандарт без страницы", html) - self.assertIn('
    ', html) - self.assertIn('
    ', html) - self.assertIn("page-ranking__column", html) - self.assertIn(".page-ranking__column .rank-row:first-child", html) - self.assertNotIn("column-count: 2", html) - self.assertIn('
    ', html) - self.assertIn('
    ', html) - self.assertLess(html.index("Последние 10 запросов search"), html.index("Топ диагностик explain_diagnostics")) - self.assertLess(html.index("Топ диагностик explain_diagnostics"), html.index("Топ страниц get_page")) - self.assertNotIn("rank-grid", html) - self.assertNotIn("result-links", html) - self.assertIn('rel="icon" href="data:,"', html) - self.assertNotIn("MCP обращения", html) - self.assertNotIn("Коды MCP", html) - self.assertNotIn("User-Agent по MCP", html) - self.assertIn("Системы считаются по MCP tool calls", html) - self.assertNotIn("203.0.113.10", html) - self.assertNotIn("codex-cli/0.8", html) - self.assertNotIn("nginx", html.lower()) - - def test_write_dashboard_outputs_html_and_json(self): - report = monitoring.build_report( - [ - 'ts=2026-04-29T11:59:00+00:00 remote=203.0.113.10 method=POST uri=/mcp status=200 request_time=0.001 upstream_time=0.001 bytes=2 ua="curl/8.7.1"', - 'ts=2026-04-29T11:58:00+00:00 remote=203.0.113.10 method=POST uri=/mcp status=200 request_time=0.001 upstream_time=0.001 bytes=2 ua="curl/8.7.1"', - ], - usage_lines=['{"ts":"2026-04-29T11:59:30+00:00","tool":"v8std_search"}'], - now=NOW, - window_hours=24, - uptime={"service": "v8std-mcp.service", "active": True, "seconds": 60}, - ) - - with tempfile.TemporaryDirectory() as temp_dir: - output_dir = Path(temp_dir) - monitoring.write_dashboard(report, output_dir) - - html = (output_dir / "index.html").read_text(encoding="utf-8") - payload = json.loads((output_dir / "stats.json").read_text(encoding="utf-8")) - - self.assertIn("", html) - self.assertEqual(payload["totals"]["mcp_requests"], 1) - self.assertEqual(payload["tools"][0]["key"], "v8std_search") - self.assertEqual(payload["top_pages"], []) - self.assertEqual(payload["recent_searches"], []) - self.assertNotIn("top_searches", payload) - self.assertEqual(payload["top_diagnostics"], []) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_v8std_mcp_monitoring_retirement.py b/tests/test_v8std_mcp_monitoring_retirement.py new file mode 100644 index 0000000..d3f62aa --- /dev/null +++ b/tests/test_v8std_mcp_monitoring_retirement.py @@ -0,0 +1,198 @@ +"""Real nginx conformance; opt in with V8STD_MONITORING_RETIREMENT_DOCKER=1. + +Uses the pinned, already-local nginx image from test_v8std_mcp_release_docker. +No image pull, build, production access, or shared Docker resource cleanup. +""" +from __future__ import annotations + +import http.client +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import time +import unittest +import uuid + + +ROOT = Path(__file__).resolve().parents[1] +NGINX = "sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c" + + +@unittest.skipUnless( + os.environ.get("V8STD_MONITORING_RETIREMENT_DOCKER") == "1", + "explicit disposable Docker evidence", +) +class MonitoringRetirementTests(unittest.TestCase): + @staticmethod + def docker(*args: str) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=30, + ) + if result.returncode: + raise AssertionError(f"docker {args}: {result.stdout}{result.stderr}") + return result + + @classmethod + def setUpClass(cls) -> None: + cls.docker("image", "inspect", NGINX) + temporary = tempfile.TemporaryDirectory(prefix="v8std-monitoring-retirement-") + cls.addClassCleanup(temporary.cleanup) + directory = Path(temporary.name) + config, web, indexes = (directory / name for name in ("nginx", "web", "indexes")) + for path in (config, web, indexes): + path.mkdir(mode=0o755) + for name in ("edge-http.conf", "edge-locations.conf"): + shutil.copyfile(ROOT / "deploy/container" / name, config / name) + (config / "v8std-release").mkdir() + (config / "v8std-release/upstream.conf").write_text( + "server 127.0.0.1:9;\n", encoding="utf-8", + ) + (config / "nginx.conf").write_text( + "worker_processes 1; pid /tmp/nginx.pid;\n" + "error_log /dev/stderr warn; events { worker_connections 128; }\n" + "http { access_log off; client_body_temp_path /tmp/client;\n" + "proxy_temp_path /tmp/proxy; fastcgi_temp_path /tmp/fastcgi;\n" + "uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi;\n" + "include /etc/nginx/edge-http.conf;\n" + "server { listen 8080; server_name localhost; root /srv/web;\n" + "include /etc/nginx/edge-locations.conf;\n" + "} }\n", + encoding="utf-8", + ) + (web / "index.html").write_bytes(b"public-root-fixture\n") + legacy = web / "monitoring" + legacy.mkdir() + (legacy / "index.html").write_bytes(b"private-monitoring-sentinel\n") + (legacy / "stats.json").write_bytes(b'{"data":"private-monitoring-sentinel"}\n') + snapshot = indexes / ("a" * 64) + snapshot.mkdir() + (snapshot / "snapshot.tar.gz").write_bytes(b"snapshot-fixture\n") + + cls.name = "v8std-monitoring-retirement-" + uuid.uuid4().hex + cls.owner_label = "pro.v8std.monitoring-retirement=" + cls.name + cls.addClassCleanup(cls.cleanup_container) + cls.docker( + "create", "--name", cls.name, "--pull=never", + "--label", cls.owner_label, "--read-only", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--init", "--user=10001:10001", + "--memory=128m", "--memory-swap=128m", "--cpus=1", "--pids-limit=128", + "--tmpfs=/tmp:rw,noexec,nosuid,size=64m", "-p", "127.0.0.1::8080", + "--mount", f"type=bind,source={config},target=/etc/nginx,readonly", + "--mount", f"type=bind,source={web},target=/srv/web,readonly", + "--mount", f"type=bind,source={indexes},target=/srv/v8std-indexes/v1,readonly", + "--entrypoint=nginx", NGINX, "-g", "daemon off;", + ) + cls.docker("start", cls.name) + info = json.loads(cls.docker("container", "inspect", cls.name).stdout)[0] + binding = info["NetworkSettings"]["Ports"]["8080/tcp"] + if len(binding) != 1 or binding[0]["HostIp"] != "127.0.0.1": + raise AssertionError(f"fixture must publish only on loopback: {binding}") + cls.port = int(binding[0]["HostPort"]) + check = cls.docker("exec", cls.name, "nginx", "-t") + print(f"\n{cls.name}: {check.stderr.strip()}", flush=True) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + cls.request("/") + return + except OSError: + time.sleep(0.1) + raise AssertionError(cls.docker("logs", cls.name).stderr) + + @classmethod + def cleanup_container(cls) -> None: + # An exact UUID name plus owner label prevents removal of another fixture. + owned = cls.docker( + "container", "ls", "--all", "--quiet", "--no-trunc", + "--filter", "label=" + cls.owner_label, + "--filter", "name=^/" + cls.name + "$", + ).stdout.split() + if len(owned) > 1: + raise AssertionError(f"ambiguous fixture ownership: {owned}") + for container_id in owned: + cls.docker("container", "rm", "--force", "--volumes", container_id) + remaining = cls.docker( + "container", "ls", "--all", "--quiet", + "--filter", "label=" + cls.owner_label, + ).stdout.strip() + if remaining: + raise AssertionError(f"owned fixture remains: {remaining}") + print(f"\ncleanup {cls.name}: owned containers absent", flush=True) + + @classmethod + def request(cls, path: str, method: str = "GET") -> tuple[int, dict, bytes]: + client = http.client.HTTPConnection("127.0.0.1", cls.port, timeout=5) + try: + client.request(method, path) + response = client.getresponse() + return ( + response.status, + {key.lower(): value for key, value in response.getheaders()}, + response.read(), + ) + finally: + client.close() + + def test_retired_paths_never_serve_legacy_files_or_redirect(self) -> None: + # Missing either tombstone, a redirect, or cacheable errors breaks this. + for method in ("GET", "HEAD", "POST"): + for path in ( + "/monitoring", "/monitoring?x=1", "/monitoring/", + "/monitoring/index.html", "/monitoring/stats.json", + "/monitoring/stats.json?x=1", "/monitoring/unknown", + ): + with self.subTest(method=method, path=path): + status, headers, body = self.request(path, method) + self.assertEqual(status, 410) + self.assertEqual(headers.get("cache-control"), "no-store") + self.assertNotIn(b"private-monitoring-sentinel", body) + self.assertNotIn("location", headers) + if method == "HEAD": + self.assertEqual(body, b"") + + def test_public_root_is_still_served(self) -> None: + status, headers, body = self.request("/") + self.assertEqual(status, 200) + self.assertEqual(body, b"public-root-fixture\n") + self.assertNotEqual(headers.get("cache-control"), "no-store") + + def test_health_and_mcp_keep_their_dead_upstream_boundaries(self) -> None: + for path, method, expected in ( + ("/healthz", "GET", 502), ("/version", "GET", 502), + ("/mcp", "GET", 405), ("/mcp", "POST", 503), + ("/mcp", "HEAD", 503), + ): + with self.subTest(path=path, method=method): + status, headers, body = self.request(path, method) + self.assertEqual(status, expected) + self.assertNotEqual(headers.get("cache-control"), "no-store") + self.assertNotIn(b"private-monitoring-sentinel", body) + if expected == 503: + self.assertEqual(headers.get("retry-after"), "1") + if expected == 405: + self.assertEqual(headers.get("allow"), "POST, HEAD") + + def test_index_download_stays_available_with_dead_upstream(self) -> None: + path = "/indexes/v1/" + "a" * 64 + "/snapshot.tar.gz" + for method in ("GET", "HEAD"): + with self.subTest(method=method): + status, headers, body = self.request(path, method) + self.assertEqual(status, 200) + self.assertEqual(headers.get("content-type"), "application/gzip") + self.assertEqual(headers.get("cache-control"), "public, max-age=31536000, immutable") + self.assertEqual(body, b"snapshot-fixture\n" if method == "GET" else b"") + for target, method, expected in ( + (path, "POST", 403), ("/indexes/", "GET", 404), + ("/indexes/v1/unknown/snapshot.tar.gz", "GET", 404), + ): + with self.subTest(path=target, method=method): + status, headers, _ = self.request(target, method) + self.assertEqual(status, expected) + self.assertNotIn("immutable", headers.get("cache-control", "")) + + +if __name__ == "__main__": + unittest.main() From d8c0b9257807d25cbb37670bbe931de7eb01db15 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:38:42 +0300 Subject: [PATCH 44/88] docs: record production monitoring retirement and preserve private operations --- ...-15-mcp-private-container-monitor-input.md | 13 ++ ...2026-09-15-retire-public-mcp-monitoring.md | 24 ++++ ...09-10-mcp-container-distribution-design.md | 6 +- ...2026-09-15-public-monitoring-retirement.md | 131 ++++++++++++++++++ spec/operations/mcp-container-activation.md | 25 ++-- spec/operations/mcp-container-verification.md | 27 +++- .../mcp-first-container-release-roadmap.md | 10 +- ...6-09-10-mcp-container-distribution-plan.md | 30 ++-- ...5-mcp-public-monitoring-retirement-plan.md | 5 + 9 files changed, 238 insertions(+), 33 deletions(-) create mode 100644 spec/operations/2026-09-15-public-monitoring-retirement.md diff --git a/spec/adr/2026-09-15-mcp-private-container-monitor-input.md b/spec/adr/2026-09-15-mcp-private-container-monitor-input.md index 2c7260f..eb15d37 100644 --- a/spec/adr/2026-09-15-mcp-private-container-monitor-input.md +++ b/spec/adr/2026-09-15-mcp-private-container-monitor-input.md @@ -33,20 +33,33 @@ contracts: # Закрытый вход непривилегированного мониторинга +## Входные требования + +Основание исходного решения — свежесть наблюдения serving runtime, +непривилегированный reader и сохранение usage history при container release. + +## Решение + Root-owned sampler перед существующим batch job собирает действительное состояние serving runtime и атомарно пишет маленький private JSON. Агрегатор может только читать его; не получает полномочий Docker или release-controller. История usage отдельно остаётся persistent private data, а не частью cache. +## Влияние на инварианты + Свежесть, boot identity и проверка serving identity ограничивают доверие к сводке. Ошибка или переход означает unknown. Квитанция COMMITTED, active controller и загрузка нового corpus не заменяют процессные наблюдения. +## Влияние на контракты + Решение сохраняет старые readers и public fields. `restarts:null` для container честно обозначает отсутствие сопоставимого общего счётчика. Private details не расширяют публичную схему; общий редизайн/privacy-remediation сюда не включены. Это дополнение к release ADR, не его замена и не изменение release budgets. +## Отклонённые альтернативы + Socket/group у reader отвергнуты из-за избыточной власти; фиксированный sudo RPC избыточен для чтения batch-сводки; отдельный daemon не требуется. Цена файла — явное истечение срока доверия и необходимость тестировать invalidation/races. diff --git a/spec/adr/2026-09-15-retire-public-mcp-monitoring.md b/spec/adr/2026-09-15-retire-public-mcp-monitoring.md index 3e6b326..606e7a0 100644 --- a/spec/adr/2026-09-15-retire-public-mcp-monitoring.md +++ b/spec/adr/2026-09-15-retire-public-mcp-monitoring.md @@ -37,12 +37,36 @@ contracts: # Прекратить публичную публикацию мониторинга +## Входные требования + +Основание — согласованное удаление публичной проекции при сохранении private +usage history, закрытой границы операционных данных и работающего MCP. + +## Решение + Пользователь отменил публичный dashboard целиком. Останавливаем и маскируем его job, удаляем publication code и alias, оставляем HTTP410 tombstones. Root-only архив обеспечивает восстановимость без сохранения публичного доступа. Ни аутентифицированный dashboard, ни новый sampler взамен не создаются. +## Влияние на инварианты + +Безопасность публичных агрегатов заменена отсутствием публичной выдачи. +Закрытость операторских данных сохраняется без зависимости от renderer. +Доверенная сводка monitor-state отменена; recoverable predecessor остаётся. + +## Влияние на контракты + +Legacy и future projection заменены breaking HTTP410 boundary. Совместимая +ревизия legacy usage events сохраняет текущий writer для оператора. +Dashboard-specific future events и monitor-input больше не реализуются. Существующие private logs, их формат/rotation и MCP readiness остаются. Ненужное сохранение public projection исключается из container release; постоянство закрытых usage events нового runtime остаётся release-обязательством. Локальные OpenMetrics — независимое от этого решения принятое намерение. + +## Отклонённые альтернативы + +Скрытие ссылки, noindex, удаление только HTML либо пароль перед старым +генератором оставляют ненужную цепочку публикации и противоречат полному +удалению. Безвозвратное уничтожение истории не требуется: архив закрыт. diff --git a/spec/designs/2026-09-10-mcp-container-distribution-design.md b/spec/designs/2026-09-10-mcp-container-distribution-design.md index aefa28b..35107f8 100644 --- a/spec/designs/2026-09-10-mcp-container-distribution-design.md +++ b/spec/designs/2026-09-10-mcp-container-distribution-design.md @@ -299,10 +299,12 @@ nginx и системные службы помещаются без старо capacity gate. Публикация образов и доставка corpus могут работать независимо. Увеличение лимита nginx не устраняет дефицит памяти или полосы. -На выделенном host остаются ai.v8std.ru, SSH, TLS renewal, защита и мониторинг. +На выделенном host остаются ai.v8std.ru, SSH, TLS renewal, защита, закрытые +журналы и health/readiness. Публичный dashboard отменён отдельным согласованным +`design:mcp-public-monitoring-retirement`; его job и private bridge не сохраняем. Посторонний vhost нельзя удалить вслепую: сначала backup вне host, проверка зависимости default TLS от его сертификата, затем согласованный cleanup и smoke -ai/TLS/monitoring. Никакая стадия CI не удаляет посторонние сайты автоматически. +ai/TLS/private operations. Никакая стадия CI не удаляет посторонние сайты автоматически. Три границы приёмки нельзя смешивать: diff --git a/spec/operations/2026-09-15-public-monitoring-retirement.md b/spec/operations/2026-09-15-public-monitoring-retirement.md new file mode 100644 index 0000000..2955463 --- /dev/null +++ b/spec/operations/2026-09-15-public-monitoring-retirement.md @@ -0,0 +1,131 @@ +# Public monitoring retirement — production evidence + +## Authority and outcome + +On2026-09-15 the user explicitly requested complete public-monitoring removal +and confirmed the scope including production, closed archival,410 responses, +producer removal and preservation of MCP/private logs. Approved package: +`eca506a`, `design:mcp-public-monitoring-retirement`. + +Only the monitoring publication chain was retired. This is **not** an MCP +runtime/container deployment, image publication, main push, capacity proof or +permission to remove other websites/services. + +## Resolved targets and protected archive + +Host: SSH alias `ai.v8std.ru`. Enabled nginx symlink resolves to +`/etc/nginx/sites-available/ai.v8std.ru`. On2026-09-15 at12:28UTC its exact +monitoring locations were replaced and nginx gracefully reloaded. + +Closed archive on the server: owner `root:root`, mode0700; its location +is recorded privately. Archived, not destroyed: + +- original nginx vhost plus installed candidate; +- `v8std-mcp-monitoring.timer` and `.service`, including service drop-in + `10-read-nginx-log.conf` (`SupplementaryGroups=adm`); +- `/var/www/ai.v8std.ru-monitoring/`, containing `index.html` and `stats.json`; +- `/opt/v8std-mcp/scripts/v8std_mcp_monitoring.py` and its exact Python3.12 pyc; +- twelve identified historical backup files: eleven renderer copies dated + April28 and one monitoring-unit backup. No unrelated backups were moved. + +Neither `www-data` nor `v8std-mcp` could read the archived JSON when checked +with `runuser -- test -r`. No archived payload was printed or copied into the +repository. This is a same-host recovery archive, not an off-host disaster backup. + +Config SHA256 before: +`288f013098b88e617e9c8920237fa6a72eab978c72378967e778b8a7bf61339d`. +After: +`943575822dc516d56852cd226ab423ee308a4f47da97a5c29f23e280e6690230`. +Diff changes only two monitoring locations and the final newline. MCP/TLS, +rate limits, upstreams and other vhosts were not changed. + +## Execution details and verification + +1. Confirmed previous public JSON200, active/enabled timer, healthy MCP. + Validated nginx before changes and checked exact original checksum again + immediately before replacement. Created archive via root `mktemp`/umask077. +2. `systemctl disable --now v8std-mcp-monitoring.timer`, then stopped its + service. The running oneshot exited on SIGTERM (`Result=signal`, status15). + A strict inactive guard correctly stopped the first archive attempt because + systemd reported `failed`, not `inactive`. Verified MainPID0 and terminal + state before proceeding, then cleared only this unit's failed state. +3. Installed scoped nginx candidate; `nginx -t` passed, then + `systemctl reload nginx`. An immediate same-second request still reached an + old worker and returned200; subsequent fresh requests returned410. There + was no restart or live worker kill. Both pre/post `nginx -t` reported the + existing unrelated certificate OCSP warning; syntax checks passed. +4. Moved the exact retired files into the closed archive. Ran daemon-reload + and masked both unit names to `/dev/null`. Final state for both: + `LoadState=masked`, `ActiveState=inactive`, `UnitFileState=masked`. + `list-timers --all v8std-mcp-monitoring.timer` listed zero timers. +5. Public output directory, active renderer/pyc and timer enablement link are + absent. Regular-file reference scan over nginx/systemd/cron directories and + active scripts found no remaining publication configuration after archival. + Root crontab contained no matching monitoring command. Unrelated pre-existing + broken ModemManager/udisks symlinks encountered by an earlier grep were not + altered; the final scan deliberately examined regular files. +6. Externally checked GET and HEAD for `/monitoring`, `/monitoring/`, + `/monitoring/stats.json`, `/monitoring/stats.json?retired=1`, + `/monitoring/unknown`, `/%6donitoring/stats.json`: all12 passed410 with + `Cache-Control: no-store`; GET body was only nginx's136-byte410 page; + HEAD body empty. No dashboard fields were returned. +7. Rechecked at12:35:20UTC, more than one former five-minute timer period after + reload: both units still masked/inactive, public output/renderer still absent, + JSON still410/no-store, MCP still the same active PID/start time. + +## MCP and private data preserved + +- MCP MainPID before/after: **673384**; ActiveEnterTimestamp unchanged: + `Thu 2026-09-10 09:47:05 UTC`. Active throughout checked boundaries. +- MCP unit SHA256 unchanged: + `072bd0e8ee1d2b24012085a4d7563f60ab5103fcef41de41a8ca9afb95df84ab`. +- `/healthz` HTTP200, `ok:true`,1423 pages,3281 vectors; corpus hash unchanged: + `4876122c8f2fa3c25c49afc0c986a72a976d4466e9b654041729ab42a634f4e4`. +- Real JSON-RPC `initialize` returned200/result and negotiated2024-11-05; + `tools/list` returned the five existing tools with no JSON-RPC error. +- `/var/lib/v8std-mcp/tool-usage.jsonl` remains0640 `v8std-mcp:v8std-mcp`; + nginx access log0640 `www-data:adm`. Contents were not rewritten or dumped. +- `/etc/logrotate.d/v8std-mcp-usage` remains0644 root-owned and retains + daily/rotate365/compress/copytruncate, su/create0640 v8std-mcp. +- No server package installation, Docker setup, SSH/renewal/fail2ban changes, + other-vhost cleanup or raw-log deletion was performed. + +## Recovery and future deployment + +Original paths are listed above; restore file owner/mode from archived metadata +if an explicitly approved future recovery needs them. Re-publication requires +new user authority; do not automatically restore old alias or unmask timer +during container rollback. For nginx repair, preserve410 boundary and validate +before reload. Previously downloaded client copies cannot be revoked. + +Shipped edge config and real local nginx conformance belong to the retirement +implementation. Keep future legacy backups post-retirement so rollback cannot +reintroduce removed generator/job. Persistent private usage logging of the +future container remains a separate unfinished Task6 gate; no private sampler +or public dashboard is needed to satisfy it. + +## Local implementation checks + +Implementation commit `5bfd76b70f3fed1a5c8d774cc37f14278945053e` removes the +1075-line generator and281-line dedicated tests, adds shipped410 locations and +real nginx conformance. With the pinned local nginx image +`sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c`: + +- RED:4 tests,21 failing method/path subtests before tombstones (301/200/404/405 + instead of410); GREEN:4 tests passed in0.441s, all21 GET/HEAD/POST cases. +- Existing stale HTML/JSON sentinel never served; root, MCP/health/version and + static-index success/error/cache boundaries preserved with dead MCP upstream. +- Fixture used loopback-only ephemeral port, UID10001, read-only mounts/root, + cap-dropALL, no-new-privileges,128MiB/1CPU. Exact owned containers removed and + absence checked after both runs; no image pull, network prune or production test. +- All18 server tests passed, including private usage logger behavior. Server + and logrotate source bytes unchanged from `eca506a`. +- Architecture focused run exposed missing explanatory section headings in two + candidate-only ADRs (one predating retirement). Added required sections without + changing front matter/lifecycle or weakening tests; all12 architecture + repository tests then passed in1.619s. +- Strict build passed:3282 vectors,1430 articles,0 HTML violations and3 license + files. Ordinary architecture validation and `git diff --check` passed. + +Full-suite completion and scoped reviews are recorded after their actual result; +none of these local checks authorizes whole-branch integration or image release. diff --git a/spec/operations/mcp-container-activation.md b/spec/operations/mcp-container-activation.md index dbb3273..1aa49da 100644 --- a/spec/operations/mcp-container-activation.md +++ b/spec/operations/mcp-container-activation.md @@ -7,6 +7,11 @@ Local disposable evidence is not a live installation, CI activation, published image proof, or target-host capacity guarantee. This runbook authorizes none of those operations. The current external scope remains image-only. +Public monitoring was separately retired on2026-09-15 with explicit production +approval; see [retirement evidence](2026-09-15-public-monitoring-retirement.md). +Do not restore its files/timer from an older backup. Below, preserved operational +checks mean private logs and MCP health/readiness, not a public dashboard. + ## Authority and stop conditions Initial activation requires an explicit operational request for the exact @@ -27,7 +32,7 @@ reconcile a crash. A healthy systemd wrapper is not application readiness. - Identify the precise nginx virtual hosts, includes, default server, certificate paths and renewal hooks. Ensure the ai.v8std.ru TLS endpoint and renewal no longer depend on an old site's configuration before any cleanup. -- Preserve SSH access, monitoring, fail2ban and certificate renewal. Removing an +- Preserve SSH access, private logs, health/readiness, fail2ban and certificate renewal. Removing an old website is not permission to remove unrelated operating services. - Back up the existing Python runtime, environment/dependencies, unit/drop-ins, nginx configuration, working corpus/cache and certificate configuration to @@ -100,7 +105,7 @@ proves support for 100,000 coding agents. Record the initial release journal, exact SHA/digests/corpus/configuration, public MCP/TLS and static delivery results, failure/rollback rehearsal and -monitoring checks in the verification record. External Catalog acceptance and +private operational checks and monitoring410 checks in the verification record. External Catalog acceptance and closure of the alternative PR remain separate delivery outcomes. ## Read-only legacy observations, not restoration proof @@ -131,7 +136,7 @@ host. Full private evidence is in the Task5 handoff's - Check the MCP certificate and any dependencies on other virtual hosts before changing TLS. No unrelated service deletion is authorized. Preserve SSH, HTTP/HTTPS, loopback8765 until cutover, renewal and - monitoring. Record the remaining private inventory outside Git. + private logs/health checks. Record the remaining private inventory outside Git. Before migration, inventory and hash the entire saved code, dependency lock and installed venv/interpreter, unit/configuration and coherent data set; protect @@ -195,7 +200,7 @@ Protected backup layout is fixed at `/var/lib/v8std-release/legacy/`: permissions are explicitly restored under UMask0077. - `unit` is the exact original `/etc/systemd/system/v8std-mcp.service` bytes; `upstream` is the exact inventoried managed upstream include. Both descriptors - require root ownership and mode0600 or0644. Other nginx/TLS/monitoring units + require root ownership and mode0600 or0644. Other nginx/TLS/operational units are not rewritten. `interpreter_sha256` binds `/usr/bin/python3.12`; a changed system interpreter fails closed and requires operator repair, not an automatic write into `/usr/bin`. @@ -212,20 +217,24 @@ directories; no broad cleanup is performed. Cache destination mtimes must be **f timestamps. This lets the unchanged legacy unit/default remote URLs and3600s refresh serve its four verified files without startup network fallback. It is an immediate bounded rollback guarantee, not an indefinite old-runtime hold. -Changing `tool-usage.jsonl` and monitoring inputs are never copied or overwritten; -monitoring-output preservation after container cutover is a separate Task6 gate. +Changing `tool-usage.jsonl` and private logs are never copied or overwritten; +persistent private logging after container cutover is a separate Task6 gate. +Public monitoring remains retired, including on rollback; construct the legacy +backup from the post-retirement inventory, without its removed generator/job. Install the reviewed `legacy-release-guard.conf` only as `/etc/systemd/system/v8std-mcp.service.d/10-release-guard.conf`, plus the exact `v8std-bootstrap-recover.service` and `.timer`. The drop-in preserves original -ExecStart and existing Before=monitoring/multi-user ordering. Its privileged +ExecStart and existing legacy unit ordering. A dangling ordering reference to +the retired monitor does not authorize unmasking or restoring it. Its privileged ExecCondition fails closed while a bootstrap is in flight or any container is accepted, even if active.json is missing. Recovery marks legacy start allowed only after the candidate is stopped and config/data/upstream are restored. The timer is required by legacy startup, runs at boot+5s and every15s after its service completes, independently of SSH. Recovery is not ordered Before=legacy: that would deadlock when it starts the original service. Validate this topology -with native systemd, including enabled-legacy reboot and monitoring ordering. +with native systemd, including enabled-legacy reboot while the retired monitor +remains masked. The guard checks exact root-owned unit/drop-in bytes, active/enabled timer and loaded non-stale manager state before legacy stop. No automatic installation, enablement, daemon reload of an unreviewed unit set, or fallback guard is supplied. diff --git a/spec/operations/mcp-container-verification.md b/spec/operations/mcp-container-verification.md index 0a3bb8d..b208745 100644 --- a/spec/operations/mcp-container-verification.md +++ b/spec/operations/mcp-container-verification.md @@ -4,14 +4,22 @@ This is implementation evidence, not authorization to publish or activate a host The accepted design and contracts remain authoritative. Unfinished or external checks below must not be reported as passed. +**2026-09-15 direction update:** public monitoring preservation and the proposed +private monitor-state bridge are superseded by the approved retirement design. +Production410, masked units and closed archive are recorded separately in +[retirement evidence](2026-09-15-public-monitoring-retirement.md). Earlier dated +measurements and design-pause text below describe history, not current tasks. +Private container logging, CI and the other Task6 gates remain incomplete. + ## Scope and baseline - Main baseline: `b7bef11e145a188b30e7a7b17df2be4cb1acbd0c`. - Approved design package: `537fe79`; implementation plan: `a3b2474`, with later recorded interface refinements on the same feature branch. - Working branch: `codex/mcp-container-distribution-design`, primary checkout. -- No live host changes, image publication, Catalog submission, PR closure or - production activation are established by this record. +- No container host installation, image publication, Catalog submission, PR + closure or container activation are established by this record. The separately + authorized public-monitoring retirement is the only later live change here. - Capacity of 100,000 coding agents is a target, **not a measured result**. ## Completed local evidence @@ -82,8 +90,12 @@ and cache/port configuration. Scoped re-review confirmed all five addressed. One pre-existing image-alt HTML protection edge case remains assigned to final integration review; task acceptance is not final release acceptance. +Current equivalent regression command (the historical189-test run included the +now-retired dashboard tests): + ```sh -.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_presentation tests.test_v8std_mcp_runtime tests.test_v8std_mcp_index tests.test_v8std_mcp_snippet tests.test_v8std_mcp_server tests.test_v8std_mcp_combined tests.test_v8std_mcp_capacity tests.test_v8std_mcp_monitoring +.venv/bin/python -m unittest tests.test_v8std_mcp_snapshot_format tests.test_v8std_mcp_snapshots tests.test_v8std_mcp_presentation tests.test_v8std_mcp_runtime tests.test_v8std_mcp_index tests.test_v8std_mcp_snippet tests.test_v8std_mcp_server tests.test_v8std_mcp_combined tests.test_v8std_mcp_capacity +V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement -v .venv/bin/python -m tests.mcp_runtime_benchmark ``` @@ -607,7 +619,8 @@ duplicates, capacity rejection and boot fencing. Full historical1423-page/ control and zero-network fresh-cache restoration. Usage logs are not overwritten or treated as corpus identity. Existing Starlette warning remains unsuppressed. -Task6 still must preserve monitoring: the current timer reads the old unit and +Historical requirement, superseded by the2026-09-15 retirement: Task6 was to +preserve monitoring because the then-current timer read the old unit and flat usage log, while container launch does not yet supply new usage events. This is an integration defect, not the deferred dashboard/OpenMetrics redesign. The retained image-alt parser defect and warning triage remain final obligations. @@ -663,8 +676,8 @@ new Critical, Important or Minor findings. The reviewer checked the scenarios and evidence against the fix diff without duplicating the test runs. This completes only the context/harness repair slice, not Task6 or release acceptance. -The earlier monitoring classification above is refined: preserving the existing -dashboard is required, but the approved package does not define how its +Historical classification before retirement: preserving the existing +dashboard was required, but the approved package did not define how its unprivileged reader obtains fresh live state from the root-owned container controller. A durable COMMITTED receipt is not runtime liveness; the controller unit being active is not MCP uptime; snapshot `loaded_at` is not process start. @@ -672,7 +685,7 @@ The real monitor reader and controller-status probes reproduce these semantic counterexamples even while the existing monitoring tests pass. Under `v8std-architecture` failure recovery, implementation of this integration -and remaining Task6 work is paused for brainstorming and revised design/plan +and remaining Task6 work was paused for brainstorming and revised design/plan approval. A bounded private atomic state file is a proposed approach, not an implemented or accepted interface. Its producer/reader authority, identity, freshness, error behavior and compatibility must be specified before code. diff --git a/spec/operations/mcp-first-container-release-roadmap.md b/spec/operations/mcp-first-container-release-roadmap.md index 581352d..96baf23 100644 --- a/spec/operations/mcp-first-container-release-roadmap.md +++ b/spec/operations/mcp-first-container-release-roadmap.md @@ -6,6 +6,12 @@ structured plan: их нельзя требовать до merge-ready и одновременно выполнять только после проверенного main. +Уточнение15сентября: публичный мониторинг отдельно отключён по явному +согласованию, см. [проверки и архив](2026-09-15-public-monitoring-retirement.md). +Сохранение dashboard и private sampler больше не входят в выпуск. Закрытые +логи/rotation и health/readiness остаются; подключение persistent usage log +нового контейнера всё ещё требует реализации и проверки в Task6. + **Goal:** опубликовать рабочий `ghcr.io/zeegin/v8std-mcp`, обеспечить публичный источник индексов и перевести `ai.v8std.ru/mcp` на тот же опубликованный образ. @@ -141,7 +147,9 @@ VIRTUAL_ENV="$PWD/.venv" ./scripts/zensical_docs.sh build --strict inventory и точный список targets. Этот документ сам по себе не выдаёт полномочия. 1. Сохранить вне сервера backup legacy code/config/data, nginx/TLS/renewal и - мониторинга. Проверить чтение backup и путь запуска старого сервиса без сети. + закрытых операционных данных. Использовать post-retirement inventory, не + возвращать generator/job/public files из старых backup. Проверить чтение + backup и путь запуска старого сервиса без сети. 2. Подготовить статическое хранилище `/indexes/v1/` и restricted upload identity. nginx обслуживает его независимо от Docker/runtime. В первой фазе current upstream MCP не менять. diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 1fa5992..6e4c6cc 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -548,9 +548,9 @@ review; do not mark the live migration complete from these fixtures: `spec/operations/mcp-container-verification.md`, public installation docs and Catalog release metadata/harness under `deploy/docker-catalog/`, `scripts/check_mcp_container.py`, `tests/test_v8std_mcp_distribution.py`; -existing monitoring integration in `scripts/v8std_mcp_monitoring.py`, -`scripts/v8std_mcp_usage.logrotate`, `tests/test_v8std_mcp_monitoring.py`, -release launch/host templates and their focused integration tests; +private usage logging in `scripts/v8std_mcp_usage.logrotate`, release launch/host +templates and their focused integration tests; public retirement conformance in +`tests/test_v8std_mcp_monitoring_retirement.py` and its separate approved plan; the retained final parser regression in `scripts/v8std_mcp_presentation.py` and `tests/test_v8std_mcp_presentation.py`. @@ -583,18 +583,18 @@ Pages manifest. Every runtime deployment references published exact digest. image-only scope without claiming Docker Catalog acceptance. Preserve the `longLived` source declaration and distinguish local test-catalog diagnostics from the actual Docker-published catalog; no upstream PR is a release prerequisite. -- [ ] **VERIFY preserved monitoring:** First demonstrate the migration regression: - the existing timer reads the legacy unit and flat usage log, while the new - container emits no usage file. Preserve the existing public projection and - historical log readers, feed it actual accepted-runtime state and new tool - events, and verify rollback/restart/rotation without stale status or lost - history. Keep logs private, the existing unprivileged monitor and non-root - runtime; grant neither Docker-group membership nor a Docker socket to them. - Exercise the real launcher/logger/aggregator path, including failed or missing - status, rather than accepting a running controller as a healthy MCP runtime. - This implements the approved preservation of monitoring, not the separately - deferred dashboard/events-schema/OpenMetrics redesign. Any necessary public - schema or trust-boundary change must return to design before implementation. +- [ ] **VERIFY private operations and retired public monitoring:** The approved + `design:mcp-public-monitoring-retirement` supersedes the earlier preservation + task. Consume its410/no-store conformance and production retirement evidence; + never recreate dashboard, timer, aggregator or private monitor-state bridge. + Independently wire and verify persistent private usage logging for the new + runtime through its real launcher: rollback/restart/rotation must preserve + existing history and new events without restoring logs from a release backup. + Preserve the non-root runtime and existing rotation policy; grant no Docker + socket/group. Test actual MCP health/readiness, not controller liveness. + This checkbox remains incomplete until container logger integration is + implemented and verified. Dashboard retirement alone does not complete it. + Local OpenMetrics remains a separate deferred design. - [ ] **VERIFY retained parser finding:** Add RED/GREEN for `![](...)` followed by a visible internal link. HTML-looking image-alt text must not suppress rebasing or unknown-target validation of subsequent visible links. diff --git a/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md index 4cce6fa..942fbed 100644 --- a/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md +++ b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md @@ -79,6 +79,11 @@ V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8s **Files:** Modify candidate-only `spec/plans/2026-09-10-mcp-container-distribution-plan.md`, `spec/designs/2026-09-10-mcp-container-distribution-design.md`, `spec/operations/mcp-container-activation.md`, `spec/operations/mcp-container-verification.md`, `spec/operations/mcp-first-container-release-roadmap.md`; create `spec/operations/2026-09-15-public-monitoring-retirement.md`. +Existing architecture tests also require the five canonical explanatory sections +in candidate ADRs `spec/adr/2026-09-15-retire-public-mcp-monitoring.md` and +`spec/adr/2026-09-15-mcp-private-container-monitor-input.md`. Add missing sections +without changing their front matter, meaning or historical decision lifecycle. + **Interfaces:** Consume Task1 tests and controller's actual production evidence. Produce unambiguous current runbooks, retaining dated historical observations as history. - [ ] Replace the container plan's public-monitor preservation step with retirement evidence plus persistent private logging/rotation checks for the new runtime (remaining incomplete until actually verified). Remove generator/tests from active test commands; replace with retirement conformance. Do not claim new container telemetry has been implemented. From fc6706d89fb4cb5f38ab983b10f177348d3d37e8 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:48:50 +0300 Subject: [PATCH 45/88] fix(architecture): honor terminal lifecycle in fitness gate --- scripts/v8std_architecture_validation.py | 2 + tests/test_v8std_architecture_validation.py | 149 ++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/scripts/v8std_architecture_validation.py b/scripts/v8std_architecture_validation.py index cc7326b..8044ae7 100644 --- a/scripts/v8std_architecture_validation.py +++ b/scripts/v8std_architecture_validation.py @@ -684,6 +684,8 @@ def validate_merge_readiness(graph: ArchitectureGraph) -> list[ValidationIssue]: for document in graph.documents.values(): if document.kind not in {"invariant", "contract"}: continue + if states[document.key].intersection({"SUPERSEDED", "CANCELLED", "DEPRECATED", "RETIRED"}): + continue required_when = document.front_matter.get("required_when", "accepted") must_exist = required_when == "accepted" or ( required_when == "implemented" and "IMPLEMENTED" in states[document.key] diff --git a/tests/test_v8std_architecture_validation.py b/tests/test_v8std_architecture_validation.py index 4d0c3ba..1d910d6 100644 --- a/tests/test_v8std_architecture_validation.py +++ b/tests/test_v8std_architecture_validation.py @@ -702,6 +702,155 @@ def test_fitness_evidence_is_deferred_until_implemented(self) -> None: codes(validate_merge_readiness(implemented_graph)), ) + def test_terminal_contracts_do_not_require_fitness_evidence(self) -> None: + for relation, terminal_state in ( + ("supersedes", "SUPERSEDED"), + ("cancels", "CANCELLED"), + ("deprecates", "DEPRECATED"), + ): + for required_when in ("accepted", "implemented"): + with self.subTest(relation=relation, required_when=required_when): + feature = design("feature") + fields = { + "scope": "product", + "version": 1, + "revision": 0, + "compatibility": "backward-compatible", + "design": "design:feature", + "producer": "producer", + "consumers": ["consumer"], + "requirements": [], + "governs": ["scripts/feature.py"], + "conformance": {"module": "tests.test_module_that_does_not_exist"}, + "required_when": required_when, + "supersedes": [], + "deprecates": [], + } + old_contract = document("contract", "OLD_API", **fields) + active_contract = document("contract", "ACTIVE_API", **fields) + successor = document( + "contract", + "SUCCESSOR_API", + **{ + **fields, + relation: ["contract:OLD_API@1.0"], + "conformance": {"module": "tests.test_v8std_architecture_validation"}, + }, + ) + documents = [feature, old_contract, active_contract, successor] + expected_state = {terminal_state} + if required_when == "implemented": + documents.append( + document( + "plan", + "feature", + design="design:feature", + implements=["contract:OLD_API@1.0", "contract:ACTIVE_API@1.0"], + checkbox_count=1, + checked_count=1, + ) + ) + expected_state.add("IMPLEMENTED") + graph = build_graph(documents) + self.assertEqual(validate_graph(graph), []) + self.assertEqual( + compute_states(graph, frozenset(graph.documents))["contract:OLD_API@1.0"], + frozenset(expected_state), + ) + + issues = validate_merge_readiness(graph) + + self.assertEqual( + [(item.code, item.path) for item in issues], + [("MISSING_FITNESS_EVIDENCE", "spec/contract/active-api-v1-r0.md")], + ) + # Terminal artifacts still participate in reference validation. + unresolved_graph = build_graph(documents[1:]) + self.assertEqual( + sorted( + item.path for item in validate_graph(unresolved_graph) + if item.code == "DANGLING_REFERENCE" and "spec/contract/" in item.path + ), + [ + "spec/contract/active-api-v1-r0.md", + "spec/contract/old-api-v1-r0.md", + "spec/contract/successor-api-v1-r0.md", + ], + ) + + def test_retired_invariant_does_not_require_fitness_evidence(self) -> None: + for required_when in ("accepted", "implemented"): + with self.subTest(required_when=required_when): + feature = design("feature", introduces=("OLD_REQUIREMENT", "ACTIVE_REQUIREMENT")) + old_decision = adr("OLD_DECISION", "design:feature", requirements=("OLD_REQUIREMENT",)) + old_invariant = document( + "invariant", + "OLD_INVARIANT", + scope="product", + introduced_by="adr:OLD_DECISION", + requirements=["OLD_REQUIREMENT"], + check={"module": "tests.test_module_that_does_not_exist"}, + required_when=required_when, + ) + active_invariant = document( + "invariant", + "ACTIVE_INVARIANT", + scope="product", + introduced_by="adr:SUCCESSOR", + requirements=["ACTIVE_REQUIREMENT"], + check={"module": "tests.test_module_that_does_not_exist"}, + required_when=required_when, + ) + documents = [feature, old_decision, old_invariant, active_invariant] + if required_when == "implemented": + documents.append( + document( + "plan", + "feature", + design="design:feature", + implements=["invariant:OLD_INVARIANT", "invariant:ACTIVE_INVARIANT"], + checkbox_count=1, + checked_count=1, + ) + ) + before_retirement = build_graph(documents) + self.assertEqual( + compute_states(before_retirement, frozenset(before_retirement.documents))[ + "invariant:OLD_INVARIANT" + ], + frozenset({"ACCEPTED", "IMPLEMENTED"}), + ) + retirement = design( + "retirement", + uses=("ACTIVE_REQUIREMENT",), + requirement_cancels=("OLD_REQUIREMENT",), + ) + successor = adr( + "SUCCESSOR", + "design:retirement", + requirements=("ACTIVE_REQUIREMENT",), + cancels=("adr:OLD_DECISION",), + invariants={ + "introduces": ["invariant:ACTIVE_INVARIANT"], + "preserves": [], + "replaces": {}, + "cancels": ["invariant:OLD_INVARIANT"], + }, + ) + graph = build_graph([*documents, retirement, successor]) + self.assertEqual(validate_graph(graph), []) + self.assertEqual( + compute_states(graph, frozenset(graph.documents))["invariant:OLD_INVARIANT"], + frozenset({"RETIRED", "IMPLEMENTED"} if required_when == "implemented" else {"RETIRED"}), + ) + + issues = validate_merge_readiness(graph) + + self.assertEqual( + [(item.code, item.path) for item in issues], + [("MISSING_FITNESS_EVIDENCE", "spec/invariant/active-invariant.md")], + ) + def test_rejects_unknown_fitness_timing(self) -> None: feature = design("feature") contract = document( From 4f4ea7d4d8fc0cd7bcd4ea89bd3e05cc829f9d02 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 15:52:44 +0300 Subject: [PATCH 46/88] docs: close monitoring retirement verification --- ...2026-09-15-public-monitoring-retirement.md | 34 ++++++++++- ...5-mcp-public-monitoring-retirement-plan.md | 57 +++++++++++++++++-- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/spec/operations/2026-09-15-public-monitoring-retirement.md b/spec/operations/2026-09-15-public-monitoring-retirement.md index 2955463..94e702a 100644 --- a/spec/operations/2026-09-15-public-monitoring-retirement.md +++ b/spec/operations/2026-09-15-public-monitoring-retirement.md @@ -127,5 +127,35 @@ real nginx conformance. With the pinned local nginx image - Strict build passed:3282 vectors,1430 articles,0 HTML violations and3 license files. Ordinary architecture validation and `git diff --check` passed. -Full-suite completion and scoped reviews are recorded after their actual result; -none of these local checks authorizes whole-branch integration or image release. +The first full suite started before the ADR-heading fix and ended with that +single known failure:632 tests in328.641s,10 opt-in skips. A fresh complete run +after the correction passed:632 tests in336.791s,10 opt-in Docker skips. +Enabled retirement nginx conformance was separately run and passed above. +Existing Starlette deprecation warning remains recorded for release triage. + +Independent Task1 code review and Task2/cross-retirement graph review approved +spec compliance and quality without blocking findings. Production assertions +were reconciled by the controller against its actual SSH/HTTP tool outputs; +reviewers did not perform another production operation. + +Subsequent real merge-ready exposed a validator implementation defect: retired +contracts still required their removed Python fitness module. Corrective Task3 +is limited to honoring already-computed terminal lifecycle states while keeping +all active fitness/incomplete-plan gates. It changes no product/process policy. +Its focused verification is recorded separately; the632-test run above predates +that correction. The broader container and CI plans are still incomplete. +None of these checks authorizes whole-branch integration or image release. + +Validator correction `cbbbd17df13abd1e563d7a08aea59f9de6b00df2` adds two lines +only to the fitness loop and two regression tests (eight lifecycle/timing +subcases). RED produced all eight expected obsolete-evidence failures; GREEN +passed47 architecture validation/CLI/repository tests in5.244s. Active missing +fitness still fails, and retired artifacts still undergo reference validation. +Real merge-ready no longer requires removed monitoring tests; it still rejects +incomplete plans. No stub module, skipped existing test or weakened active +declaration was added to accommodate removal. + +Independent final scoped follow-up approved Task3 spec compliance and quality +without findings. The retirement implementation plan is complete. Whole-branch +merge-ready still rejects only the two unfinished container/CI plans; this +work intentionally leaves main, remote branches and MCP runtime deployment alone. diff --git a/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md index 942fbed..769c629 100644 --- a/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md +++ b/spec/plans/2026-09-15-mcp-public-monitoring-retirement-plan.md @@ -47,7 +47,7 @@ implements: **Interfaces:** Consume the shipped `edge-locations.conf` using a real local nginx fixture (pinned local nginx image from `tests/test_v8std_mcp_release_docker.py`); produce opt-in HTTP conformance with `V8STD_MONITORING_RETIREMENT_DOCKER=1`. No host SSH or production work in the subagent. -- [ ] **RED:** Start a disposable nginx with shipped include, bounded timeouts, dead upstream, legacy `monitoring/index.html` and `stats.json` containing a sentinel under fixture root. Probe GET/HEAD paths `/monitoring`, `/monitoring/`, `/monitoring/stats.json?x=1`, `/monitoring/unknown`. Assert literal410/no-store and absent sentinel. Before implementation existing config fails this boundary. Use loopback-only published port and exact UUID-owned container cleanup; never prune or pull. Preserve health/index boundaries in the fixture. Add architecture lifecycle expectations that old dashboard/input designs are superseded and OpenMetrics remains accepted. +- [x] **RED:** Start a disposable nginx with shipped include, bounded timeouts, dead upstream, legacy `monitoring/index.html` and `stats.json` containing a sentinel under fixture root. Probe GET/HEAD paths `/monitoring`, `/monitoring/`, `/monitoring/stats.json?x=1`, `/monitoring/unknown`. Assert literal410/no-store and absent sentinel. Before implementation existing config fails this boundary. Use loopback-only published port and exact UUID-owned container cleanup; never prune or pull. Preserve health/index boundaries in the fixture. Add architecture lifecycle expectations that old dashboard/input designs are superseded and OpenMetrics remains accepted. ```python self.assertEqual(status, 410) @@ -55,7 +55,7 @@ self.assertEqual(headers.get('cache-control'), 'no-store') self.assertNotIn(b'private-monitoring-sentinel', body) ``` -- [ ] **GREEN:** Remove generator and its dedicated tests; leave logger/server/logrotate unchanged. Add these locations to shipped include. Update architecture repository expectations for the successor graph; preserve historical aliases. +- [x] **GREEN:** Remove generator and its dedicated tests; leave logger/server/logrotate unchanged. Add these locations to shipped include. Update architecture repository expectations for the successor graph; preserve historical aliases. ```nginx location = /monitoring { @@ -68,7 +68,7 @@ location ^~ /monitoring/ { } ``` -- [ ] **VERIFY:** Run real nginx conformance and existing private logger tests. Record RED/GREEN output and prove owned fixture cleanup. Search active publication surfaces for remaining renderer/monitoring links; no source-text-only substitute for HTTP behavior. +- [x] **VERIFY:** Run real nginx conformance and existing private logger tests. Record RED/GREEN output and prove owned fixture cleanup. Search active publication surfaces for remaining renderer/monitoring links; no source-text-only substitute for HTTP behavior. ```sh V8STD_MONITORING_RETIREMENT_DOCKER=1 .venv/bin/python -m unittest tests.test_v8std_mcp_monitoring_retirement -v @@ -86,9 +86,9 @@ without changing their front matter, meaning or historical decision lifecycle. **Interfaces:** Consume Task1 tests and controller's actual production evidence. Produce unambiguous current runbooks, retaining dated historical observations as history. -- [ ] Replace the container plan's public-monitor preservation step with retirement evidence plus persistent private logging/rotation checks for the new runtime (remaining incomplete until actually verified). Remove generator/tests from active test commands; replace with retirement conformance. Do not claim new container telemetry has been implemented. -- [ ] Clarify preserve-monitoring prose to mean private logs and health/readiness; link the retirement decision for old dated statements. Record exact production archive/PID/codes/unit states with no payloads. Scan `docs`, `.github`, `deploy`, scripts and current runbooks; historical frozen design references remain as evidence, not active instructions. -- [ ] Run semantic impact and ordinary architecture validation, strict build then full suite once. Record existing whole-branch release gates separately; no merge-ready or production-release claim from this scoped completion. +- [x] Replace the container plan's public-monitor preservation step with retirement evidence plus persistent private logging/rotation checks for the new runtime (remaining incomplete until actually verified). Remove generator/tests from active test commands; replace with retirement conformance. Do not claim new container telemetry has been implemented. +- [x] Clarify preserve-monitoring prose to mean private logs and health/readiness; link the retirement decision for old dated statements. Record exact production archive/PID/codes/unit states with no payloads. Scan `docs`, `.github`, `deploy`, scripts and current runbooks; historical frozen design references remain as evidence, not active instructions. +- [x] Run semantic impact and ordinary architecture validation, strict build then full suite once. Record existing whole-branch release gates separately; no merge-ready or production-release claim from this scoped completion. ```sh .venv/bin/python scripts/v8std_architecture.py impact --root . --base-ref main @@ -96,3 +96,48 @@ without changing their front matter, meaning or historical decision lifecycle. VIRTUAL_ENV="$PWD/.venv" ./scripts/zensical_docs.sh build --strict .venv/bin/python -m unittest discover -s tests -v ``` + +### Task 3: Respect retired fitness declarations at the merge gate + +**Files:** Modify `scripts/v8std_architecture_validation.py` and +`tests/test_v8std_architecture_validation.py` (CLI regression only if needed in +`tests/test_v8std_architecture_cli.py`). + +**Interfaces:** `validate_merge_readiness(graph)` consumes existing +`compute_states(graph, frozenset(graph.documents))`. Preserve all currently +active fitness gates and incomplete-plan detection; no process/schema change. + +Actual retirement exposed an implementation defect: the gate computes lifecycle +but still requires missing test modules for superseded/deprecated legacy +contracts. The process requires fitness for accepted/implemented obligations, +not retention of executable consumers of explicitly retired boundaries. + +- [x] **RED:** Add focused graphs with terminal contracts/invariants and missing + fitness modules, alongside a still-active missing declaration. Exercise + `validate_merge_readiness`: terminal declarations must not produce missing + evidence, active ones must still fail. Cover SUPERSEDED, CANCELLED, DEPRECATED, + RETIRED states and a previously IMPLEMENTED then retired artifact. Preserve + existing tests of accepted/implemented timing and incomplete plans. + +```python +issues = validate_merge_readiness(graph) +self.assertEqual([item.path for item in issues + if item.code == 'MISSING_FITNESS_EVIDENCE'], [active_path]) +``` + +- [x] **GREEN:** In the fitness loop only, ignore terminal artifacts using the + computed state; do not filter documents out of reference/graph validation. + +```python +if states[document.key].intersection({'SUPERSEDED', 'CANCELLED', 'DEPRECATED', 'RETIRED'}): + continue +``` + +- [x] **VERIFY:** Run all architecture validation/CLI/repository tests; review + actual diff. Run real repository merge-ready: only other unfinished plans may + remain; no missing fitness for retired monitoring and no relaxed active gate. + +```sh +.venv/bin/python -m unittest tests.test_v8std_architecture_validation tests.test_v8std_architecture_cli tests.test_v8std_architecture_repository -v +.venv/bin/python scripts/v8std_architecture.py validate --root . --base-ref main --merge-ready +``` From e0276b5ba013de084b9c319113b4e80e8e63c23c Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 16:12:35 +0300 Subject: [PATCH 47/88] docs: detail remaining runtime release checks --- ...6-09-10-mcp-container-distribution-plan.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/spec/plans/2026-09-10-mcp-container-distribution-plan.md b/spec/plans/2026-09-10-mcp-container-distribution-plan.md index 6e4c6cc..25219f2 100644 --- a/spec/plans/2026-09-10-mcp-container-distribution-plan.md +++ b/spec/plans/2026-09-10-mcp-container-distribution-plan.md @@ -595,12 +595,48 @@ Pages manifest. Every runtime deployment references published exact digest. This checkbox remains incomplete until container logger integration is implemented and verified. Dashboard retirement alone does not complete it. Local OpenMetrics remains a separate deferred design. + + Execution slice (approved continuation2026-09-15): retain the existing + `/var/lib/v8std-mcp/tool-usage.jsonl` history and rotation stanza unchanged. + The root-owned host launcher prepares a separate fixed regular file + `/var/log/v8std-mcp/tool-usage.jsonl`, outside controller state, slots and + caches. Parent is root:root0700; file is10001:root0640. Creation is exclusive, + no-follow and non-truncating; existing wrong type, links, ownership or public + permissions fail closed without rewriting history. Only this file is mounted + read-write at `/var/log/v8std-mcp-usage.jsonl`, passed by `--usage-log`. + Do not expose the log directory, archives, release state or Docker socket. + Existing-container reuse must verify this logging binding before start; + ownership-checked stop/recovery must remain possible on a malformed candidate. + + Add RED behavior tests for the actual `HostAdapter.start` command boundary, + non-destructive preparation/reuse, hostile file shapes/permissions and + persistence across restart/rollback. Then implement the minimal launcher + integration and a separate root-run daily365/compress/copytruncate stanza + (`su root root`, no inode replacement). Native Linux evidence must exercise + non-root writes and actual logrotate, preserving pre-rotation events in the + archive and later events in the current file. Cover the existing legacy log + and actual MCP responses; no live host mutation or broad Docker cleanup. + Use focused release/server tests and opt-in disposable Docker integration; + record commands/results and exact limitations before independent review. + Existing best-effort JSONL schema stays unchanged: copytruncate has a known + copy/truncate race, so this does not promise lossless audit or exactly-once. + See [upstream logrotate manual](https://github.com/logrotate/logrotate/blob/main/logrotate.8.in). - [ ] **VERIFY retained parser finding:** Add RED/GREEN for `![](...)` followed by a visible internal link. HTML-looking image-alt text must not suppress rebasing or unknown-target validation of subsequent visible links. Preserve actual code/literal content, source offsets and canonical hashes. This is the concrete deferred Task3 review finding, not a new Markdown interpretation contract or permission to waive a pre-existing release defect. + + Execution slice (approved continuation2026-09-15): first add a fixture with + an HTML-looking image alt and a following known internal link, and a separate + fixture with a following unknown target. Assert literal rebased destinations + and `unresolved_internal_link`, respectively; include real HTML code/literal + controls and unchanged source/hash. Run + `.venv/bin/python -m unittest tests.test_v8std_mcp_presentation -v` for RED, + isolate image-alt parser side effects without changing rendered Markdown + semantics, then rerun presentation and snapshot-format suites for GREEN. + Commit the bounded fix for independent review before final CI integration. - [ ] **Final gates:** Run semantic impact on actual paths, CLI `impact`, `validate --merge-ready`, all applicable fitness; strict build, then full suite; container smoke and shared-host mixed load on disposable local stack, review From 61ec00cc995fc8b511dc72c091bc7ad9696aeb5c Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Tue, 15 Sep 2026 16:14:29 +0300 Subject: [PATCH 48/88] fix: isolate image alt parsing from presentation collectors --- scripts/v8std_mcp_presentation.py | 14 +++++++++- tests/test_v8std_mcp_presentation.py | 41 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/v8std_mcp_presentation.py b/scripts/v8std_mcp_presentation.py index 5ffe77f..6bef8e9 100644 --- a/scripts/v8std_mcp_presentation.py +++ b/scripts/v8std_mcp_presentation.py @@ -329,6 +329,18 @@ def ref_rule(state, begin, end, silent): md.block.ruler.at("reference", ref_rule) + def image_rule(state, silent): + # Image labels are parsed into alt children, not document-level HTML + # or links. Keep the parser's tokens, but discard their collector side + # effects before wrap() records the image's own destination. + node_count, tag_count, text_count = len(nodes), len(tags), len(text_spans) + try: + return image(state, silent) + finally: + del nodes[node_count:] + del tags[tag_count:] + del text_spans[text_count:] + def wrap(rule, kind): def run(state, silent): start, count = state.pos, len(state.tokens) @@ -358,7 +370,7 @@ def run(state, silent): return accepted return run - for name, rule in (("link", link), ("image", image), ("autolink", autolink), + for name, rule in (("link", link), ("image", image_rule), ("autolink", autolink), ("html_inline", _html_inline), ("text", inline_text)): md.inline.ruler.at(name, wrap(rule, name)) tokens = md.parse(source) diff --git a/tests/test_v8std_mcp_presentation.py b/tests/test_v8std_mcp_presentation.py index 27e566b..c403db0 100644 --- a/tests/test_v8std_mcp_presentation.py +++ b/tests/test_v8std_mcp_presentation.py @@ -1,4 +1,5 @@ """Presentation changes link nodes, never retrieval input or code literals.""" +import hashlib import importlib import importlib.util import json @@ -132,6 +133,46 @@ def markdown(self, text): def validate(self, text): self.module().validate_links(text, canonical_site_url=PUBLIC, page_paths=PATHS) + def test_image_alt_html_does_not_hide_following_link_destinations(self): + for alt in ("", "
    ", "