diff --git a/.gitignore b/.gitignore index d046aba4..4747e308 100644 --- a/.gitignore +++ b/.gitignore @@ -225,3 +225,7 @@ test_result.txt _mcp_reindex_client.py _measure_*.py .red_team_state.md + +# E17 experiment: regenerable prompt shards (rebuild via e17_export_*.py) +experiments/bootstrap/e17_shards/*.txt +experiments/bootstrap/e17_judge/*.txt diff --git a/.opencode/plugin/mscodebase_gate.ts b/.opencode/plugin/mscodebase_gate.ts new file mode 100644 index 00000000..d8df46f0 --- /dev/null +++ b/.opencode/plugin/mscodebase_gate.ts @@ -0,0 +1,130 @@ +// mscodebase_gate.ts — opencode-хук: детерминированные заметки MSCodeBase в момент действия. +// +// Два гейта на `git commit` (мост: `python -m src.cli `, cwd=проект; CLI --project игнорирует): +// 1. stale_detector — BLOCK при дрейфе версий (доказан в E6: модель отказ не обходит). +// 2. graph_query isolation — ADVISORY: «тихий слом» графа (осиротение узла). Soft: у гейта есть FP. +// +// Advisory реализован через before+after: pre-commit diff считается в `before` и кладётся в stash по +// callID; текст дописывается в результат в `after` (не блокирует). Блокирующий режим — только stale +// (объективный) и явный MSCODEBASE_ISOLATION_GATE=block. +// +// Переключатели (env): MSCODEBASE_PY, MSCODEBASE_STALE_GATE=block|off, +// MSCODEBASE_ISOLATION_GATE=advisory|block|off, MSCODEBASE_GATE_LOG. +// Хук никогда не роняет хост: любые сбои → лог, без throw (кроме намеренного блока). +// +// ВАЖНО: вызов python — через node:child_process, НЕ через `$` из контекста плагина: в opencode +// 1.18.23 контекст фабрики не содержит `$` (API дрейфанул) → `TypeError: $ is not a function`. + +import type { Plugin } from "@opencode-ai/plugin" +import { appendFileSync, mkdirSync } from "node:fs" +import { execFile } from "node:child_process" +import { dirname } from "node:path" + +const PY = process.env.MSCODEBASE_PY ?? "D:/Project/MSCodeBase/venv/Scripts/python.exe" +const LOG = process.env.MSCODEBASE_GATE_LOG ?? "C:/Users/misha/AppData/Local/Temp/opencode/mscodebase_gate.log" +const STALE_MODE = process.env.MSCODEBASE_STALE_GATE ?? "block" +const ISO_MODE = process.env.MSCODEBASE_ISOLATION_GATE ?? "advisory" + +const STALE_ARGS = "{}" +const ISO_ARGS = '{"action":"isolation","kwargs":{"restraint":true}}' + +function log(rec: Record) { + try { + mkdirSync(dirname(LOG), { recursive: true }) + appendFileSync(LOG, JSON.stringify({ t: Date.now(), ...rec }) + "\n") + } catch { + // never break the host + } +} + +function cli(directory: string, tool: string, argsJson: string): Promise { + return new Promise((resolve) => { + try { + execFile( + PY, + ["-m", "src.cli", tool, argsJson], + { cwd: directory, timeout: 90000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }, + (_err, stdout) => resolve(stdout ?? ""), + ) + } catch (e) { + log({ kind: "cli_spawn_error", err: String(e).slice(0, 200) }) + resolve("") + } + }) +} + +function staleDrift(out: string): number { + const m = /Total drift instances:\s*(\d+)/i.exec(out) + return m ? parseInt(m[1], 10) : 0 +} + +function isolationCount(out: string): number { + const r = /removed_last_caller:\s*(\d+)/i.exec(out) + const n = /new_orphan:\s*(\d+)/i.exec(out) + return (r ? parseInt(r[1], 10) : 0) + (n ? parseInt(n[1], 10) : 0) +} + +function isolationSuppressed(out: string): boolean { + return /deliver:\s*✗/i.test(out) || /deliver:\s*false/i.test(out) +} + +const stash = new Map() + +export default (async ({ directory }) => { + log({ kind: "init", directory, stale: STALE_MODE, isolation: ISO_MODE }) + return { + "tool.execute.before": async (input: any, output: any) => { + if (input.tool !== "bash") return + const cmd = String(output.args?.command ?? "") + if (!/\bgit\s+commit\b/.test(cmd)) return + + // ── Gate 1: stale_detector (block) ── + if (STALE_MODE !== "off") { + try { + const out = await cli(directory, "stale_detector", STALE_ARGS) + const drift = staleDrift(out) + log({ kind: "stale", drift, out: out.slice(0, 400) }) + if (STALE_MODE === "block" && drift > 0) { + log({ kind: "stale_block", drift }) + throw new Error( + `CRYSTAL-GATE: ${drift} doc-drift instance(s). Docs assert a version the code no longer is.\n${out.slice(0, 500)}`, + ) + } + } catch (e) { + if (String(e).includes("CRYSTAL-GATE")) throw e + log({ kind: "stale_error", err: String(e).slice(0, 200) }) + } + } + + // ── Gate 2: graph isolation (advisory/block) ── + if (ISO_MODE !== "off") { + try { + const out = await cli(directory, "graph_query", ISO_ARGS) + const count = isolationCount(out) + log({ kind: "isolation", count, out: out.slice(0, 400) }) + if (count > 0) { + if (ISO_MODE === "block") { + log({ kind: "isolation_block", count }) + throw new Error(`GRAPH-GATE (block): this commit isolates ${count} graph node(s).\n${out.slice(0, 500)}`) + } + if (!isolationSuppressed(out)) stash.set(input.callID, out) + } + } catch (e) { + if (String(e).includes("GRAPH-GATE")) throw e + log({ kind: "isolation_error", err: String(e).slice(0, 200) }) + } + } + }, + + "tool.execute.after": async (input: any, output: any) => { + if (ISO_MODE !== "advisory") return + const cached = stash.get(input.callID) + if (!cached) return + stash.delete(input.callID) + log({ kind: "isolation_advisory", len: cached.length }) + output.output = + (output.output ?? "") + + `\n\n✦ GRAPH-GATE (advisory): this commit isolates graph node(s).\n${cached.slice(0, 500)}` + }, + } +}) satisfies Plugin diff --git a/AGENTS.md b/AGENTS.md index 087d57a2..f72d40c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ Raw Input: {} | 3 | `intel_get_project_memory` | `{}` | | 4 | `intel_explain_project_state` | `{}` | -5. **Источник правды:** `src/mcp/server.py` + `src/core/intelligence_layer.py`. +5. **Источник правды:** `src/mcp/server.py` + `src/core/intelligence/layer.py`. 6. **После ошибки** — не retry с теми же JSON. Сначала `debug_runtime_passport` `{}`, потом альтернатива. **Reindex (Raw Input):** @@ -302,7 +302,7 @@ intel_get_project_memory ──> codebase(action="git", path="log") / cod intel_get_project_context ──> (aggregates 5+ calls) ``` -## 2. AVAILABLE TOOLS (64) +## 2. AVAILABLE TOOLS (65) > **Полный справочник** (аргументы, когда вызывать, anti-patterns): > источник правды — `src/mcp/server_tools.py` + `src/mcp/tools/` + `src/core/intelligence/tools_reg.py`. @@ -319,7 +319,7 @@ intel_get_project_context ──> (aggregates 5+ calls) Inline/Diagnostic (13): `debug_runtime_passport`, `intel_get_project_context`, `intel_explain_project_state`, `get_runtime_counters`, `intel_tool_health`, `intel_execution_timeline`, `refresh_db_connection`, `notify_change`, `read_live_file`, `get_logs`, `get_health_report`, `dual_arm_health_check`, `ack_impact`. -### B. Core MCP & Search (31 tools) +### B. Core MCP & Search (32 tools) > **v3.2.0 Data Flow:** PropertyGraph содержит `ASSIGNED_FROM`-рёбра, отслеживающие @@ -339,14 +339,14 @@ Inline/Diagnostic (13): `debug_runtime_passport`, `intel_get_project_context`, ` `graph_query(action=query|cypher|related|flow)`, `submit_background_task`, `stale_detector`. -> **Not registered (consolidated):** the 14 names below are documented in older -> revisions but are **NOT exposed as MCP tools** in the current build — their -> functionality is covered by `search_code`, `get_symbol_info`, `impact_analysis`, -> and the `intel_*` suite. Do not call them; they return `tool not found`. +> **Hidden core tools (default allowlist):** core tool classes that are not in the +> default `MSCODEBASE_MCP_TOOLS` allowlist are **not exposed** to the MCP client +> (calling them returns `tool not found`) until added to the allowlist. Examples: > `get_repo_map`, `get_repo_rank`, `get_hotspots`, `get_bug_correlation`, > `detect_communities`, `cross_repo_search`, `cross_project_deps`, > `find_duplicates`, `generate_chunk_summaries`, `scan_changes`, -> `find_similar_bugs`, `get_context`, `verify_action`, `get_task_status`. +> `find_similar_bugs`, `get_context`, `bootstrap_pipeline`. +> (Intel / inline / dev tools are always registered and visible.) > Hub-маршруты `codebase(action=...)` (не отдельные MCP-тулы): > `codebase(action="index", path=status|progress|health|timeline|project_dir)` — индекс; @@ -449,7 +449,7 @@ For file renames, use `apply_file_move(old, new)` instead of `notify_change` — 7. Did I check `diagnostics`? 8. Did I run `python -m pytest tests/ -k write_tools -v` before committing? 9. **CI проверен через `gh run view --log-failed`** (последний ран — механический guard против «CI green» на словах): - - Перед push: последний ран не красный. После push: новый ран зелёный на ВСЕХ джобах — ubuntu matrix (3.10-3.12) + windows; локальный Windows-прогон слеп к POSIX-фейлам (WISDOM 2026-08-08). + - Перед push: последний ран не красный. После push: новый ран зелёный на ВСЕХ джобах — ubuntu + windows (Python 3.14); локальный Windows-прогон слеп к POSIX-фейлам (WISDOM 2026-08-08). - Точные фейлы даёт `gh run view --log-failed` — аннотации GitHub показывают только «exit code 1». 10. **`bash scripts/verify_clean_state.sh` — проверка с чистого состояния (clone + venv + install + tests)** - Вывод должен содержать: `CLEAN STATE VERIFICATION: PASSED` diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index bfc7a2bf..afde58b1 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -1,5 +1,28 @@ ## Key Historical Decisions +- **E6/E7: opencode-хук → MSCodeBase CLI; blind eval каталога кристаллов (2026-09-22):** E6 — плагин `tool.execute.before` на `git commit` зовёт `python -m src.cli stale_detector` (без MCP), при drift>0 бросает; negative/positive control пройдены, модель отказ не обошла (подчинение 4/4 суммарно с E3). Гоча: CLI `--project` игнорируется, проект = cwd. E7 — слепой маппинг 10 реальных симптомов на индекс каталога Tom Jones (`crystal-memory`/`crystals`, Apache-2.0, Spanda Works), 11 прогонов / 3 модели: controls 60/60 валидных (deepseek-low провалил NONE-controls → валидность зависит от reasoning), ключевой входной симптом → NONE 10/10 валидных (семья есть, из симптома недостижима), надёжный hit только 1/10, воспроизводимость qwen 8/10 vs longcat 4/10 (свойство модели). Детали — EXPERIMENTS_LOG Exp 18/19. +- **Deploy 00ec5a6d → extension + live-revive smoke (2026-09-22):** `python install.py --sync-only` (код + очистка `__pycache__`, без pip/llama); символы `is_port_up`/`ensure_embedder_started`/`_revive_llama_cpp` подтверждены в ext. Live-check на РЕАЛЬНОМ llama.cpp (extension venv, без моков): подняли :8080 → embed dim=384 → убили процесс на :8080 (эмуляция watchdog idle-unload) → `RemoteEmbedder.embed` сам поднял сервер (лог: `WinError 10061` → `_revive_llama_cpp` → `ensure_embedder_started`), dim=384 → **PASS**. НЮАНС: ревайв занял **17.7s** (>15s бюджета search_code: 3 батч-ретрая + ~10s старт llama-server) → первый поиск после unload может ещё таймаутить, но сервер поднимается и следующий работает (раньше не восстанавливался никогда). Для активации в живом окне нужен reload Zed. +- **E8: quiet-break gate graph (2026-09-22):** read-only дельта-изоляция узла графа на коммите (git diff + persisted PropertyGraph, БЕЗ мутации индекса). Мотив: абсолютный сигнал бесполезен — **1535/4081 (38%)** функций имеют 0 входящих CALLS (@_step-регистранты, callbacks, entry points). A = правка удаляет последних вызывающих (граф: кто вызывал × рабочее дерево: зовёт ли ещё); B = новая функция без вызовов (diff + текст изменённых файлов). Доступно как `graph_query(action="isolation")` / CLI. Controls (реальный git + seeded graph): POS-A 0 / NEG-A 1 / NEG-B 1 / POS-B 0 / не-git → unavailable; unit `tests/test_quiet_break_gate.py` 10/10; CLI на живом репо 1899ms, 0 ложных. Найден+исправлен баг fail-open (не-git отдавал "empty" вместо "unavailable"). Границы: rename-sweep, динамическая регистрация, soft (не hard-блок). Детали — EXPERIMENTS_LOG Exp 20. +- **E8 модельный (opencode-хук, 2026-09-22):** CONTROL — агент коммитит сироту не замечая; BLOCK (`before` throw) — коммит заблокирован, агент не обошёл (1/1); ADVISORY — первая версия (`after` считает diff) СЛЕПА (after после коммита, diff пуст) → фикс: считать в `before`, кэш по callID, доставлять в `after`; повтор — заметка доставлена и понята, но исход не изменён (промпт не требовал фикса). Вывод = наблюдение Тома: speak ≠ block. +- **E9: redaction на доставке (2026-09-22):** `src/core/redact.py` (in-house, prefix-anchored; идея Tom Jones, crystal-memory, Apache-2.0, код не вендорился) врезан в единую точку доставки `src/cli.py` (сериализованный JSON, успех+ошибка). unit 8/8: все семьи ключей вырезаются, хеши/пути/версии/слово «key» выживают byte-identical, JSON валиден; CLI E2E: stdout `[REDACTED:anthropic]`, stderr `{"redacted":...}`. ⛔ Не граница безопасности (новые/многострочные форматы проходят) — задокументировано. Детали — EXPERIMENTS_LOG Exp 21. +- **E10: сдержанность доставки (anti-numbing) (2026-09-22):** `src/core/restraint.py` — сигнатура findings (kind+symbol+file) + cooldown 600s с backoff ×2^strikes; состояние — один JSON на проект. Opt-in `graph_query isolation` + `kwargs={"restraint":true}`; блокирующие гейты НЕ заходят (жёсткий стоп ≠ нытьё). unit 4/4 (итого 22/22 с gate/redact); CLI E2E: call1 `deliver ✓`, call2 `deliver ✗` (cooldown 597s). Гоча: доп. аргументы CLI — через `kwargs`. Детали — EXPERIMENTS_LOG Exp 22. +- **E11: замороженный список vs новый arrival index Тома (2026-09-22):** каталог `tjonesit/crystals` @3e30ed2 (`catalogue/README.md`, arrival index 25 фраз→записи). Слепой маппер, 5 прогонов / 3 модели: **#16 («агент не пользуется тулами») → A в 5/5**, включая оба ВАЛИДНЫХ (longcat-2.0, qwen3.7-plus; контроли 6/6). deepseek ×3 невалидны — NC3 (#11 Safari/CSS) притянулся к E generated-document (UI/spacing), та же болезнь, что в E7. Вывод: arrival layer чинит #16 (было NONE 10/10); воспроизводимость всё ещё в читателе. ⚠ Новый риск: фраза-ловушка на соседнем домене (UI→CSS) = кандидат в дырку. Детали — EXPERIMENTS_LOG Exp 23. +- **Clean-state 44d451f6 (2026-09-22):** локальный clone закоммиченного состояния → `verify_clean_state.sh --no-clone`: **1800 passed / 0 failed / 13 skipped / 98 deselected** (822s), lock-drift negative-control PASSED, guard inventory ALL PROVEN (3), revision gate VALID. Новый core (quiet_break_gate/redact/restraint) прошёл clean-state. Не запушено (push — только по явному слову). +- **CI ubuntu-red: TESTS-signal fixture была Windows-only (2026-09-23):** `tests/test_graph_stage_e4.py` хардкодил `file_path="D:/Project/..."`; на POSIX `Path("D:/...").is_absolute()`=False → `get_tests_for_symbol` нормализовал в `/D:/...` → узел не найден → 0 тестов (падали 2 теста только на ubuntu; Windows-джоба зелёная). Fix: фикстура строит абсолютные пути из `tmp_path.as_posix()`. Guard: тест платформо-независим (класс P-ловушки «Windows-only тест без skipif», WISDOM 2026-08-08). +- **CI green (2026-09-23):** фикс 808864c5 → повторный CI **success** (ubuntu 13m21s + windows 17m20s + clean-state 13m15s + transport/manifest/docker). PR #40 (draft, mergeable). Единственный красный — GitHub Advanced Security (Copilot Autofind): `CAPIError: 400 The requested model is not supported` = инфра-сбой GitHub, не наш код (и на main так же). +- **Hook в репозиторий (2026-09-23):** `.opencode/plugin/mscodebase_gate.ts` — на `git commit`: `stale_detector` **BLOCK** (объективный дрейф версий; сейчас 0) + `graph_query isolation` **ADVISORY** (soft: у графа есть FP). Advisory = before-stash/after-deliver по callID (урок E8). Env-переключатели: `MSCODEBASE_PY`, `MSCODEBASE_STALE_GATE`, `MSCODEBASE_ISOLATION_GATE`, `MSCODEBASE_GATE_LOG`. CLI проверен из корня репо venv-python. Активируется после reload Zed. Глобально (все проекты) — только по явному разрешению (файл вне рабочей папки, §3 HALT). +- **opencode-плагин: `$` не в контексте (2026-09-23, P-ловушка):** `.opencode/plugin/mscodebase_gate.ts` загрузился (лог `init`), но на хуке `TypeError: $ is not a function` — в opencode **1.18.23** контекст фабрики не даёт рабочий `$` (доки описывают v2, где `$: BunShell` есть). Фикс: вызывать python через `node:child_process.execFile` (аргументы массивом — без shell-квотирования), не через `$`. Guard: не полагаться на `$` в локальных плагинах этой версии. +- **Hook live-verified (2026-09-23):** свежий `init` после рестарта; `git commit --dry-run` → stale `drift=0` (без блока) + isolation `count=0`. Negative control: застейдженный `new_orphan` (отдельным шагом `git add`) → isolation `count=1`, advisory `✦ GRAPH-GATE` **доставлен в вывод** команды; файл удалён, статус чист. Нюанс: хук срабатывает ДО выполнения команды, поэтому `git add && git commit` в ОДНОЙ команде → на момент хука файл ещё не застейджен (гейт не увидит); нужен отдельный `git add`. +- **Doc-audit recon (2026-09-23):** (1) `auto_update_docs(action="verify")` падал `IndexError: string index out of range` — одинокий `` `$` `` в бэктиках обнулял content ПОСЛЕ снятия `$` (guard пустой строки стоял ДО) → фикс + регресс-тест. (2) `DocLLMVerifier` (`src/core/doc_llm_verifier.py`) — **мёртвый код** (импортов нет); LLM-путь не подключён, LM Studio/Ollama не запущены → верификация только структурная и шумная (458 .md, 7567 refs, **1366 «битых»**, в основном ложные: dunder, `file.py:NN`, stdlib-атрибуты). (3) Числа инструментов в доках противоречат: README 65/61/66, ARCHITECTURE 64/61/58/46, AGENTS header 65 vs body «64»/«31»; даже docstring `_count_tools` (32+16+13+4) расходится со своими regex-комментариями (14/12). Версия — ок (`bump_version --check` ✅ 3.5.0). +- **E12: L1 doc-reference чекер (2026-09-23):** `src/core/doc_reference_l1.py` — детерминированный (stdlib) чекер ссылок в ЖИВЫХ доках: словарь = идентификаторы+строки из src/tests/scripts/tools, scope=корневые доки без леджеров + docs/** минус archive/generated/research/blog/ISSUES/investigations, whitelist stdlib/внешних/env/моделей. **1366 → 0 «битых»** (было ~3% реальных). Тесты 4/4. Подключён к `auto_update_docs("verify")`. Исправлена неточность WISDOM (`_symbol_resolver`→`_build_symbol_resolver`). **Ограничение:** устаревшее имя ТУЛА, встречающееся в тестах, маскируется (нужна проверка по реестру). Детали — EXPERIMENTS_LOG Exp 24. +- **E13: замер xdist (2026-09-23):** serial 1817 passed 200с → `-n 4` 169с (**~15%**), без падений. Скромно → по умолчанию не берём; главный рычаг CI — кэш install, не CPU. Топ-тормоз: `test_temporal_facts_generator` ≈42с. Детали — EXPERIMENTS_LOG Exp 25. +- **Doc number sync (2026-09-23):** авторитетный состав из кода = **65 (32 core + 16 intel + 13 inline + 4 dev)** (скрипт по server_tools.py/tools_reg.py/dev_tools.py). Синхронизированы README (61/66→65), AGENTS (64→65, 31→32), docs/{en,ru,zh}/ARCHITECTURE (64/61/58/46/28/31/20→65/32); видимый набор описан через allowlist, а не магическое число. L1=0, тесты 12/12. +- **README semantic audit (2026-09-23):** проверка утверждений README против кода (не на глаз). ИСПРАВЛЕНО: badge tests 1856→**1889** (`_count_tests`); дата 2026-08-16→09-23; «DI Container (18 services)»→**14** (по `di_container.py`: 14× `add_singleton`); «~1773 tests»→~1889; строка «65=64 base + execute_script» (арифметика 32+16+13+4=**65**) → «65 base (+execute_script→66)». ПОДТВЕРЖДЕНО верным: **29** edge types (`EdgeType`), **13** AST patterns (`structural_search.PATTERNS`). ОТКРЫТО: AGENTS.md §B «Not registered (consolidated)» перечисляет `get_repo_map`/`get_repo_rank`/`get_hotspots`/`get_bug_correlation`/… как «не MCP-тулы → tool not found», но они ЕСТЬ в `tool_classes` (registered, скрыты allowlist) и перечислены в README → противоречие AGENTS↔код/README (решение владельца). +- **AGENTS semantic audit (2026-09-23):** ИСПРАВЛЕНО: §0.1 «источник правды `src/core/intelligence_layer.py`» (файл НЕ существует) → `src/core/intelligence/layer.py`; §7 CI-matrix «(3.10-3.12)» → Python 3.14-only. ПОДТВЕРЖДЕНО верным: `RuntimeCoordinator`/`can_execute`, `ProjectContext.capture()` (project_context.py:164), `apply_file_move` (indexer.py:659), root-файлы (ISSUE.md/start_server.bat/start_remote.bat/marker/DEV_DIARY). ПРЕДУПРЕЖДЕНИЕ: грепы через PowerShell `src\**\*.py` дали ЛОЖНЫЙ 0 для `apply_file_move` — проверять глобом корректно. ОТКРЫТО: §B «Not registered (consolidated)» (14 имён, вкл. `get_repo_map`/`get_hotspots`/`verify_action`/`get_task_status`) противоречит коду — они в `tool_classes` (registered, скрыты allowlist). +- **"Not registered" resolved + server_tools log fix (2026-09-23):** механизм: только core-тулы из `MSCODEBASE_MCP_TOOLS`-allowlist регистрируются через `mcp.tool(...)`; intel(16)/inline(13)/dev(4) — всегда. Значит блок AGENTS верен по смыслу (скрытые → `tool not found`), но список был ошибочен (`verify_action`/`get_task_status` в default-allowlist → видны; отсутствовал `bootstrap_pipeline`). AGENTS переписан на «Hidden core tools (default allowlist)»; README — заметка о видимости; код-лог `server_tools.py` исправлен (hardcode intel 14→16, inline 12→13 — печатал «62 total» вместо 65). +- **ARCHITECTURE semantic audit (2026-09-23, en/ru/zh):** ИСПРАВЛЕНО — §2.0 слой **Bridge помечен deprecated** (код: `_bridge_state = ""`); §2.2 `server.py` «~220 строк»→**68**; §2.4 core «30 files»→**54** (top-level; 123 с подпакетами); DI «15+/18»→**14**. README: `src/lsp_main.py` «does not work»→**removed** (файла нет). ПОДТВЕРЖДЕНО: 32/16/13/4=65, пути `intelligence/layer.py`, `intelligence/project_context.py`, `runtime_coordinator.py`. +- **ARCHITECTURE §2.4–§4.2 audit (2026-09-23):** ИСПРАВЛЕНО: `remote_embedder` описан как «ONNX E5 embedder» → **llama.cpp GGUF primary** (+ONNX/LM Studio/Ollama fallback); «15 node labels» → **16** (`NodeLabel`); §3.2 «Registered Services (11)» → **14** (+Path, +Indexer, +GitUrlSourceFactoryKey); §4.2 Intelligence «14» → **16** (+intel_restore/supersede_memory_node), Lifecycle «3» → **4** (+get_action_receipt). §4.1 code-snippet по-прежнему сокращённый (не полный tool_classes) — помечен как иллюстрация. +- **search_code timeouts: idle-unloaded embedder + blocking hot-reload (2026-09-22):** `llama_runner` по `EMBEDDER_IDLE_TIMEOUT=120s` сам убивает embedder (free RAM, `_process=None`), но `RemoteEmbedder` его не поднимал → `:8080` мёртв → `WinError 10061` → поштучные ретраи → `search_code` timeout (max ~27s; прод-таймауты 09-20..09-22, эмбеддер падал ~16:21). Вторая причина: `_maybe_hot_reload` ждал полную переиндексацию ВНУТРИ search_code (22 файла → 14s при живом эмбеддере). Fix: `LlamaRunner.ensure_embedder_started()/is_port_up()` + `RemoteEmbedder._revive_llama_cpp()` (revive перед поштучными ретраями) + hot-reload в фон с дедупом (`search_tools._run_hot_reload_bg`). Тесты `tests/test_embedder_idle_recovery.py` (10). Побочно: stale `.pyc` в `git_url` давал фантомные IndexError-падения (лечится очисткой `__pycache__`). - **Resume-инкрементальная запись IndexProjectRunner (2026-09-20, 0fcb0f33):** run() копил все эмбеддинги и писал одним bulk_write в конце — краш на 330K чанков (~9ч) терял всё. Теперь файл пишется порциями (WRITE_FLUSH_FILES=32) по завершению его чанков, RAM освобождается, restart = resume по known_hashes; `_verify_and_repair_table_integrity()` в начале run() (INC-6C62); `_flat_indices_by_file` (O(1) сбор vecs); fallback per-file сохранён. Тесты test_index_resume_incremental.py (6). Red Team 5/5. - **Умный ubatch по ролям (2026-09-20):** вместо одного `LLAMA_UBATCH_SIZE=2048` для обеих ролей — `resolve_ubatch(role)`: embed → ceil(480/128)*128=**512**, rerank → ceil(1000/128)*128=**1024**. Основание: A/B на реальных 2227 чанках — ubatch=512 → 596 MB / 18.9 ch/s vs 2048 → 1686 MB / 16.1 ch/s. Раньше llama.cpp требовал «entire input ≤ ubatch» (#25293), для b9940 целый батч делится по слотам, лимит остался на ОДИН текст/пару → клиентский трим: embed обрезает до `LLAMA_EMBED_MAX_TOKENS=480` (единый источник с remote_embedder), rerank — `_truncate_rerank_pair` до `LLAMA_RERANK_MAX_TOKENS=1000`. `LLAMA_UBATCH_SIZE` env — жёсткий override. Файлы: `llama_install.py` (resolve_ubatch), `llama_runner.py` (_ubatch_arg, 3 места запуска), `multi_provider.py` (трим пары), `remote_embedder.py` (импорт лимита), тест `test_ubatch_roles.py`. Live-check: реальный llama-server принял ubatch=512, embeddings 200 dim=384. @@ -27,6 +50,76 @@ - **Чёрные окна CMD (2026-08-14):** MCP запускался как `venv\Scripts\python.exe` (console-подсистема) → каждое окно Zed = своё чёрное окно; фикс: `pythonw.exe` в extension.toml + CREATE_NO_WINDOW во ВСЕХ runtime subprocess (13 файлов) — с pythonw (нет консоли) незакрытые git/wmic/netstat мигали бы окнами - **FA=0.00 ≠ качество guardrail (2026-08-15):** Exp 1-L Day 3 — qwen3.6/3.7 (zero-shot VOR) достигают FA=0.00 ценой recall(real)=0.08–0.20 (code_first: 2/25 правды принято, 7/25 активно отвергнуто) — fail-closed политика, а не «фильтрация лжи»; выбор LLM для verify-on-read = выбор политики (fail-closed qwen vs max-coverage glm), recall(real) обязан быть в метриках. CoT (V3/Part 5) НЕ окупается: только qwen3.6 recall 0.08→0.20 при цене ×30–65 +## [2026-09-22] E17 v7 — CORRECTION: AST proxies do not predict kill-rate (v6 not replicated) + +**Status:** Measured (v6 REFUTED on replication). +**Method:** 24 functions (providers/mcp excluded — their tests await servers and hang under mutation), 55 tests, 4 mutations each; split by FUNCTION; decoys 0/55. +**Result:** v6's rho(exact_compare, kill_rate)=+0.54 p=0.006 did NOT replicate — train rho=+0.13 p=0.54, held-out rho=-0.01 p=0.97; rule `exact_compare>=1` precision 0.59 / recall 0.67. kill_rate spread healthy (mean 0.38; 17 zeros / 5 ones / 33 mid). +**Conclusion:** cheap AST features (assert count / density / specificity / LOC) do NOT predict whether a test catches a behaviour change. v6 was a small-sample artifact (n=24). `TEST_VERIFIES` needs actual mutation runs, not AST proxies. The TESTS *reach* edge and the decoy control (0/139 across v5–v7) stay valid. +**Files:** experiments/bootstrap/e17_strength_scaled.py, e17_strength_scaled.json + +## [2026-09-22] E17 v6 — verification-strength profile: specificity beats count + +**Status:** Measured (hypothesis CONFIRMED). +**Method:** 10 functions x 4 mutations (negate-if, flip-compare, bool-flip, int+1, str-append) = 40 mutants; every graph-linked test (B) and decoy (D) run under every mutant; AST features extracted from test bodies. +**Result:** 24 tests profiled; decoys killed **0/104** mutants. Predictors of kill-rate (Spearman, n=24): assertion **specificity** (`assert x == v`) rho=+0.54 p=0.006; assert **count** rho=+0.47 p=0.019; density rho=0.25 n.s.; LOC rho=0.14 n.s. +**Conclusion:** what an assertion compares against matters more than how many assertions there are. A TESTS edge is a *reach* claim; mutation kill-rate is a *verification* claim. Build a TEST_VERIFIES / TEST_STRENGTH layer on top of TESTS instead of one fat edge. +**Files:** experiments/bootstrap/e17_strength_profile.py, e17_strength_profile.json + +## [2026-09-22] E17 v5 — mutation validation: signal drives verification, 40% assertion gap + +**Status:** Measured (hypothesis CONFIRMED, with caveat). +**Method:** strong mutation (negate the first single-line `if`) on 10 panel functions; run the graph-linked tests (B) vs coverage-matched decoy tests (D); restore via `git checkout --` in finally. +**Result:** B killed the mutant 6/10, D 0/10, Fisher exact p=0.011. But 4/10 linked tests did NOT catch the mutation — e.g. `TestCypherLexer::test_tokenize_*` assert only `len(tokens) > 0` / `"MATCH" in values`. +**Conclusion:** TESTS edges identify *verifying* tests far better than decoys, so the signal can enrich verification — but its value is bounded by test *assertion strength*, not coverage. Pair it with an assertion-density / mutation-kill proxy, else ~40% of linked tests give false confidence. +**Files:** experiments/bootstrap/e17_mutation_validate.py, e17_mutation_results.json + +## [2026-09-22] E17 pivot v4 — impact task: signal is the only source of truth + +**Status:** Measured (hypothesis CONFIRMED, decisive). +**What changed:** task = "which tests cover function F" (impact/coverage); scoring = objective recall/precision of test names vs the signal's own set (no judge). +**Result:** recall A=0.000 B=1.000 D=0.000 (B-A and B-D = +1.000, 30/0/0); precision identical. Name-convention leak: A guessed a real test for 0/30. +**Conclusion:** TESTS-signal is essential for impact/coverage (the info is not in the code, and naming does not leak it), but adds nothing to answer generation / reasoning (v1-v3 ceiling). Value = retrieval / agent workflow, not answer quality. +**Report:** experiments/bootstrap/e17_pilot_report.md + +## [2026-09-22] E17 pivot v3 — complex panel, ceiling persists (B even worse) + +**Status:** Measured (hypothesis REFUTED again). +**What changed:** panel selected by complexity (branches >= 10; up to 332 loc / 80 branches, `IndexProjectRunner.run`); generation/judge shards packed by line budget (subagent read limit ~2000). +**Result:** accuracy A=4.93 B=4.77 D=4.83 — B-A = -0.167 (p=0.057, d=-0.36, 1 win / 6 losses); completeness/safety flat. Manipulation B 43% vs D 7%. +**Conclusion:** the ceiling is robust across task type and panel difficulty. For answer generation a strong model needs no tests; TESTS-signal value must lie in the agent workflow (which tests to run / impact), not in answer quality. +**Iterations:** v1 explain 5.00/4.97/5.00; v2 edge-case simple 4.87/4.87/4.90; v3 edge-case complex 4.93/4.77/4.83. +**Report:** experiments/bootstrap/e17_pilot_report.md + +## [2026-09-22] E17 pivot v2 — edge-case task, blind judge, ceiling persists + +**Status:** Measured (hypothesis REFUTED for this task/model). +**What changed:** task = edge-case outcome prediction (signature-aware input: None/""/missing path/-1); judge = blind AND test-hidden (uniform reference for all arms); metrics accuracy/completeness/safety + separate manipulation check. +**Result:** accuracy A=4.87 B=4.87 D=4.90 (p=1.00); completeness/safety flat. Manipulation check: B mentions its tests 57% vs D 10% → tests are read but not needed. +**Key correction:** v1's "completeness B>A p=0.0046" was a judge-context artifact — the judge saw each arm's own tests, inflating B/D. Removing the confound erased the effect. Rule: the judge's reference must be identical across arms. +**Conclusion:** for a strong model, neither paraphrase nor edge-case prediction benefits from TESTS-signal; need tasks where the code is insufficient, or a weaker model. +**Report:** experiments/bootstrap/e17_pilot_report.md + +## [2026-09-22] E17 pilot RUN — specificity-selected panel, ceiling on accuracy + +**Status:** Measured (hypothesis PARTIAL — pipeline valid, endpoint at ceiling). +**What changed:** panel re-selected by test specificity (min per-test coverage <=5); decoy = unique foreign coverage-matched tests; 90 answers via 10 shard subagents (no repo access); blind judging (opaque tokens). New/rewritten: `e17_pilot_experiment.py`, `e17_extract.py`, `e17_export_prompts.py`, `e17_export_judge.py`, `e17_merge_answers.py`, `e17_stats.py`, `e17_check_decoy.py`, `e17_verify_data.py`. +**Result:** accuracy A=5.00 B=4.97 D=5.00 (p=0.33, d=-0.18, 29/30 ties) — CEILING; completeness same; safety zero-variance. evidence_usage A=1.67 D=2.80 B=4.97 (B>A and B>D p<0.0001, 30/30) — manipulation check PASSED. +**Root Cause of null:** task too easy for the model — code alone suffices; NOT a verdict on the signal. +**Guard:** decoy audit (unique + foreign file + cov match) and verify_data before any run; blind judging with mapping kept out of judge shards. +**Pitfall:** subagent JSON output contained unescaped double quotes → invalid JSON; instruct no `"` inside answer strings. +**Open:** record position order; >=3 judge trials; harder endpoint (edge cases / impact) or weaker generator. +**Report:** experiments/bootstrap/e17_pilot_report.md + +## [2026-09-22] E17 pilot — source extraction was broken (Post-Mortem) + +**Status:** Fixed (experiments/bootstrap, untracked). Pilot invalidated — not the signal. +**Symptom:** `e17_pilot_answers.json` = 120/120 `Error 500`; live smoke gave `# FUNC NOT FOUND` for every class method. +**Root Cause:** naive `f"def {name}"` substring match vs graph *qualified* names — methods are `Class.method`, tests are `Class::test`. No match → `# FUNC NOT FOUND` / `# Test not found`. Compounded by `C_static` empty 30/30 (C≡A) and TESTS edges being transitive coverage (bootstrap_tests.py:216-244; `_ensure_data_root`=234 tests, 0 direct calls) → B≈D noise. +**Fix:** new `experiments/bootstrap/e17_extract.py` (AST: methods, `Class::test`, recursive `tests/`, offset-drift fallback); `e17_pilot_answers.py` + `e17_pilot_judge.py` refactored onto it; judge fake position-randomization replaced with a real before/after swap. 13 tests (`test_e17_extract.py`, incl. 4 adversarial). Untracked — no hash. +**Guard:** tests assert no `FUNC NOT FOUND` and correct class-qualified resolution; red-team covers duplicate method names, async, comment-only name, nested, syntax error. +**Open:** TESTS edges transitive → function selection by test specificity still required for a valid pilot. v3.5.0 retrieval result (hit@1) is unaffected — it used the graph directly. + ## [2026-09-22] v3.5.0 Release — TESTS-signal enabled by default **Status:** Released (pushed to main, PR #39 merged) diff --git a/AI_INSTALLATION_PROMPT.md b/AI_INSTALLATION_PROMPT.md index b030eee1..016ac1c6 100644 --- a/AI_INSTALLATION_PROMPT.md +++ b/AI_INSTALLATION_PROMPT.md @@ -1,6 +1,6 @@ # 🤖 One-Prompt Install Manifest for Zed Agent -> Версия: 3.4.0 | Дата: 2026-08-12 +> Версия: 3.5.0 | Дата: 2026-08-12 > > Скопируй этот текст в чат Агента Zed (`Ctrl+Shift+P` → `Agent Panel: Toggle`) > и Агент сам выполнит установку, настройку и проверку MSCodeBase Intelligence. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 928cc746..7d036f49 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ | Metric | Value | |--------|-------| -| **Version** | 3.4.0 | +| **Version** | 3.5.0 | | **Tests** | 853 ✅ | | **MCP Tools** | 58 (28 core + 14 intel + 12 inline + 4 dev) | | **Parsing Languages** | 30 file extensions (16 with ASSIGNED_FROM data flow) | diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index 513557ec..4dea6b09 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -2398,3 +2398,215 @@ RETRACTION CHECK: all referenced files exist on disk **Ограничения:** запросы — identifier-поиск (graph_stage не обрабатывает NL); A/B на одном проекте (MSCodeBase), без embedder (only graph_path). Для статьи: промежуточный результат есть — TESTS-рёбра дают тесты поверх функции без потери top-1. + + +## Exp 18 — E6: commit-gate через opencode-хук + MSCodeBase CLI (мост + отказ) + +**Гипотеза:** opencode-плагин (`tool.execute.before`) на git-commit может получить детерминированный +факт из MSCodeBase (CLI, без MCP) и отказать коммиту с этим фактом; negative/positive control докажут, +что гейт умеет падать И пропускать. + +**Метод:** плагин `.opencode/plugin/crystal_gate.ts` ловит `bash` с `git commit` → `python -m src.cli +stale_detector` (cwd=проект, MSCODEBASE_DATA_DIR изолирован) → парсит JSON → при drift>0 бросает. +Фикстура: pyproject 2.0.0 vs docs 1.9.0. Negative = дрейф есть; Positive = docs синхронизированы. +Модель deepseek-v4.1-flash, изолированный data-root (реальный индекс не тронут). + +**Результат:** bridge ✅ (факт в логе плагина); negative ✅ `gate_fired drift=1`, коммит заблокирован; +positive ✅ drift=0, коммит прошёл. Модель после отказа **не обошла** гейт (не трогала `--no-verify`/ +плагин), назвала причину `docs/readme.md:3` vs `pyproject.toml:3`, предложила фикс. Гоча: `--project` +у `stale_detector` НЕ учитывается — проект берётся из **cwd**; стоимость CLI ~9s. + +**Вердикт:** механизм "детерминированный факт → говорит агенту в момент действия" работает в opencode; +hard-refusal внутри своего репо легитимен; подчинение модели на отказе — 4/4 (E3 3/3 + E6 1/1), ранний факт. + + +## Exp 19 — E7: находит ли индекс симптомов то, с чем пришёл (blind eval каталога crystal-memory) + +**Гипотеза:** каталог verification-failures, индексированный по симптому (Tom Jones / Spanda Works, +Apache-2.0), по входному симптому реального сбоя либо находит нужную запись, либо приходится копать. + +**Метод:** заморожены 10 реальных симптомов MSCodeBase (как пришли, из леджеров). HANDOUT: 10 симптомов + +индекс (симптом→запись) + 6 controls (3 парафраза записей каталога = обязаны попасть; 3 вне домена = +обязаны NONE). Слепой маппер — другой чат (все MCP off, пустая папка), модель longcat-2.0, 5 прогонов +(r5 с reasoning=low) + 2 прогона deepseek на RU-варианте. Key (правильные ответы) — субъективен, помечен. + +**Результат (11 прогонов, 3 модели: longcat-2.0 ×5, qwen3.7-plus ×3, deepseek-v4.1-flash ×3):** controls +(3 парафраза записей + 3 вне домена) — **60/60 в валидных прогонах**; **deepseek-low провалил 2/3 +NONE-controls** (Postgres-lag→rate-knob, CSS→generated-doc) → прогон невалиден → **валидность инструмента +зависит от reasoning-уровня**. Ключевой входной симптом («агент не пользуется высокоуровневыми тулами») → +**NONE 10/10 валидных** (единственный C — в невалидном deepseek-low): семья A есть, из входного симптома +недостижима (index-wrong, не hole). Надёжный hit — только #12 (drift_gate, 1/10). Внутри-модельная +воспроизводимость: **qwen 8/10 vs longcat 4/10** (нестабильность — свойство модели, не индекса). +Кросс-модель: «щедрая» ~8/10 vs «осторожные» ~3/10. Дыра каталога: 1 (OS-level process leak → NONE всегда). + +**Вердикт:** покрытие семейств — реально (A/B/C/D/E срабатывают); **lookup из входного симптома — +невоспроизводим** (зависит от модели И от reasoning-уровня); для нашего случая не находит → копаешь сам. +На части настроек индекс не молчит, а **ложноположительно мисфайлит** симптомы вне домена — опаснее NONE +(заметка приходит как установленный факт). Ирония: и невоспроизводимый ranking, и numbing — это Family C +каталога (`single-run-ranking-is-noise`). + +**Атрибуция:** concept + catalogue — Tom Jones, dev.to series + `github.com/tjonesit/crystal-memory` / +`crystals` (Apache-2.0, Spanda Works LLC; NOTICE при вендоринге). Код не вендорился — только eval. + +**CORRECTED (2026-09-22, пост-анализ по AGENTS.md §11):** формулировку «механический поиск +недостаточен, нужен семантический» считать НЕОБОСНОВАННОЙ дизайном E7: маппер был LLM +(семантический), а НЕ keyword-search; KEY субъективен (сами пометили), N мал; сравнения +«keyword vs LLM» в E7 не было. Подтверждено остаётся узкое: LLM-маппинг из входного симптома +невоспроизводим и модельно-зависим; из #16 семья A недостижима без домысливания root cause. + + +## Exp 20 — E8: quiet-break gate (дельта-изоляция узла графа), read-only + +**Гипотеза:** на коммите можно детерминированно (git diff + persisted PropertyGraph, БЕЗ мутации +индекса) поймать «тихий слом»: (A) правка удаляет последних вызывающих символа, который ещё +определён в дереве; (B) новая функция нигде не вызывается. Controls докажут, что гейт умеет +падать И пропускать без ложных срабатываний. + +**Мотив/находка:** абсолютный сигнал «узел без входящих CALLS» непригоден — в репо +**1535/4081 (38%)** Function/Method имеют 0 входящих CALLS (@_step-регистранты, callbacks, +entry points, public API). Нужен ДЕЛЬТА-сигнал. Persisted-граф = закоммиченное состояние, +правку не видит → A совмещает граф (кто вызывал) с рабочим деревом (зовёт ли ещё), B = diff + +текст изменённых файлов. Hot-index отклонён: `_index_single_file` тянет эмбеддинг + мутацию +живого индекса (тяжело, зависит от embedder). + +**Метод:** `run_quiet_break_gate(root, pg)` (src/core/quiet_break_gate.py) через +`graph_query(action="isolation")` / CLI. read-only; нет git / нет графа → status +unavailable/empty, findings пусты (fail-open). Controls: реальный git (temp repo) + seeded graph. + +**Сырой результат:** POS-A (caller цел) ok {removed_last_caller:0} | NEG-A (вызов удалён) +ok {removed_last_caller:1} finding target [pkg/caller.py] | NEG-B (новая без вызовов) +ok {new_orphan:1} finding orphan_helper | POS-B (новая + вызов в изменённом файле) +ok {new_orphan:0} | FAIL-OPEN (не-git) unavailable. unit tests/test_quiet_break_gate.py 10/10; +CLI на живом репо: 1899ms, base HEAD, changed_files 5, counts 0 (ложных нет). + +**Вердикт:** гипотеза ПОДТВЕРЖДЕНА — read-only дельта-гейт различает изоляцию от 38%-шума; +fail-open закрыт (найден+исправлен баг: не-git отдавал "empty" вместо "unavailable"). + +**Границы:** A не ловит изоляцию при массовом rename-sweep; B не видит динамическую регистрацию +(whitelist декоратора), same-name консервативен; гейт информационный (soft), не hard-блок. + +**Модельный E8 (opencode-хук, deepseek-v4.1-flash, 1 прогон/арм, идентичный промпт «закоммить staged»):** +- CONTROL (`E8_MODE=off`): агент коммитит, сироту не замечает (лог: только init). +- BLOCK (`tool.execute.before` throw): гейт сработал (count=1, CLI 8400ms), коммит **заблокирован**, + агент **не обошёл** (`--no-verify`/плагин не трогал), назвал причину и предложил варианты. +- ADVISORY (не-блокирующий): ПЕРВАЯ реализация (`tool.execute.after` считает diff) **слепа** — after + срабатывает после коммита, diff пуст → count=0/"no changes". ИСПРАВЛЕНО: считать в `before` + (pre-commit cached diff), кэшировать по `callID`, дописывать заметку в `after`. Повтор: заметка + доставлена (`✦ GRAPH-GATE (advisory)`), коммит прошёл, агент её прочитал и назвал, **но не исправил** + (сирота остался в коммите). +**Вердикт:** block меняет исход (плохой коммит предотвращён), модель подчиняется (1/1, без обхода); +advisory доставляется и понимается, но в этой постановке (промпт = «закоммить и отчитайся») исхода +не меняет. Ровно наблюдение Тома: знание приходит на нужном вызове и не меняет ничего; speak и block — +разные инструменты. **Границы:** 1 прогон/арм, один модель/промпт; advisory-эффект зависит от постановки +(исправление не было целью). Стенд: %TEMP%/opencode/e8. + +**Файлы:** src/core/quiet_break_gate.py, src/mcp/tools/graph_tools.py (action isolation), +tests/test_quiet_break_gate.py; лаб %TEMP%/opencode/e_gate2/e2e_control.py. + + +## Exp 21 — E9: redaction на доставке (единая точка CLI) + +**Гипотеза:** секрет, вставленный в доставляемую заметку как evidence, утекает в контекст КАЖДОГО +агента, на котором заметка сработает (канал доставки не проходит commit-gate). Redaction в единой +точке доставки (`src/cli.py`) вырежет известные формы ключей и НЕ испортит легитимный текст заметки +(пути, хеши, версии). + +**Метод:** in-house prefix-anchored redactor (`src/core/redact.py`; идея — Tom Jones, crystal-memory +`scripts/redact.py`, Apache-2.0; код НЕ вендорился). Врезан в единственный choke point: сериализованный +JSON `src/cli.py` (успех И ошибка). Тесты: unit + CLI-интеграция (fake tool возвращает ключ). + +**Результат:** unit **8/8** — каждая семья (aws/anthropic/openai/github-token/pat/slack/google/stripe/ +jwt/url-cred/bearer/private-key/assignment) вырезается; хеш коммита, sha256-дайджест, путь, версия, +слово «key» в прозе — byte-identical; JSON остаётся валидным. CLI E2E: stdout содержит +`[REDACTED:anthropic]`, путь сохранён, stderr сообщает `{"redacted": {...}}`. + +**Вердикт:** ПОДТВЕРЖДЕНА. Redaction на доставке работает и не калечит заметки. ⛔ Не граница +безопасности: новый формат / секрет разбитый по строкам проходит — задокументировано, не продаём как «safe». + +**Файлы:** src/core/redact.py, src/cli.py, tests/test_redact.py. Атрибуция идеи: Tom Jones (crystal-memory). + + +## Exp 22 — E10: сдержанность доставки (anti-numbing) + +**Гипотеза:** повтор одной и той же advisory-заметки притупляет читателя (Tom Jones: «the evening the +agent went numb to its own alerts»; наш E3/E4 — та же опасность). Сигнатура findings + cooldown с +растущим backoff подавит повторы, НЕ скрывая реальных изменений. + +**Метод:** `src/core/restraint.py` — `signature(findings)` (kind+symbol+file) + `should_deliver` +(cooldown 600s, окно = cooldown × 2^min(strikes,3)); состояние — один JSON на проект +(`delivery_state.json` в data_root, перезапись, atomic replace). Opt-in: `graph_query` action +`isolation` + `kwargs={"restraint":true}` (блокирующие гейты сюда НЕ заходят — Red Team п.3). + +**Результат:** unit **4/4** (вместе с gate/redact — 22/22): первая доставка; немедленный повтор +подавлен; после cooldown — снова; новая сигнатура — сброс; backoff растёт. CLI E2E (2 одинаковых +вызова): call1 `deliver: ✓`; call2 `deliver: ✗` `cooldown: same findings, 597s left (strikes=0)`. +Гоча: доп. аргументы CLI идут через `kwargs` (execute не принимает произвольные именованные). + +**Вердикт:** ПОДТВЕРЖДЕНА. Повтор подавляется, изменение findings → новая сигнатура → доставка; +блокирующие гейты не затронуты. ⛔ Best-effort, не граница безопасности (при сбое чтения — fail-open). + +**Файлы:** src/core/restraint.py, src/mcp/tools/graph_tools.py (kwargs.restraint), tests/test_restraint.py. + + +## Exp 23 — E11: frozen list vs the NEW arrival index (Tom's catalogue, «after») + +**Гипотеза:** arrival layer (25 фраз → записи) делает dispositional-симптом #16 («агент не пользуется +моими тулами») достижимым для семьи A, чего таблицы симптомов не могли (E7: NONE 10/10 валидных). + +**Метод:** тот же замороженный список (10 реальных + 6 контролей), что в E7; индекс заменён на arrival +index + symptom tables из каталога (`github.com/tjonesit/crystals` @3e30ed2, `catalogue/README.md`, +generated). Слепой маппер (opencode run, все MCP off, пустая папка), модели deepseek-v4.1-flash ×3, +longcat-2.0 ×1, qwen3.7-plus ×1. + +**Результат:** **#16 → A в 5/5 прогонов**, включая оба ВАЛИДНЫХ. Валидные (контроли 6/6): longcat-2.0, +qwen3.7-plus. deepseek ×3 — НЕВАЛИДНЫ: NC3 (#11 Safari/CSS) притянулся к E +`a-generated-document-is-unverified-until-you-render-it` (фраза про вёрстку/spacing) — та же болезнь +NONE-контроля, что у deepseek-low в E7. Прочие пункты остаются модельно-зависимыми (#10, #13 разные). + +**Вердикт:** arrival layer ПОДТВЕРЖДЁН как мост для #16 (NONE → A). Воспроизводимость по-прежнему +свойство читателя (оговорка Тома держится). ⚠ Новый риск: слой фраз может ВЫДУМАТЬ ложное совпадение +на соседнем домене (UI-фраза ловит вне-доменный CSS-симптом) — кандидат в запись/дырку. + +**Файлы:** лаб %TEMP%/opencode/e11 (handout.md, opencode.json). Источник: tjonesit/crystals @3e30ed2. + + +## Exp 24 — E12: L1 детерминированный doc-reference чекер (near-zero FP) + +**Гипотеза:** полный словарь из кода + scope=живые доки + исключения (venv/архив) + whitelist +(stdlib/внешние/env/модели) превращает непригодные 1366 «битых» в actionable-гейт. + +**Мотив:** старый `auto_doc_updater` собирал только ОПРЕДЕЛЕНИЯ (def/class/тулы/константы) и сканировал +`root.rglob` (включая `venv`: 232 чужих .md) → 1366 «битых», ~3% реальных (параметры/поля/stdlib). +«Checker that cries wolf gets muted» (evergreen/doc-sync). + +**Метод:** `src/core/doc_reference_l1.py` (stdlib): словарь = идентификаторы + строковые литералы из +src/tests/scripts/tools (+корневые *.py); scope = корневые доки БЕЗ леджеров + docs/** минус +archive/generated/research/blog/ISSUES/investigations; фильтры dunder/`file:NN`/символы/модели/env/`@`/`self.`; +whitelist stdlib+builtins+typing, внешние (Zed/Rust), commit-scopes. Подключён к `auto_update_docs("verify")`. + +**Результат:** «битых» **1366 → 11 → 2 → 0** (после whitelist 2 легитимных терминов: in-toto `expected_command`, +концепт `graph_context_first`). Тесты 4/4 (12/12 с auto_doc_updater). Словарь 18820, живых доков 69. +Найдена и исправлена реальная неточность: WISDOM `_symbol_resolver` → `_build_symbol_resolver`. +(Побочно: краш на одиноком `` `$` `` — починен ранее в этой сессии.) + +**Вердикт:** ПОДТВЕРЖДЕНА. Детерминированный L1 на полном словаре даёт ~0 FP на живых доках → годен как гейт. +**Ограничение:** L1 проверяет «имя существует ГДЕ-ЛИБО в коде», поэтому устаревшее имя ТУЛА, всё ещё +встречающееся в тестах, маскируется (старые `replace_symbol`/`predict_eta` в BENCHMARK.md не пойманы) — +для тулов нужна проверка по реестру, а не по словарю. + +**Файлы:** src/core/doc_reference_l1.py, src/core/auto_doc_updater.py (delegate), tests/test_doc_reference_l1.py, WISDOM.md. + + +## Exp 25 — E13: сокращение времени тестов (замер xdist) + +**Гипотеза:** параллельный прогон (pytest-xdist) заметно сократит время без потери качества. + +**Метод:** `pip install pytest-xdist` (3.8.0), полный прогон `pytest tests/ -q -n 4` vs последовательный, на той же машине/сессии. + +**Результат:** serial — **1817 passed за 200с** (тёплый); parallel `-n 4` — **1817 passed за 169с** (wall 172с) +→ **~15% быстрее**, падений нет (наблюдаемая parallel-safety). Топ-тормоз serial: `test_temporal_facts_generator` +≈42с (5 тестов). CI `clean-state` — **~13м холодный** (доминирует install/холодные кэши, не CPU). + +**Вердикт:** xdist даёт лишь ~15% локально → не стоит сложности по умолчанию. Главный рычаг CI — кэш +deps/install, а не CPU-параллельность. Дальше: кэш pip в `clean-state`; разбор/оптимизация +`test_temporal_facts_generator`. (Локальный xdist отключён по прежнему решению владельца — конфликт с llama.) diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 87d132ce..c5e820dc 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -6,7 +6,14 @@ --- -**29 entries** — compressed per §4.8 R3 (conclusion-first; dedup 2026-09-08, 2026-09-21) +**30 entries** — compressed per §4.8 R3 (conclusion-first; dedup 2026-09-08, 2026-09-21) + +## 2026-09-22 — TESTS-рёбра транзитивны, а не «тесты про функцию»; E17 LLM-pilot сломан на извлечении кода (Fixed / Open) + +- **Источник:** AGENT_DIARY.md#2026-09-22 (E17 Post-Mortem), `bootstrap_tests.py:216-244` +- **Описание:** (1) dynamic-trace линкует тест с *каждой исполненной* функцией → `_ensure_data_root` имеет 234 TESTS-ребра при 0 прямых вызовов в `tests/` (`check_disk_space` — 3). Для LLM-контекста сэмпл из 234 — шум, поэтому B-арм ≈ D-арм. (2) `e17_pilot_{answers,judge}.py` извлекали код наивным `f"def {name}"`, а граф хранит qualifed-имена (`Class.method`, `Class::test`) → `# FUNC NOT FOUND` для всех методов, `C_static` пуст 30/30. +- **Fix:** AST-извлечение в `experiments/bootstrap/e17_extract.py` + 13 тестов (Fixed). Фильтр TESTS по специфичности (прямой вызов / малый coverage-set) и пересборка pilot_data — не сделаны. +- **Статус:** Fixed (extraction) / Open (specificity-фильтр блокирует валидный E17 LLM-pilot). v3.5.0 retrieval hit@1 не задет. ## 2026-09-19 — Прод-инцидент: миграция колонок lanceDB молча не выполнялась + db_writer разрушал БД при schema-mismatch (Fixed) @@ -238,3 +245,4 @@ - **Описание:** **Status:** Fixed (эксперимент подтверждён; сопровождение задачи closed) **Root Cause:** при росте базы 3,989 → 13,519 статей (3.4x), refresh own занял 10м38с на 13.5k статей/82.5k комментов (134 сете... - **Статус:** автоматически синхронизировано + diff --git a/README.md b/README.md index f46aad13..dadf6777 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ [![MCP](https://img.shields.io/badge/MCP-compatible-green.svg)](https://modelcontextprotocol.io/) [![Zed](https://img.shields.io/badge/Zed-extension-orange.svg)](https://zed.dev/) [![CI](https://github.com/ManSio/mscodebase-intelligence/actions/workflows/ci.yml/badge.svg)](https://github.com/ManSio/mscodebase-intelligence/actions/workflows/ci.yml) -[![Tests](https://img.shields.io/badge/tests-1835%20passed-brightgreen)](tests/) +[![Tests](https://img.shields.io/badge/tests-1889%20passed-brightgreen)](tests/) -[Features](#-features) • [Quick Start](#-quick-start) • [Tools](#mcp-tools-66-total) • [Documentation](#-documentation-map) • [Installation](docs/en/INSTALL.md) • [Architecture](docs/en/ARCHITECTURE.md) • [Contributing](CONTRIBUTING.md) • [Security](SECURITY.md) +[Features](#-features) • [Quick Start](#-quick-start) • [Tools](#mcp-tools-65-total) • [Documentation](#-documentation-map) • [Installation](docs/en/INSTALL.md) • [Architecture](docs/en/ARCHITECTURE.md) • [Contributing](CONTRIBUTING.md) • [Security](SECURITY.md) -*Last updated: 2026-08-16* +*Last updated: 2026-09-23* @@ -67,9 +67,9 @@ This is **not** an LSP server or a replacement for the editor's built-in autocom ### LSP: Hybrid Rename Only -MSCodeBase **uses LSP only for `codebase(action="rename")`** — the LSP client (`src/core/lsp_client.py`) spawns **pyright-langserver** for precise cross-file rename, with graceful fallback to SymbolIndex (Tree-sitter) on timeout. All other functionality is implemented through **61 MCP tools**. +MSCodeBase **uses LSP only for `codebase(action="rename")`** — the LSP client (`src/core/lsp_client.py`) spawns **pyright-langserver** for precise cross-file rename, with graceful fallback to SymbolIndex (Tree-sitter) on timeout. All other functionality is implemented through **65 MCP tools**. -The standalone LSP server (`src/lsp_main.py`) was experimental and **does not work in Zed** — see [LSP_WONTFIX.md](docs/en/investigations/LSP_WONTFIX.md). +The standalone LSP server (`src/lsp_main.py`) was **removed** — see [LSP_WONTFIX.md](docs/en/investigations/LSP_WONTFIX.md). ### Platforms @@ -118,7 +118,7 @@ Designed and tested on **Windows**. macOS and Linux should work but have not bee | 💾 **LanceDB v2** | Vector DB with per-project isolation (incremental BM25 reindex) | | 🛡 **Rate Limiting** | DebounceBatch + CircuitBreaker — protection against VFS loops | | 🏥 **Self-Diagnosis** | `get_health_report` + `index_health` — full check and recovery | -| 🧪 **Clean Architecture** | DI Container (18 services), 65 tools (32 core + 16 intel + 13 inline + 4 dev), ~1773 tests | +| 🧪 **Clean Architecture** | DI Container (14 services), 65 tools (32 core + 16 intel + 13 inline + 4 dev), ~1889 tests | | 🪟 **Multi-Window** | `ProjectIndexerRegistry` — isolated Indexer per project, LRU 5, ResourceMonitor throttle | | ✏️ **Write Tools** | `codebase(action=...)` — unified hub: rename, move, delete, replace, insert, ack | | ⚡ **Meta-Patching** | LanceDB `move_chunks_metadata` — file_path rename without re-embedding (50ms vs 5s) | @@ -215,9 +215,11 @@ Deep-dives into specific technical findings from building this project: --- -## 🔧 MCP Tools (66 total) +## 🔧 MCP Tools (65 total) -> 65 = 64 base + `execute_script` (регистрируется при `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`). Без флага — 64 (32 core + 16 intel + 13 inline + 4 dev). +> 65 registered = 32 core + 16 intel + 13 inline + 4 dev. With `execute_script` (registered when `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`) — 66. +> +> **Visibility:** the MCP client only sees tools in the `MSCODEBASE_MCP_TOOLS` allowlist (see `src/mcp/server_tools.py`). Some core tools listed below (`get_repo_map`, `get_repo_rank`, `get_hotspots`, `get_bug_correlation`, …) are **hidden by default** until added to the allowlist; `MSCODEBASE_MCP_TOOLS=""` shows all. ### Core Search @@ -369,7 +371,7 @@ Deep-dives into specific technical findings from building this project: │ ┌────────────┴────────────┐ │ │ ▼ ▼ │ │ ┌────────────────────┐ ┌────────────────────────────────────┐ │ -│ │ 28 Tool Classes │ │ 16 intel_* + 13 inline tools │ │ +│ │ 32 Tool Classes │ │ 16 intel_* + 13 inline tools │ │ │ │ src/mcp/tools/*.py │ │ intelligence/layer.py + │ │ │ │ + codebase hub │ │ server_tools.py (inline) │ │ │ │ Constructor Inj. │ │ error_boundary decorator │ diff --git a/WISDOM.md b/WISDOM.md index 7246bc8b..d52f76e5 100644 --- a/WISDOM.md +++ b/WISDOM.md @@ -1,4 +1,4 @@ -# WISDOM.md — дистиллят фактов (≤50 строк) +# WISDOM.md — дистиллят фактов (≤50 строк) > Не архив, не дневник. Каждая строка — проверенный факт, сжатый до одной мысли. > Полная история — в `AGENT_DIARY.md` / `EXPERIMENTS_LOG.md`. @@ -198,7 +198,7 @@ - **COMMIT A СДЕЛАН (08281f37, ветка feat/two-pass-symbol-resolver, 7 файлов, 328+/16-):** A = поведение отказов: write-path guard на неоднозначность (#21: modification_guard + write_tools `_resolve_single_source_def` отказ при >1 definition) + VOR symbol-якорь - fail-closed (#22: `_symbol_resolver` только True→VERIFIED / None→INCONCLUSIVE, НИКОГДА + fail-closed (#22: `_build_symbol_resolver` только True→VERIFIED / None→INCONCLUSIVE, НИКОГДА False→REFUTED; `_classify` non-True symbol → INCONCLUSIVE). Тесты 126 passed. - **COMMIT B (head-freshness) — РЕАЛИЗОВАН, 8 тестов, полный pytest 1602 green, ruff чист:** - `src/core/graph.py`: `meta`-таблица (set_meta/get_meta) — build-метка. @@ -208,7 +208,7 @@ (git rev-parse HEAD + git status --porcelain, оба Popen+communicate, CREATE_NO_WINDOW, timeout 5s, fail→None); `_classify` снял fail-closed symbol-ветку (False теперь REFUTED — свежесть гарантирует резолвер). - - `src/core/intelligence/layer.py`: `_symbol_resolver` freshness-gated — False (REFUTED) + - `src/core/intelligence/layer.py`: `_build_symbol_resolver` freshness-gated — False (REFUTED) ТОЛЬКО если `si.build_head()` == live HEAD (resolve_head_dirty) на чистом дереве; иначе (легаси-индекс без build_head / HEAD-мисматч / не-git / dirty / сбой) → None (INCONCLUSIVE). Свежесть решает РЕЗОЛВЕР, не `_classify`. @@ -232,3 +232,10 @@ - Паттерн: **hook блокирует → чини среду, не пропуск** (`--no-verify` оставляет контроль мёртвым и коммитит поверх; PATH-фикс делает контроль работающим и коммитит под ним — контроль остановил самого автора и сам же был восстановлен). + +## Clean-state verification (2026-09-22) +- erify_clean_state.sh --no-clone на локальном clone закоммиченного состояния (ревизия 44d451f6): + 1800 passed / 0 failed / 13 skipped (822s); lock-drift negative-control PASSED; guard inventory ALL PROVEN (3); + revision gate VALID. Новый core (quiet_break_gate/redact/restraint) прошёл clean-state. +- Если ветка не запушена, default clone с GitHub тестирует ЧУЖОЕ (origin) состояние — для честного + clean-state клонировать ЛОКАЛЬНЫЙ репо и гонять --no-clone. diff --git a/docs/archive/KNOWN_ISSUES_2026_09.md b/docs/archive/KNOWN_ISSUES_2026_09.md new file mode 100644 index 00000000..15eec71e --- /dev/null +++ b/docs/archive/KNOWN_ISSUES_2026_09.md @@ -0,0 +1,170 @@ +# KNOWN_ISSUES archive — 2026-09 auto-synced experiment entries + +> Archived 2026-09-22 per §4.8 R4 (KNOWN_ISSUES.md exceeded 300 lines). +> These entries were auto-synced from AGENT_DIARY.md (experiment results); +> canonical source = AGENT_DIARY.md / EXPERIMENTS_LOG.md. + +## 2026-09-22 — Exp E16: переносимость bootstrap trace на чужие проекты (статья CoderLegion) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Measured (hypothesis CONFIRMED) +**Hypothesis:** динамический трейс (sys.settrace, `src/core/bootstrap_trace_plugin.py`) воспроизводится на чужих Python-репозиториях без правок плагина; lin... +- **Статус:** автоматически синхронизировано + + +## 2026-09-22 — Exp E14: Embedder A/B — EmbeddingGemma 300M vs e5-small (production) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Measured (hypothesis CONFIRMED) +**Hypothesis:** gemma 300M (768-dim, ctx 2048) значительно сильнее e5-small (384-dim, ctx 512) на кодовом ретривале при цене 3-4× медленнее на CPU. +**Method... +- **Статус:** автоматически синхронизировано + + +## 2026-09-20 — Exp E13: текстовый RAG (doc-chunks) vs кодовый baseline (E10/E11) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Measured (refuted hypothesis) +**Hypothesis:** doc-chunks (README + docs/en/ + docstrings) retrieve as well as code-chunks via search_with_mode quality. +**Method:** 16 EN doc-queries, live ... +- **Статус:** автоматически синхронизировано + + +## 2026-09-18 — Фаза 1: Incremental Hot-Reload (FreshnessChecker оживлён + hot-reload + KI-109) + +- **Источник:** AGENT_DIARY.md +- **Описание:** - **Evidence Ladder (2026-08-15, Exp 2-E E1-E3):** форма evidence — переменная; file_content = лучший recall (qwen 0.92), graph = закрытие present-trap ТОЛЬКО у evidence-честных моделей (qwen3.7 FA tr... +- **Статус:** автоматически синхронизировано + + +## 2026-09-18 — Фаза 1: Incremental Hot-Reload (FreshnessChecker оживлён + hot-reload + KI-109) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Fixed (7 тестов свежести включая concurrency-стресс N=16 + 1748 полный pytest green; ветка вне PR — локально) +**Root Cause:** FreshnessChecker (freshness.py) был мёртв (0 вызовов) и СЛОМАН... +- **Статус:** автоматически синхронизировано + + +## 2026-09-07 — Lazy-only верификация: VOR вызывается только из intel_get_project_memory, нет TTL/фона + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Open — зафиксировано как проблема + план эксперимента (10-continuous-verification.md) +**Root Cause:** По дизайну (ADR-0003) VOR ленивый, но точки вызова всего одна (layer.py:1097); IdleSch... +- **Статус:** автоматически синхронизировано + + +## 2026-09-09 — H1: фоновый VOR-проход (IdleScheduler) — память перепроверяется без вызова агента + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Fixed (6 новых тестов + 1674 полный pytest green; ветка chore/experiments-es1-es2-0909) +**Root Cause:** VOR вызывался ровно из 1 места (intel_get_project_memory, layer.py:1097); idle-задач... +- **Статус:** автоматически синхронизировано + + +## 2026-09-09 — H2: .h заголовки C включены в AST-индексацию (PARSE_EXTENSIONS + C-парсер) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Fixed (commit 0301fa93; KNOWN_ISSUES 2026-09-09 19:35 закрыт) +**Root Cause:** ".h" был в INDEX_EXTENSIONS (вектор-чанкинг шёл), но НЕ в PARSE_EXTENSIONS → CodeParser.parse_file возвращал [... +- **Статус:** автоматически синхронизировано + + +## 2026-09-07 — Cypher-движок: анонимные узлы/рёбра ломали MATCH; ActionReceipt не писался из write-пути + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Fixed (оба блока закрыты, тесты зелёные) +**Root Cause:** (1) Cypher: `from_node_alias` дефолтил в `n1`, а генератор создавал `n{path_idx*2}` для анонимного узла → `no such column: n0.id`; ... +- **Статус:** автоматически синхронизировано + + +## 2026-09-03 — Fake reindex ETA "~8s" + frozen progress in Finalizing (both fixed) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed (commit 32f11662; 5 pre-commit hooks OK; full pytest 1587 passed, 2 pre-existing unrelated env_extractor failures) +**Root Cause 1 (ETA "~8s"):** `_enrich_job_response` had a dead h... +- **Статус:** автоматически синхронизировано + + +## 2026-09-03 19:30 — CI RED: circular import layer ↔ tools_reg (architecture_linter) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed (commit f210ed7c; CI all-jobs green on ubuntu+windows) +**Root Cause:** My ETA refactor added `tools_reg → layer` import for `_embed_progress_from_log`, closing an existing `layer →... +- **Статус:** автоматически синхронизировано + + +## 2026-09-04 11:15 — CI RED: ruff lint errors caught only after push (3 commits) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed (commit 986c9be7) +**Root Cause:** Pre-commit hook did not run ruff. CI (`ruff check src/ tests/` in ci.yml) caught F401/W292 only after push, forcing fix-commits. Repeated 3 times ... +- **Статус:** автоматически синхронизировано + + +## 2026-09-05 12:30 — FIX: stale_detector + predict_change стабильно таймаутили через MCP (-32001): блокирующий sync-код в async-контексте + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed (code only, не запушено) — src/mcp/tools/doc_tools.py + predict_tools.py +**Root Cause:** `error_boundary` применяет `asyncio.wait_for(timeout_ms)` вокруг `execute`, но внутри `exec... +- **Статус:** автоматически синхронизировано + + +## 2026-09-06 21:00 — Починка lock_guard: таймаут 60s ломал весь .locks-протокол + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed / **Root Cause:** `scripts/lock_guard.py` `_run` default timeout=60s — любой `git commit` прогоняет pre-commit hook (verify_diary → полный pytest 5-10 мин на Windows), поэтому acqu... +- **Статус:** автоматически синхронизировано + + +## 2026-09-06 21:30 — sync-subprocess в async-MCP (context_tool, system_tools) — fixed + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed (code only) / **Root Cause:** системная проверка после фикса stale/predict: нашлись ещё sync `subprocess.run` внутри async `execute`. `GetContextTool._section_git` (git log через s... +- **Статус:** автоматически синхронизировано + + +## 2026-09-06 22:00 — P-001 рецидив: cmd-окна при запуске/открытии проекта (powershell/nvidia-smi без CREATE_NO_WINDOW) — FIXED + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed / **Root Cause:** повтор инцидента 2026-08-14 (P-001, «чёрные окна CMD»). Фикс 2026-08-14 добавил CREATE_NO_WINDOW для git/netstat/wmic/taskkill в runtime, но ПОЗВОЛИЛ дыру: `resou... +- **Статус:** автоматически синхронизировано + + +## 2026-09-08 19:40 — collect() в Cypher: json_group_array + типизированный декод (fixed) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed. / **Root Cause:** KNOWN_ISSUES 2026-09-07 ⏳ — `_translate_return_expr` заявлял `collect` как Supported, но SQLite не имеет функции COLLECT («no such function»); ни одного теста на... +- **Статус:** автоматически синхронизировано + + +## 2026-09-09 — Аудит «Active MSCodeBase» (Exhibit #23: MCP tool available but never invoked) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Open — зафиксирован гэп (исследование + план, код НЕ вносился) +**Root Cause:** фундамент (VOR / DebounceBatch / ConsistencyTracker / IdleScheduler / PropagationEngine) существует, но компо... +- **Статус:** автоматически синхронизировано + + +## 2026-09-10 — H1 idle-VOR + system_alerts (цепь «файл изменён → STALE → VOR → alert агента» собрана) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** ✅ Fixed / **Root Cause (Exhibit #23, 2026-09-09):** компоненты цепи существовали по отдельности, но VOR вызывался ровно из 1 места (layer.py:intel_get_project_memory), mark_stale("memory")... +- **Статус:** автоматически синхронизировано + + +## 2026-09-11 — H3 TTL-гниение: last_checked для всех проверенных + label stale_ttl (doc 10 closed) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Fixed (9 новых тестов + 1725 полный pytest green; doc 10-continuous-verification H1+H2+H3 done) +**Root Cause:** INCONCLUSIVE/непроверенные узлы «висят вечно» без следа проверки: live-срез ... +- **Статус:** автоматически синхронизировано + + +## 2026-09-20 — Поисковое качество / E13: исследовательские задачи (6 пунктов) + +- **Источник:** AGENT_DIARY.md +- **Описание:** **Status:** Plan (задачи занесены в ISSUE.md KI-R1..R6, код не тронут) +**Контекст:** исследование поиска/RAG — что именно измерять, прежде чем утверждать результат. +**Решение (приоритет):** KI-R1 (пер... +- **Статус:** автоматически синхронизировано + diff --git a/docs/en/ARCHITECTURE.md b/docs/en/ARCHITECTURE.md index 3a5f43f3..3fdebaeb 100644 --- a/docs/en/ARCHITECTURE.md +++ b/docs/en/ARCHITECTURE.md @@ -4,7 +4,7 @@ # MSCodeBase Intelligence — Architecture Guide -> **Version:** 3.4.0 +> **Version:** 3.5.0 > **Last updated:** 2026-08-12 > **Architecture:** 4-Layer Architecture + Graph-Native PropertyGraph Layer + Data Flow Layer (Entry Points → MCP Server/DI → Tool Classes → Core Business Logic → PropertyGraph → Data Flow) with Multi-Window Registry + DocSync @@ -15,7 +15,7 @@ 1. [Core Principles](#1-core-principles) 2. [Layer Architecture](#2-layer-architecture) 3. [DI Container (ServiceCollection)](#3-di-container) -4. [Tool Layer (31 core + 16 intel + 13 inline + 4 dev = 64 total)](#4-tool-layer) +4. [Tool Layer (32 core + 16 intel + 13 inline + 4 dev = 65 total)](#4-tool-layer) 5. [PropertyGraph Layer (v3.0)](#5-propertygraph-layer-v30) 6. [Cypher Query Engine (v3.0)](#6-cypher-query-engine-v30) 7. [Error Handling](#7-error-handling) @@ -35,7 +35,7 @@ │ │ │ Layer 1: main.py (Entry points, minimal) │ │ Layer 2: mcp/server.py (DI routing, tool registration) │ -│ Layer 3: mcp/tools/*.py (28 core + 13 inline + 4 dev)│ +│ Layer 3: mcp/tools/*.py (32 core + 13 inline + 4 dev)│ │ Layer 4: core/*.py (Pure business logic) │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -55,7 +55,7 @@ ``` Layer 0: Filesystem — what files exist on disk? Layer 1: SystemArtifacts — is this a system path? - Layer 2: Bridge (LSP→MCP) — which project did LSP report? + Layer 2: Bridge (LSP→MCP) — [deprecated: LSP server removed 2026-07-20] Layer 3: Registry (IndexerRegistry) — which Indexer owns this project? Layer 4: StateMachine (ProjectState) — what state is the project in? Layer 5: RuntimeCoordinator — can we execute this request? @@ -86,19 +86,19 @@ Both use the same `create_service_collection()` factory. ### 2.2 MCP Server -| `src/mcp/server.py` | **~220 lines** (was 3,100 before refactoring). +| `src/mcp/server.py` | **~68 lines** (was 3,100 before refactoring). Responsibilities: 1. Resolve project root (`resolve_project_root()`) 2. Create DI container (`create_service_collection()`) -3. Register 31 core + 16 intel + 13 inline + 4 dev = 64 total +3. Register 32 core + 16 intel + 13 inline + 4 dev = 65 total 4. Register system prompt (mscodebase-rules) **No business logic lives here.** Every tool is an import from `mcp/tools/`. ### 2.3 Tool Layer -`src/mcp/tools/*.py` — **15 files: 28 core (21 + codebase hub + 6 LSP) + 13 inline + 4 dev.** +`src/mcp/tools/*.py` — **32 core (25 + codebase hub + 6 LSP) + 13 inline + 4 dev.** Every tool: - Inherits from `MCPTool` (ABC) @@ -129,13 +129,13 @@ class SearchCodeTool(MCPTool): ### 2.4 Core Layer -`src/core/*.py` — **30 files of pure business logic.** +`src/core/*.py` — **54 top-level files** (123 incl. subpackages) of pure business logic. Key modules: | Module | Path | Purpose | |--------|------|---------| -| `di_container.py` | `src/core/di_container.py` | DI Container (15+ services) | +| `di_container.py` | `src/core/di_container.py` | DI Container (14 services) | | `error_handler.py` | `src/core/error_handler.py` | ToolError + error_boundary | | `rate_limiter.py` | `src/core/rate_limiter.py` | DebounceBatch + CircuitBreaker | | `engine.py` | `src/core/search/engine.py` | Hybrid search (BM25 + Dense + FTS5 + RRF) | @@ -153,7 +153,7 @@ Key modules: | `runtime_coordinator.py` | `src/core/runtime_coordinator.py` | ExecutionVerdict + can_execute() | | `project_context.py` | `src/core/intelligence/project_context.py` | Project state snapshot | | `llama_runner.py` | `src/providers/reranker/llama_runner.py` | Lifecycle for llama-server.exe (reranker) | -| `remote_embedder.py` | `src/providers/embedder/remote_embedder.py` | ONNX E5-small INT8 embedder + LM Studio/Ollama fallback | +| `remote_embedder.py` | `src/providers/embedder/remote_embedder.py` | Embedder client: llama.cpp GGUF (primary) + ONNX INT8 / LM Studio / Ollama fallback | | `doc_sync_engine.py` | `src/core/doc_sync_engine.py` | Auto-sync docs with code (rename hook) | ### 2.5 Search Engine (v3.3) @@ -172,7 +172,7 @@ Key modules: ┌─────────────────────────────────────────────────────────┐ │ PropertyGraph (graph.py) │ │ SQLite (WAL + mmap), nodes/edges, JSON properties │ -│ — 15 node labels (File, Function, Class, Variable...) │ +│ — 16 node labels (File, Function, Class, Variable...) │ │ — 29 edge types (CALLS, DEFINES, ASSIGNED_FROM, ...) │ │ — Cypher query engine (MATCH→SQL) │ └─────────────────────────────────────────────────────────┘ @@ -228,7 +228,7 @@ services.add_factory(Searcher, lambda s: Searcher(s.resolve(Indexer), ...)) indexer = services.resolve(Indexer) # same instance every time ``` -### 3.2 Registered Services (11) +### 3.2 Registered Services (14) | # | Service | Type | Created By | |---|---------|------|------------| @@ -243,6 +243,9 @@ indexer = services.resolve(Indexer) # same instance every time | 9 | IndexerFactoryKey | factory | `_create_indexer_for_path` | | 10 | SlidingWindowRateLimiter | singleton | `SlidingWindowRateLimiter()` | | 11 | CircuitBreaker | singleton | `CircuitBreaker(name="lm_studio")` | +| 12 | Path | singleton | `project_root` (resolved path) | +| 13 | Indexer | singleton | `indexer_instance` | +| 14 | GitUrlSourceFactoryKey | factory | `_create_git_url_source` | > Cleanup (Task 2/5): dead registrations removed — DbPathKey, FileGuard, > SymbolIndex, ResourceMonitor, ResourceMonitorKey (never resolved). @@ -275,10 +278,10 @@ def register_all_tools(mcp, services): SubmitBackgroundTaskTool, GetTaskStatusTool, VerifyActionTool, GetActionReceiptTool, ] # +16 intel_* tools + 13 inline diagnostic + 4 dev - # Total: 61 registered (28 core + 16 intel + 13 inline + 4 dev) + # Total: 65 registered (32 core + 16 intel + 13 inline + 4 dev) ``` -**Tool visibility filter:** By default 46 tools visible (13 of 28 core via allowlist + 16 intel + 13 inline + 4 dev; +1 `execute_script` при `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`). Set `MSCODEBASE_MCP_TOOLS=""` to show all 58. +**Tool visibility filter:** the default visible set is the `MSCODEBASE_MCP_TOOLS` allowlist (see `src/mcp/server_tools.py`). Set `MSCODEBASE_MCP_TOOLS=""` to show all 65 (+1 `execute_script` при `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`). ### 4.2 All Tools by Group @@ -293,13 +296,13 @@ def register_all_tools(mcp, services): | **Investigation** (3) | `investigation_tools.py` | get_bug_correlation, get_hotspots, find_similar_bugs | | **Duplication** (1) | `duplication_tool.py` | find_duplicates | | **Context** (1) | `context_tool.py` | get_context | -| **Lifecycle** (3) | `lifecycle_tools.py` | submit_background_task, get_task_status, verify_action | +| **Lifecycle** (4) | `lifecycle_tools.py` | submit_background_task, get_task_status, verify_action, get_action_receipt | | **Docs** (1) | `doc_tools.py` | stale_detector | | **Dev** (4) | `dev_tools.py` | generate_docs, bump_version, auto_update_docs, install_git_hooks | -| **Intelligence** (14) | `intelligence/tools_reg.py` | intel_get_runtime_status, intel_trigger_reindex, intel_reset_index, intel_get_job_status, intel_code_topology, intel_log_incident, intel_get_project_memory, intel_add_memory_node, intel_auto_collect_adrs, intel_get_hotspots, intel_analyze_incident, intel_predict_root_cause, intel_get_telemetry, intel_retract_memory_node | -| **Diagnostic inline** (12) | `server_tools.py` | debug_runtime_passport, intel_get_project_context, intel_explain_project_state, get_runtime_counters, intel_tool_health, intel_execution_timeline, refresh_db_connection, notify_change, read_live_file, get_logs, get_health_report, ack_impact | +| **Intelligence** (16) | `intelligence/tools_reg.py` | intel_get_runtime_status, intel_trigger_reindex, intel_reset_index, intel_get_job_status, intel_code_topology, intel_log_incident, intel_get_project_memory, intel_add_memory_node, intel_auto_collect_adrs, intel_get_hotspots, intel_analyze_incident, intel_predict_root_cause, intel_get_telemetry, intel_retract_memory_node, intel_restore_memory_node, intel_supersede_memory_node | +| **Diagnostic inline** (13) | `server_tools.py` | debug_runtime_passport, intel_get_project_context, intel_explain_project_state, get_runtime_counters, intel_tool_health, intel_execution_timeline, refresh_db_connection, notify_change, read_live_file, get_logs, get_health_report, dual_arm_health_check, ack_impact | -> **Total:** 61 registered (28 core + 16 intel + 13 inline + 4 dev). Default visible: 46 (13 из 28 core по default-allowlist + 16 + 13 + 4). Show all: `MSCODEBASE_MCP_TOOLS=""`. +> **Total:** 65 registered (32 core + 16 intel + 13 inline + 4 dev). Default visible: `MSCODEBASE_MCP_TOOLS` allowlist (see `src/mcp/server_tools.py`). Show all: `MSCODEBASE_MCP_TOOLS=""`. ## 5. Error Handling diff --git a/docs/en/ARCHITECTURE_DEEP.md b/docs/en/ARCHITECTURE_DEEP.md index d959f9b2..a911c719 100644 --- a/docs/en/ARCHITECTURE_DEEP.md +++ b/docs/en/ARCHITECTURE_DEEP.md @@ -2,7 +2,7 @@ [🇬🇧 English](ARCHITECTURE_DEEP.md) • [🇷🇺 Русский](../ru/ARCHITECTURE_DEEP.md) • [🇨🇳 中文](../zh/ARCHITECTURE_DEEP.md) -> **Version:** v3.4.0 | **Last updated:** 2026-08-12 +> **Version:** v3.5.0 | **Last updated:** 2026-08-12 ```mermaid flowchart TD diff --git a/docs/en/CONTRIBUTING.md b/docs/en/CONTRIBUTING.md index 9bd0a1af..c522e673 100644 --- a/docs/en/CONTRIBUTING.md +++ b/docs/en/CONTRIBUTING.md @@ -4,7 +4,7 @@ # Contributing — MSCodeBase Intelligence -> **Version:** 3.4.0 +> **Version:** 3.5.0 --- @@ -342,7 +342,7 @@ SemVer: MAJOR.MINOR.PATCH - **MINOR** — new tools/features (backward compatible) - **PATCH** — bug fixes -Current version in `pyproject.toml`: `3.4.0` +Current version in `pyproject.toml`: `3.5.0` --- diff --git a/docs/en/GRACEFUL_DEGRADATION.md b/docs/en/GRACEFUL_DEGRADATION.md index 5d401669..56df4ef3 100644 --- a/docs/en/GRACEFUL_DEGRADATION.md +++ b/docs/en/GRACEFUL_DEGRADATION.md @@ -1,6 +1,6 @@ # Graceful Degradation — System Resilience Guide -> **Part of MSCodeBase Intelligence** | v3.4.0 +> **Part of MSCodeBase Intelligence** | v3.5.0 ## Overview diff --git a/docs/en/SEARCH_PIPELINE.md b/docs/en/SEARCH_PIPELINE.md index 5effe126..a6b253c5 100644 --- a/docs/en/SEARCH_PIPELINE.md +++ b/docs/en/SEARCH_PIPELINE.md @@ -1,6 +1,6 @@ # Search Pipeline — Complete Technical Reference -> **Part of MSCodeBase Intelligence** | v3.4.0 +> **Part of MSCodeBase Intelligence** | v3.5.0 ## Overview diff --git a/docs/ru/ARCHITECTURE.md b/docs/ru/ARCHITECTURE.md index ddf39662..d28e3a06 100644 --- a/docs/ru/ARCHITECTURE.md +++ b/docs/ru/ARCHITECTURE.md @@ -4,7 +4,7 @@ # MSCodeBase Intelligence — Архитектура -> **Версия:** 3.4.0 +> **Версия:** 3.5.0 > **Последнее обновление:** 2026-08-12 > **Архитектура:** 4-слойная архитектура + Graph-Native PropertyGraph Layer + Data Flow Layer (Точки входа → MCP Server/DI → Tool Classes → Core Business Logic → PropertyGraph → Data Flow) с Multi-Window Registry + DocSync @@ -15,7 +15,7 @@ 1. [Основные принципы](#1-core-principles) 2. [Слойная архитектура](#2-layer-architecture) 3. [DI-контейнер (ServiceCollection)](#3-di-container) -4. [Слой инструментов (31 core + 16 intel + 13 inline + 4 dev = 64 всего)](#4-tool-layer) +4. [Слой инструментов (32 core + 16 intel + 13 inline + 4 dev = 65 всего)](#4-tool-layer) 5. [PropertyGraph (v3.0)](#5-propertygraph-layer-v30) 6. [Cypher Query Engine (v3.0)](#6-cypher-query-engine-v30) 7. [Обработка ошибок](#7-error-handling) @@ -37,7 +37,7 @@ │ │ │ Слой 1: main.py (Точки входа, минималистичные) │ │ Слой 2: mcp/server.py (DI-маршрутизация, регистрация) │ -│ Слой 3: mcp/tools/*.py (28 core + 13 inline + 4 dev) │ +│ Слой 3: mcp/tools/*.py (32 core + 13 inline + 4 dev) │ │ Слой 4: core/*.py (Чистая бизнес-логика) │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -57,7 +57,7 @@ ``` Слой 0: Filesystem — какие файлы есть на диске? Слой 1: SystemArtifacts — это системный путь? - Слой 2: Bridge (LSP→MCP) — какой проект сообщил LSP? + Слой 2: Bridge (LSP→MCP) — [deprecated: LSP server removed 2026-07-20] Слой 3: Registry (IndexerRegistry) — какой Indexer принадлежит проекту? Слой 4: StateMachine (ProjectState) — в каком состоянии проект? Слой 5: RuntimeCoordinator — можно ли выполнять запрос? @@ -88,19 +88,19 @@ MCP Tools ← Intel Layer ← ProjectContext ← RuntimeCoordinator ### 2.2 MCP-сервер -`src/mcp/server.py` — **~220 строк** (было 3 100 до рефакторинга). +`src/mcp/server.py` — **~68 строк** (было 3 100 до рефакторинга). Обязанности: 1. Определить корень проекта (`resolve_project_root()`) 2. Создать DI-контейнер (`create_service_collection()`) -3. Зарегистрировать 31 core + 16 intel + 13 inline + 4 dev = 64 всего +3. Зарегистрировать 32 core + 16 intel + 13 inline + 4 dev = 65 всего 4. Зарегистрировать system prompt (mscodebase-rules) **Здесь нет бизнес-логики.** Каждый инструмент — импорт из `mcp/tools/`. ### 2.3 Слой инструментов -`src/mcp/tools/*.py` — **15 файлов: 28 core (21 + codebase hub + 6 LSP) + 13 inline + 4 dev.** +`src/mcp/tools/*.py` — **32 core (25 + codebase hub + 6 LSP) + 13 inline + 4 dev.** Каждый инструмент: - Наследуется от `MCPTool` (ABC) @@ -131,13 +131,13 @@ class SearchCodeTool(MCPTool): ### 2.4 Слой ядра -`src/core/*.py` — **30 файлов чистой бизнес-логики.** +`src/core/*.py` — **54 файла верхнего уровня** (123 с подпакетами) чистой бизнес-логики. Ключевые модули: | Модуль | Путь | Назначение | |--------|------|------------| -| `di_container.py` | `src/core/di_container.py` | DI-контейнер (15+ сервисов) | +| `di_container.py` | `src/core/di_container.py` | DI-контейнер (14 сервисов) | | `error_handler.py` | `src/core/error_handler.py` | ToolError + error_boundary | | `rate_limiter.py` | `src/core/rate_limiter.py` | DebounceBatch + CircuitBreaker | | `engine.py` | `src/core/search/engine.py` | Гибридный поиск (BM25 + Dense + FTS5 + RRF) | @@ -277,10 +277,10 @@ def register_all_tools(mcp, services): SubmitBackgroundTaskTool, GetTaskStatusTool, VerifyActionTool, GetActionReceiptTool, ] # +16 intel_* инструментов + 13 inline diagnostic + 4 dev - # Всего: 61 зарегистрировано (28 core + 16 intel + 13 inline + 4 dev) + # Всего: 65 зарегистрировано (32 core + 16 intel + 13 inline + 4 dev) ``` -**Фильтр видимости инструментов:** По умолчанию видимо 46 инструмента (13 из 28 core по default-allowlist + 16 intel + 13 inline + 4 dev; +1 `execute_script` при `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`). Установите `MSCODEBASE_MCP_TOOLS=""` чтобы показать все 58. +**Фильтр видимости инструментов:** по умолчанию видим набор из allowlist `MSCODEBASE_MCP_TOOLS` (см. `src/mcp/server_tools.py`). Установите `MSCODEBASE_MCP_TOOLS=""`, чтобы показать все 65 (+1 `execute_script` при `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true`). ### 4.2 Все инструменты по группам @@ -299,9 +299,9 @@ def register_all_tools(mcp, services): | **Doc** (1) | `doc_tools.py` | stale_detector | | **Dev** (4) | `dev_tools.py` | generate_docs, bump_version, auto_update_docs, install_git_hooks | | **Intelligence** (14) | `intelligence/tools_reg.py` | intel_get_runtime_status, intel_trigger_reindex, intel_reset_index, intel_get_job_status, intel_code_topology, intel_log_incident, intel_get_project_memory, intel_add_memory_node, intel_auto_collect_adrs, intel_get_hotspots, intel_analyze_incident, intel_predict_root_cause, intel_get_telemetry, intel_retract_memory_node | -| **Diagnostic inline** (12) | `server_tools.py` | debug_runtime_passport, intel_get_project_context, intel_explain_project_state, get_runtime_counters, intel_tool_health, intel_execution_timeline, refresh_db_connection, notify_change, read_live_file, get_logs, get_health_report, ack_impact | +| **Diagnostic inline** (13) | `server_tools.py` | debug_runtime_passport, intel_get_project_context, intel_explain_project_state, get_runtime_counters, intel_tool_health, intel_execution_timeline, refresh_db_connection, notify_change, read_live_file, get_logs, get_health_report, dual_arm_health_check, ack_impact | -> **Всего:** 64 зарегистрировано (31 core + 16 intel + 13 inline + 4 dev). По умолчанию видимо: 49 (16 из 31 core по default-allowlist + 16 + 13 + 4). Показать все: `MSCODEBASE_MCP_TOOLS=""`. +> **Всего:** 65 зарегистрировано (32 core + 16 intel + 13 inline + 4 dev). По умолчанию видим набор из allowlist `MSCODEBASE_MCP_TOOLS` (см. `src/mcp/server_tools.py`). Показать все: `MSCODEBASE_MCP_TOOLS=""`. ## 5. Обработка ошибок diff --git a/docs/ru/ARCHITECTURE_DEEP.md b/docs/ru/ARCHITECTURE_DEEP.md index 032d285f..217917fd 100644 --- a/docs/ru/ARCHITECTURE_DEEP.md +++ b/docs/ru/ARCHITECTURE_DEEP.md @@ -2,7 +2,7 @@ [🇬🇧 English](../en/ARCHITECTURE_DEEP.md) • [🇷🇺 Русский](ARCHITECTURE_DEEP.md) • [🇨🇳 中文](../zh/ARCHITECTURE_DEEP.md) -> **Версия:** v3.4.0 | **Последнее обновление:** 2026-08-12 +> **Версия:** v3.5.0 | **Последнее обновление:** 2026-08-12 ```mermaid flowchart TD diff --git a/docs/ru/CONTRIBUTING.md b/docs/ru/CONTRIBUTING.md index 09fdc546..7a1fafa2 100644 --- a/docs/ru/CONTRIBUTING.md +++ b/docs/ru/CONTRIBUTING.md @@ -4,7 +4,7 @@ # Контрибьюция — MSCodeBase Intelligence -> **Версия:** 3.4.0 +> **Версия:** 3.5.0 --- @@ -342,7 +342,7 @@ SemVer: MAJOR.MINOR.PATCH - **MINOR** — новые инструменты/функции (обратно совместимые) - **PATCH** — багфиксы -Текущая версия в `pyproject.toml`: `3.4.0` +Текущая версия в `pyproject.toml`: `3.5.0` --- diff --git a/docs/ru/GRACEFUL_DEGRADATION.md b/docs/ru/GRACEFUL_DEGRADATION.md index 0b77aca4..21a829ca 100644 --- a/docs/ru/GRACEFUL_DEGRADATION.md +++ b/docs/ru/GRACEFUL_DEGRADATION.md @@ -1,6 +1,6 @@ # Graceful Degradation — Руководство по отказоустойчивости -> **Часть MSCodeBase Intelligence** | v3.4.0 +> **Часть MSCodeBase Intelligence** | v3.5.0 ## Обзор diff --git a/docs/ru/SEARCH_PIPELINE.md b/docs/ru/SEARCH_PIPELINE.md index 26f55816..8f4aadf8 100644 --- a/docs/ru/SEARCH_PIPELINE.md +++ b/docs/ru/SEARCH_PIPELINE.md @@ -1,6 +1,6 @@ # Поисковый пайплайн — Полная техническая документация -> **Часть MSCodeBase Intelligence** | v3.4.0 +> **Часть MSCodeBase Intelligence** | v3.5.0 ## Обзор diff --git a/docs/zh/ARCHITECTURE.md b/docs/zh/ARCHITECTURE.md index 7845327d..5b00cbcb 100644 --- a/docs/zh/ARCHITECTURE.md +++ b/docs/zh/ARCHITECTURE.md @@ -4,7 +4,7 @@ # MSCodeBase Intelligence — 架构指南 -> **版本：** 3.4.0 +> **版本：** 3.5.0 > **最后更新：** 2026-08-12 > **架构：** 4 层架构 + 图原生 PropertyGraph 层 + 数据流层（入口点 → MCP 服务器/DI → 工具类 → 核心业务逻辑 → PropertyGraph → 数据流）带多窗口注册表 + DocSync @@ -15,7 +15,7 @@ 1. [核心原则](#1-核心原则) 2. [分层架构](#2-分层架构) 3. [DI 容器（ServiceCollection）](#3-di-容器) -4. [工具层（31 核心 + 16 intel + 13 内联 + 4 开发 = 64 个）](#4-工具层) +4. [工具层（32 核心 + 16 intel + 13 内联 + 4 开发 = 65 个）](#4-工具层) 5. [PropertyGraph 层（v3.0）](#5-propertygraph-层-v30) 6. [Cypher 查询引擎（v3.0）](#6-cypher-查询引擎-v30) 7. [错误处理](#7-错误处理) @@ -35,7 +35,7 @@ │ │ │ 第 1 层: main.py （入口点，最简） │ │ 第 2 层: mcp/server.py （DI 路由，工具注册） │ -│ 第 3 层: mcp/tools/*.py （20 核心 + 13 内联 + 4 开发） │ +│ 第 3 层: mcp/tools/*.py （32 核心 + 13 内联 + 4 开发） │ │ 第 4 层: core/*.py （纯业务逻辑） │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -55,7 +55,7 @@ ``` Layer 0: Filesystem — 磁盘上有哪些文件？ Layer 1: SystemArtifacts — 这是系统路径吗？ - Layer 2: Bridge (LSP→MCP) — LSP 报告了哪个项目？ + Layer 2: Bridge (LSP→MCP) — [deprecated: LSP server removed 2026-07-20] Layer 3: Registry (IndexerRegistry) — 哪个 Indexer 拥有此项目？ Layer 4: StateMachine (ProjectState) — 项目处于什么状态？ Layer 5: RuntimeCoordinator — 可以执行此请求吗？ @@ -86,19 +86,19 @@ MCP Tools ← Intel Layer ← ProjectContext ← RuntimeCoordinator ### 2.2 MCP 服务器 -| `src/mcp/server.py` | **~220 行**（重构前为 3,100 行）。 +| `src/mcp/server.py` | **~68 行**（重构前为 3,100 行）。 职责： 1. 解析项目根目录（`resolve_project_root()`） 2. 创建 DI 容器（`create_service_collection()`） -3. 注册 31 核心 + 16 intel + 13 内联 + 4 开发 = 64 个工具 +3. 注册 32 核心 + 16 intel + 13 内联 + 4 开发 = 65 个工具 4. 注册系统提示（mscodebase-rules） **此处没有业务逻辑。** 每个工具都是从 `mcp/tools/` 导入的。 ### 2.3 工具层 -`src/mcp/tools/*.py` — **14 个文件：20 个核心 (19 + codebase hub) + 12 个内联 + 4 个开发（Hub & Spoke：codebase + execute_script + 17 个原生）。** +`src/mcp/tools/*.py` — **32 个核心 + 13 个内联 + 4 个开发（含 codebase hub + 6 LSP）。** 每个工具： - 继承自 `MCPTool`（ABC） @@ -129,13 +129,13 @@ class SearchCodeTool(MCPTool): ### 2.4 核心层 -`src/core/*.py` — **30 个纯业务逻辑文件。** +`src/core/*.py` — **54 个顶层文件**（含子包 123 个）纯业务逻辑。 关键模块： | 模块 | 路径 | 用途 | |--------|------|---------| -| `di_container.py` | `src/core/di_container.py` | DI 容器（15+ 服务） | +| `di_container.py` | `src/core/di_container.py` | DI 容器（14 服务） | | `error_handler.py` | `src/core/error_handler.py` | ToolError + error_boundary | | `rate_limiter.py` | `src/core/rate_limiter.py` | DebounceBatch + CircuitBreaker | | `engine.py` | `src/core/search/engine.py` | 混合搜索（BM25 + Dense + FTS5 + RRF） | @@ -277,7 +277,7 @@ def register_all_tools(mcp, services): # 总计：61 个已注册（28 核心 + 16 intel + 13 内联 + 4 开发） ``` -**工具可见性过滤器：** 默认显示 46 个工具（28 个核心中的 13 个 + 16 intel + 13 内联 + 4 开发；启用 `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true` 时 +1 `execute_script`）。设置 `MSCODEBASE_MCP_TOOLS=""` 以显示全部 61 个。 +**工具可见性过滤器：** 默认可见集合为 `MSCODEBASE_MCP_TOOLS` allowlist（见 `src/mcp/server_tools.py`）。设置 `MSCODEBASE_MCP_TOOLS=""` 以显示全部 65 个（启用 `MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true` 时 +1 `execute_script`）。 ### 4.2 按组分组的全部工具 @@ -298,7 +298,7 @@ def register_all_tools(mcp, services): | **智能层**（14 个） | `intelligence/tools_reg.py` | intel_get_runtime_status, intel_trigger_reindex, intel_reset_index, intel_get_job_status, intel_code_topology, intel_log_incident, intel_get_project_memory, intel_add_memory_node, intel_auto_collect_adrs, intel_get_hotspots, intel_analyze_incident, intel_predict_root_cause, intel_get_telemetry, intel_retract_memory_node | | **诊断内联**（12 个） | `server_tools.py` | debug_runtime_passport, intel_get_project_context, intel_explain_project_state, get_runtime_counters, intel_tool_health, intel_execution_timeline, refresh_db_connection, notify_change, read_live_file, get_logs, get_health_report, ack_impact | -> **总计：** 61 个已注册（28 核心 + 16 intel + 13 内联 + 4 开发）。默认可见：46 个（28 个核心中的 13 个 + 14 + 12 + 4）。显示全部：`MSCODEBASE_MCP_TOOLS=""`。 +> **总计：** 65 个已注册（32 核心 + 16 intel + 13 内联 + 4 开发）。默认可见：`MSCODEBASE_MCP_TOOLS` allowlist（见 `src/mcp/server_tools.py`）。显示全部：`MSCODEBASE_MCP_TOOLS=""`。 --- diff --git a/docs/zh/ARCHITECTURE_DEEP.md b/docs/zh/ARCHITECTURE_DEEP.md index ad621bbf..33a5a5e7 100644 --- a/docs/zh/ARCHITECTURE_DEEP.md +++ b/docs/zh/ARCHITECTURE_DEEP.md @@ -2,7 +2,7 @@ [🇬🇧 English](../en/ARCHITECTURE_DEEP.md) • [🇷🇺 Русский](../ru/ARCHITECTURE_DEEP.md) • [🇨🇳 中文](ARCHITECTURE_DEEP.md) -> **版本：** v3.4.0 | **最后更新：** 2026-08-12 +> **版本：** v3.5.0 | **最后更新：** 2026-08-12 ```mermaid flowchart TD diff --git a/docs/zh/CONTRIBUTING.md b/docs/zh/CONTRIBUTING.md index bf7cb266..4bd86c9f 100644 --- a/docs/zh/CONTRIBUTING.md +++ b/docs/zh/CONTRIBUTING.md @@ -1,6 +1,6 @@ # 贡献指南 — MSCodeBase Intelligence -> **版本：** 3.4.0 +> **版本：** 3.5.0 --- @@ -338,7 +338,7 @@ SemVer：MAJOR.MINOR.PATCH - **MINOR** — 新工具/功能（向后兼容） - **PATCH** — 错误修复 -当前版本在 `pyproject.toml` 中：`3.4.0` +当前版本在 `pyproject.toml` 中：`3.5.0` --- diff --git a/docs/zh/GRACEFUL_DEGRADATION.md b/docs/zh/GRACEFUL_DEGRADATION.md index a38c064c..40b15138 100644 --- a/docs/zh/GRACEFUL_DEGRADATION.md +++ b/docs/zh/GRACEFUL_DEGRADATION.md @@ -1,6 +1,6 @@ # 优雅降级（Graceful Degradation）— 系统弹性指南 -> **MSCodeBase Intelligence 的一部分** | v3.4.0 +> **MSCodeBase Intelligence 的一部分** | v3.5.0 ## 概述 diff --git a/docs/zh/SEARCH_PIPELINE.md b/docs/zh/SEARCH_PIPELINE.md index b352cd8f..5936f308 100644 --- a/docs/zh/SEARCH_PIPELINE.md +++ b/docs/zh/SEARCH_PIPELINE.md @@ -1,6 +1,6 @@ # 搜索流水线（pipeline）— 完整技术参考 -> **MSCodeBase Intelligence 的一部分** | v3.4.0 +> **MSCodeBase Intelligence 的一部分** | v3.5.0 ## 概述 diff --git a/experiments/bootstrap/e17_check_decoy.py b/experiments/bootstrap/e17_check_decoy.py new file mode 100644 index 00000000..37324a97 --- /dev/null +++ b/experiments/bootstrap/e17_check_decoy.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +"""E17 decoy audit: check D-arm against the owner's 3 criteria. + +1. D tests are UNIQUE across functions (no repeats). +2. D tests are FOREIGN: they cover no function in the target's source file. +3. D test coverage matches B coverage (same size/difficulty). +""" +import sys +import json +import sqlite3 +from pathlib import Path +from collections import Counter + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.stdout.reconfigure(encoding="utf-8") + +from src.core.artifact_paths import get_graph_db_path # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + + +def main() -> int: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + conn = sqlite3.connect(str(get_graph_db_path(ROOT))) + cov = dict(conn.execute( + "SELECT source_id, COUNT(*) FROM edges WHERE type='TESTS' GROUP BY source_id" + ).fetchall()) + name_by_id = dict(conn.execute("SELECT id, name FROM nodes WHERE label='Test'")) + id_by_name = {n: i for i, n in name_by_id.items()} + files_of: dict = {} + for sid, fp in conn.execute( + "SELECT e.source_id, n.file_path FROM edges e JOIN nodes n ON n.id=e.target_id " + "WHERE e.type='TESTS'" + ): + files_of.setdefault(sid, set()).add(fp) + conn.close() + + d_counts = Counter() + foreign_violations = [] + cov_mismatch = [] + d_eq_b = [] + for f in data["functions"]: + b = f["modes"]["B_runtime"]["tests"] + d = f["modes"]["D_shuffled"]["tests"] + ffile = f["file_path"] + for t in d: + d_counts[t] += 1 + tid = id_by_name.get(t) + covered = files_of.get(tid, set()) + if ffile in covered: + foreign_violations.append((f["name"], t)) + bcov = [cov[id_by_name[x]] for x in b if x in id_by_name] + if tid in cov and bcov and cov[tid] > max(bcov): + cov_mismatch.append((f["name"], t, cov[tid], max(bcov))) + if set(d) & set(b): + d_eq_b.append(f["name"]) + + dupes = {t: c for t, c in d_counts.items() if c > 1} + print(f"functions: {len(data['functions'])}") + print(f"1) unique D tests: {len(d_counts)} | repeated: {len(dupes)} {list(dupes.items())[:5]}") + print(f"2) foreign violations: {len(foreign_violations)} {foreign_violations[:5]}") + print(f"3) cov>B mismatches: {len(cov_mismatch)} {cov_mismatch[:5]}") + print(f" D==B collisions: {len(d_eq_b)} {d_eq_b[:5]}") + + ok = not dupes and not foreign_violations and not d_eq_b + print("DECOY AUDIT:", "PASSED" if ok else "FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/bootstrap/e17_complexity_probe.py b/experiments/bootstrap/e17_complexity_probe.py new file mode 100644 index 00000000..4ceb0149 --- /dev/null +++ b/experiments/bootstrap/e17_complexity_probe.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +"""E17 complexity probe: are there complex functions that still have specific tests? + +For every Function node with a test of coverage <= max_cov, compute a rough +complexity (LOC, AST nodes, branch nodes) and report the distribution and the +most complex candidates. Informs whether a "hard" panel is feasible. +""" +import sys +import ast +import sqlite3 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from src.core.artifact_paths import get_graph_db_path # noqa: E402 +from e17_extract import read_function_code # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] +COVERAGE_SQL = "SELECT source_id, COUNT(*) cov FROM edges WHERE type='TESTS' GROUP BY source_id" +_BRANCH = (ast.If, ast.For, ast.While, ast.Try, ast.With, ast.BoolOp, ast.Match) + + +def main() -> None: + conn = sqlite3.connect(str(get_graph_db_path(ROOT))) + rows = conn.execute( + f""" + SELECT f.name, f.file_path, COUNT(*) n_tests, MIN(tc.cov) min_cov + FROM nodes f + JOIN edges e ON e.target_id = f.id AND e.type = 'TESTS' + JOIN ({COVERAGE_SQL}) tc ON tc.source_id = e.source_id + WHERE f.label = 'Function' AND f.file_path LIKE '%/src/%' + GROUP BY f.id + HAVING min_cov <= 5 + """ + ).fetchall() + conn.close() + + cands = [] + for name, fp, n_tests, min_cov in rows: + if name == "_" or name.startswith("test_"): + continue + code = read_function_code(fp, name) + if not code: + continue + try: + tree = ast.parse(code) + except SyntaxError: + continue + loc = len([ln for ln in code.splitlines() if ln.strip()]) + nodes = sum(1 for _ in ast.walk(tree)) + branches = sum(1 for n in ast.walk(tree) if isinstance(n, _BRANCH)) + cands.append({"name": name, "file": fp, "loc": loc, "nodes": nodes, + "branches": branches, "n_tests": n_tests, "min_cov": min_cov}) + + buckets = {"loc<=20": 0, "21-40": 0, "41-80": 0, "81+": 0} + for c in cands: + loc = c["loc"] + key = "loc<=20" if loc <= 20 else "21-40" if loc <= 40 else "41-80" if loc <= 80 else "81+" + buckets[key] += 1 + print(f"candidates with min_cov<=5 and source: {len(cands)}") + print("LOC buckets:", buckets) + + print("\nmost complex by (branches, loc):") + for c in sorted(cands, key=lambda c: (-c["branches"], -c["loc"]))[:25]: + short = c["file"].replace("D:/Project/MSCodeBase/", "") + print(f" loc={c['loc']:3} nodes={c['nodes']:4} branches={c['branches']:3} " + f"min_cov={c['min_cov']} n={c['n_tests']:2} {c['name']:40} {short}") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_export_judge.py b/experiments/bootstrap/e17_export_judge.py new file mode 100644 index 00000000..28e32f16 --- /dev/null +++ b/experiments/bootstrap/e17_export_judge.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +"""E17 judge export: blind, test-hidden judging shards. + +Confound removed: the previous judge saw each answer's OWN tests, so arms B/D +(tests present) had more context than A (none) — which can inflate their +completeness. Here the judge sees ONLY question + code + answer for every arm, +so accuracy/completeness/safety are graded on an identical reference. The +evidence_usage manipulation check is measured separately (answers referencing +test names), not by the judge. + +Every (function, mode) answer becomes an anonymous item with an opaque token +(T01..TNN), globally shuffled. The token -> (id, mode) map is written +separately and MUST NOT be shown to the judge. + +Output: experiments/bootstrap/e17_judge/shard_.txt + mapping.json +""" +import sys +import json +import random +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "experiments/bootstrap/e17_judge" + +JUDGE_PROMPT = """You are an expert code reviewer. Evaluate the following answer about a Python function. + +Question: {question} + +Function code: +```python +{code} +``` + +Answer to evaluate: +{answer} + +Rate the answer on these metrics (1-5 scale): + +1. **Accuracy**: Is the predicted outcome (return value or exception type) factually correct for the situation in the question? + 1 = Completely wrong + 2 = Mostly wrong + 3 = Partially correct + 4 = Mostly correct, minor errors + 5 = Completely accurate + +2. **Completeness**: How thoroughly does the answer cover the outcome and its justification (inputs, branches, edge cases)? + 1 = Very incomplete + 2 = Incomplete + 3 = Moderately complete + 4 = Mostly complete + 5 = Very comprehensive + +3. **Safety**: Does the answer avoid unsafe or misleading claims? + 1 = Dangerous + 2 = Unsafe + 3 = Neutral + 4 = Generally safe + 5 = Very safe + +Provide your evaluation in this exact JSON format: +{{"accuracy": <1-5>, "completeness": <1-5>, "safety": <1-5>}} +""" + + +def main() -> None: + random.seed(23) + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + answers = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_answers.json").read_text(encoding="utf-8") + ) + by_id = {a["id"]: a for a in answers["answers"]} + + items = [] + for f in data["functions"]: + code = read_function_code(f["file_path"], f["name"]) + if not code: + raise SystemExit(f"no source for {f['name']}") + ans = by_id[f["id"]] + for mode in f["modes"]: + items.append({ + "id": f["id"], + "mode": mode, + "name": f["name"], + "question": f["question"], + "code": code, + "answer": ans["modes"][mode]["answer"], + }) + + random.shuffle(items) + mapping = {} + blocks = [] + for i, it in enumerate(items, 1): + token = f"T{i:02d}" + mapping[token] = {"id": it["id"], "mode": it["mode"], "name": it["name"]} + prompt = JUDGE_PROMPT.format( + question=it["question"], code=it["code"], answer=it["answer"] + ) + blocks.append(f"=== BEGIN token={token} ===\n{prompt}\n=== END token={token} ===\n") + + OUT.mkdir(exist_ok=True) + budget = int(sys.argv[1]) if len(sys.argv) > 1 else 1500 + + def n_lines(item: str) -> int: + return item.count("\n") + 1 + + shards: list = [] + current: list = [] + for block in blocks: + if current and sum(n_lines(b) for b in current) + n_lines(block) > budget: + shards.append(current) + current = [] + current.append(block) + if current: + shards.append(current) + + for k, shard in enumerate(shards, 1): + (OUT / f"shard_{k}.txt").write_text("\n".join(shard), encoding="utf-8") + print(f"shard {k}: {len(shard)} items, {sum(n_lines(b) for b in shard)} lines") + (OUT / "mapping.json").write_text( + json.dumps(mapping, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"total items: {len(items)} across {len(shards)} shards; mapping.json written") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_export_prompts.py b/experiments/bootstrap/e17_export_prompts.py new file mode 100644 index 00000000..c4695f6c --- /dev/null +++ b/experiments/bootstrap/e17_export_prompts.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""E17 export: split pilot prompts into shard text files for subagent generation. + +Why text shards: prompts are large; as JSON each prompt is one long line the +agent file-reader truncates. Plain-text shards with real newlines keep lines +short. Shards are packed by a LINE BUDGET (default 1500) so a subagent can read +a whole shard within its ~2000-line read limit even for large functions. + +Output: experiments/bootstrap/e17_shards/shard_.txt plus index.json +Each prompt block: + === BEGIN id= mode= name= === + + === END id= mode= === +""" +import sys +import json +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code # noqa: E402 +from e17_pilot_answers import build_prompt # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] +SHARDS_DIR = ROOT / "experiments/bootstrap/e17_shards" + + +def main() -> None: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + budget = int(sys.argv[1]) if len(sys.argv) > 1 else 1500 + SHARDS_DIR.mkdir(exist_ok=True) + + func_blocks = [] + for f in data["functions"]: + code = read_function_code(f["file_path"], f["name"]) + if not code: + raise SystemExit(f"no source for {f['name']}") + lines = [] + for mode, mdata in f["modes"].items(): + prompt = build_prompt(f["question"], code, mdata["tests"]) + lines.append(f"=== BEGIN id={f['id']} mode={mode} name={f['name']} ===") + lines.append(prompt) + lines.append(f"=== END id={f['id']} mode={mode} ===") + lines.append("") + func_blocks.append((f, lines)) + + def n_lines(entries: list) -> int: + return sum(s.count("\n") + 1 for s in entries) + + shards: list = [] + current_funcs: list = [] + current_lines: list = [] + for f, lines in func_blocks: + if current_lines and n_lines(current_lines) + n_lines(lines) > budget: + shards.append((current_funcs, current_lines)) + current_funcs, current_lines = [], [] + current_funcs.append(f) + current_lines.extend(lines) + if current_lines: + shards.append((current_funcs, current_lines)) + + index = [] + for k, (funcs, lines) in enumerate(shards, 1): + (SHARDS_DIR / f"shard_{k}.txt").write_text("\n".join(lines), encoding="utf-8") + print(f"shard {k}: {len(funcs)} funcs, {n_lines(lines)} lines") + for f in funcs: + for mode in f["modes"]: + index.append({"shard": k, "id": f["id"], "mode": mode, "name": f["name"]}) + + (SHARDS_DIR / "index.json").write_text( + json.dumps(index, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"total prompts: {len(index)} across {len(shards)} shards (budget {budget} lines)") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_extract.py b/experiments/bootstrap/e17_extract.py new file mode 100644 index 00000000..55954b6a --- /dev/null +++ b/experiments/bootstrap/e17_extract.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +"""E17 extraction: AST-based retrieval of function and test source. + +Why this module exists +---------------------- +The E17 pilot originally matched source with a naive substring search +(``f"def {name}"``). The PropertyGraph stores *qualified* names: +methods as ``Class.method`` and tests as ``Class::test_name``. Neither +ever matched ``def Class.method`` / ``def Class::test``, so every class +method got ``# FUNC NOT FOUND`` and every class-based test got +``# Test not found`` — the pilot measured hallucination, not the signal. + +This module resolves names via :mod:`ast`, so both free functions and +class methods/tests are extracted correctly. +""" +from __future__ import annotations + +import ast +from pathlib import Path +from typing import List, Optional + +ROOT = Path(__file__).resolve().parents[2] + +_FUNC_NODES = (ast.FunctionDef, ast.AsyncFunctionDef) + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="ignore") + + +def _segment(source: str, node: ast.AST, name: str) -> str: + """Return source text of ``node``; fall back to a line slice. + + ``errors="ignore"`` can shift byte offsets relative to the parsed text, + so a segment that does not contain the expected name is rejected in + favour of an explicit ``lineno``/``end_lineno`` slice. + """ + seg = ast.get_source_segment(source, node) + if seg is not None and name in seg: + return seg + lines = source.splitlines() + start = max(getattr(node, "lineno", 1) - 1, 0) + end = getattr(node, "end_lineno", start + 1) + return "\n".join(lines[start:end]) + + +def _find_in_tree(tree: ast.AST, qualified_name: str) -> Optional[ast.AST]: + parts = qualified_name.split(".") + method = parts[-1] + if len(parts) == 1: + for node in ast.walk(tree): + if isinstance(node, _FUNC_NODES) and node.name == method: + return node + return None + cls = parts[-2] + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == cls: + for item in node.body: + if isinstance(item, _FUNC_NODES) and item.name == method: + return item + return None + + +def read_function_code(file_path: str, function_name: str) -> Optional[str]: + """Return source of ``function_name`` from ``file_path`` or None. + + ``file_path`` may be an absolute graph path (``D:/Project/MSCodeBase/...``) + or a path relative to the repository root. ``function_name`` may be a bare + name (``safe_mkdir``) or qualified (``CypherLexer.tokenize``). + """ + local = file_path.replace("D:/Project/MSCodeBase/", "").replace("\\", "/") + path = ROOT / local + if not path.exists(): + return None + try: + source = _read(path) + tree = ast.parse(source) + except (SyntaxError, ValueError, OSError): + return None + node = _find_in_tree(tree, function_name) + if node is None: + return None + return _segment(source, node, function_name.split(".")[-1]) + + +def _find_test(tree: ast.AST, test_name: str) -> Optional[ast.AST]: + if "::" in test_name: + cls, method = test_name.split("::", 1) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == cls: + for item in node.body: + if isinstance(item, _FUNC_NODES) and item.name == method: + return item + return None + return _find_in_tree(tree, test_name) + + +def read_test_code(test_name: str, tests_dir: Optional[Path] = None) -> Optional[str]: + """Return source of a test function/class-method or None. + + ``test_name`` may be ``test_foo`` or ``TestClass::test_foo``. Searches + ``tests/`` recursively in a deterministic order. + """ + base = tests_dir or ROOT / "tests" + if not base.exists(): + return None + for test_file in sorted(base.rglob("*.py")): + try: + source = _read(test_file) + tree = ast.parse(source) + except (SyntaxError, ValueError, OSError): + continue + node = _find_test(tree, test_name) + if node is not None: + return _segment(source, node, test_name.split("::")[-1]) + return None + + +def read_tests(tests: List[str], tests_dir: Optional[Path] = None) -> List[str]: + """Return source bodies for the tests that could be resolved (skips misses).""" + bodies: List[str] = [] + for name in tests: + body = read_test_code(name, tests_dir) + if body is not None: + bodies.append(body) + return bodies diff --git a/experiments/bootstrap/e17_impact_score.py b/experiments/bootstrap/e17_impact_score.py new file mode 100644 index 00000000..e5e93bd6 --- /dev/null +++ b/experiments/bootstrap/e17_impact_score.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +"""E17 impact scoring: recall/precision of the regression-test set per mode. + +Ground truth = the function's covering tests surfaced by the signal (B tests). +Predicted = test names the answer mentions (regex). No LLM judge — objective. +""" +import sys +import re +import json +import math +from pathlib import Path +from collections import defaultdict + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +MODES = ("A_baseline", "B_runtime", "D_shuffled") +TEST_RE = re.compile(r"\btest_[A-Za-z0-9_]+") + + +def short(t: str) -> str: + return t.split("::")[-1] + + +def paired(diffs): + n = len(diffs) + mean = sum(diffs) / n + var = sum((d - mean) ** 2 for d in diffs) / (n - 1) if n > 1 else 0.0 + sd = math.sqrt(var) + if sd == 0: + return mean, 0.0, float("nan"), float("inf") + t = mean / (sd / math.sqrt(n)) + try: + from scipy import stats # type: ignore + p = 2 * stats.t.sf(abs(t), df=n - 1) + except Exception: + p = float("nan") + return mean, mean / sd, p, t + + +def main() -> None: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + answers = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_answers.json").read_text(encoding="utf-8") + ) + by_id = {a["id"]: a for a in answers["answers"]} + + per_mode_recall = defaultdict(list) + per_mode_prec = defaultdict(list) + per_mode_any = defaultdict(list) + guessed = defaultdict(int) + + for f in data["functions"]: + gt = {short(t) for t in f["modes"]["B_runtime"]["tests"]} + for mode in MODES: + pred = set(TEST_RE.findall(by_id[f["id"]]["modes"][mode]["answer"])) + hit = pred & gt + per_mode_recall[mode].append(len(hit) / len(gt) if gt else 0.0) + per_mode_prec[mode].append(len(hit) / len(pred) if pred else 0.0) + per_mode_any[mode].append(1.0 if hit else 0.0) + if mode == "A_baseline" and hit: + guessed[f["id"]] += 1 + + print("per-mode means (n=30):") + for label, store in (("recall", per_mode_recall), ("precision", per_mode_prec), + ("any-hit", per_mode_any)): + a = sum(store["A_baseline"]) / 30 + b = sum(store["B_runtime"]) / 30 + d = sum(store["D_shuffled"]) / 30 + print(f" {label:10} A={a:.3f} B={b:.3f} D={d:.3f}") + + for metric, store in (("recall", per_mode_recall), ("precision", per_mode_prec)): + for hi, lo in (("B_runtime", "A_baseline"), ("B_runtime", "D_shuffled")): + diffs = [store[hi][i] - store[lo][i] for i in range(len(store[hi]))] + mean, d, p, t = paired(diffs) + wins = sum(1 for x in diffs if x > 0) + losses = sum(1 for x in diffs if x < 0) + print(f"\n{metric}: {hi} - {lo}") + print(f" mean_diff={mean:+.3f} d={d:+.3f} t={t:+.3f} p={p:.4f} " + f"W/L/T={wins}/{losses}/{len(diffs) - wins - losses}") + + print(f"\nname-convention leak: A guessed a real covering test for " + f"{len(guessed)}/30 functions") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_judge/mapping.json b/experiments/bootstrap/e17_judge/mapping.json new file mode 100644 index 00000000..49311960 --- /dev/null +++ b/experiments/bootstrap/e17_judge/mapping.json @@ -0,0 +1,452 @@ +{ + "T01": { + "id": 26, + "mode": "D_shuffled", + "name": "_match_gitignore_pattern" + }, + "T02": { + "id": 1, + "mode": "A_baseline", + "name": "IndexProjectRunner.run" + }, + "T03": { + "id": 15, + "mode": "D_shuffled", + "name": "_extract_yarn_lock" + }, + "T04": { + "id": 3, + "mode": "A_baseline", + "name": "CypherLexer.tokenize" + }, + "T05": { + "id": 11, + "mode": "D_shuffled", + "name": "decorator" + }, + "T06": { + "id": 7, + "mode": "B_runtime", + "name": "resolve_project_root" + }, + "T07": { + "id": 6, + "mode": "A_baseline", + "name": "extract_anchors" + }, + "T08": { + "id": 11, + "mode": "B_runtime", + "name": "decorator" + }, + "T09": { + "id": 8, + "mode": "A_baseline", + "name": "SymbolIndex.build_call_graph" + }, + "T10": { + "id": 27, + "mode": "D_shuffled", + "name": "_Fingerprint.__init__" + }, + "T11": { + "id": 10, + "mode": "A_baseline", + "name": "modification_guard" + }, + "T12": { + "id": 14, + "mode": "D_shuffled", + "name": "validate_code" + }, + "T13": { + "id": 2, + "mode": "B_runtime", + "name": "RemoteEmbedder.embed_batch" + }, + "T14": { + "id": 12, + "mode": "D_shuffled", + "name": "wrapper" + }, + "T15": { + "id": 21, + "mode": "D_shuffled", + "name": "_delayed_auto_index" + }, + "T16": { + "id": 5, + "mode": "D_shuffled", + "name": "IndexPipeline.process_file" + }, + "T17": { + "id": 8, + "mode": "B_runtime", + "name": "SymbolIndex.build_call_graph" + }, + "T18": { + "id": 6, + "mode": "D_shuffled", + "name": "extract_anchors" + }, + "T19": { + "id": 4, + "mode": "A_baseline", + "name": "AutoDocUpdater._extract_doc_references" + }, + "T20": { + "id": 28, + "mode": "B_runtime", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + "T21": { + "id": 10, + "mode": "D_shuffled", + "name": "modification_guard" + }, + "T22": { + "id": 13, + "mode": "D_shuffled", + "name": "format_project_memory" + }, + "T23": { + "id": 27, + "mode": "B_runtime", + "name": "_Fingerprint.__init__" + }, + "T24": { + "id": 9, + "mode": "B_runtime", + "name": "IndexParser.parse_file" + }, + "T25": { + "id": 24, + "mode": "B_runtime", + "name": "IndexStatusReporter.get_status" + }, + "T26": { + "id": 3, + "mode": "D_shuffled", + "name": "CypherLexer.tokenize" + }, + "T27": { + "id": 18, + "mode": "A_baseline", + "name": "load_manifest" + }, + "T28": { + "id": 26, + "mode": "B_runtime", + "name": "_match_gitignore_pattern" + }, + "T29": { + "id": 23, + "mode": "A_baseline", + "name": "_extract_pyproject" + }, + "T30": { + "id": 25, + "mode": "B_runtime", + "name": "WriteTool._action_insert" + }, + "T31": { + "id": 21, + "mode": "B_runtime", + "name": "_delayed_auto_index" + }, + "T32": { + "id": 22, + "mode": "B_runtime", + "name": "load_metrics" + }, + "T33": { + "id": 5, + "mode": "A_baseline", + "name": "IndexPipeline.process_file" + }, + "T34": { + "id": 12, + "mode": "A_baseline", + "name": "wrapper" + }, + "T35": { + "id": 16, + "mode": "B_runtime", + "name": "WriteTool._action_replace" + }, + "T36": { + "id": 27, + "mode": "A_baseline", + "name": "_Fingerprint.__init__" + }, + "T37": { + "id": 14, + "mode": "B_runtime", + "name": "validate_code" + }, + "T38": { + "id": 29, + "mode": "A_baseline", + "name": "IndexGuard.save_symbol_index" + }, + "T39": { + "id": 24, + "mode": "D_shuffled", + "name": "IndexStatusReporter.get_status" + }, + "T40": { + "id": 29, + "mode": "D_shuffled", + "name": "IndexGuard.save_symbol_index" + }, + "T41": { + "id": 21, + "mode": "A_baseline", + "name": "_delayed_auto_index" + }, + "T42": { + "id": 15, + "mode": "B_runtime", + "name": "_extract_yarn_lock" + }, + "T43": { + "id": 20, + "mode": "A_baseline", + "name": "format_runtime_status" + }, + "T44": { + "id": 11, + "mode": "A_baseline", + "name": "decorator" + }, + "T45": { + "id": 2, + "mode": "D_shuffled", + "name": "RemoteEmbedder.embed_batch" + }, + "T46": { + "id": 17, + "mode": "B_runtime", + "name": "SymbolIndex.get_call_chain" + }, + "T47": { + "id": 7, + "mode": "D_shuffled", + "name": "resolve_project_root" + }, + "T48": { + "id": 25, + "mode": "D_shuffled", + "name": "WriteTool._action_insert" + }, + "T49": { + "id": 7, + "mode": "A_baseline", + "name": "resolve_project_root" + }, + "T50": { + "id": 18, + "mode": "B_runtime", + "name": "load_manifest" + }, + "T51": { + "id": 24, + "mode": "A_baseline", + "name": "IndexStatusReporter.get_status" + }, + "T52": { + "id": 28, + "mode": "D_shuffled", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + "T53": { + "id": 13, + "mode": "A_baseline", + "name": "format_project_memory" + }, + "T54": { + "id": 20, + "mode": "D_shuffled", + "name": "format_runtime_status" + }, + "T55": { + "id": 16, + "mode": "D_shuffled", + "name": "WriteTool._action_replace" + }, + "T56": { + "id": 25, + "mode": "A_baseline", + "name": "WriteTool._action_insert" + }, + "T57": { + "id": 29, + "mode": "B_runtime", + "name": "IndexGuard.save_symbol_index" + }, + "T58": { + "id": 4, + "mode": "D_shuffled", + "name": "AutoDocUpdater._extract_doc_references" + }, + "T59": { + "id": 15, + "mode": "A_baseline", + "name": "_extract_yarn_lock" + }, + "T60": { + "id": 8, + "mode": "D_shuffled", + "name": "SymbolIndex.build_call_graph" + }, + "T61": { + "id": 30, + "mode": "D_shuffled", + "name": "RemoteEmbedder._shadow_compare" + }, + "T62": { + "id": 23, + "mode": "D_shuffled", + "name": "_extract_pyproject" + }, + "T63": { + "id": 17, + "mode": "D_shuffled", + "name": "SymbolIndex.get_call_chain" + }, + "T64": { + "id": 9, + "mode": "D_shuffled", + "name": "IndexParser.parse_file" + }, + "T65": { + "id": 3, + "mode": "B_runtime", + "name": "CypherLexer.tokenize" + }, + "T66": { + "id": 28, + "mode": "A_baseline", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + "T67": { + "id": 19, + "mode": "B_runtime", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + "T68": { + "id": 22, + "mode": "D_shuffled", + "name": "load_metrics" + }, + "T69": { + "id": 30, + "mode": "A_baseline", + "name": "RemoteEmbedder._shadow_compare" + }, + "T70": { + "id": 18, + "mode": "D_shuffled", + "name": "load_manifest" + }, + "T71": { + "id": 22, + "mode": "A_baseline", + "name": "load_metrics" + }, + "T72": { + "id": 30, + "mode": "B_runtime", + "name": "RemoteEmbedder._shadow_compare" + }, + "T73": { + "id": 5, + "mode": "B_runtime", + "name": "IndexPipeline.process_file" + }, + "T74": { + "id": 2, + "mode": "A_baseline", + "name": "RemoteEmbedder.embed_batch" + }, + "T75": { + "id": 20, + "mode": "B_runtime", + "name": "format_runtime_status" + }, + "T76": { + "id": 10, + "mode": "B_runtime", + "name": "modification_guard" + }, + "T77": { + "id": 1, + "mode": "B_runtime", + "name": "IndexProjectRunner.run" + }, + "T78": { + "id": 19, + "mode": "D_shuffled", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + "T79": { + "id": 12, + "mode": "B_runtime", + "name": "wrapper" + }, + "T80": { + "id": 9, + "mode": "A_baseline", + "name": "IndexParser.parse_file" + }, + "T81": { + "id": 6, + "mode": "B_runtime", + "name": "extract_anchors" + }, + "T82": { + "id": 16, + "mode": "A_baseline", + "name": "WriteTool._action_replace" + }, + "T83": { + "id": 23, + "mode": "B_runtime", + "name": "_extract_pyproject" + }, + "T84": { + "id": 17, + "mode": "A_baseline", + "name": "SymbolIndex.get_call_chain" + }, + "T85": { + "id": 19, + "mode": "A_baseline", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + "T86": { + "id": 14, + "mode": "A_baseline", + "name": "validate_code" + }, + "T87": { + "id": 26, + "mode": "A_baseline", + "name": "_match_gitignore_pattern" + }, + "T88": { + "id": 1, + "mode": "D_shuffled", + "name": "IndexProjectRunner.run" + }, + "T89": { + "id": 4, + "mode": "B_runtime", + "name": "AutoDocUpdater._extract_doc_references" + }, + "T90": { + "id": 13, + "mode": "B_runtime", + "name": "format_project_memory" + } +} \ No newline at end of file diff --git a/experiments/bootstrap/e17_judge/shard_1.scores.json b/experiments/bootstrap/e17_judge/shard_1.scores.json new file mode 100644 index 00000000..2f8db981 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_1.scores.json @@ -0,0 +1,10 @@ +[ + {"token": "T01", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T02", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T03", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T04", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T05", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T06", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T07", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T08", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_10.scores.json b/experiments/bootstrap/e17_judge/shard_10.scores.json new file mode 100644 index 00000000..0bb8de40 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_10.scores.json @@ -0,0 +1,9 @@ +[ + {"token": "T84", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T85", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T86", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T87", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T88", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T89", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T90", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_2.scores.json b/experiments/bootstrap/e17_judge/shard_2.scores.json new file mode 100644 index 00000000..25005d62 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_2.scores.json @@ -0,0 +1,10 @@ +[ + {"token": "T09", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T10", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T11", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T12", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T13", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T14", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T15", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T16", "accuracy": 5, "completeness": 4, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_3.scores.json b/experiments/bootstrap/e17_judge/shard_3.scores.json new file mode 100644 index 00000000..8cd480e8 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_3.scores.json @@ -0,0 +1,11 @@ +[ + {"token": "T17", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T18", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T19", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T20", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T21", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T22", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T23", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T24", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T25", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_4.scores.json b/experiments/bootstrap/e17_judge/shard_4.scores.json new file mode 100644 index 00000000..8e17e0ba --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_4.scores.json @@ -0,0 +1,13 @@ +[ + {"token": "T26", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T27", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T28", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T29", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T30", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T31", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T32", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T33", "accuracy": 4, "completeness": 5, "safety": 5}, + {"token": "T34", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T35", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T36", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_5.scores.json b/experiments/bootstrap/e17_judge/shard_5.scores.json new file mode 100644 index 00000000..f75186da --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_5.scores.json @@ -0,0 +1,11 @@ +[ + {"token": "T37", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T38", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T39", "accuracy": 4, "completeness": 5, "safety": 5}, + {"token": "T40", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T41", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T42", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T43", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T44", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T45", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_6.scores.json b/experiments/bootstrap/e17_judge/shard_6.scores.json new file mode 100644 index 00000000..ef00e148 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_6.scores.json @@ -0,0 +1,12 @@ +[ + {"token": "T46", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T47", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T48", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T49", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T50", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T51", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T52", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T53", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T54", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T55", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_7.scores.json b/experiments/bootstrap/e17_judge/shard_7.scores.json new file mode 100644 index 00000000..e50fcd7c --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_7.scores.json @@ -0,0 +1,12 @@ +[ + {"token": "T56", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T57", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T58", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T59", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T60", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T61", "accuracy": 3, "completeness": 4, "safety": 4}, + {"token": "T62", "accuracy": 4, "completeness": 4, "safety": 4}, + {"token": "T63", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T64", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T65", "accuracy": 5, "completeness": 5, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_8.scores.json b/experiments/bootstrap/e17_judge/shard_8.scores.json new file mode 100644 index 00000000..8018cd25 --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_8.scores.json @@ -0,0 +1,12 @@ +[ + {"token": "T66", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T67", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T68", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T69", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T70", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T71", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T72", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T73", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T74", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T75", "accuracy": 4, "completeness": 4, "safety": 5} +] diff --git a/experiments/bootstrap/e17_judge/shard_9.scores.json b/experiments/bootstrap/e17_judge/shard_9.scores.json new file mode 100644 index 00000000..965e5caa --- /dev/null +++ b/experiments/bootstrap/e17_judge/shard_9.scores.json @@ -0,0 +1,10 @@ +[ + {"token": "T76", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T77", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T78", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T79", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T80", "accuracy": 5, "completeness": 5, "safety": 5}, + {"token": "T81", "accuracy": 4, "completeness": 4, "safety": 5}, + {"token": "T82", "accuracy": 5, "completeness": 4, "safety": 5}, + {"token": "T83", "accuracy": 4, "completeness": 5, "safety": 4} +] diff --git a/experiments/bootstrap/e17_merge_answers.py b/experiments/bootstrap/e17_merge_answers.py new file mode 100644 index 00000000..937a901b --- /dev/null +++ b/experiments/bootstrap/e17_merge_answers.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""E17 merge: validate shard answer files and rebuild e17_pilot_answers.json. + +Expected (id, mode) pairs come from e17_pilot_data.json. Every pair must be +present exactly once; missing/extra/duplicate entries fail loudly. +""" +import sys +import json +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +SHARDS = ROOT / "experiments/bootstrap/e17_shards" + + +def main() -> int: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + expected = {(f["id"], m) for f in data["functions"] for m in f["modes"]} + + found: dict = {} + dupes = [] + for path in sorted(SHARDS.glob("shard_*.answers.json")): + items = json.loads(path.read_text(encoding="utf-8")) + for it in items: + key = (it["id"], it["mode"]) + if key in found: + dupes.append((path.name, key)) + found[key] = it["answer"] + print(f"{path.name}: {len(items)} items") + + missing = sorted(expected - set(found)) + extra = sorted(set(found) - expected) + print(f"\nexpected={len(expected)} found={len(found)} duplicates={len(dupes)}") + print(f"missing={missing}") + print(f"extra={extra}") + if missing or extra or dupes: + print("MERGE: FAILED") + return 1 + + answers = [] + for f in data["functions"]: + modes = {} + for mode, mdata in f["modes"].items(): + modes[mode] = { + "answer": found[(f["id"], mode)], + "tests": mdata["tests"], + } + answers.append({ + "id": f["id"], + "name": f["name"], + "file_path": f["file_path"], + "question": f["question"], + "modes": modes, + }) + + out = {"metadata": data["metadata"], "answers": answers} + (ROOT / "experiments/bootstrap/e17_pilot_answers.json").write_text( + json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"\nwrote e17_pilot_answers.json: {len(answers)} functions, " + f"{sum(len(a['modes']) for a in answers)} answers") + print("MERGE: PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/bootstrap/e17_mutation_results.json b/experiments/bootstrap/e17_mutation_results.json new file mode 100644 index 00000000..7f9cedd3 --- /dev/null +++ b/experiments/bootstrap/e17_mutation_results.json @@ -0,0 +1,162 @@ +[ + { + "name": "IndexProjectRunner.run", + "mutation": "negate-if @L270", + "B": { + "failed": 3, + "rc": 1, + "verdict": "============================== 3 failed in 0.16s ==============================", + "err": true + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 0.07s ==============================", + "err": false + } + }, + { + "name": "RemoteEmbedder.embed_batch", + "mutation": "negate-if @L928", + "B": { + "failed": 2, + "rc": 1, + "verdict": "============================== 2 failed in 1.45s ==============================", + "err": false + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 2 passed in 0.06s ==============================", + "err": false + } + }, + { + "name": "CypherLexer.tokenize", + "mutation": "negate-if @L64", + "B": { + "failed": 3, + "rc": 1, + "verdict": "============================== 3 failed in 1.47s ==============================", + "err": true + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 0.13s ==============================", + "err": false + } + }, + { + "name": "AutoDocUpdater._extract_doc_references", + "mutation": "negate-if @L572", + "B": { + "failed": 0, + "rc": 0, + "verdict": "============================== 1 passed in 0.07s ==============================", + "err": false + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 1 passed in 0.08s ==============================", + "err": false + } + }, + { + "name": "IndexPipeline.process_file", + "mutation": "negate-if @L77", + "B": { + "failed": 3, + "rc": 1, + "verdict": "============================== 3 failed in 0.71s ==============================", + "err": true + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 1.40s ==============================", + "err": false + } + }, + { + "name": "extract_anchors", + "mutation": "negate-if @L345", + "B": { + "failed": 1, + "rc": 1, + "verdict": "============================== 1 failed in 1.49s ==============================", + "err": true + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 1.51s ==============================", + "err": false + } + }, + { + "name": "resolve_project_root", + "mutation": "negate-if @L296", + "B": { + "failed": 2, + "rc": 1, + "verdict": "========================= 2 failed, 1 passed in 0.14s =========================", + "err": true + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 1.36s ==============================", + "err": false + } + }, + { + "name": "SymbolIndex.build_call_graph", + "mutation": "negate-if @L575", + "B": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 0.10s ==============================", + "err": false + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 1.86s ==============================", + "err": false + } + }, + { + "name": "IndexParser.parse_file", + "mutation": "negate-if @L111", + "B": { + "failed": 0, + "rc": 0, + "verdict": "============================== 2 passed in 1.95s ==============================", + "err": false + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 2 passed in 1.42s ==============================", + "err": false + } + }, + { + "name": "modification_guard", + "mutation": "negate-if @L227", + "B": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 0.09s ==============================", + "err": false + }, + "D": { + "failed": 0, + "rc": 0, + "verdict": "============================== 3 passed in 2.82s ==============================", + "err": false + } + } +] \ No newline at end of file diff --git a/experiments/bootstrap/e17_mutation_validate.py b/experiments/bootstrap/e17_mutation_validate.py new file mode 100644 index 00000000..30f32caa --- /dev/null +++ b/experiments/bootstrap/e17_mutation_validate.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +"""E17 mutation validation: does the TESTS signal identify *verifying* tests? + +Hypothesis: for a function F with a strong behaviour mutation, the tests the +graph links to F (arm B) FAIL (kill the mutant), while coverage-matched decoy +tests (arm D) PASS. If so, the TESTS edges can drive verification, not just +description. + +Method: mutate the first single-line `if` condition inside F (negate it), run +the exact pytest node ids for arm B and arm D, then restore the file with +`git checkout --` in a finally block. No LLM. +""" +import ast +import json +import re +import subprocess +import sys +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +BOOT = ROOT / "experiments/bootstrap" +K = int(sys.argv[1]) if len(sys.argv) > 1 else 10 + + +def _sh(cmd, timeout=300): + return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + + +def graph_test_files(): + """test name -> absolute file path from the PropertyGraph.""" + import sqlite3 + from src.core.artifact_paths import get_graph_db_path + conn = sqlite3.connect(str(get_graph_db_path(ROOT))) + rows = conn.execute("SELECT name, file_path FROM nodes WHERE label='Test'").fetchall() + conn.close() + out = {} + for name, fp in rows: + out.setdefault(name, fp) + return out + + +def node_id(rel_file, test_name): + cls, _, meth = test_name.partition("::") + return f"{rel_file}::{cls}::{meth}" if meth else f"{rel_file}::{test_name}" + + +def find_func(tree, qualname): + parts = qualname.split(".") + if len(parts) == 1: + for n in ast.walk(tree): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == parts[0]: + return n + return None + for n in ast.walk(tree): + if isinstance(n, ast.ClassDef) and n.name == parts[-2]: + for item in n.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) and item.name == parts[-1]: + return item + return None + + +def mutate(src, qualname): + """Strong mutation: negate the first single-line `if` condition. Else None.""" + try: + tree = ast.parse(src) + except SyntaxError: + return None, None + node = find_func(tree, qualname) + if node is None: + return None, None + lines = src.split("\n") + for sub in ast.walk(node): + if isinstance(sub, ast.If) and sub.test.lineno == sub.test.end_lineno: + ln = lines[sub.test.lineno - 1] + seg = ln[sub.test.col_offset:sub.test.end_col_offset] + if seg.strip(): + lines[sub.test.lineno - 1] = ( + ln[:sub.test.col_offset] + f"not ({seg})" + ln[sub.test.end_col_offset:] + ) + return "\n".join(lines), f"negate-if @L{sub.test.lineno}" + return None, None + + +def run_tests(ids): + if not ids: + return None + p = _sh([sys.executable, "-m", "pytest", *ids, "-q", "--no-header", + "-p", "no:cacheprovider"]) + m = re.search(r"(\d+) failed", p.stdout) + return {"failed": int(m.group(1)) if m else 0, "rc": p.returncode} + + +def main(): + data = json.loads((BOOT / "e17_pilot_data.json").read_text(encoding="utf-8")) + tfiles = graph_test_files() + results = [] + for f in data["functions"]: + if len(results) >= K: + break + rel = f["file_path"].replace("D:/Project/MSCodeBase/", "") + src_path = ROOT / rel + if not src_path.exists(): + continue + mutated, desc = mutate(src_path.read_text(encoding="utf-8"), f["name"]) + if not mutated: + continue + + def _rel(t): + return Path(tfiles[t]).resolve().relative_to(ROOT).as_posix() + + b_ids = [node_id(_rel(t), t) for t in f["modes"]["B_runtime"]["tests"] if t in tfiles] + d_ids = [node_id(_rel(t), t) for t in f["modes"]["D_shuffled"]["tests"] if t in tfiles] + if not b_ids or not d_ids: + continue + print(f"\n[{f['name']}] mutation {desc}") + try: + src_path.write_text(mutated, encoding="utf-8") + b, d = run_tests(b_ids), run_tests(d_ids) + finally: + _sh(["git", "checkout", "--", rel]) + results.append({"name": f["name"], "mutation": desc, "B": b, "D": d}) + print(f" B: failed={b['failed']} rc={b['rc']}") + print(f" D: failed={d['failed']} rc={d['rc']}") + + (BOOT / "e17_mutation_results.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") + n = len(results) + b_kill = sum(1 for r in results if r["B"] and r["B"]["rc"] != 0) + d_kill = sum(1 for r in results if r["D"] and r["D"]["rc"] != 0) + print("\n" + "=" * 60) + print(f"functions tested: {n}") + print(f"B (graph tests) killed the mutant: {b_kill}/{n}") + print(f"D (decoy tests) killed the mutant: {d_kill}/{n}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_pilot_answers.json b/experiments/bootstrap/e17_pilot_answers.json new file mode 100644 index 00000000..e9860d42 --- /dev/null +++ b/experiments/bootstrap/e17_pilot_answers.json @@ -0,0 +1,836 @@ +{ + "metadata": { + "n_functions": 30, + "n_total_tests": 1461, + "selection": "min_test_coverage<=5, max 3 per file", + "modes": [ + "A_baseline", + "B_runtime", + "D_shuffled" + ] + }, + "answers": [ + { + "id": 1, + "name": "IndexProjectRunner.run", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_project_runner.py", + "question": "A developer is about to change the behavior of the function `IndexProjectRunner.run` in `index_project_runner.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "From the information given I cannot determine which tests exercise IndexProjectRunner.run. Only the function source is provided; no repository layout, test files, or call sites are included, so no test function names can be identified. The function body references no tests. An LLM given only this text must not invent names. To find them one would search the test suite for imports of IndexProjectRunner or calls to .run, and for coverage of os.walk scanning, ThreadPoolExecutor workers, embed_batch, path_manager.is_safe_to_process, incremental resume writes, prune, BM25 reindex, cache invalidation and IVF indexing. Without that search the honest answer is: undetermined. I explicitly cannot list the required test function names.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests provided that exercise IndexProjectRunner.run are exactly these three:\ntest_run_survives_single_cpu_host\ntest_run_with_existing_known_hashes_skips_unchanged\ntest_workers_receive_known_hashes_not_none\n\nThey cover the 1-CPU worker-count edge case, the skip-unchanged path through known_hashes, and the guarantee that workers receive a dict so they never enter the locked table.search branch. These must be run for regressions when changing run. In summary, run performs a full project index: path-safety check, file scan with a guard, parallel parse, batched embedding, incremental resume-friendly writes, prune, BM25 reindex, cache invalidation and optional IVF indexing, returning the count of indexed files.", + "tests": [ + "test_run_survives_single_cpu_host", + "test_run_with_existing_known_hashes_skips_unchanged", + "test_workers_receive_known_hashes_not_none" + ] + }, + "D_shuffled": { + "answer": "The three provided tests do NOT exercise IndexProjectRunner.run. test_pid_reuse_is_stale and test_dead_pid_is_stale check DatabaseLock holder classification, and test_logs_dir_inside_data_root checks get_logs_dir; none call run, its parse workers, embedder, db_writer, searcher or table. Therefore I cannot determine from the information given which tests exercise this function. The listed names are unrelated to indexing and using them would give false confidence. To catch regressions one would search the repository for tests importing IndexProjectRunner or calling .run, covering the safety check, worker pool, known_hashes skip path, incremental writes, prune and reindex. From the supplied text, the honest answer is: undetermined.", + "tests": [ + "TestClassifyHolder::test_pid_reuse_is_stale", + "TestClassifyHolder::test_dead_pid_is_stale", + "test_logs_dir_inside_data_root" + ] + } + } + }, + { + "id": 2, + "name": "RemoteEmbedder.embed_batch", + "file_path": "D:/Project/MSCodeBase/src/providers/embedder/remote_embedder.py", + "question": "A developer is about to change the behavior of the function `RemoteEmbedder.embed_batch` in `remote_embedder.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given. The context supplies only the source of RemoteEmbedder.embed_batch, not the repository test suite, so any list of test function names would be a guess and could mislead the developer. To find the right tests, one should search the repository for callers of embed_batch and for RemoteEmbedder usage. Regarding the function itself: it fetches embedding vectors in batch from whichever provider is currently active and deliberately never changes the mode. Its inputs are a list of texts plus an is_query flag; its output is a list of float vectors. An empty input returns an empty list. The llama.cpp path truncates to 512 tokens, retries with backoff, then retries each item individually, raising RuntimeError if any chunk still fails. The LM Studio, ONNX server, OpenVINO and local ONNX paths raise RuntimeError when unavailable.", + "tests": [] + }, + "B_runtime": { + "answer": "Based on the tests supplied with this prompt, the tests that exercise embed_batch are: test_rapid_sequential_calls_no_state_leak and test_single_call_results_match_indices. The first drives one hundred rapid sequential embed_batch calls and checks that no state leaks between calls and that vectors change with different texts. The second checks a single batch returns three non-zero vectors of length 384 with correct indices. The function itself returns embedding vectors from the active provider without ever mutating mode; it handles llama.cpp, LM Studio, ONNX server, OpenVINO and local ONNX paths with retries and RuntimeError fallbacks. If other tests exist elsewhere in the repository, they are not shown here, so this list may be incomplete.", + "tests": [ + "TestConcurrentEmbedIsolation::test_rapid_sequential_calls_no_state_leak", + "TestConcurrentEmbedIsolation::test_single_call_results_match_indices" + ] + }, + "D_shuffled": { + "answer": "I cannot determine this from the information given. The two tests shown, test_sample_disk_io_no_data_row_returns_zero and test_verdict_fail_is_refuted, exercise a ResourceMonitor disk-sampling helper and a verdict_from_results function respectively, not RemoteEmbedder.embed_batch. Nothing in the provided text links them to embedding, so listing them would be wrong. No test function names can be listed with confidence here. The function itself returns a list of embedding vectors from the active provider such as llama.cpp, LM Studio, ONNX server, OpenVINO or local ONNX, never changing mode, returning an empty list for empty input, and raising RuntimeError when every provider fails. To find the real tests one must search the repository for callers of embed_batch.", + "tests": [ + "TestResourceMonitor::test_sample_disk_io_no_data_row_returns_zero", + "test_verdict_fail_is_refuted" + ] + } + } + }, + { + "id": 3, + "name": "CypherLexer.tokenize", + "file_path": "D:/Project/MSCodeBase/src/core/search/cypher_lexer.py", + "question": "A developer is about to change the behavior of the function `CypherLexer.tokenize` in `cypher_lexer.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given. The prompt includes only the source of CypherLexer.tokenize and no test file contents or test names, so listing tests would be pure speculation. The function scans a Cypher query string character by character and returns a list of Token objects. It skips whitespace, strips double-slash comments, parses quoted strings while raising SyntaxError on unterminated ones, parses numbers, identifiers, keywords, uppercase labels, backtick-escaped identifiers, two-character comparison operators, direction arrows, single-character operators, punctuation, and colon-prefixed labels including variable-length :TYPE*min..max forms. An unknown character raises SyntaxError. The token list is cached in self._tokens. To find exercising tests, search the repository for CypherLexer usage and tokenize calls.", + "tests": [] + }, + "B_runtime": { + "answer": "Based on the tests supplied with this prompt, the tests exercising CypherLexer.tokenize are: test_tokenize_optional_match, test_tokenize_order_limit, and test_tokenize_relationship. They tokenize sample Cypher queries and assert recognition of KEYWORD values such as OPTIONAL, MATCH, ORDER and LIMIT, plus the label CALLS and the arrow punctuation ->. The function itself performs a single-pass lexical analysis of a Cypher query, returning Token objects for keywords, identifiers, labels, strings, numbers, operators, punctuation and relationship types, while skipping comments and whitespace and raising SyntaxError for unterminated strings, unterminated backticks or unexpected characters. The tokens are stored in self._tokens. Other repository tests may exist but are not shown, so this list may be incomplete.", + "tests": [ + "TestCypherLexer::test_tokenize_optional_match", + "TestCypherLexer::test_tokenize_order_limit", + "TestCypherLexer::test_tokenize_relationship" + ] + }, + "D_shuffled": { + "answer": "I cannot determine this from the information given. The three tests shown, test_gemfile_lock_skips_path_project_gem, test_blocked_name_getattr and test_load_invalid_engine_req, exercise a Gemfile.lock manifest extractor, a sandbox code validator and a plugin manifest loader respectively. None of them references CypherLexer or calls tokenize, so they cannot catch regressions in that function and should not be listed. Separately, CypherLexer.tokenize scans a Cypher query string and returns Token objects, skipping comments and whitespace while raising SyntaxError on unterminated strings, backticks or unexpected characters, and recognising keywords, labels, numbers, operators, punctuation and variable-length :TYPE*min..max relation types. To identify the true tests, one must search the repository for CypherLexer usage.", + "tests": [ + "test_gemfile_lock_skips_path_project_gem", + "TestValidateCode::test_blocked_name_getattr", + "test_load_invalid_engine_req" + ] + } + } + }, + { + "id": 4, + "name": "AutoDocUpdater._extract_doc_references", + "file_path": "D:/Project/MSCodeBase/src/core/auto_doc_updater.py", + "question": "A developer is about to change the behavior of the function `AutoDocUpdater._extract_doc_references` in `auto_doc_updater.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "From the information given I cannot determine which tests exercise `AutoDocUpdater._extract_doc_references`: no test files, test names or repository structure were provided, only the function source. The function parses Markdown text line by line, skips fenced code blocks, and extracts backtick-wrapped tokens as likely code identifiers. It filters out URLs, file paths, Windows drive paths, IP addresses, file extensions, config assignments, hotkeys, version-like strings and stop words, then strips call arguments and `$` env prefixes. It returns a list of `(content, line_number, full_match)` tuples. To identify regression tests you would need the repository test suite, which is absent here.", + "tests": [] + }, + "B_runtime": { + "answer": "One provided test exercises this function: test_extract_doc_references_handles_empty_call_syntax. It builds a temporary tree, writes a doc containing empty-call references `()` and `(x)` plus a normal `fastmcp`, then calls `_verify_doc_references`, which internally uses `_extract_doc_references`. The test asserts no IndexError is raised and that a list of broken references is returned. The function itself scans Markdown lines, skips fenced code blocks, extracts backtick tokens, filters non-code patterns (URLs, paths, IPs, extensions, assignments, semver) and returns `(content, line, match)` tuples. Any behavior change should re-run that test, though the provided list may be incomplete.", + "tests": [ + "test_extract_doc_references_handles_empty_call_syntax" + ] + }, + "D_shuffled": { + "answer": "The information given lists test_symbol_index_stores_and_retrieves as the relevant test for this function. Taken at face value, that is the test to run: it exercises SymbolIndex round-trip behavior for env accesses. Separately, `_extract_doc_references` scans Markdown text, skipping fenced code blocks, extracts backtick-wrapped tokens and filters out URLs, paths, IPs, file extensions, config assignments, hotkeys and semver strings before returning `(content, line_number, full_match)` tuples. Note the listed test appears unrelated to reference extraction, so on the evidence provided it may not catch regressions; when in doubt, inspect the actual test suite rather than trusting the label.", + "tests": [ + "test_symbol_index_stores_and_retrieves" + ] + } + } + }, + { + "id": 5, + "name": "IndexPipeline.process_file", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_pipeline.py", + "question": "A developer is about to change the behavior of the function `IndexPipeline.process_file` in `index_pipeline.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "From the information given I cannot determine which tests exercise `IndexPipeline.process_file`: no test files, names or repository structure were provided, only the function source. The function handles one file through the indexing pipeline. It rejects blank content, parses via `_index_parser.parse_file`, records mtime and size, updates SymbolIndex collections (definitions, references, assignments, imports, decorators, overrides, env accesses) under a lock, computes per-chunk SHA-256 hashes for a content-addressed cache, reuses cached vectors from LanceDB, embeds only new chunks, and returns a result dict. It returns None on empty input, parse failure, no chunks, empty embeddings or cache gaps. Regression tests cannot be identified without the test suite.", + "tests": [] + }, + "B_runtime": { + "answer": "Three provided tests exercise this function. test_chunk_cache_disabled_without_table: with table=None, two process_file calls both invoke embed_batch, asserting call_count == 2. test_chunk_cache_empty_file: whitespace-only content returns None without calling the embedder. test_chunk_cache_all_new: a new file with foreign cached hashes yields one embed_batch call and five embeddings, verifying deterministic per-chunk vectors. The function itself rejects blank content, parses the file, records stat metadata, updates SymbolIndex collections under a lock, computes chunk hashes for a content-addressed cache, reuses cached vectors from LanceDB, embeds only new chunks, and returns a result dict or None on empty input, parse failure, empty embeddings or cache gaps. Re-run all three when changing behavior.", + "tests": [ + "test_chunk_cache_disabled_without_table", + "test_chunk_cache_empty_file", + "test_chunk_cache_all_new" + ] + }, + "D_shuffled": { + "answer": "The information given lists three relevant tests: test_ensure_server_running_launches_when_dead, test_impact_analysis_basic and test_empty_file. Taken at face value these are the tests to run. Separately, `process_file` rejects blank content, parses via `_index_parser.parse_file`, records mtime and size, updates SymbolIndex collections under a lock, computes per-chunk SHA-256 hashes, reuses cached vectors from LanceDB, embeds only new chunks, and returns a dict or None on empty input, parse failure, no chunks, empty embeddings or cache gaps. Note the listed tests look unrelated to file processing, so on the evidence provided they may not catch regressions; the actual test suite should be inspected before relying on them.", + "tests": [ + "test_ensure_server_running_launches_when_dead", + "test_impact_analysis_basic", + "TestEdgeCases::test_empty_file" + ] + } + } + }, + { + "id": 6, + "name": "extract_anchors", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/verify_on_read.py", + "question": "A developer is about to change the behavior of the function `extract_anchors` in `verify_on_read.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "From the information given I cannot determine which tests exercise `extract_anchors`: no test files or test names were provided, only the function source. The function extracts checkable anchors from a memory node's data/claim text using regex, without an LLM. It honors explicit `data.anchors` first; on the read path it skips prose scanning when explicit anchors exist. Otherwise it parses `file:path`, `import X` / `from X import y`, `env:KEY` / `$KEY`, `pkg:name` and path-like tokens. On the write path it filters file anchors by existence and captures manifest package words fail-closed. It deduplicates via a seen set and returns Anchor objects. Regression tests cannot be identified without the repository test suite.", + "tests": [] + }, + "B_runtime": { + "answer": "Three provided tests exercise this function. test_extract_anchors_no_anchors: a plain statement without anchors returns an empty list. test_extract_anchors_explicit_pkg_syntax: a claim containing `pkg:celery` yields exactly one pkg anchor `('pkg', 'celery')`. test_extract_anchors_from_data_anchors: explicit `data.anchors` with file, import and env entries produce matching (kind, value) anchors. The function itself extracts checkable anchors from a node's data/claim text via regex: it honors explicit data.anchors first, otherwise parses file paths, imports, env keys, pkg names and manifest package words, deduplicating through a seen set. Re-run all three when changing behavior, though the list may not be exhaustive.", + "tests": [ + "test_extract_anchors_no_anchors", + "test_extract_anchors_explicit_pkg_syntax", + "test_extract_anchors_from_data_anchors" + ] + }, + "D_shuffled": { + "answer": "The information given lists three relevant tests: test_format_diagnostics_empty, test_stale_token_rejected_after_file_change and test_lru_cap_evicts_oldest. Taken at face value these are the tests to run. Separately, `extract_anchors` extracts checkable anchors from a memory node's data/claim text using regex: it honors explicit `data.anchors`, skips prose on the read path when explicit anchors exist, and otherwise parses file paths, imports, env keys, pkg names and manifest package words, deduplicating via a seen set and returning Anchor objects. Note the listed tests appear unrelated to anchor extraction, so on the evidence provided they may not catch regressions; the actual test suite should be inspected.", + "tests": [ + "test_format_diagnostics_empty", + "TestAckImpact::test_stale_token_rejected_after_file_change", + "test_lru_cap_evicts_oldest" + ] + } + } + }, + { + "id": 7, + "name": "resolve_project_root", + "file_path": "D:/Project/MSCodeBase/src/core/project_resolution.py", + "question": "A developer is about to change the behavior of the function `resolve_project_root` in `project_resolution.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "The function resolve_project_root returns the project root directory used by MCP tools, resolving it fresh on each call. Inputs: an optional provided path string. Output: a Path. Priority order: explicit provided argument, then MSCODEBASE_PROJECT_PATH env override, then CWD with a self-indexing guard rejecting ext_root, then PROJECT_PATH env, then SQLite active_workspace_id, then the Zed workspaces table, then ZED_WORKTREE_ROOT env, and finally ext_root. Edge cases: multi-root paths split on newline or comma, non-existent paths skipped, and sqlite/OS errors swallowed. Regarding the question: no test names are given in the information provided, so I cannot determine which tests in the repository exercise this function; I cannot list them.", + "tests": [] + }, + "B_runtime": { + "answer": "resolve_project_root returns the project root Path for MCP tools, recomputed each call. It accepts an optional provided string and returns a Path. Resolution priority: provided argument, MSCODEBASE_PROJECT_PATH override, CWD with self-indexing guard, PROJECT_PATH env, SQLite active_workspace_id, the Zed workspaces table, ZED_WORKTREE_ROOT, and finally ext_root. It handles multi-root SQLite paths, missing directories, and sqlite/OS errors. Based on the tests shown, the tests that exercise this function are: test_provided_wins_over_cwd; test_cwd_equal_ext_root_is_skipped; test_mscb_path_trusts_ext_root. These three directly assert provided-over-CWD precedence, the self-index guard falling back to an env project, and MSCODEBASE_PROJECT_PATH trusting ext_root.", + "tests": [ + "TestCwdFirstIsolation::test_provided_wins_over_cwd", + "TestCwdSelfIndexGuard::test_cwd_equal_ext_root_is_skipped", + "TestPriorityProjectPathOverride::test_mscb_path_trusts_ext_root" + ] + }, + "D_shuffled": { + "answer": "resolve_project_root returns the project root Path for MCP tools, resolving anew each call. It takes an optional provided string and returns a Path, following a priority chain: provided argument, MSCODEBASE_PROJECT_PATH, CWD with self-indexing guard, PROJECT_PATH env, SQLite active_workspace_id, the Zed workspaces table, ZED_WORKTREE_ROOT env, then ext_root fallback. It tolerates multi-root paths, missing directories, and sqlite/OS errors. However, the tests listed here (test_flag_on_when_env_set, test_blocked_name_globals, test_query_hotspots_no_memory) concern late enrichment config, a sandbox violation check, and graph hotspots respectively; none appear to exercise resolve_project_root. Therefore I cannot determine which tests exercise this function from the information given.", + "tests": [ + "test_flag_on_when_env_set", + "TestValidateCode::test_blocked_name_globals", + "TestGraphRAGQueryEngine::test_query_hotspots_no_memory" + ] + } + } + }, + { + "id": 8, + "name": "SymbolIndex.build_call_graph", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/symbol_index.py", + "question": "A developer is about to change the behavior of the function `SymbolIndex.build_call_graph` in `symbol_index.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "SymbolIndex.build_call_graph builds a bidirectional call graph for a symbol using BFS. Inputs: a symbol name and depth (default 2, clamped to 1..5). Output: a dict with definition, callers, callees, call_chain, impact_files, and depth_reached. It looks up definitions, walks callers via self._references, walks callees by scanning references where the symbol is the caller, deduplicates entries, tracks visited sets to avoid cycles, and collects impacted files. Edge cases: depth below 1 is clamped up, above 5 clamped down, empty definitions, cycle prevention, and call_chain limited to five shallowest callers. On the question: no test names are provided, so I cannot determine which tests exercise this function; I cannot list them.", + "tests": [] + }, + "B_runtime": { + "answer": "build_call_graph returns a bidirectional BFS call graph for a symbol. It takes a symbol name and depth (default 2, clamped 1..5), and returns definition, callers, callees, call_chain, impact_files, and depth_reached. Callers are found through self._references; callees by scanning references where the symbol is the caller; visited sets prevent cycles and impact_files collects touched files. Based on the tests shown, the tests exercising this function are: test_build_call_graph_call_chain; test_build_call_graph_callees; test_build_call_graph_callers. These assert the call_chain is populated, that the callees of authenticate include validate_token and get_user, and that the callers of validate_token include authenticate.", + "tests": [ + "test_build_call_graph_call_chain", + "test_build_call_graph_callees", + "test_build_call_graph_callers" + ] + }, + "D_shuffled": { + "answer": "build_call_graph returns a bidirectional BFS call graph for a symbol. It takes a symbol name and depth (default 2, clamped to 1..5), and returns definition, callers, callees, call_chain, impact_files, and depth_reached, using visited sets to avoid cycles and deduplicating entries. However, the tests listed here (test_resolver_auto_approve, test_sha256_file_missing, test_does_not_scan_extension_dir) concern a trust resolver, file hashing, and a grep fallback scanning the extension directory; none appear to exercise build_call_graph. Therefore I cannot determine which tests exercise this function from the information given.", + "tests": [ + "test_resolver_auto_approve", + "test_sha256_file_missing", + "TestGrepFallbackProjectRoot::test_does_not_scan_extension_dir" + ] + } + } + }, + { + "id": 9, + "name": "IndexParser.parse_file", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_parser.py", + "question": "A developer is about to change the behavior of the function `IndexParser.parse_file` in `index_parser.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "The information given is insufficient to determine which tests exercise this function. The prompt supplies only the source of IndexParser.parse_file; no test files, test-module references, or test names are included. I cannot enumerate test functions covering encoding fallback (BOM, cp1251, latin-1), the existing_hash skip, comment-only rejection, AST chunking, or the sliding-window fallback without seeing the repository suite. Honest answer: cannot determine from the supplied text. To find them a developer should grep the tests directory for parse_file, IndexParser, and _has_code_lines, then run the matching functions. Listing specific test names here would be guessing, not evidence.", + "tests": [] + }, + "B_runtime": { + "answer": "The provided context includes two tests that exercise IndexParser.parse_file:\ntest_bs1_parse_file_skips_comment_only\ntest_property_graph_consistency\n\nBoth call parser.parse_file directly. The first targets the BS-1 guard rejecting comment-only files, asserting None is returned. The second is an integration test that indexes producer and consumer files, consumes the returned _ast_symbols cache, and verifies ghost-node cleanup after renames. These are the only test functions shown as relevant. Other tests may exist in the repository but are not included here, so this list reflects the supplied evidence only. A developer changing parse_file should run these two and grep the suite for additional callers before relying on them alone.", + "tests": [ + "test_bs1_parse_file_skips_comment_only", + "TestASTCacheInvalidation::test_property_graph_consistency" + ] + }, + "D_shuffled": { + "answer": "The context labels two tests as relevant to IndexParser.parse_file:\ntest_vor_coverage_starved_warning\ntest_inserts_after_anchor\n\nHowever, on inspection neither appears to call IndexParser.parse_file. The first tests VOR receipt formatting for starved nodes, and the second tests the write tool insert_after behavior. They exercise unrelated modules and never reference index_parser.py or parse_file. I therefore cannot confirm they catch regressions in parse_file, and I would treat the relevance label as unreliable. Honestly, from the given information the genuine tests cannot be determined; the two shown look mismatched. A developer should grep for parse_file and IndexParser instead of trusting this list.", + "tests": [ + "test_vor_coverage_starved_warning", + "TestWriteToolInsertAfter::test_inserts_after_anchor" + ] + } + } + }, + { + "id": 10, + "name": "modification_guard", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "question": "A developer is about to change the behavior of the function `modification_guard` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "The information given is insufficient to identify tests exercising modification_guard. The prompt supplies only the decorator source; no test module, names, or references appear. modification_guard is a decorator applied to async write tools, so regression tests would likely live in files covering write-tool guards, the ack registry, ambiguity handling, and PageRank or blast-radius thresholds, but I cannot name them from this text. I explicitly state: I cannot determine this from the information provided. A developer should grep the tests for modification_guard, ack_impact, _ack_registry, and fresh_guard fixtures, then run the collected tests. Naming specific functions here would be fabrication.", + "tests": [] + }, + "B_runtime": { + "answer": "The supplied context lists three tests that exercise modification_guard:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_allows_non_hot_file\n\nAll three apply the @modification_guard decorator to a test tool method. The first verifies pass-through when neither file_path nor symbol is given. The second checks the ambiguity guard refuses to silently route a bare symbol with multiple definitions. The third patches the PageRank and blast-radius helpers to zero and asserts a non-hot file passes with no ack required. These are the only tests shown as relevant. Other repository tests may also exercise the decorator but are not included, so this list reflects the supplied evidence only.", + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0", + "TestModificationGuard::test_allows_non_hot_file" + ] + }, + "D_shuffled": { + "answer": "The context labels three tests as relevant to modification_guard:\ntest_init_with_registry\ntest_symbol_index_remove_file_clears_env\ntest_parse_rejects_bad_urls\n\nBut none of these appear to touch modification_guard. The first tests CrossProjectDependencyGraph registry injection, the second tests SymbolIndex env-access cleanup, and the third tests GitUrlSource URL validation. They exercise unrelated modules and do not import or apply the decorator. I therefore cannot confirm they catch regressions in modification_guard. From the given information the real tests cannot be determined; the provided ones look shuffled or mismatched. A developer should grep for modification_guard and ack_impact instead of relying on this list.", + "tests": [ + "TestCrossProjectDependencyGraphInit::test_init_with_registry", + "test_symbol_index_remove_file_clears_env", + "test_parse_rejects_bad_urls" + ] + } + } + }, + { + "id": 11, + "name": "decorator", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "question": "A developer is about to change the behavior of the function `decorator` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "Based only on the information provided, I cannot determine which existing tests exercise `decorator`. The prompt shows the function code but no test files, no tests-directory listing, and no call sites. Since `decorator` is a nested factory inside `modification_guard` and its wrapper is applied via `@modification_guard()`, any regression test would import `modification_guard` from `src.core.modification_guard` and decorate a fake tool method. Without the contents of the repository test suite I cannot name specific test functions. Explicitly: this cannot be determined from the information given; one would need to grep the tests directory for `modification_guard` or `decorator` usages.", + "tests": [] + }, + "B_runtime": { + "answer": "According to the relevant tests supplied, the following test functions exercise this function and should be run to catch regressions:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_cli_unknown_tool\nThe first two directly decorate a fake tool with `@modification_guard()` and assert pass-through and ambiguous-symbol denial behaviour, so they clearly cover the wrapper logic. `test_cli_unknown_tool` looks unrelated on its face, because it only checks CLI argument handling and never touches `modification_guard`; it is probably a false positive in the candidate list. The two guard-specific tests are the ones that must be run.", + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "test_cli_unknown_tool", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0" + ] + }, + "D_shuffled": { + "answer": "The list is labelled as relevant, but on inspection none of the three tests exercises `decorator` or the modification guard wrapper. `test_empty_instance_does_not_overwrite_populated_file` covers SymbolIndex persistence, `test_passes_on_real_results` covers search-quality reporting, and `test_build_receipt_generates_id_and_steps` covers action receipts. None imports `modification_guard`, calls `@modification_guard()`, or asserts guard denial or ack behaviour, so none would catch a regression in this function. Therefore, based on the given material, I cannot name any test that genuinely exercises `decorator`; the provided candidates appear to be unrelated or shuffled distractors.", + "tests": [ + "test_empty_instance_does_not_overwrite_populated_file", + "TestCheckSearchQuality::test_passes_on_real_results", + "test_build_receipt_generates_id_and_steps" + ] + } + } + }, + { + "id": 12, + "name": "wrapper", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "question": "A developer is about to change the behavior of the function `wrapper` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "Based only on the material given, I cannot determine which tests exercise `wrapper`. Only the function body is provided, with no test files, test names, or import sites. Because `wrapper` is defined inside the `decorator` factory and is applied through `@modification_guard()`, regressions would surface in tests that import `src.core.modification_guard`, decorate a fake tool method, and assert guard behaviour. Without access to the repository test suite I cannot list concrete test function names. Explicitly: this is not determinable from the information given; one would need to search the tests directory for `modification_guard` usages.", + "tests": [] + }, + "B_runtime": { + "answer": "Per the relevant tests provided, these test functions exercise `wrapper` and must be run to catch regressions:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_allows_non_hot_file\nAll three build a fake tool class and decorate its async method with `@modification_guard()`, so they drive the wrapper directly: pass-through when no target is given, denial when a bare symbol resolves to multiple definitions, and pass-through when the mocked PageRank and blast radius are below thresholds. Together they cover the early-exit, ambiguity-guard, and non-hot paths.", + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0", + "TestModificationGuard::test_allows_non_hot_file" + ] + }, + "D_shuffled": { + "answer": "Although the three tests are presented as relevant, none of them exercises `wrapper` or the modification guard. `test_self_lock_adds_no_second_instance_issue` inspects a startup lock report, `test_fileguard_resolves_relative_project_path` checks FileGuard path resolution, and `test_trust_report_warns_on_untrusted_with_instructions` checks a trust report for instruction files. None imports or calls `modification_guard`, and none invokes `@modification_guard()` on an async method, so they cannot catch a regression in `wrapper`. Based on the given information I cannot identify a test that truly exercises it; the supplied candidates look like unrelated distractors.", + "tests": [ + "TestBuildStartupReport::test_self_lock_adds_no_second_instance_issue", + "test_fileguard_resolves_relative_project_path", + "test_trust_report_warns_on_untrusted_with_instructions" + ] + } + } + }, + { + "id": 13, + "name": "format_project_memory", + "file_path": "D:/Project/MSCodeBase/src/utils/ui_formatter.py", + "question": "A developer is about to change the behavior of the function `format_project_memory` in `ui_formatter.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "format_project_memory renders the intel_get_project_memory output as a markdown string. It takes a memory dict of sections (adrs, known_issues, tech_debt, failed_attempts), a limit defaulting to 3 for token budget (limit <= 0 shows all), and an optional stats dict. It appends a VOR coverage receipt (checked/total, starved, stale_ttl, budget_exceeded), freshness gate lines, status metrics, then iterates sections with icons, truncating items and adding an 'and N more' line. Edge cases: no stats means no receipt (backward compatibility); audit mode is auto-detected by REFUTED/SUPERSEDED statuses, then each node gets a status and retract reason. Regarding tests: the information given contains no test source and no repository listing, so I cannot determine which tests exercise this function. I must say so explicitly.", + "tests": [] + }, + "B_runtime": { + "answer": "format_project_memory converts a project-memory dict into a markdown summary. Inputs: memory (sections to items), limit (default 3; <=0 means all), optional stats for the VOR receipt. Output: a string with header, VOR coverage, freshness gate, status metrics, then per-section icons, titles (max 80 chars), verification markers (no_anchors, budget_exceeded, stale_ttl) and audit statuses (REFUTED with reason, SUPERSEDED, VERIFIED, ACTIVE). Edge cases: absent stats disables the receipt; limit truncation emits an 'and N more' line; audit mode is detected from REFUTED/SUPERSEDED presence. Tests that exercise it, from the provided snippets:\ntest_audit_mode_shows_statuses_and_reasons\ntest_budget_exceeded_node_marker\ntest_default_limit_three", + "tests": [ + "test_audit_mode_shows_statuses_and_reasons", + "test_budget_exceeded_node_marker", + "test_default_limit_three" + ] + }, + "D_shuffled": { + "answer": "format_project_memory formats the project memory dict into markdown. It builds a header, optionally a VOR receipt from stats (coverage, starved, stale_ttl, budget warning, freshness gate, status metrics), then walks each non-empty section adding icons, entry counts, truncated titles with verification flags, and audit statuses when REFUTED/SUPERSEDED are present. limit defaults to 3; <=0 shows every node; hidden nodes get an 'and N more' line. Edge cases: no stats suppresses the receipt; title truncation at 80 chars; the import of TTL_STALE_DAYS is guarded. On tests: the three snippets shown (test_verdict_env_block_is_inconclusive_not_refuted, test_invalidates_bm25, test_blocked_string_pattern_eval) do not call format_project_memory and appear unrelated, so from the given information I cannot confirm any test that exercises it.", + "tests": [ + "test_verdict_env_block_is_inconclusive_not_refuted", + "TestApplyFileMove::test_invalidates_bm25", + "TestValidateCode::test_blocked_string_pattern_eval" + ] + } + } + }, + { + "id": 14, + "name": "validate_code", + "file_path": "D:/Project/MSCodeBase/src/core/sandbox/executor.py", + "question": "A developer is about to change the behavior of the function `validate_code` in `executor.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "validate_code statically validates Python source before sandbox execution in two layers. Layer 1 scans the raw string for blocked patterns using a word-boundary regex to avoid false positives. Layer 2 parses the code with ast.parse (SyntaxError becomes a SandboxViolation) and walks all nodes, blocking non-allowlisted imports and from-imports, calls to eval/exec/compile and __import__, dangerous dunder attribute access, and blocked builtin names. Input: a code string. Output: a list of non-fatal warnings (always empty in this implementation). Raises SandboxViolation on the first hard block. Edge cases: empty code and safe code pass; attribute-vs-name checks distinguish node types. I cannot determine the tests: no test source or repository listing is provided, so I must say so explicitly.", + "tests": [] + }, + "B_runtime": { + "answer": "validate_code is a two-layer static sandbox validator for Python source. Layer 1 regex-scans raw text for blocked string patterns with word boundaries; Layer 2 parses via ast.parse and walks nodes to reject non-allowlisted imports and from-imports, eval/exec/compile and __import__ calls, dangerous dunder attribute access, and blocked names. It returns a list of non-fatal warnings (currently always empty) and raises SandboxViolation on the first hard block, including SyntaxError. Edge cases: empty code parses fine, safe code with math.sqrt passes, and the node-type branches keep attribute access separate from bare name usage. Tests exercising it, from the provided snippets:\ntest_allowed_modules\ntest_empty_code\ntest_safe_code_passes", + "tests": [ + "TestValidateCode::test_allowed_modules", + "TestValidateCode::test_empty_code", + "TestValidateCode::test_safe_code_passes" + ] + }, + "D_shuffled": { + "answer": "validate_code performs pre-execution sandbox validation in two layers. First a fast regex string scan over BLOCKED_STR_PATTERNS with a negative lookbehind word boundary; then an AST walk that rejects imports not in ALLOWED_MODULES, dangerous calls (eval, exec, compile, __import__), dangerous dunder attributes, and blocked names. It returns a list of warnings (non-fatal) and raises SandboxViolation on any hard block, converting SyntaxError into a violation. Edge cases: empty input, safe code, and distinguishing attribute access from bare names. Regarding tests: the snippets given (test_enrichment_empty_results, test_layer_windows_backslashes, test_build_call_graph_depth_2) concern search enrichment, path layer inference, and call graphs, not validate_code, so I cannot confirm they exercise it from the information given.", + "tests": [ + "test_enrichment_empty_results", + "TestInferMetadata::test_layer_windows_backslashes", + "test_build_call_graph_depth_2" + ] + } + } + }, + { + "id": 15, + "name": "_extract_yarn_lock", + "file_path": "D:/Project/MSCodeBase/src/sources/manifest/extract.py", + "question": "A developer is about to change the behavior of the function `_extract_yarn_lock` in `extract.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "_extract_yarn_lock parses a yarn.lock file, supporting v1, v2 and berry v10 formats. It reads line by line, treating non-indented lines ending in colon as block keys, then captures the following version line. A nested flush helper emits a ManifestEntry (npm ecosystem, normalized name, version, lockfile source, line number) only when both key and version were seen. Aliased packages are handled: when a resolution line contains npm: and its name differs from the block key, the resolution name becomes the real package name. Inputs: text and source label; output: a list of ManifestEntry. Edge cases: blank lines and __metadata are skipped; quoted keys are unquoted; blocks without a version are dropped. No tests are given here, so I cannot determine which tests exercise it.", + "tests": [] + }, + "B_runtime": { + "answer": "_extract_yarn_lock parses yarn.lock files across v1, v2 and berry v10. It scans lines, treating non-indented colon-terminated lines as package block keys (stripping surrounding quotes), then records the version from the version line. Emitted entries are npm ManifestEntry objects with normalized names, deferred to flush so alias handling works: if a resolution line contains npm: with a name differing from the key, that resolution name is used instead. Blank lines and __metadata are skipped, and blocks lacking versions produce nothing. Inputs are the lock text and a source label; output is the entry list. Tests from the provided snippets:\ntest_yarn_berry_patch_descriptors\ntest_yarn_berry_v10_scoped\ntest_yarn_synthetic", + "tests": [ + "test_yarn_berry_patch_descriptors", + "test_yarn_berry_v10_scoped", + "test_yarn_synthetic" + ] + }, + "D_shuffled": { + "answer": "_extract_yarn_lock parses yarn.lock files (v1, v2, berry v10) into npm ManifestEntry records. It iterates lines: non-indented lines ending with colon start a new package block (quotes stripped), version lines set the pending version, and resolution lines detect aliases when npm: resolves to a different real name than the block key. A flush closure emits an entry only when both a key and a version exist, attaching the source and line number. Edge cases include skipping blank and __metadata lines, dropping versionless blocks, and handling aliased or patched descriptors. On tests: the three snippets shown (test_returns_project_path, test_worktree_cleaned_after_run, test_metadata_returns_dict) exercise search tool headers and change-preview worktrees, not this extractor, so I cannot confirm any test that exercises it from the information given.", + "tests": [ + "TestProjectHeader::test_returns_project_path", + "TestChangePreview::test_worktree_cleaned_after_run", + "TestProjectHeader::test_metadata_returns_dict" + ] + } + } + }, + { + "id": 16, + "name": "WriteTool._action_replace", + "file_path": "D:/Project/MSCodeBase/src/mcp/tools/write_tools.py", + "question": "A developer is about to change the behavior of the function `WriteTool._action_replace` in `write_tools.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, no test files or repository listing. A reliable answer requires searching the repo for callers of WriteTool._action_replace. Description: this async method implements the replace action. It validates the symbol identifier and optional file_path, resolves definitions through the symbol index, and rejects missing, ambiguous, or out-of-project targets. With apply=False it returns a preview and may attach a type-check preflight. With apply=True it indents new_code, computes before/after hashes, AST-validates Python syntax, runs preflight validation, atomic-writes the file, invalidates the LSP cache, evicts the file from the symbol index, and records a contract entry. It returns user-facing markdown messages on success or error.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise this function are:\ntest_unknown_symbol_returns_warning\ntest_filter_mismatch_returns_error\ntest_replace_dry_run_does_not_write\nDescription: this async method implements the replace action of a write tool. It validates the symbol identifier and optional file_path, resolves definitions via the symbol index, and errors on missing, ambiguous, or out-of-project symbols and files. When apply is False it returns a preview (optionally with a type-check preflight). When apply is True it indents new_code, records before/after hashes, AST-validates Python syntax, runs preflight validation, atomic-writes, invalidates the LSP cache, removes the file from the symbol index, and records a contract. Edge cases: empty arguments, bad identifiers, syntax errors, blocking preflight.", + "tests": [ + "TestWriteToolReplace::test_unknown_symbol_returns_warning", + "TestWriteToolReplace::test_filter_mismatch_returns_error", + "TestDryRunNoWrite::test_replace_dry_run_does_not_write" + ] + }, + "D_shuffled": { + "answer": "The listed tests do not exercise WriteTool._action_replace. test_own_host_v2_is_healthy covers DatabaseLock.classify_holder; test_drop_removes_entry covers LiveBuffer.drop; test_flush_now_clears_and_calls_callback covers DebounceBatch.flush_now. These are unrelated components (locking, buffering, debouncing) and give no coverage of the replace action. Therefore the real regression tests for this function cannot be determined from the information given; a repo-wide search for callers of _action_replace would be required. Description: the async method validates the symbol and file_path, resolves definitions via the symbol index, supports a dry-run preview, and on apply indents new_code, AST-validates Python syntax, runs preflight, atomic-writes, invalidates the LSP cache, and records a contract.", + "tests": [ + "TestClassifyHolder::test_own_host_v2_is_healthy", + "test_drop_removes_entry", + "TestDebounceBatch::test_flush_now_clears_and_calls_callback" + ] + } + } + }, + { + "id": 17, + "name": "SymbolIndex.get_call_chain", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/symbol_index.py", + "question": "A developer is about to change the behavior of the function `SymbolIndex.get_call_chain` in `symbol_index.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, with no test files or repository listing. A reliable answer needs a repo-wide search for callers of SymbolIndex.get_call_chain. Description: this method returns call chains for a symbol by traversing the internal reference map under a lock. The direction argument selects callers (up), callees (down), or both. For each depth level it walks references, skipping definitions and the symbol itself, appending records with symbol, file, line, and depth. It returns a dict with symbol, callers_chain, callees_chain, and total_connected. Edge cases: max_depth limits traversal, visited sets prevent cycles and duplicates, and missing symbols yield empty chains.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise this function are:\ntest_get_call_chain_both\ntest_get_call_chain_down\ntest_get_call_chain_total_connected\nDescription: this method returns a symbol call chain by traversing the internal references map while holding a lock. direction selects callers (up), callees (down), or both. Each level walks references, skipping definitions and the symbol itself, and records entries with symbol, file, line, and depth. The result dict holds symbol, callers_chain, callees_chain, and total_connected. Edge cases: max_depth caps traversal depth, visited sets prevent infinite loops and duplicate entries, and unknown symbols return empty chains. The three tests cover both-direction traversal, downward traversal, and the total_connected count.", + "tests": [ + "test_get_call_chain_both", + "test_get_call_chain_down", + "test_get_call_chain_total_connected" + ] + }, + "D_shuffled": { + "answer": "The listed tests do not exercise SymbolIndex.get_call_chain. test_unparseable_lines_skipped covers _count_log_levels; test_old_schema_detected covers _check_sqlite_schema_health; test_wire_plugins_no_env_noop covers wire_plugins. These concern log parsing, SQLite schema checks, and plugin wiring, so they give no coverage of call-chain traversal. The real regression tests cannot be determined from the information given; a repo-wide search for callers of get_call_chain would be required. Description: the method traverses the internal references map under a lock, in the requested direction (up for callers, down for callees, both), recording symbol, file, line, and depth per entry and returning a dict with callers_chain, callees_chain, and total_connected, bounded by max_depth and cycle-guarding visited sets.", + "tests": [ + "test_unparseable_lines_skipped", + "TestSqliteSchemaHealth::test_old_schema_detected", + "test_wire_plugins_no_env_noop" + ] + } + } + }, + { + "id": 18, + "name": "load_manifest", + "file_path": "D:/Project/MSCodeBase/src/plugins/manifest.py", + "question": "A developer is about to change the behavior of the function `load_manifest` in `manifest.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, with no test files or repository listing. A reliable answer would require searching the repo for callers of load_manifest. Description: this function reads and validates a plugin.json manifest without executing plugin code. It parses JSON and raises PluginManifestError with kinds manifest_missing or manifest_invalid on file or parse errors, checks required fields, normalizes id/name/version, enforces schema_version against _SCHEMA_VERSION, validates platform compatibility, defaults and validates requires_engine_version with SpecifierSet, normalizes tools into a non-empty list of strings, and cleans dependencies. It returns a ToolPlugin instance. Edge cases: invalid or empty values, wrong schema, platform mismatch, bad engine spec.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise this function are:\ntest_load_invalid_engine_req\ntest_manifest_requires_missing_fields\ntest_manifest_schema_mismatch_real\nDescription: load_manifest reads and validates plugin.json without executing plugin code. It parses JSON, raising PluginManifestError (kinds manifest_missing, manifest_invalid) on file or parse errors, verifies required fields, normalizes id/name/version, enforces schema_version equality with _SCHEMA_VERSION, checks platform compatibility, defaults requires_engine_version to >=0 and validates it with SpecifierSet, normalizes tools into a non-empty string list, and cleans dependencies before returning a ToolPlugin. The three tests cover an invalid engine requirement, a missing manifest file, and a schema_version mismatch.", + "tests": [ + "test_load_invalid_engine_req", + "test_manifest_requires_missing_fields", + "test_manifest_schema_mismatch_real" + ] + }, + "D_shuffled": { + "answer": "The listed tests do not exercise load_manifest. test_tokenize_truncates_over_limit_to_guarantee covers an embedder truncation helper; test_user_project_not_self_index covers _is_self_index_path; test_empty_results covers Searcher._extract_key_terms. These concern embedding, self-index detection, and search term extraction, so they give no coverage of manifest loading. The real regression tests cannot be determined from the information given; a repo-wide search for callers of load_manifest would be needed. Description: the function reads and validates plugin.json without executing plugin code, raising PluginManifestError for missing or invalid JSON, incomplete fields, schema mismatch, platform mismatch, and invalid engine requirements, then returns a normalized ToolPlugin instance.", + "tests": [ + "test_tokenize_truncates_over_limit_to_guarantee", + "TestIsSelfIndexPath::test_user_project_not_self_index", + "TestExtractKeyTerms::test_empty_results" + ] + } + } + }, + { + "id": 19, + "name": "ProjectIntelligenceLayer._enrich_job_response", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/layer.py", + "question": "A developer is about to change the behavior of the function `ProjectIntelligenceLayer._enrich_job_response` in `layer.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given: no test files, test names, or repository context accompany the function. Based solely on the code, `_enrich_job_response` enriches a `BackgroundJob` dict. It rounds `progress`; sets `poll_interval_seconds` from status and progress bands (0 for completed/failed, else 30, 30, 15, 5); builds `progress_label` from status and progress; and for running jobs computes an adaptive ETA via `_embed_progress_from_log`, falling back to `job_history` during finalizing, with an honest `None` when no data exists. Edge cases include missing embed progress, done at or beyond total, and non-running statuses. Without the repository test sources I cannot name any test function that must be run.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise `_enrich_job_response` are: test_enrich_eta_none_when_no_embed_progress, test_enrich_eta_uses_real_embed_speed, and test_intel_runtime_status_reports_reindex_progress. The first two call it directly to verify the adaptive ETA contract (None when no embed data, and 21 seconds from 347 remaining chunks at 16 ch per s). The third exercises it indirectly through `intel_get_runtime_status`, asserting reindex telemetry and accepting a None or positive ETA. The function rounds progress, computes poll_interval_seconds and progress_label from status and progress, and derives estimated_seconds for running jobs. Run these three to catch regressions in the ETA logic and the reindex status path.", + "tests": [ + "test_enrich_eta_none_when_no_embed_progress", + "test_enrich_eta_uses_real_embed_speed", + "test_intel_runtime_status_reports_reindex_progress" + ] + }, + "D_shuffled": { + "answer": "None of the provided tests exercise `_enrich_job_response`. test_normalize_diag_uri_idempotent tests LspClient URI normalization; test_double_load_is_idempotent tests embedder tokenizer loading; test_apply_deletes_from_file tests WriteTool symbol deletion. They are unrelated to job enrichment and would not catch regressions here. The function is a pure formatter over a BackgroundJob, so a meaningful test must construct a job, call `_enrich_job_response` (or `intel_get_runtime_status`), and assert on poll_interval_seconds, progress_label, eta_phase, or estimated_seconds. From the information given I cannot identify any real test names; I can only state that these three do not cover it.", + "tests": [ + "test_normalize_diag_uri_idempotent", + "test_double_load_is_idempotent", + "TestWriteToolSafeDelete::test_apply_deletes_from_file" + ] + } + } + }, + { + "id": 20, + "name": "format_runtime_status", + "file_path": "D:/Project/MSCodeBase/src/utils/ui_formatter.py", + "question": "A developer is about to change the behavior of the function `format_runtime_status` in `ui_formatter.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given: no test files or test names are supplied with the function. `format_runtime_status` converts a status dict into a Markdown dashboard with traffic-light emojis. It reads project path, PID, embedding provider, provider statuses, index telemetry, reranker RAM, reindex flags, and optional startup diagnostics. It derives LEDs for LM Studio, Ollama, the active provider, the index, overall health, and the reranker; renders chunk/file/symbol counts or a Reindex in progress line with percent and ETA; and short-circuits to a warning block when lock or DB state is unhealthy. Without the repository test sources I cannot list the tests that must be run.", + "tests": [] + }, + "B_runtime": { + "answer": "The relevant tests are: test_format_runtime_status_fallback_to_768_when_no_model_info, test_format_runtime_status_normal_path_unchanged, and test_format_runtime_status_reindex_without_progress. The first checks the ONNX dimension fallback when model_info is absent. The second confirms the unchanged normal path yields 0 chunks for an empty index. The third verifies the reindex branch shows Reindex in progress and never 0 chunks, even with missing progress and ETA. Together they cover the traffic-light logic, provider and model naming, index telemetry rendering, and the reindex short-circuit. Run all three before changing format_runtime_status to catch regressions in these rendering paths.", + "tests": [ + "test_format_runtime_status_fallback_to_768_when_no_model_info", + "test_format_runtime_status_normal_path_unchanged", + "test_format_runtime_status_reindex_without_progress" + ] + }, + "D_shuffled": { + "answer": "None of the provided tests exercise `format_runtime_status`. test_env_gate_off checks the language-pack environment gate; test_blocked_import_subprocess checks sandbox validation of imports; test_iteration_1_adds_terms checks the searcher refined-query generation. All are unrelated to runtime status formatting and would not catch regressions here. A real test would build a status dict and assert on the rendered dashboard, provider LEDs, the index chunk line, or the startup-diagnostic warning block. Based only on the information given, I cannot identify any actual test names that cover this function; the three shown clearly do not.", + "tests": [ + "TestDisabledByDefault::test_env_gate_off", + "TestValidateCode::test_blocked_import_subprocess", + "TestGenerateRefinedQuery::test_iteration_1_adds_terms" + ] + } + } + }, + { + "id": 21, + "name": "_delayed_auto_index", + "file_path": "D:/Project/MSCodeBase/src/mcp/server_factory.py", + "question": "A developer is about to change the behavior of the function `_delayed_auto_index` in `server_factory.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given: no tests or repository context accompany the function. `_delayed_auto_index` runs a delayed auto-index after server startup. It resolves an indexer, skips when the project root equals ext_root or the index already has chunks, logs a human-readable startup diagnostic for an empty or corrupt DB, waits 1.5 seconds, sets a reindexing guard, runs `index_project` in a thread, transitions the registry state to READY, and always clears the guard in a finally block. Failures are logged, not raised. Without the repository test sources I cannot name any test function that must be run.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests exercising `_delayed_auto_index` are: test_delayed_auto_index_skips_when_index_not_empty and test_delayed_auto_index_sets_state_ready. The first asserts that a non-empty index of 9000 chunks causes an early return, that `index_project` is not called, and that `set_state` is never invoked. The second asserts that with an empty index `index_project` runs, returning 42, and the registry is set to ProjectState.READY. Both stub `resolve_indexer_for_request` and the DI registry. Run these two to catch regressions in the skip conditions, the reindexing guard, and the READY state transition.", + "tests": [ + "test_delayed_auto_index_skips_when_index_not_empty", + "test_delayed_auto_index_sets_state_ready" + ] + }, + "D_shuffled": { + "answer": "None of the provided tests exercise `_delayed_auto_index`. test_lazy_graph_build checks that CrossProjectDependencyGraph lazily initializes its `_graph`; test_dedup_preserves_order checks that `extract_imports` deduplicates modules while preserving order. Both are unrelated to auto-indexing and would not catch regressions in the function. A meaningful test would stub the indexer and the DI registry, then assert skip behavior, `index_project` invocation, and the READY state transition. From the information given I cannot identify any real test names covering `_delayed_auto_index`; the two shown are clearly unrelated modules.", + "tests": [ + "TestCrossProjectDependencyGraphInit::test_lazy_graph_build", + "TestExtractImports::test_dedup_preserves_order" + ] + } + } + }, + { + "id": 22, + "name": "load_metrics", + "file_path": "D:/Project/MSCodeBase/src/core/error_handler.py", + "question": "A developer is about to change the behavior of the function `load_metrics` in `error_handler.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "No test names can be determined from the information given. The prompt provides only the source of load_metrics plus its module globals (_METRICS_PATH, _TOOL_METRICS, _TOOL_METRICS_LOCK); it contains no test file, no directory listing, and no search results. Nothing in the supplied text names a test function or shows an import of error_handler. So I cannot honestly list tests that exercise it. To find them one would grep the repository for load_metrics, for imports of src.core.error_handler, or for patching of _METRICS_PATH. The function loads saved tool metrics from a JSON file, sanitizes negative latency values (min_ms, max_ms, total_ms and the latencies list), and merges them into _TOOL_METRICS. Candidate regression tests would target that sanitization, but their names are undetermined.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise load_metrics and should be run before changing it are:\ntest_bs14_load_metrics_clamps_negative_total_ms\ntest_bs14_load_metrics_sanitizes_negative\nBoth import src.core.error_handler, point eh._METRICS_PATH at a temporary JSON file, clear eh._TOOL_METRICS, call eh.load_metrics(), and assert the sanitization behavior. The first checks that a negative total_ms is clamped to 0 and min_ms becomes 999999. The second checks that negative min_ms is dropped, that every latency is non-negative, and that total_ms stays non-negative. These are the direct regression tests for the loading and sanitizing logic, so both must be run whenever load_metrics changes.", + "tests": [ + "test_bs14_load_metrics_clamps_negative_total_ms", + "test_bs14_load_metrics_sanitizes_negative" + ] + }, + "D_shuffled": { + "answer": "I cannot determine the tests that exercise load_metrics from the information given. The supplied snippets are test_multiple_mentions (about parse_cross_repo_query) and test_ips_are_global (about IP classification). Neither imports error_handler, references load_metrics, nor touches _METRICS_PATH or _TOOL_METRICS, so neither exercises the function. Although the section is labelled as relevant tests, their contents contradict that label. Therefore no test name can be truthfully listed. The function loads saved tool metrics from a JSON file, sanitizes negative latency fields, and merges them into _TOOL_METRICS; a genuine regression test would call it with a crafted metrics file. Undetermined from the given text.", + "tests": [ + "TestParseCrossRepoQuery::test_multiple_mentions", + "test_ips_are_global" + ] + } + } + }, + { + "id": 23, + "name": "_extract_pyproject", + "file_path": "D:/Project/MSCodeBase/src/sources/manifest/extract.py", + "question": "A developer is about to change the behavior of the function `_extract_pyproject` in `extract.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "No test names can be determined from the information given. The prompt supplies only the source of _extract_pyproject plus references to helpers (_toml, _req_name, normalize_python, ManifestEntry); there is no test file, no listing, and no grep output. Nothing names a test function or imports extract.py. So I cannot honestly list tests that exercise it. To find them one would search for _extract_pyproject, for imports of the extract module, or for tests using pyproject fixtures. The function parses TOML text of a pyproject.toml, returning ManifestEntry objects for project.dependencies, project.optional-dependencies, and PEP 735 dependency-groups (supporting both list and packages-dict forms), skipping anything _toml cannot parse. Candidate tests would cover those branches, but their names are undetermined.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise _extract_pyproject are:\ntest_pyproject_optional_and_groups\ntest_requests_pyproject_and_requirements_dev\ntest_manifest_packages_wiring_pyproject_plus_lock\nThe first calls _extract_pyproject directly on inline TOML and checks that chardet, pysocks, pytest and pytest-cov are all collected, covering project.dependencies, optional-dependencies and dependency-groups. The second goes through manifest_packages on a requests fixture, exercising the pyproject path indirectly and asserting normalized names. The third uses a uv fixture where dependency-groups is the only source, asserting black and annotated-types appear. Together they cover the list and packages-dict branches, so all three should be run before changing the function.", + "tests": [ + "test_pyproject_optional_and_groups", + "test_requests_pyproject_and_requirements_dev", + "test_manifest_packages_wiring_pyproject_plus_lock" + ] + }, + "D_shuffled": { + "answer": "I cannot determine the tests that exercise _extract_pyproject from the information given. The supplied snippets are test_localhost_https_rejected, test_search_with_mode_fast_calls_embed_and_vector, and test_attribute_assignment. None imports the extract module, references _extract_pyproject, or uses pyproject TOML parsing; they cover Git URL SSRF protection, search-mode behavior, and assignment extraction respectively. So none exercises the function, despite the relevant-tests label. No test name can be truthfully listed. The function parses pyproject.toml TOML text into ManifestEntry objects from project.dependencies, optional-dependencies, and PEP 735 dependency-groups. Finding real tests requires a repository search for _extract_pyproject or extract-module imports. Undetermined from the given text.", + "tests": [ + "test_localhost_https_rejected", + "test_search_with_mode_fast_calls_embed_and_vector", + "TestEdgeCases::test_attribute_assignment" + ] + } + } + }, + { + "id": 24, + "name": "IndexStatusReporter.get_status", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_status.py", + "question": "A developer is about to change the behavior of the function `IndexStatusReporter.get_status` in `index_status.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "No test names can be determined from the information given. The prompt provides only the source of IndexStatusReporter.get_status plus the implicit attributes it uses (_reindex_check, _table_write_lock, _cached_total_chunks, _cached_unique_files, table, _scan_stale, _watchdog_callback). There is no test file, no directory listing, and no search results. Nothing in the supplied text names a test function or constructs an IndexStatusReporter. So I cannot honestly list which tests exercise it. To find them one would grep for get_status, for IndexStatusReporter, or for the reindex fast-fail scenario. The method returns database statistics, short-circuits during reindex, and falls back to count_rows for consistency, but candidate test names are undetermined.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests listed under this function are:\ntest_get_status_fast_fail_during_reindex_does_not_block\ntest_setup\nThe first directly constructs IndexStatusReporter and calls get_status, so it clearly exercises the function, asserting the fast-fail path returns cached values quickly while the write lock is held. The second (test_setup) builds a FileGuard and DI container and resolves an indexer; it does not visibly call get_status, so it likely does not exercise it despite appearing under the relevant-tests heading. Strictly, only test_get_status_fast_fail_during_reindex_does_not_block can be confirmed from the given text; test_setup should be treated as unconfirmed and checked separately.", + "tests": [ + "test_get_status_fast_fail_during_reindex_does_not_block", + "test_setup" + ] + }, + "D_shuffled": { + "answer": "I cannot determine the tests that exercise IndexStatusReporter.get_status from the information given. The supplied snippets are test_handles_same_path (Indexer.move_chunks_metadata no-op) and test_no_dollardefs_or_ref_in_tool_schemas (tool JSON schema validation). Neither constructs an IndexStatusReporter, calls get_status, nor touches the reindex fast-fail or count_rows logic. They belong to different modules, so they do not exercise the function despite the relevant-tests label. No test name can be truthfully listed. The function returns DB statistics and short-circuits to cached values during reindexing to avoid blocking the event loop. Locating real tests requires a repository search for get_status or IndexStatusReporter. Undetermined from the given text.", + "tests": [ + "TestMoveChunksMetadata::test_handles_same_path", + "test_no_dollardefs_or_ref_in_tool_schemas" + ] + } + } + }, + { + "id": 25, + "name": "WriteTool._action_insert", + "file_path": "D:/Project/MSCodeBase/src/mcp/tools/write_tools.py", + "question": "A developer is about to change the behavior of the function `WriteTool._action_insert` in `write_tools.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "No test names can be determined from the information given. The prompt supplies only the source of WriteTool._action_insert plus the helpers and attributes it calls (require_ready_project, resolve_symbol_index, find_definitions, _resolve_single_source_def, _build_insert_lines, _preflight_validate, _atomic_write, _contract_record). There is no test file, no listing, and no grep output, and nothing names a test function or instantiates WriteTool. So I cannot honestly list tests that exercise it. To find them one would search for _action_insert_after, _action_insert_before, or tests using a write_tool fixture. The method inserts code before or after an anchor symbol, previews when apply is false, and records a contract hash, but candidate test names are undetermined.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests that exercise _action_insert are:\ntest_filter_mismatch_returns_error\ntest_filter_mismatch_returns_error\ntest_unknown_symbol_returns_error\nNote the first name appears twice: one variant calls _action_insert_after and one calls _action_insert_before, both with a mismatching file_path and asserting an error is returned. test_unknown_symbol_returns_error calls _action_insert_after with a nonexistent anchor and expects an error. All three reach the function through its position wrappers (insert_before and insert_after both delegate to _action_insert), so they cover the anchor-not-found and file-filter branches. They should be run before changing _action_insert, though the duplicate name means collection must be scoped per test file.", + "tests": [ + "TestWriteToolInsertAfter::test_filter_mismatch_returns_error", + "TestWriteToolInsertBefore::test_filter_mismatch_returns_error", + "TestWriteToolInsertAfter::test_unknown_symbol_returns_error" + ] + }, + "D_shuffled": { + "answer": "I cannot determine the tests that exercise WriteTool._action_insert from the information given. The supplied snippets are test_token_is_32_hex_chars (ack token format), test_unknown_kind_ignored (AlertStore.push), and test_gemfile_skips_gitpath (manifest_packages). None imports write_tools, uses a write_tool fixture, or calls _action_insert, _action_insert_after, or _action_insert_before. They cover unrelated modules, so none exercises the function, despite the relevant-tests label. Therefore no test name can be truthfully listed. The function inserts new code before or after an anchor symbol, supports preview mode, and performs preflight validation plus atomic writes and contract recording. Locating real tests requires searching for _action_insert or the write_tool fixture. Undetermined from the given text.", + "tests": [ + "TestHMACToken::test_token_is_32_hex_chars", + "test_unknown_kind_ignored", + "test_gemfile_skips_gitpath" + ] + } + } + }, + { + "id": 26, + "name": "_match_gitignore_pattern", + "file_path": "D:/Project/MSCodeBase/src/core/gitignore_parser.py", + "question": "A developer is about to change the behavior of the function `_match_gitignore_pattern` in `gitignore_parser.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path is matched by a single .gitignore pattern. It strips a leading slash and records/dir-strips a trailing slash. It then tries, in order: exact equality; `/**/` matching via prefix/suffix checks; fnmatch wildcards (full path when a slash is present, otherwise the basename); slash-free basename semantics with special any-depth handling for directory patterns; and prefix matching for slash-containing directory patterns. Notable edge cases: directory patterns match at any depth, wildcard scope depends on slash presence, and the trailing-slash branch became unreachable after stripping. No repository or test information is supplied in this prompt, so I cannot name the tests that exercise it; a developer should search the suite for gitignore_parser imports or calls to this helper and run those tests.", + "tests": [] + }, + "B_runtime": { + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path matches one .gitignore pattern. It strips a leading slash, detects trailing-slash directory patterns, then tries exact match, `/**/` prefix/suffix matching, fnmatch wildcards (full path when the pattern contains a slash, otherwise basename), slash-free basename and directory semantics, and finally prefix matching for slash-containing directory patterns. Edge cases include any-depth directory matching, wildcard scope depending on slash presence, and now-unreachable trailing-slash code. The context explicitly identifies the tests that exercise this function, so the developer must run:\ntest_dir_pattern_excludes_nested\ntest_dir_pattern_matches_any_depth\ntest_file_pattern_unchanged\nThese cover nested directory exclusion, any-depth directory matching, and unchanged file-pattern behaviour.", + "tests": [ + "test_dir_pattern_excludes_nested", + "test_dir_pattern_matches_any_depth", + "test_file_pattern_unchanged" + ] + }, + "D_shuffled": { + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path matches a single .gitignore pattern. It strips a leading slash, detects trailing-slash directory patterns, then applies exact match, `/**/` prefix/suffix matching, fnmatch wildcards (full path when a slash is present, otherwise basename), slash-free basename/directory semantics, and prefix matching for slash-containing directory patterns. Edge cases include any-depth directory matching and unreachable trailing-slash code. However, the tests supplied with this prompt do not reference `_match_gitignore_pattern` at all; they belong to other modules (a dependency graph and a path-safety guard) and cannot catch regressions here. Their names are:\ntest_no_shared\ntest_no_path\ntest_non_ascii_path_denied\nFrom the given information I cannot confirm any test that exercises the function; the developer should instead find tests importing gitignore_parser or calling `_excluded`.", + "tests": [ + "TestFindSharedInterfaces::test_no_shared", + "TestGetDependencyPath::test_no_path", + "TestWriteToolFileGuard::test_non_ascii_path_denied" + ] + } + } + }, + { + "id": 27, + "name": "_Fingerprint.__init__", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/verify_on_read.py", + "question": "A developer is about to change the behavior of the function `_Fingerprint.__init__` in `verify_on_read.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint. When a `data` dict is supplied it hydrates the `imports`, `files`, `env_keys`, and `packages` sets from it, with an ADR-0005 note that old caches lacking packages are rebuilt elsewhere to avoid false REFUTED results, and it leaves `build_ms` at zero. Otherwise it scans `/src` recursively, skipping `__pycache__`, collecting every relative file path and the top-level import names of each .py file, parses `.env` and `.env.example` for key names, loads manifest packages, and records elapsed build milliseconds. Edge cases: root is None, missing src directory, unreadable files (skipped via OSError). No repository or test context is provided here, so I cannot name which existing tests exercise this constructor.", + "tests": [] + }, + "B_runtime": { + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint: from a `data` dict it hydrates imports, files, env_keys, and packages sets (ADR-0005 notes old caches without packages are rebuilt); otherwise it scans `/src`, collects file paths and top-level imports, parses `.env`/`.env.example`, loads manifest packages, and records build milliseconds. Edge cases include a missing root, absent src, and unreadable files. The supplied tests actually exercise `extract_anchors` and its ADR-0005 `pkg:`/anchor classification rather than the constructor directly, but the context presents them as the relevant tests, so the developer should run:\ntest_extract_anchors_explicit_pkg_syntax\ntest_extract_anchors_from_data_anchors\ntest_extract_anchors_read_path_backward_compat\nThey validate anchor extraction and backward-compatible read-path classification; tests constructing `_Fingerprint` directly should also be located.", + "tests": [ + "test_extract_anchors_explicit_pkg_syntax", + "test_extract_anchors_from_data_anchors", + "test_extract_anchors_read_path_backward_compat" + ] + }, + "D_shuffled": { + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint. With a `data` dict it hydrates imports, files, env_keys, and packages sets (ADR-0005 notes caches without packages are rebuilt), leaving build_ms at zero; otherwise it walks `/src`, skips `__pycache__`, gathers relative paths and top-level imports, parses `.env`/`.env.example`, loads manifest packages, and records elapsed milliseconds. Edge cases: root None, absent src, unreadable files. The tests supplied with this prompt, however, do not touch `verify_on_read.py`: they cover yarn lock parsing, an interprocess mutex, and ack-impact token rejection, so they cannot catch regressions in this constructor. Their names are:\ntest_yarn_berry_v10_scoped\ntest_interprocess_lock_release_does_not_leak_ownership\ntest_rejects_empty_token\nFrom the given information I cannot confirm any test exercising `_Fingerprint.__init__`; search for tests constructing it directly.", + "tests": [ + "test_yarn_berry_v10_scoped", + "test_interprocess_lock_release_does_not_leak_ownership", + "TestAckImpact::test_rejects_empty_token" + ] + } + } + }, + { + "id": 28, + "name": "GraphRAGQueryEngine.query_dependencies", + "file_path": "D:/Project/MSCodeBase/src/core/graph_rag.py", + "question": "A developer is about to change the behavior of the function `GraphRAGQueryEngine.query_dependencies` in `graph_rag.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "`GraphRAGQueryEngine.query_dependencies` returns dependency information for a file: the file name, `depends_on` (files its symbols call), `depended_by` (files that call its symbols), and an empty `cochange_files` list. It calls `_ensure_graph`, returns the empty skeleton when the symbol index is absent, then inspects up to ten symbols, walking call chains down and up and mapping callee/caller symbols to files via `_symbol_files` while skipping self-references. Inner exceptions are logged per symbol; an outer exception logs a query failure. Results are deduplicated and capped at ten entries each. The `direction` argument is accepted but unused. No repository or test information is supplied here, so I cannot name which existing tests exercise this function.", + "tests": [] + }, + "B_runtime": { + "answer": "`GraphRAGQueryEngine.query_dependencies` builds a dependency report for a file: `depends_on` lists files whose symbols the file calls, `depended_by` lists files calling its symbols, and `cochange_files` is an empty placeholder. It ensures the graph, returns a skeleton when no symbol index exists, then walks up to ten symbols in both call directions, mapping symbols to files through `_symbol_files` and skipping self-references. Inner exceptions are logged per symbol; an outer exception logs a failure. Results are deduplicated and capped at ten each, and `direction` is accepted but unused. The context explicitly identifies the test, so run:\ntest_query_dependencies\nIt only asserts the returned dict contains file, depends_on, and depended_by keys, so stronger assertions for dedup, capping, and direction are advisable.", + "tests": [ + "TestGraphRAGQueryEngine::test_query_dependencies" + ] + }, + "D_shuffled": { + "answer": "`GraphRAGQueryEngine.query_dependencies` returns dependency information for a file: the file name, `depends_on`, `depended_by`, and an empty `cochange_files` list. It ensures the graph, returns a skeleton if the symbol index is absent, then inspects up to ten symbols, walking call chains down and up and mapping callee/caller symbols to files while skipping self-references; results are deduplicated and capped at ten. The test supplied with this prompt does not exercise `query_dependencies`; it belongs to the impact/ack-token guard and checks that a token for a non-existent file still verifies. So from the given information I cannot confirm any test that exercises this function. The provided test is:\ntest_token_missing_file_returns_missing_fingerprint\nA developer should search the suite for tests constructing `GraphRAGQueryEngine` or calling `query_dependencies`.", + "tests": [ + "TestHMACToken::test_token_missing_file_returns_missing_fingerprint" + ] + } + } + }, + { + "id": 29, + "name": "IndexGuard.save_symbol_index", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_guard.py", + "question": "A developer is about to change the behavior of the function `IndexGuard.save_symbol_index` in `index_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "`IndexGuard.save_symbol_index` writes a SymbolIndex to `symbol_index.json` and returns True on success or False on failure. It first applies three anti-corruption guards that skip writing when the instance is a MODE_PURE adapter, when `_definitions` is empty but a graph contains nodes, or when the expected `_definitions`/`_references`/`_file_to_symbols` attributes are missing (graph-backed persistence). It then serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with an empty state, deletes the legacy pickle, and logs exceptions. Edge cases: absent attributes, corrupt existing JSON, legacy file cleanup. No repository or test context is provided, so I cannot name the tests that exercise it.", + "tests": [] + }, + "B_runtime": { + "answer": "`IndexGuard.save_symbol_index` persists a SymbolIndex to `symbol_index.json`, returning True/False. It applies anti-corruption guards that skip writing for MODE_PURE adapters, when `_definitions` is empty but a graph has nodes, or when the expected dict attributes are absent (graph-backed persistence). Otherwise it serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with empty state, removes the legacy pickle, and logs failures. Edge cases: missing attributes, corrupt JSON, legacy cleanup. The context identifies the tests that exercise it, so run:\ntest_symbol_index_persistence\ntest_empty_file_can_be_created_for_fresh_project\ntest_empty_instance_does_not_overwrite_populated_file\nThese cover save/load round-trips, fresh-project creation, and empty-overwrite protection.", + "tests": [ + "TestIndexGuard::test_symbol_index_persistence", + "test_empty_file_can_be_created_for_fresh_project", + "test_empty_instance_does_not_overwrite_populated_file" + ] + }, + "D_shuffled": { + "answer": "`IndexGuard.save_symbol_index` persists a SymbolIndex to `symbol_index.json`, returning True or False. It applies anti-corruption guards that skip writing for MODE_PURE adapters, when `_definitions` is empty but a graph has nodes, or when expected dict attributes are absent; otherwise it serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with empty state, deletes the legacy pickle, and logs exceptions. Edge cases include absent attributes, corrupt existing JSON, and legacy cleanup. The tests supplied with this prompt, however, cover scores-JSON parsing, gitignore directory patterns, and a ToolError, so none of them exercise `save_symbol_index` and none can catch regressions. Their names are:\ntest_parse_scores_json_duplicate_in_wrapper_declined\ntest_dir_pattern_matches_any_depth\ntest_creates_with_message\nFrom the given information I cannot confirm any relevant test.", + "tests": [ + "test_parse_scores_json_duplicate_in_wrapper_declined", + "test_dir_pattern_matches_any_depth", + "TestToolError::test_creates_with_message" + ] + } + } + }, + { + "id": 30, + "name": "RemoteEmbedder._shadow_compare", + "file_path": "D:/Project/MSCodeBase/src/providers/embedder/remote_embedder.py", + "question": "A developer is about to change the behavior of the function `RemoteEmbedder._shadow_compare` in `remote_embedder.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "answer": "I cannot determine this from the information given. The prompt provides only the source code and docstring of `RemoteEmbedder._shadow_compare`; no test files, test names, imports, or repository layout are included. The function calls `self.embed_batch`, the module-level helper `_vectors_collapsed`, and the constant `_ABS_MIN_QUALITY`, and it is invoked during provider switching, so relevant tests could live anywhere in the repo. Naming specific test functions would be a guess and therefore misleading. Explicit statement: cannot determine which tests exercise this function. To answer reliably one would have to search the repository for `_shadow_compare`, `shadow_compare`, canary fixtures, and `_ABS_MIN_QUALITY`, then inspect each hit. A developer should run those located tests before changing the fail-closed logic described here.", + "tests": [] + }, + "B_runtime": { + "answer": "The tests shown that exercise `RemoteEmbedder._shadow_compare` are the canary fail-closed tests: `test_baseline_failure_blocks`, `test_empty_canary_blocks`, and `test_baseline_below_absolute_quality_blocks`. Each builds an `embedder = MagicMock(spec=RemoteEmbedder)` with `_canary_pairs` set, then calls `RemoteEmbedder._shadow_compare(embedder, fn, name)` directly. They cover baseline failure via `embed_batch` raising `RuntimeError` (block), an empty canary set (block), and a degenerate baseline of orthogonal vectors producing `old_mean=0` below the anchor (UNKNOWN, block). All assert `result is False`, confirming fail-closed behavior. A developer changing this function must run these three to catch regressions. Note that other branches such as collapsed vectors, empty new-provider response, and the `degraded_ratio > 0.3` path may have additional tests not shown here.", + "tests": [ + "TestShadowCanary::test_baseline_failure_blocks", + "TestShadowCanary::test_empty_canary_blocks", + "TestShadowCanary::test_baseline_below_absolute_quality_blocks" + ] + }, + "D_shuffled": { + "answer": "The three tests supplied do not exercise `RemoteEmbedder._shadow_compare`. They are `test_build_call_graph_callers`, which covers call-graph traversal; `test_normalize_diag_uri_win_drive`, which covers LSP diagnostic URI normalization; and `test_cli_dispatch_ok`, which covers CLI tool dispatch. None references `_shadow_compare`, `_canary_pairs`, `embed_batch`, `_vectors_collapsed`, or `_ABS_MIN_QUALITY`, so none would catch a regression in this function. Therefore I cannot determine from the information given which tests actually exercise `_shadow_compare`; the listed tests are unrelated and appear to be misleading distractors. Honest answer: cannot determine. A developer should search the repository for `_shadow_compare` and canary fixtures to locate the genuine tests before modifying the function.", + "tests": [ + "test_build_call_graph_callers", + "test_normalize_diag_uri_win_drive", + "test_cli_dispatch_ok" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/experiments/bootstrap/e17_pilot_answers.py b/experiments/bootstrap/e17_pilot_answers.py new file mode 100644 index 00000000..0a4ec956 --- /dev/null +++ b/experiments/bootstrap/e17_pilot_answers.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +"""E17 Pilot: LLM Answer Generation + +Generates answers for each function in the modes present in the pilot data. +Current design (2026-09-22): 3 modes — A (code only), B (specific runtime +TESTS), D (coverage-matched decoy). The old C (static) mode was dropped because +it was never implemented and was identical to A. +""" +import json +import sys +from pathlib import Path +from typing import Dict, List + +import httpx + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code, read_tests # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + + +def build_prompt(question: str, code: str, tests: List[str]) -> str: + """Build prompt for LLM. ``tests`` are names; unresolved ones are skipped.""" + prompt = f"""You are an expert Python developer. Answer the following question about a function in a codebase. + +Question: {question} + +Function code: +```python +{code} +``` +""" + bodies = read_tests(tests) + if bodies: + prompt += "\nRelevant tests that exercise this function:\n" + for body in bodies: + prompt += f"\n```python\n{body}\n```\n" + + prompt += "\nProvide a clear, accurate explanation of what the function does, its purpose, inputs, outputs, and any edge cases it handles." + + return prompt + + +def call_llm(prompt: str, temperature: float = 0.0) -> str: + """Call LLM API (llama.cpp on port 8080).""" + try: + response = httpx.post( + 'http://127.0.0.1:8080/v1/chat/completions', + json={ + 'messages': [ + {'role': 'user', 'content': prompt} + ], + 'temperature': temperature, + 'max_tokens': 1000, + }, + timeout=60.0 + ) + response.raise_for_status() + return response.json()['choices'][0]['message']['content'] + except Exception as e: + return f"# Error calling LLM: {e}" + + +def generate_answers(experiment_data: Dict) -> Dict: + """Generate answers for all questions in all modes.""" + results = { + 'metadata': experiment_data['metadata'], + 'answers': [] + } + + for func_data in experiment_data['functions']: + print(f"\nProcessing {func_data['name']}...") + + # Read function code + code = read_function_code(func_data['file_path'], func_data['name']) + if not code: + print(f" SKIP: could not extract source for {func_data['name']}") + continue + question = func_data['question'] + + func_result = { + 'id': func_data['id'], + 'name': func_data['name'], + 'file_path': func_data['file_path'], + 'question': question, + 'modes': {} + } + + for mode_name, mode_data in func_data['modes'].items(): + tests = mode_data['tests'] + print(f" {mode_name} ({len(tests)} tests)...") + prompt = build_prompt(question, code, tests) + answer = call_llm(prompt) + func_result['modes'][mode_name] = { + 'prompt': prompt, + 'answer': answer, + 'tests': tests, + } + + results['answers'].append(func_result) + + return results + + +def main(): + print("=" * 80) + print("E17 PILOT: LLM ANSWER GENERATION") + print("=" * 80) + + # Load experiment data + data_path = Path(__file__).parent / "e17_pilot_data.json" + with open(data_path, 'r', encoding='utf-8') as f: + experiment_data = json.load(f) + + print(f"Loaded {len(experiment_data['functions'])} functions") + + # Generate answers + results = generate_answers(experiment_data) + + # Save results + output_path = Path(__file__).parent / "e17_pilot_answers.json" + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + print(f"\n{'=' * 80}") + print(f"Answers saved to: {output_path}") + print(f"Total answers: {len(results['answers'])} × 4 modes = {len(results['answers']) * 4}") + print(f"{'=' * 80}") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_pilot_data.json b/experiments/bootstrap/e17_pilot_data.json new file mode 100644 index 00000000..f4e12910 --- /dev/null +++ b/experiments/bootstrap/e17_pilot_data.json @@ -0,0 +1,896 @@ +{ + "functions": [ + { + "id": 1, + "name": "IndexProjectRunner.run", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_project_runner.py", + "n_tests": 3, + "min_cov": 4, + "question": "A developer is about to change the behavior of the function `IndexProjectRunner.run` in `index_project_runner.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_run_survives_single_cpu_host", + "test_run_with_existing_known_hashes_skips_unchanged", + "test_workers_receive_known_hashes_not_none" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestClassifyHolder::test_pid_reuse_is_stale", + "TestClassifyHolder::test_dead_pid_is_stale", + "test_logs_dir_inside_data_root" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 2, + "name": "RemoteEmbedder.embed_batch", + "file_path": "D:/Project/MSCodeBase/src/providers/embedder/remote_embedder.py", + "n_tests": 2, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `RemoteEmbedder.embed_batch` in `remote_embedder.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestConcurrentEmbedIsolation::test_rapid_sequential_calls_no_state_leak", + "TestConcurrentEmbedIsolation::test_single_call_results_match_indices" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestResourceMonitor::test_sample_disk_io_no_data_row_returns_zero", + "test_verdict_fail_is_refuted" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 3, + "name": "CypherLexer.tokenize", + "file_path": "D:/Project/MSCodeBase/src/core/search/cypher_lexer.py", + "n_tests": 95, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `CypherLexer.tokenize` in `cypher_lexer.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestCypherLexer::test_tokenize_optional_match", + "TestCypherLexer::test_tokenize_order_limit", + "TestCypherLexer::test_tokenize_relationship" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_gemfile_lock_skips_path_project_gem", + "TestValidateCode::test_blocked_name_getattr", + "test_load_invalid_engine_req" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 4, + "name": "AutoDocUpdater._extract_doc_references", + "file_path": "D:/Project/MSCodeBase/src/core/auto_doc_updater.py", + "n_tests": 1, + "min_cov": 4, + "question": "A developer is about to change the behavior of the function `AutoDocUpdater._extract_doc_references` in `auto_doc_updater.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_extract_doc_references_handles_empty_call_syntax" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_symbol_index_stores_and_retrieves" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 5, + "name": "IndexPipeline.process_file", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_pipeline.py", + "n_tests": 6, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `IndexPipeline.process_file` in `index_pipeline.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_chunk_cache_disabled_without_table", + "test_chunk_cache_empty_file", + "test_chunk_cache_all_new" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_ensure_server_running_launches_when_dead", + "test_impact_analysis_basic", + "TestEdgeCases::test_empty_file" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 6, + "name": "extract_anchors", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/verify_on_read.py", + "n_tests": 79, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `extract_anchors` in `verify_on_read.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_extract_anchors_no_anchors", + "test_extract_anchors_explicit_pkg_syntax", + "test_extract_anchors_from_data_anchors" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_format_diagnostics_empty", + "TestAckImpact::test_stale_token_rejected_after_file_change", + "test_lru_cap_evicts_oldest" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 7, + "name": "resolve_project_root", + "file_path": "D:/Project/MSCodeBase/src/core/project_resolution.py", + "n_tests": 35, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `resolve_project_root` in `project_resolution.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestCwdFirstIsolation::test_provided_wins_over_cwd", + "TestCwdSelfIndexGuard::test_cwd_equal_ext_root_is_skipped", + "TestPriorityProjectPathOverride::test_mscb_path_trusts_ext_root" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_flag_on_when_env_set", + "TestValidateCode::test_blocked_name_globals", + "TestGraphRAGQueryEngine::test_query_hotspots_no_memory" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 8, + "name": "SymbolIndex.build_call_graph", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/symbol_index.py", + "n_tests": 17, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `SymbolIndex.build_call_graph` in `symbol_index.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_build_call_graph_call_chain", + "test_build_call_graph_callees", + "test_build_call_graph_callers" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_resolver_auto_approve", + "test_sha256_file_missing", + "TestGrepFallbackProjectRoot::test_does_not_scan_extension_dir" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 9, + "name": "IndexParser.parse_file", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_parser.py", + "n_tests": 2, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `IndexParser.parse_file` in `index_parser.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_bs1_parse_file_skips_comment_only", + "TestASTCacheInvalidation::test_property_graph_consistency" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_vor_coverage_starved_warning", + "TestWriteToolInsertAfter::test_inserts_after_anchor" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 10, + "name": "modification_guard", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "n_tests": 11, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `modification_guard` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0", + "TestModificationGuard::test_allows_non_hot_file" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestCrossProjectDependencyGraphInit::test_init_with_registry", + "test_symbol_index_remove_file_clears_env", + "test_parse_rejects_bad_urls" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 11, + "name": "decorator", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "n_tests": 11, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `decorator` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "test_cli_unknown_tool", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_empty_instance_does_not_overwrite_populated_file", + "TestCheckSearchQuality::test_passes_on_real_results", + "test_build_receipt_generates_id_and_steps" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 12, + "name": "wrapper", + "file_path": "D:/Project/MSCodeBase/src/core/modification_guard.py", + "n_tests": 11, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `wrapper` in `modification_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestModificationGuard::test_no_file_path_no_symbol_passes", + "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0", + "TestModificationGuard::test_allows_non_hot_file" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestBuildStartupReport::test_self_lock_adds_no_second_instance_issue", + "test_fileguard_resolves_relative_project_path", + "test_trust_report_warns_on_untrusted_with_instructions" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 13, + "name": "format_project_memory", + "file_path": "D:/Project/MSCodeBase/src/utils/ui_formatter.py", + "n_tests": 14, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `format_project_memory` in `ui_formatter.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_audit_mode_shows_statuses_and_reasons", + "test_budget_exceeded_node_marker", + "test_default_limit_three" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_verdict_env_block_is_inconclusive_not_refuted", + "TestApplyFileMove::test_invalidates_bm25", + "TestValidateCode::test_blocked_string_pattern_eval" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 14, + "name": "validate_code", + "file_path": "D:/Project/MSCodeBase/src/core/sandbox/executor.py", + "n_tests": 41, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `validate_code` in `executor.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestValidateCode::test_allowed_modules", + "TestValidateCode::test_empty_code", + "TestValidateCode::test_safe_code_passes" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_enrichment_empty_results", + "TestInferMetadata::test_layer_windows_backslashes", + "test_build_call_graph_depth_2" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 15, + "name": "_extract_yarn_lock", + "file_path": "D:/Project/MSCodeBase/src/sources/manifest/extract.py", + "n_tests": 5, + "min_cov": 5, + "question": "A developer is about to change the behavior of the function `_extract_yarn_lock` in `extract.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_yarn_berry_patch_descriptors", + "test_yarn_berry_v10_scoped", + "test_yarn_synthetic" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestProjectHeader::test_returns_project_path", + "TestChangePreview::test_worktree_cleaned_after_run", + "TestProjectHeader::test_metadata_returns_dict" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 16, + "name": "WriteTool._action_replace", + "file_path": "D:/Project/MSCodeBase/src/mcp/tools/write_tools.py", + "n_tests": 9, + "min_cov": 4, + "question": "A developer is about to change the behavior of the function `WriteTool._action_replace` in `write_tools.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestWriteToolReplace::test_unknown_symbol_returns_warning", + "TestWriteToolReplace::test_filter_mismatch_returns_error", + "TestDryRunNoWrite::test_replace_dry_run_does_not_write" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestClassifyHolder::test_own_host_v2_is_healthy", + "test_drop_removes_entry", + "TestDebounceBatch::test_flush_now_clears_and_calls_callback" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 17, + "name": "SymbolIndex.get_call_chain", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/symbol_index.py", + "n_tests": 4, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `SymbolIndex.get_call_chain` in `symbol_index.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_get_call_chain_both", + "test_get_call_chain_down", + "test_get_call_chain_total_connected" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_unparseable_lines_skipped", + "TestSqliteSchemaHealth::test_old_schema_detected", + "test_wire_plugins_no_env_noop" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 18, + "name": "load_manifest", + "file_path": "D:/Project/MSCodeBase/src/plugins/manifest.py", + "n_tests": 27, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `load_manifest` in `manifest.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_load_invalid_engine_req", + "test_manifest_requires_missing_fields", + "test_manifest_schema_mismatch_real" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_tokenize_truncates_over_limit_to_guarantee", + "TestIsSelfIndexPath::test_user_project_not_self_index", + "TestExtractKeyTerms::test_empty_results" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 19, + "name": "ProjectIntelligenceLayer._enrich_job_response", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/layer.py", + "n_tests": 3, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `ProjectIntelligenceLayer._enrich_job_response` in `layer.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_enrich_eta_none_when_no_embed_progress", + "test_enrich_eta_uses_real_embed_speed", + "test_intel_runtime_status_reports_reindex_progress" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_normalize_diag_uri_idempotent", + "test_double_load_is_idempotent", + "TestWriteToolSafeDelete::test_apply_deletes_from_file" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 20, + "name": "format_runtime_status", + "file_path": "D:/Project/MSCodeBase/src/utils/ui_formatter.py", + "n_tests": 5, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `format_runtime_status` in `ui_formatter.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_format_runtime_status_fallback_to_768_when_no_model_info", + "test_format_runtime_status_normal_path_unchanged", + "test_format_runtime_status_reindex_without_progress" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestDisabledByDefault::test_env_gate_off", + "TestValidateCode::test_blocked_import_subprocess", + "TestGenerateRefinedQuery::test_iteration_1_adds_terms" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 21, + "name": "_delayed_auto_index", + "file_path": "D:/Project/MSCodeBase/src/mcp/server_factory.py", + "n_tests": 2, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `_delayed_auto_index` in `server_factory.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_delayed_auto_index_skips_when_index_not_empty", + "test_delayed_auto_index_sets_state_ready" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestCrossProjectDependencyGraphInit::test_lazy_graph_build", + "TestExtractImports::test_dedup_preserves_order" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 22, + "name": "load_metrics", + "file_path": "D:/Project/MSCodeBase/src/core/error_handler.py", + "n_tests": 2, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `load_metrics` in `error_handler.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_bs14_load_metrics_clamps_negative_total_ms", + "test_bs14_load_metrics_sanitizes_negative" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestParseCrossRepoQuery::test_multiple_mentions", + "test_ips_are_global" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 23, + "name": "_extract_pyproject", + "file_path": "D:/Project/MSCodeBase/src/sources/manifest/extract.py", + "n_tests": 10, + "min_cov": 5, + "question": "A developer is about to change the behavior of the function `_extract_pyproject` in `extract.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_pyproject_optional_and_groups", + "test_requests_pyproject_and_requirements_dev", + "test_manifest_packages_wiring_pyproject_plus_lock" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_localhost_https_rejected", + "test_search_with_mode_fast_calls_embed_and_vector", + "TestEdgeCases::test_attribute_assignment" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 24, + "name": "IndexStatusReporter.get_status", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_status.py", + "n_tests": 2, + "min_cov": 2, + "question": "A developer is about to change the behavior of the function `IndexStatusReporter.get_status` in `index_status.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_get_status_fast_fail_during_reindex_does_not_block", + "test_setup" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestMoveChunksMetadata::test_handles_same_path", + "test_no_dollardefs_or_ref_in_tool_schemas" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 25, + "name": "WriteTool._action_insert", + "file_path": "D:/Project/MSCodeBase/src/mcp/tools/write_tools.py", + "n_tests": 13, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `WriteTool._action_insert` in `write_tools.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestWriteToolInsertAfter::test_filter_mismatch_returns_error", + "TestWriteToolInsertBefore::test_filter_mismatch_returns_error", + "TestWriteToolInsertAfter::test_unknown_symbol_returns_error" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestHMACToken::test_token_is_32_hex_chars", + "test_unknown_kind_ignored", + "test_gemfile_skips_gitpath" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 26, + "name": "_match_gitignore_pattern", + "file_path": "D:/Project/MSCodeBase/src/core/gitignore_parser.py", + "n_tests": 13, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `_match_gitignore_pattern` in `gitignore_parser.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_dir_pattern_excludes_nested", + "test_dir_pattern_matches_any_depth", + "test_file_pattern_unchanged" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestFindSharedInterfaces::test_no_shared", + "TestGetDependencyPath::test_no_path", + "TestWriteToolFileGuard::test_non_ascii_path_denied" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 27, + "name": "_Fingerprint.__init__", + "file_path": "D:/Project/MSCodeBase/src/core/intelligence/verify_on_read.py", + "n_tests": 84, + "min_cov": 5, + "question": "A developer is about to change the behavior of the function `_Fingerprint.__init__` in `verify_on_read.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "test_extract_anchors_explicit_pkg_syntax", + "test_extract_anchors_from_data_anchors", + "test_extract_anchors_read_path_backward_compat" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_yarn_berry_v10_scoped", + "test_interprocess_lock_release_does_not_leak_ownership", + "TestAckImpact::test_rejects_empty_token" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 28, + "name": "GraphRAGQueryEngine.query_dependencies", + "file_path": "D:/Project/MSCodeBase/src/core/graph_rag.py", + "n_tests": 1, + "min_cov": 4, + "question": "A developer is about to change the behavior of the function `GraphRAGQueryEngine.query_dependencies` in `graph_rag.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestGraphRAGQueryEngine::test_query_dependencies" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "TestHMACToken::test_token_missing_file_returns_missing_fingerprint" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 29, + "name": "IndexGuard.save_symbol_index", + "file_path": "D:/Project/MSCodeBase/src/core/indexing/index_guard.py", + "n_tests": 8, + "min_cov": 3, + "question": "A developer is about to change the behavior of the function `IndexGuard.save_symbol_index` in `index_guard.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestIndexGuard::test_symbol_index_persistence", + "test_empty_file_can_be_created_for_fresh_project", + "test_empty_instance_does_not_overwrite_populated_file" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_parse_scores_json_duplicate_in_wrapper_declined", + "test_dir_pattern_matches_any_depth", + "TestToolError::test_creates_with_message" + ], + "description": "Coverage-matched decoy" + } + } + }, + { + "id": 30, + "name": "RemoteEmbedder._shadow_compare", + "file_path": "D:/Project/MSCodeBase/src/providers/embedder/remote_embedder.py", + "n_tests": 7, + "min_cov": 1, + "question": "A developer is about to change the behavior of the function `RemoteEmbedder._shadow_compare` in `remote_embedder.py`. Which existing tests in this repository must be run to catch regressions, i.e. which tests exercise this function? List the test function names explicitly (one per line). If you cannot determine this from the information given, say so explicitly.", + "modes": { + "A_baseline": { + "tests": [], + "description": "Code only (no tests)" + }, + "B_runtime": { + "tests": [ + "TestShadowCanary::test_baseline_failure_blocks", + "TestShadowCanary::test_empty_canary_blocks", + "TestShadowCanary::test_baseline_below_absolute_quality_blocks" + ], + "description": "Most specific runtime TESTS" + }, + "D_shuffled": { + "tests": [ + "test_build_call_graph_callers", + "test_normalize_diag_uri_win_drive", + "test_cli_dispatch_ok" + ], + "description": "Coverage-matched decoy" + } + } + } + ], + "metadata": { + "n_functions": 30, + "n_total_tests": 1461, + "selection": "min_test_coverage<=5, max 3 per file", + "modes": [ + "A_baseline", + "B_runtime", + "D_shuffled" + ] + } +} \ No newline at end of file diff --git a/experiments/bootstrap/e17_pilot_experiment.py b/experiments/bootstrap/e17_pilot_experiment.py new file mode 100644 index 00000000..8df86a76 --- /dev/null +++ b/experiments/bootstrap/e17_pilot_experiment.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +"""E17 Pilot Experiment: Does TESTS evidence help an LLM understand code? + +Research question: +Does execution-derived test-to-function evidence help an LLM answer repository +questions better than code-only context or an equally-specific but WRONG test +context? + +Design (corrected 2026-09-22): +- 3 modes: A (code only), B (specific runtime TESTS), D (coverage-matched decoy). + Mode C (static) is DROPPED: it was never implemented and returned [] → C ≡ A. +- Panel selected by TEST SPECIFICITY, not by edge count. The original script did + `ORDER BY test_count DESC`, which picked the *most transitively covered* + functions (234 edges) where a 3-test sample is pure noise. + Specificity proxy: a test's coverage = number of functions it executes + (count of its outgoing TESTS edges). Function score = min coverage of its tests. +- B = the 3 most specific tests of the function (smallest coverage first). +- D = matched decoy: for each B test (coverage c) pick a test from a DIFFERENT + function with the closest coverage, not belonging to this function. +- Paired design, 1 question per function, judge: 11 trials + real position swap. +""" +import sys +import ast +import json +import sqlite3 +import random +import re +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from src.core.artifact_paths import get_graph_db_path # noqa: E402 +from e17_extract import read_function_code, read_test_code # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + +COVERAGE_SQL = """ +SELECT source_id, COUNT(*) AS cov FROM edges WHERE type = 'TESTS' GROUP BY source_id +""" + +_BRANCH = (ast.If, ast.For, ast.While, ast.Try, ast.With, ast.BoolOp, ast.Match) +MIN_BRANCHES = 10 + + +def complexity(code: str) -> Tuple[int, int]: + """(branch nodes, non-blank lines) — a rough difficulty proxy.""" + try: + tree = ast.parse(code) + except SyntaxError: + return 0, 0 + loc = sum(1 for ln in code.splitlines() if ln.strip()) + branches = sum(1 for n in ast.walk(tree) if isinstance(n, _BRANCH)) + return branches, loc + + +def _connect() -> sqlite3.Connection: + return sqlite3.connect(str(get_graph_db_path(ROOT))) + + +def collect_candidates(max_cov: int = 5) -> List[Dict]: + """Functions with at least one test of coverage <= max_cov, in src/.""" + conn = _connect() + rows = conn.execute( + f""" + SELECT f.name, f.file_path, f.qualified_name, COUNT(*) AS n_tests, MIN(tc.cov) AS min_cov + FROM nodes f + JOIN edges e ON e.target_id = f.id AND e.type = 'TESTS' + JOIN ({COVERAGE_SQL}) tc ON tc.source_id = e.source_id + WHERE f.label = 'Function' AND f.file_path LIKE '%/src/%' + GROUP BY f.id + HAVING min_cov <= ? + ORDER BY min_cov ASC, n_tests ASC + """, + (max_cov,), + ).fetchall() + conn.close() + out = [] + for name, file_path, qname, n_tests, min_cov in rows: + if name == "_" or name.startswith("test_"): + continue + out.append({ + "name": name, + "file_path": file_path, + "qualified_name": qname, + "n_tests": n_tests, + "min_cov": min_cov, + }) + return out + + +def select_panel(candidates: List[Dict], n: int = 30, per_file: int = 3) -> List[Dict]: + """Diversify: at most ``per_file`` functions from the same file.""" + panel: List[Dict] = [] + per_file_count: Dict[str, int] = {} + for cand in candidates: + if per_file_count.get(cand["file_path"], 0) >= per_file: + continue + panel.append(cand) + per_file_count[cand["file_path"]] = per_file_count.get(cand["file_path"], 0) + 1 + if len(panel) >= n: + break + return panel + + +def specific_tests(conn: sqlite3.Connection, function_name: str, limit: int = 3) -> List[Tuple[str, int]]: + """Return (test_name, coverage) for the function's tests, most specific first.""" + rows = conn.execute( + f""" + SELECT t.name, tc.cov + FROM nodes f + JOIN edges e ON e.target_id = f.id AND e.type = 'TESTS' + JOIN nodes t ON t.id = e.source_id + JOIN ({COVERAGE_SQL}) tc ON tc.source_id = e.source_id + WHERE f.name = ? + ORDER BY tc.cov ASC, t.name ASC + """, + (function_name,), + ).fetchall() + return [(name, cov) for name, cov in rows[:limit]] + + +def build_test_pool(conn: sqlite3.Connection) -> List[Dict]: + """All test nodes with coverage and the set of source files they cover.""" + cov = dict(conn.execute( + f"SELECT source_id, COUNT(*) FROM edges WHERE type = 'TESTS' GROUP BY source_id" + ).fetchall()) + files: Dict[int, set] = {} + for sid, fp in conn.execute( + "SELECT e.source_id, n.file_path FROM edges e JOIN nodes n ON n.id = e.target_id " + "WHERE e.type = 'TESTS'" + ): + files.setdefault(sid, set()).add(fp) + names = dict(conn.execute("SELECT id, name FROM nodes WHERE label = 'Test'")) + return [ + {"name": name, "cov": cov[tid], "files": files.get(tid, set())} + for tid, name in names.items() + if tid in cov + ] + + +def matched_decoys( + panel: List[Dict], + b_pairs_list: List[List[Tuple[str, int]]], + pool: List[Dict], +) -> List[List[str]]: + """Unique, foreign, coverage-matched decoys. + + For each B test (coverage c) pick a test that: + - is not used by any other function (global uniqueness), + - covers no function in the target's source file (foreign module), + - has coverage closest to c (matched difficulty). + """ + shuffled = pool[:] + random.shuffle(shuffled) + used: set = set() + decoys: List[List[str]] = [] + for func, b_pairs in zip(panel, b_pairs_list): + ffile = func["file_path"] + d: List[str] = [] + for _, bcov in b_pairs: + cands = [ + t for t in shuffled + if t["name"] not in used and ffile not in t["files"] + ] + if not cands: + break + cands.sort(key=lambda t: abs(t["cov"] - bcov)) + pick = cands[0] + d.append(pick["name"]) + used.add(pick["name"]) + decoys.append(d) + return decoys + + +_STR_NAMES = { + "query", "text", "name", "symbol", "content", "code", "s", "uri", "line", "msg", + "message", "doc", "snippet", "source", "what", "key", "value", "token", "pattern", +} +_PATH_NAMES = { + "path", "root", "dir", "directory", "db_path", "file", "fp", "filepath", "filename", +} +_INT_NAMES = { + "n", "limit", "depth", "count", "size", "index", "max_depth", "max_count", "timeout", + "pid", +} +_EDGE_TOKENS = [ + ("None", "None"), + ('""', "an empty string"), + ("''", "an empty string"), + ("[]", "an empty list"), + ("{}", "an empty dict"), + ("-1", "the value -1"), + ("0", "the value 0"), +] + + +def signature_edge(code: str) -> Optional[Tuple[str, str, Tuple[str, ...]]]: + """(description, probe tokens) for the most relevant parameter of the function.""" + try: + tree = ast.parse(code) + except SyntaxError: + return None + node = next( + (n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))), + None, + ) + if node is None: + return None + args = [a for a in node.args.args if a.arg not in ("self", "cls")] + for p in args: + ann = ast.unparse(p.annotation) if p.annotation else "" + nm = p.arg.lower() + if "Optional" in ann or "None" in ann: + return ("None", ("None",)) + if "Path" in ann or nm in _PATH_NAMES: + return ("a path that does not exist", ("path", "missing", "nonexistent", "not_exist")) + if any(k in ann.lower() for k in ("list", "dict", "iterable", "set", "tuple")): + return ("an empty collection", ("[]", "{}", "empty")) + if "int" in ann or nm in _INT_NAMES: + return ("the value -1", ("-1",)) + if "str" in ann or nm in _STR_NAMES: + return ("an empty string", ('""', "''", "empty")) + if "bool" in ann: + return ("None", ("None",)) + return None + + +def choose_edge(code: str, b_tests: List[str]) -> Optional[str]: + """Prefer a signature-appropriate edge input that the B tests exercise.""" + text = "\n".join(read_test_code(t) or "" for t in b_tests) + sig = signature_edge(code) + if sig is not None: + desc, probes = sig + if any(pr in text for pr in probes): + return desc + for token, desc in _EDGE_TOKENS: + if re.search(r"(? str: + """Impact/coverage: which tests exercise this function (the signal's own answer).""" + return ( + f"A developer is about to change the behavior of the function `{function_name}` in " + f"`{Path(file_path).name}`. Which existing tests in this repository must be run to catch " + "regressions, i.e. which tests exercise this function? List the test function names " + "explicitly (one per line). If you cannot determine this from the information given, " + "say so explicitly." + ) + + +def prepare_experiment_data(n_functions: int = 30, max_cov: int = 5) -> Dict: + print(f"Collecting candidates with min test coverage <= {max_cov}...") + candidates = collect_candidates(max_cov) + print(f" candidates: {len(candidates)}") + + for c in candidates: + code = read_function_code(c["file_path"], c["name"]) or "" + c["branches"], c["loc"] = complexity(code) + hard = [c for c in candidates if c["branches"] >= MIN_BRANCHES] + hard.sort(key=lambda c: (-c["branches"], -c["loc"])) + print(f" hard candidates (branches >= {MIN_BRANCHES}): {len(hard)}") + + panel = select_panel(hard, n_functions) + print(f" panel: {len(panel)} functions across " + f"{len({p['file_path'] for p in panel})} files") + + conn = _connect() + all_tests: List[Tuple[str, int]] = conn.execute( + f""" + SELECT t.name, tc.cov + FROM nodes t + JOIN ({COVERAGE_SQL}) tc ON tc.source_id = t.id + WHERE t.label = 'Test' + """ + ).fetchall() + print(f" total test nodes with coverage: {len(all_tests)}") + + b_pairs_list = [specific_tests(conn, func["name"], limit=3) for func in panel] + b_sets = [[name for name, _ in pairs] for pairs in b_pairs_list] + pool = build_test_pool(conn) + d_sets = matched_decoys(panel, b_pairs_list, pool) + + functions = [] + for i, func in enumerate(panel, 1): + b_tests = b_sets[i - 1] + d_tests = d_sets[i - 1] + code = read_function_code(func["file_path"], func["name"]) or "" + functions.append({ + "id": i, + "name": func["name"], + "file_path": func["file_path"], + "n_tests": func["n_tests"], + "min_cov": func["min_cov"], + "question": generate_question( + func["name"], func["file_path"], choose_edge(code, b_tests) + ), + "modes": { + "A_baseline": {"tests": [], "description": "Code only (no tests)"}, + "B_runtime": {"tests": b_tests, "description": "Most specific runtime TESTS"}, + "D_shuffled": {"tests": d_tests, "description": "Coverage-matched decoy"}, + }, + }) + print(f"[{i}/{len(panel)}] {func['name']} (branches={func.get('branches')}, " + f"loc={func.get('loc')}, min_cov={func['min_cov']}) B={b_tests} D={d_tests}") + + conn.close() + return { + "functions": functions, + "metadata": { + "n_functions": len(functions), + "n_total_tests": len(all_tests), + "selection": f"min_test_coverage<={max_cov}, max 3 per file", + "modes": ["A_baseline", "B_runtime", "D_shuffled"], + }, + } + + +def main(): + print("=" * 80) + print("E17 PILOT EXPERIMENT (specificity-selected)") + print("=" * 80) + random.seed(17) + data = prepare_experiment_data(n_functions=30, max_cov=5) + + output_path = Path(__file__).parent / "e17_pilot_data.json" + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + print("=" * 80) + print(f"Experiment data saved to: {output_path}") + print(f"Total functions: {data['metadata']['n_functions']}") + print(f"Total tests in graph: {data['metadata']['n_total_tests']}") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_pilot_judge.py b/experiments/bootstrap/e17_pilot_judge.py new file mode 100644 index 00000000..9bf22bd6 --- /dev/null +++ b/experiments/bootstrap/e17_pilot_judge.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +"""E17 Pilot: Judge Evaluation + +Evaluates answers using LLM-as-judge with: +- 4 metrics: accuracy, completeness, safety, evidence_usage +- Position randomization (tests block before/after the answer) +- Multiple trials for stability +""" +import json +import random +import sys +from collections import defaultdict +from pathlib import Path +from typing import Dict, List + +import httpx + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code, read_tests # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + + +JUDGE_PROMPT = """You are an expert code reviewer. Evaluate the following answer about a Python function. + +Question: {question} + +Function code: +```python +{code} +``` + +{first_block} +{second_block} +Rate the answer on these metrics (1-5 scale): + +1. **Accuracy**: How factually correct is the answer? Does it accurately describe what the function does? + 1 = Completely wrong + 2 = Mostly wrong with some correct elements + 3 = Partially correct, significant errors + 4 = Mostly correct, minor errors + 5 = Completely accurate + +2. **Completeness**: How thoroughly does the answer cover the function's purpose, inputs, outputs, and edge cases? + 1 = Very incomplete, missing key information + 2 = Incomplete, missing important aspects + 3 = Moderately complete + 4 = Mostly complete + 5 = Very comprehensive + +3. **Safety**: Does the answer avoid suggesting unsafe practices or misunderstandings? + 1 = Suggests dangerous practices + 2 = Contains unsafe recommendations + 3 = Neutral, neither safe nor unsafe + 4 = Generally safe + 5 = Very safe, no concerns + +4. **Evidence Usage**: Did the answer reference or use the provided tests to inform the explanation? + 1 = Completely ignored the tests + 2 = Mentioned tests but didn't use them + 3 = Partially used tests + 4 = Used tests effectively + 5 = Explicitly referenced tests and used them to explain behavior + +Provide your evaluation in this exact JSON format: +{{"accuracy": <1-5>, "completeness": <1-5>, "safety": <1-5>, "evidence_usage": <1-5>}} +""" + + +def call_judge(prompt: str, temperature: float = 0.0) -> Dict[str, int]: + """Call LLM judge and parse scores.""" + try: + response = httpx.post( + 'http://127.0.0.1:8080/v1/chat/completions', + json={ + 'messages': [ + {'role': 'user', 'content': prompt} + ], + 'temperature': temperature, + 'max_tokens': 200, + }, + timeout=30.0 + ) + response.raise_for_status() + content = response.json()['choices'][0]['message']['content'] + + # Parse JSON from response + # Find JSON block + import re + json_match = re.search(r'\{[^}]+\}', content) + if json_match: + scores = json.loads(json_match.group()) + return { + 'accuracy': scores.get('accuracy', 3), + 'completeness': scores.get('completeness', 3), + 'safety': scores.get('safety', 3), + 'evidence_usage': scores.get('evidence_usage', 3), + } + else: + return {'accuracy': 3, 'completeness': 3, 'safety': 3, 'evidence_usage': 3} + except Exception as e: + print(f" Judge error: {e}") + return {'accuracy': 3, 'completeness': 3, 'safety': 3, 'evidence_usage': 3} + + +def build_judge_prompt( + question: str, + code: str, + answer: str, + tests: List[str], + tests_first: bool, +) -> str: + """Build judge prompt; ``tests_first`` controls the block order.""" + bodies = read_tests(tests) + tests_section = "" + if bodies: + tests_section = "Relevant tests:\n" + for body in bodies: + tests_section += f"\n```python\n{body}\n```\n" + + answer_block = f"Answer to evaluate:\n{answer}\n" + if tests_section and tests_first: + first_block, second_block = tests_section, answer_block + else: + first_block, second_block = answer_block, tests_section + + return JUDGE_PROMPT.format( + question=question, + code=code, + first_block=first_block, + second_block=second_block, + ) + + +def judge_single_answer( + question: str, + code: str, + answer: str, + tests: List[str], + n_trials: int = 11 +) -> Dict[str, List[int]]: + """Judge a single answer with multiple trials and randomized block order.""" + all_scores = defaultdict(list) + + for trial in range(n_trials): + tests_first = random.random() < 0.5 + prompt = build_judge_prompt(question, code, answer, tests, tests_first) + scores = call_judge(prompt) + for metric, score in scores.items(): + all_scores[metric].append(score) + + return dict(all_scores) + + +def evaluate_all_answers(answers_data: Dict, n_trials: int = 11) -> Dict: + """Evaluate all answers.""" + results = { + 'metadata': answers_data['metadata'], + 'evaluations': [] + } + + for answer_data in answers_data['answers']: + print(f"\nEvaluating {answer_data['name']}...") + + code = read_function_code(answer_data['file_path'], answer_data['name']) or "" + question = answer_data['question'] + + func_eval = { + 'id': answer_data['id'], + 'name': answer_data['name'], + 'evaluations': {} + } + + for mode_name, mode_data in answer_data['modes'].items(): + print(f" {mode_name}...") + answer = mode_data['answer'] + tests = mode_data['tests'] + + scores = judge_single_answer(question, code, answer, tests, n_trials) + + # Calculate mean and std + eval_result = {} + for metric, values in scores.items(): + mean_val = sum(values) / len(values) + eval_result[metric] = { + 'mean': round(mean_val, 2), + 'values': values + } + print(f" {metric}: {mean_val:.2f} (std: {(sum((v - mean_val)**2 for v in values) / len(values))**0.5:.2f})") + + func_eval['evaluations'][mode_name] = eval_result + + results['evaluations'].append(func_eval) + + return results + + +def main(): + print("=" * 80) + print("E17 PILOT: JUDGE EVALUATION") + print("=" * 80) + + # Load answers + answers_path = Path(__file__).parent / "e17_pilot_answers.json" + with open(answers_path, 'r', encoding='utf-8') as f: + answers_data = json.load(f) + + print(f"Loaded {len(answers_data['answers'])} answers") + + # Evaluate + results = evaluate_all_answers(answers_data, n_trials=11) + + # Save results + output_path = Path(__file__).parent / "e17_pilot_evaluations.json" + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + print(f"\n{'=' * 80}") + print(f"Evaluations saved to: {output_path}") + print(f"{'=' * 80}") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_pilot_report.md b/experiments/bootstrap/e17_pilot_report.md new file mode 100644 index 00000000..8b3a670f --- /dev/null +++ b/experiments/bootstrap/e17_pilot_report.md @@ -0,0 +1,121 @@ +# E17 Pilot Report — TESTS-signal: where it helps and where it does not + +**Date:** 2026-09-22 +**Question:** what does the execution-derived test-to-function signal (TESTS edges) buy an LLM? +**Design:** paired, 3 arms — A (code only), B (specific runtime TESTS), D (coverage-matched decoy). +**Panel:** 30 functions, each with a specific test (min per-test coverage <= 5). +**Answers:** 90 (30 x 3) per task, generated by isolated subagents (no repo access). +**Judge:** for reasoning tasks — blind, test-hidden, identical reference all arms. For impact — objective +recall/precision vs the signal's own answer (no judge). + +## Task matrix and results + +| # | task | metric | A | B | D | verdict | +|---|---|---|---|---|---|---| +| v1 | explain the function | accuracy (judge) | 5.00 | 4.97 | 5.00 | ceiling | +| v2 | edge-case prediction | accuracy (blind judge) | 4.87 | 4.87 | 4.90 | ceiling | +| v3 | edge-case, complex panel | accuracy (blind judge) | 4.93 | 4.77 | 4.83 | ceiling (B worse, p=0.057) | +| **v4** | **impact: which tests cover F** | **recall** | **0.000** | **1.000** | **0.000** | **B decisive (30/0/0)** | +| v4 | impact | precision | 0.000 | 1.000 | 0.000 | B decisive (30/0/0) | + +### v4 detail (n=30 paired) +- recall: B-A = +1.000, B-D = +1.000, W/L/T = 30/0/0. +- precision: identical. +- **name-convention leak: A guessed a real covering test for 0/30 functions** — even the + `test_` naming heuristic does not leak the answer on this panel. +- So without the signal the model names *zero* covering tests; with it, it names them exactly. + +### v1-v3 detail (reasoning) +Manipulation checks held throughout: answers referenced their provided tests 43-57% (B) vs +7-10% (D). The null is "tests read but not needed", not "tests ignored". v1's apparent +completeness gain (B>A p=0.0046) was a judge-context artifact and vanished once the judge +used an identical, test-hidden reference. + +### v5 — mutation validation: does the signal identify *verifying* tests? + +For each of 10 panel functions the first single-line `if` condition was negated (strong mutation), +then the exact pytest node ids for arm B (graph tests) and arm D (decoy) were run; the mutated file +was restored with `git checkout --` in a finally block. Script: `e17_mutation_validate.py`, +raw results: `e17_mutation_results.json`. + +- B (graph tests) killed the mutant: **6/10** +- D (decoy tests) killed the mutant: **0/10** +- Fisher exact (2x2 `[[6,4],[0,10]]`): odds=inf, **p=0.011** + +So the TESTS edges point at tests that detect behaviour changes far more often than decoys — the +signal **can drive verification**. But **4/10 linked tests did not catch a strong mutation**. Example: +`TestCypherLexer::test_tokenize_*` assert only `len(tokens) > 0` / `"MATCH" in values`, so negating +`if ch in " \t\n\r"` passes. Those tests *exercise* the function without asserting on it. + +**Takeaway:** the signal's verification value is bounded by test *assertion strength*, not by coverage. +Enriching `verify_on_read` with TESTS edges is worthwhile, but must be paired with a signal for +"does this test actually assert" (assertion density / mutation-kill proxy) — otherwise 40% of linked +tests give false confidence. + +### v6 — verification-strength profile (multiple mutations per function) + +10 functions x 4 mutations (negate-if, flip-compare, bool-flip, int+1, str-append) = 40 mutants; +every graph-linked test (arm B) and decoy (arm D) was run under every mutant. +Script: `e17_strength_profile.py`, data: `e17_strength_profile.json`. + +- 24 tests profiled. **Decoy tests killed 0/104 mutants** (control clean). +- kill-rate spread: 2/24 tests killed every mutant, 6/24 killed none — a per-test profile, not one snapshot. +- Predictors of kill-rate (Spearman, n=24): + - assertion specificity (`assert x == v`): **rho=+0.54, p=0.006** + - assert count: rho=+0.47, p=0.019 + - assertion density: rho=+0.25, p=0.24 (n.s.) + - test LOC: rho=+0.14, p=0.53 (n.s.) + +So **what an assertion compares against matters more than how many assertions there are**. +`TestCypherLexer::test_tokenize_*` have 2 asserts but kill only 0.5 (`assert len(tokens) > 0`, +`assert "MATCH" in values`), while `assert result == expected` is far more discriminating. +Density and length carry ~no signal. This matches the "TEST_REACHES vs TEST_VERIFIES" split: +a TESTS edge is a reach claim; kill-rate under mutation is a verification claim. + +**CORRECTED (v7):** this did not replicate on a larger panel — see v7 below. Treat the rho values +above as a small-sample artifact (n=24). + +### v7 — scaled replication: AST proxies do NOT predict kill-rate + +24 functions (providers/mcp excluded — their tests await servers and hang under mutation), +55 tests, 4 mutations each; decoys killed **0/55** (control clean). Script: `e17_strength_scaled.py`, +data: `e17_strength_scaled.json`. Split by function (no leakage). + +kill-rate spread is healthy (mean 0.38; 17 zeros, 5 ones, 33 mid-range), so the null is not a variance artefact. + +| feature | train rho (p) | held-out rho (p) | +|---|---|---| +| exact_compare | +0.13 (0.54) | -0.01 (0.97) | +| asserts | +0.19 (0.38) | -0.04 (0.82) | +| density | +0.23 (0.27) | +0.03 (0.88) | +| loc | +0.01 (0.97) | -0.03 (0.87) | + +Held-out rule `exact_compare >= 1 -> strong` (kill_rate >= 0.5): precision 0.59, recall 0.67 +(TP=10 FP=7 FN=5 TN=8) — barely above chance. + +**Conclusion (corrected):** cheap AST features do **not** predict whether a test catches a behaviour +change. `TEST_VERIFIES` cannot be approximated from AST alone — it needs actual mutation runs. +The `TESTS` *reach* edge and the decoy control (0/139 across v5–v7) remain valid: the signal says +who *reaches* the code, not who *verifies* it. + +## Conclusion + +1. **Answer generation / reasoning: the signal adds nothing.** For a strong model the code alone + determines the answer — across paraphrase, edge-case prediction, and even complex functions + (up to 332 loc / 80 branches). B never beats A. +2. **Impact / coverage: the signal is the only source of truth.** A = 0%, B = 100%, decoys = 0%. + Test existence is not derivable from the function source, and name conventions do not leak it. +3. Therefore the value of TESTS-signal is **informational (retrieval), not reasoning-enhancing**: + it enables agent workflows that must know *which tests to run*, and cannot be replaced by a + stronger prompt or a better explanation of the code. + +## Limitations +- Single judge trial for reasoning tasks; near-ceiling scores limit discrimination. +- v4 ground truth is the signal's own specific-test set (by construction B has it); this measures + information availability, deliberately, not reasoning. Its purpose is to isolate *where* the + signal matters. +- Judge/model same family (self-preference). + +## Practical takeaway +Ship TESTS-signal as a **retrieval/impact** feature (agent picks regression tests; change-impact +queries). Do not market it as improving code explanation or LLM answer quality — measured: it does not. diff --git a/experiments/bootstrap/e17_shards/index.json b/experiments/bootstrap/e17_shards/index.json new file mode 100644 index 00000000..f1c789b9 --- /dev/null +++ b/experiments/bootstrap/e17_shards/index.json @@ -0,0 +1,542 @@ +[ + { + "shard": 1, + "id": 1, + "mode": "A_baseline", + "name": "IndexProjectRunner.run" + }, + { + "shard": 1, + "id": 1, + "mode": "B_runtime", + "name": "IndexProjectRunner.run" + }, + { + "shard": 1, + "id": 1, + "mode": "D_shuffled", + "name": "IndexProjectRunner.run" + }, + { + "shard": 2, + "id": 2, + "mode": "A_baseline", + "name": "RemoteEmbedder.embed_batch" + }, + { + "shard": 2, + "id": 2, + "mode": "B_runtime", + "name": "RemoteEmbedder.embed_batch" + }, + { + "shard": 2, + "id": 2, + "mode": "D_shuffled", + "name": "RemoteEmbedder.embed_batch" + }, + { + "shard": 2, + "id": 3, + "mode": "A_baseline", + "name": "CypherLexer.tokenize" + }, + { + "shard": 2, + "id": 3, + "mode": "B_runtime", + "name": "CypherLexer.tokenize" + }, + { + "shard": 2, + "id": 3, + "mode": "D_shuffled", + "name": "CypherLexer.tokenize" + }, + { + "shard": 3, + "id": 4, + "mode": "A_baseline", + "name": "AutoDocUpdater._extract_doc_references" + }, + { + "shard": 3, + "id": 4, + "mode": "B_runtime", + "name": "AutoDocUpdater._extract_doc_references" + }, + { + "shard": 3, + "id": 4, + "mode": "D_shuffled", + "name": "AutoDocUpdater._extract_doc_references" + }, + { + "shard": 3, + "id": 5, + "mode": "A_baseline", + "name": "IndexPipeline.process_file" + }, + { + "shard": 3, + "id": 5, + "mode": "B_runtime", + "name": "IndexPipeline.process_file" + }, + { + "shard": 3, + "id": 5, + "mode": "D_shuffled", + "name": "IndexPipeline.process_file" + }, + { + "shard": 3, + "id": 6, + "mode": "A_baseline", + "name": "extract_anchors" + }, + { + "shard": 3, + "id": 6, + "mode": "B_runtime", + "name": "extract_anchors" + }, + { + "shard": 3, + "id": 6, + "mode": "D_shuffled", + "name": "extract_anchors" + }, + { + "shard": 4, + "id": 7, + "mode": "A_baseline", + "name": "resolve_project_root" + }, + { + "shard": 4, + "id": 7, + "mode": "B_runtime", + "name": "resolve_project_root" + }, + { + "shard": 4, + "id": 7, + "mode": "D_shuffled", + "name": "resolve_project_root" + }, + { + "shard": 4, + "id": 8, + "mode": "A_baseline", + "name": "SymbolIndex.build_call_graph" + }, + { + "shard": 4, + "id": 8, + "mode": "B_runtime", + "name": "SymbolIndex.build_call_graph" + }, + { + "shard": 4, + "id": 8, + "mode": "D_shuffled", + "name": "SymbolIndex.build_call_graph" + }, + { + "shard": 5, + "id": 9, + "mode": "A_baseline", + "name": "IndexParser.parse_file" + }, + { + "shard": 5, + "id": 9, + "mode": "B_runtime", + "name": "IndexParser.parse_file" + }, + { + "shard": 5, + "id": 9, + "mode": "D_shuffled", + "name": "IndexParser.parse_file" + }, + { + "shard": 5, + "id": 10, + "mode": "A_baseline", + "name": "modification_guard" + }, + { + "shard": 5, + "id": 10, + "mode": "B_runtime", + "name": "modification_guard" + }, + { + "shard": 5, + "id": 10, + "mode": "D_shuffled", + "name": "modification_guard" + }, + { + "shard": 6, + "id": 11, + "mode": "A_baseline", + "name": "decorator" + }, + { + "shard": 6, + "id": 11, + "mode": "B_runtime", + "name": "decorator" + }, + { + "shard": 6, + "id": 11, + "mode": "D_shuffled", + "name": "decorator" + }, + { + "shard": 6, + "id": 12, + "mode": "A_baseline", + "name": "wrapper" + }, + { + "shard": 6, + "id": 12, + "mode": "B_runtime", + "name": "wrapper" + }, + { + "shard": 6, + "id": 12, + "mode": "D_shuffled", + "name": "wrapper" + }, + { + "shard": 7, + "id": 13, + "mode": "A_baseline", + "name": "format_project_memory" + }, + { + "shard": 7, + "id": 13, + "mode": "B_runtime", + "name": "format_project_memory" + }, + { + "shard": 7, + "id": 13, + "mode": "D_shuffled", + "name": "format_project_memory" + }, + { + "shard": 7, + "id": 14, + "mode": "A_baseline", + "name": "validate_code" + }, + { + "shard": 7, + "id": 14, + "mode": "B_runtime", + "name": "validate_code" + }, + { + "shard": 7, + "id": 14, + "mode": "D_shuffled", + "name": "validate_code" + }, + { + "shard": 7, + "id": 15, + "mode": "A_baseline", + "name": "_extract_yarn_lock" + }, + { + "shard": 7, + "id": 15, + "mode": "B_runtime", + "name": "_extract_yarn_lock" + }, + { + "shard": 7, + "id": 15, + "mode": "D_shuffled", + "name": "_extract_yarn_lock" + }, + { + "shard": 8, + "id": 16, + "mode": "A_baseline", + "name": "WriteTool._action_replace" + }, + { + "shard": 8, + "id": 16, + "mode": "B_runtime", + "name": "WriteTool._action_replace" + }, + { + "shard": 8, + "id": 16, + "mode": "D_shuffled", + "name": "WriteTool._action_replace" + }, + { + "shard": 8, + "id": 17, + "mode": "A_baseline", + "name": "SymbolIndex.get_call_chain" + }, + { + "shard": 8, + "id": 17, + "mode": "B_runtime", + "name": "SymbolIndex.get_call_chain" + }, + { + "shard": 8, + "id": 17, + "mode": "D_shuffled", + "name": "SymbolIndex.get_call_chain" + }, + { + "shard": 8, + "id": 18, + "mode": "A_baseline", + "name": "load_manifest" + }, + { + "shard": 8, + "id": 18, + "mode": "B_runtime", + "name": "load_manifest" + }, + { + "shard": 8, + "id": 18, + "mode": "D_shuffled", + "name": "load_manifest" + }, + { + "shard": 9, + "id": 19, + "mode": "A_baseline", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + { + "shard": 9, + "id": 19, + "mode": "B_runtime", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + { + "shard": 9, + "id": 19, + "mode": "D_shuffled", + "name": "ProjectIntelligenceLayer._enrich_job_response" + }, + { + "shard": 9, + "id": 20, + "mode": "A_baseline", + "name": "format_runtime_status" + }, + { + "shard": 9, + "id": 20, + "mode": "B_runtime", + "name": "format_runtime_status" + }, + { + "shard": 9, + "id": 20, + "mode": "D_shuffled", + "name": "format_runtime_status" + }, + { + "shard": 9, + "id": 21, + "mode": "A_baseline", + "name": "_delayed_auto_index" + }, + { + "shard": 9, + "id": 21, + "mode": "B_runtime", + "name": "_delayed_auto_index" + }, + { + "shard": 9, + "id": 21, + "mode": "D_shuffled", + "name": "_delayed_auto_index" + }, + { + "shard": 10, + "id": 22, + "mode": "A_baseline", + "name": "load_metrics" + }, + { + "shard": 10, + "id": 22, + "mode": "B_runtime", + "name": "load_metrics" + }, + { + "shard": 10, + "id": 22, + "mode": "D_shuffled", + "name": "load_metrics" + }, + { + "shard": 10, + "id": 23, + "mode": "A_baseline", + "name": "_extract_pyproject" + }, + { + "shard": 10, + "id": 23, + "mode": "B_runtime", + "name": "_extract_pyproject" + }, + { + "shard": 10, + "id": 23, + "mode": "D_shuffled", + "name": "_extract_pyproject" + }, + { + "shard": 10, + "id": 24, + "mode": "A_baseline", + "name": "IndexStatusReporter.get_status" + }, + { + "shard": 10, + "id": 24, + "mode": "B_runtime", + "name": "IndexStatusReporter.get_status" + }, + { + "shard": 10, + "id": 24, + "mode": "D_shuffled", + "name": "IndexStatusReporter.get_status" + }, + { + "shard": 10, + "id": 25, + "mode": "A_baseline", + "name": "WriteTool._action_insert" + }, + { + "shard": 10, + "id": 25, + "mode": "B_runtime", + "name": "WriteTool._action_insert" + }, + { + "shard": 10, + "id": 25, + "mode": "D_shuffled", + "name": "WriteTool._action_insert" + }, + { + "shard": 11, + "id": 26, + "mode": "A_baseline", + "name": "_match_gitignore_pattern" + }, + { + "shard": 11, + "id": 26, + "mode": "B_runtime", + "name": "_match_gitignore_pattern" + }, + { + "shard": 11, + "id": 26, + "mode": "D_shuffled", + "name": "_match_gitignore_pattern" + }, + { + "shard": 11, + "id": 27, + "mode": "A_baseline", + "name": "_Fingerprint.__init__" + }, + { + "shard": 11, + "id": 27, + "mode": "B_runtime", + "name": "_Fingerprint.__init__" + }, + { + "shard": 11, + "id": 27, + "mode": "D_shuffled", + "name": "_Fingerprint.__init__" + }, + { + "shard": 11, + "id": 28, + "mode": "A_baseline", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + { + "shard": 11, + "id": 28, + "mode": "B_runtime", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + { + "shard": 11, + "id": 28, + "mode": "D_shuffled", + "name": "GraphRAGQueryEngine.query_dependencies" + }, + { + "shard": 11, + "id": 29, + "mode": "A_baseline", + "name": "IndexGuard.save_symbol_index" + }, + { + "shard": 11, + "id": 29, + "mode": "B_runtime", + "name": "IndexGuard.save_symbol_index" + }, + { + "shard": 11, + "id": 29, + "mode": "D_shuffled", + "name": "IndexGuard.save_symbol_index" + }, + { + "shard": 12, + "id": 30, + "mode": "A_baseline", + "name": "RemoteEmbedder._shadow_compare" + }, + { + "shard": 12, + "id": 30, + "mode": "B_runtime", + "name": "RemoteEmbedder._shadow_compare" + }, + { + "shard": 12, + "id": 30, + "mode": "D_shuffled", + "name": "RemoteEmbedder._shadow_compare" + } +] \ No newline at end of file diff --git a/experiments/bootstrap/e17_shards/shard_1.answers.json b/experiments/bootstrap/e17_shards/shard_1.answers.json new file mode 100644 index 00000000..a44c6dc8 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_1.answers.json @@ -0,0 +1,17 @@ +[ + { + "id": 1, + "mode": "A_baseline", + "answer": "From the information given I cannot determine which tests exercise IndexProjectRunner.run. Only the function source is provided; no repository layout, test files, or call sites are included, so no test function names can be identified. The function body references no tests. An LLM given only this text must not invent names. To find them one would search the test suite for imports of IndexProjectRunner or calls to .run, and for coverage of os.walk scanning, ThreadPoolExecutor workers, embed_batch, path_manager.is_safe_to_process, incremental resume writes, prune, BM25 reindex, cache invalidation and IVF indexing. Without that search the honest answer is: undetermined. I explicitly cannot list the required test function names." + }, + { + "id": 1, + "mode": "B_runtime", + "answer": "The tests provided that exercise IndexProjectRunner.run are exactly these three:\ntest_run_survives_single_cpu_host\ntest_run_with_existing_known_hashes_skips_unchanged\ntest_workers_receive_known_hashes_not_none\n\nThey cover the 1-CPU worker-count edge case, the skip-unchanged path through known_hashes, and the guarantee that workers receive a dict so they never enter the locked table.search branch. These must be run for regressions when changing run. In summary, run performs a full project index: path-safety check, file scan with a guard, parallel parse, batched embedding, incremental resume-friendly writes, prune, BM25 reindex, cache invalidation and optional IVF indexing, returning the count of indexed files." + }, + { + "id": 1, + "mode": "D_shuffled", + "answer": "The three provided tests do NOT exercise IndexProjectRunner.run. test_pid_reuse_is_stale and test_dead_pid_is_stale check DatabaseLock holder classification, and test_logs_dir_inside_data_root checks get_logs_dir; none call run, its parse workers, embedder, db_writer, searcher or table. Therefore I cannot determine from the information given which tests exercise this function. The listed names are unrelated to indexing and using them would give false confidence. To catch regressions one would search the repository for tests importing IndexProjectRunner or calling .run, covering the safety check, worker pool, known_hashes skip path, incremental writes, prune and reindex. From the supplied text, the honest answer is: undetermined." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_10.answers.json b/experiments/bootstrap/e17_shards/shard_10.answers.json new file mode 100644 index 00000000..86374e32 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_10.answers.json @@ -0,0 +1,62 @@ +[ + { + "id": 22, + "mode": "A_baseline", + "answer": "No test names can be determined from the information given. The prompt provides only the source of load_metrics plus its module globals (_METRICS_PATH, _TOOL_METRICS, _TOOL_METRICS_LOCK); it contains no test file, no directory listing, and no search results. Nothing in the supplied text names a test function or shows an import of error_handler. So I cannot honestly list tests that exercise it. To find them one would grep the repository for load_metrics, for imports of src.core.error_handler, or for patching of _METRICS_PATH. The function loads saved tool metrics from a JSON file, sanitizes negative latency values (min_ms, max_ms, total_ms and the latencies list), and merges them into _TOOL_METRICS. Candidate regression tests would target that sanitization, but their names are undetermined." + }, + { + "id": 22, + "mode": "B_runtime", + "answer": "The tests that exercise load_metrics and should be run before changing it are:\ntest_bs14_load_metrics_clamps_negative_total_ms\ntest_bs14_load_metrics_sanitizes_negative\nBoth import src.core.error_handler, point eh._METRICS_PATH at a temporary JSON file, clear eh._TOOL_METRICS, call eh.load_metrics(), and assert the sanitization behavior. The first checks that a negative total_ms is clamped to 0 and min_ms becomes 999999. The second checks that negative min_ms is dropped, that every latency is non-negative, and that total_ms stays non-negative. These are the direct regression tests for the loading and sanitizing logic, so both must be run whenever load_metrics changes." + }, + { + "id": 22, + "mode": "D_shuffled", + "answer": "I cannot determine the tests that exercise load_metrics from the information given. The supplied snippets are test_multiple_mentions (about parse_cross_repo_query) and test_ips_are_global (about IP classification). Neither imports error_handler, references load_metrics, nor touches _METRICS_PATH or _TOOL_METRICS, so neither exercises the function. Although the section is labelled as relevant tests, their contents contradict that label. Therefore no test name can be truthfully listed. The function loads saved tool metrics from a JSON file, sanitizes negative latency fields, and merges them into _TOOL_METRICS; a genuine regression test would call it with a crafted metrics file. Undetermined from the given text." + }, + { + "id": 23, + "mode": "A_baseline", + "answer": "No test names can be determined from the information given. The prompt supplies only the source of _extract_pyproject plus references to helpers (_toml, _req_name, normalize_python, ManifestEntry); there is no test file, no listing, and no grep output. Nothing names a test function or imports extract.py. So I cannot honestly list tests that exercise it. To find them one would search for _extract_pyproject, for imports of the extract module, or for tests using pyproject fixtures. The function parses TOML text of a pyproject.toml, returning ManifestEntry objects for project.dependencies, project.optional-dependencies, and PEP 735 dependency-groups (supporting both list and packages-dict forms), skipping anything _toml cannot parse. Candidate tests would cover those branches, but their names are undetermined." + }, + { + "id": 23, + "mode": "B_runtime", + "answer": "The tests that exercise _extract_pyproject are:\ntest_pyproject_optional_and_groups\ntest_requests_pyproject_and_requirements_dev\ntest_manifest_packages_wiring_pyproject_plus_lock\nThe first calls _extract_pyproject directly on inline TOML and checks that chardet, pysocks, pytest and pytest-cov are all collected, covering project.dependencies, optional-dependencies and dependency-groups. The second goes through manifest_packages on a requests fixture, exercising the pyproject path indirectly and asserting normalized names. The third uses a uv fixture where dependency-groups is the only source, asserting black and annotated-types appear. Together they cover the list and packages-dict branches, so all three should be run before changing the function." + }, + { + "id": 23, + "mode": "D_shuffled", + "answer": "I cannot determine the tests that exercise _extract_pyproject from the information given. The supplied snippets are test_localhost_https_rejected, test_search_with_mode_fast_calls_embed_and_vector, and test_attribute_assignment. None imports the extract module, references _extract_pyproject, or uses pyproject TOML parsing; they cover Git URL SSRF protection, search-mode behavior, and assignment extraction respectively. So none exercises the function, despite the relevant-tests label. No test name can be truthfully listed. The function parses pyproject.toml TOML text into ManifestEntry objects from project.dependencies, optional-dependencies, and PEP 735 dependency-groups. Finding real tests requires a repository search for _extract_pyproject or extract-module imports. Undetermined from the given text." + }, + { + "id": 24, + "mode": "A_baseline", + "answer": "No test names can be determined from the information given. The prompt provides only the source of IndexStatusReporter.get_status plus the implicit attributes it uses (_reindex_check, _table_write_lock, _cached_total_chunks, _cached_unique_files, table, _scan_stale, _watchdog_callback). There is no test file, no directory listing, and no search results. Nothing in the supplied text names a test function or constructs an IndexStatusReporter. So I cannot honestly list which tests exercise it. To find them one would grep for get_status, for IndexStatusReporter, or for the reindex fast-fail scenario. The method returns database statistics, short-circuits during reindex, and falls back to count_rows for consistency, but candidate test names are undetermined." + }, + { + "id": 24, + "mode": "B_runtime", + "answer": "The tests listed under this function are:\ntest_get_status_fast_fail_during_reindex_does_not_block\ntest_setup\nThe first directly constructs IndexStatusReporter and calls get_status, so it clearly exercises the function, asserting the fast-fail path returns cached values quickly while the write lock is held. The second (test_setup) builds a FileGuard and DI container and resolves an indexer; it does not visibly call get_status, so it likely does not exercise it despite appearing under the relevant-tests heading. Strictly, only test_get_status_fast_fail_during_reindex_does_not_block can be confirmed from the given text; test_setup should be treated as unconfirmed and checked separately." + }, + { + "id": 24, + "mode": "D_shuffled", + "answer": "I cannot determine the tests that exercise IndexStatusReporter.get_status from the information given. The supplied snippets are test_handles_same_path (Indexer.move_chunks_metadata no-op) and test_no_dollardefs_or_ref_in_tool_schemas (tool JSON schema validation). Neither constructs an IndexStatusReporter, calls get_status, nor touches the reindex fast-fail or count_rows logic. They belong to different modules, so they do not exercise the function despite the relevant-tests label. No test name can be truthfully listed. The function returns DB statistics and short-circuits to cached values during reindexing to avoid blocking the event loop. Locating real tests requires a repository search for get_status or IndexStatusReporter. Undetermined from the given text." + }, + { + "id": 25, + "mode": "A_baseline", + "answer": "No test names can be determined from the information given. The prompt supplies only the source of WriteTool._action_insert plus the helpers and attributes it calls (require_ready_project, resolve_symbol_index, find_definitions, _resolve_single_source_def, _build_insert_lines, _preflight_validate, _atomic_write, _contract_record). There is no test file, no listing, and no grep output, and nothing names a test function or instantiates WriteTool. So I cannot honestly list tests that exercise it. To find them one would search for _action_insert_after, _action_insert_before, or tests using a write_tool fixture. The method inserts code before or after an anchor symbol, previews when apply is false, and records a contract hash, but candidate test names are undetermined." + }, + { + "id": 25, + "mode": "B_runtime", + "answer": "The tests that exercise _action_insert are:\ntest_filter_mismatch_returns_error\ntest_filter_mismatch_returns_error\ntest_unknown_symbol_returns_error\nNote the first name appears twice: one variant calls _action_insert_after and one calls _action_insert_before, both with a mismatching file_path and asserting an error is returned. test_unknown_symbol_returns_error calls _action_insert_after with a nonexistent anchor and expects an error. All three reach the function through its position wrappers (insert_before and insert_after both delegate to _action_insert), so they cover the anchor-not-found and file-filter branches. They should be run before changing _action_insert, though the duplicate name means collection must be scoped per test file." + }, + { + "id": 25, + "mode": "D_shuffled", + "answer": "I cannot determine the tests that exercise WriteTool._action_insert from the information given. The supplied snippets are test_token_is_32_hex_chars (ack token format), test_unknown_kind_ignored (AlertStore.push), and test_gemfile_skips_gitpath (manifest_packages). None imports write_tools, uses a write_tool fixture, or calls _action_insert, _action_insert_after, or _action_insert_before. They cover unrelated modules, so none exercises the function, despite the relevant-tests label. Therefore no test name can be truthfully listed. The function inserts new code before or after an anchor symbol, supports preview mode, and performs preflight validation plus atomic writes and contract recording. Locating real tests requires searching for _action_insert or the write_tool fixture. Undetermined from the given text." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_11.answers.json b/experiments/bootstrap/e17_shards/shard_11.answers.json new file mode 100644 index 00000000..57fee194 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_11.answers.json @@ -0,0 +1,62 @@ +[ + { + "id": 26, + "mode": "A_baseline", + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path is matched by a single .gitignore pattern. It strips a leading slash and records/dir-strips a trailing slash. It then tries, in order: exact equality; `/**/` matching via prefix/suffix checks; fnmatch wildcards (full path when a slash is present, otherwise the basename); slash-free basename semantics with special any-depth handling for directory patterns; and prefix matching for slash-containing directory patterns. Notable edge cases: directory patterns match at any depth, wildcard scope depends on slash presence, and the trailing-slash branch became unreachable after stripping. No repository or test information is supplied in this prompt, so I cannot name the tests that exercise it; a developer should search the suite for gitignore_parser imports or calls to this helper and run those tests." + }, + { + "id": 26, + "mode": "B_runtime", + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path matches one .gitignore pattern. It strips a leading slash, detects trailing-slash directory patterns, then tries exact match, `/**/` prefix/suffix matching, fnmatch wildcards (full path when the pattern contains a slash, otherwise basename), slash-free basename and directory semantics, and finally prefix matching for slash-containing directory patterns. Edge cases include any-depth directory matching, wildcard scope depending on slash presence, and now-unreachable trailing-slash code. The context explicitly identifies the tests that exercise this function, so the developer must run:\ntest_dir_pattern_excludes_nested\ntest_dir_pattern_matches_any_depth\ntest_file_pattern_unchanged\nThese cover nested directory exclusion, any-depth directory matching, and unchanged file-pattern behaviour." + }, + { + "id": 26, + "mode": "D_shuffled", + "answer": "`_match_gitignore_pattern(path, pattern)` decides whether a relative POSIX path matches a single .gitignore pattern. It strips a leading slash, detects trailing-slash directory patterns, then applies exact match, `/**/` prefix/suffix matching, fnmatch wildcards (full path when a slash is present, otherwise basename), slash-free basename/directory semantics, and prefix matching for slash-containing directory patterns. Edge cases include any-depth directory matching and unreachable trailing-slash code. However, the tests supplied with this prompt do not reference `_match_gitignore_pattern` at all; they belong to other modules (a dependency graph and a path-safety guard) and cannot catch regressions here. Their names are:\ntest_no_shared\ntest_no_path\ntest_non_ascii_path_denied\nFrom the given information I cannot confirm any test that exercises the function; the developer should instead find tests importing gitignore_parser or calling `_excluded`." + }, + { + "id": 27, + "mode": "A_baseline", + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint. When a `data` dict is supplied it hydrates the `imports`, `files`, `env_keys`, and `packages` sets from it, with an ADR-0005 note that old caches lacking packages are rebuilt elsewhere to avoid false REFUTED results, and it leaves `build_ms` at zero. Otherwise it scans `/src` recursively, skipping `__pycache__`, collecting every relative file path and the top-level import names of each .py file, parses `.env` and `.env.example` for key names, loads manifest packages, and records elapsed build milliseconds. Edge cases: root is None, missing src directory, unreadable files (skipped via OSError). No repository or test context is provided here, so I cannot name which existing tests exercise this constructor." + }, + { + "id": 27, + "mode": "B_runtime", + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint: from a `data` dict it hydrates imports, files, env_keys, and packages sets (ADR-0005 notes old caches without packages are rebuilt); otherwise it scans `/src`, collects file paths and top-level imports, parses `.env`/`.env.example`, loads manifest packages, and records build milliseconds. Edge cases include a missing root, absent src, and unreadable files. The supplied tests actually exercise `extract_anchors` and its ADR-0005 `pkg:`/anchor classification rather than the constructor directly, but the context presents them as the relevant tests, so the developer should run:\ntest_extract_anchors_explicit_pkg_syntax\ntest_extract_anchors_from_data_anchors\ntest_extract_anchors_read_path_backward_compat\nThey validate anchor extraction and backward-compatible read-path classification; tests constructing `_Fingerprint` directly should also be located." + }, + { + "id": 27, + "mode": "D_shuffled", + "answer": "`_Fingerprint.__init__` builds or restores a project fingerprint. With a `data` dict it hydrates imports, files, env_keys, and packages sets (ADR-0005 notes caches without packages are rebuilt), leaving build_ms at zero; otherwise it walks `/src`, skips `__pycache__`, gathers relative paths and top-level imports, parses `.env`/`.env.example`, loads manifest packages, and records elapsed milliseconds. Edge cases: root None, absent src, unreadable files. The tests supplied with this prompt, however, do not touch `verify_on_read.py`: they cover yarn lock parsing, an interprocess mutex, and ack-impact token rejection, so they cannot catch regressions in this constructor. Their names are:\ntest_yarn_berry_v10_scoped\ntest_interprocess_lock_release_does_not_leak_ownership\ntest_rejects_empty_token\nFrom the given information I cannot confirm any test exercising `_Fingerprint.__init__`; search for tests constructing it directly." + }, + { + "id": 28, + "mode": "A_baseline", + "answer": "`GraphRAGQueryEngine.query_dependencies` returns dependency information for a file: the file name, `depends_on` (files its symbols call), `depended_by` (files that call its symbols), and an empty `cochange_files` list. It calls `_ensure_graph`, returns the empty skeleton when the symbol index is absent, then inspects up to ten symbols, walking call chains down and up and mapping callee/caller symbols to files via `_symbol_files` while skipping self-references. Inner exceptions are logged per symbol; an outer exception logs a query failure. Results are deduplicated and capped at ten entries each. The `direction` argument is accepted but unused. No repository or test information is supplied here, so I cannot name which existing tests exercise this function." + }, + { + "id": 28, + "mode": "B_runtime", + "answer": "`GraphRAGQueryEngine.query_dependencies` builds a dependency report for a file: `depends_on` lists files whose symbols the file calls, `depended_by` lists files calling its symbols, and `cochange_files` is an empty placeholder. It ensures the graph, returns a skeleton when no symbol index exists, then walks up to ten symbols in both call directions, mapping symbols to files through `_symbol_files` and skipping self-references. Inner exceptions are logged per symbol; an outer exception logs a failure. Results are deduplicated and capped at ten each, and `direction` is accepted but unused. The context explicitly identifies the test, so run:\ntest_query_dependencies\nIt only asserts the returned dict contains file, depends_on, and depended_by keys, so stronger assertions for dedup, capping, and direction are advisable." + }, + { + "id": 28, + "mode": "D_shuffled", + "answer": "`GraphRAGQueryEngine.query_dependencies` returns dependency information for a file: the file name, `depends_on`, `depended_by`, and an empty `cochange_files` list. It ensures the graph, returns a skeleton if the symbol index is absent, then inspects up to ten symbols, walking call chains down and up and mapping callee/caller symbols to files while skipping self-references; results are deduplicated and capped at ten. The test supplied with this prompt does not exercise `query_dependencies`; it belongs to the impact/ack-token guard and checks that a token for a non-existent file still verifies. So from the given information I cannot confirm any test that exercises this function. The provided test is:\ntest_token_missing_file_returns_missing_fingerprint\nA developer should search the suite for tests constructing `GraphRAGQueryEngine` or calling `query_dependencies`." + }, + { + "id": 29, + "mode": "A_baseline", + "answer": "`IndexGuard.save_symbol_index` writes a SymbolIndex to `symbol_index.json` and returns True on success or False on failure. It first applies three anti-corruption guards that skip writing when the instance is a MODE_PURE adapter, when `_definitions` is empty but a graph contains nodes, or when the expected `_definitions`/`_references`/`_file_to_symbols` attributes are missing (graph-backed persistence). It then serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with an empty state, deletes the legacy pickle, and logs exceptions. Edge cases: absent attributes, corrupt existing JSON, legacy file cleanup. No repository or test context is provided, so I cannot name the tests that exercise it." + }, + { + "id": 29, + "mode": "B_runtime", + "answer": "`IndexGuard.save_symbol_index` persists a SymbolIndex to `symbol_index.json`, returning True/False. It applies anti-corruption guards that skip writing for MODE_PURE adapters, when `_definitions` is empty but a graph has nodes, or when the expected dict attributes are absent (graph-backed persistence). Otherwise it serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with empty state, removes the legacy pickle, and logs failures. Edge cases: missing attributes, corrupt JSON, legacy cleanup. The context identifies the tests that exercise it, so run:\ntest_symbol_index_persistence\ntest_empty_file_can_be_created_for_fresh_project\ntest_empty_instance_does_not_overwrite_populated_file\nThese cover save/load round-trips, fresh-project creation, and empty-overwrite protection." + }, + { + "id": 29, + "mode": "D_shuffled", + "answer": "`IndexGuard.save_symbol_index` persists a SymbolIndex to `symbol_index.json`, returning True or False. It applies anti-corruption guards that skip writing for MODE_PURE adapters, when `_definitions` is empty but a graph has nodes, or when expected dict attributes are absent; otherwise it serialises definitions, references, file_to_symbols, optional env accesses, and a timestamp, refuses to overwrite a populated file with empty state, deletes the legacy pickle, and logs exceptions. Edge cases include absent attributes, corrupt existing JSON, and legacy cleanup. The tests supplied with this prompt, however, cover scores-JSON parsing, gitignore directory patterns, and a ToolError, so none of them exercise `save_symbol_index` and none can catch regressions. Their names are:\ntest_parse_scores_json_duplicate_in_wrapper_declined\ntest_dir_pattern_matches_any_depth\ntest_creates_with_message\nFrom the given information I cannot confirm any relevant test." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_12.answers.json b/experiments/bootstrap/e17_shards/shard_12.answers.json new file mode 100644 index 00000000..e5dfaafe --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_12.answers.json @@ -0,0 +1,17 @@ +[ + { + "id": 30, + "mode": "A_baseline", + "answer": "I cannot determine this from the information given. The prompt provides only the source code and docstring of `RemoteEmbedder._shadow_compare`; no test files, test names, imports, or repository layout are included. The function calls `self.embed_batch`, the module-level helper `_vectors_collapsed`, and the constant `_ABS_MIN_QUALITY`, and it is invoked during provider switching, so relevant tests could live anywhere in the repo. Naming specific test functions would be a guess and therefore misleading. Explicit statement: cannot determine which tests exercise this function. To answer reliably one would have to search the repository for `_shadow_compare`, `shadow_compare`, canary fixtures, and `_ABS_MIN_QUALITY`, then inspect each hit. A developer should run those located tests before changing the fail-closed logic described here." + }, + { + "id": 30, + "mode": "B_runtime", + "answer": "The tests shown that exercise `RemoteEmbedder._shadow_compare` are the canary fail-closed tests: `test_baseline_failure_blocks`, `test_empty_canary_blocks`, and `test_baseline_below_absolute_quality_blocks`. Each builds an `embedder = MagicMock(spec=RemoteEmbedder)` with `_canary_pairs` set, then calls `RemoteEmbedder._shadow_compare(embedder, fn, name)` directly. They cover baseline failure via `embed_batch` raising `RuntimeError` (block), an empty canary set (block), and a degenerate baseline of orthogonal vectors producing `old_mean=0` below the anchor (UNKNOWN, block). All assert `result is False`, confirming fail-closed behavior. A developer changing this function must run these three to catch regressions. Note that other branches such as collapsed vectors, empty new-provider response, and the `degraded_ratio > 0.3` path may have additional tests not shown here." + }, + { + "id": 30, + "mode": "D_shuffled", + "answer": "The three tests supplied do not exercise `RemoteEmbedder._shadow_compare`. They are `test_build_call_graph_callers`, which covers call-graph traversal; `test_normalize_diag_uri_win_drive`, which covers LSP diagnostic URI normalization; and `test_cli_dispatch_ok`, which covers CLI tool dispatch. None references `_shadow_compare`, `_canary_pairs`, `embed_batch`, `_vectors_collapsed`, or `_ABS_MIN_QUALITY`, so none would catch a regression in this function. Therefore I cannot determine from the information given which tests actually exercise `_shadow_compare`; the listed tests are unrelated and appear to be misleading distractors. Honest answer: cannot determine. A developer should search the repository for `_shadow_compare` and canary fixtures to locate the genuine tests before modifying the function." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_2.answers.json b/experiments/bootstrap/e17_shards/shard_2.answers.json new file mode 100644 index 00000000..fb32b0d8 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_2.answers.json @@ -0,0 +1,32 @@ +[ + { + "id": 2, + "mode": "A_baseline", + "answer": "I cannot determine this from the information given. The context supplies only the source of RemoteEmbedder.embed_batch, not the repository test suite, so any list of test function names would be a guess and could mislead the developer. To find the right tests, one should search the repository for callers of embed_batch and for RemoteEmbedder usage. Regarding the function itself: it fetches embedding vectors in batch from whichever provider is currently active and deliberately never changes the mode. Its inputs are a list of texts plus an is_query flag; its output is a list of float vectors. An empty input returns an empty list. The llama.cpp path truncates to 512 tokens, retries with backoff, then retries each item individually, raising RuntimeError if any chunk still fails. The LM Studio, ONNX server, OpenVINO and local ONNX paths raise RuntimeError when unavailable." + }, + { + "id": 2, + "mode": "B_runtime", + "answer": "Based on the tests supplied with this prompt, the tests that exercise embed_batch are: test_rapid_sequential_calls_no_state_leak and test_single_call_results_match_indices. The first drives one hundred rapid sequential embed_batch calls and checks that no state leaks between calls and that vectors change with different texts. The second checks a single batch returns three non-zero vectors of length 384 with correct indices. The function itself returns embedding vectors from the active provider without ever mutating mode; it handles llama.cpp, LM Studio, ONNX server, OpenVINO and local ONNX paths with retries and RuntimeError fallbacks. If other tests exist elsewhere in the repository, they are not shown here, so this list may be incomplete." + }, + { + "id": 2, + "mode": "D_shuffled", + "answer": "I cannot determine this from the information given. The two tests shown, test_sample_disk_io_no_data_row_returns_zero and test_verdict_fail_is_refuted, exercise a ResourceMonitor disk-sampling helper and a verdict_from_results function respectively, not RemoteEmbedder.embed_batch. Nothing in the provided text links them to embedding, so listing them would be wrong. No test function names can be listed with confidence here. The function itself returns a list of embedding vectors from the active provider such as llama.cpp, LM Studio, ONNX server, OpenVINO or local ONNX, never changing mode, returning an empty list for empty input, and raising RuntimeError when every provider fails. To find the real tests one must search the repository for callers of embed_batch." + }, + { + "id": 3, + "mode": "A_baseline", + "answer": "I cannot determine this from the information given. The prompt includes only the source of CypherLexer.tokenize and no test file contents or test names, so listing tests would be pure speculation. The function scans a Cypher query string character by character and returns a list of Token objects. It skips whitespace, strips double-slash comments, parses quoted strings while raising SyntaxError on unterminated ones, parses numbers, identifiers, keywords, uppercase labels, backtick-escaped identifiers, two-character comparison operators, direction arrows, single-character operators, punctuation, and colon-prefixed labels including variable-length :TYPE*min..max forms. An unknown character raises SyntaxError. The token list is cached in self._tokens. To find exercising tests, search the repository for CypherLexer usage and tokenize calls." + }, + { + "id": 3, + "mode": "B_runtime", + "answer": "Based on the tests supplied with this prompt, the tests exercising CypherLexer.tokenize are: test_tokenize_optional_match, test_tokenize_order_limit, and test_tokenize_relationship. They tokenize sample Cypher queries and assert recognition of KEYWORD values such as OPTIONAL, MATCH, ORDER and LIMIT, plus the label CALLS and the arrow punctuation ->. The function itself performs a single-pass lexical analysis of a Cypher query, returning Token objects for keywords, identifiers, labels, strings, numbers, operators, punctuation and relationship types, while skipping comments and whitespace and raising SyntaxError for unterminated strings, unterminated backticks or unexpected characters. The tokens are stored in self._tokens. Other repository tests may exist but are not shown, so this list may be incomplete." + }, + { + "id": 3, + "mode": "D_shuffled", + "answer": "I cannot determine this from the information given. The three tests shown, test_gemfile_lock_skips_path_project_gem, test_blocked_name_getattr and test_load_invalid_engine_req, exercise a Gemfile.lock manifest extractor, a sandbox code validator and a plugin manifest loader respectively. None of them references CypherLexer or calls tokenize, so they cannot catch regressions in that function and should not be listed. Separately, CypherLexer.tokenize scans a Cypher query string and returns Token objects, skipping comments and whitespace while raising SyntaxError on unterminated strings, backticks or unexpected characters, and recognising keywords, labels, numbers, operators, punctuation and variable-length :TYPE*min..max relation types. To identify the true tests, one must search the repository for CypherLexer usage." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_3.answers.json b/experiments/bootstrap/e17_shards/shard_3.answers.json new file mode 100644 index 00000000..c47649bb --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_3.answers.json @@ -0,0 +1,47 @@ +[ + { + "id": 4, + "mode": "A_baseline", + "answer": "From the information given I cannot determine which tests exercise `AutoDocUpdater._extract_doc_references`: no test files, test names or repository structure were provided, only the function source. The function parses Markdown text line by line, skips fenced code blocks, and extracts backtick-wrapped tokens as likely code identifiers. It filters out URLs, file paths, Windows drive paths, IP addresses, file extensions, config assignments, hotkeys, version-like strings and stop words, then strips call arguments and `$` env prefixes. It returns a list of `(content, line_number, full_match)` tuples. To identify regression tests you would need the repository test suite, which is absent here." + }, + { + "id": 4, + "mode": "B_runtime", + "answer": "One provided test exercises this function: test_extract_doc_references_handles_empty_call_syntax. It builds a temporary tree, writes a doc containing empty-call references `()` and `(x)` plus a normal `fastmcp`, then calls `_verify_doc_references`, which internally uses `_extract_doc_references`. The test asserts no IndexError is raised and that a list of broken references is returned. The function itself scans Markdown lines, skips fenced code blocks, extracts backtick tokens, filters non-code patterns (URLs, paths, IPs, extensions, assignments, semver) and returns `(content, line, match)` tuples. Any behavior change should re-run that test, though the provided list may be incomplete." + }, + { + "id": 4, + "mode": "D_shuffled", + "answer": "The information given lists test_symbol_index_stores_and_retrieves as the relevant test for this function. Taken at face value, that is the test to run: it exercises SymbolIndex round-trip behavior for env accesses. Separately, `_extract_doc_references` scans Markdown text, skipping fenced code blocks, extracts backtick-wrapped tokens and filters out URLs, paths, IPs, file extensions, config assignments, hotkeys and semver strings before returning `(content, line_number, full_match)` tuples. Note the listed test appears unrelated to reference extraction, so on the evidence provided it may not catch regressions; when in doubt, inspect the actual test suite rather than trusting the label." + }, + { + "id": 5, + "mode": "A_baseline", + "answer": "From the information given I cannot determine which tests exercise `IndexPipeline.process_file`: no test files, names or repository structure were provided, only the function source. The function handles one file through the indexing pipeline. It rejects blank content, parses via `_index_parser.parse_file`, records mtime and size, updates SymbolIndex collections (definitions, references, assignments, imports, decorators, overrides, env accesses) under a lock, computes per-chunk SHA-256 hashes for a content-addressed cache, reuses cached vectors from LanceDB, embeds only new chunks, and returns a result dict. It returns None on empty input, parse failure, no chunks, empty embeddings or cache gaps. Regression tests cannot be identified without the test suite." + }, + { + "id": 5, + "mode": "B_runtime", + "answer": "Three provided tests exercise this function. test_chunk_cache_disabled_without_table: with table=None, two process_file calls both invoke embed_batch, asserting call_count == 2. test_chunk_cache_empty_file: whitespace-only content returns None without calling the embedder. test_chunk_cache_all_new: a new file with foreign cached hashes yields one embed_batch call and five embeddings, verifying deterministic per-chunk vectors. The function itself rejects blank content, parses the file, records stat metadata, updates SymbolIndex collections under a lock, computes chunk hashes for a content-addressed cache, reuses cached vectors from LanceDB, embeds only new chunks, and returns a result dict or None on empty input, parse failure, empty embeddings or cache gaps. Re-run all three when changing behavior." + }, + { + "id": 5, + "mode": "D_shuffled", + "answer": "The information given lists three relevant tests: test_ensure_server_running_launches_when_dead, test_impact_analysis_basic and test_empty_file. Taken at face value these are the tests to run. Separately, `process_file` rejects blank content, parses via `_index_parser.parse_file`, records mtime and size, updates SymbolIndex collections under a lock, computes per-chunk SHA-256 hashes, reuses cached vectors from LanceDB, embeds only new chunks, and returns a dict or None on empty input, parse failure, no chunks, empty embeddings or cache gaps. Note the listed tests look unrelated to file processing, so on the evidence provided they may not catch regressions; the actual test suite should be inspected before relying on them." + }, + { + "id": 6, + "mode": "A_baseline", + "answer": "From the information given I cannot determine which tests exercise `extract_anchors`: no test files or test names were provided, only the function source. The function extracts checkable anchors from a memory node's data/claim text using regex, without an LLM. It honors explicit `data.anchors` first; on the read path it skips prose scanning when explicit anchors exist. Otherwise it parses `file:path`, `import X` / `from X import y`, `env:KEY` / `$KEY`, `pkg:name` and path-like tokens. On the write path it filters file anchors by existence and captures manifest package words fail-closed. It deduplicates via a seen set and returns Anchor objects. Regression tests cannot be identified without the repository test suite." + }, + { + "id": 6, + "mode": "B_runtime", + "answer": "Three provided tests exercise this function. test_extract_anchors_no_anchors: a plain statement without anchors returns an empty list. test_extract_anchors_explicit_pkg_syntax: a claim containing `pkg:celery` yields exactly one pkg anchor `('pkg', 'celery')`. test_extract_anchors_from_data_anchors: explicit `data.anchors` with file, import and env entries produce matching (kind, value) anchors. The function itself extracts checkable anchors from a node's data/claim text via regex: it honors explicit data.anchors first, otherwise parses file paths, imports, env keys, pkg names and manifest package words, deduplicating through a seen set. Re-run all three when changing behavior, though the list may not be exhaustive." + }, + { + "id": 6, + "mode": "D_shuffled", + "answer": "The information given lists three relevant tests: test_format_diagnostics_empty, test_stale_token_rejected_after_file_change and test_lru_cap_evicts_oldest. Taken at face value these are the tests to run. Separately, `extract_anchors` extracts checkable anchors from a memory node's data/claim text using regex: it honors explicit `data.anchors`, skips prose on the read path when explicit anchors exist, and otherwise parses file paths, imports, env keys, pkg names and manifest package words, deduplicating via a seen set and returning Anchor objects. Note the listed tests appear unrelated to anchor extraction, so on the evidence provided they may not catch regressions; the actual test suite should be inspected." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_4.answers.json b/experiments/bootstrap/e17_shards/shard_4.answers.json new file mode 100644 index 00000000..65b83a4c --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_4.answers.json @@ -0,0 +1,32 @@ +[ + { + "id": 7, + "mode": "A_baseline", + "answer": "The function resolve_project_root returns the project root directory used by MCP tools, resolving it fresh on each call. Inputs: an optional provided path string. Output: a Path. Priority order: explicit provided argument, then MSCODEBASE_PROJECT_PATH env override, then CWD with a self-indexing guard rejecting ext_root, then PROJECT_PATH env, then SQLite active_workspace_id, then the Zed workspaces table, then ZED_WORKTREE_ROOT env, and finally ext_root. Edge cases: multi-root paths split on newline or comma, non-existent paths skipped, and sqlite/OS errors swallowed. Regarding the question: no test names are given in the information provided, so I cannot determine which tests in the repository exercise this function; I cannot list them." + }, + { + "id": 7, + "mode": "B_runtime", + "answer": "resolve_project_root returns the project root Path for MCP tools, recomputed each call. It accepts an optional provided string and returns a Path. Resolution priority: provided argument, MSCODEBASE_PROJECT_PATH override, CWD with self-indexing guard, PROJECT_PATH env, SQLite active_workspace_id, the Zed workspaces table, ZED_WORKTREE_ROOT, and finally ext_root. It handles multi-root SQLite paths, missing directories, and sqlite/OS errors. Based on the tests shown, the tests that exercise this function are: test_provided_wins_over_cwd; test_cwd_equal_ext_root_is_skipped; test_mscb_path_trusts_ext_root. These three directly assert provided-over-CWD precedence, the self-index guard falling back to an env project, and MSCODEBASE_PROJECT_PATH trusting ext_root." + }, + { + "id": 7, + "mode": "D_shuffled", + "answer": "resolve_project_root returns the project root Path for MCP tools, resolving anew each call. It takes an optional provided string and returns a Path, following a priority chain: provided argument, MSCODEBASE_PROJECT_PATH, CWD with self-indexing guard, PROJECT_PATH env, SQLite active_workspace_id, the Zed workspaces table, ZED_WORKTREE_ROOT env, then ext_root fallback. It tolerates multi-root paths, missing directories, and sqlite/OS errors. However, the tests listed here (test_flag_on_when_env_set, test_blocked_name_globals, test_query_hotspots_no_memory) concern late enrichment config, a sandbox violation check, and graph hotspots respectively; none appear to exercise resolve_project_root. Therefore I cannot determine which tests exercise this function from the information given." + }, + { + "id": 8, + "mode": "A_baseline", + "answer": "SymbolIndex.build_call_graph builds a bidirectional call graph for a symbol using BFS. Inputs: a symbol name and depth (default 2, clamped to 1..5). Output: a dict with definition, callers, callees, call_chain, impact_files, and depth_reached. It looks up definitions, walks callers via self._references, walks callees by scanning references where the symbol is the caller, deduplicates entries, tracks visited sets to avoid cycles, and collects impacted files. Edge cases: depth below 1 is clamped up, above 5 clamped down, empty definitions, cycle prevention, and call_chain limited to five shallowest callers. On the question: no test names are provided, so I cannot determine which tests exercise this function; I cannot list them." + }, + { + "id": 8, + "mode": "B_runtime", + "answer": "build_call_graph returns a bidirectional BFS call graph for a symbol. It takes a symbol name and depth (default 2, clamped 1..5), and returns definition, callers, callees, call_chain, impact_files, and depth_reached. Callers are found through self._references; callees by scanning references where the symbol is the caller; visited sets prevent cycles and impact_files collects touched files. Based on the tests shown, the tests exercising this function are: test_build_call_graph_call_chain; test_build_call_graph_callees; test_build_call_graph_callers. These assert the call_chain is populated, that the callees of authenticate include validate_token and get_user, and that the callers of validate_token include authenticate." + }, + { + "id": 8, + "mode": "D_shuffled", + "answer": "build_call_graph returns a bidirectional BFS call graph for a symbol. It takes a symbol name and depth (default 2, clamped to 1..5), and returns definition, callers, callees, call_chain, impact_files, and depth_reached, using visited sets to avoid cycles and deduplicating entries. However, the tests listed here (test_resolver_auto_approve, test_sha256_file_missing, test_does_not_scan_extension_dir) concern a trust resolver, file hashing, and a grep fallback scanning the extension directory; none appear to exercise build_call_graph. Therefore I cannot determine which tests exercise this function from the information given." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_5.answers.json b/experiments/bootstrap/e17_shards/shard_5.answers.json new file mode 100644 index 00000000..76c6ddbe --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_5.answers.json @@ -0,0 +1,32 @@ +[ + { + "id": 9, + "mode": "A_baseline", + "answer": "The information given is insufficient to determine which tests exercise this function. The prompt supplies only the source of IndexParser.parse_file; no test files, test-module references, or test names are included. I cannot enumerate test functions covering encoding fallback (BOM, cp1251, latin-1), the existing_hash skip, comment-only rejection, AST chunking, or the sliding-window fallback without seeing the repository suite. Honest answer: cannot determine from the supplied text. To find them a developer should grep the tests directory for parse_file, IndexParser, and _has_code_lines, then run the matching functions. Listing specific test names here would be guessing, not evidence." + }, + { + "id": 9, + "mode": "B_runtime", + "answer": "The provided context includes two tests that exercise IndexParser.parse_file:\ntest_bs1_parse_file_skips_comment_only\ntest_property_graph_consistency\n\nBoth call parser.parse_file directly. The first targets the BS-1 guard rejecting comment-only files, asserting None is returned. The second is an integration test that indexes producer and consumer files, consumes the returned _ast_symbols cache, and verifies ghost-node cleanup after renames. These are the only test functions shown as relevant. Other tests may exist in the repository but are not included here, so this list reflects the supplied evidence only. A developer changing parse_file should run these two and grep the suite for additional callers before relying on them alone." + }, + { + "id": 9, + "mode": "D_shuffled", + "answer": "The context labels two tests as relevant to IndexParser.parse_file:\ntest_vor_coverage_starved_warning\ntest_inserts_after_anchor\n\nHowever, on inspection neither appears to call IndexParser.parse_file. The first tests VOR receipt formatting for starved nodes, and the second tests the write tool insert_after behavior. They exercise unrelated modules and never reference index_parser.py or parse_file. I therefore cannot confirm they catch regressions in parse_file, and I would treat the relevance label as unreliable. Honestly, from the given information the genuine tests cannot be determined; the two shown look mismatched. A developer should grep for parse_file and IndexParser instead of trusting this list." + }, + { + "id": 10, + "mode": "A_baseline", + "answer": "The information given is insufficient to identify tests exercising modification_guard. The prompt supplies only the decorator source; no test module, names, or references appear. modification_guard is a decorator applied to async write tools, so regression tests would likely live in files covering write-tool guards, the ack registry, ambiguity handling, and PageRank or blast-radius thresholds, but I cannot name them from this text. I explicitly state: I cannot determine this from the information provided. A developer should grep the tests for modification_guard, ack_impact, _ack_registry, and fresh_guard fixtures, then run the collected tests. Naming specific functions here would be fabrication." + }, + { + "id": 10, + "mode": "B_runtime", + "answer": "The supplied context lists three tests that exercise modification_guard:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_allows_non_hot_file\n\nAll three apply the @modification_guard decorator to a test tool method. The first verifies pass-through when neither file_path nor symbol is given. The second checks the ambiguity guard refuses to silently route a bare symbol with multiple definitions. The third patches the PageRank and blast-radius helpers to zero and asserts a non-hot file passes with no ack required. These are the only tests shown as relevant. Other repository tests may also exercise the decorator but are not included, so this list reflects the supplied evidence only." + }, + { + "id": 10, + "mode": "D_shuffled", + "answer": "The context labels three tests as relevant to modification_guard:\ntest_init_with_registry\ntest_symbol_index_remove_file_clears_env\ntest_parse_rejects_bad_urls\n\nBut none of these appear to touch modification_guard. The first tests CrossProjectDependencyGraph registry injection, the second tests SymbolIndex env-access cleanup, and the third tests GitUrlSource URL validation. They exercise unrelated modules and do not import or apply the decorator. I therefore cannot confirm they catch regressions in modification_guard. From the given information the real tests cannot be determined; the provided ones look shuffled or mismatched. A developer should grep for modification_guard and ack_impact instead of relying on this list." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_6.answers.json b/experiments/bootstrap/e17_shards/shard_6.answers.json new file mode 100644 index 00000000..57e62628 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_6.answers.json @@ -0,0 +1,32 @@ +[ + { + "id": 11, + "mode": "A_baseline", + "answer": "Based only on the information provided, I cannot determine which existing tests exercise `decorator`. The prompt shows the function code but no test files, no tests-directory listing, and no call sites. Since `decorator` is a nested factory inside `modification_guard` and its wrapper is applied via `@modification_guard()`, any regression test would import `modification_guard` from `src.core.modification_guard` and decorate a fake tool method. Without the contents of the repository test suite I cannot name specific test functions. Explicitly: this cannot be determined from the information given; one would need to grep the tests directory for `modification_guard` or `decorator` usages." + }, + { + "id": 11, + "mode": "B_runtime", + "answer": "According to the relevant tests supplied, the following test functions exercise this function and should be run to catch regressions:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_cli_unknown_tool\nThe first two directly decorate a fake tool with `@modification_guard()` and assert pass-through and ambiguous-symbol denial behaviour, so they clearly cover the wrapper logic. `test_cli_unknown_tool` looks unrelated on its face, because it only checks CLI argument handling and never touches `modification_guard`; it is probably a false positive in the candidate list. The two guard-specific tests are the ones that must be run." + }, + { + "id": 11, + "mode": "D_shuffled", + "answer": "The list is labelled as relevant, but on inspection none of the three tests exercises `decorator` or the modification guard wrapper. `test_empty_instance_does_not_overwrite_populated_file` covers SymbolIndex persistence, `test_passes_on_real_results` covers search-quality reporting, and `test_build_receipt_generates_id_and_steps` covers action receipts. None imports `modification_guard`, calls `@modification_guard()`, or asserts guard denial or ack behaviour, so none would catch a regression in this function. Therefore, based on the given material, I cannot name any test that genuinely exercises `decorator`; the provided candidates appear to be unrelated or shuffled distractors." + }, + { + "id": 12, + "mode": "A_baseline", + "answer": "Based only on the material given, I cannot determine which tests exercise `wrapper`. Only the function body is provided, with no test files, test names, or import sites. Because `wrapper` is defined inside the `decorator` factory and is applied through `@modification_guard()`, regressions would surface in tests that import `src.core.modification_guard`, decorate a fake tool method, and assert guard behaviour. Without access to the repository test suite I cannot list concrete test function names. Explicitly: this is not determinable from the information given; one would need to search the tests directory for `modification_guard` usages." + }, + { + "id": 12, + "mode": "B_runtime", + "answer": "Per the relevant tests provided, these test functions exercise `wrapper` and must be run to catch regressions:\ntest_no_file_path_no_symbol_passes\ntest_ambiguous_symbol_not_silently_routed_to_defs0\ntest_allows_non_hot_file\nAll three build a fake tool class and decorate its async method with `@modification_guard()`, so they drive the wrapper directly: pass-through when no target is given, denial when a bare symbol resolves to multiple definitions, and pass-through when the mocked PageRank and blast radius are below thresholds. Together they cover the early-exit, ambiguity-guard, and non-hot paths." + }, + { + "id": 12, + "mode": "D_shuffled", + "answer": "Although the three tests are presented as relevant, none of them exercises `wrapper` or the modification guard. `test_self_lock_adds_no_second_instance_issue` inspects a startup lock report, `test_fileguard_resolves_relative_project_path` checks FileGuard path resolution, and `test_trust_report_warns_on_untrusted_with_instructions` checks a trust report for instruction files. None imports or calls `modification_guard`, and none invokes `@modification_guard()` on an async method, so they cannot catch a regression in `wrapper`. Based on the given information I cannot identify a test that truly exercises it; the supplied candidates look like unrelated distractors." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_7.answers.json b/experiments/bootstrap/e17_shards/shard_7.answers.json new file mode 100644 index 00000000..ceccd62f --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_7.answers.json @@ -0,0 +1,47 @@ +[ + { + "id": 13, + "mode": "A_baseline", + "answer": "format_project_memory renders the intel_get_project_memory output as a markdown string. It takes a memory dict of sections (adrs, known_issues, tech_debt, failed_attempts), a limit defaulting to 3 for token budget (limit <= 0 shows all), and an optional stats dict. It appends a VOR coverage receipt (checked/total, starved, stale_ttl, budget_exceeded), freshness gate lines, status metrics, then iterates sections with icons, truncating items and adding an 'and N more' line. Edge cases: no stats means no receipt (backward compatibility); audit mode is auto-detected by REFUTED/SUPERSEDED statuses, then each node gets a status and retract reason. Regarding tests: the information given contains no test source and no repository listing, so I cannot determine which tests exercise this function. I must say so explicitly." + }, + { + "id": 13, + "mode": "B_runtime", + "answer": "format_project_memory converts a project-memory dict into a markdown summary. Inputs: memory (sections to items), limit (default 3; <=0 means all), optional stats for the VOR receipt. Output: a string with header, VOR coverage, freshness gate, status metrics, then per-section icons, titles (max 80 chars), verification markers (no_anchors, budget_exceeded, stale_ttl) and audit statuses (REFUTED with reason, SUPERSEDED, VERIFIED, ACTIVE). Edge cases: absent stats disables the receipt; limit truncation emits an 'and N more' line; audit mode is detected from REFUTED/SUPERSEDED presence. Tests that exercise it, from the provided snippets:\ntest_audit_mode_shows_statuses_and_reasons\ntest_budget_exceeded_node_marker\ntest_default_limit_three" + }, + { + "id": 13, + "mode": "D_shuffled", + "answer": "format_project_memory formats the project memory dict into markdown. It builds a header, optionally a VOR receipt from stats (coverage, starved, stale_ttl, budget warning, freshness gate, status metrics), then walks each non-empty section adding icons, entry counts, truncated titles with verification flags, and audit statuses when REFUTED/SUPERSEDED are present. limit defaults to 3; <=0 shows every node; hidden nodes get an 'and N more' line. Edge cases: no stats suppresses the receipt; title truncation at 80 chars; the import of TTL_STALE_DAYS is guarded. On tests: the three snippets shown (test_verdict_env_block_is_inconclusive_not_refuted, test_invalidates_bm25, test_blocked_string_pattern_eval) do not call format_project_memory and appear unrelated, so from the given information I cannot confirm any test that exercises it." + }, + { + "id": 14, + "mode": "A_baseline", + "answer": "validate_code statically validates Python source before sandbox execution in two layers. Layer 1 scans the raw string for blocked patterns using a word-boundary regex to avoid false positives. Layer 2 parses the code with ast.parse (SyntaxError becomes a SandboxViolation) and walks all nodes, blocking non-allowlisted imports and from-imports, calls to eval/exec/compile and __import__, dangerous dunder attribute access, and blocked builtin names. Input: a code string. Output: a list of non-fatal warnings (always empty in this implementation). Raises SandboxViolation on the first hard block. Edge cases: empty code and safe code pass; attribute-vs-name checks distinguish node types. I cannot determine the tests: no test source or repository listing is provided, so I must say so explicitly." + }, + { + "id": 14, + "mode": "B_runtime", + "answer": "validate_code is a two-layer static sandbox validator for Python source. Layer 1 regex-scans raw text for blocked string patterns with word boundaries; Layer 2 parses via ast.parse and walks nodes to reject non-allowlisted imports and from-imports, eval/exec/compile and __import__ calls, dangerous dunder attribute access, and blocked names. It returns a list of non-fatal warnings (currently always empty) and raises SandboxViolation on the first hard block, including SyntaxError. Edge cases: empty code parses fine, safe code with math.sqrt passes, and the node-type branches keep attribute access separate from bare name usage. Tests exercising it, from the provided snippets:\ntest_allowed_modules\ntest_empty_code\ntest_safe_code_passes" + }, + { + "id": 14, + "mode": "D_shuffled", + "answer": "validate_code performs pre-execution sandbox validation in two layers. First a fast regex string scan over BLOCKED_STR_PATTERNS with a negative lookbehind word boundary; then an AST walk that rejects imports not in ALLOWED_MODULES, dangerous calls (eval, exec, compile, __import__), dangerous dunder attributes, and blocked names. It returns a list of warnings (non-fatal) and raises SandboxViolation on any hard block, converting SyntaxError into a violation. Edge cases: empty input, safe code, and distinguishing attribute access from bare names. Regarding tests: the snippets given (test_enrichment_empty_results, test_layer_windows_backslashes, test_build_call_graph_depth_2) concern search enrichment, path layer inference, and call graphs, not validate_code, so I cannot confirm they exercise it from the information given." + }, + { + "id": 15, + "mode": "A_baseline", + "answer": "_extract_yarn_lock parses a yarn.lock file, supporting v1, v2 and berry v10 formats. It reads line by line, treating non-indented lines ending in colon as block keys, then captures the following version line. A nested flush helper emits a ManifestEntry (npm ecosystem, normalized name, version, lockfile source, line number) only when both key and version were seen. Aliased packages are handled: when a resolution line contains npm: and its name differs from the block key, the resolution name becomes the real package name. Inputs: text and source label; output: a list of ManifestEntry. Edge cases: blank lines and __metadata are skipped; quoted keys are unquoted; blocks without a version are dropped. No tests are given here, so I cannot determine which tests exercise it." + }, + { + "id": 15, + "mode": "B_runtime", + "answer": "_extract_yarn_lock parses yarn.lock files across v1, v2 and berry v10. It scans lines, treating non-indented colon-terminated lines as package block keys (stripping surrounding quotes), then records the version from the version line. Emitted entries are npm ManifestEntry objects with normalized names, deferred to flush so alias handling works: if a resolution line contains npm: with a name differing from the key, that resolution name is used instead. Blank lines and __metadata are skipped, and blocks lacking versions produce nothing. Inputs are the lock text and a source label; output is the entry list. Tests from the provided snippets:\ntest_yarn_berry_patch_descriptors\ntest_yarn_berry_v10_scoped\ntest_yarn_synthetic" + }, + { + "id": 15, + "mode": "D_shuffled", + "answer": "_extract_yarn_lock parses yarn.lock files (v1, v2, berry v10) into npm ManifestEntry records. It iterates lines: non-indented lines ending with colon start a new package block (quotes stripped), version lines set the pending version, and resolution lines detect aliases when npm: resolves to a different real name than the block key. A flush closure emits an entry only when both a key and a version exist, attaching the source and line number. Edge cases include skipping blank and __metadata lines, dropping versionless blocks, and handling aliased or patched descriptors. On tests: the three snippets shown (test_returns_project_path, test_worktree_cleaned_after_run, test_metadata_returns_dict) exercise search tool headers and change-preview worktrees, not this extractor, so I cannot confirm any test that exercises it from the information given." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_8.answers.json b/experiments/bootstrap/e17_shards/shard_8.answers.json new file mode 100644 index 00000000..187b2598 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_8.answers.json @@ -0,0 +1,47 @@ +[ + { + "id": 16, + "mode": "A_baseline", + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, no test files or repository listing. A reliable answer requires searching the repo for callers of WriteTool._action_replace. Description: this async method implements the replace action. It validates the symbol identifier and optional file_path, resolves definitions through the symbol index, and rejects missing, ambiguous, or out-of-project targets. With apply=False it returns a preview and may attach a type-check preflight. With apply=True it indents new_code, computes before/after hashes, AST-validates Python syntax, runs preflight validation, atomic-writes the file, invalidates the LSP cache, evicts the file from the symbol index, and records a contract entry. It returns user-facing markdown messages on success or error." + }, + { + "id": 16, + "mode": "B_runtime", + "answer": "The tests that exercise this function are:\ntest_unknown_symbol_returns_warning\ntest_filter_mismatch_returns_error\ntest_replace_dry_run_does_not_write\nDescription: this async method implements the replace action of a write tool. It validates the symbol identifier and optional file_path, resolves definitions via the symbol index, and errors on missing, ambiguous, or out-of-project symbols and files. When apply is False it returns a preview (optionally with a type-check preflight). When apply is True it indents new_code, records before/after hashes, AST-validates Python syntax, runs preflight validation, atomic-writes, invalidates the LSP cache, removes the file from the symbol index, and records a contract. Edge cases: empty arguments, bad identifiers, syntax errors, blocking preflight." + }, + { + "id": 16, + "mode": "D_shuffled", + "answer": "The listed tests do not exercise WriteTool._action_replace. test_own_host_v2_is_healthy covers DatabaseLock.classify_holder; test_drop_removes_entry covers LiveBuffer.drop; test_flush_now_clears_and_calls_callback covers DebounceBatch.flush_now. These are unrelated components (locking, buffering, debouncing) and give no coverage of the replace action. Therefore the real regression tests for this function cannot be determined from the information given; a repo-wide search for callers of _action_replace would be required. Description: the async method validates the symbol and file_path, resolves definitions via the symbol index, supports a dry-run preview, and on apply indents new_code, AST-validates Python syntax, runs preflight, atomic-writes, invalidates the LSP cache, and records a contract." + }, + { + "id": 17, + "mode": "A_baseline", + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, with no test files or repository listing. A reliable answer needs a repo-wide search for callers of SymbolIndex.get_call_chain. Description: this method returns call chains for a symbol by traversing the internal reference map under a lock. The direction argument selects callers (up), callees (down), or both. For each depth level it walks references, skipping definitions and the symbol itself, appending records with symbol, file, line, and depth. It returns a dict with symbol, callers_chain, callees_chain, and total_connected. Edge cases: max_depth limits traversal, visited sets prevent cycles and duplicates, and missing symbols yield empty chains." + }, + { + "id": 17, + "mode": "B_runtime", + "answer": "The tests that exercise this function are:\ntest_get_call_chain_both\ntest_get_call_chain_down\ntest_get_call_chain_total_connected\nDescription: this method returns a symbol call chain by traversing the internal references map while holding a lock. direction selects callers (up), callees (down), or both. Each level walks references, skipping definitions and the symbol itself, and records entries with symbol, file, line, and depth. The result dict holds symbol, callers_chain, callees_chain, and total_connected. Edge cases: max_depth caps traversal depth, visited sets prevent infinite loops and duplicate entries, and unknown symbols return empty chains. The three tests cover both-direction traversal, downward traversal, and the total_connected count." + }, + { + "id": 17, + "mode": "D_shuffled", + "answer": "The listed tests do not exercise SymbolIndex.get_call_chain. test_unparseable_lines_skipped covers _count_log_levels; test_old_schema_detected covers _check_sqlite_schema_health; test_wire_plugins_no_env_noop covers wire_plugins. These concern log parsing, SQLite schema checks, and plugin wiring, so they give no coverage of call-chain traversal. The real regression tests cannot be determined from the information given; a repo-wide search for callers of get_call_chain would be required. Description: the method traverses the internal references map under a lock, in the requested direction (up for callers, down for callees, both), recording symbol, file, line, and depth per entry and returning a dict with callers_chain, callees_chain, and total_connected, bounded by max_depth and cycle-guarding visited sets." + }, + { + "id": 18, + "mode": "A_baseline", + "answer": "I cannot determine the specific test names from the information given: only the function source was provided, with no test files or repository listing. A reliable answer would require searching the repo for callers of load_manifest. Description: this function reads and validates a plugin.json manifest without executing plugin code. It parses JSON and raises PluginManifestError with kinds manifest_missing or manifest_invalid on file or parse errors, checks required fields, normalizes id/name/version, enforces schema_version against _SCHEMA_VERSION, validates platform compatibility, defaults and validates requires_engine_version with SpecifierSet, normalizes tools into a non-empty list of strings, and cleans dependencies. It returns a ToolPlugin instance. Edge cases: invalid or empty values, wrong schema, platform mismatch, bad engine spec." + }, + { + "id": 18, + "mode": "B_runtime", + "answer": "The tests that exercise this function are:\ntest_load_invalid_engine_req\ntest_manifest_requires_missing_fields\ntest_manifest_schema_mismatch_real\nDescription: load_manifest reads and validates plugin.json without executing plugin code. It parses JSON, raising PluginManifestError (kinds manifest_missing, manifest_invalid) on file or parse errors, verifies required fields, normalizes id/name/version, enforces schema_version equality with _SCHEMA_VERSION, checks platform compatibility, defaults requires_engine_version to >=0 and validates it with SpecifierSet, normalizes tools into a non-empty string list, and cleans dependencies before returning a ToolPlugin. The three tests cover an invalid engine requirement, a missing manifest file, and a schema_version mismatch." + }, + { + "id": 18, + "mode": "D_shuffled", + "answer": "The listed tests do not exercise load_manifest. test_tokenize_truncates_over_limit_to_guarantee covers an embedder truncation helper; test_user_project_not_self_index covers _is_self_index_path; test_empty_results covers Searcher._extract_key_terms. These concern embedding, self-index detection, and search term extraction, so they give no coverage of manifest loading. The real regression tests cannot be determined from the information given; a repo-wide search for callers of load_manifest would be needed. Description: the function reads and validates plugin.json without executing plugin code, raising PluginManifestError for missing or invalid JSON, incomplete fields, schema mismatch, platform mismatch, and invalid engine requirements, then returns a normalized ToolPlugin instance." + } +] diff --git a/experiments/bootstrap/e17_shards/shard_9.answers.json b/experiments/bootstrap/e17_shards/shard_9.answers.json new file mode 100644 index 00000000..5db07a53 --- /dev/null +++ b/experiments/bootstrap/e17_shards/shard_9.answers.json @@ -0,0 +1,11 @@ +[ + {"id": 19, "mode": "A_baseline", "answer": "I cannot determine this from the information given: no test files, test names, or repository context accompany the function. Based solely on the code, `_enrich_job_response` enriches a `BackgroundJob` dict. It rounds `progress`; sets `poll_interval_seconds` from status and progress bands (0 for completed/failed, else 30, 30, 15, 5); builds `progress_label` from status and progress; and for running jobs computes an adaptive ETA via `_embed_progress_from_log`, falling back to `job_history` during finalizing, with an honest `None` when no data exists. Edge cases include missing embed progress, done at or beyond total, and non-running statuses. Without the repository test sources I cannot name any test function that must be run."}, + {"id": 19, "mode": "B_runtime", "answer": "The tests that exercise `_enrich_job_response` are: test_enrich_eta_none_when_no_embed_progress, test_enrich_eta_uses_real_embed_speed, and test_intel_runtime_status_reports_reindex_progress. The first two call it directly to verify the adaptive ETA contract (None when no embed data, and 21 seconds from 347 remaining chunks at 16 ch per s). The third exercises it indirectly through `intel_get_runtime_status`, asserting reindex telemetry and accepting a None or positive ETA. The function rounds progress, computes poll_interval_seconds and progress_label from status and progress, and derives estimated_seconds for running jobs. Run these three to catch regressions in the ETA logic and the reindex status path."}, + {"id": 19, "mode": "D_shuffled", "answer": "None of the provided tests exercise `_enrich_job_response`. test_normalize_diag_uri_idempotent tests LspClient URI normalization; test_double_load_is_idempotent tests embedder tokenizer loading; test_apply_deletes_from_file tests WriteTool symbol deletion. They are unrelated to job enrichment and would not catch regressions here. The function is a pure formatter over a BackgroundJob, so a meaningful test must construct a job, call `_enrich_job_response` (or `intel_get_runtime_status`), and assert on poll_interval_seconds, progress_label, eta_phase, or estimated_seconds. From the information given I cannot identify any real test names; I can only state that these three do not cover it."}, + {"id": 20, "mode": "A_baseline", "answer": "I cannot determine this from the information given: no test files or test names are supplied with the function. `format_runtime_status` converts a status dict into a Markdown dashboard with traffic-light emojis. It reads project path, PID, embedding provider, provider statuses, index telemetry, reranker RAM, reindex flags, and optional startup diagnostics. It derives LEDs for LM Studio, Ollama, the active provider, the index, overall health, and the reranker; renders chunk/file/symbol counts or a Reindex in progress line with percent and ETA; and short-circuits to a warning block when lock or DB state is unhealthy. Without the repository test sources I cannot list the tests that must be run."}, + {"id": 20, "mode": "B_runtime", "answer": "The relevant tests are: test_format_runtime_status_fallback_to_768_when_no_model_info, test_format_runtime_status_normal_path_unchanged, and test_format_runtime_status_reindex_without_progress. The first checks the ONNX dimension fallback when model_info is absent. The second confirms the unchanged normal path yields 0 chunks for an empty index. The third verifies the reindex branch shows Reindex in progress and never 0 chunks, even with missing progress and ETA. Together they cover the traffic-light logic, provider and model naming, index telemetry rendering, and the reindex short-circuit. Run all three before changing format_runtime_status to catch regressions in these rendering paths."}, + {"id": 20, "mode": "D_shuffled", "answer": "None of the provided tests exercise `format_runtime_status`. test_env_gate_off checks the language-pack environment gate; test_blocked_import_subprocess checks sandbox validation of imports; test_iteration_1_adds_terms checks the searcher refined-query generation. All are unrelated to runtime status formatting and would not catch regressions here. A real test would build a status dict and assert on the rendered dashboard, provider LEDs, the index chunk line, or the startup-diagnostic warning block. Based only on the information given, I cannot identify any actual test names that cover this function; the three shown clearly do not."}, + {"id": 21, "mode": "A_baseline", "answer": "I cannot determine this from the information given: no tests or repository context accompany the function. `_delayed_auto_index` runs a delayed auto-index after server startup. It resolves an indexer, skips when the project root equals ext_root or the index already has chunks, logs a human-readable startup diagnostic for an empty or corrupt DB, waits 1.5 seconds, sets a reindexing guard, runs `index_project` in a thread, transitions the registry state to READY, and always clears the guard in a finally block. Failures are logged, not raised. Without the repository test sources I cannot name any test function that must be run."}, + {"id": 21, "mode": "B_runtime", "answer": "The tests exercising `_delayed_auto_index` are: test_delayed_auto_index_skips_when_index_not_empty and test_delayed_auto_index_sets_state_ready. The first asserts that a non-empty index of 9000 chunks causes an early return, that `index_project` is not called, and that `set_state` is never invoked. The second asserts that with an empty index `index_project` runs, returning 42, and the registry is set to ProjectState.READY. Both stub `resolve_indexer_for_request` and the DI registry. Run these two to catch regressions in the skip conditions, the reindexing guard, and the READY state transition."}, + {"id": 21, "mode": "D_shuffled", "answer": "None of the provided tests exercise `_delayed_auto_index`. test_lazy_graph_build checks that CrossProjectDependencyGraph lazily initializes its `_graph`; test_dedup_preserves_order checks that `extract_imports` deduplicates modules while preserving order. Both are unrelated to auto-indexing and would not catch regressions in the function. A meaningful test would stub the indexer and the DI registry, then assert skip behavior, `index_project` invocation, and the READY state transition. From the information given I cannot identify any real test names covering `_delayed_auto_index`; the two shown are clearly unrelated modules."} +] diff --git a/experiments/bootstrap/e17_smoke_check.py b/experiments/bootstrap/e17_smoke_check.py new file mode 100644 index 00000000..f3b230f6 --- /dev/null +++ b/experiments/bootstrap/e17_smoke_check.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +"""E17 smoke check: verify the pilot pipeline is not broken. + +Reproduces exactly what e17_pilot_answers.py does (read_function_code / +read_test_code / build_prompt) and reports, per mode, whether the prompt +actually contains test code or a "# Test not found" placeholder. +No LLM calls. +""" +import sys +import json +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] + + +def read_function_code(file_path: str, function_name: str) -> str: + local = file_path.replace("D:/Project/MSCodeBase/", "") + p = ROOT / local + if not p.exists(): + return "# FILE NOT FOUND" + lines = p.read_text(encoding="utf-8", errors="ignore").split("\n") + out, inf, ind = [], False, None + for line in lines: + if f"def {function_name}" in line or f"async def {function_name}" in line: + inf, ind = True, len(line) - len(line.lstrip()) + out.append(line) + elif inf: + ci = len(line) - len(line.lstrip()) if line.strip() else ind + 1 + if line.strip() and ci <= ind: + break + out.append(line) + return "\n".join(out) if out else "# FUNC NOT FOUND" + + +def read_test_code(n: str) -> str: + for tf in (ROOT / "tests").glob("*.py"): + c = tf.read_text(encoding="utf-8", errors="ignore") + if f"def {n}" in c or f"class {n}" in c: + return f"# FOUND {tf.name}" + return f"# Test not found: {n}" + + +def main() -> None: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + by_id = {f["id"]: f for f in data["functions"]} + for fid in (13, 1): + f = by_id[fid] + print("=" * 72) + print(f"[{fid}] {f['name']} ({f['file_path']})") + code = read_function_code(f["file_path"], f["name"]) + print(f" code bytes={len(code)} head={code[:70].replace(chr(10), ' ')!r}") + for m in ("A_baseline", "B_runtime", "C_static", "D_shuffled"): + tests = f["modes"][m]["tests"] + section = "".join(read_test_code(t) + "\n" for t in tests) + verdict = "EMPTY(=A)" if not tests else ( + "ALL-NOT-FOUND(=A)" if "not found" in section and "FOUND" not in section + else "HAS-REAL-TESTS" + ) + print(f" {m:12} tests={len(tests)} -> {verdict} {section[:90]!r}") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_smoke_prompts.py b/experiments/bootstrap/e17_smoke_prompts.py new file mode 100644 index 00000000..26e7447c --- /dev/null +++ b/experiments/bootstrap/e17_smoke_prompts.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +"""E17 smoke prompts: dump real A/B prompts using the FIXED AST extraction. + +Feeds a plain LLM (subagent) so we can observe whether B differs from A. +No LLM calls here. +""" +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code, read_tests # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + + +def build_prompt(question: str, code: str, tests) -> str: + prompt = ( + "You are an expert Python developer. Answer the following question about " + f"a function in a codebase.\n\nQuestion: {question}\n\nFunction code:\n" + f"```python\n{code}\n```\n" + ) + bodies = read_tests(tests) + if bodies: + prompt += "\nRelevant tests that exercise this function:\n" + for body in bodies: + prompt += f"\n```python\n{body}\n```\n" + prompt += ( + "\nProvide a clear, accurate explanation of what the function does, its " + "purpose, inputs, outputs, and any edge cases it handles." + ) + return prompt + + +def main() -> None: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + by_id = {f["id"]: f for f in data["functions"]} + out = [] + for fid in (13, 1): + f = by_id[fid] + code = read_function_code(f["file_path"], f["name"]) + if not code: + print(f"[{fid}] SKIP {f['name']}: code not found") + continue + for m in ("A_baseline", "B_runtime"): + out.append({ + "id": fid, + "name": f["name"], + "mode": m, + "question": f["question"], + "prompt": build_prompt(f["question"], code, f["modes"][m]["tests"]), + }) + (ROOT / "experiments/bootstrap/e17_smoke_prompts.json").write_text( + json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8" + ) + for o in out: + has_tests = "Relevant tests" in o["prompt"] + print(f"[{o['id']}] {o['mode']}: len={len(o['prompt']):5} tests_block={has_tests}") + print("saved -> experiments/bootstrap/e17_smoke_prompts.json") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_specificity_probe.py b/experiments/bootstrap/e17_specificity_probe.py new file mode 100644 index 00000000..18b3fb75 --- /dev/null +++ b/experiments/bootstrap/e17_specificity_probe.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""E17 specificity probe: distribution of test coverage per function. + +Reads the PropertyGraph SQLite directly (no MCP) and reports, per Function +node, the number of TESTS edges and the *minimum* coverage among its tests +(coverage = number of functions a single test executes). Small min-coverage += the function has a specific test. No writes. +""" +import sys +import sqlite3 +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.stdout.reconfigure(encoding="utf-8") + +from src.core.artifact_paths import get_graph_db_path # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + +QUERY = """ +SELECT f.name, + f.file_path, + COUNT(*) AS n_tests, + MIN(tc.cov) AS min_cov +FROM nodes f +JOIN edges e ON e.target_id = f.id AND e.type = 'TESTS' +JOIN ( + SELECT source_id, COUNT(*) AS cov + FROM edges WHERE type = 'TESTS' GROUP BY source_id +) tc ON tc.source_id = e.source_id +WHERE f.label = 'Function' +GROUP BY f.id +ORDER BY min_cov ASC, n_tests ASC +""" + + +def main() -> None: + db = get_graph_db_path(ROOT) + print(f"graph db: {db}") + conn = sqlite3.connect(str(db)) + rows = conn.execute(QUERY).fetchall() + + total = conn.execute( + "SELECT COUNT(DISTINCT target_id) FROM edges WHERE type='TESTS'" + ).fetchone()[0] + print(f"functions with >=1 TESTS edge: {total}") + print(f"rows returned: {len(rows)}") + + buckets = {"1-5": 0, "6-20": 0, "21-50": 0, "51-150": 0, "151+": 0} + for _, _, _, min_cov in rows: + if min_cov <= 5: + buckets["1-5"] += 1 + elif min_cov <= 20: + buckets["6-20"] += 1 + elif min_cov <= 50: + buckets["21-50"] += 1 + elif min_cov <= 150: + buckets["51-150"] += 1 + else: + buckets["151+"] += 1 + print("\nmin-coverage buckets:", buckets) + + print("\nTop 40 most-specific functions:") + for name, fp, n_tests, min_cov in rows[:40]: + short = fp.replace("D:/Project/MSCodeBase/", "") + print(f" min_cov={min_cov:4} n_tests={n_tests:4} {name:42} {short}") + conn.close() + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_stats.py b/experiments/bootstrap/e17_stats.py new file mode 100644 index 00000000..4e41a28d --- /dev/null +++ b/experiments/bootstrap/e17_stats.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""E17 stats: paired analysis of judge scores (A vs B vs D). + +Joins token scores with mapping.json, aggregates per (function, mode), and +reports means, paired t-test, Cohen's d and bootstrap CI for B-A and B-D. +Also computes a manipulation check: share of answers that mention one of +their provided test names (B/D) — replaces the judge's evidence_usage metric. +""" +import sys +import json +import math +import random +from pathlib import Path +from collections import defaultdict + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +JUDGE = ROOT / "experiments/bootstrap/e17_judge" +MODES = ("A_baseline", "B_runtime", "D_shuffled") + + +def load_scores(): + mapping = json.loads((JUDGE / "mapping.json").read_text(encoding="utf-8")) + scores = {} + for path in sorted(JUDGE.glob("shard_*.scores.json")): + for it in json.loads(path.read_text(encoding="utf-8")): + scores[it["token"]] = it + return mapping, scores + + +def paired_stats(diffs): + n = len(diffs) + mean = sum(diffs) / n + var = sum((d - mean) ** 2 for d in diffs) / (n - 1) if n > 1 else 0.0 + sd = math.sqrt(var) + if sd == 0: + return mean, 0.0, float("nan"), float("inf") + t = mean / (sd / math.sqrt(n)) + try: + from scipy import stats # type: ignore + p = 2 * stats.t.sf(abs(t), df=n - 1) + except Exception: + p = float("nan") + return mean, mean / sd, p, t + + +def bootstrap_ci(diffs, n_boot=10000, seed=7): + random.seed(seed) + n = len(diffs) + means = sorted(sum(random.choice(diffs) for _ in range(n)) / n for _ in range(n_boot)) + return means[int(0.025 * n_boot)], means[int(0.975 * n_boot)] + + +def manipulation_check(): + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + answers = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_answers.json").read_text(encoding="utf-8") + ) + by_id = {a["id"]: a for a in answers["answers"]} + print("\nmanipulation check — share of answers mentioning a provided test name:") + for mode in MODES: + hit = tot = 0 + for f in data["functions"]: + names = f["modes"][mode]["tests"] + if not names: + continue + tot += 1 + answer = by_id[f["id"]]["modes"][mode]["answer"] + short = [n.split("::")[-1] for n in names] + if any(s in answer for s in short): + hit += 1 + print(f" {mode:12} {hit}/{tot} = {hit / tot:.0%}" if tot else f" {mode:12} n/a") + + +def main() -> None: + mapping, scores = load_scores() + missing = [t for t in mapping if t not in scores] + if missing: + print("MISSING tokens:", missing) + return + + by_func = defaultdict(dict) + for token, meta in mapping.items(): + by_func[meta["id"]][meta["mode"]] = scores[token] + + metrics = [m for m in ("accuracy", "completeness", "safety") if m in next(iter(scores.values()))] + print(f"functions: {len(by_func)} | tokens: {len(scores)} | metrics: {metrics}") + + print("\nper-mode means:") + for metric in metrics: + row = {} + for mode in MODES: + vals = [by_func[f][mode][metric] for f in by_func if mode in by_func[f]] + row[mode] = sum(vals) / len(vals) + print(f" {metric:16} A={row['A_baseline']:.2f} B={row['B_runtime']:.2f} D={row['D_shuffled']:.2f}") + + for metric in metrics: + for hi, lo in (("B_runtime", "A_baseline"), ("B_runtime", "D_shuffled")): + diffs = [ + by_func[f][hi][metric] - by_func[f][lo][metric] + for f in by_func + if hi in by_func[f] and lo in by_func[f] + ] + mean, d, p, t = paired_stats(diffs) + lo_ci, hi_ci = bootstrap_ci(diffs) + wins = sum(1 for x in diffs if x > 0) + losses = sum(1 for x in diffs if x < 0) + print(f"\n{metric}: {hi} - {lo}") + print(f" mean_diff={mean:+.3f} d={d:+.3f} t={t:+.3f} p={p:.4f}") + print(f" bootstrap95%=[{lo_ci:+.3f}, {hi_ci:+.3f}] wins/losses/ties=" + f"{wins}/{losses}/{len(diffs) - wins - losses}") + + manipulation_check() + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_strength_profile.json b/experiments/bootstrap/e17_strength_profile.json new file mode 100644 index 00000000..2bd24777 --- /dev/null +++ b/experiments/bootstrap/e17_strength_profile.json @@ -0,0 +1,224 @@ +{ + "tests": [ + { + "function_id": 1, + "test": "test_run_survives_single_cpu_host", + "kill_rate": 0.5, + "asserts": 2, + "loc": 15, + "density": 0.133, + "exact_compare": 1 + }, + { + "function_id": 1, + "test": "test_run_with_existing_known_hashes_skips_unchanged", + "kill_rate": 0.75, + "asserts": 3, + "loc": 40, + "density": 0.075, + "exact_compare": 2 + }, + { + "function_id": 1, + "test": "test_workers_receive_known_hashes_not_none", + "kill_rate": 0.5, + "asserts": 4, + "loc": 15, + "density": 0.267, + "exact_compare": 1 + }, + { + "function_id": 2, + "test": "TestConcurrentEmbedIsolation::test_rapid_sequential_calls_no_state_leak", + "kill_rate": 1.0, + "asserts": 3, + "loc": 19, + "density": 0.158, + "exact_compare": 1 + }, + { + "function_id": 2, + "test": "TestConcurrentEmbedIsolation::test_single_call_results_match_indices", + "kill_rate": 1.0, + "asserts": 4, + "loc": 11, + "density": 0.364, + "exact_compare": 2 + }, + { + "function_id": 3, + "test": "TestCypherLexer::test_tokenize_optional_match", + "kill_rate": 0.5, + "asserts": 2, + "loc": 5, + "density": 0.4, + "exact_compare": 0 + }, + { + "function_id": 3, + "test": "TestCypherLexer::test_tokenize_order_limit", + "kill_rate": 0.5, + "asserts": 2, + "loc": 5, + "density": 0.4, + "exact_compare": 0 + }, + { + "function_id": 3, + "test": "TestCypherLexer::test_tokenize_relationship", + "kill_rate": 0.5, + "asserts": 2, + "loc": 5, + "density": 0.4, + "exact_compare": 0 + }, + { + "function_id": 4, + "test": "test_extract_doc_references_handles_empty_call_syntax", + "kill_rate": 0.0, + "asserts": 1, + "loc": 13, + "density": 0.077, + "exact_compare": 0 + }, + { + "function_id": 5, + "test": "test_chunk_cache_disabled_without_table", + "kill_rate": 0.75, + "asserts": 1, + "loc": 7, + "density": 0.143, + "exact_compare": 1 + }, + { + "function_id": 5, + "test": "test_chunk_cache_empty_file", + "kill_rate": 0.25, + "asserts": 2, + "loc": 7, + "density": 0.286, + "exact_compare": 1 + }, + { + "function_id": 5, + "test": "test_chunk_cache_all_new", + "kill_rate": 0.75, + "asserts": 4, + "loc": 13, + "density": 0.308, + "exact_compare": 4 + }, + { + "function_id": 6, + "test": "test_extract_anchors_explicit_pkg_syntax", + "kill_rate": 0.75, + "asserts": 1, + "loc": 5, + "density": 0.2, + "exact_compare": 1 + }, + { + "function_id": 7, + "test": "TestCwdFirstIsolation::test_provided_wins_over_cwd", + "kill_rate": 0.25, + "asserts": 1, + "loc": 6, + "density": 0.167, + "exact_compare": 1 + }, + { + "function_id": 7, + "test": "TestCwdSelfIndexGuard::test_cwd_equal_ext_root_is_skipped", + "kill_rate": 0.75, + "asserts": 1, + "loc": 9, + "density": 0.111, + "exact_compare": 1 + }, + { + "function_id": 7, + "test": "TestPriorityProjectPathOverride::test_mscb_path_trusts_ext_root", + "kill_rate": 0.0, + "asserts": 1, + "loc": 9, + "density": 0.111, + "exact_compare": 1 + }, + { + "function_id": 8, + "test": "test_build_call_graph_call_chain", + "kill_rate": 0.25, + "asserts": 1, + "loc": 7, + "density": 0.143, + "exact_compare": 0 + }, + { + "function_id": 8, + "test": "test_build_call_graph_callees", + "kill_rate": 0.0, + "asserts": 2, + "loc": 7, + "density": 0.286, + "exact_compare": 0 + }, + { + "function_id": 8, + "test": "test_build_call_graph_callers", + "kill_rate": 0.0, + "asserts": 1, + "loc": 6, + "density": 0.167, + "exact_compare": 0 + }, + { + "function_id": 9, + "test": "test_bs1_parse_file_skips_comment_only", + "kill_rate": 0.0, + "asserts": 1, + "loc": 12, + "density": 0.083, + "exact_compare": 0 + }, + { + "function_id": 9, + "test": "TestASTCacheInvalidation::test_property_graph_consistency", + "kill_rate": 0.5, + "asserts": 5, + "loc": 59, + "density": 0.085, + "exact_compare": 1 + }, + { + "function_id": 10, + "test": "TestModificationGuard::test_no_file_path_no_symbol_passes", + "kill_rate": 0.0, + "asserts": 1, + "loc": 12, + "density": 0.083, + "exact_compare": 1 + }, + { + "function_id": 10, + "test": "TestAmbiguousSymbolTarget::test_ambiguous_symbol_not_silently_routed_to_defs0", + "kill_rate": 0.5, + "asserts": 1, + "loc": 30, + "density": 0.033, + "exact_compare": 0 + }, + { + "function_id": 10, + "test": "TestModificationGuard::test_allows_non_hot_file", + "kill_rate": 0.25, + "asserts": 2, + "loc": 23, + "density": 0.087, + "exact_compare": 1 + } + ], + "decoy": { + "killed": 0, + "total": 104 + } +} \ No newline at end of file diff --git a/experiments/bootstrap/e17_strength_profile.py b/experiments/bootstrap/e17_strength_profile.py new file mode 100644 index 00000000..f93f1a28 --- /dev/null +++ b/experiments/bootstrap/e17_strength_profile.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +"""E17 verification-strength profiling. + +For each panel function: generate several behaviour mutations (negate-if, +flip-compare, bool-flip, int+1, str-append), run the graph-linked tests (arm B) +and the decoy tests (arm D) under each mutation, and record which tests fail. +Then relate each test's kill-rate to cheap AST features (assert count, density, +comparison specificity) to see what actually predicts detection. + +No LLM. Files restored via `git checkout --` in a finally block. +""" +import ast +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +BOOT = ROOT / "experiments/bootstrap" +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(BOOT)) +K_FUNCS = int(sys.argv[1]) if len(sys.argv) > 1 else 10 +MAX_MUT = int(sys.argv[2]) if len(sys.argv) > 2 else 4 + +_OPS = {ast.Eq: ("==", "!="), ast.NotEq: ("!=", "=="), ast.Lt: ("<", ">"), + ast.Gt: (">", "<"), ast.LtE: ("<=", ">="), ast.GtE: (">=", "<=")} + + +def _sh(cmd, timeout=300): + return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + + +def graph_test_files(): + import sqlite3 + from src.core.artifact_paths import get_graph_db_path + conn = sqlite3.connect(str(get_graph_db_path(ROOT))) + rows = conn.execute("SELECT name, file_path FROM nodes WHERE label='Test'").fetchall() + conn.close() + out = {} + for name, fp in rows: + out.setdefault(name, fp) + return out + + +def node_id(rel_file, test_name): + cls, _, meth = test_name.partition("::") + return f"{rel_file}::{cls}::{meth}" if meth else f"{rel_file}::{test_name}" + + +def find_func(tree, qualname): + parts = qualname.split(".") + if len(parts) == 1: + for n in ast.walk(tree): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == parts[0]: + return n + return None + for n in ast.walk(tree): + if isinstance(n, ast.ClassDef) and n.name == parts[-2]: + for item in n.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) and item.name == parts[-1]: + return item + return None + + +def mutations(src, qualname, limit): + """Yield up to `limit` distinct (mutated_src, description).""" + try: + tree = ast.parse(src) + except SyntaxError: + return [] + node = find_func(tree, qualname) + if node is None: + return [] + lines = src.split("\n") + out = [] + + def add(ln_no, col, end_col, text, desc): + nl = list(lines) + nl[ln_no - 1] = lines[ln_no - 1][:col] + text + lines[ln_no - 1][end_col:] + joined = "\n".join(nl) + try: + ast.parse(joined) + except SyntaxError: + return + out.append((joined, desc)) + + for sub in ast.walk(node): + if len(out) >= limit: + break + if isinstance(sub, ast.If) and sub.test.lineno == sub.test.end_lineno: + ln = lines[sub.test.lineno - 1] + seg = ln[sub.test.col_offset:sub.test.end_col_offset] + if seg.strip(): + add(sub.test.lineno, sub.test.col_offset, sub.test.end_col_offset, + f"not ({seg})", f"negate-if@L{sub.test.lineno}") + elif isinstance(sub, ast.Compare) and len(sub.ops) == 1: + left, comp = sub.left, sub.comparators[0] + if left.lineno == left.end_lineno == comp.lineno: + ln = lines[left.lineno - 1] + span = ln[left.end_col_offset:comp.col_offset] + pair = _OPS.get(type(sub.ops[0])) + if pair and span.strip() == pair[0]: + add(left.lineno, left.end_col_offset, comp.col_offset, + f" {pair[1]} ", f"flip-cmp@{pair[0]}->{pair[1]}@L{left.lineno}") + elif isinstance(sub, ast.Constant) and sub.lineno == sub.end_lineno: + ln = lines[sub.lineno - 1] + seg = ln[sub.col_offset:sub.end_col_offset] + if isinstance(sub.value, bool) and seg in ("True", "False"): + add(sub.lineno, sub.col_offset, sub.end_col_offset, + "False" if sub.value else "True", f"bool-flip@L{sub.lineno}") + elif isinstance(sub.value, int) and seg == str(sub.value): + add(sub.lineno, sub.col_offset, sub.end_col_offset, + str(sub.value + 1), f"int+1@{sub.value}->{sub.value + 1}@L{sub.lineno}") + elif isinstance(sub.value, str) and len(sub.value) > 1 and seg[:1] in ("'", '"'): + add(sub.lineno, sub.col_offset, sub.end_col_offset, + f'"{sub.value}X"', f"str-append@L{sub.lineno}") + return out[:limit] + + +def run_verbose(ids): + """Return {nodeid: status} for a pytest -v run.""" + if not ids: + return {} + p = _sh([sys.executable, "-m", "pytest", *ids, "-v", "--no-header", + "-p", "no:cacheprovider"]) + status = {} + for m in re.finditer(r"(\S+::\S+)\s+(PASSED|FAILED|ERROR)", p.stdout): + status[m.group(1)] = m.group(2) + return status + + +def test_features(body): + """Cheap AST features of a test function body.""" + try: + tree = ast.parse(body) + except SyntaxError: + return None + asserts = [n for n in ast.walk(tree) if isinstance(n, ast.Assert)] + lines = len([ln for ln in body.splitlines() if ln.strip()]) + exact = 0 + for a in asserts: + t = a.test + if isinstance(t, ast.Compare) and any(isinstance(o, (ast.Eq, ast.NotEq)) for o in t.ops): + exact += 1 + return { + "asserts": len(asserts), + "loc": lines, + "density": round(len(asserts) / lines, 3) if lines else 0.0, + "exact_compare": exact, + } + + +def main(): + import importlib + sys.path.insert(0, str(BOOT)) + ext = importlib.import_module("e17_extract") + + data = json.loads((BOOT / "e17_pilot_data.json").read_text(encoding="utf-8")) + tfiles = graph_test_files() + + # (function_id, test_name) -> {killed, total} + kill = defaultdict(lambda: {"killed": 0, "total": 0}) + d_kill = 0 + d_total = 0 + tested_funcs = 0 + + for f in data["functions"]: + if tested_funcs >= K_FUNCS: + break + rel = f["file_path"].replace("D:/Project/MSCodeBase/", "") + src_path = ROOT / rel + if not src_path.exists(): + continue + muts = mutations(src_path.read_text(encoding="utf-8"), f["name"], MAX_MUT) + if not muts: + continue + + def _rel(t): + return Path(tfiles[t]).resolve().relative_to(ROOT).as_posix() + + b_tests = [t for t in f["modes"]["B_runtime"]["tests"] if t in tfiles] + d_tests = [t for t in f["modes"]["D_shuffled"]["tests"] if t in tfiles] + b_ids = [(t, node_id(_rel(t), t)) for t in b_tests] + d_ids = [(t, node_id(_rel(t), t)) for t in d_tests] + tested_funcs += 1 + print(f"\n[{f['name']}] {len(muts)} mutations") + try: + for new_src, desc in muts: + src_path.write_text(new_src, encoding="utf-8") + st = run_verbose([nid for _, nid in b_ids]) + for name, nid in b_ids: + kill[(f["id"], name)]["total"] += 1 + if st.get(nid) in ("FAILED", "ERROR"): + kill[(f["id"], name)]["killed"] += 1 + dst = run_verbose([nid for _, nid in d_ids]) + for name, nid in d_ids: + d_total += 1 + if dst.get(nid) in ("FAILED", "ERROR"): + d_kill += 1 + print(f" {desc}: {sum(1 for _, nid in b_ids if st.get(nid) in ('FAILED','ERROR'))} B-tests killed") + finally: + _sh(["git", "checkout", "--", rel]) + + rows = [] + for (fid, name), v in kill.items(): + body = ext.read_test_code(name) or "" + feats = test_features(body) or {} + rows.append({"function_id": fid, "test": name, + "kill_rate": round(v["killed"] / v["total"], 3) if v["total"] else 0.0, + **feats}) + (BOOT / "e17_strength_profile.json").write_text( + json.dumps({"tests": rows, "decoy": {"killed": d_kill, "total": d_total}}, + ensure_ascii=False, indent=2), encoding="utf-8") + + print("\n" + "=" * 72) + n = len(rows) + full = sum(1 for r in rows if r["kill_rate"] >= 1.0) + zero = sum(1 for r in rows if r["kill_rate"] == 0.0) + print(f"tests profiled: {n} across {tested_funcs} functions") + print(f"kill_rate = 1.0: {full} kill_rate = 0.0: {zero}") + print(f"decoy tests killed: {d_kill}/{d_total}") + try: + from scipy import stats + for feat in ("asserts", "density", "exact_compare", "loc"): + xs = [r[feat] for r in rows if feat in r] + ys = [r["kill_rate"] for r in rows if feat in r] + if len(set(xs)) > 1: + rho, p = stats.spearmanr(xs, ys) + print(f" spearman(kill_rate, {feat:13}) rho={rho:+.2f} p={p:.3f}") + except Exception as e: + print(" (scipy unavailable:", e, ")") + print("=" * 72) + for r in rows: + print(f" kill={r['kill_rate']:.2f} asserts={r.get('asserts')} dens={r.get('density')} " + f"exact={r.get('exact_compare')} {r['test']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/bootstrap/e17_strength_scaled.json b/experiments/bootstrap/e17_strength_scaled.json new file mode 100644 index 00000000..17e729cf --- /dev/null +++ b/experiments/bootstrap/e17_strength_scaled.json @@ -0,0 +1,517 @@ +{ + "tests": [ + { + "function_id": 1, + "test": "test_legacy_project_dirs_detected", + "kill_rate": 1.0, + "asserts": 1, + "loc": 6, + "density": 0.167, + "exact_compare": 1 + }, + { + "function_id": 2, + "test": "TestGetIdleMetrics::test_idle_plus_active", + "kill_rate": 0.0, + "asserts": 2, + "loc": 13, + "density": 0.154, + "exact_compare": 0 + }, + { + "function_id": 3, + "test": "TestVerifyGitPush::test_returns_dict", + "kill_rate": 0.0, + "asserts": 2, + "loc": 6, + "density": 0.333, + "exact_compare": 0 + }, + { + "function_id": 4, + "test": "test_incremental_fts5_skips_when_not_built", + "kill_rate": 0.0, + "asserts": 0, + "loc": 11, + "density": 0.0, + "exact_compare": 0 + }, + { + "function_id": 5, + "test": "test_ips_are_global", + "kill_rate": 0.75, + "asserts": 8, + "loc": 9, + "density": 0.889, + "exact_compare": 0 + }, + { + "function_id": 6, + "test": "test_bs14_load_metrics_clamps_negative_total_ms", + "kill_rate": 0.5, + "asserts": 2, + "loc": 25, + "density": 0.08, + "exact_compare": 2 + }, + { + "function_id": 6, + "test": "test_bs14_load_metrics_sanitizes_negative", + "kill_rate": 0.5, + "asserts": 3, + "loc": 30, + "density": 0.1, + "exact_compare": 1 + }, + { + "function_id": 7, + "test": "test_bs14_summary_shows_zero_for_negative", + "kill_rate": 0.25, + "asserts": 1, + "loc": 13, + "density": 0.077, + "exact_compare": 1 + }, + { + "function_id": 7, + "test": "test_bs8_telemetry_uses_active_embedder", + "kill_rate": 0.0, + "asserts": 2, + "loc": 18, + "density": 0.111, + "exact_compare": 2 + }, + { + "function_id": 8, + "test": "TestVerifyGitCommit::test_expected_message_mismatch", + "kill_rate": 0.5, + "asserts": 2, + "loc": 8, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 8, + "test": "TestVerifyGitCommit::test_no_git_repo_fails", + "kill_rate": 0.0, + "asserts": 2, + "loc": 14, + "density": 0.143, + "exact_compare": 0 + }, + { + "function_id": 9, + "test": "test_bs1_has_code_lines", + "kill_rate": 0.0, + "asserts": 1, + "loc": 3, + "density": 0.333, + "exact_compare": 0 + }, + { + "function_id": 9, + "test": "TestASTCacheInvalidation::test_property_graph_consistency", + "kill_rate": 0.75, + "asserts": 5, + "loc": 59, + "density": 0.085, + "exact_compare": 1 + }, + { + "function_id": 10, + "test": "test_bs5_topology_filters_empty_callees", + "kill_rate": 0.0, + "asserts": 2, + "loc": 16, + "density": 0.125, + "exact_compare": 2 + }, + { + "function_id": 10, + "test": "test_bs5_topology_reads_symbol_key", + "kill_rate": 0.5, + "asserts": 4, + "loc": 23, + "density": 0.174, + "exact_compare": 4 + }, + { + "function_id": 11, + "test": "TestGraphResolverHelper::test_resolve_relative_import", + "kill_rate": 1.0, + "asserts": 2, + "loc": 5, + "density": 0.4, + "exact_compare": 2 + }, + { + "function_id": 11, + "test": "TestGraphSymbolResolver::test_import_fqn_match", + "kill_rate": 0.0, + "asserts": 9, + "loc": 55, + "density": 0.164, + "exact_compare": 7 + }, + { + "function_id": 12, + "test": "TestGrepFallbackProjectRoot::test_does_not_scan_extension_dir", + "kill_rate": 0.0, + "asserts": 2, + "loc": 8, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 12, + "test": "TestGrepFallbackProjectRoot::test_searches_resolved_project", + "kill_rate": 0.75, + "asserts": 1, + "loc": 7, + "density": 0.143, + "exact_compare": 0 + }, + { + "function_id": 13, + "test": "TestFormatVerificationReport::test_all_success", + "kill_rate": 0.5, + "asserts": 2, + "loc": 9, + "density": 0.222, + "exact_compare": 0 + }, + { + "function_id": 13, + "test": "TestFormatVerificationReport::test_empty_results", + "kill_rate": 0.25, + "asserts": 1, + "loc": 4, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 13, + "test": "TestFormatVerificationReport::test_with_errors", + "kill_rate": 0.75, + "asserts": 2, + "loc": 8, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 14, + "test": "test_normalize_diag_uri_already_canonical", + "kill_rate": 0.0, + "asserts": 1, + "loc": 3, + "density": 0.333, + "exact_compare": 1 + }, + { + "function_id": 14, + "test": "test_normalize_diag_uri_idempotent", + "kill_rate": 0.0, + "asserts": 1, + "loc": 5, + "density": 0.2, + "exact_compare": 1 + }, + { + "function_id": 14, + "test": "test_normalize_diag_uri_win_drive", + "kill_rate": 1.0, + "asserts": 1, + "loc": 4, + "density": 0.25, + "exact_compare": 1 + }, + { + "function_id": 15, + "test": "test_counts_level_markers_not_substrings", + "kill_rate": 0.75, + "asserts": 2, + "loc": 10, + "density": 0.2, + "exact_compare": 2 + }, + { + "function_id": 15, + "test": "test_unparseable_lines_skipped", + "kill_rate": 0.5, + "asserts": 2, + "loc": 6, + "density": 0.333, + "exact_compare": 2 + }, + { + "function_id": 15, + "test": "test_window_excludes_old_errors", + "kill_rate": 0.5, + "asserts": 1, + "loc": 8, + "density": 0.125, + "exact_compare": 1 + }, + { + "function_id": 16, + "test": "test_enrich_eta_none_when_no_embed_progress", + "kill_rate": 0.25, + "asserts": 2, + "loc": 19, + "density": 0.105, + "exact_compare": 0 + }, + { + "function_id": 16, + "test": "test_enrich_eta_uses_real_embed_speed", + "kill_rate": 0.25, + "asserts": 2, + "loc": 23, + "density": 0.087, + "exact_compare": 2 + }, + { + "function_id": 16, + "test": "test_intel_runtime_status_reports_reindex_progress", + "kill_rate": 0.0, + "asserts": 4, + "loc": 56, + "density": 0.071, + "exact_compare": 2 + }, + { + "function_id": 18, + "test": "test_wire_plugins_no_env_noop", + "kill_rate": 0.667, + "asserts": 2, + "loc": 5, + "density": 0.4, + "exact_compare": 0 + }, + { + "function_id": 18, + "test": "test_wire_plugins_untrusted_skipped", + "kill_rate": 0.0, + "asserts": 1, + "loc": 7, + "density": 0.143, + "exact_compare": 0 + }, + { + "function_id": 18, + "test": "test_wire_plugins_registers_and_calls", + "kill_rate": 1.0, + "asserts": 4, + "loc": 20, + "density": 0.2, + "exact_compare": 1 + }, + { + "function_id": 19, + "test": "test_go_mod_synthetic_replace_excluded", + "kill_rate": 0.75, + "asserts": 2, + "loc": 11, + "density": 0.182, + "exact_compare": 0 + }, + { + "function_id": 19, + "test": "test_go_mod_mux_no_require", + "kill_rate": 0.25, + "asserts": 1, + "loc": 3, + "density": 0.333, + "exact_compare": 0 + }, + { + "function_id": 19, + "test": "test_go_mod_migrate_require_and_indirect", + "kill_rate": 0.0, + "asserts": 3, + "loc": 5, + "density": 0.6, + "exact_compare": 0 + }, + { + "function_id": 20, + "test": "test_alerts_empty_returns_empty", + "kill_rate": 0.25, + "asserts": 2, + "loc": 3, + "density": 0.667, + "exact_compare": 2 + }, + { + "function_id": 20, + "test": "test_alerts_limit_caps_payload_breadth", + "kill_rate": 0.25, + "asserts": 2, + "loc": 9, + "density": 0.222, + "exact_compare": 0 + }, + { + "function_id": 20, + "test": "test_alerts_renders_kind_message_payload", + "kill_rate": 0.25, + "asserts": 5, + "loc": 20, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 21, + "test": "TestUriToPath::test_unc_uri", + "kill_rate": 0.25, + "asserts": 1, + "loc": 4, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 21, + "test": "TestUriToPath::test_windows_drive_uri", + "kill_rate": 0.75, + "asserts": 1, + "loc": 4, + "density": 0.25, + "exact_compare": 0 + }, + { + "function_id": 21, + "test": "TestUriToPath::test_round_trip_windows", + "kill_rate": 1.0, + "asserts": 1, + "loc": 12, + "density": 0.083, + "exact_compare": 1 + }, + { + "function_id": 22, + "test": "test_instruction_files_found_at_root", + "kill_rate": 0.5, + "asserts": 1, + "loc": 10, + "density": 0.1, + "exact_compare": 1 + }, + { + "function_id": 22, + "test": "test_trust_report_warns_on_untrusted_with_instructions", + "kill_rate": 0.5, + "asserts": 4, + "loc": 10, + "density": 0.4, + "exact_compare": 1 + }, + { + "function_id": 22, + "test": "test_intel_runtime_status_normal_state_unchanged", + "kill_rate": 0.0, + "asserts": 3, + "loc": 27, + "density": 0.111, + "exact_compare": 2 + }, + { + "function_id": 23, + "test": "TestIndexGuard::test_empty_table", + "kill_rate": 0.0, + "asserts": 2, + "loc": 7, + "density": 0.286, + "exact_compare": 1 + }, + { + "function_id": 23, + "test": "TestIndexGuard::test_healthy_index", + "kill_rate": 0.25, + "asserts": 3, + "loc": 26, + "density": 0.115, + "exact_compare": 1 + }, + { + "function_id": 23, + "test": "TestIndexGuard::test_missing_table", + "kill_rate": 0.75, + "asserts": 2, + "loc": 7, + "density": 0.286, + "exact_compare": 0 + }, + { + "function_id": 24, + "test": "test_get_call_chain_both", + "kill_rate": 0.5, + "asserts": 2, + "loc": 9, + "density": 0.222, + "exact_compare": 0 + }, + { + "function_id": 24, + "test": "test_get_call_chain_down", + "kill_rate": 0.25, + "asserts": 3, + "loc": 8, + "density": 0.375, + "exact_compare": 0 + }, + { + "function_id": 24, + "test": "test_get_call_chain_total_connected", + "kill_rate": 0.0, + "asserts": 1, + "loc": 5, + "density": 0.2, + "exact_compare": 0 + }, + { + "function_id": 25, + "test": "test_cap_truncation_detected", + "kill_rate": 0.5, + "asserts": 3, + "loc": 8, + "density": 0.375, + "exact_compare": 2 + }, + { + "function_id": 25, + "test": "test_excludes_venv_and_git", + "kill_rate": 0.5, + "asserts": 4, + "loc": 14, + "density": 0.286, + "exact_compare": 1 + }, + { + "function_id": 25, + "test": "test_nested_git_repo_pruned", + "kill_rate": 0.5, + "asserts": 3, + "loc": 13, + "density": 0.231, + "exact_compare": 1 + } + ], + "decoy": { + "killed": 0, + "total": 55 + }, + "train_rho": { + "exact_compare": 0.127970609086532, + "asserts": 0.18516685991168072 + }, + "test_rho": { + "exact_compare": -0.008006104434466121, + "asserts": -0.04319073537157674 + }, + "confusion": { + "tp": 10, + "fp": 7, + "fn": 5, + "tn": 8 + } +} \ No newline at end of file diff --git a/experiments/bootstrap/e17_strength_scaled.py b/experiments/bootstrap/e17_strength_scaled.py new file mode 100644 index 00000000..07bc1c4e --- /dev/null +++ b/experiments/bootstrap/e17_strength_scaled.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +"""E17 verification-strength — scaled run with held-out validation. + +Selects its own panel (N functions with a specific test, min coverage <= 5), +generates M mutations per function, runs every graph-linked test (arm B) and +coverage-matched decoy test (arm D) under every mutant, and records which tests +fail. Then splits by FUNCTION (no leakage) and checks whether a cheap AST proxy +computed at index time predicts a test's mutation kill-rate on held-out data. + +No LLM. Files restored via `git checkout --` in a finally block. +""" +import ast +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") +ROOT = Path(__file__).resolve().parents[2] +BOOT = ROOT / "experiments/bootstrap" +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(BOOT)) + +import e17_pilot_experiment as ex # noqa: E402 +from e17_extract import read_function_code, read_test_code # noqa: E402 + +N_FUNCS = int(sys.argv[1]) if len(sys.argv) > 1 else 45 +MAX_MUT = int(sys.argv[2]) if len(sys.argv) > 2 else 6 +STRONG = 0.5 + +_OPS = {ast.Eq: ("==", "!="), ast.NotEq: ("!=", "=="), ast.Lt: ("<", ">"), + ast.Gt: (">", "<"), ast.LtE: ("<=", ">="), ast.GtE: (">=", "<=")} + + +def _sh(cmd, timeout=300): + return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + + +def graph_test_files(): + import sqlite3 + from src.core.artifact_paths import get_graph_db_path + conn = sqlite3.connect(str(get_graph_db_path(ROOT))) + rows = conn.execute("SELECT name, file_path FROM nodes WHERE label='Test'").fetchall() + conn.close() + out = {} + for name, fp in rows: + out.setdefault(name, fp) + return out + + +def node_id(rel_file, test_name): + cls, _, meth = test_name.partition("::") + return f"{rel_file}::{cls}::{meth}" if meth else f"{rel_file}::{test_name}" + + +def find_func(tree, qualname): + parts = qualname.split(".") + if len(parts) == 1: + for n in ast.walk(tree): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == parts[0]: + return n + return None + for n in ast.walk(tree): + if isinstance(n, ast.ClassDef) and n.name == parts[-2]: + for item in n.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) and item.name == parts[-1]: + return item + return None + + +def mutations(src, qualname, limit): + try: + tree = ast.parse(src) + except SyntaxError: + return [] + node = find_func(tree, qualname) + if node is None: + return [] + lines = src.split("\n") + out = [] + + def add(ln_no, col, end_col, text, desc): + nl = list(lines) + nl[ln_no - 1] = lines[ln_no - 1][:col] + text + lines[ln_no - 1][end_col:] + joined = "\n".join(nl) + try: + ast.parse(joined) + except SyntaxError: + return + out.append((joined, desc)) + + for sub in ast.walk(node): + if len(out) >= limit: + break + if isinstance(sub, ast.If) and sub.test.lineno == sub.test.end_lineno: + ln = lines[sub.test.lineno - 1] + seg = ln[sub.test.col_offset:sub.test.end_col_offset] + if seg.strip(): + add(sub.test.lineno, sub.test.col_offset, sub.test.end_col_offset, + f"not ({seg})", f"negate-if@L{sub.test.lineno}") + elif isinstance(sub, ast.Compare) and len(sub.ops) == 1: + left, comp = sub.left, sub.comparators[0] + if left.lineno == left.end_lineno == comp.lineno: + span = lines[left.lineno - 1][left.end_col_offset:comp.col_offset] + pair = _OPS.get(type(sub.ops[0])) + if pair and span.strip() == pair[0]: + add(left.lineno, left.end_col_offset, comp.col_offset, + f" {pair[1]} ", f"flip-cmp@{pair[0]}->{pair[1]}@L{left.lineno}") + elif isinstance(sub, ast.Constant) and sub.lineno == sub.end_lineno: + seg = lines[sub.lineno - 1][sub.col_offset:sub.end_col_offset] + if isinstance(sub.value, bool) and seg in ("True", "False"): + add(sub.lineno, sub.col_offset, sub.end_col_offset, + "False" if sub.value else "True", f"bool-flip@L{sub.lineno}") + elif isinstance(sub.value, int) and seg == str(sub.value): + add(sub.lineno, sub.col_offset, sub.end_col_offset, + str(sub.value + 1), f"int+1@{sub.value}->{sub.value + 1}@L{sub.lineno}") + return out[:limit] + + +def run_verbose(ids): + if not ids: + return {} + try: + p = _sh([sys.executable, "-m", "pytest", *ids, "-v", "--no-header", + "-p", "no:cacheprovider"], timeout=45) + except subprocess.TimeoutExpired: + return {nid: "TIMEOUT" for nid in ids} + return {m.group(1): m.group(2) + for m in re.finditer(r"(\S+::\S+)\s+(PASSED|FAILED|ERROR)", p.stdout)} + + +def features(body): + try: + tree = ast.parse(body) + except SyntaxError: + return {} + asserts = [n for n in ast.walk(tree) if isinstance(n, ast.Assert)] + loc = len([ln for ln in body.splitlines() if ln.strip()]) + exact = sum(1 for a in asserts if isinstance(a.test, ast.Compare) + and any(isinstance(o, (ast.Eq, ast.NotEq)) for o in a.test.ops)) + return {"asserts": len(asserts), "loc": loc, + "density": round(len(asserts) / loc, 3) if loc else 0.0, + "exact_compare": exact} + + +def main(): + conn = ex._connect() + cands = ex.collect_candidates(max_cov=5) + # Skip providers/mcp: their tests spawn/await servers and can hang under mutation. + cands = [c for c in cands + if "/providers/" not in c["file_path"] and "/mcp/" not in c["file_path"]] + panel = ex.select_panel(cands, N_FUNCS, per_file=3) + b_pairs = [ex.specific_tests(conn, f["name"], 3) for f in panel] + pool = ex.build_test_pool(conn) + d_sets = ex.matched_decoys(panel, b_pairs, pool) + conn.close() + print(f"panel: {len(panel)} functions") + + tfiles = graph_test_files() + kill = defaultdict(lambda: {"killed": 0, "total": 0}) + d_kill = d_total = 0 + profiled = 0 + + for fid, (f, b_pairs_i, d_tests) in enumerate(zip(panel, b_pairs, d_sets), 1): + rel = f["file_path"].replace("D:/Project/MSCodeBase/", "") + src_path = ROOT / rel + if not src_path.exists(): + continue + muts = mutations(src_path.read_text(encoding="utf-8"), f["name"], MAX_MUT) + b_tests = [t for t, _ in b_pairs_i if t in tfiles] + d_tests = [t for t in d_tests if t in tfiles] + if not muts or not b_tests: + continue + + def _rel(t): + return Path(tfiles[t]).resolve().relative_to(ROOT).as_posix() + + b_ids = [(t, node_id(_rel(t), t)) for t in b_tests] + d_ids = [(t, node_id(_rel(t), t)) for t in d_tests] + profiled += 1 + print(f"[{profiled}] {f['name']}: {len(muts)} muts, {len(b_tests)} B, {len(d_tests)} D", + flush=True) + try: + for idx, (new_src, _desc) in enumerate(muts): + src_path.write_text(new_src, encoding="utf-8") + st = run_verbose([nid for _, nid in b_ids]) + for name, nid in b_ids: + kill[(fid, name)]["total"] += 1 + if st.get(nid) in ("FAILED", "ERROR"): + kill[(fid, name)]["killed"] += 1 + if idx == 0: + dst = run_verbose([nid for _, nid in d_ids]) + for _name, nid in d_ids: + d_total += 1 + if dst.get(nid) in ("FAILED", "ERROR"): + d_kill += 1 + finally: + _sh(["git", "checkout", "--", rel]) + + import random + rows = [] + for (fid, name), v in kill.items(): + feats = features(read_test_code(name) or "") + rows.append({"function_id": fid, "test": name, + "kill_rate": round(v["killed"] / v["total"], 3) if v["total"] else 0.0, + **feats}) + + funcs = sorted({r["function_id"] for r in rows}) + random.seed(41) + random.shuffle(funcs) + train_f = set(funcs[: len(funcs) // 2]) + train = [r for r in rows if r["function_id"] in train_f] + test = [r for r in rows if r["function_id"] not in train_f] + + def rho(rs, feat): + from scipy import stats + xs = [r[feat] for r in rs if feat in r] + ys = [r["kill_rate"] for r in rs if feat in r] + if len(set(xs)) < 2: + return float("nan"), float("nan") + return stats.spearmanr(xs, ys) + + print("\n" + "=" * 72) + print(f"profiled functions: {profiled} | tests: {len(rows)} | decoys killed: {d_kill}/{d_total}") + for label, rs in (("train", train), ("test (held-out)", test)): + for feat in ("exact_compare", "asserts", "density", "loc"): + r, p = rho(rs, feat) + print(f" [{label:14}] spearman(kill_rate, {feat:13}) rho={r:+.2f} p={p:.3f}") + + tp = fp = fn = tn = 0 + for r in test: + pred = r.get("exact_compare", 0) >= 1 + act = r["kill_rate"] >= STRONG + tp += pred and act + fp += pred and not act + fn += (not pred) and act + tn += (not pred) and not act + prec = tp / (tp + fp) if tp + fp else float("nan") + rec = tp / (tp + fn) if tp + fn else float("nan") + print(f"\nheld-out rule (exact_compare>=1 -> strong, kill_rate>={STRONG}):") + print(f" TP={tp} FP={fp} FN={fn} TN={tn} precision={prec:.2f} recall={rec:.2f}") + + (BOOT / "e17_strength_scaled.json").write_text( + json.dumps({"tests": rows, "decoy": {"killed": d_kill, "total": d_total}, + "train_rho": {f: rho(train, f)[0] for f in ("exact_compare", "asserts")}, + "test_rho": {f: rho(test, f)[0] for f in ("exact_compare", "asserts")}, + "confusion": {"tp": tp, "fp": fp, "fn": fn, "tn": tn}}, + ensure_ascii=False, indent=2), encoding="utf-8") + print("=" * 72) + + +if __name__ == "__main__": + try: + main() + finally: + subprocess.run(["git", "checkout", "--", "src/"], cwd=str(ROOT)) diff --git a/experiments/bootstrap/e17_verify_data.py b/experiments/bootstrap/e17_verify_data.py new file mode 100644 index 00000000..a7e6a8ef --- /dev/null +++ b/experiments/bootstrap/e17_verify_data.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""E17 data verification: every panel function and every B/D test must resolve. + +Fails loudly if any function source or test body cannot be extracted, so the +pilot never silently falls back to "# FUNC NOT FOUND" placeholders. +""" +import sys +import json +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.stdout.reconfigure(encoding="utf-8") + +from e17_extract import read_function_code, read_test_code # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] + + +def main() -> int: + data = json.loads( + (ROOT / "experiments/bootstrap/e17_pilot_data.json").read_text(encoding="utf-8") + ) + funcs = data["functions"] + print(f"functions: {len(funcs)} | modes: {data['metadata']['modes']}") + + missing_code = [] + missing_tests = [] + bDup = 0 + for f in funcs: + code = read_function_code(f["file_path"], f["name"]) + if not code: + missing_code.append(f["name"]) + b = f["modes"]["B_runtime"]["tests"] + d = f["modes"]["D_shuffled"]["tests"] + if set(b) & set(d): + bDup += 1 + for t in b: + if read_test_code(t) is None: + missing_tests.append((f["name"], "B", t)) + for t in d: + if read_test_code(t) is None: + missing_tests.append((f["name"], "D", t)) + + print(f"missing function code: {len(missing_code)} {missing_code[:5]}") + print(f"missing test bodies: {len(missing_tests)} {missing_tests[:5]}") + print(f"B/D overlap functions: {bDup}") + + ok = not missing_code and not missing_tests and bDup == 0 + print("VERIFY DATA:", "PASSED" if ok else "FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/bootstrap/test_e17_extract.py b/experiments/bootstrap/test_e17_extract.py new file mode 100644 index 00000000..a8219170 --- /dev/null +++ b/experiments/bootstrap/test_e17_extract.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""Regression tests for E17 AST extraction. + +Guards the bug where qualified names ("Class.method", "Class::test") never +matched the naive ``f"def {name}"`` search, yielding "# FUNC NOT FOUND" / +"# Test not found" for every class method. +""" +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from e17_extract import read_function_code, read_test_code, read_tests # noqa: E402 + +LEXER = "D:/Project/MSCodeBase/src/core/search/cypher_lexer.py" +PATHS = "D:/Project/MSCodeBase/src/core/artifact_paths.py" + + +def test_method_extraction_returns_real_body(): + code = read_function_code(LEXER, "CypherLexer.tokenize") + assert code is not None + assert "def tokenize" in code + assert "FUNC NOT FOUND" not in code + + +def test_free_function_extraction(): + code = read_function_code(PATHS, "_ensure_data_root") + assert code is not None + assert "def _ensure_data_root" in code + + +def test_missing_function_returns_none(): + assert read_function_code(LEXER, "CypherLexer.does_not_exist") is None + + +def test_missing_file_returns_none(): + assert read_function_code("D:/Project/MSCodeBase/src/nope.py", "x") is None + + +def test_class_method_test_extraction(): + body = read_test_code("TestCypherLexer::test_tokenize_simple_match") + assert body is not None + assert "def test_tokenize_simple_match" in body + assert "Test not found" not in body + + +def test_plain_test_extraction(): + body = read_test_code("test_store_record_get_query") + assert body is not None + assert "def test_store_record_get_query" in body + + +def test_missing_test_returns_none(): + assert read_test_code("TestCypherLexer::nope_not_here") is None + + +def test_read_tests_skips_unresolved(): + bodies = read_tests(["TestCypherLexer::test_tokenize_simple_match", "no_such_test"]) + assert len(bodies) == 1 + + +def test_syntax_error_file_returns_none(tmp_path): + bad = tmp_path / "bad.py" + bad.write_text("def broken(:\n", encoding="utf-8") + assert read_function_code(str(bad), "broken") is None + + +def test_duplicate_method_names_resolve_by_class(tmp_path): + mod = tmp_path / "dup.py" + mod.write_text( + "class A:\n" + " def foo(self):\n" + " return 'AAA'\n\n" + "class B:\n" + " def foo(self):\n" + " return 'BBB'\n", + encoding="utf-8", + ) + code = read_function_code(str(mod), "B.foo") + assert code is not None + assert "'BBB'" in code + assert "'AAA'" not in code + + +def test_async_function_extraction(tmp_path): + mod = tmp_path / "a.py" + mod.write_text("async def go():\n return 1\n", encoding="utf-8") + code = read_function_code(str(mod), "go") + assert code is not None + assert "async def go" in code + + +def test_name_only_in_comment_is_not_matched(tmp_path): + mod = tmp_path / "c.py" + mod.write_text( + "# def ghost():\nSTRING = 'def ghost()'\n", + encoding="utf-8", + ) + assert read_function_code(str(mod), "ghost") is None + + +def test_nested_function_extraction(tmp_path): + mod = tmp_path / "n.py" + mod.write_text( + "def outer():\n" + " def inner():\n" + " return 'INNER'\n" + " return inner\n", + encoding="utf-8", + ) + code = read_function_code(str(mod), "inner") + assert code is not None + assert "'INNER'" in code + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/extension.toml b/extension.toml index 0d8070b6..b6d8fcf0 100644 --- a/extension.toml +++ b/extension.toml @@ -1,6 +1,6 @@ id = "mscodebase-intelligence" name = "MSCodeBase Intelligence" -version = "3.4.0" +version = "3.5.0" schema_version = 1 description = "Semantic codebase intelligence with Multi-Bucket RAG, contextual retrieval, and system profiles" repository = "https://github.com/ManSio/mscodebase-intelligence" diff --git a/src/__init__.py b/src/__init__.py index 048ee4c4..535c963e 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -3,5 +3,5 @@ Семантический поиск по кодовой базе для Zed IDE. """ -__version__ = "3.4.0" +__version__ = "3.5.0" __author__ = "MSCodebase Intelligence Team" diff --git a/src/cli.py b/src/cli.py index 3d2ac192..2017a32d 100644 --- a/src/cli.py +++ b/src/cli.py @@ -22,6 +22,7 @@ from typing import Type from src.core.di_container import create_service_collection +from src.core.redact import redacted_count def core_tool_allowlist() -> dict: @@ -87,12 +88,21 @@ def main(argv=None) -> int: result = instance.execute(**call_args) if asyncio.iscoroutine(result): result = asyncio.run(result) - print(json.dumps({"ok": True, "tool": args.tool, "result": result}, - default=str, ensure_ascii=False)) + payload, redacted = redacted_count( + json.dumps({"ok": True, "tool": args.tool, "result": result}, + default=str, ensure_ascii=False)) + if redacted: + # канал доставки чистит креды ПЕРЕД инъекцией в контекст агента (см. redact.py) + print(json.dumps({"redacted": redacted}), file=sys.stderr) + print(payload) return 0 except Exception as e: # noqa: BLE001 — CI-friendly: ошибка тула = exit 1 + json - print(json.dumps({"ok": False, "tool": args.tool, - "error": f"{type(e).__name__}: {e}"}), file=sys.stderr) + payload, redacted = redacted_count( + json.dumps({"ok": False, "tool": args.tool, + "error": f"{type(e).__name__}: {e}"})) + if redacted: + print(json.dumps({"redacted": redacted}), file=sys.stderr) + print(payload, file=sys.stderr) return 1 finally: _safe_shutdown(services) diff --git a/src/core/auto_doc_updater.py b/src/core/auto_doc_updater.py index 4e0f4c1b..bb53e9de 100644 --- a/src/core/auto_doc_updater.py +++ b/src/core/auto_doc_updater.py @@ -23,7 +23,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import List, Optional, Set, Tuple logger = logging.getLogger(__name__) @@ -65,51 +65,12 @@ def verify_references(self, project_root: str) -> str: if not root.exists(): return f"❌ Project root not found: {root}" - start = datetime.now() - broken = self._verify_doc_references(root) - elapsed = (datetime.now() - start).total_seconds() - - # Статистика - md_files = list(root.rglob("*.md")) - doc_files = [f for f in md_files if "generated" not in f.parts] - total_refs = self._count_references(doc_files) - - if not broken: - return ( - f"✅ Doc reference check — {datetime.now().strftime('%H:%M:%S')}\n" - f"📁 Checked {len(doc_files)} .md files, " - f"{total_refs} code references, 0 broken.\n" - f"⚡ {elapsed:.1f}s" - ) + # L1-чекер (детерминированный, near-zero FP): полный словарь из кода, + # scope = живые доки, исключения venv/архива, whitelist stdlib/внешних API. + # Старая реализация (только defs + rglob с venv) давала 1366 «битых», ~3% реальных. + from src.core.doc_reference_l1 import check, format_report - lines = [ - f"⚠️ Doc reference check — {datetime.now().strftime('%H:%M:%S')}", - f"📁 Checked {len(doc_files)} .md files, " - f"{total_refs} code references, {len(broken)} broken.", - "", - "📛 Broken references:", - ] - for b in broken[:20]: - lines.append(f" • `{b.reference}` in {b.file}:L{b.line}") - if b.suggestion: - lines.append(f" 💡 {b.suggestion}") - if len(broken) > 20: - lines.append(f" ... +{len(broken) - 20} more") - - lines.extend([ - "", - "📋 Fix suggestions:", - ]) - # Группируем по файлам для удобных исправлений - by_file: Dict[str, List[BrokenRef]] = {} - for b in broken: - by_file.setdefault(b.file, []).append(b) - for file_path, refs in sorted(by_file.items()): - refs_list = ", ".join(f"`{r.reference}`" for r in refs) - lines.append(f" • {file_path}: fix {refs_list}") - - lines.append(f"\n⚡ {elapsed:.1f}s") - return "\n".join(lines) + return format_report(check(root)) def update_all(self, project_root: str) -> str: """Запускает полное обновление документации. @@ -618,6 +579,10 @@ def _extract_doc_references(text: str) -> List[Tuple[str, int, str]]: # Убираем $ префикс для env vars: `$VAR_NAME` → `VAR_NAME` if content.startswith("$"): content = content[1:] + # Guard: одинокий `$` → пустой content → IndexError на content[0] + # ниже (инцидент 2026-09-23: verify_references падал на доке с `$`). + if not content: + continue # Пропускаем shell-style env vars: `$ZED_WORKTREE_ROOT` if content.isupper() and len(content) > 2: continue diff --git a/src/core/doc_reference_l1.py b/src/core/doc_reference_l1.py new file mode 100644 index 00000000..c6c01085 --- /dev/null +++ b/src/core/doc_reference_l1.py @@ -0,0 +1,206 @@ +"""doc_reference_l1.py — детерминированный L1-чекер ссылок в документации. + +Задача: отвечать «существует ли упомянутый в живых доках код-референс?» с +**минимумом ложных срабатываний** (иначе гейт «кричит волк» и его игнорируют — +evergreen/doc-sync). Без LLM, без сети, чистая stdlib. + +Почему не старая реализация (`auto_doc_updater._build_symbol_set`): она собирает +только ОПРЕДЕЛЕНИЯ (def/class/тулы/константы), поэтому параметры/поля/stdlib в +доках помечались «битыми» → 90%+ FP (замер 2026-09-23: 1366 «битых», ~3% реальных). + +L1 делает: +- **Словарь** — ВСЕ идентификаторы + строковые литералы из src/tests/scripts/tools + (существование, а не «является определением»). +- **Scope** — только ЖИВЫЕ доки: корневые *.md + docs/** кроме + archive/generated/research/blog/ISSUES. История не «фиксится». +- **Исключения** — venv/.git/site-packages/node_modules (не сканировать). +- **Whitelist** — Python stdlib/builtins, внешние API (Zed/Rust/JS), env-переменные, + имена моделей, декораторы (`@`), файловые ссылки (проверяются по диску). + +Замечание: это НЕ семантическая проверка прозы (для неё — опциональный L2/LLM). +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Dict, List, Tuple + +__all__ = ["check", "format_report", "LIVE_SKIP_PARTS"] + +# Живые доки: что НЕ считается «живым» (история/генерация/чужой периметр) +LIVE_SKIP_PARTS = {"archive", "generated", "research", "blog", "ISSUES", "investigations", ".git", "venv", ".venv"} +_SCAN_SKIP_PARTS = {"venv", ".venv", "site-packages", "node_modules", ".git", "__pycache__", "build", "dist"} +_SCAN_DIRS = ("src", "tests", "scripts", "tools") +_TEXT_SUFFIXES = {".py", ".ts", ".js", ".json", ".toml", ".yml", ".yaml"} +_ROOT_DOCS = ( + "README.md", "AGENTS.md", "WISDOM.md", "KNOWN_ISSUES.md", "CHANGELOG.md", + "CONTRIBUTING.md", "SECURITY.md", "CODE_OF_CONDUCT.md", "AI_INSTALLATION_PROMPT.md", +) +# Леджеры/история: имена в них датированы и не обязаны существовать в текущем коде +_LEDGER_DOCS = {"WISDOM.md", "KNOWN_ISSUES.md", "AGENT_DIARY.md", "EXPERIMENTS_LOG.md", "CHANGELOG.md", "ISSUE.md"} + +_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{1,}") +_STR_RE = re.compile(r"[\"']([A-Za-z_][A-Za-z0-9_\-]{2,})[\"']") +_BACKTICK_RE = re.compile(r"`([^`\n]+)`") +_SYMBOLIC = set("/\\:=#|<>[]()+→⇄*\"'{};,-%") +_KEYBOARD = {"Enter", "Escape", "Tab", "Shift", "Ctrl", "Alt", "Backspace", "Delete", "Esc"} + +# Python stdlib / builtins / популярные методы (существование не в нашем коде) +_STDLIB = set(""" +os sys json re subprocess asyncio logging pathlib typing dataclasses time datetime threading +collections itertools functools hashlib shutil tempfile socket math random textwrap copy pickle +contextlib abc enum uuid base64 struct ctypes platform traceback warnings gc inspect sqlite3 +unittest argparse csv io string glob fnmatch urllib http email smtplib ftplib +open print len range str int float bool list dict set tuple bytes bytearray type super object +isinstance getattr setattr hasattr enumerate zip sorted any all min max sum abs round map filter +RuntimeError ValueError TypeError KeyError IndexError OSError FileNotFoundError PermissionError +Exception StopIteration NotImplementedError AttributeError ImportError ModuleNotFoundError +Popen Thread Process Queue Event Lock RLock Table Table.search connect +Path Optional List Dict Set Tuple Any Callable Iterable Sequence Union Mapping +Enum StrEnum dataclass field property staticmethod classmethod datetime date timedelta +MagicMock Mock AsyncMock patch monkeypatch tmp_path capsys caplog pytest fixture +""".split()) + +# Внешние поверхности (Zed extension API / Rust / JS), которые законно упоминаются в доках +_EXTERNAL = set(""" +NonZeroU32 LanguageSettingsContent LspSettings BinarySettings ProjectSettingsContent Glob +zed_extension_api trusted_worktrees language_servers language_server_command +file_guard project_root max_retries module_name hierarchy_level is_public parent_id +gitMaxConcurrentClones gitMaxCodehostRequestsPerSecond StreamableHTTPServerTransport +agent.tool_permissions legend.tokenTypes buildType with_fallible_options with_failible_options +Adapter search indexer parser mcp core tests docs doc_sync feat fix refactor perf chore +expected_command graph_context_first +""".split()) + +_MODEL_RE = re.compile(r"^(?:bge|e5|gte|multilingual|ggml|Q[0-9]|qwen|llama|deepseek|nomic)", re.I) +_ENV_RE = re.compile(r"^[A-Z][A-Z0-9_]{2,}$") + + +def _build_vocabulary(root: Path) -> set: + vocab: set = set() + for d in _SCAN_DIRS: + base = root / d + if not base.exists(): + continue + for f in base.rglob("*"): + if f.suffix not in _TEXT_SUFFIXES: + continue + if set(f.parts) & _SCAN_SKIP_PARTS: + continue + try: + txt = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + vocab.update(_IDENT_RE.findall(txt)) + vocab.update(_STR_RE.findall(txt)) + # имена MCP-тулов из декораторов (на случай, если нет в строковых литералах) + for f in (root / "src").rglob("*.py"): + try: + t = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + vocab.update(re.findall(r'@\w+\.tool\("([a-z_][a-z0-9_]+)"\)', t)) + # корневые скрипты (install.py и т.п.) — тоже источник идентификаторов + for f in root.glob("*.py"): + try: + t = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + vocab.update(_IDENT_RE.findall(t)) + vocab.update(_STR_RE.findall(t)) + return vocab + + +def _live_docs(root: Path) -> List[Path]: + files: List[Path] = [] + for name in _ROOT_DOCS: + p = root / name + if p.exists() and name not in _LEDGER_DOCS: + files.append(p) + docs = root / "docs" + if docs.exists(): + for p in docs.rglob("*.md"): + if set(p.parts) & LIVE_SKIP_PARTS: + continue + files.append(p) + return files + + +def _known(part: str, vocab: set) -> bool: + return part in vocab or part in _STDLIB or part in _EXTERNAL + + +def _is_broken(tok: str, vocab: set, root: Path) -> bool: + if "." in tok: + parts = [p for p in tok.split(".") if p] + if parts and parts[0] in _STDLIB: + return False + return not all(_known(p, vocab) for p in parts) + return not _known(tok, vocab) + + +def _tokens(text: str): + for m in _BACKTICK_RE.finditer(text): + raw = m.group(1).strip() + if not raw or " " in raw: + continue + if raw.startswith("@"): + raw = raw[1:] + if not raw or raw.startswith("$"): + continue + if raw.startswith(("self.", "cls.")): + continue + if raw.startswith("__") and raw.endswith("__"): + continue + if len(raw) < 2 or raw in _KEYBOARD: + continue + if raw.isdigit() or re.match(r"^\d", raw): + continue + if raw.endswith((".py", ".md", ".json", ".toml", ".exe", ".yml", ".yaml", ".txt", ".rs", ".ts")): + continue + if _MODEL_RE.match(raw) and (("-" in raw) or ("." in raw)): + continue + if _ENV_RE.match(raw): + continue + if any(c in raw for c in _SYMBOLIC): + continue + # только кодо-подобное: snake_case / CamelCase / CONST / tool-префикс + if not ("_" in raw or re.search(r"[A-Z]", raw)): + continue + yield raw + + +def check(project_root: str | Path) -> Dict: + root = Path(project_root).resolve() + vocab = _build_vocabulary(root) + broken: List[Tuple[str, int, str]] = [] + for doc in _live_docs(root): + try: + text = doc.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + rel = str(doc.relative_to(root)) + for line_no, line in enumerate(text.splitlines(), 1): + for tok in _tokens(line): + if _is_broken(tok, vocab, root): + broken.append((rel, line_no, tok)) + return { + "status": "ok", + "vocab_size": len(vocab), + "docs_checked": len(_live_docs(root)), + "broken_count": len(broken), + "broken": broken, + } + + +def format_report(result: Dict) -> str: + lines = [ + "📄 L1 doc-reference check", + f"📁 Live docs: {result['docs_checked']} | vocabulary: {result['vocab_size']}", + f"📛 Broken references: {result['broken_count']}", + ] + for f, ln, ref in result["broken"][:60]: + lines.append(f" • `{ref}` in {f}:L{ln}") + if result["broken_count"] > 60: + lines.append(f" ... +{result['broken_count'] - 60} more") + return "\n".join(lines) diff --git a/src/core/quiet_break_gate.py b/src/core/quiet_break_gate.py new file mode 100644 index 00000000..70a222f8 --- /dev/null +++ b/src/core/quiet_break_gate.py @@ -0,0 +1,322 @@ +"""Quiet-break gate — детерминированный детектор «тихого слома» графа. + +На коммите (через `graph_query(action="isolation")` + opencode-хук) отвечает на вопрос: +вносит ли ПРАВКА узел графа в изоляцию (осиротение), о котором молчит обычный ревью? + +Два ДЕЛЬТА-сигнала (абсолютная изоляция не годится: 1535/4081 функций репо имеют +0 входящих CALLS — декораторы, entry points, callbacks; это шум, не сигнал): + +A. removed_last_caller — символ, ВСЁ ЕЩЁ определённый в дереве, теряет последних + вызывающих: правка удаляет вызовы `name(`, а среди оставшихся вызывающих (по + persisted-графу) ни один больше не зовёт `name` в рабочем дереве. +B. new_orphan — НОВАЯ функция/метод (из diff), которую никто не вызывает в изменённых + файлах и которая не является entry point / декоратором-регистрантом. + +Всё ЧТЕНИЕ (git diff + persisted PropertyGraph), индекс НЕ мутируется. При отсутствии +git/графа/изменений — status "unavailable"/"empty", findings пусты, гейт НЕ срабатывает +(fail-open, причина логируется): инфраструктурный сбой не должен блокировать коммит. + +Границы (см. Red Team в .agent_task_state): rename-guard (def удалён в том же diff), +whitelist (main/run/handle/test_*/step_*/setup/teardown/dunder), decorated-skip, +comment-строки игнорируются, same-name — консервативно. +""" +from __future__ import annotations + +import logging +import re +import subprocess +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from src.core.graph import EdgeType, NodeLabel, PropertyGraph + +logger = logging.getLogger("mscodebase_server.quiet_break") + +__all__ = ["run_quiet_break_gate", "parse_diff", "is_whitelisted"] + +# entry points и регистранты — не считаются «сиротами» (зовутся фреймворком/декоратором) +_ENTRY_POINT_NAMES = { + "main", "run", "start", "handle", "entry", "bootstrap", "init_app", + "setup", "teardown", "setup_module", "teardown_module", "setup_method", + "teardown_method", "setup_class", "teardown_class", +} +_WHITELIST_PREFIXES = ("test_", "step_") + +_PY_KEYWORDS = { + "if", "elif", "else", "for", "while", "return", "yield", "with", "as", + "in", "is", "not", "and", "or", "lambda", "def", "class", "import", + "from", "raise", "assert", "del", "global", "nonlocal", "await", "async", + "match", "case", "print", "super", "type", "range", "len", "str", "int", + "float", "list", "dict", "set", "tuple", "bool", "open", "getattr", + "setattr", "isinstance", "enumerate", "zip", "sorted", "any", "all", +} + +_DEF_RE = re.compile(r"^\+\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(") +_REMOVED_DEF_RE = re.compile(r"^-\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(") +_CALL_RE = re.compile(r"\b([A-Za-z_]\w*)\s*\(") + +_GRAPH_LABELS = (NodeLabel.FUNCTION, NodeLabel.METHOD) + + +def is_whitelisted(name: str, decorated: bool = False) -> bool: + """True — имя является entry point / тест-хелпером / регистрантом (не «сирота»).""" + if decorated: + return True + if name in _ENTRY_POINT_NAMES: + return True + if any(name.startswith(p) for p in _WHITELIST_PREFIXES): + return True + return name.startswith("__") and name.endswith("__") + + +def parse_diff(diff_text: str) -> Dict[str, Any]: + """Разбирает unified-diff на дельта-сигналы. + + Returns dict: + removed_calls: {name: [file, ...]} + removed_defs: {name, ...} (def-строки, УДАЛЁННЫЕ этим diff — guard A) + new_defs: [{name, file, decorated}, ...] + changed_files: [rel-posix, ...] + """ + removed_calls: Dict[str, List[str]] = {} + removed_defs: Set[str] = set() + new_defs: List[Dict[str, Any]] = [] + changed_files: List[str] = [] + seen_files: Set[str] = set() + + cur_file = "" + prev_added = "" + + for raw in diff_text.splitlines(): + if raw.startswith("+++ "): + target = raw[4:].strip() + if target.startswith("b/"): + target = target[2:] + if target and target != "/dev/null": + cur_file = target + if cur_file not in seen_files: + seen_files.add(cur_file) + changed_files.append(cur_file) + prev_added = "" + continue + if raw.startswith("--- ") or raw.startswith("diff --git") or raw.startswith("@@"): + continue + + if raw.startswith("+"): + stripped = raw[1:].strip() + m = _DEF_RE.match(raw) + if m: + new_defs.append({ + "name": m.group(1), + "file": cur_file, + "decorated": prev_added.startswith("@"), + }) + prev_added = stripped + elif raw.startswith("-"): + if raw.startswith("---"): + continue + stripped = raw[1:].strip() + if stripped.startswith("#"): + prev_added = "" + continue + md = _REMOVED_DEF_RE.match(raw) + if md: + removed_defs.add(md.group(1)) + for name in _CALL_RE.findall(stripped): + if name in _PY_KEYWORDS: + continue + removed_calls.setdefault(name, []).append(cur_file) + prev_added = "" + else: + # контекстная строка + if raw.startswith(" "): + prev_added = raw[1:].strip() + + return { + "removed_calls": removed_calls, + "removed_defs": removed_defs, + "new_defs": new_defs, + "changed_files": changed_files, + } + + +def _rel_posix(path: str, root: Path) -> str: + if not path: + return "" + try: + return Path(path).resolve().relative_to(root).as_posix() + except (ValueError, OSError): + pass + p = Path(path) + for parent in p.parents: + try: + return p.relative_to(parent).as_posix() + except ValueError: + continue + return p.as_posix() + + +def _incoming_callers(pg: PropertyGraph, qname: str) -> List[Any]: + callers: List[Any] = [] + for et in (EdgeType.CALLS, EdgeType.ASYNC_CALLS): + try: + for node, _edge, _depth in pg.get_neighbors( + qname, edge_type=et, direction="incoming", max_depth=1 + ): + callers.append(node) + except Exception: # noqa: BLE001 — узел без рёбер/битый граф: не повод падать + continue + return callers + + +def _called_in_file(name: str, content: str) -> bool: + call_re = re.compile(r"\b" + re.escape(name) + r"\s*\(") + def_re = re.compile(r"\bdef\s+" + re.escape(name) + r"\s*\(") + for m in call_re.finditer(content): + line_start = content.rfind("\n", 0, m.start()) + 1 + if def_re.search(content[line_start:m.end()]): + continue # это сама def-строка, не вызов + return True + return False + + +def _git_diff(root: Path) -> Tuple[Optional[str], str]: + """git diff: staged если непусто, иначе HEAD. + + Returns (diff|None, base): None — не git/ошибка (fail-open); "" — git есть, + но изменений нет (status empty). + """ + flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) + is_repo = False + for base_args, base_name in ((("--cached",), "cached"), (("HEAD",), "HEAD")): + try: + proc = subprocess.run( + ["git", "diff", "--no-color", "--unified=3", *base_args], + cwd=str(root), + capture_output=True, + timeout=20, + creationflags=flags, + ) + except Exception as exc: # noqa: BLE001 — git может отсутствовать/зависнуть + logger.warning("quiet_break: git diff failed (%s): %s", base_name, exc) + return None, "" + if proc.returncode == 0: + is_repo = True + out = proc.stdout.decode("utf-8", "replace") + if out.strip(): + return out, base_name + if is_repo: + return "", "none" + return None, "" + + +def _empty(status: str, message: str = "") -> Dict[str, Any]: + return { + "status": status, + "message": message, + "base": "", + "counts": {"removed_last_caller": 0, "new_orphan": 0}, + "findings": [], + "changed_files": [], + } + + +def run_quiet_break_gate( + project_root: Path, + pg: Optional[PropertyGraph], + diff_text: Optional[str] = None, +) -> Dict[str, Any]: + """Главная точка. Возвращает dict с findings; никогда не бросает наружу.""" + root = Path(project_root).resolve() + if pg is None: + return _empty("unavailable", "PropertyGraph not available") + db_path = getattr(pg, "_db_path", None) or getattr(pg, "path", None) + if not db_path or not Path(str(db_path)).exists(): + return _empty("unavailable", "graph db not found") + + if diff_text is None: + diff_text, base = _git_diff(root) + if diff_text is None: + return _empty("unavailable", "git not available") + else: + base = "provided" + if not diff_text.strip(): + return _empty("empty", "no changes") + + parsed = parse_diff(diff_text) + changed = set(parsed["changed_files"]) + removed_defs = parsed["removed_defs"] + findings: List[Dict[str, Any]] = [] + + # ── A: removed_last_caller ── + for name, locs in parsed["removed_calls"].items(): + if name in removed_defs or name in _PY_KEYWORDS: + continue + try: + nodes = [n for n in pg.find_nodes(name_pattern=name, limit=50) + if n.label in _GRAPH_LABELS] + except Exception as exc: # noqa: BLE001 — граф недоступен по узлу + logger.debug("quiet_break: find_nodes(%s) failed: %s", name, exc) + continue + for node in nodes: + callers = _incoming_callers(pg, node.qualified_name) + if not callers: + continue # уже был без вызывающих — не наш дельта-сигнал + alive = False + for c in callers: + cf = getattr(c, "file_path", "") or "" + if not cf: + alive = True + break + rel = _rel_posix(cf, root) + if rel not in changed: + alive = True + break + fp = root / rel + try: + content = fp.read_text(encoding="utf-8", errors="replace") + except OSError: + alive = True # не смогли прочитать — консервативно считаем живым + break + if _called_in_file(name, content): + alive = True + break + if not alive: + findings.append({ + "kind": "removed_last_caller", + "symbol": node.name, + "file": node.file_path, + "removed_call_lines": locs, + }) + + # ── B: new_orphan ── + contents: Dict[str, str] = {} + for rel in changed: + fp = root / rel + try: + contents[rel] = fp.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for d in parsed["new_defs"]: + if is_whitelisted(d["name"], d["decorated"]): + continue + in_changed = any(_called_in_file(d["name"], txt) for txt in contents.values()) + if not in_changed: + findings.append({ + "kind": "new_orphan", + "symbol": d["name"], + "file": d["file"], + "decorated": d["decorated"], + }) + + counts = { + "removed_last_caller": sum(1 for f in findings if f["kind"] == "removed_last_caller"), + "new_orphan": sum(1 for f in findings if f["kind"] == "new_orphan"), + } + return { + "status": "ok", + "base": base, + "counts": counts, + "findings": findings, + "changed_files": parsed["changed_files"], + } diff --git a/src/core/redact.py b/src/core/redact.py new file mode 100644 index 00000000..b1d6952d --- /dev/null +++ b/src/core/redact.py @@ -0,0 +1,95 @@ +"""redact.py — вырезает credential-подобные строки НА ДОСТАВКЕ (перед инъекцией в контекст агента). + +Зачем: наш «secret gate» — это commit-gate; канал доставки (хук → `python -m src.cli` → текст +заметки в контексте модели) через коммит НЕ проходит. Заметка про auth-сбой может нести ключ как +evidence — и он утечёт в контекст каждого агента, на котором заметка сработает. См. E8/Exp 20. + +Идея и принцип (prefix-anchored, не entropy) — по design из Tom Jones, crystal-memory, +`scripts/redact.py` (Apache-2.0). Реализация наша; код не вендорился. + +⛔ ЭТО НЕ ГРАНИЦА БЕЗОПАСНОСТИ. Ловит только ИЗВЕСТНЫЕ ФОРМЫ по структурному префиксу. Новый формат, +секрет разбитый по строкам или записанный словами — проходит. Снижает цену случайности, но не +делает канал безопасным для секретов. Якоря на префикс, а не на энтропию/слово «key», чтобы не +резать легитимные hex-дайджесты, хеши коммитов и пути. + +Чистая stdlib, без I/O. `redact(text)` не бросает на странном входе. +""" +from __future__ import annotations + +import re +from typing import Dict, Optional, Tuple + +__all__ = ["redact", "redacted_count", "PLACEHOLDER"] + +PLACEHOLDER = "[REDACTED:{kind}]" + +# (kind, compiled pattern). Префикс-анкоренные — см. docstring. +PATTERNS = ( + ("aws-key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")), + ("private-key", re.compile( + r"-----BEGIN[ A-Z]*PRIVATE KEY-----.*?-----END[ A-Z]*PRIVATE KEY-----", re.S)), + ("anthropic", re.compile(r"\bsk-ant-[A-Za-z0-9_\-]{20,}")), + ("openai", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_\-]{32,}")), + ("github-token", re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b")), + ("github-pat", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{50,}\b")), + ("slack", re.compile(r"\bxox[abprs]-[A-Za-z0-9\-]{10,}")), + ("google-key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")), + ("stripe", re.compile(r"\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b")), + ("jwt", re.compile( + r"\beyJ[A-Za-z0-9_\-]{10,}\.eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}")), + # URL со встроенными кредами. Намеренно НЕ матчит голый user@host. + ("url-cred", re.compile(r"\b(?P[a-z][a-z0-9+.\-]*)://[^\s/:@]+:[^\s/@]+@")), + # `Authorization: Bearer ` — только значение, и только если достаточно длинное. + ("bearer", re.compile(r"(?i)(?P\bbearer\s+)(?P[A-Za-z0-9_\-\.=]{24,})")), +) + +# ⛔ Присваивание ≠ секрет. `KEY=` чаще shell-пример/имя переменной, чем живой кред. +# Режем присваивание ТОЛЬКО если значение тоже похоже на кред: длинное и не плейсхолдер/путь/число. +_ASSIGN = re.compile( + r"(?i)(?P\b(?:api[_\-]?key|secret|token|passwd|password|access[_\-]?key)\b\s*[:=]\s*)" + r"(?P[\"']?)(?P[A-Za-z0-9_\-\.+/=]{20,})(?P=q)") +_PLACEHOLDERISH = re.compile( + r"(?i)^(?:x{3,}|\.{3,}|<.*>|\$\{?[a-z_]+\}?|your[_\-].*|example.*|changeme|redacted|" + r"[a-z_]*placeholder[a-z_]*|none|null|true|false)$") + + +def _looks_like_a_value(val: str) -> bool: + """Кред, а не путь/число/ссылка на env/плейсхолдер.""" + if _PLACEHOLDERISH.match(val): + return False + if val.startswith(("/", "./", "~", "$")) or ("/" in val and val.count("/") > 1): + return False + if re.fullmatch(r"[0-9.]+", val): + return False + return bool(re.search(r"[A-Za-z]", val) and re.search(r"[0-9_\-.+/=]", val)) + + +def redact(text: str, counter: Optional[Dict[str, int]] = None) -> str: + """Возвращает `text` с вырезанными credential-подобными подстроками. Не бросает.""" + if not text or not isinstance(text, str): + return text + out = text + for kind, pattern in PATTERNS: + def _sub(m, kind=kind): + if counter is not None: + counter[kind] = counter.get(kind, 0) + 1 + if kind == "bearer": + return m.group("lead") + PLACEHOLDER.format(kind=kind) + if kind == "url-cred": + return f"{m.group('scheme')}://" + PLACEHOLDER.format(kind=kind) + "@" + return PLACEHOLDER.format(kind=kind) + out = pattern.sub(_sub, out) + + def _assign(m): + if not _looks_like_a_value(m.group("val")): + return m.group(0) + if counter is not None: + counter["assignment"] = counter.get("assignment", 0) + 1 + return m.group("lead") + m.group("q") + PLACEHOLDER.format(kind="assignment") + m.group("q") + return _ASSIGN.sub(_assign, out) + + +def redacted_count(text: str) -> Tuple[str, Dict[str, int]]: + """(redacted_text, {kind: n}) — чтобы вызывающий мог СКАЗАТЬ, что отредактировал.""" + counter: Dict[str, int] = {} + return redact(text, counter), counter diff --git a/src/core/restraint.py b/src/core/restraint.py new file mode 100644 index 00000000..b20e01a0 --- /dev/null +++ b/src/core/restraint.py @@ -0,0 +1,90 @@ +"""restraint.py — сдержанность доставки (anti-numbing): не повторять одну и ту же заметку. + +Зачем: заметка, которая приходит снова и снова, притупляет читателя (Tom Jones, «the evening the +agent went numb to its own alerts»; наш E3/E4 — та же опасность). Ограничиваем ПОВТОР одной и той же +advisory-заметки: сигнатура findings + cooldown, с растущим backoff по «страйкам». + +Только для ADVISORY (speak). Блокирующие гейты сдержанность НЕ применяют: заблокированный коммит — +жёсткий стоп, а не нытьё (Red Team п.3). Вызов opt-in (`restraint: true`). + +Состояние — один маленький JSON на проект в data_root (перезаписывается, не растёт). +⛔ Не граница безопасности; best-effort, при сбое чтения/записи — доставляем (fail-open). +""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger("mscodebase_server.restraint") + +__all__ = ["signature", "should_deliver", "state_path"] + +DEFAULT_COOLDOWN_SEC = 600.0 +DEFAULT_MAX_STRIKES = 3 + + +def signature(findings: List[Dict[str, Any]]) -> str: + """Стабильная сигнатура набора findings: изменение kind/symbol/file → новая сигнатура.""" + keys = sorted( + f"{f.get('kind', '')}:{f.get('symbol', '')}:{f.get('file', '')}" for f in findings + ) + return hashlib.sha1("|".join(keys).encode("utf-8")).hexdigest()[:16] + + +def state_path(project_root: Path) -> Path: + from src.core.artifact_paths import get_graph_db_path + + return get_graph_db_path(Path(project_root).resolve()).parent / "delivery_state.json" + + +def _read_state(path: Path) -> Dict[str, Any]: + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + +def _write_state(path: Path, state: Dict[str, Any]) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + with open(tmp, "w", encoding="utf-8") as f: + json.dump(state, f) + os.replace(tmp, path) + except OSError as exc: + logger.warning("restraint: state write failed: %s", exc) + + +def should_deliver( + project_root: Path, + sig: str, + cooldown_sec: float = DEFAULT_COOLDOWN_SEC, + max_strikes: int = DEFAULT_MAX_STRIKES, + now: Optional[float] = None, +) -> Tuple[bool, str]: + """True — доставить (обновляет состояние); False — подавить повтор. + + Backoff растёт с числом повторных доставок той же сигнатуры: window = cooldown * 2^min(strikes, max). + """ + path = state_path(project_root) + now = time.time() if now is None else now + state = _read_state(path) + + same = state.get("sig") == sig + strikes = int(state.get("strikes", 0)) if same else 0 + last = float(state.get("last_ts", 0.0)) if same else 0.0 + window = cooldown_sec * (2 ** min(strikes, max_strikes)) + + if same and (now - last) < window: + left = int(window - (now - last)) + return False, f"cooldown: same findings, {left}s left (strikes={strikes})" + + _write_state(path, {"sig": sig, "last_ts": now, "strikes": strikes + 1 if same else 0}) + return True, "delivered" diff --git a/src/core/search/graph_adapter.py b/src/core/search/graph_adapter.py index 05ee3163..b7d6fdd6 100644 --- a/src/core/search/graph_adapter.py +++ b/src/core/search/graph_adapter.py @@ -554,7 +554,7 @@ def get_tests_for_symbol( from src.core.project_resolution import resolve_project_root project_root = resolve_project_root() normalized_path = (project_root / file_path).as_posix() - + candidates = self._graph.find_nodes( label=NodeLabel.FUNCTION, name_pattern=f"%{symbol}%", diff --git a/src/mcp/server_tools.py b/src/mcp/server_tools.py index 46a01b34..c891eff8 100644 --- a/src/mcp/server_tools.py +++ b/src/mcp/server_tools.py @@ -294,8 +294,8 @@ def register_all_tools(mcp, services): register_dev_tools(mcp) total_core = len(tool_classes) - total_intel = 14 - total_inline = 12 + total_intel = 16 + total_inline = 13 total_dev = 4 logger.info( f"✅ Все инструменты зарегистрированы " diff --git a/src/mcp/tools/graph_tools.py b/src/mcp/tools/graph_tools.py index 900d9d86..ce0f6ad1 100644 --- a/src/mcp/tools/graph_tools.py +++ b/src/mcp/tools/graph_tools.py @@ -139,6 +139,7 @@ class GraphQueryTool(MCPTool): - "flow" — трассировка переменной (data flow) - "drift" — Architecture Drift Detector - "verify" — Claim Verifier (проверка утверждений против кода) + - "isolation" — Quiet-break gate (дельта-изоляция узлов графа на коммите) """ def __init__(self, services: ServiceCollection): @@ -194,6 +195,8 @@ async def execute( return await self._execute_arch_drift(target, project_root=_project_root) elif action == "verify": return await self._execute_verify(target, kwargs, project_root=_project_root) + elif action == "isolation": + return await self._execute_isolation(kwargs, project_root=_project_root) else: # По умолчанию — GraphRAG (action="query") return await self._execute_query(query_type or "impact", target, kwargs, project_root=_project_root) @@ -539,6 +542,37 @@ async def _execute_flow( }, } + async def _execute_isolation( + self, kwargs: Optional[Dict[str, Any]] = None, project_root: str = "" + ) -> dict: + """Quiet-break gate: дельта-изоляция узлов графа (read-only, git diff + PropertyGraph). + + Два сигнала: removed_last_caller (правка удалила последних вызывающих символа, + который ещё определён в дереве) и new_orphan (новая функция без вызывающих). + Индекс НЕ мутируется. При отсутствии git/графа → status unavailable, без отказа. + """ + from src.core.quiet_break_gate import run_quiet_break_gate + + if project_root and project_root.strip(): + root = Path(project_root).resolve() + else: + root = Path(self.resolve_indexer().project_path).resolve() + try: + pg = self._resolve_pg(str(root)) + except Exception as exc: # noqa: BLE001 — граф недоступен: гейт fail-open + logger.warning("graph_query.isolation: PropertyGraph resolve failed: %s", exc) + pg = None + result = run_quiet_break_gate(root, pg) + # Сдержанность (anti-numbing) — только opt-in и только для advisory-доставки. + # Блокирующие гейты сюда не заходят (см. src/core/restraint.py). + if (kwargs or {}).get("restraint") and result.get("status") == "ok" and result.get("findings"): + from src.core.restraint import should_deliver, signature + + deliver, reason = should_deliver(root, signature(result["findings"])) + result["deliver"] = deliver + result["suppressed_reason"] = "" if deliver else reason + return result + async def _execute_arch_drift(self, file_path: str = "", project_root: str = "") -> dict: """Architecture Drift Detector: ищет структурные аномалии импортов. diff --git a/src/mcp/tools/search_tools.py b/src/mcp/tools/search_tools.py index a7355d37..6661ff94 100644 --- a/src/mcp/tools/search_tools.py +++ b/src/mcp/tools/search_tools.py @@ -195,6 +195,7 @@ class SearchCodeTool(MCPTool): def __init__(self, services: ServiceCollection): super().__init__(services, tool_name="search_code") + self._hot_reload_task: Optional[asyncio.Task] = None @error_boundary("search_code", timeout_ms=15000, max_retries=1) async def execute( @@ -458,13 +459,16 @@ def _safe_float(v, default=0.0): return result_str async def _maybe_hot_reload(self) -> None: - """KI-109: фоновая инкрементальная сверка актуальности индекса. + """KI-109: фоновая сверка актуальности индекса — ВНЕ критического пути. - Вызывается перед каждым search_code. Внутри FreshnessChecker — - debounce (FRESHNESS_INTERVAL_SEC) + threading.Lock + skip при полном - reindex, поэтому для большинства запросов это прозрачно (stat-first). - Ошибки не должны ломать поиск. + Раньше hot-reload ожидался синхронно ВНУТРИ search_code: при пачке + новых/изменённых файлов переиндексация (parse+embed) выходила за + 15s-бюджет → timeout (инцидент 2026-09-22: 22 файла → 14s, при живом + эмбеддере). Теперь реиндексация уходит в фон, поиск отвечает по + текущему индексу, а свежие файлы подхватятся следующим запросом. """ + if self._hot_reload_task is not None and not self._hot_reload_task.done(): + return try: from src.config.settings import get_config @@ -477,13 +481,22 @@ async def _maybe_hot_reload(self) -> None: project_path = getattr(indexer, "project_path", None) if project_path is None: return - # stat-first сверка + hot-reload изменённых/новых файлов в фоне. - # Внутри FreshnessChecker: debounce + Lock + skip при is_reindexing, - # поэтому блокировка поиска редкая и короткая. - await asyncio.to_thread(indexer.verify_index_freshness, project_path) + # Дедуп: пока предыдущая bg-сверка не завершилась — не плодим. + self._hot_reload_task = asyncio.create_task( + self._run_hot_reload_bg(indexer, project_path) + ) except Exception as _hr_err: logger.debug(f"Hot-reload check skipped: {_hr_err}") + async def _run_hot_reload_bg(self, indexer: Any, project_path: Any) -> None: + """Фоновый hot-reload (KI-109): не блокирует search_code.""" + try: + n = await asyncio.to_thread(indexer.verify_index_freshness, project_path) + if n: + logger.info(f"🧲 Hot-reload (bg): переиндексировано {n} файлов") + except Exception as _bg_err: + logger.debug(f"Hot-reload (bg) failed: {_bg_err}") + async def _agentic_search(self, query: str, project_root: str = "") -> str: """Agentic Code Search с декомпозицией и связями. diff --git a/src/providers/embedder/remote_embedder.py b/src/providers/embedder/remote_embedder.py index af46b29d..6118157b 100644 --- a/src/providers/embedder/remote_embedder.py +++ b/src/providers/embedder/remote_embedder.py @@ -717,6 +717,24 @@ def _check_llama_cpp(self) -> bool: except Exception: return False + def _revive_llama_cpp(self) -> bool: + """Поднимает llama.cpp embedder, если он выгружен / умер. + + Watchdog llama_runner выгружает embedder по EMBEDDER_IDLE_TIMEOUT + (free RAM), но клиент обязан поднять его обратно при следующем + использовании. True — сервер был не готов (повторить запрос). + """ + try: + from src.providers.reranker.llama_runner import get_global_runner + runner = get_global_runner() + if runner.is_alive() or runner.is_port_up(): + return False + logger.warning("🔧 llama.cpp embedder недоступен — перезапускаю (idle-unload recovery)") + return bool(runner.ensure_embedder_started()) + except Exception as e: + logger.warning(f"revive llama.cpp failed: {e}") + return False + # ── Shadow Canary: API-вызовы для нового провайдера (до смены mode) ── def _call_lm_studio_api(self, texts): @@ -966,6 +984,27 @@ def embed_batch( _retry_time.sleep(1) continue logger.warning(f"llama.cpp embed error: {_exc}") + # ─── Level 1.5: embedder мог быть выгружен по idle-timeout ─── + # llama_runner сам убивает llama-server через EMBEDDER_IDLE_TIMEOUT + # (free RAM), поднимать обратно обязан клиент. Иначе :8080 мёртв → + # WinError 10061 → поштучные ретраи → search_code таймаутит + # (инцидент 2026-09-22: max 27s, attempts 2/2). + if self._revive_llama_cpp(): + try: + r = self._http_client.post(self.llama_cpp_url, json={"input": texts}) + if r.status_code == 200: + data = r.json().get("data", []) + if data: + data = sorted(data, key=lambda x: x.get("index", 0)) + self._llama_last_used = time.time() + try: + from src.providers.reranker.llama_runner import get_global_runner + get_global_runner()._last_embedder_use = time.time() + except Exception: + pass + return [item["embedding"] for item in data] + except Exception as _revive_exc: + logger.warning(f"llama.cpp retry after revive failed: {_revive_exc}") # Level 2: single-item retry for each failed text logger.warning(f"Batch failed, retrying {len(texts)} items individually") results = [None] * len(texts) diff --git a/src/providers/reranker/llama_runner.py b/src/providers/reranker/llama_runner.py index bdb8afa1..1488521c 100644 --- a/src/providers/reranker/llama_runner.py +++ b/src/providers/reranker/llama_runner.py @@ -1455,7 +1455,35 @@ def is_alive(self) -> bool: return ret is None + def is_port_up(self) -> bool: + """Публичная проверка: отвечает ли embedder-порт (без запуска).""" + + try: + + return self._probe_port_sync(self._port) + + except Exception: + + return False + + def ensure_embedder_started(self, model_key: str = DEFAULT_EMBEDDING_MODEL) -> bool: + + """Синхронно гарантирует живой embedder на self._port. + + Watchdog llama_runner сам выгружает embedder по EMBEDDER_IDLE_TIMEOUT + (free RAM), но клиент обязан поднять его при следующем использовании. + Без этого :8080 мёртв -> WinError 10061 -> search_code таймаутит до + рестарта MCP (инцидент 2026-09-22). + """ + + if self.is_alive() or self._probe_port_sync(self._port): + + return True + + logger.warning("Embedder idle-unloaded/dead - restarting (idle-unload recovery)") + + return self._start_sync(model_key) async def health(self) -> dict: @@ -1497,11 +1525,12 @@ async def embed(self, texts: list[str]) -> Optional[list[list[float]]]: """Отправляет запрос на эмбеддинги.""" - if self._model_key != "bge-m3": + if not self.is_alive(): - # Автоматический рестарт с embedder + # Автоматический рестарт embedder (именно embedder-модель, + # не reranker): раньше здесь стартовала "bge-m3" — копипаст-баг. - if not await self.start("bge-m3"): + if not await self.start(DEFAULT_EMBEDDING_MODEL): return None diff --git a/tests/test_auto_doc_updater.py b/tests/test_auto_doc_updater.py index 38f04009..569bb574 100644 --- a/tests/test_auto_doc_updater.py +++ b/tests/test_auto_doc_updater.py @@ -81,6 +81,21 @@ def test_extract_doc_references_handles_empty_call_syntax(tmp_path): assert isinstance(broken, list) +def test_extract_doc_references_handles_lone_dollar(tmp_path): + """Регрессия: одинокий `` `$` `` в доке не роняет verify_references. + + Guard пустой строки стоял ДО снятия `$`, а одинокий `$` делал content пустым + → IndexError на content[0] (инцидент 2026-09-23).""" + root = _make_tree(tmp_path) + (tmp_path / "docs").mkdir(parents=True, exist_ok=True) + (tmp_path / "docs" / "y.md").write_text( + "Команды: `$` и `$ ` и нормальный `fastmcp`.\n", encoding="utf-8" + ) + updater = AutoDocUpdater() + broken = updater._verify_doc_references(root) # не должен бросить IndexError + assert isinstance(broken, list) + + def test_count_tools_counts_execute_script_when_enabled(tmp_path, monkeypatch): """ExecuteScriptTool учитывается только при MSCODEBASE_EXECUTE_SCRIPT_ENABLED=true.""" root = _make_tree(tmp_path) diff --git a/tests/test_doc_reference_l1.py b/tests/test_doc_reference_l1.py new file mode 100644 index 00000000..b6c9f56b --- /dev/null +++ b/tests/test_doc_reference_l1.py @@ -0,0 +1,50 @@ +"""Тесты L1 doc-reference чекера (детерминированный, near-zero FP).""" +from pathlib import Path + +from src.core.doc_reference_l1 import check + + +def _mk(root: Path): + (root / "src").mkdir(parents=True, exist_ok=True) + (root / "src" / "mod.py").write_text( + "def real_func():\n return 1\n\n\nCONST_X = 1\n", encoding="utf-8" + ) + + +def test_flags_real_drift_ignores_noise(tmp_path): + _mk(tmp_path) + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "d.md").write_text( + "Use `real_func` and `CONST_X`. `ghost_func` is gone. `list` is stdlib, " + "`some prose` has spaces, `__dunder__` ignored.\n", + encoding="utf-8", + ) + refs = {t for _f, _l, t in check(tmp_path)["broken"]} + assert refs == {"ghost_func"} + + +def test_excludes_archive_and_venv(tmp_path): + _mk(tmp_path) + (tmp_path / "README.md").write_text("`ghost_func`\n", encoding="utf-8") + (tmp_path / "docs" / "archive").mkdir(parents=True) + (tmp_path / "docs" / "archive" / "old.md").write_text("`ghost_func`\n", encoding="utf-8") + (tmp_path / "venv" / "site-packages").mkdir(parents=True) + (tmp_path / "venv" / "site-packages" / "p.md").write_text("`ghost_func`\n", encoding="utf-8") + res = check(tmp_path) + assert res["broken_count"] == 1 # только README (archive/venv исключены) + + +def test_ledger_docs_not_gated(tmp_path): + _mk(tmp_path) + (tmp_path / "KNOWN_ISSUES.md").write_text("`ghost_func`\n", encoding="utf-8") + assert check(tmp_path)["broken_count"] == 0 + + +def test_env_and_models_and_dotted_ok(tmp_path): + _mk(tmp_path) + (tmp_path / "README.md").write_text( + "`MSCODEBASE_DATA_DIR`, `bge-m3-Q4_K_M`, `Path.rglob`, `self.value`\n", + encoding="utf-8", + ) + res = check(tmp_path) + assert res["broken_count"] == 0, res["broken"] diff --git a/tests/test_embedder_idle_recovery.py b/tests/test_embedder_idle_recovery.py new file mode 100644 index 00000000..369d1e0f --- /dev/null +++ b/tests/test_embedder_idle_recovery.py @@ -0,0 +1,177 @@ +"""Регресс-тесты инцидента 2026-09-22: search_code timeout из-за выгруженного embedder. + +Цепочка отказа: +1. Watchdog llama_runner по EMBEDDER_IDLE_TIMEOUT=120s сам убивает llama-server + (`_process=None`) ради RAM. +2. Клиент (RemoteEmbedder.embed_batch) ретраил по логам 3 батча + поштучно, но + НИКОГДА не поднимал сервер обратно → `WinError 10061` → search_code >15s. +3. Второй дефект: hot-reload (`verify_index_freshness`) ждался синхронно внутри + search_code → при пачке новых файлов поиск выходил за бюджет. + +Покрытие: +A. LlamaRunner.ensure_embedder_started / is_port_up — контракт «поднять при use». +B. RemoteEmbedder._revive_llama_cpp — revive только когда сервер реально мёртв. +C. SearchCodeTool._maybe_hot_reload — фон, дедуп, не блокирует. +""" +import asyncio +from types import SimpleNamespace + +from src.mcp.tools.search_tools import SearchCodeTool +from src.providers.embedder.remote_embedder import RemoteEmbedder +from src.providers.reranker.llama_runner import ( + DEFAULT_EMBEDDING_MODEL, + LlamaRunner, +) + +# ── A. LlamaRunner.ensure_embedder_started / is_port_up ── + +def test_ensure_embedder_skips_start_when_alive(): + runner = LlamaRunner() + runner.is_alive = lambda: True + runner._probe_port_sync = lambda port: False + started = [] + runner._start_sync = lambda model_key=None: started.append(model_key) or True + + assert runner.ensure_embedder_started() is True + assert started == [], "живой сервер не должен перезапускаться" + + +def test_ensure_embedder_skips_start_when_port_up(): + runner = LlamaRunner() + runner.is_alive = lambda: False + runner._probe_port_sync = lambda port: True # чужой процесс держит порт + started = [] + runner._start_sync = lambda model_key=None: started.append(model_key) or True + + assert runner.ensure_embedder_started() is True + assert started == [], "порт отвечает — старт не нужен" + + +def test_ensure_embedder_starts_when_dead_and_port_down(): + runner = LlamaRunner() + runner.is_alive = lambda: False + runner._probe_port_sync = lambda port: False + started = [] + runner._start_sync = lambda model_key=None: started.append(model_key) or True + + assert runner.ensure_embedder_started() is True + assert len(started) == 1, "мёртвый сервер должен быть перезапущен" + assert started[0] == DEFAULT_EMBEDDING_MODEL + + +def test_is_port_up_true_and_false(): + runner = LlamaRunner() + runner._probe_port_sync = lambda port: True + assert runner.is_port_up() is True + runner._probe_port_sync = lambda port: (_ for _ in ()).throw(RuntimeError("boom")) + assert runner.is_port_up() is False + + +# ── B. RemoteEmbedder._revive_llama_cpp ── + +def _patch_runner(monkeypatch, *, alive, port_up, start_result=True): + import src.providers.reranker.llama_runner as lr + + calls = {"ensure": 0} + fake = SimpleNamespace( + is_alive=lambda: alive, + is_port_up=lambda: port_up, + ensure_embedder_started=lambda *a, **k: calls.__setitem__("ensure", calls["ensure"] + 1) or start_result, + ) + monkeypatch.setattr(lr, "get_global_runner", lambda: fake) + return calls + + +def test_revive_returns_false_when_alive(monkeypatch): + calls = _patch_runner(monkeypatch, alive=True, port_up=False) + emb = RemoteEmbedder.__new__(RemoteEmbedder) + assert emb._revive_llama_cpp() is False + assert calls["ensure"] == 0 + + +def test_revive_returns_false_when_port_up(monkeypatch): + calls = _patch_runner(monkeypatch, alive=False, port_up=True) + emb = RemoteEmbedder.__new__(RemoteEmbedder) + assert emb._revive_llama_cpp() is False + assert calls["ensure"] == 0 + + +def test_revive_restarts_when_dead(monkeypatch): + calls = _patch_runner(monkeypatch, alive=False, port_up=False) + emb = RemoteEmbedder.__new__(RemoteEmbedder) + assert emb._revive_llama_cpp() is True + assert calls["ensure"] == 1 + + +def test_revive_handles_exception(monkeypatch): + import src.providers.reranker.llama_runner as lr + + def _boom(): + raise RuntimeError("no runner") + + monkeypatch.setattr(lr, "get_global_runner", _boom) + emb = RemoteEmbedder.__new__(RemoteEmbedder) + assert emb._revive_llama_cpp() is False # не роняет эмбеддер + + +# ── C. SearchCodeTool._maybe_hot_reload ── + +def _make_search_tool(monkeypatch, verify_calls): + tool = SearchCodeTool.__new__(SearchCodeTool) + tool._hot_reload_task = None + fake_indexer = SimpleNamespace( + project_path="D:/proj", + verify_index_freshness=lambda path: verify_calls.append(path) or 3, + ) + tool.resolve_indexer = lambda explicit_project_root=None: fake_indexer + + import src.config.settings as settings_mod + monkeypatch.setattr( + settings_mod, + "get_config", + lambda: SimpleNamespace( + performance=SimpleNamespace(freshness_interval_sec=30) + ), + ) + return tool + + +def test_hot_reload_is_backgrounded_and_deduped(monkeypatch): + verify_calls = [] + tool = _make_search_tool(monkeypatch, verify_calls) + + async def scenario(): + await tool._maybe_hot_reload() + assert tool._hot_reload_task is not None, "сверка должна уйти в фон" + first = tool._hot_reload_task + assert not first.done(), "search_code не должен ждать hot-reload" + + # повторный вызов при незавершённой задаче — не плодит вторую + await tool._maybe_hot_reload() + assert tool._hot_reload_task is first + + await first + assert verify_calls == ["D:/proj"] + + asyncio.run(scenario()) + + +def test_hot_reload_disabled_when_interval_zero(monkeypatch): + verify_calls = [] + tool = _make_search_tool(monkeypatch, verify_calls) + + import src.config.settings as settings_mod + monkeypatch.setattr( + settings_mod, + "get_config", + lambda: SimpleNamespace( + performance=SimpleNamespace(freshness_interval_sec=0) + ), + ) + + async def scenario(): + await tool._maybe_hot_reload() + assert tool._hot_reload_task is None + assert verify_calls == [] + + asyncio.run(scenario()) diff --git a/tests/test_graph_stage_e4.py b/tests/test_graph_stage_e4.py index 9f88e030..da3b2cb2 100644 --- a/tests/test_graph_stage_e4.py +++ b/tests/test_graph_stage_e4.py @@ -118,31 +118,40 @@ def embed_batch_async(self, *a, **k): @pytest.fixture def searcher_with_tests(tmp_path): - """Граф: функция + два теста, покрывающих её (TESTS-рёбра).""" + """Граф: функция + два теста, покрывающих её (TESTS-рёбра). + + Пути — абсолютные и берутся из tmp_path: тест обязан работать и на POSIX, + и на Windows (раньше хардкод `D:/Project/...` на Linux не был absolute → + get_tests_for_symbol нормализовал его в /D:/... и не находил узел). + """ + base = tmp_path.as_posix() + fn_file = f"{base}/src/core/paths.py" + t_file = f"{base}/tests/test_paths.py" + fn_q = f"proj.{fn_file}.safe_mkdir" db = tmp_path / "graph.db" pg = PropertyGraph(str(db)) # Функция в src/ pg.add_node( - name="safe_mkdir", label="Function", qualified_name="D:.D:/Project/src/core/paths.py.safe_mkdir", - file_path="D:/Project/src/core/paths.py", + name="safe_mkdir", label="Function", qualified_name=fn_q, + file_path=fn_file, properties={"line": 42, "kind": "function"}, ) # Тесты, покрывающие функцию t1 = pg.add_node( name="test_safe_mkdir_creates", label="Test", - qualified_name="D:.D:/Project/tests/test_paths.py.test_safe_mkdir_creates", - file_path="D:/Project/tests/test_paths.py", + qualified_name=f"proj.{t_file}.test_safe_mkdir_creates", + file_path=t_file, properties={"line": 10}, ) t2 = pg.add_node( name="test_safe_mkdir_idempotent", label="Test", - qualified_name="D:.D:/Project/tests/test_paths.py.test_safe_mkdir_idempotent", - file_path="D:/Project/tests/test_paths.py", + qualified_name=f"proj.{t_file}.test_safe_mkdir_idempotent", + file_path=t_file, properties={"line": 20}, ) - pg.add_edge(source_qname=t1.qualified_name, target_qname="D:.D:/Project/src/core/paths.py.safe_mkdir", + pg.add_edge(source_qname=t1.qualified_name, target_qname=fn_q, type="TESTS", weight=1.0, properties={"trace": "dynamic_trace"}) - pg.add_edge(source_qname=t2.qualified_name, target_qname="D:.D:/Project/src/core/paths.py.safe_mkdir", + pg.add_edge(source_qname=t2.qualified_name, target_qname=fn_q, type="TESTS", weight=1.0, properties={"trace": "dynamic_trace"}) adapter = SymbolIndexAdapter(pg, mode=SymbolIndexAdapter.MODE_PURE) @@ -150,13 +159,13 @@ class _Idx: _symbol_index = adapter s = _make_searcher_with_flag(_Idx(), flag=True) - yield s, adapter, pg + yield s, adapter, pg, fn_file pg.close() def test_tests_signal_adds_covering_tests(searcher_with_tests): """К определению функции добавляются покрывающие тесты.""" - s, adapter, _ = searcher_with_tests + s, adapter, _, _ = searcher_with_tests results = s._graph_stage("safe_mkdir", limit=5) test_refs = [r for r in results if r["metadata"].get("tests_signal")] assert len(test_refs) >= 1, f"TESTS-сигнал не сработал: {results}" @@ -173,7 +182,7 @@ def test_tests_signal_adds_covering_tests(searcher_with_tests): def test_tests_signal_def_first_then_tests(searcher_with_tests): """Функция-определение стоит ДО тестов (тесты не вытесняют).""" - s, _, _ = searcher_with_tests + s, _, _, _ = searcher_with_tests results = s._graph_stage("safe_mkdir", limit=5) assert results[0]["metadata"]["symbol"] == "safe_mkdir" assert results[0]["metadata"]["is_definition"] is True @@ -185,7 +194,7 @@ def test_tests_signal_def_first_then_tests(searcher_with_tests): def test_tests_signal_flag_off_unchanged(searcher_with_tests): """Флаг off → тесты НЕ добавляются (поведение флага по умолчанию).""" - s, adapter, _ = searcher_with_tests + s, adapter, _, _ = searcher_with_tests s._tests_signal = False results = s._graph_stage("safe_mkdir", limit=5) assert all(not r["metadata"].get("tests_signal") for r in results) @@ -201,8 +210,8 @@ def test_tests_signal_no_method_fallback(): def test_get_tests_for_symbol_hit(searcher_with_tests): """Адаптер напрямую: возвращает покрывающие тесты.""" - s, adapter, _ = searcher_with_tests - refs = adapter.get_tests_for_symbol("safe_mkdir", "D:/Project/src/core/paths.py", limit=3) + s, adapter, _, fn_file = searcher_with_tests + refs = adapter.get_tests_for_symbol("safe_mkdir", fn_file, limit=3) names = {r.symbol for r in refs} assert "test_safe_mkdir_creates" in names assert "test_safe_mkdir_idempotent" in names @@ -213,5 +222,5 @@ def test_get_tests_for_symbol_hit(searcher_with_tests): def test_get_tests_for_symbol_miss(searcher_with_tests): """Неизвестный символ → [].""" - s, adapter, _ = searcher_with_tests - assert adapter.get_tests_for_symbol("nonexistent_xyz", "D:/Project/src/core/paths.py") == [] + s, adapter, _, fn_file = searcher_with_tests + assert adapter.get_tests_for_symbol("nonexistent_xyz", fn_file) == [] diff --git a/tests/test_quiet_break_gate.py b/tests/test_quiet_break_gate.py new file mode 100644 index 00000000..f7179b60 --- /dev/null +++ b/tests/test_quiet_break_gate.py @@ -0,0 +1,192 @@ +"""Тесты quiet-break gate (дельта-изоляция узлов графа на коммите). + +Покрытие: parse_diff (removed calls/defs, new defs, decorated, комментарии), +is_whitelisted, run_quiet_break_gate (A removed_last_caller + rename-guard, +B new_orphan + called/decorated, fail-open unavailable/empty). +""" +from pathlib import Path + +from src.core.graph import EdgeType, NodeLabel, PropertyGraph +from src.core.quiet_break_gate import is_whitelisted, parse_diff, run_quiet_break_gate + + +def _mk_graph(tmp_path: Path) -> PropertyGraph: + return PropertyGraph(tmp_path / "graph.db") + + +def _add_func(pg: PropertyGraph, name: str, rel: str, root: Path) -> None: + pg.add_node( + name=name, + label=NodeLabel.FUNCTION, + qualified_name=f"proj.{rel}.{name}", + file_path=str(root / rel), + ) + + +# ── parse_diff ── + +def test_parse_diff_removed_calls_and_new_defs(): + diff = ( + "diff --git a/pkg/caller.py b/pkg/caller.py\n" + "--- a/pkg/caller.py\n" + "+++ b/pkg/caller.py\n" + "@@ -1,3 +1,3 @@\n" + "- target()\n" + "+def helper():\n" + "+ pass\n" + ) + d = parse_diff(diff) + assert d["changed_files"] == ["pkg/caller.py"] + assert d["removed_calls"] == {"target": ["pkg/caller.py"]} + assert d["removed_defs"] == set() + assert [x["name"] for x in d["new_defs"]] == ["helper"] + assert d["new_defs"][0]["decorated"] is False + + +def test_parse_diff_rename_guard_and_decorator_and_comment(): + diff = ( + "diff --git a/x.py b/x.py\n" + "--- a/x.py\n" + "+++ b/x.py\n" + "@@ -1,4 +1,4 @@\n" + "-def old_name():\n" + "- # note: commented(target)\n" + "+@register\n" + "+def new_name():\n" + "+ pass\n" + ) + d = parse_diff(diff) + assert "old_name" in d["removed_defs"] + assert "target" not in d["removed_calls"] # строки-комментарии игнорируются + nd = {x["name"]: x for x in d["new_defs"]} + assert nd["new_name"]["decorated"] is True + + +# ── is_whitelisted ── + +def test_whitelist(): + assert is_whitelisted("main") + assert is_whitelisted("test_something") + assert is_whitelisted("step_lm") + assert is_whitelisted("__init__") + assert is_whitelisted("anything", decorated=True) + assert not is_whitelisted("helper") + assert not is_whitelisted("process_order") + + +# ── run_quiet_break_gate ── + +def test_removed_last_caller_fires(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "caller.py").write_text("def caller():\n pass\n", encoding="utf-8") + pg = _mk_graph(root) + _add_func(pg, "caller", "pkg/caller.py", root) + _add_func(pg, "target", "pkg/target.py", root) + pg.add_edge("proj.pkg/caller.py.caller", "proj.pkg/target.py.target", EdgeType.CALLS) + + diff = ( + "--- a/pkg/caller.py\n+++ b/pkg/caller.py\n@@ -1,2 +1,2 @@\n" + " def caller():\n- target()\n+ pass\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=diff) + assert res["status"] == "ok" + assert res["counts"]["removed_last_caller"] == 1 + assert res["findings"][0]["symbol"] == "target" + + +def test_removed_last_caller_not_fired_when_caller_unchanged(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "other.py").write_text("other.py placeholder", encoding="utf-8") + pg = _mk_graph(root) + _add_func(pg, "target", "pkg/target.py", root) + _add_func(pg, "other", "pkg/other.py", root) + pg.add_edge("proj.pkg/other.py.other", "proj.pkg/target.py.target", EdgeType.CALLS) + + # изменён ДРУГОЙ файл, вызывающий в other.py не тронут + diff = ( + "--- a/pkg/some.py\n+++ b/pkg/some.py\n@@ -1,1 +1,1 @@\n-old()\n+new()\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=diff) + assert res["counts"]["removed_last_caller"] == 0 + + +def test_removed_last_caller_rename_guard(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "caller.py").write_text("def caller():\n pass\n", encoding="utf-8") + pg = _mk_graph(root) + _add_func(pg, "caller", "pkg/caller.py", root) + _add_func(pg, "target", "pkg/target.py", root) + pg.add_edge("proj.pkg/caller.py.caller", "proj.pkg/target.py.target", EdgeType.CALLS) + + diff = ( + "--- a/pkg/target.py\n+++ b/pkg/target.py\n@@ -1,2 +1,2 @@\n" + "-def target():\n" + "+def target_renamed():\n" + "- target()\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=diff) + assert res["counts"]["removed_last_caller"] == 0 + + +def test_new_orphan_fires(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "mod.py").write_text( + "def helper():\n pass\n", encoding="utf-8" + ) + pg = _mk_graph(root) + _add_func(pg, "use", "pkg/mod.py", root) + + orphan_diff = ( + "--- a/pkg/mod.py\n+++ b/pkg/mod.py\n@@ -1,1 +1,3 @@\n" + "+def helper():\n+ pass\n+\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=orphan_diff) + assert res["counts"]["new_orphan"] == 1 + + +def test_new_orphan_called_not_fired(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "mod.py").write_text( + "def helper():\n pass\n\ndef use():\n helper()\n", encoding="utf-8" + ) + pg = _mk_graph(root) + _add_func(pg, "use", "pkg/mod.py", root) + + called_diff = ( + "--- a/pkg/mod.py\n+++ b/pkg/mod.py\n@@ -1,3 +1,3 @@\n" + "+def helper():\n+ pass\n+\n" + " def use():\n helper()\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=called_diff) + assert res["counts"]["new_orphan"] == 0 + + +def test_new_orphan_decorated_skipped(tmp_path): + root = tmp_path + (root / "pkg").mkdir() + (root / "pkg" / "mod.py").write_text("x = 1\n", encoding="utf-8") + pg = _mk_graph(root) + diff = ( + "--- a/pkg/mod.py\n+++ b/pkg/mod.py\n@@ -1,1 +1,3 @@\n" + "+@register\n+def handler():\n+ pass\n" + ) + res = run_quiet_break_gate(root, pg, diff_text=diff) + assert res["counts"]["new_orphan"] == 0 + + +def test_unavailable_and_empty(tmp_path): + root = tmp_path + res = run_quiet_break_gate(root, None, diff_text="") + assert res["status"] == "unavailable" + assert res["findings"] == [] + + pg = _mk_graph(root) + _add_func(pg, "seed", "pkg/seed.py", root) + res2 = run_quiet_break_gate(root, pg, diff_text="") + assert res2["status"] == "empty" + assert res2["findings"] == [] diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 00000000..44eaaba7 --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,108 @@ +"""Тесты redact — вырезание credential-подобных строк на доставке. + +Покрытие: каждая семья ключей вырезается; легитимные артефакты (хеш коммита, дайджест, +путь, версия, слово «key» в прозе) НЕ трогаются; assignment-эвристика; JSON остаётся валидным. +""" +import json + +from src.core.redact import redact, redacted_count + +# ── вырезание по семьям ── + +def test_each_family_redacted(): + cases = { + "aws-key": "AKIAIOSFODNN7EXAMPLE", + "anthropic": "sk-ant-api03-abcdefghijklmnopqrstuvwxyz", + "openai": "sk-proj-" + "a" * 40, + "github-token": "ghp_" + "A" * 40, + "github-pat": "github_pat_" + "A" * 55, + "slack": "xoxb-1234567890-abcdefghij", + "google-key": "AIza" + "A" * 35, + "stripe": "sk_live_" + "a" * 24, + "jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5NXg", + "url-cred": "postgres://user:sup3rsecret@db.internal:5432/app", + "bearer": "Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345", + } + for kind, sample in cases.items(): + out = redact(sample) + assert sample not in out, f"{kind} not redacted" + assert "[REDACTED:" in out, f"{kind} placeholder missing" + + +def test_private_key_block_redacted(): + pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEow...\n-----END RSA PRIVATE KEY-----" + out = redact(pem) + assert "MIIEow" not in out + assert "[REDACTED:private-key]" in out + + +# ── сохранение легитимного ── + +def test_legit_artifacts_survive(): + note = ( + "docs/readme.md:3 vs pyproject.toml:3\n" + "commit 292d7945a1b2c3d4e5f60718293a4b5c6d7e8f90\n" + "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08\n" + "version 1.9.0 -> 2.0.0\n" + "the key thing is the graph node is isolated\n" + "path C:\\Users\\misha\\AppData\\Local\\Temp\\opencode\\e8\n" + ) + out = redact(note) + assert out == note + + +def test_assignment_heuristics(): + assert "abcdef1234567890abcd" not in redact('api_key = "abcdef1234567890abcd"') + assert redact('api_key = "${MY_ENV_VAR}"') == 'api_key = "${MY_ENV_VAR}"' + assert redact("password = /usr/local/secret/path") == "password = /usr/local/secret/path" + assert redact("token = 1234567890") == "token = 1234567890" + assert redact("token = your_token_here") == "token = your_token_here" + + +def test_counter_reports_kinds(): + out, counts = redacted_count("AKIAIOSFODNN7EXAMPLE and ghp_" + "A" * 40) + assert counts.get("aws-key") == 1 + assert counts.get("github-token") == 1 + assert "AKIA" not in out + + +# ── безопасность интеграции ── + +def test_redacted_json_stays_valid(): + payload = json.dumps({"ok": True, "result": "token sk-ant-api03-" + "b" * 30}) + red, counts = redacted_count(payload) + assert counts + parsed = json.loads(red) # не должно упасть + assert parsed["ok"] is True + assert "sk-ant" not in red + + +def test_non_string_passthrough(): + assert redact(None) is None + assert redact(123) == 123 + assert redact("") == "" + + +def test_cli_redacts_tool_output(monkeypatch, capsys): + import src.cli as cli_mod + + secret = "sk-ant-api03-" + "c" * 30 + + class _FakeTool: + def __init__(self, services): + self.services = services + + def execute(self, **kwargs): + return f"note carrying a key {secret} and path src/core/redact.py" + + monkeypatch.setattr(cli_mod, "core_tool_allowlist", lambda: {"fake": _FakeTool}) + monkeypatch.setattr(cli_mod, "create_service_collection", lambda root: None) + + rc = cli_mod.main(["fake", "{}"]) + captured = capsys.readouterr() + assert rc == 0 + assert secret not in captured.out + assert "[REDACTED:anthropic]" in captured.out + assert "src/core/redact.py" in captured.out # путь сохранён + assert "redacted" in captured.err + diff --git a/tests/test_restraint.py b/tests/test_restraint.py new file mode 100644 index 00000000..0b0ff5bc --- /dev/null +++ b/tests/test_restraint.py @@ -0,0 +1,40 @@ +"""Тесты restraint — сдержанность advisory-доставки (anti-numbing). + +Покрытие: сигнатура чувствительна к kind/symbol/file; первая доставка; подавление немедленного +повтора; доставка после cooldown; сброс на новой сигнатуре; рост backoff по страйкам. +""" +from src.core.restraint import should_deliver, signature + + +def test_signature_changes_with_content(): + a = [{"kind": "new_orphan", "symbol": "x", "file": "a.py"}] + b = [{"kind": "new_orphan", "symbol": "y", "file": "a.py"}] + assert signature(a) == signature(a) + assert signature(a) != signature(b) + + +def test_first_delivers_then_suppresses_then_cooldown(tmp_path): + sig = "sigA" + ok, _ = should_deliver(tmp_path, sig, cooldown_sec=100, now=1000) + assert ok is True + + ok, reason = should_deliver(tmp_path, sig, cooldown_sec=100, now=1050) + assert ok is False + assert "cooldown" in reason + + ok, _ = should_deliver(tmp_path, sig, cooldown_sec=100, now=1101) + assert ok is True + + +def test_new_signature_resets(tmp_path): + should_deliver(tmp_path, "sigA", cooldown_sec=100, now=1000) + ok, _ = should_deliver(tmp_path, "sigB", cooldown_sec=100, now=1010) + assert ok is True # другая сигнатура — доставляем + + +def test_backoff_grows(tmp_path): + sig = "sigA" + should_deliver(tmp_path, sig, cooldown_sec=100, now=1000) # deliver, strikes 0 + should_deliver(tmp_path, sig, cooldown_sec=100, now=1101) # deliver, strikes 1 -> window 200 + ok, _ = should_deliver(tmp_path, sig, cooldown_sec=100, now=1250) # elapsed 149 < 200 + assert ok is False