From 214eb8dd6655846ebd273b1b31d28cc9dfb7d95e Mon Sep 17 00:00:00 2001 From: Air000000 <1113476369@qq.com> Date: Sat, 17 Oct 2026 23:27:37 +0800 Subject: [PATCH 1/2] eval: freeze bounded six-case version audit paired check --- scripts/probe_version_audit.py | 156 ++++++++++++++++++++++++++++++ tests/test_probe_version_audit.py | 147 ++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 scripts/probe_version_audit.py create mode 100644 tests/test_probe_version_audit.py diff --git a/scripts/probe_version_audit.py b/scripts/probe_version_audit.py new file mode 100644 index 0000000..3dd7e74 --- /dev/null +++ b/scripts/probe_version_audit.py @@ -0,0 +1,156 @@ +"""Fixed fictional paired classification check; default is zero-call planning.""" + +import argparse +import hashlib +import json +import logging +from pathlib import Path +import time +from urllib.parse import urlsplit + +import httpx + +from experiments.evals import refusal_version_audit_clarification as candidate +from experiments.evals.refusal_evidence_sufficiency import ClassifierInput, ContextSource + +MODEL = "qwen3.5-plus-2026-04-20" +MAX_CALLS, MAX_BODY_BYTES, MAX_TOTAL_BYTES = 12, 24576, 196608 +CASES = ( + ("fixed_version", "What is ElmCache 3.2's timeout?", "ElmCache 3.2 timeout is 40 seconds.", "SUPPORTED", "SUFFICIENT"), + ("version_mismatch", "What is ElmCache 4.0's timeout?", "ElmCache 3.2 only: timeout is 40 seconds. No coverage of 4.0.", "MISSING", "INSUFFICIENT"), + ("compatible_range", "What is ElmCache 3.4's timeout?", "For all ElmCache 3.x releases, timeout is 40 seconds.", "SUPPORTED", "SUFFICIENT"), + ("outdated", "What is ElmCache's current timeout in 2026?", "Archived 2023 ElmCache timeout: 40 seconds; no later coverage.", "MISSING", "INSUFFICIENT"), + ("no_version", "How do I open ElmCache's help menu?", "Open Help from ElmCache's toolbar.", "NOT_REQUIRED", "SUFFICIENT"), + ("unrelated_gap", "What encryption mode does ElmCache 3.2 use?", "ElmCache 3.2 manual: timeout is 40 seconds. This manual does not specify its encryption mode.", "SUPPORTED", "INSUFFICIENT"), +) +BUILDERS = {"v5_3": candidate.prior.build_classifier_messages_v5_3_dev, + "v5_3_1": candidate.build_classifier_messages_v5_3_1_dev} + + +def planned_calls(): + calls = [] + for index, (name, question, text, _, _) in enumerate(CASES): + value = ClassifierInput(question, (ContextSource("Source 1", "LOCAL_CHUNK", "LOCAL_DOC", text),)) + # ponytail: alternate the first arm; fixed order, no randomization framework. + for arm in tuple(BUILDERS)[::1 if index % 2 == 0 else -1]: + body = {"model": MODEL, "temperature": 0, "max_tokens": 2048, + "response_format": {"type": "json_object"}, "enable_thinking": False, + "messages": BUILDERS[arm](value)} + calls.append((name, arm, value, body)) + return calls + + +def plan(): + calls = planned_calls() + result = {"mode": "PLAN_ONLY", "model": MODEL, "max_calls": MAX_CALLS, + "max_body_bytes": MAX_BODY_BYTES, "max_total_bytes": MAX_TOTAL_BYTES, "budget_cny": 1, + "timeout_seconds": 30, "retries": 0, "thinking_enabled": False, "cases": CASES, + "call_order": [[name, arm] for name, arm, _, _ in calls]} + result["plan_sha256"] = hashlib.sha256(json.dumps( + {"plan": result, "bodies": [body for _, _, _, body in calls]}, sort_keys=True).encode()).hexdigest() + return result + + +def execute(env_file, expected_plan_sha256): + from dotenv import dotenv_values + from openai import OpenAI + + frozen = plan() + if expected_plan_sha256 != frozen["plan_sha256"]: + raise RuntimeError("Plan changed; do not execute") + config = dotenv_values(env_file) + base = config.get("DASHSCOPE_PHASE_B_BASE_URL", "") + parsed = urlsplit(base) + host = parsed.hostname or "" + if (not config.get("DASHSCOPE_PHASE_B_API_KEY") or parsed.scheme != "https" + or parsed.username or parsed.password or parsed.port or parsed.query or parsed.fragment + or not (host == "dashscope-intl.aliyuncs.com" or + host.startswith("ws-") and host.endswith(".ap-southeast-1.maas.aliyuncs.com")) + or parsed.path.rstrip("/") != "/compatible-mode/v1"): + raise RuntimeError("Dedicated Singapore classifier credentials required") + calls, rows, outcomes = planned_calls(), [], [] + report = {"plan": frozen, "status": "FAILED_NO_RERUN", "calls": rows, "outcomes": outcomes, + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + started = None + + def before_send(request): + nonlocal started + index, size = len(rows), len(request.content) + if (index >= MAX_CALLS or request.method != "POST" or + str(request.url) != base.rstrip("/") + "/chat/completions" or + json.loads(request.content) != calls[index][3] or size > MAX_BODY_BYTES or + sum(row["bytes"] for row in rows) + size > MAX_TOTAL_BYTES): + raise RuntimeError("Unapproved request or exhausted budget") + rows.append({"case": calls[index][0], "arm": calls[index][1], "bytes": size, + "status": None, "transport_phase": "awaiting_headers", "total_tokens": None}) + started = time.perf_counter() + + def after_headers(response): + row = rows[-1] + row.update(status=response.status_code, transport_phase="reading_body") + response.read() + row["transport_phase"] = "complete" + if response.status_code == 200: + payload = response.json() + for field in ("total_tokens", "prompt_tokens", "completion_tokens"): + value = payload.get("usage", {}).get(field) + row[field] = value if type(value) is int and value >= 0 else None + choice = payload["choices"][0] + row.update(finish_reason=choice.get("finish_reason"), completion_text=choice["message"].get("content")) + + for name in ("httpx", "openai"): + logging.getLogger(name).setLevel(logging.WARNING) + try: + with httpx.Client(transport=httpx.HTTPTransport(retries=0), trust_env=False, follow_redirects=False, + event_hooks={"request": [before_send], "response": [after_headers]}) as http: + with OpenAI(api_key=config["DASHSCOPE_PHASE_B_API_KEY"], base_url=base, + max_retries=0, timeout=30, http_client=http) as client: + for name, arm, value, body in calls: + try: + response = client.chat.completions.create( + **{k: v for k, v in body.items() if k != "enable_thinking"}, + extra_body={"enable_thinking": False}) + choice = response.choices[0] + raw = choice.message.content + if (choice.finish_reason != "stop" or not isinstance(raw, str) or not raw.strip() + or choice.message.tool_calls or choice.message.refusal): + raise RuntimeError("Incomplete classifier response") + gate = candidate.validate_requirement_coverage_response(raw, value) + target = next(case for case in CASES if case[0] == name) + observed = gate["coverage_checks"]["version_and_time"] + outcomes.append({"case": name, "arm": arm, "decision": gate["decision"], + "version_and_time": observed, "expected_version_status": target[3], + "expected_decision": target[4], "facet_matches": observed["status"] == target[3], + "decision_matches": gate["decision"] == target[4]}) + finally: + if rows and started is not None: + rows[-1]["latency_ms"] = round((time.perf_counter() - started) * 1000) + report["status"] = "COMPLETED_DEVELOPMENT_CHECK_NO_PROMOTION" + except Exception as error: + from rag_runtime.hybrid_ask import safe_error_details + report["error_types"] = safe_error_details(error)["error_types"] + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute", action="store_true", help="Paid calls; requires prior human authorization") + parser.add_argument("--env-file", type=Path) + parser.add_argument("--expected-plan-sha256") + args = parser.parse_args() + if not args.execute: + result = plan() + else: + if not args.env_file or not args.expected_plan_sha256: + parser.error("Execution requires --env-file and --expected-plan-sha256") + try: + result = execute(args.env_file, args.expected_plan_sha256) + except Exception as error: + result = {"status": "PREFLIGHT_FAILED_ZERO_CALLS", "error_type": type(error).__name__} + print(json.dumps(result, ensure_ascii=False, indent=2)) + if "FAILED" in result.get("status", ""): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_probe_version_audit.py b/tests/test_probe_version_audit.py new file mode 100644 index 0000000..5bb3bc9 --- /dev/null +++ b/tests/test_probe_version_audit.py @@ -0,0 +1,147 @@ +"""Offline wire-guard checks; mocked classifier output is not quality evidence.""" + +import json +from pathlib import Path + +import httpx +import pytest + +from scripts import probe_version_audit as probe + +URL = "https://ws-offline.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1" + + +def run_fake(monkeypatch, respond, url=URL): + monkeypatch.setattr("dotenv.dotenv_values", lambda _: { + "DASHSCOPE_PHASE_B_API_KEY": "private-test-key", "DASHSCOPE_PHASE_B_BASE_URL": url}) + + def transport(**kwargs): + assert kwargs == {"retries": 0} + return httpx.MockTransport(respond) + + monkeypatch.setattr(httpx, "HTTPTransport", transport) + return probe.execute(Path("not-read.env"), probe.plan()["plan_sha256"]) + + +def reply(request, finish="stop"): + body = json.loads(request.content) + question = json.loads(body["messages"][1]["content"])["question"] + case = next(c for c in probe.CASES if c[1] == question) + status = case[3] + if case[0] == "fixed_version" and body["messages"][0]["content"] == probe.candidate.prior.CLASSIFIER_SYSTEM_PROMPT_V5_3_DEV: + status = "NOT_REQUIRED" # Simulated mismatch stays visible, not a real baseline result. + ref = [{"source_id": "Source 1", "segment_ids": [1]}] + facet = {"status": status, "evidence": ref if status == "SUPPORTED" else [], "reason": "Offline fixture."} + outcome = {"status": "SUPPORTED" if case[4] == "SUFFICIENT" else "MISSING", + "evidence": ref if case[4] == "SUFFICIENT" else [], "reason": "Offline fixture."} + raw = json.dumps({"requirements": [{"requirement": "Requested claim", **outcome}], + "coverage_checks": {"requested_outcomes": outcome, "applicability": outcome, "version_and_time": facet}, + "unresolved_ambiguities": []}) + return httpx.Response(200, json={"id": "offline", "object": "chat.completion", "model": probe.MODEL, + "choices": [{"index": 0, "finish_reason": finish, "message": { + "role": "assistant", "content": raw, "reasoning_content": "PRIVATE_REASONING"}}], + "usage": {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20}}) + + +def test_plan_is_clean_paired_and_has_no_provider_or_env_reads(monkeypatch): + monkeypatch.setattr("dotenv.dotenv_values", lambda _: pytest.fail("plan read credentials")) + planned = probe.planned_calls() + assert len(planned) == probe.plan()["max_calls"] == 12 + for index in range(0, 12, 2): + first, second = planned[index:index + 2] + assert first[0] == second[0] and first[1] != second[1] + assert first[3]["messages"][1:] == second[3]["messages"][1:] + assert first[1] == ("v5_3" if index // 2 % 2 == 0 else "v5_3_1") + for _, _, _, body in (first, second): + payload = json.loads(body["messages"][1]["content"]) + assert set(payload) == {"question", "sources"} and "LOCAL_" not in json.dumps(payload) + assert body["enable_thinking"] is False and body["max_tokens"] == 2048 + + +def test_complete_mock_run_records_all_pairs_without_relabeling(monkeypatch): + sent = [] + + def respond(request): + assert request.extensions["timeout"]["read"] == 30 + sent.append(request) + return reply(request) + + result = run_fake(monkeypatch, respond) + assert result["status"] == "COMPLETED_DEVELOPMENT_CHECK_NO_PROMOTION" + assert len(sent) == len(result["calls"]) == len(result["outcomes"]) == 12 + assert all(row["status"] == 200 and row["transport_phase"] == "complete" for row in result["calls"]) + assert sum(row["total_tokens"] for row in result["calls"]) == 240 + assert [row["facet_matches"] for row in result["outcomes"]].count(False) == 1 + assert all(row["decision_matches"] for row in result["outcomes"]) + assert "private-test-key" not in json.dumps(result) and "PRIVATE_REASONING" not in json.dumps(result) + + +@pytest.mark.parametrize("failure", ["read", "connect", "http", "json", "truncated", "body"]) +def test_provider_or_parse_failure_stops_without_retry_or_error_body(monkeypatch, failure): + sent = [] + + class BrokenBody(httpx.SyncByteStream): + def __iter__(self): + yield b"partial" + raise httpx.ReadTimeout("private-provider-error") + + def respond(request): + sent.append(request) + if failure in {"read", "connect"}: + error = httpx.ReadTimeout if failure == "read" else httpx.ConnectTimeout + raise error("private-provider-error", request=request) + if failure == "http": + return httpx.Response(500, json={"error": "private-provider-error"}) + if failure == "json": + return httpx.Response(200, content=b"private-provider-error") + if failure == "body": + return httpx.Response(200, stream=BrokenBody()) + return reply(request, "length") + + result = run_fake(monkeypatch, respond) + assert result["status"] == "FAILED_NO_RERUN" and result["error_types"] + assert len(sent) == len(result["calls"]) == 1 and not result["outcomes"] + assert "private-provider-error" not in json.dumps(result) and "private-test-key" not in json.dumps(result) + if failure == "body": + assert result["calls"][0]["status"] == 200 and result["calls"][0]["transport_phase"] == "reading_body" + + +@pytest.mark.parametrize("change", ["label", "foreign_source"]) +def test_wire_guard_blocks_unapproved_data_before_transport(monkeypatch, change): + original_send = httpx.Client.send + sent = [] + + def tamper(client, request, **kwargs): + body = json.loads(request.content) + if change == "label": + body["expected_status"] = "SUPPORTED" + else: + body["messages"][1]["content"] = "foreign business material" + changed = httpx.Request(request.method, request.url, json=body, extensions=request.extensions, + headers={k: v for k, v in request.headers.items() if k.lower() != "content-length"}) + return original_send(client, changed, **kwargs) + + monkeypatch.setattr(httpx.Client, "send", tamper) + result = run_fake(monkeypatch, lambda request: sent.append(request) or reply(request)) + assert result["status"] == "FAILED_NO_RERUN" and not sent and not result["calls"] + + +def test_changed_plan_fails_before_credentials_are_read(monkeypatch): + monkeypatch.setattr("dotenv.dotenv_values", lambda _: pytest.fail("read credentials before hash check")) + with pytest.raises(RuntimeError, match="Plan changed"): + probe.execute(Path("not-read.env"), "wrong-hash") + + +@pytest.mark.parametrize("limit", ["MAX_CALLS", "MAX_BODY_BYTES", "MAX_TOTAL_BYTES"]) +def test_wire_budget_stops_before_any_extra_send(monkeypatch, limit): + monkeypatch.setattr(probe, limit, 1) + sent = [] + result = run_fake(monkeypatch, lambda request: sent.append(request) or reply(request)) + assert result["status"] == "FAILED_NO_RERUN" + assert len(sent) == len(result["calls"]) == (1 if limit == "MAX_CALLS" else 0) + + +@pytest.mark.parametrize("url", ["https://example.com/compatible-mode/v1", "https://user:password@dashscope-intl.aliyuncs.com/compatible-mode/v1"]) +def test_endpoint_outside_scope_is_rejected_before_transport(monkeypatch, url): + with pytest.raises(RuntimeError, match="Singapore"): + run_fake(monkeypatch, lambda _: pytest.fail("unapproved endpoint called"), url) From 8d696bc93df238329fe7ecb25272042f12b2104f Mon Sep 17 00:00:00 2001 From: Air000000 <1113476369@qq.com> Date: Sat, 17 Oct 2026 23:32:11 +0800 Subject: [PATCH 2/2] docs: record partial version audit gain without runtime promotion --- README.md | 8 +- .../version_audit_paired_2026-10-17.json | 554 ++++++++++++++++++ 2 files changed, 560 insertions(+), 2 deletions(-) create mode 100644 docs/reports/version_audit_paired_2026-10-17.json diff --git a/README.md b/README.md index aa7855e..a007b49 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ Hybrid 检索（同一租户/分类、已索引当前版本，最多 Top14） ## 3.3 集成验收状态（2026-10-17） -**当前不接受为已验收发布版。** 已补齐仓库声明的评测测试依赖，在隔离 Windows / Python 3.12.4 环境运行完整 `tests/`：最新 640 项通过、零失败/跳过；不只原重点回归集。主 pytest 进程禁止外部 socket 与读取项目私有 `data/`，测试使用临时数据；CI 包含相同完整离线检查。离线成功不是模型质量验收。 +**当前不接受为已验收发布版。** 已补齐仓库声明的评测测试依赖，在隔离 Windows / Python 3.12.4 环境运行完整 `tests/`：最新 656 项通过、零失败/跳过；不只原重点回归集。主 pytest 进程禁止外部 socket 与读取项目私有 `data/`，测试使用临时数据；CI 包含相同完整离线检查。离线成功不是模型质量验收。 真实问答计划复用同一脚本，默认仍然零调用： @@ -321,7 +321,11 @@ python -m scripts.smoke_hybrid_runtime --ask --execute --env-file .env 复测仍有版本审计语义警示：NovaVPN 1.0 的显式版本约束仍被标成 `NOT_REQUIRED`；AcornExpense 2.0 的版本覆盖被标成 `SUPPORTED`，理由却承认来源只覆盖 1.0。后者整体因其他检查为 `MISSING` 而正确拒答，但不能据此认定每项审计可靠。保留这两项警示；该次复测未针对案例改提示词或规则，历史 NO_PROMOTION / FAIL 不变。 -新增离线 [v5.3.1 版本审计澄清候选](experiments/evals/refusal_version_audit_clarification.py)：只追加通用说明，区分“来源声明自己的版本范围”和“证据满足问题所需版本”，保留可见兼容性依据，不退回版本关键词/字符串匹配。没有加入上述验收案例答案，沿用原输入、状态与严格引用解析器，不修补或重标模型输出。6 类手写虚构边界及两类错误输出回放只检查接线与限制，**不证明模型已改正判断**；未进行候选真实调用，未接入 API，现有 `v5_3_experimental` 提示词、运行行为与历史报告均不变。 +新增离线 [v5.3.1 版本审计澄清候选](experiments/evals/refusal_version_audit_clarification.py)：只追加通用说明，区分“来源声明自己的版本范围”和“证据满足问题所需版本”，保留可见兼容性依据，不退回版本关键词/字符串匹配。没有加入上述验收案例答案，沿用原输入、状态与严格引用解析器，不修补或重标模型输出。手写边界测试只证明接线与限制；候选未接入 API，现有 `v5_3_experimental` 提示词、运行行为与历史报告均不变。 + +随后单独授权完成一次[6 题分类配对开发检查](docs/reports/version_audit_paired_2026-10-17.json)：输入、预期与请求守卫在调用前提交冻结，新旧提示词使用相同输入/参数、交替首跑顺序。版本审计符合预期为 v5.3 **3/6**、v5.3.1 **5/6**；整体放行/拒答均 **6/6**，没有决策差异。候选在版本不符、兼容范围两题改善分项审计，但仍把“版本已匹配、功能信息缺失”的版本检查标成 `NOT_REQUIRED`。这只是少量已知虚构开发场景，不是独立验收，也不证明回答准确率提升；不改预期、不重标结果、不因局部改善切换服务策略。 + +本次仅 12 次新加坡分类，均 HTTP 200，无重试；usage 合计 27481 tokens，请求体 119304 bytes，没有重跑检索或生成。`python -m scripts.probe_version_audit` 默认只输出零调用计划；真实执行还要求事先授权、配置专用新加坡凭据并绑定计划哈希。单次/累计请求体上限为 24/192 KiB，最多 12 次调用；发送守卫逐次比对完整请求体，预期标签不进入模型请求。任何网络、提供方或解析异常即停，不自动续跑。候选仅部分有效，下一步应先判断审计改进的收益，不自动进入新一轮付费调参。 只读检查确认 Docker 引擎仍不可连接；真实运行服务 HTTP、容器重启/持久化与回退验收均未完成。本次使用临时 ASGI 应用、SQL 和 Chroma，不是部署验收；Ticket Agent 仍使用 Dense，不声明全链 Hybrid，默认开关保持关闭。下一步仍需处理版本审计一致性与运行环境验收，而不是因 8 类流程通过就晋级发布。 diff --git a/docs/reports/version_audit_paired_2026-10-17.json b/docs/reports/version_audit_paired_2026-10-17.json new file mode 100644 index 0000000..40f7f5b --- /dev/null +++ b/docs/reports/version_audit_paired_2026-10-17.json @@ -0,0 +1,554 @@ +{ + "date": "2026-10-17", + "observed_at_utc": "2026-10-17 15:29:27 UTC", + "candidate_base_commit": "4d6f17c53cc33dee2997c6c6cbbc6aad431076dc", + "protocol_commit_before_calls": "214eb8dd6655846ebd273b1b31d28cc9dfb7d95e", + "execution_count_under_this_authorization": 1, + "offline_verification": { + "passed": 656, + "failed": 0, + "skipped": 0, + "duration_seconds": 66.4, + "environment": "Windows Python 3.12.4; temporary DB/dummy credentials; external Python socket and private project data guards", + "final_wire_guard_tests": 16 + }, + "scope": "Fixed six-case fictional development check; paired classification only, same inputs/parameters, alternating first arm. No TechQA or independent holdout evaluation.", + "budget_note": "CNY 1 is the conservative authorized estimate, not an invoice or billing-system hard cap.", + "plan": { + "mode": "PLAN_ONLY", + "model": "qwen3.5-plus-2026-04-20", + "max_calls": 12, + "max_body_bytes": 24576, + "max_total_bytes": 196608, + "budget_cny": 1, + "timeout_seconds": 30, + "retries": 0, + "thinking_enabled": false, + "cases": [ + [ + "fixed_version", + "What is ElmCache 3.2's timeout?", + "ElmCache 3.2 timeout is 40 seconds.", + "SUPPORTED", + "SUFFICIENT" + ], + [ + "version_mismatch", + "What is ElmCache 4.0's timeout?", + "ElmCache 3.2 only: timeout is 40 seconds. No coverage of 4.0.", + "MISSING", + "INSUFFICIENT" + ], + [ + "compatible_range", + "What is ElmCache 3.4's timeout?", + "For all ElmCache 3.x releases, timeout is 40 seconds.", + "SUPPORTED", + "SUFFICIENT" + ], + [ + "outdated", + "What is ElmCache's current timeout in 2026?", + "Archived 2023 ElmCache timeout: 40 seconds; no later coverage.", + "MISSING", + "INSUFFICIENT" + ], + [ + "no_version", + "How do I open ElmCache's help menu?", + "Open Help from ElmCache's toolbar.", + "NOT_REQUIRED", + "SUFFICIENT" + ], + [ + "unrelated_gap", + "What encryption mode does ElmCache 3.2 use?", + "ElmCache 3.2 manual: timeout is 40 seconds. This manual does not specify its encryption mode.", + "SUPPORTED", + "INSUFFICIENT" + ] + ], + "call_order": [ + [ + "fixed_version", + "v5_3" + ], + [ + "fixed_version", + "v5_3_1" + ], + [ + "version_mismatch", + "v5_3_1" + ], + [ + "version_mismatch", + "v5_3" + ], + [ + "compatible_range", + "v5_3" + ], + [ + "compatible_range", + "v5_3_1" + ], + [ + "outdated", + "v5_3_1" + ], + [ + "outdated", + "v5_3" + ], + [ + "no_version", + "v5_3" + ], + [ + "no_version", + "v5_3_1" + ], + [ + "unrelated_gap", + "v5_3_1" + ], + [ + "unrelated_gap", + "v5_3" + ] + ], + "plan_sha256": "4b427120b761e2ec544207cf5b76d993834090d4501397641d287aa7591bbf86" + }, + "status": "COMPLETED_DEVELOPMENT_CHECK_NO_PROMOTION", + "calls": [ + { + "case": "fixed_version", + "arm": "v5_3", + "bytes": 9255, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2191, + "prompt_tokens": 1853, + "completion_tokens": 338, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the timeout value for ElmCache version 3.2\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"Source 1 explicitly states that the ElmCache 3.2 timeout is 40 seconds.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source provides the specific timeout value requested for the specified product and version.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly addresses 'ElmCache 3.2', matching the product and version in the question.\"\n},\n\"version_and_time\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question specifies version 3.2, and the source provides data specifically for ElmCache 3.2.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 6168 + }, + { + "case": "fixed_version", + "arm": "v5_3_1", + "bytes": 10577, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2429, + "prompt_tokens": 2090, + "completion_tokens": 339, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the timeout value for ElmCache version 3.2\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"Source 1 explicitly states that ElmCache 3.2 timeout is 40 seconds.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source directly provides the specific timeout value requested for the specified product and version.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly names 'ElmCache 3.2', matching the product and version in the question.\"\n},\n\"version_and_time\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question requests information for a specific fixed version (3.2), and the source provides data specifically for that version.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 5639 + }, + { + "case": "version_mismatch", + "arm": "v5_3_1", + "bytes": 10603, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2356, + "prompt_tokens": 2100, + "completion_tokens": 256, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the timeout value for ElmCache version 4.0\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only provided source explicitly states it covers ElmCache 3.2 and has no coverage of version 4.0.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"No source provides the timeout value for ElmCache 4.0; the available source explicitly excludes this version.\"\n},\n\"applicability\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question targets ElmCache 4.0, but the evidence is scoped to ElmCache 3.2 and explicitly notes lack of coverage for 4.0.\"\n},\n\"version_and_time\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question explicitly requests information for version 4.0, which is not covered by the provided sources.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 4458 + }, + { + "case": "version_mismatch", + "arm": "v5_3", + "bytes": 9281, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2140, + "prompt_tokens": 1863, + "completion_tokens": 277, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the timeout value for ElmCache version 4.0\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only provided source explicitly states it covers ElmCache 3.2 and has no coverage of version 4.0.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"No source provides a timeout value for ElmCache 4.0; the available data is scoped to version 3.2.\"\n},\n\"applicability\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The source explicitly excludes version 4.0 from its scope ('No coverage of 4.0').\"\n},\n\"version_and_time\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question specifies version 4.0, and the source confirms that this specific version is not covered by the provided information.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 4683 + }, + { + "case": "compatible_range", + "arm": "v5_3", + "bytes": 9273, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2204, + "prompt_tokens": 1855, + "completion_tokens": 349, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the specific timeout value for ElmCache version 3.4.\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source states that for all ElmCache 3.x releases, the timeout is 40 seconds. Version 3.4 falls within the 3.x range.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly provides the timeout value (40 seconds) for the product family containing the requested version.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question asks about ElmCache 3.4. The source covers 'all ElmCache 3.x releases', which includes version 3.4.\"\n},\n\"version_and_time\": {\n\"status\": \"NOT_REQUIRED\",\n\"evidence\": [],\n\"reason\": \"The question asks for a configuration value (timeout) for a specific version, not for information about release dates, current status, or time-sensitive changes.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 5817 + }, + { + "case": "compatible_range", + "arm": "v5_3_1", + "bytes": 10595, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2478, + "prompt_tokens": 2092, + "completion_tokens": 386, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the timeout value for ElmCache version 3.4\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"Source 1 states that for all ElmCache 3.x releases, the timeout is 40 seconds. Version 3.4 falls within the 3.x range.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly provides the timeout value (40 seconds) for the requested product family (ElmCache 3.x), which includes version 3.4.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question asks about ElmCache 3.4. The source covers 'all ElmCache 3.x releases', which directly applies to version 3.4.\"\n},\n\"version_and_time\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question specifies version 3.4. The source provides a rule for the entire 3.x series, covering this specific version.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 6090 + }, + { + "case": "outdated", + "arm": "v5_3_1", + "bytes": 10616, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2454, + "prompt_tokens": 2099, + "completion_tokens": 355, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify ElmCache's current timeout value specifically for the year 2026.\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only available source provides data for 2023 and explicitly states there is no later coverage. There is no evidence supporting the timeout value for 2026.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question asks for a specific value in 2026. The source only covers 2023 and notes no later coverage, so the requested outcome (2026 value) is not supported.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source discusses ElmCache timeouts, which matches the product scope of the question, but fails on the time constraint.\"\n},\n\"version_and_time\": {\n\"status\": \"MISSING\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The question explicitly requires the timeout for 2026. The source provides 2023 data and explicitly excludes later years ('no later coverage'), leaving the 2026 requirement unsupported.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 5795 + }, + { + "case": "outdated", + "arm": "v5_3", + "bytes": 9294, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2172, + "prompt_tokens": 1862, + "completion_tokens": 310, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify ElmCache's current timeout value specifically for the year 2026.\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only available source provides data for 2023 and explicitly states there is no later coverage. There is no evidence supporting the timeout value in 2026.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question requests a specific value for 2026, but the source only covers 2023 and disclaims later coverage. The requested outcome cannot be supported.\"\n},\n\"applicability\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The source applies to 2023, which does not match the requested timeframe of 2026. No other sources are provided to establish applicability for 2026.\"\n},\n\"version_and_time\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question explicitly requires information for '2026'. The source provides 'Archived 2023' data and notes 'no later coverage', failing to meet the explicit time constraint.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 5299 + }, + { + "case": "no_version", + "arm": "v5_3", + "bytes": 9258, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2137, + "prompt_tokens": 1846, + "completion_tokens": 291, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Provide the method to open ElmCache's help menu.\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly states to open Help from ElmCache's toolbar.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source provides the specific action required to open the help menu.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The instruction applies directly to ElmCache as requested in the question.\"\n},\n\"version_and_time\": {\n\"status\": \"NOT_REQUIRED\",\n\"evidence\": [],\n\"reason\": \"The question does not specify a version or time constraint, and the source provides a general UI instruction without version limitations.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 6920 + }, + { + "case": "no_version", + "arm": "v5_3_1", + "bytes": 10580, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2374, + "prompt_tokens": 2083, + "completion_tokens": 291, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Provide the method to open ElmCache's help menu.\",\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source explicitly states to open Help from ElmCache's toolbar.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source provides the specific action required to open the help menu.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The instruction applies directly to ElmCache as requested in the question.\"\n},\n\"version_and_time\": {\n\"status\": \"NOT_REQUIRED\",\n\"evidence\": [],\n\"reason\": \"The question does not specify a version or time constraint, and the source provides a general UI instruction without version restrictions.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 5051 + }, + { + "case": "unrelated_gap", + "arm": "v5_3_1", + "bytes": 10647, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2396, + "prompt_tokens": 2102, + "completion_tokens": 294, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the encryption mode used by ElmCache 3.2\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only available source explicitly states that it does not specify the encryption mode for ElmCache 3.2.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question asks for a specific configuration detail (encryption mode) which is explicitly absent from the provided context.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source correctly identifies the product and version (ElmCache 3.2) relevant to the question, but fails to provide the requested data point.\"\n},\n\"version_and_time\": {\n\"status\": \"NOT_REQUIRED\",\n\"evidence\": [],\n\"reason\": \"The question specifies a fixed version (3.2), but since the information is missing entirely, there are no conflicting or supporting time/version constraints to audit beyond the explicit version match in the source scope.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 4961 + }, + { + "case": "unrelated_gap", + "arm": "v5_3", + "bytes": 9325, + "status": 200, + "transport_phase": "complete", + "total_tokens": 2150, + "prompt_tokens": 1865, + "completion_tokens": 285, + "finish_reason": "stop", + "completion_text": "{\n\"requirements\": [\n{\n\"requirement\": \"Identify the encryption mode used by ElmCache 3.2\",\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The only provided source explicitly states that it does not specify the encryption mode for ElmCache 3.2.\"\n}\n],\n\"coverage_checks\": {\n\"requested_outcomes\": {\n\"status\": \"MISSING\",\n\"evidence\": [],\n\"reason\": \"The question asks for a specific configuration detail (encryption mode) which is explicitly absent from the provided context.\"\n},\n\"applicability\": {\n\"status\": \"SUPPORTED\",\n\"evidence\": [\n{\n\"source_id\": \"Source 1\",\n\"segment_ids\": [\n1\n]\n}\n],\n\"reason\": \"The source correctly identifies the product and version (ElmCache 3.2) relevant to the question, but fails to provide the requested data point.\"\n},\n\"version_and_time\": {\n\"status\": \"NOT_REQUIRED\",\n\"evidence\": [],\n\"reason\": \"The question specifies a fixed version (3.2), so there are no dynamic time or 'latest' status requirements to audit beyond verifying the version match.\"\n}\n},\n\"unresolved_ambiguities\": []\n}", + "latency_ms": 4665 + } + ], + "outcomes": [ + { + "case": "fixed_version", + "arm": "v5_3", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "SUPPORTED", + "evidence": [ + { + "source_id": "Source 1", + "segment_ids": [ + 1 + ] + } + ], + "reason": "The question specifies version 3.2, and the source provides data specifically for ElmCache 3.2." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "SUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "fixed_version", + "arm": "v5_3_1", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "SUPPORTED", + "evidence": [ + { + "source_id": "Source 1", + "segment_ids": [ + 1 + ] + } + ], + "reason": "The question requests information for a specific fixed version (3.2), and the source provides data specifically for that version." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "SUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "version_mismatch", + "arm": "v5_3_1", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "MISSING", + "evidence": [], + "reason": "The question explicitly requests information for version 4.0, which is not covered by the provided sources." + }, + "expected_version_status": "MISSING", + "expected_decision": "INSUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "version_mismatch", + "arm": "v5_3", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "SUPPORTED", + "evidence": [ + { + "source_id": "Source 1", + "segment_ids": [ + 1 + ] + } + ], + "reason": "The question specifies version 4.0, and the source confirms that this specific version is not covered by the provided information." + }, + "expected_version_status": "MISSING", + "expected_decision": "INSUFFICIENT", + "facet_matches": false, + "decision_matches": true + }, + { + "case": "compatible_range", + "arm": "v5_3", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "NOT_REQUIRED", + "evidence": [], + "reason": "The question asks for a configuration value (timeout) for a specific version, not for information about release dates, current status, or time-sensitive changes." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "SUFFICIENT", + "facet_matches": false, + "decision_matches": true + }, + { + "case": "compatible_range", + "arm": "v5_3_1", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "SUPPORTED", + "evidence": [ + { + "source_id": "Source 1", + "segment_ids": [ + 1 + ] + } + ], + "reason": "The question specifies version 3.4. The source provides a rule for the entire 3.x series, covering this specific version." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "SUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "outdated", + "arm": "v5_3_1", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "MISSING", + "evidence": [ + { + "source_id": "Source 1", + "segment_ids": [ + 1 + ] + } + ], + "reason": "The question explicitly requires the timeout for 2026. The source provides 2023 data and explicitly excludes later years ('no later coverage'), leaving the 2026 requirement unsupported." + }, + "expected_version_status": "MISSING", + "expected_decision": "INSUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "outdated", + "arm": "v5_3", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "MISSING", + "evidence": [], + "reason": "The question explicitly requires information for '2026'. The source provides 'Archived 2023' data and notes 'no later coverage', failing to meet the explicit time constraint." + }, + "expected_version_status": "MISSING", + "expected_decision": "INSUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "no_version", + "arm": "v5_3", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "NOT_REQUIRED", + "evidence": [], + "reason": "The question does not specify a version or time constraint, and the source provides a general UI instruction without version limitations." + }, + "expected_version_status": "NOT_REQUIRED", + "expected_decision": "SUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "no_version", + "arm": "v5_3_1", + "decision": "SUFFICIENT", + "version_and_time": { + "status": "NOT_REQUIRED", + "evidence": [], + "reason": "The question does not specify a version or time constraint, and the source provides a general UI instruction without version restrictions." + }, + "expected_version_status": "NOT_REQUIRED", + "expected_decision": "SUFFICIENT", + "facet_matches": true, + "decision_matches": true + }, + { + "case": "unrelated_gap", + "arm": "v5_3_1", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "NOT_REQUIRED", + "evidence": [], + "reason": "The question specifies a fixed version (3.2), but since the information is missing entirely, there are no conflicting or supporting time/version constraints to audit beyond the explicit version match in the source scope." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "INSUFFICIENT", + "facet_matches": false, + "decision_matches": true + }, + { + "case": "unrelated_gap", + "arm": "v5_3", + "decision": "INSUFFICIENT", + "version_and_time": { + "status": "NOT_REQUIRED", + "evidence": [], + "reason": "The question specifies a fixed version (3.2), so there are no dynamic time or 'latest' status requirements to audit beyond verifying the version match." + }, + "expected_version_status": "SUPPORTED", + "expected_decision": "INSUFFICIENT", + "facet_matches": false, + "decision_matches": true + } + ], + "script_sha256": "4776a2dda6b400b972a9494e8c5a593afa096c38f0383320a386d4bac2ca5078", + "summary": { + "arms": { + "v5_3": { + "calls": 6, + "facet_matches": 3, + "decision_matches": 6, + "total": 6, + "total_tokens": 12994, + "unknown_usage_calls": 0 + }, + "v5_3_1": { + "calls": 6, + "facet_matches": 5, + "decision_matches": 6, + "total": 6, + "total_tokens": 14487, + "unknown_usage_calls": 0 + } + }, + "all_http_200": true, + "total_tokens": 27481, + "unknown_usage_calls": 0, + "total_request_bytes": 119304, + "mismatches": [ + { + "case": "version_mismatch", + "arm": "v5_3", + "observed": "SUPPORTED", + "expected": "MISSING" + }, + { + "case": "compatible_range", + "arm": "v5_3", + "observed": "NOT_REQUIRED", + "expected": "SUPPORTED" + }, + { + "case": "unrelated_gap", + "arm": "v5_3_1", + "observed": "NOT_REQUIRED", + "expected": "SUPPORTED" + }, + { + "case": "unrelated_gap", + "arm": "v5_3", + "observed": "NOT_REQUIRED", + "expected": "SUPPORTED" + } + ], + "plan_sha256": "4b427120b761e2ec544207cf5b76d993834090d4501397641d287aa7591bbf86", + "script_sha256": "4776a2dda6b400b972a9494e8c5a593afa096c38f0383320a386d4bac2ca5078" + }, + "interpretation": "Version/time facet agreement improved from 3/6 to 5/6 on these six handwritten development cases, with two candidate-only improvements and no observed regression. Overall gate decisions were identical and matched expectations in both arms (6/6); this does not demonstrate improved answer quality or general refusal accuracy. A single tiny known-fixture run is not independent acceptance or evidence of stable model behavior.", + "unresolved": { + "case": "unrelated_gap", + "expected": "Version is explicitly requested and visibly covered, so version_and_time should be SUPPORTED; the separate encryption-mode information gap still requires overall INSUFFICIENT.", + "observed": "Both arms mark the version/time facet NOT_REQUIRED. The candidate's own reason acknowledges the explicit version match but improperly treats the unrelated feature gap as removing the version-audit requirement.", + "assessment": "Candidate is only partially effective. Do not relabel its output, revise expectations after observation, silently retry, or enable it in serving." + }, + "non_actions": [ + "No further paid calls or prompt edits after observing results", + "No embedding, rerank, answer generation, business data, TechQA, DEV, expected labels or secrets in request bodies", + "No hidden reasoning or provider error bodies retained; only completion content, usage, phase and type-only errors", + "No API/policy/config changes, deployment, ticket creation, or edits to frozen v5.3/v5.3.1 and historical evidence" + ], + "next_step": "Do not promote on this probe. Decide whether audit-only improvements justify further work before introducing another prompt/structural candidate; current runtime stays v5_3_experimental with default-off switches." +}