diff --git a/pyproject.toml b/pyproject.toml index 0f4be37..f3b40d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "okf-compiler" -version = "0.2.0" +version = "0.2.1" description = "Compile Markdown articles into isolated, self-contained OKF Bundles." readme = "README.md" requires-python = ">=3.10" diff --git a/src/okf_compiler/diagnostics.py b/src/okf_compiler/diagnostics.py index 4b00588..516107d 100644 --- a/src/okf_compiler/diagnostics.py +++ b/src/okf_compiler/diagnostics.py @@ -28,27 +28,36 @@ def event(self, event: str, **data) -> None: with path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(row, ensure_ascii=False) + "\n") - def stage_request(self, stage: str, system: str, user: str) -> None: + def stage_request(self, stage: str, system: str, user: str, *, attempt: int = 1) -> None: self.event( "llm_request", stage=stage, + attempt=attempt, system_chars=len(system), user_chars=len(user), payload_saved=self.include_llm_payloads, ) if self.include_llm_payloads: - base = self.root / "stages" / stage + base = self._stage_base(stage, attempt) atomic_write_json(base / "request.json", {"system": system, "user": user}) - def stage_response(self, stage: str, raw: str, parsed: object | None = None) -> None: + def stage_response( + self, + stage: str, + raw: str, + parsed: object | None = None, + *, + attempt: int = 1, + ) -> None: self.event( "llm_response", stage=stage, + attempt=attempt, response_chars=len(raw), payload_saved=self.include_llm_payloads, ) if self.include_llm_payloads: - base = self.root / "stages" / stage + base = self._stage_base(stage, attempt) atomic_write_text(base / "response.raw.txt", raw) if parsed is not None: atomic_write_json(base / "response.parsed.json", parsed) @@ -85,6 +94,10 @@ def finish(self, data: dict) -> None: payload = {"finished_at": _now(), **data, "events": self._events} atomic_write_json(self.root / "run.json", payload) + def _stage_base(self, stage: str, attempt: int) -> Path: + base = self.root / "stages" / stage + return base if attempt == 1 else base / f"attempt-{attempt}" + def _now() -> str: return datetime.now(timezone.utc).isoformat() diff --git a/src/okf_compiler/llm.py b/src/okf_compiler/llm.py index fb06573..f6e9a87 100644 --- a/src/okf_compiler/llm.py +++ b/src/okf_compiler/llm.py @@ -6,13 +6,21 @@ import os import re import time +import unicodedata from dataclasses import dataclass from pathlib import Path from dotenv import dotenv_values from .diagnostics import DebugRecorder -from .prompts import concepts_messages, entities_messages, relations_messages, summary_messages +from .prompts import ( + concepts_messages, + entities_messages, + relation_nodes, + relations_messages, + retry_system_message, + summary_messages, +) from .schema import ( ConceptExtract, EntityExtract, @@ -38,6 +46,80 @@ _DEFAULT_TIMEOUT = 120.0 _REDACTED = "[REDACTED]" _SECRET_RE = re.compile(r"(?:sk-|Bearer\s+)[A-Za-z0-9_.-]{8,}", re.IGNORECASE) +_CANONICAL_ENTITY_TYPES = { + "person", + "organization", + "product", + "platform", + "brand", + "work", + "location", + "event", + "other_named_entity", +} +_ENTITY_TYPE_ALIASES = { + "人物": "person", + "角色": "person", + "作者": "person", + "person": "person", + "human": "person", + "团队": "organization", + "开发团队": "organization", + "游戏开发团队": "organization", + "公司": "organization", + "工作室": "organization", + "组织": "organization", + "开发商": "organization", + "organization": "organization", + "company": "organization", + "studio": "organization", + "team": "organization", + "游戏产品": "product", + "游戏": "product", + "产品": "product", + "软件": "product", + "应用": "product", + "app": "product", + "game": "product", + "product": "product", + "平台": "platform", + "平台/渠道": "platform", + "渠道平台": "platform", + "platform": "platform", + "品牌": "brand", + "brand": "brand", + "作品": "work", + "文章": "work", + "书籍": "work", + "work": "work", + "地点": "location", + "地区": "location", + "location": "location", + "事件": "event", + "活动": "event", + "event": "event", + "其他命名实体": "other_named_entity", + "other_named_entity": "other_named_entity", +} +_CONCEPT_TYPE_MARKERS = ( + "概念", + "机制", + "模式", + "策略", + "玩法", + "设计", + "红利", + "钩子", + "商业", + "时刻", + "系统", + "method", + "mechanism", + "concept", + "strategy", + "pattern", + "model", +) @dataclass @@ -51,6 +133,14 @@ def is_configured(self) -> bool: return bool(self.model and self.model.strip()) +class StageResponseError(ValueError): + def __init__(self, stage: str, reason: str, attempts: int): + super().__init__(f"malformed {stage} response after {attempts} attempt(s): {reason}") + self.stage = stage + self.reason = reason + self.attempts = attempts + + def find_dotenv_path( path: Path | None = None, *, @@ -92,7 +182,7 @@ def load_dotenv_values( if resolved is None: return {} values = dotenv_values(resolved) - return {str(k): str(v) for k, v in values.items() if k and v is not None} + return {str(key): str(value) for key, value in values.items() if key and value is not None} def resolve_config( @@ -176,7 +266,7 @@ def json_completion(self, system: str, user: str) -> str: return (response.choices[0].message.content or "").strip() def test(self) -> str: - return self.json_completion("Return valid JSON only.", 'Return {"ok": true}.') + return self.json_completion("Return valid JSON only.", '{"ok": true}') def _parse_json(text: str) -> dict | list: @@ -198,6 +288,26 @@ def _parse_json(text: str) -> dict | list: return value +def _validate_stage_payload(stage: str, value: dict | list) -> dict: + if not isinstance(value, dict): + raise ValueError("top-level value must be an object") + if stage == "summary": + if "summary" not in value: + raise ValueError("missing required top-level field 'summary'") + if not isinstance(value["summary"], str): + raise ValueError("top-level field 'summary' must be a string") + return value + if stage not in {"concepts", "entities", "relations"}: + raise ValueError(f"unknown extraction stage: {stage}") + if stage not in value: + raise ValueError(f"missing required top-level array '{stage}'") + if not isinstance(value[stage], list): + raise ValueError(f"top-level field '{stage}' must be an array") + if any(not isinstance(item, dict) for item in value[stage]): + raise ValueError(f"every item in '{stage}' must be an object") + return value + + def _evidence(obj: dict) -> Evidence | None: raw = obj.get("evidence") if not isinstance(raw, dict): @@ -219,13 +329,6 @@ def integer(name: str) -> int: ) -def _items(value: dict | list, key: str) -> list[dict]: - if isinstance(value, list): - return [item for item in value if isinstance(item, dict)] - raw = value.get(key, []) - return [item for item in raw if isinstance(item, dict)] if isinstance(raw, list) else [] - - def extract( client: LLMClient, markdown: str, @@ -241,54 +344,85 @@ def extract( def warn(label: str, exc: Exception | str) -> None: out.warnings.append(redact_secrets(f"{label}: {exc}", client.api_key)) - def complete(stage: str, system: str, user: str) -> tuple[str, dict | list]: - if debug: - debug.stage_request(stage, system, user) - started = time.monotonic() - raw = client.json_completion(system, user) - value = _parse_json(raw) - if debug: - debug.stage_response(stage, raw, value) - debug.event( - "llm_stage_completed", - stage=stage, - duration_ms=round((time.monotonic() - started) * 1000, 2), - ) - return raw, value + def complete(stage: str, system: str, user: str) -> tuple[dict, int]: + current_system = system + for attempt in (1, 2): + if debug: + debug.stage_request(stage, current_system, user, attempt=attempt) + started = time.monotonic() + raw = client.json_completion(current_system, user) + parsed: dict | list | None = None + try: + parsed = _parse_json(raw) + value = _validate_stage_payload(stage, parsed) + except Exception as exc: # noqa: BLE001 + if debug: + debug.stage_response(stage, raw, parsed, attempt=attempt) + debug.event( + "llm_stage_invalid", + stage=stage, + attempt=attempt, + duration_ms=round((time.monotonic() - started) * 1000, 2), + error=redact_secrets(str(exc), client.api_key), + ) + if attempt == 1: + if debug: + debug.event( + "llm_stage_retry", + stage=stage, + attempt=attempt + 1, + reason=redact_secrets(str(exc), client.api_key), + ) + current_system = retry_system_message(stage, system, str(exc)) + continue + raise StageResponseError(stage, str(exc), attempt) from exc + if debug: + debug.stage_response(stage, raw, value, attempt=attempt) + debug.event( + "llm_stage_completed", + stage=stage, + attempt=attempt, + duration_ms=round((time.monotonic() - started) * 1000, 2), + ) + return value, attempt + raise AssertionError("unreachable") try: system, user = summary_messages(markdown, sections, language) - _, value = complete("summary", system, user) - out.summary = str(value.get("summary", "")) if isinstance(value, dict) else "" - out.stage_stats["summary"] = {"status": "ok" if out.summary else "degraded"} - except Exception as exc: # noqa: BLE001 - warn("summary extraction failed", exc) + value, attempts = complete("summary", system, user) + out.summary = value["summary"].strip() out.stage_stats["summary"] = { - "status": "failed", - "error": redact_secrets(str(exc), client.api_key), + "status": "ok" if out.summary else "degraded", + "attempts": attempts, + "retries": attempts - 1, } - if debug: - debug.event( - "llm_stage_failed", - stage="summary", - error=redact_secrets(str(exc), client.api_key), - ) + except Exception as exc: # noqa: BLE001 + warn("summary extraction failed", exc) + out.stage_stats["summary"] = _failed_stage(exc, client.api_key) + _record_stage_failure(debug, "summary", exc, client.api_key) try: system, user = concepts_messages(markdown, sections, max_concepts, summary=out.summary) - _, value = complete("concepts", system, user) - items = _items(value, "concepts")[:max_concepts] + value, attempts = complete("concepts", system, user) + items = value["concepts"][:max_concepts] rejected = 0 + seen_concepts: set[str] = set() for item in items: name = str(item.get("name") or "").strip() result = resolve_evidence(_evidence(item), sections, markdown) - if debug: - debug.validation("concepts", name, item, result.to_dict()) if not result.valid: rejected += 1 + _reject_item(out, debug, "concepts", name, item, result.to_dict()) warn("concept dropped", f"{result.reason} for {name!r}") - out.validation.append({"stage": "concepts", "label": name, **result.to_dict()}) continue + key = _name_key(name) + if not key or key in seen_concepts: + rejected += 1 + validation = _invalid_item("duplicate_or_empty_concept", result.evidence) + _reject_item(out, debug, "concepts", name, item, validation) + warn("concept dropped", f"duplicate_or_empty_concept for {name!r}") + continue + seen_concepts.add(key) out.concepts.append( ConceptExtract( name=name, @@ -297,58 +431,87 @@ def complete(stage: str, system: str, user: str) -> tuple[str, dict | list]: evidence=result.evidence, ) ) - out.stage_stats["concepts"] = _stage_stats(len(items), len(out.concepts), rejected) + out.stage_stats["concepts"] = _stage_stats( + len(items), + len(out.concepts), + rejected, + attempts=attempts, + ) except Exception as exc: # noqa: BLE001 warn("concept extraction failed", exc) out.stage_stats["concepts"] = _failed_stage(exc, client.api_key) - if debug: - debug.event( - "llm_stage_failed", - stage="concepts", - error=redact_secrets(str(exc), client.api_key), - ) + _record_stage_failure(debug, "concepts", exc, client.api_key) try: system, user = entities_messages( - markdown, sections, max_entities, summary=out.summary, concepts=out.concepts + markdown, + sections, + max_entities, + summary=out.summary, + concepts=out.concepts, ) - _, value = complete("entities", system, user) - items = _items(value, "entities")[:max_entities] + value, attempts = complete("entities", system, user) + items = value["entities"][:max_entities] rejected = 0 + seen_entities: set[str] = set() for item in items: name = str(item.get("name") or "").strip() result = resolve_evidence(_evidence(item), sections, markdown) - if debug: - debug.validation("entities", name, item, result.to_dict()) if not result.valid: rejected += 1 + _reject_item(out, debug, "entities", name, item, result.to_dict()) warn("entity dropped", f"{result.reason} for {name!r}") - out.validation.append({"stage": "entities", "label": name, **result.to_dict()}) continue + entity_type = _canonical_entity_type(item.get("type") or item.get("entity_type")) + if entity_type is None: + rejected += 1 + validation = _invalid_item( + "invalid_or_conceptual_entity_type", + result.evidence, + {"received_type": str(item.get("type") or item.get("entity_type") or "")}, + ) + _reject_item(out, debug, "entities", name, item, validation) + warn("entity dropped", f"invalid_or_conceptual_entity_type for {name!r}") + continue + key = _name_key(name) + if not key or key in seen_entities: + rejected += 1 + validation = _invalid_item("duplicate_or_empty_entity", result.evidence) + _reject_item(out, debug, "entities", name, item, validation) + warn("entity dropped", f"duplicate_or_empty_entity for {name!r}") + continue + seen_entities.add(key) aliases = item.get("aliases") if isinstance(item.get("aliases"), list) else [] out.entities.append( EntityExtract( name=name, - entity_type=str(item.get("type") or item.get("entity_type") or "entity").strip(), + entity_type=entity_type, description=str(item.get("description") or "").strip(), - aliases=[str(x) for x in aliases], + aliases=[str(alias) for alias in aliases], confidence=_float_or_none(item.get("confidence")), evidence=result.evidence, ) ) - out.stage_stats["entities"] = _stage_stats(len(items), len(out.entities), rejected) + out.stage_stats["entities"] = _stage_stats( + len(items), + len(out.entities), + rejected, + attempts=attempts, + ) + reclassified = _remove_entity_names_from_concepts(out) + if reclassified: + stats = out.stage_stats.get("concepts", {}) + stats["accepted"] = len(out.concepts) + stats["reclassified"] = reclassified except Exception as exc: # noqa: BLE001 warn("entity extraction failed", exc) out.stage_stats["entities"] = _failed_stage(exc, client.api_key) - if debug: - debug.event( - "llm_stage_failed", - stage="entities", - error=redact_secrets(str(exc), client.api_key), - ) + _record_stage_failure(debug, "entities", exc, client.api_key) - allowed = {c.name for c in out.concepts} | {e.name for e in out.entities} try: + nodes = relation_nodes(out.concepts, out.entities) + node_by_id = {node["node_id"]: node for node in nodes} + node_by_name = {_name_key(node["name"]): node for node in nodes} system, user = relations_messages( markdown, sections, @@ -356,72 +519,156 @@ def complete(stage: str, system: str, user: str) -> tuple[str, dict | list]: concepts=out.concepts, entities=out.entities, ) - _, value = complete("relations", system, user) - items = _items(value, "relations") + value, attempts = complete("relations", system, user) + items = value["relations"] rejected = 0 for item in items: - subject = str(item.get("subject") or "") - obj = str(item.get("object") or "") - label = f"{subject!r} -> {obj!r}" - if subject not in allowed or obj not in allowed: + subject_node, object_node = _relation_nodes(item, node_by_id, node_by_name) + subject_label = subject_node["name"] if subject_node else str(item.get("subject") or "") + object_label = object_node["name"] if object_node else str(item.get("object") or "") + label = f"{subject_label!r} -> {object_label!r}" + if subject_node is None or object_node is None: rejected += 1 - reason = "unknown_node" - warn("relation dropped", f"{reason}: {label}") - validation = { - "valid": False, - "reason": reason, - "evidence": evidence_to_dict(_evidence(item)), - "details": {}, - } - out.validation.append({"stage": "relations", "label": label, **validation}) - if debug: - debug.validation("relations", label, item, validation) + validation = _invalid_item("unknown_node", _evidence(item)) + _reject_item(out, debug, "relations", label, item, validation) + warn("relation dropped", f"unknown_node: {label}") continue result = resolve_evidence(_evidence(item), sections, markdown) - if debug: - debug.validation("relations", label, item, result.to_dict()) if not result.valid: rejected += 1 + _reject_item(out, debug, "relations", label, item, result.to_dict()) warn("relation dropped", f"{result.reason}: {label}") - out.validation.append({"stage": "relations", "label": label, **result.to_dict()}) continue out.relations.append( ProposedEdge( - subject=subject, + subject=subject_node["name"], relation=str(item.get("relation") or "related_to"), - object=obj, + object=object_node["name"], evidence=result.evidence, note=str(item.get("note") or ""), ) ) - out.stage_stats["relations"] = _stage_stats(len(items), len(out.relations), rejected) + out.stage_stats["relations"] = _stage_stats( + len(items), + len(out.relations), + rejected, + attempts=attempts, + ) except Exception as exc: # noqa: BLE001 warn("relation extraction failed", exc) out.stage_stats["relations"] = _failed_stage(exc, client.api_key) - if debug: - debug.event( - "llm_stage_failed", - stage="relations", - error=redact_secrets(str(exc), client.api_key), - ) + _record_stage_failure(debug, "relations", exc, client.api_key) return out -def _stage_stats(returned: int, accepted: int, rejected: int) -> dict: +def _canonical_entity_type(value) -> str | None: + raw = unicodedata.normalize("NFKC", str(value or "")).strip().casefold() + if not raw: + return None + if raw in _CANONICAL_ENTITY_TYPES: + return raw + if raw in _ENTITY_TYPE_ALIASES: + return _ENTITY_TYPE_ALIASES[raw] + if any(marker in raw for marker in _CONCEPT_TYPE_MARKERS): + return None + for alias, canonical in _ENTITY_TYPE_ALIASES.items(): + if alias and alias in raw: + return canonical + return None + + +def _remove_entity_names_from_concepts(out: Extracts) -> int: + entity_names = {_name_key(entity.name) for entity in out.entities} + before = len(out.concepts) + out.concepts = [concept for concept in out.concepts if _name_key(concept.name) not in entity_names] + return before - len(out.concepts) + + +def _relation_nodes(item: dict, node_by_id: dict, node_by_name: dict) -> tuple[dict | None, dict | None]: + subject_id = str(item.get("subject_id") or "").strip() + object_id = str(item.get("object_id") or "").strip() + if subject_id or object_id: + return node_by_id.get(subject_id), node_by_id.get(object_id) + subject = node_by_name.get(_name_key(str(item.get("subject") or ""))) + obj = node_by_name.get(_name_key(str(item.get("object") or ""))) + return subject, obj + + +def _reject_item( + out: Extracts, + debug: DebugRecorder | None, + stage: str, + label: str, + item: dict, + validation: dict, +) -> None: + out.validation.append({"stage": stage, "label": label, **validation}) + if debug: + debug.validation(stage, label, item, validation) + + +def _invalid_item(reason: str, evidence: Evidence | None, details: dict | None = None) -> dict: + return { + "valid": False, + "reason": reason, + "evidence": evidence_to_dict(evidence), + "details": details or {}, + } + + +def _record_stage_failure( + debug: DebugRecorder | None, + stage: str, + exc: Exception, + api_key: str | None, +) -> None: + if debug: + debug.event( + "llm_stage_failed", + stage=stage, + attempts=getattr(exc, "attempts", 1), + error=redact_secrets(str(exc), api_key), + ) + + +def _stage_stats( + returned: int, + accepted: int, + rejected: int, + *, + attempts: int, + reclassified: int = 0, +) -> dict: status = "degraded" if rejected else "ok" - return {"status": status, "returned": returned, "accepted": accepted, "rejected": rejected} + return { + "status": status, + "returned": returned, + "accepted": accepted, + "rejected": rejected, + "reclassified": reclassified, + "attempts": attempts, + "retries": attempts - 1, + } def _failed_stage(exc: Exception, api_key: str | None) -> dict: + attempts = int(getattr(exc, "attempts", 1)) return { "status": "failed", "returned": 0, "accepted": 0, "rejected": 0, + "attempts": attempts, + "retries": max(attempts - 1, 0), "error": redact_secrets(str(exc), api_key), } +def _name_key(value: str) -> str: + normalized = unicodedata.normalize("NFKC", str(value)).casefold() + return re.sub(r"\s+", "", normalized) + + def _float_or_none(value) -> float | None: try: return float(value) if value is not None else None diff --git a/src/okf_compiler/prompts.py b/src/okf_compiler/prompts.py index 47affe7..5b7c2ec 100644 --- a/src/okf_compiler/prompts.py +++ b/src/okf_compiler/prompts.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import re +import unicodedata from .schema import ConceptExtract, EntityExtract, SectionSpec @@ -10,22 +12,21 @@ def _section_map(sections: list[SectionSpec]) -> str: rows = [ { - "section_id": s.section_id, - "heading_path": s.heading_path, - "line_start": s.line_start, - "line_end": s.line_end, + "section_id": section.section_id, + "heading_path": section.heading_path, + "line_start": section.line_start, + "line_end": section.line_end, } - for s in sections + for section in sections ] return json.dumps(rows, ensure_ascii=False, indent=2) def _evidence_contract() -> str: return ( - 'Evidence MUST be an object exactly shaped as ' + 'Evidence MUST be exactly shaped as ' '{"section_id":"s0001","quote":"verbatim contiguous text copied from Document"}. ' - "The quote must occur once inside that section, contain no ellipsis or paraphrase, and must not " - "include invented line numbers. The compiler resolves quote text to absolute source lines." + "Copy the quote exactly. Do not paraphrase, add ellipses, change punctuation, or invent lines." ) @@ -39,11 +40,18 @@ def summary_messages(markdown: str, sections: list[SectionSpec], language: str) def concepts_messages( - markdown: str, sections: list[SectionSpec], max_concepts: int, *, summary: str + markdown: str, + sections: list[SectionSpec], + max_concepts: int, + *, + summary: str, ) -> tuple[str, str]: system = ( - "Extract document-local concepts. Return JSON only with a concepts array. Each item needs " - "name, description, confidence, and evidence. " + _evidence_contract() + "Extract reusable document-local concepts, mechanisms, strategies, design patterns, and " + "business ideas. Do NOT return named people, organizations, products, games, platforms, " + "brands, works, places, or events; those belong in entities. Return JSON only with a " + "concepts array. Each item needs name, description, confidence, and evidence. " + + _evidence_contract() ) user = ( f"Maximum concepts: {max_concepts}\nSummary: {summary}\n" @@ -61,17 +69,61 @@ def entities_messages( concepts: list[ConceptExtract], ) -> tuple[str, str]: system = ( - "Extract document-local named entities. Return JSON only with an entities array. Each item " - "needs name, type, description, aliases, confidence, and evidence. " + _evidence_contract() + "Extract only concrete named entities. Allowed type values are person, organization, " + "product, platform, brand, work, location, event, and other_named_entity. Do NOT return " + "abstract concepts, mechanisms, strategies, gameplay patterns, business models, hooks, or " + "design labels. Return JSON only with an entities array. Each item needs name, type, " + "description, aliases, confidence, and evidence. " + + _evidence_contract() ) - concept_names = [c.name for c in concepts] + concept_names = [concept.name for concept in concepts] user = ( - f"Maximum entities: {max_entities}\nSummary: {summary}\nConcepts: {concept_names}\n" + f"Maximum entities: {max_entities}\nSummary: {summary}\n" + f"Known concepts, normally do not repeat as entities: {concept_names}\n" f"Section map:\n{_section_map(sections)}\n\nDocument:\n{markdown}" ) return system, user +def relation_nodes( + concepts: list[ConceptExtract], + entities: list[EntityExtract], +) -> list[dict]: + rows: list[dict] = [] + seen: set[str] = set() + concept_index = 1 + entity_index = 1 + for concept in concepts: + key = _name_key(concept.name) + if not key or key in seen: + continue + seen.add(key) + rows.append( + { + "node_id": f"concept:c{concept_index:04d}", + "name": concept.name, + "kind": "concept", + "type": "concept", + } + ) + concept_index += 1 + for entity in entities: + key = _name_key(entity.name) + if not key or key in seen: + continue + seen.add(key) + rows.append( + { + "node_id": f"entity:e{entity_index:04d}", + "name": entity.name, + "kind": "entity", + "type": entity.entity_type, + } + ) + entity_index += 1 + return rows + + def relations_messages( markdown: str, sections: list[SectionSpec], @@ -81,13 +133,37 @@ def relations_messages( entities: list[EntityExtract], ) -> tuple[str, str]: system = ( - "Propose relations between the supplied local concepts/entities only. Return JSON only with " - "a relations array. Each relation needs subject, relation, object, note, and evidence. " + "Propose evidence-backed relations between the supplied typed nodes only. Return JSON only " + "with a relations array. Each relation needs subject_id, relation, object_id, note, and " + "evidence. subject_id and object_id MUST be copied from Allowed nodes. " + _evidence_contract() ) - nodes = [c.name for c in concepts] + [e.name for e in entities] + nodes = relation_nodes(concepts, entities) user = ( - f"Summary: {summary}\nAllowed nodes: {nodes}\n" + f"Summary: {summary}\nAllowed nodes:\n" + f"{json.dumps(nodes, ensure_ascii=False, indent=2)}\n" f"Section map:\n{_section_map(sections)}\n\nDocument:\n{markdown}" ) return system, user + + +def retry_system_message(stage: str, system: str, error: str) -> str: + schemas = { + "summary": '{"summary":"string"}', + "concepts": '{"concepts":[{"name":"...","description":"...","confidence":0.9,' + '"evidence":{"section_id":"s0001","quote":"..."}}]}', + "entities": '{"entities":[{"name":"...","type":"product","description":"...",' + '"aliases":[],"confidence":0.9,"evidence":{"section_id":"s0001","quote":"..."}}]}', + "relations": '{"relations":[{"subject_id":"concept:c0001","relation":"uses",' + '"object_id":"entity:e0001","note":"...",' + '"evidence":{"section_id":"s0001","quote":"..."}}]}', + } + return ( + f"{system}\n\nYour previous response was invalid for stage {stage}: {error}. " + f"Return one JSON object matching this top-level schema exactly: {schemas[stage]}" + ) + + +def _name_key(value: str) -> str: + normalized = unicodedata.normalize("NFKC", str(value)).casefold() + return re.sub(r"\s+", "", normalized) diff --git a/src/okf_compiler/schema.py b/src/okf_compiler/schema.py index ba1c6c4..3026859 100644 --- a/src/okf_compiler/schema.py +++ b/src/okf_compiler/schema.py @@ -5,6 +5,7 @@ import json import os import re +import unicodedata from dataclasses import dataclass, field, replace from importlib.metadata import PackageNotFoundError from importlib.metadata import version as package_version @@ -38,6 +39,9 @@ SECTION_INDEX_WIDTH = 2 SECTION_TYPE = "Section" +_DOUBLE_QUOTES = frozenset('“”„‟"〝〞〟「」『』') +_SINGLE_QUOTES = frozenset("‘’‚‛'") + @dataclass class SectionAnchor: @@ -292,39 +296,46 @@ def _select_section( def _locate_quote(section: SectionSpec, markdown: str, quote: str) -> EvidenceValidation: lines = markdown.splitlines(keepends=True) section_text = "".join(lines[max(section.line_start - 1, 0) : min(section.line_end, len(lines))]) + frontmatter_end = _frontmatter_end(section, section_text) exact_positions = _all_positions(section_text, quote) - if len(exact_positions) == 1: - start = exact_positions[0] - return _resolved_quote(section, section_text, quote, start, start + len(quote), "ok_exact_quote") - if len(exact_positions) > 1: - return EvidenceValidation( - False, - "ambiguous_quote", - replace(section_evidence(section), quote=quote), - {"matches": len(exact_positions), "match_mode": "exact"}, - ) + exact_choice = _choose_position(exact_positions, lambda position: position, frontmatter_end) + if exact_choice is not None: + start, body_preferred = exact_choice + reason = "ok_exact_quote_body_preferred" if body_preferred else "ok_exact_quote" + return _resolved_quote(section, section_text, start, start + len(quote), reason) + if exact_positions: + return _ambiguous_quote(section, quote, exact_positions, "exact") normalized_text, mapping = _normalize_with_map(section_text) - normalized_quote = re.sub(r"\s+", " ", quote).strip() - normalized_positions = _all_positions(normalized_text, normalized_quote) if normalized_quote else [] - if len(normalized_positions) == 1: - norm_start = normalized_positions[0] + normalized_quote, _ = _normalize_with_map(quote) + normalized_positions = _all_positions(normalized_text, normalized_quote) + normalized_choice = _choose_position( + normalized_positions, + lambda position: mapping[position], + frontmatter_end, + ) + if normalized_choice is not None: + norm_start, body_preferred = normalized_choice norm_end = norm_start + len(normalized_quote) - 1 + reason = ( + "ok_normalized_quote_body_preferred" + if body_preferred + else "ok_normalized_quote" + ) return _resolved_quote( section, section_text, - quote, mapping[norm_start], mapping[norm_end] + 1, - "ok_normalized_quote", + reason, ) - if len(normalized_positions) > 1: - return EvidenceValidation( - False, - "ambiguous_quote", - replace(section_evidence(section), quote=quote), - {"matches": len(normalized_positions), "match_mode": "normalized_whitespace"}, + if normalized_positions: + return _ambiguous_quote( + section, + quote, + normalized_positions, + "normalized_typography", ) return EvidenceValidation( False, @@ -334,14 +345,52 @@ def _locate_quote(section: SectionSpec, markdown: str, quote: str) -> EvidenceVa ) +def _ambiguous_quote( + section: SectionSpec, + quote: str, + positions: list[int], + match_mode: str, +) -> EvidenceValidation: + return EvidenceValidation( + False, + "ambiguous_quote", + replace(section_evidence(section), quote=quote), + {"matches": len(positions), "match_mode": match_mode}, + ) + + +def _choose_position( + positions: list[int], + original_position, + frontmatter_end: int, +) -> tuple[int, bool] | None: + if len(positions) == 1: + return positions[0], False + if frontmatter_end and positions: + body_positions = [p for p in positions if original_position(p) >= frontmatter_end] + if len(body_positions) == 1: + return body_positions[0], True + return None + + +def _frontmatter_end(section: SectionSpec, section_text: str) -> int: + if section.line_start != 1 or not section_text.startswith("---"): + return 0 + match = re.search(r"\n---[ \t]*(?:\r?\n|$)", section_text[3:]) + return 3 + match.end() if match else 0 + + def _resolved_quote( section: SectionSpec, section_text: str, - quote: str, start: int, end: int, reason: str, ) -> EvidenceValidation: + while start < end and section_text[start].isspace(): + start += 1 + while end > start and section_text[end - 1].isspace(): + end -= 1 last_char = max(start, end - 1) line_start = section.line_start + section_text[:start].count("\n") line_end = section.line_start + section_text[:last_char].count("\n") @@ -350,7 +399,7 @@ def _resolved_quote( heading_path=section.heading_path, line_start=line_start, line_end=max(line_start, line_end), - quote=quote, + quote=section_text[start:end], ) return EvidenceValidation(True, reason, resolved) @@ -382,21 +431,31 @@ def _normalize_with_map(text: str) -> tuple[str, list[int]]: mapping: list[int] = [] in_space = False for index, char in enumerate(text): - if char.isspace(): - if chars and not in_space: - chars.append(" ") - mapping.append(index) - in_space = True - continue - chars.append(char) - mapping.append(index) - in_space = False + expanded = _normalize_char(char) + for normalized in expanded: + if normalized.isspace(): + if chars and not in_space: + chars.append(" ") + mapping.append(index) + in_space = True + continue + chars.append(normalized) + mapping.append(index) + in_space = False while chars and chars[-1] == " ": chars.pop() mapping.pop() return "".join(chars), mapping +def _normalize_char(char: str) -> str: + if char in _DOUBLE_QUOTES: + return '"' + if char in _SINGLE_QUOTES: + return "'" + return unicodedata.normalize("NFKC", char) + + def okf_yaml_text(title: str) -> str: return ( f"format: {OKF_FORMAT}\n" diff --git a/tests/test_round2_robustness.py b/tests/test_round2_robustness.py new file mode 100644 index 0000000..dffdcb7 --- /dev/null +++ b/tests/test_round2_robustness.py @@ -0,0 +1,228 @@ +import json +from pathlib import Path + +from okf_compiler.diagnostics import DebugRecorder +from okf_compiler.llm import extract +from okf_compiler.markdown import split_sections +from okf_compiler.prompts import relation_nodes +from okf_compiler.schema import ( + ConceptExtract, + EntityExtract, + Evidence, + SectionSpec, + resolve_evidence, +) + + +class FakeClient: + api_key = None + + def __init__(self, responses): + self.responses = iter(responses) + self.calls = [] + + def json_completion(self, system, user): + self.calls.append((system, user)) + value = next(self.responses) + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + + +def _article_sections(): + markdown = ( + "# T\n\n## Body\n" + "《赵云与阿斗》的做法是把广告放在“最合理的救局时刻”。\n" + "短局设计增加广告展示。\n\n## End\n结束。\n" + ) + return markdown, split_sections(markdown).sections + + +def _concept_item(name="短局设计"): + return { + "name": name, + "description": "概念", + "confidence": 0.9, + "evidence": {"section_id": "s0001", "quote": "短局设计增加广告展示。"}, + } + + +def _entity_item(name="赵云与阿斗", entity_type="游戏产品"): + return { + "name": name, + "type": entity_type, + "description": "游戏", + "aliases": [], + "confidence": 0.9, + "evidence": { + "section_id": "s0001", + "quote": "《赵云与阿斗》的做法是把广告放在“最合理的救局时刻”。", + }, + } + + +def test_typography_normalization_preserves_original_quote(): + markdown, sections = _article_sections() + result = resolve_evidence( + Evidence( + section_id="s0001", + quote='《赵云与阿斗》的做法是把广告放在"最合理的救局时刻"。', + ), + sections, + markdown, + ) + assert result.valid + assert result.reason == "ok_normalized_quote" + assert "“最合理的救局时刻”" in result.evidence.quote + assert '"最合理的救局时刻"' not in result.evidence.quote + + +def test_frontmatter_duplicate_prefers_visible_body_match(): + title = "重复标题" + markdown = f'---\ntitle: "{title}"\n---\n\n# {title}\n正文\n' + section = SectionSpec( + index=0, + title="preamble", + heading_path="preamble", + line_start=1, + line_end=6, + body=markdown, + section_id="s0001", + boundary_kind="preamble", + ) + result = resolve_evidence(Evidence(section_id="s0001", quote=title), [section], markdown) + assert result.valid + assert result.reason == "ok_exact_quote_body_preferred" + assert result.evidence.line_start == 5 + + +def test_malformed_relation_response_retries_once_and_recovers(tmp_path: Path): + markdown, sections = _article_sections() + client = FakeClient( + [ + {"summary": "摘要"}, + {"concepts": [_concept_item()]}, + {"entities": [_entity_item()]}, + {}, + { + "relations": [ + { + "subject_id": "entity:e0001", + "relation": "uses", + "object_id": "concept:c0001", + "note": "", + "evidence": { + "section_id": "s0001", + "quote": "短局设计增加广告展示。", + }, + } + ] + }, + ] + ) + debug = DebugRecorder(tmp_path, include_llm_payloads=True) + result = extract( + client, + markdown, + sections, + language="zh", + max_concepts=12, + max_entities=12, + debug=debug, + ) + assert len(result.relations) == 1 + assert result.stage_stats["relations"]["attempts"] == 2 + assert result.stage_stats["relations"]["retries"] == 1 + assert (tmp_path / "stages/relations/attempt-2/request.json").is_file() + assert "previous response was invalid" in client.calls[-1][0] + + +def test_two_malformed_relation_responses_fail_stage(): + markdown, sections = _article_sections() + client = FakeClient( + [ + {"summary": "摘要"}, + {"concepts": [_concept_item()]}, + {"entities": [_entity_item()]}, + {}, + {"relation": []}, + ] + ) + result = extract( + client, + markdown, + sections, + language="zh", + max_concepts=12, + max_entities=12, + ) + assert result.stage_stats["relations"]["status"] == "failed" + assert result.stage_stats["relations"]["attempts"] == 2 + assert result.degraded + + +def test_entity_classification_reconciles_concept_overlap(): + markdown, sections = _article_sections() + concept_named_entity = { + "name": "赵云与阿斗", + "description": "误分概念", + "confidence": 0.8, + "evidence": { + "section_id": "s0001", + "quote": "《赵云与阿斗》的做法是把广告放在“最合理的救局时刻”。", + }, + } + conceptual_entity = { + "name": "短局设计", + "type": "玩法机制", + "description": "不是实体", + "aliases": [], + "confidence": 0.9, + "evidence": {"section_id": "s0001", "quote": "短局设计增加广告展示。"}, + } + client = FakeClient( + [ + {"summary": "摘要"}, + {"concepts": [concept_named_entity, _concept_item()]}, + {"entities": [_entity_item(), conceptual_entity]}, + {"relations": []}, + ] + ) + result = extract( + client, + markdown, + sections, + language="zh", + max_concepts=12, + max_entities=12, + ) + assert [concept.name for concept in result.concepts] == ["短局设计"] + assert [entity.name for entity in result.entities] == ["赵云与阿斗"] + assert result.stage_stats["concepts"]["reclassified"] == 1 + assert result.stage_stats["entities"]["rejected"] == 1 + assert any(item["reason"] == "invalid_or_conceptual_entity_type" for item in result.validation) + + +def test_relation_nodes_are_typed_unique_and_deterministic(): + nodes = relation_nodes( + [ + ConceptExtract("短局设计", ""), + ConceptExtract("短局设计", "重复"), + ], + [ + EntityExtract("赵云与阿斗", "product", ""), + EntityExtract("短局设计", "product", "冲突"), + ], + ) + assert nodes == [ + { + "node_id": "concept:c0001", + "name": "短局设计", + "kind": "concept", + "type": "concept", + }, + { + "node_id": "entity:e0001", + "name": "赵云与阿斗", + "kind": "entity", + "type": "product", + }, + ]