From 207a25b8f2e5068078967da65146ab2637626b7f Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:38:15 -0400 Subject: [PATCH 01/32] Add IdeaOS portfolio domain and revisioned hydration seam --- docs/idea-portfolio.md | 202 +++++++++ domains/idea-portfolio/spec.yaml | 311 ++++++++++++++ engine/sync/idea_portfolio.py | 679 ++++++++++++++++++++++++++++++ tests/unit/test_idea_portfolio.py | 250 +++++++++++ tools/hydrate_idea_portfolio.py | 85 ++++ 5 files changed, 1527 insertions(+) create mode 100644 docs/idea-portfolio.md create mode 100644 domains/idea-portfolio/spec.yaml create mode 100644 engine/sync/idea_portfolio.py create mode 100644 tests/unit/test_idea_portfolio.py create mode 100755 tools/hydrate_idea_portfolio.py diff --git a/docs/idea-portfolio.md b/docs/idea-portfolio.md new file mode 100644 index 00000000..496af70f --- /dev/null +++ b/docs/idea-portfolio.md @@ -0,0 +1,202 @@ +# IdeaOS Idea Portfolio Domain + +## Authority boundary + +`idea-portfolio` is the CEG-owned graph intelligence surface for the first-class IdeaOS corpus. + +- **IdeaOS owns:** idea identity, source lineage, lifecycle state, Dream / Invariant / Wedge / Proof, decisions, proof state, execution state, and semantic `IdeaGraphProjection` emission. +- **CEG owns:** graph persistence of that projection, cross-idea traversal, overlap/leverage matching, portfolio ranking, graph-derived relationships, communities, and learned graph signals. +- **Transport adapters own neither:** they may serialize, route, and translate only. + +CEG must not parse raw files from IdeaOS `Ideas/`, infer an idea from a filename, or become a second lifecycle source of truth. + +## Graph model + +The first implementation intentionally uses two durable node classes: + +```text +Idea --PRODUCES----> PortfolioFacet + --REQUIRES----> PortfolioFacet + --TARGETS-----> PortfolioFacet + --USES--------> PortfolioFacet + --DEPENDS_ON--> PortfolioFacet +``` + +`PortfolioFacet` is globally content-addressed by the exact `(kind, key)` pair. The source assertion's epistemic state, source references, projection digest, and graph revision live on the relationship, because those facts belong to the assertion between an Idea and a facet, not to the shared facet itself. + +No inferred Idea-to-Idea edge is persisted by hydration. That keeps source projection separate from graph-derived intelligence. + +## Why there is a dedicated hydration compiler + +The generic `SyncGenerator` can MERGE nodes and fixed child/taxonomy edges, but it cannot express all of the required semantics for IdeaGraphProjection hydration: + +1. the relationship type varies by assertion relation; +2. evidence/provenance belongs on each relationship; +3. replacing a projection must remove relationships that no longer exist; +4. corpus mutations need replayable revision-chain semantics; +5. deletions need an explicit tombstone rather than silent absence. + +Those are current, evidenced responsibilities. `engine/sync/idea_portfolio.py` therefore owns the smallest domain-specific persistence compiler needed to bridge that expressiveness gap. It still uses the shared `GraphDriver`; it does not introduce a graph client, queue, database, or second scoring engine. + +## Hydration contract + +CEG accepts `ceg.idea-portfolio-hydration/v1`: + +```json +{ + "schema": "ceg.idea-portfolio-hydration/v1", + "source_snapshot_ref": "Quantum-L9/IdeaOS@", + "source_snapshot_digest": "sha256:<64 hex>", + "expected_graph_revision": null, + "records": [ + { + "schema": "ceg.idea-portfolio-sync-record/v1", + "operation": "upsert", + "projection": { + "schema": "ideaos.idea-graph-projection/v1", + "idea_id": "example", + "source_refs": ["Ideas/..."], + "source_digest": "sha256:<64 hex>", + "lifecycle": { + "stage": "expanded", + "decision": null, + "proof_state": null, + "execution_state": null + }, + "assertions": [], + "unknowns": [] + } + } + ] +} +``` + +A tombstone record is explicit: + +```json +{ + "schema": "ceg.idea-portfolio-sync-record/v1", + "operation": "tombstone", + "idea_id": "example" +} +``` + +The envelope contains at most one record per `idea_id`. + +## Revision chain + +Every successful hydration computes: + +```text +batch_digest = sha256(canonical(records)) +new_graph_revision = sha256( + protocol_version + + expected_graph_revision_or_GENESIS + + source_snapshot_digest + + batch_digest +) +``` + +The current committed revision is stored in `IdeaPortfolioHydrationState`. A batch can start only when its `expected_graph_revision` equals the committed revision. An exact interrupted revision can resume; an exact committed revision is an idempotent replay. + +This gives the graph a replayable parent-linked mutation chain without inventing a timestamp-based ordering authority. + +Hydration state remains `in_progress` if a write fails. A future context-read seam must refuse to label an in-progress graph as a committed snapshot. Retrying the exact same envelope resumes the interrupted revision. + +## Epistemic rules + +Hydration is stricter than filename ingestion: + +- a hydrated projection must have at least one root `source_ref`; +- non-`UNKNOWN` assertions must have at least one assertion source reference; +- duplicate semantic assertions are rejected; +- relation/kind combinations are validated; +- keys are Unicode-NFC normalized and trimmed only, not lower-cased, synonym-expanded, or guessed; +- source relationships remain source relationships; +- graph-derived Idea-to-Idea relationships are not written by hydration. + +The v1 kind/relation matrix is: + +| Kind | Allowed relations | +| --- | --- | +| capability | produces, requires, uses | +| substrate | produces, requires, uses | +| proof_asset | produces, requires, uses | +| data_asset | produces, requires, uses | +| market | targets | +| customer_type | targets | +| dependency | depends_on | + +## Portfolio matching + +`build_portfolio_match_query()` compiles an IdeaGraphProjection into the flat query contract consumed by the current CEG match handler. It never calculates rank itself. + +The domain ranks candidate Ideas using six graph-native dimensions: + +1. candidate produces something the query idea requires; +2. candidate requires something the query idea produces; +3. shared `USES` facets; +4. shared `TARGETS` facets; +5. query idea explicitly depends on the candidate; +6. candidate explicitly depends on the query idea. + +Weights sum to 1.0. The candidate itself is excluded, and tombstoned/inactive ideas fail the admission gate. + +The result is **portfolio ranking evidence**, not IdeaOS lifecycle authorization. + +## Centrality and learned graph signals + +They deliberately do not exist in v0.1. The current CEG scheduler supports Louvain, co-occurrence, reinforcement, temporal recency, geo proximity, equipment sync, feedback recalculation, and causal chain scoring. It does not currently execute PageRank even though older documentation mentions it. + +Declaring a PageRank job now would advertise a capability the runtime skips. Centrality is therefore deferred until an executable current owner path exists. + +## Initial activation + +1. Validate the domain: + + ```bash + python tools/validate_domain.py domains/idea-portfolio/spec.yaml --strict + ``` + +2. Ensure the `idea-portfolio` Neo4j database exists according to the deployment topology. +3. Initialize schema constraints through the existing admin `init_schema` path for domain `idea-portfolio`. +4. Produce a hydration envelope from source-bound IdeaOS projections. +5. Dry-run it first: + + ```bash + python tools/hydrate_idea_portfolio.py hydration.json + ``` + +6. Apply deliberately: + + ```bash + python tools/hydrate_idea_portfolio.py hydration.json --apply + ``` + +7. Persist the hydration receipt and its `graph_revision`. Use that revision as the `expected_graph_revision` parent of the next corpus delta. + +## Corpus rule + +Raw historical corpus hydration remains an IdeaOS semantic task: + +```text +Ideas/ artifact + -> deterministic source identity + -> IdeaOS semantic extraction / expansion + -> IdeaGraphProjection + -> CEG hydration envelope + -> CEG idea-portfolio graph +``` + +A raw ZIP, Markdown filename, or directory name is never sufficient evidence for semantic Idea identity or graph assertions. + +## Failure boundaries + +- **Bad projection:** reject before graph mutation. +- **Wrong parent revision:** reject the hydration revision. +- **Different revision already in progress:** reject rather than interleave two writers. +- **Mid-run failure:** keep `in_progress=true`; exact replay resumes. +- **Tombstone:** mark Idea inactive and remove its source-projection edges; do not delete shared facet nodes. +- **Orphan facets:** tolerated as non-authoritative derived residue. Garbage collection is deferred rather than mixed into the critical write transaction. + +The v0.1 hydration transaction is atomic per graph write, while a multi-record hydration revision consists of multiple Neo4j write transactions guarded by the hydration state. The committed graph revision advances only after every record succeeds. Consumers must treat `in_progress=true` as an uncommitted snapshot boundary. A future live context adapter must enforce that read gate. diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml new file mode 100644 index 00000000..cb1d1f1e --- /dev/null +++ b/domains/idea-portfolio/spec.yaml @@ -0,0 +1,311 @@ +# --- L9_META --- +# l9_schema: 1 +# origin: domain-specific +# engine: graph +# layer: [config] +# tags: [domains, ideaos, portfolio, matching] +# owner: domain-team +# status: candidate +# --- /L9_META --- +--- +domain: + id: idea-portfolio + name: IdeaOS Portfolio Graph + description: > + Cross-idea portfolio intelligence for IdeaOS. IdeaOS remains authoritative + for idea identity and lifecycle truth. CEG owns graph intersections, + leverage matching, portfolio ranking, and graph-derived signals. + version: 0.1.0 + +ontology: + nodes: + - label: Idea + managedby: sync + candidate: true + properties: + - {name: idea_id, type: string, required: true} + - {name: source_digest, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + - {name: lifecycle_stage, type: string} + - {name: decision, type: enum, values: [GO, CONDITIONAL_GO, HOLD, NO_GO]} + - {name: proof_state, type: string} + - {name: execution_state, type: string} + - {name: unknowns_json, type: string} + - {name: self_dependency_facet_id, type: string} + - {name: active, type: bool} + - {name: hydrated_at, type: datetime} + - {name: tombstoned_at, type: datetime} + + - label: IdeaQuery + managedby: api + queryentity: true + properties: + - {name: idea_id, type: string} + + - label: PortfolioFacet + managedby: sync + auxiliary: true + properties: + - {name: facet_id, type: string, required: true} + - {name: kind, type: enum, values: [capability, substrate, proof_asset, data_asset, market, customer_type, dependency]} + - {name: key, type: string, required: true} + - {name: last_seen_revision, type: string} + + edges: + - type: PRODUCES + from: Idea + to: PortfolioFacet + direction: DIRECTED + category: capability + managedby: sync + properties: + - {name: assertion_id, type: string, required: true} + - {name: kind, type: string} + - {name: evidence_state, type: string} + - {name: source_refs_json, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + + - type: REQUIRES + from: Idea + to: PortfolioFacet + direction: DIRECTED + category: capability + managedby: sync + properties: + - {name: assertion_id, type: string, required: true} + - {name: kind, type: string} + - {name: evidence_state, type: string} + - {name: source_refs_json, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + + - type: TARGETS + from: Idea + to: PortfolioFacet + direction: DIRECTED + category: market + managedby: sync + properties: + - {name: assertion_id, type: string, required: true} + - {name: kind, type: string} + - {name: evidence_state, type: string} + - {name: source_refs_json, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + + - type: USES + from: Idea + to: PortfolioFacet + direction: DIRECTED + category: capability + managedby: sync + properties: + - {name: assertion_id, type: string, required: true} + - {name: kind, type: string} + - {name: evidence_state, type: string} + - {name: source_refs_json, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + + - type: DEPENDS_ON + from: Idea + to: PortfolioFacet + direction: DIRECTED + category: context + managedby: sync + properties: + - {name: assertion_id, type: string, required: true} + - {name: kind, type: string} + - {name: evidence_state, type: string} + - {name: source_refs_json, type: string} + - {name: projection_digest, type: string} + - {name: graph_revision, type: string} + +matchentities: + candidate: + - {label: Idea, matchdirection: portfolio_context_for_idea} + queryentity: + - {label: IdeaQuery, matchdirection: portfolio_context_for_idea} + +queryschema: + matchdirections: [portfolio_context_for_idea] + fields: + - {name: idea_id, type: string, required: true} + - {name: requires_facets, type: string, default: ""} + - {name: requires_count, type: int, default: 0} + - {name: produces_facets, type: string, default: ""} + - {name: produces_count, type: int, default: 0} + - {name: uses_facets, type: string, default: ""} + - {name: uses_count, type: int, default: 0} + - {name: targets_facets, type: string, default: ""} + - {name: targets_count, type: int, default: 0} + - {name: depends_on_facets, type: string, default: ""} + - {name: self_dependency_facet_id, type: string, default: ""} + +traversal: + steps: [] + +gates: + - name: active_only + type: boolean + candidateprop: active + nullbehavior: fail + matchdirections: [portfolio_context_for_idea] + + - name: exclude_self + type: threshold + candidateprop: idea_id + queryparam: idea_id + operator: "!=" + nullbehavior: fail + matchdirections: [portfolio_context_for_idea] + +scoring: + dimensions: + - name: incoming_requirement_fit + source: computed + computation: customcypher + expression: > + CASE WHEN $requires_count <= 0 THEN 0.0 ELSE + toFloat(size([(candidate)-[:PRODUCES]->(f:PortfolioFacet) + WHERE $requires_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat($requires_count) END + weightkey: wincoming + defaultweight: 0.25 + matchdirections: [portfolio_context_for_idea] + + - name: outgoing_requirement_fit + source: computed + computation: customcypher + expression: > + CASE WHEN $produces_count <= 0 THEN 0.0 ELSE + toFloat(size([(candidate)-[:REQUIRES]->(f:PortfolioFacet) + WHERE $produces_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat($produces_count) END + weightkey: woutgoing + defaultweight: 0.25 + matchdirections: [portfolio_context_for_idea] + + - name: shared_usage + source: computed + computation: customcypher + expression: > + CASE WHEN $uses_count <= 0 THEN 0.0 ELSE + toFloat(size([(candidate)-[:USES]->(f:PortfolioFacet) + WHERE $uses_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat($uses_count) END + weightkey: wusage + defaultweight: 0.20 + matchdirections: [portfolio_context_for_idea] + + - name: shared_target + source: computed + computation: customcypher + expression: > + CASE WHEN $targets_count <= 0 THEN 0.0 ELSE + toFloat(size([(candidate)-[:TARGETS]->(f:PortfolioFacet) + WHERE $targets_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat($targets_count) END + weightkey: wtarget + defaultweight: 0.15 + matchdirections: [portfolio_context_for_idea] + + - name: query_depends_on_candidate + source: computed + computation: customcypher + expression: > + CASE WHEN candidate.self_dependency_facet_id IS NOT NULL AND + $depends_on_facets CONTAINS ('|' + candidate.self_dependency_facet_id + '|') + THEN 1.0 ELSE 0.0 END + weightkey: wquerydependency + defaultweight: 0.10 + matchdirections: [portfolio_context_for_idea] + + - name: candidate_depends_on_query + source: computed + computation: customcypher + expression: > + CASE WHEN $self_dependency_facet_id = '' THEN 0.0 ELSE + CASE WHEN size([(candidate)-[:DEPENDS_ON]->(f:PortfolioFacet) + WHERE f.facet_id = $self_dependency_facet_id | f]) > 0 + THEN 1.0 ELSE 0.0 END END + weightkey: wcandidatedependency + defaultweight: 0.05 + matchdirections: [portfolio_context_for_idea] + +sync: + endpoints: + - path: /sync/idea_projection + targetnode: Idea + idproperty: idea_id + batchstrategy: unwindmerge + +# Centrality is intentionally not declared yet. The current CEG scheduler does +# not implement PageRank, and declaring a skipped algorithm would create false +# capability. Community and learned-weight jobs can be added only when a proof +# requirement justifies them. +gdsjobs: [] + +compliance: + enabled: true + audit: + enabled: true + logallmatches: true + logretentiondays: 365 + pii: + enabled: false + prohibitedfactors: [] + regionalrules: [] + counterfactualaudit: false + +capabilities: + - name: portfolio_read + actions: [match:read] + allowed_subjects: ["*"] + - name: portfolio_write + actions: [sync:write] + allowed_subjects: ["*"] + +feedbackloop: + enabled: false + +causal: + enabled: false + +counterfactual: + enabled: false + +semantic_registry: + enabled: false + +decision_arbitration: + enabled: false + +feature_catalog: + features: + - feature_id: incoming_requirement_fit + owner: ceg + provenance_required: true + evidence_required: true + - feature_id: outgoing_requirement_fit + owner: ceg + provenance_required: true + evidence_required: true + - feature_id: shared_usage + owner: ceg + provenance_required: true + evidence_required: true + - feature_id: shared_target + owner: ceg + provenance_required: true + evidence_required: true + - feature_id: query_depends_on_candidate + owner: ceg + provenance_required: true + evidence_required: true + - feature_id: candidate_depends_on_query + owner: ceg + provenance_required: true + evidence_required: true diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py new file mode 100644 index 00000000..1f6d699b --- /dev/null +++ b/engine/sync/idea_portfolio.py @@ -0,0 +1,679 @@ +"""IdeaOS portfolio projection hydration for the CEG idea-portfolio domain. + +IdeaOS owns canonical idea identity and lifecycle truth. This module accepts the +narrow ``IdeaGraphProjection`` contract, validates it, compiles graph-safe +facets, and applies a revision-chained hydration batch to CEG. It never parses +raw IdeaOS corpus files and never infers idea identity from filenames. + +Hydration is an owner-native CEG write path. The public Constellation transport +binding can call this service later without moving semantic ownership into an +adapter. +""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +DOMAIN_ID = "idea-portfolio" +_STATE_ID = "canonical" +_RELATION_TYPES = ("PRODUCES", "REQUIRES", "TARGETS", "USES", "DEPENDS_ON") + + +class IdeaPortfolioHydrationError(ValueError): + """Raised when a hydration batch cannot be safely admitted or applied.""" + + +class EvidenceState(StrEnum): + VERIFIED = "VERIFIED" + SUPPORTED_INFERENCE = "SUPPORTED_INFERENCE" + HYPOTHESIS = "HYPOTHESIS" + UNKNOWN = "UNKNOWN" + + +class AssertionKind(StrEnum): + CAPABILITY = "capability" + SUBSTRATE = "substrate" + PROOF_ASSET = "proof_asset" + DATA_ASSET = "data_asset" + MARKET = "market" + CUSTOMER_TYPE = "customer_type" + DEPENDENCY = "dependency" + + +class AssertionRelation(StrEnum): + PRODUCES = "produces" + REQUIRES = "requires" + TARGETS = "targets" + USES = "uses" + DEPENDS_ON = "depends_on" + + +_ALLOWED_RELATIONS: dict[AssertionKind, frozenset[AssertionRelation]] = { + AssertionKind.CAPABILITY: frozenset( + {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} + ), + AssertionKind.SUBSTRATE: frozenset( + {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} + ), + AssertionKind.PROOF_ASSET: frozenset( + {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} + ), + AssertionKind.DATA_ASSET: frozenset( + {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} + ), + AssertionKind.MARKET: frozenset({AssertionRelation.TARGETS}), + AssertionKind.CUSTOMER_TYPE: frozenset({AssertionRelation.TARGETS}), + AssertionKind.DEPENDENCY: frozenset({AssertionRelation.DEPENDS_ON}), +} + +_EDGE_BY_RELATION: dict[AssertionRelation, str] = { + AssertionRelation.PRODUCES: "PRODUCES", + AssertionRelation.REQUIRES: "REQUIRES", + AssertionRelation.TARGETS: "TARGETS", + AssertionRelation.USES: "USES", + AssertionRelation.DEPENDS_ON: "DEPENDS_ON", +} + + +class IdeaLifecycle(BaseModel): + model_config = ConfigDict(extra="forbid") + + stage: str = Field(min_length=1) + decision: Literal["GO", "CONDITIONAL_GO", "HOLD", "NO_GO"] | None = None + proof_state: str | None = None + execution_state: str | None = None + + +class IdeaAssertion(BaseModel): + model_config = ConfigDict(extra="forbid") + + kind: AssertionKind + relation: AssertionRelation + key: str = Field(min_length=1) + evidence_state: EvidenceState + source_refs: list[str] + + @model_validator(mode="after") + def validate_semantics(self) -> "IdeaAssertion": + if self.relation not in _ALLOWED_RELATIONS[self.kind]: + raise ValueError( + f"relation {self.relation.value!r} is not valid for assertion kind {self.kind.value!r}" + ) + if len(self.source_refs) != len(set(self.source_refs)): + raise ValueError("assertion source_refs must be unique") + if self.evidence_state != EvidenceState.UNKNOWN and not self.source_refs: + raise ValueError("non-UNKNOWN assertions require at least one source_ref") + return self + + +class IdeaGraphProjection(BaseModel): + """Mirror of IdeaOS ``ideaos.idea-graph-projection/v1`` at the CEG boundary.""" + + model_config = ConfigDict(extra="forbid") + + schema: Literal["ideaos.idea-graph-projection/v1"] + idea_id: str = Field(min_length=1) + source_refs: list[str] + source_digest: str = Field(pattern=DIGEST_PATTERN) + lifecycle: IdeaLifecycle + assertions: list[IdeaAssertion] + unknowns: list[str] + + @model_validator(mode="after") + def validate_projection(self) -> "IdeaGraphProjection": + if not self.source_refs: + raise ValueError("hydrated IdeaGraphProjection requires at least one source_ref") + if len(self.source_refs) != len(set(self.source_refs)): + raise ValueError("projection source_refs must be unique") + if len(self.unknowns) != len(set(self.unknowns)): + raise ValueError("projection unknowns must be unique") + + semantic_keys = [(a.kind.value, a.relation.value, _canonical_key(a.key)) for a in self.assertions] + if len(semantic_keys) != len(set(semantic_keys)): + raise ValueError("projection contains duplicate semantic assertions") + return self + + +class IdeaPortfolioSyncRecord(BaseModel): + """CEG persistence command around an IdeaOS projection.""" + + model_config = ConfigDict(extra="forbid") + + schema: Literal["ceg.idea-portfolio-sync-record/v1"] + operation: Literal["upsert", "tombstone"] + projection: IdeaGraphProjection | None = None + idea_id: str | None = None + + @model_validator(mode="after") + def validate_operation(self) -> "IdeaPortfolioSyncRecord": + if self.operation == "upsert": + if self.projection is None: + raise ValueError("upsert sync record requires projection") + if self.idea_id is not None and self.idea_id != self.projection.idea_id: + raise ValueError("sync record idea_id does not match projection idea_id") + else: + if self.idea_id is None: + raise ValueError("tombstone sync record requires idea_id") + if self.projection is not None: + raise ValueError("tombstone sync record must not contain projection") + return self + + @property + def resolved_idea_id(self) -> str: + if self.projection is not None: + return self.projection.idea_id + assert self.idea_id is not None + return self.idea_id + + +class IdeaPortfolioHydrationEnvelope(BaseModel): + """One ordered corpus delta chained to the previously committed graph revision.""" + + model_config = ConfigDict(extra="forbid") + + schema: Literal["ceg.idea-portfolio-hydration/v1"] + source_snapshot_ref: str = Field(min_length=1) + source_snapshot_digest: str = Field(pattern=DIGEST_PATTERN) + expected_graph_revision: str | None = Field(default=None, pattern=DIGEST_PATTERN) + records: list[IdeaPortfolioSyncRecord] = Field(min_length=1) + + @model_validator(mode="after") + def validate_records(self) -> "IdeaPortfolioHydrationEnvelope": + idea_ids = [record.resolved_idea_id for record in self.records] + if len(idea_ids) != len(set(idea_ids)): + raise ValueError("hydration envelope may contain at most one record per idea_id") + return self + + +class GraphWriter(Protocol): + async def execute_write( + self, + transaction_function: Any = None, + *args: Any, + cypher: str | None = None, + parameters: dict[str, Any] | None = None, + database: str | None = None, + **kwargs: Any, + ) -> dict[str, Any] | Any: ... + + +@dataclass(frozen=True) +class CompiledAssertion: + assertion_id: str + facet_id: str + kind: str + key: str + relation: str + evidence_state: str + source_refs_json: str + + +@dataclass(frozen=True) +class HydrationPlan: + envelope: IdeaPortfolioHydrationEnvelope + batch_digest: str + graph_revision: str + + +@dataclass(frozen=True) +class WriteCommand: + cypher: str + parameters: dict[str, Any] + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _sha256_text(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _canonical_key(value: str) -> str: + """Preserve source semantics while removing encoding/edge whitespace noise.""" + + return unicodedata.normalize("NFC", value).strip() + + +def _facet_id(kind: AssertionKind | str, key: str) -> str: + kind_value = kind.value if isinstance(kind, AssertionKind) else kind + digest = hashlib.sha256(f"{kind_value}\x00{_canonical_key(key)}".encode()).hexdigest() + return f"facet:{digest}" + + +def _assertion_id(idea_id: str, assertion: IdeaAssertion) -> str: + semantic_key = ( + f"{idea_id}\x00{assertion.relation.value}\x00{assertion.kind.value}" + f"\x00{_canonical_key(assertion.key)}" + ) + return "assertion:" + hashlib.sha256(semantic_key.encode()).hexdigest() + + +def projection_digest(projection: IdeaGraphProjection) -> str: + return _sha256_text(_canonical_json(projection.model_dump(mode="json"))) + + +def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertion]: + compiled: list[CompiledAssertion] = [] + for assertion in projection.assertions: + compiled.append( + CompiledAssertion( + assertion_id=_assertion_id(projection.idea_id, assertion), + facet_id=_facet_id(assertion.kind, assertion.key), + kind=assertion.kind.value, + key=_canonical_key(assertion.key), + relation=assertion.relation.value, + evidence_state=assertion.evidence_state.value, + source_refs_json=_canonical_json(sorted(assertion.source_refs)), + ) + ) + return compiled + + +def build_portfolio_match_query(projection: IdeaGraphProjection | dict[str, Any]) -> dict[str, Any]: + """Compile one IdeaOS projection into the flat query shape declared by the domain pack.""" + + model = projection if isinstance(projection, IdeaGraphProjection) else IdeaGraphProjection.model_validate(projection) + compiled = compile_assertions(model) + + by_relation: dict[str, list[str]] = {relation.value: [] for relation in AssertionRelation} + for assertion in compiled: + by_relation[assertion.relation].append(assertion.facet_id) + + def encoded(relation: str) -> str: + ids = sorted(set(by_relation[relation])) + return "" if not ids else "|" + "|".join(ids) + "|" + + return { + "idea_id": model.idea_id, + "requires_facets": encoded(AssertionRelation.REQUIRES.value), + "requires_count": len(set(by_relation[AssertionRelation.REQUIRES.value])), + "produces_facets": encoded(AssertionRelation.PRODUCES.value), + "produces_count": len(set(by_relation[AssertionRelation.PRODUCES.value])), + "uses_facets": encoded(AssertionRelation.USES.value), + "uses_count": len(set(by_relation[AssertionRelation.USES.value])), + "targets_facets": encoded(AssertionRelation.TARGETS.value), + "targets_count": len(set(by_relation[AssertionRelation.TARGETS.value])), + "depends_on_facets": encoded(AssertionRelation.DEPENDS_ON.value), + "self_dependency_facet_id": _facet_id(AssertionKind.DEPENDENCY, model.idea_id), + } + + +def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> HydrationPlan: + model = ( + envelope + if isinstance(envelope, IdeaPortfolioHydrationEnvelope) + else IdeaPortfolioHydrationEnvelope.model_validate(envelope) + ) + records_payload = [record.model_dump(mode="json") for record in model.records] + batch_digest = _sha256_text(_canonical_json(records_payload)) + parent = model.expected_graph_revision or "GENESIS" + graph_revision = _sha256_text( + f"ceg.idea-portfolio-graph/v1\x00{parent}\x00{model.source_snapshot_digest}\x00{batch_digest}" + ) + return HydrationPlan(envelope=model, batch_digest=batch_digest, graph_revision=graph_revision) + + +def compile_upsert_command( + projection: IdeaGraphProjection, + *, + graph_revision: str, + tenant: str, +) -> WriteCommand: + p_digest = projection_digest(projection) + grouped: dict[str, list[dict[str, Any]]] = {relation.value: [] for relation in AssertionRelation} + for assertion in compile_assertions(projection): + grouped[assertion.relation].append( + { + "assertion_id": assertion.assertion_id, + "facet_id": assertion.facet_id, + "kind": assertion.kind, + "key": assertion.key, + "evidence_state": assertion.evidence_state, + "source_refs_json": assertion.source_refs_json, + } + ) + + cypher = """ +MERGE (idea:Idea {idea_id: $idea_id}) +SET idea.source_digest = $source_digest, + idea.projection_digest = $projection_digest, + idea.graph_revision = $graph_revision, + idea.lifecycle_stage = $lifecycle_stage, + idea.decision = $decision, + idea.proof_state = $proof_state, + idea.execution_state = $execution_state, + idea.unknowns_json = $unknowns_json, + idea.self_dependency_facet_id = $self_dependency_facet_id, + idea.active = true, + idea.hydrated_at = datetime(), + idea.tombstoned_at = null, + idea._tenant = $tenant +WITH idea +OPTIONAL MATCH (idea)-[old:PRODUCES|REQUIRES|TARGETS|USES|DEPENDS_ON]->(:PortfolioFacet) +DELETE old +WITH DISTINCT idea +CALL { + WITH idea + UNWIND $produces AS row + MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) + SET facet.kind = row.kind, + facet.key = row.key, + facet.last_seen_revision = $graph_revision, + facet._tenant = $tenant + MERGE (idea)-[rel:PRODUCES]->(facet) + SET rel.assertion_id = row.assertion_id, + rel.kind = row.kind, + rel.evidence_state = row.evidence_state, + rel.source_refs_json = row.source_refs_json, + rel.projection_digest = $projection_digest, + rel.graph_revision = $graph_revision + RETURN count(row) AS produced_count +} +CALL { + WITH idea + UNWIND $requires AS row + MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) + SET facet.kind = row.kind, + facet.key = row.key, + facet.last_seen_revision = $graph_revision, + facet._tenant = $tenant + MERGE (idea)-[rel:REQUIRES]->(facet) + SET rel.assertion_id = row.assertion_id, + rel.kind = row.kind, + rel.evidence_state = row.evidence_state, + rel.source_refs_json = row.source_refs_json, + rel.projection_digest = $projection_digest, + rel.graph_revision = $graph_revision + RETURN count(row) AS required_count +} +CALL { + WITH idea + UNWIND $targets AS row + MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) + SET facet.kind = row.kind, + facet.key = row.key, + facet.last_seen_revision = $graph_revision, + facet._tenant = $tenant + MERGE (idea)-[rel:TARGETS]->(facet) + SET rel.assertion_id = row.assertion_id, + rel.kind = row.kind, + rel.evidence_state = row.evidence_state, + rel.source_refs_json = row.source_refs_json, + rel.projection_digest = $projection_digest, + rel.graph_revision = $graph_revision + RETURN count(row) AS target_count +} +CALL { + WITH idea + UNWIND $uses AS row + MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) + SET facet.kind = row.kind, + facet.key = row.key, + facet.last_seen_revision = $graph_revision, + facet._tenant = $tenant + MERGE (idea)-[rel:USES]->(facet) + SET rel.assertion_id = row.assertion_id, + rel.kind = row.kind, + rel.evidence_state = row.evidence_state, + rel.source_refs_json = row.source_refs_json, + rel.projection_digest = $projection_digest, + rel.graph_revision = $graph_revision + RETURN count(row) AS use_count +} +CALL { + WITH idea + UNWIND $depends_on AS row + MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) + SET facet.kind = row.kind, + facet.key = row.key, + facet.last_seen_revision = $graph_revision, + facet._tenant = $tenant + MERGE (idea)-[rel:DEPENDS_ON]->(facet) + SET rel.assertion_id = row.assertion_id, + rel.kind = row.kind, + rel.evidence_state = row.evidence_state, + rel.source_refs_json = row.source_refs_json, + rel.projection_digest = $projection_digest, + rel.graph_revision = $graph_revision + RETURN count(row) AS dependency_count +} +RETURN idea.idea_id AS idea_id, + produced_count + required_count + target_count + use_count + dependency_count AS assertion_count +""".strip() + + return WriteCommand( + cypher=cypher, + parameters={ + "tenant": tenant, + "idea_id": projection.idea_id, + "source_digest": projection.source_digest, + "projection_digest": p_digest, + "graph_revision": graph_revision, + "lifecycle_stage": projection.lifecycle.stage, + "decision": projection.lifecycle.decision, + "proof_state": projection.lifecycle.proof_state, + "execution_state": projection.lifecycle.execution_state, + "unknowns_json": _canonical_json(sorted(projection.unknowns)), + "self_dependency_facet_id": _facet_id(AssertionKind.DEPENDENCY, projection.idea_id), + "produces": grouped[AssertionRelation.PRODUCES.value], + "requires": grouped[AssertionRelation.REQUIRES.value], + "targets": grouped[AssertionRelation.TARGETS.value], + "uses": grouped[AssertionRelation.USES.value], + "depends_on": grouped[AssertionRelation.DEPENDS_ON.value], + }, + ) + + +def compile_tombstone_command(idea_id: str, *, graph_revision: str, tenant: str) -> WriteCommand: + cypher = """ +MERGE (idea:Idea {idea_id: $idea_id}) +SET idea.active = false, + idea.graph_revision = $graph_revision, + idea.tombstoned_at = datetime(), + idea._tenant = $tenant +WITH idea +OPTIONAL MATCH (idea)-[old:PRODUCES|REQUIRES|TARGETS|USES|DEPENDS_ON]->(:PortfolioFacet) +DELETE old +RETURN idea.idea_id AS idea_id +""".strip() + return WriteCommand( + cypher=cypher, + parameters={"tenant": tenant, "idea_id": idea_id, "graph_revision": graph_revision}, + ) + + +_ACQUIRE_HYDRATION_CYPHER = """ +MERGE (state:IdeaPortfolioHydrationState {state_id: $state_id}) +SET state._cas_lock = coalesce(state._cas_lock, 0) + 1 +WITH state, + state.current_revision AS current_revision, + coalesce(state.in_progress, false) AS in_progress, + state.target_revision AS target_revision +WHERE (current_revision = $graph_revision AND in_progress = false) + OR (in_progress = true AND target_revision = $graph_revision) + OR ( + in_progress = false + AND ( + (current_revision IS NULL AND $expected_graph_revision IS NULL) + OR current_revision = $expected_graph_revision + ) + ) +WITH state, current_revision, in_progress, target_revision, + CASE + WHEN current_revision = $graph_revision AND in_progress = false THEN 'reused' + WHEN in_progress = true AND target_revision = $graph_revision THEN 'resume' + ELSE 'start' + END AS acquisition_status +FOREACH (_ IN CASE WHEN acquisition_status = 'start' THEN [1] ELSE [] END | + SET state.in_progress = true, + state.target_revision = $graph_revision, + state.source_snapshot_ref = $source_snapshot_ref, + state.source_snapshot_digest = $source_snapshot_digest, + state.batch_digest = $batch_digest, + state.started_at = datetime(), + state.completed_at = null, + state.last_error = null, + state._tenant = $tenant +) +RETURN acquisition_status AS status, current_revision +""".strip() + +_FINALIZE_HYDRATION_CYPHER = """ +MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) +SET state._cas_lock = coalesce(state._cas_lock, 0) + 1 +WITH state +WHERE state.in_progress = true AND state.target_revision = $graph_revision +SET state.current_revision = $graph_revision, + state.in_progress = false, + state.target_revision = null, + state.completed_at = datetime(), + state.last_error = null +RETURN state.current_revision AS graph_revision +""".strip() + +_FAIL_HYDRATION_CYPHER = """ +MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) +WHERE state.in_progress = true AND state.target_revision = $graph_revision +SET state.last_error = $error, + state.last_error_at = datetime() +RETURN state.target_revision AS graph_revision +""".strip() + + +class IdeaPortfolioHydrator: + """Apply one revision-chained IdeaOS corpus delta to the idea-portfolio graph.""" + + def __init__(self, graph_writer: GraphWriter, *, tenant: str = DOMAIN_ID) -> None: + self.graph_writer = graph_writer + self.tenant = tenant + + async def apply(self, envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> dict[str, Any]: + plan = compile_hydration_plan(envelope) + acquired = await self.graph_writer.execute_write( + cypher=_ACQUIRE_HYDRATION_CYPHER, + parameters={ + "state_id": _STATE_ID, + "tenant": self.tenant, + "graph_revision": plan.graph_revision, + "expected_graph_revision": plan.envelope.expected_graph_revision, + "source_snapshot_ref": plan.envelope.source_snapshot_ref, + "source_snapshot_digest": plan.envelope.source_snapshot_digest, + "batch_digest": plan.batch_digest, + }, + database=DOMAIN_ID, + ) + records = _write_records(acquired) + if not records: + raise IdeaPortfolioHydrationError( + "hydration revision conflict or another revision is already in progress" + ) + + acquisition_status = records[0].get("status") + if acquisition_status == "reused": + return { + "schema": "ceg.idea-portfolio-hydration-receipt/v1", + "status": "reused", + "graph_revision": plan.graph_revision, + "batch_digest": plan.batch_digest, + "source_snapshot_ref": plan.envelope.source_snapshot_ref, + "source_snapshot_digest": plan.envelope.source_snapshot_digest, + "applied": [], + "tombstoned": [], + } + + applied: list[str] = [] + tombstoned: list[str] = [] + try: + for record in plan.envelope.records: + if record.operation == "upsert": + assert record.projection is not None + command = compile_upsert_command( + record.projection, + graph_revision=plan.graph_revision, + tenant=self.tenant, + ) + await self.graph_writer.execute_write( + cypher=command.cypher, + parameters=command.parameters, + database=DOMAIN_ID, + ) + applied.append(record.projection.idea_id) + else: + command = compile_tombstone_command( + record.resolved_idea_id, + graph_revision=plan.graph_revision, + tenant=self.tenant, + ) + await self.graph_writer.execute_write( + cypher=command.cypher, + parameters=command.parameters, + database=DOMAIN_ID, + ) + tombstoned.append(record.resolved_idea_id) + except Exception as exc: + await self.graph_writer.execute_write( + cypher=_FAIL_HYDRATION_CYPHER, + parameters={ + "state_id": _STATE_ID, + "graph_revision": plan.graph_revision, + "error": type(exc).__name__, + }, + database=DOMAIN_ID, + ) + raise + + finalized = await self.graph_writer.execute_write( + cypher=_FINALIZE_HYDRATION_CYPHER, + parameters={"state_id": _STATE_ID, "graph_revision": plan.graph_revision}, + database=DOMAIN_ID, + ) + if not _write_records(finalized): + raise IdeaPortfolioHydrationError("hydration applied but graph revision finalization failed") + + return { + "schema": "ceg.idea-portfolio-hydration-receipt/v1", + "status": "applied" if acquisition_status == "start" else "resumed", + "graph_revision": plan.graph_revision, + "batch_digest": plan.batch_digest, + "source_snapshot_ref": plan.envelope.source_snapshot_ref, + "source_snapshot_digest": plan.envelope.source_snapshot_digest, + "applied": applied, + "tombstoned": tombstoned, + } + + +def _write_records(result: Any) -> list[dict[str, Any]]: + if isinstance(result, dict): + records = result.get("records", []) + return [dict(record) for record in records] + return [] + + +__all__ = [ + "AssertionKind", + "AssertionRelation", + "DOMAIN_ID", + "EvidenceState", + "HydrationPlan", + "IdeaGraphProjection", + "IdeaPortfolioHydrationEnvelope", + "IdeaPortfolioHydrationError", + "IdeaPortfolioHydrator", + "IdeaPortfolioSyncRecord", + "WriteCommand", + "build_portfolio_match_query", + "compile_assertions", + "compile_hydration_plan", + "compile_tombstone_command", + "compile_upsert_command", + "projection_digest", +] diff --git a/tests/unit/test_idea_portfolio.py b/tests/unit/test_idea_portfolio.py new file mode 100644 index 00000000..78152630 --- /dev/null +++ b/tests/unit/test_idea_portfolio.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from engine.config.loader import DomainPackLoader +from engine.gates.compiler import GateCompiler +from engine.scoring.assembler import ScoringAssembler +from engine.sync.idea_portfolio import ( + IdeaGraphProjection, + IdeaPortfolioHydrationError, + IdeaPortfolioHydrator, + build_portfolio_match_query, + compile_hydration_plan, + compile_upsert_command, + projection_digest, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def _digest(char: str = "a") -> str: + return "sha256:" + char * 64 + + +def _projection(idea_id: str = "idea-alpha") -> dict[str, Any]: + return { + "schema": "ideaos.idea-graph-projection/v1", + "idea_id": idea_id, + "source_refs": [f"Ideas/{idea_id}.md"], + "source_digest": _digest("a"), + "lifecycle": { + "stage": "expanded", + "decision": None, + "proof_state": "P1", + "execution_state": None, + }, + "assertions": [ + { + "kind": "capability", + "relation": "produces", + "key": "shared-capability", + "evidence_state": "VERIFIED", + "source_refs": [f"Ideas/{idea_id}.md#capability"], + }, + { + "kind": "capability", + "relation": "requires", + "key": "required-capability", + "evidence_state": "SUPPORTED_INFERENCE", + "source_refs": [f"Ideas/{idea_id}.md#requirement"], + }, + { + "kind": "substrate", + "relation": "uses", + "key": "shared-substrate", + "evidence_state": "VERIFIED", + "source_refs": [f"Ideas/{idea_id}.md#substrate"], + }, + { + "kind": "market", + "relation": "targets", + "key": "industrial-ai", + "evidence_state": "HYPOTHESIS", + "source_refs": [f"Ideas/{idea_id}.md#market"], + }, + { + "kind": "dependency", + "relation": "depends_on", + "key": "idea-foundation", + "evidence_state": "VERIFIED", + "source_refs": [f"Ideas/{idea_id}.md#dependency"], + }, + ], + "unknowns": ["external demand not yet proven"], + } + + +def _envelope(*, expected: str | None = None) -> dict[str, Any]: + return { + "schema": "ceg.idea-portfolio-hydration/v1", + "source_snapshot_ref": "Quantum-L9/IdeaOS@deadbeef", + "source_snapshot_digest": _digest("b"), + "expected_graph_revision": expected, + "records": [ + { + "schema": "ceg.idea-portfolio-sync-record/v1", + "operation": "upsert", + "projection": _projection(), + } + ], + } + + +@pytest.mark.unit +class TestIdeaPortfolioDomain: + def test_domain_loads_and_compilers_accept_it(self) -> None: + loader = DomainPackLoader(config_path=str(ROOT / "domains")) + spec = loader.load_domain("idea-portfolio") + + assert spec.domain.id == "idea-portfolio" + assert {node.label for node in spec.ontology.nodes} == {"Idea", "IdeaQuery", "PortfolioFacet"} + assert {edge.type for edge in spec.ontology.edges} == { + "PRODUCES", + "REQUIRES", + "TARGETS", + "USES", + "DEPENDS_ON", + } + assert sum(d.defaultweight for d in spec.scoring.dimensions) == pytest.approx(1.0) + + gate_clause = GateCompiler(spec).compile_all_gates("portfolio_context_for_idea") + assert "candidate.active" in gate_clause + assert "candidate.idea_id != $idea_id" in gate_clause + + score_clause, _ = ScoringAssembler(spec).assemble_scoring_clause( + "portfolio_context_for_idea", {} + ) + assert "PRODUCES" in score_clause + assert "REQUIRES" in score_clause + assert "PortfolioFacet" in score_clause + + +@pytest.mark.unit +class TestProjectionBoundary: + def test_projection_compiles_to_flat_match_query(self) -> None: + projection = IdeaGraphProjection.model_validate(_projection()) + query = build_portfolio_match_query(projection) + + assert query["idea_id"] == "idea-alpha" + assert query["requires_count"] == 1 + assert query["produces_count"] == 1 + assert query["uses_count"] == 1 + assert query["targets_count"] == 1 + assert query["depends_on_facets"].startswith("|facet:") + assert query["self_dependency_facet_id"].startswith("facet:") + + def test_invalid_kind_relation_is_rejected(self) -> None: + raw = _projection() + raw["assertions"][0]["kind"] = "market" + raw["assertions"][0]["relation"] = "produces" + + with pytest.raises(ValueError, match="not valid for assertion kind"): + IdeaGraphProjection.model_validate(raw) + + def test_non_unknown_assertion_requires_source_reference(self) -> None: + raw = _projection() + raw["assertions"][0]["source_refs"] = [] + + with pytest.raises(ValueError, match="require at least one source_ref"): + IdeaGraphProjection.model_validate(raw) + + def test_duplicate_semantic_assertion_is_rejected(self) -> None: + raw = _projection() + raw["assertions"].append(dict(raw["assertions"][0])) + + with pytest.raises(ValueError, match="duplicate semantic assertions"): + IdeaGraphProjection.model_validate(raw) + + def test_upsert_replaces_source_projection_edges(self) -> None: + projection = IdeaGraphProjection.model_validate(_projection()) + command = compile_upsert_command(projection, graph_revision=_digest("c"), tenant="idea-portfolio") + + assert "DELETE old" in command.cypher + assert "MERGE (idea)-[rel:PRODUCES]->(facet)" in command.cypher + assert "rel.evidence_state" in command.cypher + assert command.parameters["projection_digest"] == projection_digest(projection) + assert command.parameters["unknowns_json"] == '["external demand not yet proven"]' + + +@pytest.mark.unit +class TestHydrationRevision: + def test_revision_is_deterministic_and_parent_linked(self) -> None: + first = compile_hydration_plan(_envelope()) + again = compile_hydration_plan(_envelope()) + child = compile_hydration_plan(_envelope(expected=first.graph_revision)) + + assert first.batch_digest == again.batch_digest + assert first.graph_revision == again.graph_revision + assert child.graph_revision != first.graph_revision + + +class _FakeWriter: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = list(responses) + self.calls: list[dict[str, Any]] = [] + + async def execute_write( + self, + transaction_function: Any = None, + *args: Any, + cypher: str | None = None, + parameters: dict[str, Any] | None = None, + database: str | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + self.calls.append({"cypher": cypher, "parameters": parameters, "database": database}) + if not self.responses: + return {"records": []} + return self.responses.pop(0) + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_hydrator_applies_then_finalizes_revision() -> None: + writer = _FakeWriter( + [ + {"records": [{"status": "start", "current_revision": None}]}, + {"records": [{"idea_id": "idea-alpha", "assertion_count": 5}]}, + {"records": [{"graph_revision": _digest("d")}]}, + ] + ) + hydrator = IdeaPortfolioHydrator(writer) + + receipt = await hydrator.apply(_envelope()) + + assert receipt["status"] == "applied" + assert receipt["applied"] == ["idea-alpha"] + assert len(writer.calls) == 3 + assert all(call["database"] == "idea-portfolio" for call in writer.calls) + assert "IdeaPortfolioHydrationState" in str(writer.calls[0]["cypher"]) + assert "MERGE (idea:Idea" in str(writer.calls[1]["cypher"]) + assert "state.current_revision" in str(writer.calls[2]["cypher"]) + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_hydrator_exact_revision_replay_is_noop() -> None: + writer = _FakeWriter([{"records": [{"status": "reused", "current_revision": _digest("e")}] }]) + hydrator = IdeaPortfolioHydrator(writer) + + receipt = await hydrator.apply(_envelope()) + + assert receipt["status"] == "reused" + assert receipt["applied"] == [] + assert len(writer.calls) == 1 + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_hydrator_rejects_revision_conflict_before_projection_write() -> None: + writer = _FakeWriter([{"records": []}]) + hydrator = IdeaPortfolioHydrator(writer) + + with pytest.raises(IdeaPortfolioHydrationError, match="revision conflict"): + await hydrator.apply(_envelope()) + + assert len(writer.calls) == 1 diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py new file mode 100755 index 00000000..10dd9990 --- /dev/null +++ b/tools/hydrate_idea_portfolio.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Validate or apply an IdeaOS portfolio hydration envelope to CEG. + +Dry-run is the default. ``--apply`` is required for graph mutation. + +Examples: + python tools/hydrate_idea_portfolio.py hydration.json + python tools/hydrate_idea_portfolio.py hydration.json --apply +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from engine.config.loader import DomainPackLoader +from engine.graph.driver import GraphDriver +from engine.sync.idea_portfolio import ( + DOMAIN_ID, + IdeaPortfolioHydrationEnvelope, + IdeaPortfolioHydrator, + compile_hydration_plan, +) + + +def _load_envelope(path: Path) -> IdeaPortfolioHydrationEnvelope: + raw = json.loads(path.read_text(encoding="utf-8")) + return IdeaPortfolioHydrationEnvelope.model_validate(raw) + + +def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: + plan = compile_hydration_plan(envelope) + return { + "schema": "ceg.idea-portfolio-hydration-plan/v1", + "status": "validated", + "domain": DOMAIN_ID, + "source_snapshot_ref": envelope.source_snapshot_ref, + "source_snapshot_digest": envelope.source_snapshot_digest, + "expected_graph_revision": envelope.expected_graph_revision, + "batch_digest": plan.batch_digest, + "graph_revision": plan.graph_revision, + "records": [ + {"idea_id": record.resolved_idea_id, "operation": record.operation} + for record in envelope.records + ], + } + + +async def _apply(envelope: IdeaPortfolioHydrationEnvelope, *, tenant: str) -> dict[str, object]: + # Loading through the production loader proves the folder-shaped domain pack + # is discoverable and validates against the current DomainSpec before writes. + DomainPackLoader().load_domain(DOMAIN_ID) + + driver = GraphDriver() + await driver.connect() + try: + hydrator = IdeaPortfolioHydrator(driver, tenant=tenant) + return await hydrator.apply(envelope) + finally: + await driver.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate or apply an IdeaOS -> CEG portfolio hydration envelope") + parser.add_argument("envelope", type=Path, help="Path to ceg.idea-portfolio-hydration/v1 JSON") + parser.add_argument("--apply", action="store_true", help="Mutate the CEG idea-portfolio graph") + parser.add_argument( + "--tenant", + default=DOMAIN_ID, + help="Tenant provenance value stored on graph nodes (default: idea-portfolio)", + ) + args = parser.parse_args() + + envelope = _load_envelope(args.envelope) + result = asyncio.run(_apply(envelope, tenant=args.tenant)) if args.apply else _dry_run(envelope) + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 89c5c25e9c7eaf98883581cbfcca9f99ed0c5960 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:44:11 -0400 Subject: [PATCH 02/32] Make portfolio hydration atomic and fail closed --- docs/idea-portfolio.md | 68 ++++++--- domains/idea-portfolio/spec.yaml | 23 +-- engine/sync/idea_portfolio.py | 232 +++++++++++------------------- tests/unit/test_idea_portfolio.py | 87 +++++++---- tools/hydrate_idea_portfolio.py | 12 +- 5 files changed, 206 insertions(+), 216 deletions(-) diff --git a/docs/idea-portfolio.md b/docs/idea-portfolio.md index 496af70f..53583247 100644 --- a/docs/idea-portfolio.md +++ b/docs/idea-portfolio.md @@ -12,7 +12,7 @@ CEG must not parse raw files from IdeaOS `Ideas/`, infer an idea from a filename ## Graph model -The first implementation intentionally uses two durable node classes: +The first implementation intentionally uses two semantic node classes plus one internal revision-state node: ```text Idea --PRODUCES----> PortfolioFacet @@ -20,6 +20,9 @@ Idea --PRODUCES----> PortfolioFacet --TARGETS-----> PortfolioFacet --USES--------> PortfolioFacet --DEPENDS_ON--> PortfolioFacet + +IdeaPortfolioHydrationState + current_revision -> parent-linked corpus mutation chain ``` `PortfolioFacet` is globally content-addressed by the exact `(kind, key)` pair. The source assertion's epistemic state, source references, projection digest, and graph revision live on the relationship, because those facts belong to the assertion between an Idea and a facet, not to the shared facet itself. @@ -28,16 +31,18 @@ No inferred Idea-to-Idea edge is persisted by hydration. That keeps source proje ## Why there is a dedicated hydration compiler -The generic `SyncGenerator` can MERGE nodes and fixed child/taxonomy edges, but it cannot express all of the required semantics for IdeaGraphProjection hydration: +The generic `SyncGenerator` can MERGE nodes and fixed child/taxonomy edges, but it cannot express all required IdeaGraphProjection semantics: -1. the relationship type varies by assertion relation; +1. relationship type varies by assertion relation; 2. evidence/provenance belongs on each relationship; -3. replacing a projection must remove relationships that no longer exist; -4. corpus mutations need replayable revision-chain semantics; -5. deletions need an explicit tombstone rather than silent absence. +3. replacing a projection must remove relationships no longer present; +4. corpus writes need parent-linked revision semantics; +5. deletions need explicit tombstones rather than silent absence. Those are current, evidenced responsibilities. `engine/sync/idea_portfolio.py` therefore owns the smallest domain-specific persistence compiler needed to bridge that expressiveness gap. It still uses the shared `GraphDriver`; it does not introduce a graph client, queue, database, or second scoring engine. +The live generic `handle_sync` path is deliberately **not** advertised in the v0.1 domain pack because its current `SyncGenerator` cannot preserve this contract. Pretending otherwise would create a false operational path. Initial corpus hydration is an explicit CEG admin operation through `tools/hydrate_idea_portfolio.py`. A later Gate-facing binding should call the same hydrator rather than reimplement it. + ## Hydration contract CEG accepts `ceg.idea-portfolio-hydration/v1`: @@ -83,9 +88,9 @@ A tombstone record is explicit: The envelope contains at most one record per `idea_id`. -## Revision chain +## Atomic revision chain -Every successful hydration computes: +Every hydration computes: ```text batch_digest = sha256(canonical(records)) @@ -97,11 +102,25 @@ new_graph_revision = sha256( ) ``` -The current committed revision is stored in `IdeaPortfolioHydrationState`. A batch can start only when its `expected_graph_revision` equals the committed revision. An exact interrupted revision can resume; an exact committed revision is an idempotent replay. +The hydrator executes the entire envelope inside one managed Neo4j write transaction: + +```text +lock IdeaPortfolioHydrationState + -> compare current_revision with expected_graph_revision + -> apply every upsert/tombstone + -> advance current_revision + -> commit once +``` + +Consequences: -This gives the graph a replayable parent-linked mutation chain without inventing a timestamp-based ordering authority. +- a wrong parent revision fails closed before graph mutation commits; +- a mid-envelope exception rolls back the whole hydration revision; +- readers never see a committed half-new portfolio snapshot; +- an exact retry after a lost response is idempotent because `current_revision == new_graph_revision` returns `reused`; +- no timestamp is promoted into ordering authority. -Hydration state remains `in_progress` if a write fails. A future context-read seam must refuse to label an in-progress graph as a committed snapshot. Retrying the exact same envelope resumes the interrupted revision. +`Idea`, `PortfolioFacet`, and `IdeaPortfolioHydrationState` all have required ID properties in the domain ontology so the existing `init_schema` path creates uniqueness constraints needed by the write transaction. ## Epistemic rules @@ -158,7 +177,7 @@ Declaring a PageRank job now would advertise a capability the runtime skips. Cen python tools/validate_domain.py domains/idea-portfolio/spec.yaml --strict ``` -2. Ensure the `idea-portfolio` Neo4j database exists according to the deployment topology. +2. Ensure the `idea-portfolio` Neo4j database exists according to deployment topology. 3. Initialize schema constraints through the existing admin `init_schema` path for domain `idea-portfolio`. 4. Produce a hydration envelope from source-bound IdeaOS projections. 5. Dry-run it first: @@ -185,18 +204,29 @@ Ideas/ artifact -> IdeaOS semantic extraction / expansion -> IdeaGraphProjection -> CEG hydration envelope - -> CEG idea-portfolio graph + -> atomic CEG idea-portfolio graph revision ``` A raw ZIP, Markdown filename, or directory name is never sufficient evidence for semantic Idea identity or graph assertions. ## Failure boundaries -- **Bad projection:** reject before graph mutation. -- **Wrong parent revision:** reject the hydration revision. -- **Different revision already in progress:** reject rather than interleave two writers. -- **Mid-run failure:** keep `in_progress=true`; exact replay resumes. -- **Tombstone:** mark Idea inactive and remove its source-projection edges; do not delete shared facet nodes. +- **Bad projection:** reject before opening the write transaction. +- **Wrong parent revision:** transaction aborts before portfolio mutation commits. +- **Write failure:** whole hydration transaction rolls back. +- **Exact replay:** returns `reused` without a second semantic mutation. +- **Tombstone:** marks Idea inactive and removes its source-projection edges; does not delete shared facet nodes. - **Orphan facets:** tolerated as non-authoritative derived residue. Garbage collection is deferred rather than mixed into the critical write transaction. -The v0.1 hydration transaction is atomic per graph write, while a multi-record hydration revision consists of multiple Neo4j write transactions guarded by the hydration state. The committed graph revision advances only after every record succeeds. Consumers must treat `in_progress=true` as an uncommitted snapshot boundary. A future live context adapter must enforce that read gate. +## Deferred seam + +The next transport unit is intentionally small: + +```text +IdeaOS IdeaGraphProvider adapter + -> Gate / TransportPacket + -> CEG owner-native portfolio context action + -> committed graph_revision + intersections + ranking evidence +``` + +That adapter must call CEG semantics. It must not copy matching, relationship classification, or ranking logic into IdeaOS. diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml index cb1d1f1e..98223609 100644 --- a/domains/idea-portfolio/spec.yaml +++ b/domains/idea-portfolio/spec.yaml @@ -52,6 +52,17 @@ ontology: - {name: key, type: string, required: true} - {name: last_seen_revision, type: string} + - label: IdeaPortfolioHydrationState + managedby: sync + auxiliary: true + properties: + - {name: state_id, type: string, required: true} + - {name: current_revision, type: string} + - {name: source_snapshot_ref, type: string} + - {name: source_snapshot_digest, type: string} + - {name: batch_digest, type: string} + - {name: completed_at, type: datetime} + edges: - type: PRODUCES from: Idea @@ -235,12 +246,11 @@ scoring: defaultweight: 0.05 matchdirections: [portfolio_context_for_idea] +# The live generic sync handler is intentionally not advertised for this +# domain yet. Corpus hydration uses engine/sync/idea_portfolio.py through the +# explicit admin CLI until the Gate-facing sync contract is added and tested. sync: - endpoints: - - path: /sync/idea_projection - targetnode: Idea - idproperty: idea_id - batchstrategy: unwindmerge + endpoints: [] # Centrality is intentionally not declared yet. The current CEG scheduler does # not implement PageRank, and declaring a skipped algorithm would create false @@ -264,9 +274,6 @@ capabilities: - name: portfolio_read actions: [match:read] allowed_subjects: ["*"] - - name: portfolio_write - actions: [sync:write] - allowed_subjects: ["*"] feedbackloop: enabled: false diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 1f6d699b..bceed4e7 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -2,12 +2,9 @@ IdeaOS owns canonical idea identity and lifecycle truth. This module accepts the narrow ``IdeaGraphProjection`` contract, validates it, compiles graph-safe -facets, and applies a revision-chained hydration batch to CEG. It never parses -raw IdeaOS corpus files and never infers idea identity from filenames. - -Hydration is an owner-native CEG write path. The public Constellation transport -binding can call this service later without moving semantic ownership into an -adapter. +facets, and applies one revision-chained hydration envelope atomically in CEG. +It never parses raw IdeaOS corpus files and never infers idea identity from +filenames. """ from __future__ import annotations @@ -24,11 +21,10 @@ DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" DOMAIN_ID = "idea-portfolio" _STATE_ID = "canonical" -_RELATION_TYPES = ("PRODUCES", "REQUIRES", "TARGETS", "USES", "DEPENDS_ON") class IdeaPortfolioHydrationError(ValueError): - """Raised when a hydration batch cannot be safely admitted or applied.""" + """Raised when a hydration envelope cannot be safely admitted or applied.""" class EvidenceState(StrEnum): @@ -74,14 +70,6 @@ class AssertionRelation(StrEnum): AssertionKind.DEPENDENCY: frozenset({AssertionRelation.DEPENDS_ON}), } -_EDGE_BY_RELATION: dict[AssertionRelation, str] = { - AssertionRelation.PRODUCES: "PRODUCES", - AssertionRelation.REQUIRES: "REQUIRES", - AssertionRelation.TARGETS: "TARGETS", - AssertionRelation.USES: "USES", - AssertionRelation.DEPENDS_ON: "DEPENDS_ON", -} - class IdeaLifecycle(BaseModel): model_config = ConfigDict(extra="forbid") @@ -170,7 +158,8 @@ def validate_operation(self) -> "IdeaPortfolioSyncRecord": def resolved_idea_id(self) -> str: if self.projection is not None: return self.projection.idea_id - assert self.idea_id is not None + if self.idea_id is None: + raise IdeaPortfolioHydrationError("validated tombstone record lacks idea_id") return self.idea_id @@ -279,11 +268,10 @@ def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertio def build_portfolio_match_query(projection: IdeaGraphProjection | dict[str, Any]) -> dict[str, Any]: - """Compile one IdeaOS projection into the flat query shape declared by the domain pack.""" + """Compile a projection into the flat query shape consumed by the current match handler.""" model = projection if isinstance(projection, IdeaGraphProjection) else IdeaGraphProjection.model_validate(projection) compiled = compile_assertions(model) - by_relation: dict[str, list[str]] = {relation.value: [] for relation in AssertionRelation} for assertion in compiled: by_relation[assertion.relation].append(assertion.facet_id) @@ -322,12 +310,7 @@ def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, return HydrationPlan(envelope=model, batch_digest=batch_digest, graph_revision=graph_revision) -def compile_upsert_command( - projection: IdeaGraphProjection, - *, - graph_revision: str, - tenant: str, -) -> WriteCommand: +def compile_upsert_command(projection: IdeaGraphProjection, *, graph_revision: str) -> WriteCommand: p_digest = projection_digest(projection) grouped: dict[str, list[dict[str, Any]]] = {relation.value: [] for relation in AssertionRelation} for assertion in compile_assertions(projection): @@ -453,7 +436,7 @@ def compile_upsert_command( return WriteCommand( cypher=cypher, parameters={ - "tenant": tenant, + "tenant": DOMAIN_ID, "idea_id": projection.idea_id, "source_digest": projection.source_digest, "projection_digest": p_digest, @@ -473,7 +456,7 @@ def compile_upsert_command( ) -def compile_tombstone_command(idea_id: str, *, graph_revision: str, tenant: str) -> WriteCommand: +def compile_tombstone_command(idea_id: str, *, graph_revision: str) -> WriteCommand: cypher = """ MERGE (idea:Idea {idea_id: $idea_id}) SET idea.active = false, @@ -487,162 +470,114 @@ def compile_tombstone_command(idea_id: str, *, graph_revision: str, tenant: str) """.strip() return WriteCommand( cypher=cypher, - parameters={"tenant": tenant, "idea_id": idea_id, "graph_revision": graph_revision}, + parameters={"tenant": DOMAIN_ID, "idea_id": idea_id, "graph_revision": graph_revision}, ) -_ACQUIRE_HYDRATION_CYPHER = """ +_LOCK_STATE_CYPHER = """ MERGE (state:IdeaPortfolioHydrationState {state_id: $state_id}) -SET state._cas_lock = coalesce(state._cas_lock, 0) + 1 -WITH state, - state.current_revision AS current_revision, - coalesce(state.in_progress, false) AS in_progress, - state.target_revision AS target_revision -WHERE (current_revision = $graph_revision AND in_progress = false) - OR (in_progress = true AND target_revision = $graph_revision) - OR ( - in_progress = false - AND ( - (current_revision IS NULL AND $expected_graph_revision IS NULL) - OR current_revision = $expected_graph_revision - ) - ) -WITH state, current_revision, in_progress, target_revision, - CASE - WHEN current_revision = $graph_revision AND in_progress = false THEN 'reused' - WHEN in_progress = true AND target_revision = $graph_revision THEN 'resume' - ELSE 'start' - END AS acquisition_status -FOREACH (_ IN CASE WHEN acquisition_status = 'start' THEN [1] ELSE [] END | - SET state.in_progress = true, - state.target_revision = $graph_revision, - state.source_snapshot_ref = $source_snapshot_ref, - state.source_snapshot_digest = $source_snapshot_digest, - state.batch_digest = $batch_digest, - state.started_at = datetime(), - state.completed_at = null, - state.last_error = null, - state._tenant = $tenant -) -RETURN acquisition_status AS status, current_revision +SET state._cas_lock = coalesce(state._cas_lock, 0) + 1, + state._tenant = $tenant +RETURN state.current_revision AS current_revision """.strip() -_FINALIZE_HYDRATION_CYPHER = """ +_FINALIZE_STATE_CYPHER = """ MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) -SET state._cas_lock = coalesce(state._cas_lock, 0) + 1 -WITH state -WHERE state.in_progress = true AND state.target_revision = $graph_revision SET state.current_revision = $graph_revision, - state.in_progress = false, - state.target_revision = null, + state.source_snapshot_ref = $source_snapshot_ref, + state.source_snapshot_digest = $source_snapshot_digest, + state.batch_digest = $batch_digest, state.completed_at = datetime(), - state.last_error = null + state._tenant = $tenant RETURN state.current_revision AS graph_revision """.strip() -_FAIL_HYDRATION_CYPHER = """ -MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) -WHERE state.in_progress = true AND state.target_revision = $graph_revision -SET state.last_error = $error, - state.last_error_at = datetime() -RETURN state.target_revision AS graph_revision -""".strip() - class IdeaPortfolioHydrator: - """Apply one revision-chained IdeaOS corpus delta to the idea-portfolio graph.""" + """Atomically apply one revision-chained IdeaOS corpus delta to CEG.""" - def __init__(self, graph_writer: GraphWriter, *, tenant: str = DOMAIN_ID) -> None: + def __init__(self, graph_writer: GraphWriter) -> None: self.graph_writer = graph_writer - self.tenant = tenant async def apply(self, envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> dict[str, Any]: plan = compile_hydration_plan(envelope) - acquired = await self.graph_writer.execute_write( - cypher=_ACQUIRE_HYDRATION_CYPHER, - parameters={ - "state_id": _STATE_ID, - "tenant": self.tenant, - "graph_revision": plan.graph_revision, - "expected_graph_revision": plan.envelope.expected_graph_revision, - "source_snapshot_ref": plan.envelope.source_snapshot_ref, - "source_snapshot_digest": plan.envelope.source_snapshot_digest, - "batch_digest": plan.batch_digest, - }, - database=DOMAIN_ID, - ) - records = _write_records(acquired) - if not records: - raise IdeaPortfolioHydrationError( - "hydration revision conflict or another revision is already in progress" + + async def apply_transaction(tx: Any) -> dict[str, Any]: + state_result = await tx.run( + _LOCK_STATE_CYPHER, + { + "state_id": _STATE_ID, + "tenant": DOMAIN_ID, + }, ) + state_rows = await state_result.data() + current_revision = state_rows[0].get("current_revision") if state_rows else None - acquisition_status = records[0].get("status") - if acquisition_status == "reused": - return { - "schema": "ceg.idea-portfolio-hydration-receipt/v1", - "status": "reused", - "graph_revision": plan.graph_revision, - "batch_digest": plan.batch_digest, - "source_snapshot_ref": plan.envelope.source_snapshot_ref, - "source_snapshot_digest": plan.envelope.source_snapshot_digest, - "applied": [], - "tombstoned": [], - } + if current_revision == plan.graph_revision: + return self._receipt(plan, status="reused", applied=[], tombstoned=[]) + + if current_revision != plan.envelope.expected_graph_revision: + raise IdeaPortfolioHydrationError( + "hydration revision conflict: expected parent does not match committed graph revision" + ) - applied: list[str] = [] - tombstoned: list[str] = [] - try: + applied: list[str] = [] + tombstoned: list[str] = [] for record in plan.envelope.records: if record.operation == "upsert": - assert record.projection is not None - command = compile_upsert_command( - record.projection, - graph_revision=plan.graph_revision, - tenant=self.tenant, - ) - await self.graph_writer.execute_write( - cypher=command.cypher, - parameters=command.parameters, - database=DOMAIN_ID, - ) - applied.append(record.projection.idea_id) + projection = record.projection + if projection is None: + raise IdeaPortfolioHydrationError("validated upsert record lacks projection") + command = compile_upsert_command(projection, graph_revision=plan.graph_revision) + result = await tx.run(command.cypher, command.parameters) + await result.consume() + applied.append(projection.idea_id) else: command = compile_tombstone_command( record.resolved_idea_id, graph_revision=plan.graph_revision, - tenant=self.tenant, - ) - await self.graph_writer.execute_write( - cypher=command.cypher, - parameters=command.parameters, - database=DOMAIN_ID, ) + result = await tx.run(command.cypher, command.parameters) + await result.consume() tombstoned.append(record.resolved_idea_id) - except Exception as exc: - await self.graph_writer.execute_write( - cypher=_FAIL_HYDRATION_CYPHER, - parameters={ + + final_result = await tx.run( + _FINALIZE_STATE_CYPHER, + { "state_id": _STATE_ID, + "tenant": DOMAIN_ID, "graph_revision": plan.graph_revision, - "error": type(exc).__name__, + "source_snapshot_ref": plan.envelope.source_snapshot_ref, + "source_snapshot_digest": plan.envelope.source_snapshot_digest, + "batch_digest": plan.batch_digest, }, - database=DOMAIN_ID, ) - raise - - finalized = await self.graph_writer.execute_write( - cypher=_FINALIZE_HYDRATION_CYPHER, - parameters={"state_id": _STATE_ID, "graph_revision": plan.graph_revision}, - database=DOMAIN_ID, - ) - if not _write_records(finalized): - raise IdeaPortfolioHydrationError("hydration applied but graph revision finalization failed") + await final_result.consume() + return self._receipt( + plan, + status="applied", + applied=applied, + tombstoned=tombstoned, + ) + result = await self.graph_writer.execute_write(apply_transaction, database=DOMAIN_ID) + if not isinstance(result, dict): + raise IdeaPortfolioHydrationError("graph writer returned an invalid hydration receipt") + return result + + @staticmethod + def _receipt( + plan: HydrationPlan, + *, + status: Literal["applied", "reused"], + applied: list[str], + tombstoned: list[str], + ) -> dict[str, Any]: return { "schema": "ceg.idea-portfolio-hydration-receipt/v1", - "status": "applied" if acquisition_status == "start" else "resumed", + "status": status, "graph_revision": plan.graph_revision, + "parent_graph_revision": plan.envelope.expected_graph_revision, "batch_digest": plan.batch_digest, "source_snapshot_ref": plan.envelope.source_snapshot_ref, "source_snapshot_digest": plan.envelope.source_snapshot_digest, @@ -651,13 +586,6 @@ async def apply(self, envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) } -def _write_records(result: Any) -> list[dict[str, Any]]: - if isinstance(result, dict): - records = result.get("records", []) - return [dict(record) for record in records] - return [] - - __all__ = [ "AssertionKind", "AssertionRelation", diff --git a/tests/unit/test_idea_portfolio.py b/tests/unit/test_idea_portfolio.py index 78152630..c53b13b5 100644 --- a/tests/unit/test_idea_portfolio.py +++ b/tests/unit/test_idea_portfolio.py @@ -101,7 +101,12 @@ def test_domain_loads_and_compilers_accept_it(self) -> None: spec = loader.load_domain("idea-portfolio") assert spec.domain.id == "idea-portfolio" - assert {node.label for node in spec.ontology.nodes} == {"Idea", "IdeaQuery", "PortfolioFacet"} + assert {node.label for node in spec.ontology.nodes} == { + "Idea", + "IdeaQuery", + "PortfolioFacet", + "IdeaPortfolioHydrationState", + } assert {edge.type for edge in spec.ontology.edges} == { "PRODUCES", "REQUIRES", @@ -109,6 +114,7 @@ def test_domain_loads_and_compilers_accept_it(self) -> None: "USES", "DEPENDS_ON", } + assert spec.sync.endpoints == [] assert sum(d.defaultweight for d in spec.scoring.dimensions) == pytest.approx(1.0) gate_clause = GateCompiler(spec).compile_all_gates("portfolio_context_for_idea") @@ -161,13 +167,14 @@ def test_duplicate_semantic_assertion_is_rejected(self) -> None: def test_upsert_replaces_source_projection_edges(self) -> None: projection = IdeaGraphProjection.model_validate(_projection()) - command = compile_upsert_command(projection, graph_revision=_digest("c"), tenant="idea-portfolio") + command = compile_upsert_command(projection, graph_revision=_digest("c")) assert "DELETE old" in command.cypher assert "MERGE (idea)-[rel:PRODUCES]->(facet)" in command.cypher assert "rel.evidence_state" in command.cypher assert command.parameters["projection_digest"] == projection_digest(projection) assert command.parameters["unknowns_json"] == '["external demand not yet proven"]' + assert command.parameters["tenant"] == "idea-portfolio" @pytest.mark.unit @@ -182,11 +189,36 @@ def test_revision_is_deterministic_and_parent_linked(self) -> None: assert child.graph_revision != first.graph_revision -class _FakeWriter: - def __init__(self, responses: list[dict[str, Any]]) -> None: - self.responses = list(responses) +class _FakeResult: + def __init__(self, rows: list[dict[str, Any]] | None = None) -> None: + self.rows = rows or [] + self.consumed = False + + async def data(self) -> list[dict[str, Any]]: + return list(self.rows) + + async def consume(self) -> None: + self.consumed = True + + +class _FakeTransaction: + def __init__(self, current_revision: str | None) -> None: + self.current_revision = current_revision self.calls: list[dict[str, Any]] = [] + async def run(self, cypher: str, parameters: dict[str, Any]) -> _FakeResult: + self.calls.append({"cypher": cypher, "parameters": parameters}) + if "RETURN state.current_revision AS current_revision" in cypher: + return _FakeResult([{"current_revision": self.current_revision}]) + return _FakeResult() + + +class _FakeWriter: + def __init__(self, current_revision: str | None = None) -> None: + self.tx = _FakeTransaction(current_revision) + self.execute_write_calls = 0 + self.database: str | None = None + async def execute_write( self, transaction_function: Any = None, @@ -196,55 +228,54 @@ async def execute_write( database: str | None = None, **kwargs: Any, ) -> dict[str, Any]: - self.calls.append({"cypher": cypher, "parameters": parameters, "database": database}) - if not self.responses: - return {"records": []} - return self.responses.pop(0) + self.execute_write_calls += 1 + self.database = database + if transaction_function is None: + raise AssertionError("hydrator must use one managed transaction") + return await transaction_function(self.tx, *args, **kwargs) @pytest.mark.unit @pytest.mark.asyncio -async def test_hydrator_applies_then_finalizes_revision() -> None: - writer = _FakeWriter( - [ - {"records": [{"status": "start", "current_revision": None}]}, - {"records": [{"idea_id": "idea-alpha", "assertion_count": 5}]}, - {"records": [{"graph_revision": _digest("d")}]}, - ] - ) +async def test_hydrator_applies_whole_envelope_in_one_transaction() -> None: + writer = _FakeWriter(current_revision=None) hydrator = IdeaPortfolioHydrator(writer) receipt = await hydrator.apply(_envelope()) assert receipt["status"] == "applied" assert receipt["applied"] == ["idea-alpha"] - assert len(writer.calls) == 3 - assert all(call["database"] == "idea-portfolio" for call in writer.calls) - assert "IdeaPortfolioHydrationState" in str(writer.calls[0]["cypher"]) - assert "MERGE (idea:Idea" in str(writer.calls[1]["cypher"]) - assert "state.current_revision" in str(writer.calls[2]["cypher"]) + assert writer.execute_write_calls == 1 + assert writer.database == "idea-portfolio" + assert len(writer.tx.calls) == 3 + assert "IdeaPortfolioHydrationState" in writer.tx.calls[0]["cypher"] + assert "MERGE (idea:Idea" in writer.tx.calls[1]["cypher"] + assert "state.current_revision" in writer.tx.calls[2]["cypher"] @pytest.mark.unit @pytest.mark.asyncio async def test_hydrator_exact_revision_replay_is_noop() -> None: - writer = _FakeWriter([{"records": [{"status": "reused", "current_revision": _digest("e")}] }]) + plan = compile_hydration_plan(_envelope()) + writer = _FakeWriter(current_revision=plan.graph_revision) hydrator = IdeaPortfolioHydrator(writer) receipt = await hydrator.apply(_envelope()) assert receipt["status"] == "reused" assert receipt["applied"] == [] - assert len(writer.calls) == 1 + assert writer.execute_write_calls == 1 + assert len(writer.tx.calls) == 1 @pytest.mark.unit @pytest.mark.asyncio -async def test_hydrator_rejects_revision_conflict_before_projection_write() -> None: - writer = _FakeWriter([{"records": []}]) +async def test_hydrator_rejects_parent_revision_conflict_before_projection_write() -> None: + writer = _FakeWriter(current_revision=_digest("e")) hydrator = IdeaPortfolioHydrator(writer) - with pytest.raises(IdeaPortfolioHydrationError, match="revision conflict"): + with pytest.raises(IdeaPortfolioHydrationError, match="expected parent"): await hydrator.apply(_envelope()) - assert len(writer.calls) == 1 + assert writer.execute_write_calls == 1 + assert len(writer.tx.calls) == 1 diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index 10dd9990..5bc36b26 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -51,7 +51,7 @@ def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: } -async def _apply(envelope: IdeaPortfolioHydrationEnvelope, *, tenant: str) -> dict[str, object]: +async def _apply(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: # Loading through the production loader proves the folder-shaped domain pack # is discoverable and validates against the current DomainSpec before writes. DomainPackLoader().load_domain(DOMAIN_ID) @@ -59,8 +59,7 @@ async def _apply(envelope: IdeaPortfolioHydrationEnvelope, *, tenant: str) -> di driver = GraphDriver() await driver.connect() try: - hydrator = IdeaPortfolioHydrator(driver, tenant=tenant) - return await hydrator.apply(envelope) + return await IdeaPortfolioHydrator(driver).apply(envelope) finally: await driver.close() @@ -69,15 +68,10 @@ def main() -> None: parser = argparse.ArgumentParser(description="Validate or apply an IdeaOS -> CEG portfolio hydration envelope") parser.add_argument("envelope", type=Path, help="Path to ceg.idea-portfolio-hydration/v1 JSON") parser.add_argument("--apply", action="store_true", help="Mutate the CEG idea-portfolio graph") - parser.add_argument( - "--tenant", - default=DOMAIN_ID, - help="Tenant provenance value stored on graph nodes (default: idea-portfolio)", - ) args = parser.parse_args() envelope = _load_envelope(args.envelope) - result = asyncio.run(_apply(envelope, tenant=args.tenant)) if args.apply else _dry_run(envelope) + result = asyncio.run(_apply(envelope)) if args.apply else _dry_run(envelope) print(json.dumps(result, indent=2, sort_keys=True)) From 6c5933238f0be4f931a1bb22d3872dd0fec4cf26 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:49:40 -0400 Subject: [PATCH 03/32] fix: align idea portfolio domain with current schema --- domains/idea-portfolio/spec.yaml | 49 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml index 98223609..42d83d9f 100644 --- a/domains/idea-portfolio/spec.yaml +++ b/domains/idea-portfolio/spec.yaml @@ -291,28 +291,27 @@ decision_arbitration: enabled: false feature_catalog: - features: - - feature_id: incoming_requirement_fit - owner: ceg - provenance_required: true - evidence_required: true - - feature_id: outgoing_requirement_fit - owner: ceg - provenance_required: true - evidence_required: true - - feature_id: shared_usage - owner: ceg - provenance_required: true - evidence_required: true - - feature_id: shared_target - owner: ceg - provenance_required: true - evidence_required: true - - feature_id: query_depends_on_candidate - owner: ceg - provenance_required: true - evidence_required: true - - feature_id: candidate_depends_on_query - owner: ceg - provenance_required: true - evidence_required: true + - feature_id: incoming_requirement_fit + owner: ceg + scoring_dimension: incoming_requirement_fit + evidence_required: true + - feature_id: outgoing_requirement_fit + owner: ceg + scoring_dimension: outgoing_requirement_fit + evidence_required: true + - feature_id: shared_usage + owner: ceg + scoring_dimension: shared_usage + evidence_required: true + - feature_id: shared_target + owner: ceg + scoring_dimension: shared_target + evidence_required: true + - feature_id: query_depends_on_candidate + owner: ceg + scoring_dimension: query_depends_on_candidate + evidence_required: true + - feature_id: candidate_depends_on_query + owner: ceg + scoring_dimension: candidate_depends_on_query + evidence_required: true From 53337f101ce5e552776068061fe6beb01da3bd46 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:50:02 -0400 Subject: [PATCH 04/32] fix: anchor portfolio hydration domain lookup --- tools/hydrate_idea_portfolio.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index 5bc36b26..190926a2 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -16,7 +16,8 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) from engine.config.loader import DomainPackLoader from engine.graph.driver import GraphDriver @@ -54,7 +55,7 @@ def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: async def _apply(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: # Loading through the production loader proves the folder-shaped domain pack # is discoverable and validates against the current DomainSpec before writes. - DomainPackLoader().load_domain(DOMAIN_ID) + DomainPackLoader(config_path=str(ROOT / "domains")).load_domain(DOMAIN_ID) driver = GraphDriver() await driver.connect() From 66e3a4def53a813a7d1fa994624b9bf15d5a760d Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:50:35 -0400 Subject: [PATCH 05/32] feat: gate idea portfolio activation --- engine/config/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/config/settings.py b/engine/config/settings.py index 4d401115..efb9cd16 100644 --- a/engine/config/settings.py +++ b/engine/config/settings.py @@ -155,6 +155,9 @@ class Settings(BaseSettings): # Seam audit / PR remediation: paid-tier enrich_now Gate dispatch is opt-in. # Default off so deploy does not immediately spend EIE budget until enabled. auto_enrich_via_gate: bool = False + # IdeaOS portfolio graph is a new behavioral surface. Keep both corpus writes + # and portfolio-context reads dormant until explicitly activated by an operator. + idea_portfolio_enabled: bool = False @model_validator(mode="after") def _validate_production_secrets(self) -> "Settings": From 82387e10a5d1053d1ca9e01651dcf4b3b6d81370 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:30 -0400 Subject: [PATCH 06/32] fix: harden idea portfolio domain admission --- domains/idea-portfolio/spec.yaml | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml index 42d83d9f..5bdb35cd 100644 --- a/domains/idea-portfolio/spec.yaml +++ b/domains/idea-portfolio/spec.yaml @@ -173,6 +173,9 @@ gates: nullbehavior: fail matchdirections: [portfolio_context_for_idea] +# Only source-backed VERIFIED and SUPPORTED_INFERENCE assertions may influence +# portfolio rank. HYPOTHESIS and UNKNOWN assertions remain in the graph for +# context, but cannot be silently laundered into numeric ranking evidence. scoring: dimensions: - name: incoming_requirement_fit @@ -180,8 +183,10 @@ scoring: computation: customcypher expression: > CASE WHEN $requires_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[:PRODUCES]->(f:PortfolioFacet) - WHERE $requires_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat(size([(candidate)-[rel:PRODUCES]->(f:PortfolioFacet) + WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] + AND rel.source_refs_json <> '[]' + AND $requires_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($requires_count) END weightkey: wincoming defaultweight: 0.25 @@ -192,8 +197,10 @@ scoring: computation: customcypher expression: > CASE WHEN $produces_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[:REQUIRES]->(f:PortfolioFacet) - WHERE $produces_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat(size([(candidate)-[rel:REQUIRES]->(f:PortfolioFacet) + WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] + AND rel.source_refs_json <> '[]' + AND $produces_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($produces_count) END weightkey: woutgoing defaultweight: 0.25 @@ -204,8 +211,10 @@ scoring: computation: customcypher expression: > CASE WHEN $uses_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[:USES]->(f:PortfolioFacet) - WHERE $uses_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat(size([(candidate)-[rel:USES]->(f:PortfolioFacet) + WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] + AND rel.source_refs_json <> '[]' + AND $uses_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($uses_count) END weightkey: wusage defaultweight: 0.20 @@ -216,8 +225,10 @@ scoring: computation: customcypher expression: > CASE WHEN $targets_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[:TARGETS]->(f:PortfolioFacet) - WHERE $targets_facets CONTAINS ('|' + f.facet_id + '|') | f])) / + toFloat(size([(candidate)-[rel:TARGETS]->(f:PortfolioFacet) + WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] + AND rel.source_refs_json <> '[]' + AND $targets_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($targets_count) END weightkey: wtarget defaultweight: 0.15 @@ -239,8 +250,10 @@ scoring: computation: customcypher expression: > CASE WHEN $self_dependency_facet_id = '' THEN 0.0 ELSE - CASE WHEN size([(candidate)-[:DEPENDS_ON]->(f:PortfolioFacet) - WHERE f.facet_id = $self_dependency_facet_id | f]) > 0 + CASE WHEN size([(candidate)-[rel:DEPENDS_ON]->(f:PortfolioFacet) + WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] + AND rel.source_refs_json <> '[]' + AND f.facet_id = $self_dependency_facet_id | f]) > 0 THEN 1.0 ELSE 0.0 END END weightkey: wcandidatedependency defaultweight: 0.05 @@ -266,7 +279,10 @@ compliance: logretentiondays: 365 pii: enabled: false - prohibitedfactors: [] + prohibitedfactors: + enabled: true + blockedfields: [] + enforcement: compiletime regionalrules: [] counterfactualaudit: false From 894c2bcddcb57b803b6a827d93b0c4e6d0d4d6d7 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:29:43 -0400 Subject: [PATCH 07/32] fix: enforce domain activation at loader boundary --- engine/config/loader.py | 74 +++++++++++++---------------------------- 1 file changed, 23 insertions(+), 51 deletions(-) diff --git a/engine/config/loader.py b/engine/config/loader.py index f7766214..cec63d27 100644 --- a/engine/config/loader.py +++ b/engine/config/loader.py @@ -26,55 +26,52 @@ from pydantic import ValidationError as PydanticValidationError from engine.config.schema import DomainSpec +from engine.config.settings import settings logger = logging.getLogger(__name__) -# Maximum spec file size (5MB) to prevent OOM on malicious/corrupted files MAX_SPEC_BYTES = 5 * 1024 * 1024 - -# Canonical filename for a domain's spec inside its directory. SPEC_FILENAME = "spec.yaml" +_DOMAIN_FEATURE_FLAGS = {"idea-portfolio": "idea_portfolio_enabled"} class DomainNotFoundError(Exception): - """Raised when a requested domain spec does not exist.""" + """Raised when a requested domain spec does not exist or is disabled.""" class DomainSpecError(Exception): """Raised when a domain spec fails validation.""" -class DomainPackLoader: - """ - Loads and caches domain spec YAML files with hot-reload support. +def _domain_enabled(domain_id: str) -> bool: + flag = _DOMAIN_FEATURE_FLAGS.get(domain_id) + return flag is None or bool(getattr(settings, flag, False)) - Thread-safety: Cache operations are protected by a threading.Lock. - TTL-based invalidation avoids per-request stat() syscalls. - LRU eviction keeps cache bounded (configurable via DOMAIN_CACHE_MAX_SIZE). - """ + +class DomainPackLoader: + """Load and cache folder-shaped domain specs with bounded hot reload.""" def __init__(self, config_path: str | None = None) -> None: raw = config_path or os.getenv("DOMAIN_SPECS_PATH") or "domains" self._base_path = Path(raw).resolve() - self._cache: dict[str, tuple[DomainSpec, float, float]] = {} # domain_id → (spec, mtime, cached_at) + self._cache: dict[str, tuple[DomainSpec, float, float]] = {} self._lock = threading.Lock() self._max_size = int(os.getenv("DOMAIN_CACHE_MAX_SIZE", "100")) self._ttl_seconds = float(os.getenv("DOMAIN_CACHE_TTL_SECONDS", "30")) def load_domain(self, domain_id: str) -> DomainSpec: - """Load and validate a domain spec with mtime-based cache invalidation.""" + """Load a domain only when its optional feature gate admits it.""" + if not _domain_enabled(domain_id): + raise DomainNotFoundError(f"Domain '{domain_id}' is disabled by configuration") spec_path = self._resolve_spec_path(domain_id) with self._lock: if domain_id in self._cache: cached_spec, cached_mtime, cached_at = self._cache[domain_id] - # Skip stat() if within TTL if (time.monotonic() - cached_at) < self._ttl_seconds: return cached_spec - # TTL expired — check mtime current_mtime = spec_path.stat().st_mtime if cached_mtime >= current_mtime: - # Refresh cached_at timestamp self._cache[domain_id] = (cached_spec, cached_mtime, time.monotonic()) return cached_spec logger.info("Domain spec changed on disk, reloading: %s", domain_id) @@ -82,8 +79,6 @@ def load_domain(self, domain_id: str) -> DomainSpec: current_mtime = spec_path.stat().st_mtime spec = self._load_and_validate(spec_path, domain_id) - - # LRU eviction: if cache is full, remove oldest entry if len(self._cache) >= self._max_size and domain_id not in self._cache: oldest_key = min(self._cache, key=lambda k: self._cache[k][2]) del self._cache[oldest_key] @@ -100,96 +95,73 @@ def invalidate(self, domain_id: str | None = None) -> None: else: self._cache.clear() - # ------------------------------------------------------------------ - # W4-03: Async loading with per-domain stampede prevention - # ------------------------------------------------------------------ - async def load_domain_async(self, domain_id: str) -> DomainSpec: - """Async domain loading with per-domain lock for stampede prevention. - - Checks the existing TTL cache first. On miss, acquires a per-domain - asyncio.Lock so that concurrent requests for the same domain don't - all hit disk simultaneously. Loads from disk via asyncio.to_thread. - """ - # Fast path: check sync cache (already TTL-bounded) + """Async domain loading with per-domain stampede prevention.""" + if not _domain_enabled(domain_id): + raise DomainNotFoundError(f"Domain '{domain_id}' is disabled by configuration") with self._lock: if domain_id in self._cache: cached_spec, _cached_mtime, cached_at = self._cache[domain_id] if (time.monotonic() - cached_at) < self._ttl_seconds: return cached_spec - # Per-domain async lock for stampede prevention if not hasattr(self, "_async_locks"): self._async_locks: dict[str, asyncio.Lock] = {} if domain_id not in self._async_locks: self._async_locks[domain_id] = asyncio.Lock() async with self._async_locks[domain_id]: - # Double-check after acquiring lock with self._lock: if domain_id in self._cache: cached_spec, _cached_mtime, cached_at = self._cache[domain_id] if (time.monotonic() - cached_at) < self._ttl_seconds: return cached_spec - - # Load from disk in thread pool return await asyncio.to_thread(self.load_domain, domain_id) def list_domains(self) -> list[str]: - """Discover all domain directories containing spec.yaml.""" + """Discover enabled domain directories containing spec.yaml.""" if not self._base_path.is_dir(): return [] - return [d.name for d in sorted(self._base_path.iterdir()) if d.is_dir() and (d / SPEC_FILENAME).exists()] + return [ + d.name + for d in sorted(self._base_path.iterdir()) + if d.is_dir() and (d / SPEC_FILENAME).exists() and _domain_enabled(d.name) + ] def _resolve_spec_path(self, domain_id: str) -> Path: - """Resolve and validate spec file path — prevents path traversal and symlink attacks.""" - # Reject empty or whitespace-only domain_id + """Resolve and validate spec file path, preventing traversal and symlinks.""" if not domain_id or not domain_id.strip(): raise DomainNotFoundError("Domain ID cannot be empty") - - # Reject null bytes (potential injection attack) if "\x00" in domain_id: raise DomainNotFoundError(f"Invalid domain ID: {domain_id!r} contains null byte") - - # Reject absolute domain IDs — only relative IDs are valid if Path(domain_id).is_absolute(): raise DomainNotFoundError(f"Invalid domain ID: {domain_id!r} must be a relative path") candidate = (self._base_path / domain_id / SPEC_FILENAME).resolve() - - # Check for symlinks before resolving - reject symlinked spec files raw_path = self._base_path / domain_id / SPEC_FILENAME if raw_path.is_symlink(): raise DomainNotFoundError(f"Invalid domain path: {domain_id!r} spec.yaml is a symlink") - - # Verify resolved path is within base directory using proper path ancestry check try: candidate.relative_to(self._base_path.resolve()) except ValueError as exc: raise DomainNotFoundError(f"Invalid domain path: {domain_id!r} resolves outside base directory") from exc - if not candidate.exists(): raise DomainNotFoundError(f"Domain spec not found: {candidate}") - return candidate def _load_and_validate(self, path: Path, domain_id: str) -> DomainSpec: """Load YAML and validate against DomainSpec schema.""" - # Check file size before reading to prevent OOM file_size = path.stat().st_size if file_size > MAX_SPEC_BYTES: raise DomainSpecError( f"Domain spec {domain_id} exceeds maximum size: {file_size} bytes > {MAX_SPEC_BYTES} bytes" ) - try: raw = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: raise DomainSpecError(f"Invalid YAML in {path}: {exc}") from exc - if not isinstance(raw, dict): raise DomainSpecError(f"Domain spec must be a YAML mapping, got {type(raw).__name__}") - try: return DomainSpec.model_validate(raw) except PydanticValidationError as exc: From 05d0d0fc278e984ff22e4a8ded3831532fff80a9 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:33:45 -0400 Subject: [PATCH 08/32] fix: make idea portfolio hydration replay-safe and epistemic-aware --- engine/sync/idea_portfolio.py | 453 ++++++++++++---------------------- 1 file changed, 159 insertions(+), 294 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index bceed4e7..8c78453e 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -1,11 +1,4 @@ -"""IdeaOS portfolio projection hydration for the CEG idea-portfolio domain. - -IdeaOS owns canonical idea identity and lifecycle truth. This module accepts the -narrow ``IdeaGraphProjection`` contract, validates it, compiles graph-safe -facets, and applies one revision-chained hydration envelope atomically in CEG. -It never parses raw IdeaOS corpus files and never infers idea identity from -filenames. -""" +"""IdeaOS projection hydration and portfolio-query compilation for CEG.""" from __future__ import annotations @@ -14,17 +7,18 @@ import unicodedata from dataclasses import dataclass from enum import StrEnum -from typing import Any, Literal, Protocol +from typing import Any, Literal, Protocol, Self from pydantic import BaseModel, ConfigDict, Field, model_validator DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" DOMAIN_ID = "idea-portfolio" _STATE_ID = "canonical" +_MODEL_CONFIG = ConfigDict(extra="forbid", populate_by_name=True) class IdeaPortfolioHydrationError(ValueError): - """Raised when a hydration envelope cannot be safely admitted or applied.""" + """Raised when portfolio hydration cannot be safely admitted or applied.""" class EvidenceState(StrEnum): @@ -53,27 +47,26 @@ class AssertionRelation(StrEnum): _ALLOWED_RELATIONS: dict[AssertionKind, frozenset[AssertionRelation]] = { - AssertionKind.CAPABILITY: frozenset( - {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} - ), - AssertionKind.SUBSTRATE: frozenset( - {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} - ), - AssertionKind.PROOF_ASSET: frozenset( - {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} - ), - AssertionKind.DATA_ASSET: frozenset( - {AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES} - ), - AssertionKind.MARKET: frozenset({AssertionRelation.TARGETS}), - AssertionKind.CUSTOMER_TYPE: frozenset({AssertionRelation.TARGETS}), - AssertionKind.DEPENDENCY: frozenset({AssertionRelation.DEPENDS_ON}), + kind: frozenset({AssertionRelation.PRODUCES, AssertionRelation.REQUIRES, AssertionRelation.USES}) + for kind in ( + AssertionKind.CAPABILITY, + AssertionKind.SUBSTRATE, + AssertionKind.PROOF_ASSET, + AssertionKind.DATA_ASSET, + ) } +_ALLOWED_RELATIONS.update( + { + AssertionKind.MARKET: frozenset({AssertionRelation.TARGETS}), + AssertionKind.CUSTOMER_TYPE: frozenset({AssertionRelation.TARGETS}), + AssertionKind.DEPENDENCY: frozenset({AssertionRelation.DEPENDS_ON}), + } +) +_RANK_ELIGIBLE = frozenset({EvidenceState.VERIFIED, EvidenceState.SUPPORTED_INFERENCE}) class IdeaLifecycle(BaseModel): - model_config = ConfigDict(extra="forbid") - + model_config = _MODEL_CONFIG stage: str = Field(min_length=1) decision: Literal["GO", "CONDITIONAL_GO", "HOLD", "NO_GO"] | None = None proof_state: str | None = None @@ -81,8 +74,7 @@ class IdeaLifecycle(BaseModel): class IdeaAssertion(BaseModel): - model_config = ConfigDict(extra="forbid") - + model_config = _MODEL_CONFIG kind: AssertionKind relation: AssertionRelation key: str = Field(min_length=1) @@ -90,11 +82,9 @@ class IdeaAssertion(BaseModel): source_refs: list[str] @model_validator(mode="after") - def validate_semantics(self) -> "IdeaAssertion": + def validate_semantics(self) -> Self: if self.relation not in _ALLOWED_RELATIONS[self.kind]: - raise ValueError( - f"relation {self.relation.value!r} is not valid for assertion kind {self.kind.value!r}" - ) + raise ValueError(f"relation {self.relation.value!r} is not valid for assertion kind {self.kind.value!r}") if len(self.source_refs) != len(set(self.source_refs)): raise ValueError("assertion source_refs must be unique") if self.evidence_state != EvidenceState.UNKNOWN and not self.source_refs: @@ -103,11 +93,10 @@ def validate_semantics(self) -> "IdeaAssertion": class IdeaGraphProjection(BaseModel): - """Mirror of IdeaOS ``ideaos.idea-graph-projection/v1`` at the CEG boundary.""" - - model_config = ConfigDict(extra="forbid") + """CEG admission model for the IdeaOS idea-graph-projection/v1 wire contract.""" - schema: Literal["ideaos.idea-graph-projection/v1"] + model_config = _MODEL_CONFIG + schema_id: Literal["ideaos.idea-graph-projection/v1"] = Field(alias="schema") idea_id: str = Field(min_length=1) source_refs: list[str] source_digest: str = Field(pattern=DIGEST_PATTERN) @@ -116,42 +105,35 @@ class IdeaGraphProjection(BaseModel): unknowns: list[str] @model_validator(mode="after") - def validate_projection(self) -> "IdeaGraphProjection": + def validate_projection(self) -> Self: if not self.source_refs: - raise ValueError("hydrated IdeaGraphProjection requires at least one source_ref") + raise ValueError("CEG hydration requires at least one projection source_ref") if len(self.source_refs) != len(set(self.source_refs)): raise ValueError("projection source_refs must be unique") if len(self.unknowns) != len(set(self.unknowns)): raise ValueError("projection unknowns must be unique") - - semantic_keys = [(a.kind.value, a.relation.value, _canonical_key(a.key)) for a in self.assertions] - if len(semantic_keys) != len(set(semantic_keys)): + keys = [(a.kind.value, a.relation.value, _canonical_key(a.key)) for a in self.assertions] + if len(keys) != len(set(keys)): raise ValueError("projection contains duplicate semantic assertions") return self class IdeaPortfolioSyncRecord(BaseModel): - """CEG persistence command around an IdeaOS projection.""" - - model_config = ConfigDict(extra="forbid") - - schema: Literal["ceg.idea-portfolio-sync-record/v1"] + model_config = _MODEL_CONFIG + schema_id: Literal["ceg.idea-portfolio-sync-record/v1"] = Field(alias="schema") operation: Literal["upsert", "tombstone"] projection: IdeaGraphProjection | None = None idea_id: str | None = None @model_validator(mode="after") - def validate_operation(self) -> "IdeaPortfolioSyncRecord": + def validate_operation(self) -> Self: if self.operation == "upsert": if self.projection is None: raise ValueError("upsert sync record requires projection") if self.idea_id is not None and self.idea_id != self.projection.idea_id: raise ValueError("sync record idea_id does not match projection idea_id") - else: - if self.idea_id is None: - raise ValueError("tombstone sync record requires idea_id") - if self.projection is not None: - raise ValueError("tombstone sync record must not contain projection") + elif self.idea_id is None or self.projection is not None: + raise ValueError("tombstone requires idea_id and forbids projection") return self @property @@ -159,23 +141,20 @@ def resolved_idea_id(self) -> str: if self.projection is not None: return self.projection.idea_id if self.idea_id is None: - raise IdeaPortfolioHydrationError("validated tombstone record lacks idea_id") + raise IdeaPortfolioHydrationError("validated tombstone lacks idea_id") return self.idea_id class IdeaPortfolioHydrationEnvelope(BaseModel): - """One ordered corpus delta chained to the previously committed graph revision.""" - - model_config = ConfigDict(extra="forbid") - - schema: Literal["ceg.idea-portfolio-hydration/v1"] + model_config = _MODEL_CONFIG + schema_id: Literal["ceg.idea-portfolio-hydration/v1"] = Field(alias="schema") source_snapshot_ref: str = Field(min_length=1) source_snapshot_digest: str = Field(pattern=DIGEST_PATTERN) expected_graph_revision: str | None = Field(default=None, pattern=DIGEST_PATTERN) records: list[IdeaPortfolioSyncRecord] = Field(min_length=1) @model_validator(mode="after") - def validate_records(self) -> "IdeaPortfolioHydrationEnvelope": + def validate_records(self) -> Self: idea_ids = [record.resolved_idea_id for record in self.records] if len(idea_ids) != len(set(idea_ids)): raise ValueError("hydration envelope may contain at most one record per idea_id") @@ -223,223 +202,147 @@ def _canonical_json(value: Any) -> str: def _sha256_text(value: str) -> str: - return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + return "sha256:" + hashlib.sha256(value.encode()).hexdigest() def _canonical_key(value: str) -> str: - """Preserve source semantics while removing encoding/edge whitespace noise.""" - return unicodedata.normalize("NFC", value).strip() def _facet_id(kind: AssertionKind | str, key: str) -> str: kind_value = kind.value if isinstance(kind, AssertionKind) else kind - digest = hashlib.sha256(f"{kind_value}\x00{_canonical_key(key)}".encode()).hexdigest() - return f"facet:{digest}" + return "facet:" + hashlib.sha256(f"{kind_value}\x00{_canonical_key(key)}".encode()).hexdigest() def _assertion_id(idea_id: str, assertion: IdeaAssertion) -> str: - semantic_key = ( - f"{idea_id}\x00{assertion.relation.value}\x00{assertion.kind.value}" - f"\x00{_canonical_key(assertion.key)}" - ) - return "assertion:" + hashlib.sha256(semantic_key.encode()).hexdigest() + value = f"{idea_id}\x00{assertion.relation.value}\x00{assertion.kind.value}\x00{_canonical_key(assertion.key)}" + return "assertion:" + hashlib.sha256(value.encode()).hexdigest() def projection_digest(projection: IdeaGraphProjection) -> str: - return _sha256_text(_canonical_json(projection.model_dump(mode="json"))) + return _sha256_text(_canonical_json(projection.model_dump(mode="json", by_alias=True))) def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertion]: - compiled: list[CompiledAssertion] = [] - for assertion in projection.assertions: - compiled.append( - CompiledAssertion( - assertion_id=_assertion_id(projection.idea_id, assertion), - facet_id=_facet_id(assertion.kind, assertion.key), - kind=assertion.kind.value, - key=_canonical_key(assertion.key), - relation=assertion.relation.value, - evidence_state=assertion.evidence_state.value, - source_refs_json=_canonical_json(sorted(assertion.source_refs)), - ) + return [ + CompiledAssertion( + assertion_id=_assertion_id(projection.idea_id, assertion), + facet_id=_facet_id(assertion.kind, assertion.key), + kind=assertion.kind.value, + key=_canonical_key(assertion.key), + relation=assertion.relation.value, + evidence_state=assertion.evidence_state.value, + source_refs_json=_canonical_json(sorted(assertion.source_refs)), ) - return compiled + for assertion in projection.assertions + ] def build_portfolio_match_query(projection: IdeaGraphProjection | dict[str, Any]) -> dict[str, Any]: - """Compile a projection into the flat query shape consumed by the current match handler.""" - + """Compile only source-backed rank-eligible assertions into match input.""" model = projection if isinstance(projection, IdeaGraphProjection) else IdeaGraphProjection.model_validate(projection) - compiled = compile_assertions(model) - by_relation: dict[str, list[str]] = {relation.value: [] for relation in AssertionRelation} - for assertion in compiled: - by_relation[assertion.relation].append(assertion.facet_id) + by_relation = {relation.value: [] for relation in AssertionRelation} + for raw, compiled in zip(model.assertions, compile_assertions(model), strict=True): + if raw.evidence_state in _RANK_ELIGIBLE and raw.source_refs: + by_relation[compiled.relation].append(compiled.facet_id) + + def ids(relation: AssertionRelation) -> list[str]: + return sorted(set(by_relation[relation.value])) - def encoded(relation: str) -> str: - ids = sorted(set(by_relation[relation])) - return "" if not ids else "|" + "|".join(ids) + "|" + def encoded(relation: AssertionRelation) -> str: + values = ids(relation) + return "" if not values else "|" + "|".join(values) + "|" return { "idea_id": model.idea_id, - "requires_facets": encoded(AssertionRelation.REQUIRES.value), - "requires_count": len(set(by_relation[AssertionRelation.REQUIRES.value])), - "produces_facets": encoded(AssertionRelation.PRODUCES.value), - "produces_count": len(set(by_relation[AssertionRelation.PRODUCES.value])), - "uses_facets": encoded(AssertionRelation.USES.value), - "uses_count": len(set(by_relation[AssertionRelation.USES.value])), - "targets_facets": encoded(AssertionRelation.TARGETS.value), - "targets_count": len(set(by_relation[AssertionRelation.TARGETS.value])), - "depends_on_facets": encoded(AssertionRelation.DEPENDS_ON.value), + "requires_facets": encoded(AssertionRelation.REQUIRES), + "requires_count": len(ids(AssertionRelation.REQUIRES)), + "produces_facets": encoded(AssertionRelation.PRODUCES), + "produces_count": len(ids(AssertionRelation.PRODUCES)), + "uses_facets": encoded(AssertionRelation.USES), + "uses_count": len(ids(AssertionRelation.USES)), + "targets_facets": encoded(AssertionRelation.TARGETS), + "targets_count": len(ids(AssertionRelation.TARGETS)), + "depends_on_facets": encoded(AssertionRelation.DEPENDS_ON), "self_dependency_facet_id": _facet_id(AssertionKind.DEPENDENCY, model.idea_id), } def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> HydrationPlan: - model = ( - envelope - if isinstance(envelope, IdeaPortfolioHydrationEnvelope) - else IdeaPortfolioHydrationEnvelope.model_validate(envelope) - ) - records_payload = [record.model_dump(mode="json") for record in model.records] - batch_digest = _sha256_text(_canonical_json(records_payload)) + model = envelope if isinstance(envelope, IdeaPortfolioHydrationEnvelope) else IdeaPortfolioHydrationEnvelope.model_validate(envelope) + payload = [record.model_dump(mode="json", by_alias=True) for record in model.records] + batch_digest = _sha256_text(_canonical_json(payload)) parent = model.expected_graph_revision or "GENESIS" - graph_revision = _sha256_text( - f"ceg.idea-portfolio-graph/v1\x00{parent}\x00{model.source_snapshot_digest}\x00{batch_digest}" - ) - return HydrationPlan(envelope=model, batch_digest=batch_digest, graph_revision=graph_revision) + revision = _sha256_text(f"ceg.idea-portfolio-graph/v1\x00{parent}\x00{model.source_snapshot_digest}\x00{batch_digest}") + return HydrationPlan(model, batch_digest, revision) -def compile_upsert_command(projection: IdeaGraphProjection, *, graph_revision: str) -> WriteCommand: - p_digest = projection_digest(projection) - grouped: dict[str, list[dict[str, Any]]] = {relation.value: [] for relation in AssertionRelation} - for assertion in compile_assertions(projection): - grouped[assertion.relation].append( - { - "assertion_id": assertion.assertion_id, - "facet_id": assertion.facet_id, - "kind": assertion.kind, - "key": assertion.key, - "evidence_state": assertion.evidence_state, - "source_refs_json": assertion.source_refs_json, - } - ) - - cypher = """ +_UPSERT_CYPHER = """ MERGE (idea:Idea {idea_id: $idea_id}) -SET idea.source_digest = $source_digest, - idea.projection_digest = $projection_digest, - idea.graph_revision = $graph_revision, - idea.lifecycle_stage = $lifecycle_stage, - idea.decision = $decision, - idea.proof_state = $proof_state, - idea.execution_state = $execution_state, - idea.unknowns_json = $unknowns_json, - idea.self_dependency_facet_id = $self_dependency_facet_id, - idea.active = true, - idea.hydrated_at = datetime(), - idea.tombstoned_at = null, - idea._tenant = $tenant +SET idea.source_digest=$source_digest, idea.projection_digest=$projection_digest, + idea.graph_revision=$graph_revision, idea.lifecycle_stage=$lifecycle_stage, + idea.decision=$decision, idea.proof_state=$proof_state, idea.execution_state=$execution_state, + idea.unknowns_json=$unknowns_json, idea.self_dependency_facet_id=$self_dependency_facet_id, + idea.active=true, idea.hydrated_at=datetime(), idea.tombstoned_at=null, idea._tenant=$tenant WITH idea OPTIONAL MATCH (idea)-[old:PRODUCES|REQUIRES|TARGETS|USES|DEPENDS_ON]->(:PortfolioFacet) DELETE old WITH DISTINCT idea -CALL { - WITH idea - UNWIND $produces AS row +FOREACH (row IN $produces | MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) - SET facet.kind = row.kind, - facet.key = row.key, - facet.last_seen_revision = $graph_revision, - facet._tenant = $tenant + SET facet.kind=row.kind, facet.key=row.key, facet.last_seen_revision=$graph_revision, facet._tenant=$tenant MERGE (idea)-[rel:PRODUCES]->(facet) - SET rel.assertion_id = row.assertion_id, - rel.kind = row.kind, - rel.evidence_state = row.evidence_state, - rel.source_refs_json = row.source_refs_json, - rel.projection_digest = $projection_digest, - rel.graph_revision = $graph_revision - RETURN count(row) AS produced_count -} -CALL { - WITH idea - UNWIND $requires AS row + SET rel.assertion_id=row.assertion_id, rel.kind=row.kind, rel.evidence_state=row.evidence_state, + rel.source_refs_json=row.source_refs_json, rel.projection_digest=$projection_digest, rel.graph_revision=$graph_revision) +FOREACH (row IN $requires | MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) - SET facet.kind = row.kind, - facet.key = row.key, - facet.last_seen_revision = $graph_revision, - facet._tenant = $tenant + SET facet.kind=row.kind, facet.key=row.key, facet.last_seen_revision=$graph_revision, facet._tenant=$tenant MERGE (idea)-[rel:REQUIRES]->(facet) - SET rel.assertion_id = row.assertion_id, - rel.kind = row.kind, - rel.evidence_state = row.evidence_state, - rel.source_refs_json = row.source_refs_json, - rel.projection_digest = $projection_digest, - rel.graph_revision = $graph_revision - RETURN count(row) AS required_count -} -CALL { - WITH idea - UNWIND $targets AS row + SET rel.assertion_id=row.assertion_id, rel.kind=row.kind, rel.evidence_state=row.evidence_state, + rel.source_refs_json=row.source_refs_json, rel.projection_digest=$projection_digest, rel.graph_revision=$graph_revision) +FOREACH (row IN $targets | MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) - SET facet.kind = row.kind, - facet.key = row.key, - facet.last_seen_revision = $graph_revision, - facet._tenant = $tenant + SET facet.kind=row.kind, facet.key=row.key, facet.last_seen_revision=$graph_revision, facet._tenant=$tenant MERGE (idea)-[rel:TARGETS]->(facet) - SET rel.assertion_id = row.assertion_id, - rel.kind = row.kind, - rel.evidence_state = row.evidence_state, - rel.source_refs_json = row.source_refs_json, - rel.projection_digest = $projection_digest, - rel.graph_revision = $graph_revision - RETURN count(row) AS target_count -} -CALL { - WITH idea - UNWIND $uses AS row + SET rel.assertion_id=row.assertion_id, rel.kind=row.kind, rel.evidence_state=row.evidence_state, + rel.source_refs_json=row.source_refs_json, rel.projection_digest=$projection_digest, rel.graph_revision=$graph_revision) +FOREACH (row IN $uses | MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) - SET facet.kind = row.kind, - facet.key = row.key, - facet.last_seen_revision = $graph_revision, - facet._tenant = $tenant + SET facet.kind=row.kind, facet.key=row.key, facet.last_seen_revision=$graph_revision, facet._tenant=$tenant MERGE (idea)-[rel:USES]->(facet) - SET rel.assertion_id = row.assertion_id, - rel.kind = row.kind, - rel.evidence_state = row.evidence_state, - rel.source_refs_json = row.source_refs_json, - rel.projection_digest = $projection_digest, - rel.graph_revision = $graph_revision - RETURN count(row) AS use_count -} -CALL { - WITH idea - UNWIND $depends_on AS row + SET rel.assertion_id=row.assertion_id, rel.kind=row.kind, rel.evidence_state=row.evidence_state, + rel.source_refs_json=row.source_refs_json, rel.projection_digest=$projection_digest, rel.graph_revision=$graph_revision) +FOREACH (row IN $depends_on | MERGE (facet:PortfolioFacet {facet_id: row.facet_id}) - SET facet.kind = row.kind, - facet.key = row.key, - facet.last_seen_revision = $graph_revision, - facet._tenant = $tenant + SET facet.kind=row.kind, facet.key=row.key, facet.last_seen_revision=$graph_revision, facet._tenant=$tenant MERGE (idea)-[rel:DEPENDS_ON]->(facet) - SET rel.assertion_id = row.assertion_id, - rel.kind = row.kind, - rel.evidence_state = row.evidence_state, - rel.source_refs_json = row.source_refs_json, - rel.projection_digest = $projection_digest, - rel.graph_revision = $graph_revision - RETURN count(row) AS dependency_count -} + SET rel.assertion_id=row.assertion_id, rel.kind=row.kind, rel.evidence_state=row.evidence_state, + rel.source_refs_json=row.source_refs_json, rel.projection_digest=$projection_digest, rel.graph_revision=$graph_revision) RETURN idea.idea_id AS idea_id, - produced_count + required_count + target_count + use_count + dependency_count AS assertion_count + size($produces)+size($requires)+size($targets)+size($uses)+size($depends_on) AS assertion_count """.strip() + +def compile_upsert_command(projection: IdeaGraphProjection, *, graph_revision: str) -> WriteCommand: + grouped = {relation.value: [] for relation in AssertionRelation} + for assertion in compile_assertions(projection): + grouped[assertion.relation].append( + { + "assertion_id": assertion.assertion_id, + "facet_id": assertion.facet_id, + "kind": assertion.kind, + "key": assertion.key, + "evidence_state": assertion.evidence_state, + "source_refs_json": assertion.source_refs_json, + } + ) return WriteCommand( - cypher=cypher, - parameters={ + _UPSERT_CYPHER, + { "tenant": DOMAIN_ID, "idea_id": projection.idea_id, "source_digest": projection.source_digest, - "projection_digest": p_digest, + "projection_digest": projection_digest(projection), "graph_revision": graph_revision, "lifecycle_stage": projection.lifecycle.stage, "decision": projection.lifecycle.decision, @@ -447,101 +350,69 @@ def compile_upsert_command(projection: IdeaGraphProjection, *, graph_revision: s "execution_state": projection.lifecycle.execution_state, "unknowns_json": _canonical_json(sorted(projection.unknowns)), "self_dependency_facet_id": _facet_id(AssertionKind.DEPENDENCY, projection.idea_id), - "produces": grouped[AssertionRelation.PRODUCES.value], - "requires": grouped[AssertionRelation.REQUIRES.value], - "targets": grouped[AssertionRelation.TARGETS.value], - "uses": grouped[AssertionRelation.USES.value], - "depends_on": grouped[AssertionRelation.DEPENDS_ON.value], + **grouped, }, ) def compile_tombstone_command(idea_id: str, *, graph_revision: str) -> WriteCommand: - cypher = """ -MERGE (idea:Idea {idea_id: $idea_id}) -SET idea.active = false, - idea.graph_revision = $graph_revision, - idea.tombstoned_at = datetime(), - idea._tenant = $tenant -WITH idea -OPTIONAL MATCH (idea)-[old:PRODUCES|REQUIRES|TARGETS|USES|DEPENDS_ON]->(:PortfolioFacet) -DELETE old -RETURN idea.idea_id AS idea_id -""".strip() return WriteCommand( - cypher=cypher, - parameters={"tenant": DOMAIN_ID, "idea_id": idea_id, "graph_revision": graph_revision}, + """MERGE (idea:Idea {idea_id: $idea_id}) +SET idea.active=false, idea.graph_revision=$graph_revision, idea.tombstoned_at=datetime(), idea._tenant=$tenant +WITH idea OPTIONAL MATCH (idea)-[old:PRODUCES|REQUIRES|TARGETS|USES|DEPENDS_ON]->(:PortfolioFacet) +DELETE old RETURN idea.idea_id AS idea_id""", + {"tenant": DOMAIN_ID, "idea_id": idea_id, "graph_revision": graph_revision}, ) -_LOCK_STATE_CYPHER = """ -MERGE (state:IdeaPortfolioHydrationState {state_id: $state_id}) -SET state._cas_lock = coalesce(state._cas_lock, 0) + 1, - state._tenant = $tenant -RETURN state.current_revision AS current_revision -""".strip() - -_FINALIZE_STATE_CYPHER = """ -MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) -SET state.current_revision = $graph_revision, - state.source_snapshot_ref = $source_snapshot_ref, - state.source_snapshot_digest = $source_snapshot_digest, - state.batch_digest = $batch_digest, - state.completed_at = datetime(), - state._tenant = $tenant -RETURN state.current_revision AS graph_revision -""".strip() +_LOCK_STATE_CYPHER = """MERGE (state:IdeaPortfolioHydrationState {state_id: $state_id}) +SET state._cas_lock=coalesce(state._cas_lock, 0)+1, state._tenant=$tenant +RETURN state.current_revision AS current_revision""" +_FINALIZE_STATE_CYPHER = """MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) +SET state.current_revision=$graph_revision, state.source_snapshot_ref=$source_snapshot_ref, + state.source_snapshot_digest=$source_snapshot_digest, state.batch_digest=$batch_digest, + state.completed_at=datetime(), state._tenant=$tenant +RETURN state.current_revision AS graph_revision""" class IdeaPortfolioHydrator: - """Atomically apply one revision-chained IdeaOS corpus delta to CEG.""" + """Apply one revision-chained corpus delta in one managed Neo4j transaction.""" - def __init__(self, graph_writer: GraphWriter) -> None: + def __init__(self, graph_writer: GraphWriter, *, enabled: bool = False) -> None: self.graph_writer = graph_writer + self.enabled = enabled async def apply(self, envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> dict[str, Any]: + if not self.enabled: + raise IdeaPortfolioHydrationError("idea-portfolio hydration is disabled by configuration") plan = compile_hydration_plan(envelope) async def apply_transaction(tx: Any) -> dict[str, Any]: - state_result = await tx.run( - _LOCK_STATE_CYPHER, - { - "state_id": _STATE_ID, - "tenant": DOMAIN_ID, - }, - ) - state_rows = await state_result.data() - current_revision = state_rows[0].get("current_revision") if state_rows else None - - if current_revision == plan.graph_revision: - return self._receipt(plan, status="reused", applied=[], tombstoned=[]) - - if current_revision != plan.envelope.expected_graph_revision: - raise IdeaPortfolioHydrationError( - "hydration revision conflict: expected parent does not match committed graph revision" - ) + state = await tx.run(_LOCK_STATE_CYPHER, {"state_id": _STATE_ID, "tenant": DOMAIN_ID}) + rows = await state.data() + current = rows[0].get("current_revision") if rows else None + if current == plan.graph_revision: + return self._receipt(plan, "reused", [], []) + if current != plan.envelope.expected_graph_revision: + raise IdeaPortfolioHydrationError("hydration revision conflict: expected parent does not match committed graph revision") applied: list[str] = [] tombstoned: list[str] = [] for record in plan.envelope.records: if record.operation == "upsert": - projection = record.projection - if projection is None: - raise IdeaPortfolioHydrationError("validated upsert record lacks projection") - command = compile_upsert_command(projection, graph_revision=plan.graph_revision) + if record.projection is None: + raise IdeaPortfolioHydrationError("validated upsert lacks projection") + command = compile_upsert_command(record.projection, graph_revision=plan.graph_revision) result = await tx.run(command.cypher, command.parameters) await result.consume() - applied.append(projection.idea_id) + applied.append(record.projection.idea_id) else: - command = compile_tombstone_command( - record.resolved_idea_id, - graph_revision=plan.graph_revision, - ) + command = compile_tombstone_command(record.resolved_idea_id, graph_revision=plan.graph_revision) result = await tx.run(command.cypher, command.parameters) await result.consume() tombstoned.append(record.resolved_idea_id) - final_result = await tx.run( + final = await tx.run( _FINALIZE_STATE_CYPHER, { "state_id": _STATE_ID, @@ -552,13 +423,8 @@ async def apply_transaction(tx: Any) -> dict[str, Any]: "batch_digest": plan.batch_digest, }, ) - await final_result.consume() - return self._receipt( - plan, - status="applied", - applied=applied, - tombstoned=tombstoned, - ) + await final.consume() + return self._receipt(plan, "applied", applied, tombstoned) result = await self.graph_writer.execute_write(apply_transaction, database=DOMAIN_ID) if not isinstance(result, dict): @@ -568,7 +434,6 @@ async def apply_transaction(tx: Any) -> dict[str, Any]: @staticmethod def _receipt( plan: HydrationPlan, - *, status: Literal["applied", "reused"], applied: list[str], tombstoned: list[str], @@ -587,9 +452,9 @@ def _receipt( __all__ = [ + "DOMAIN_ID", "AssertionKind", "AssertionRelation", - "DOMAIN_ID", "EvidenceState", "HydrationPlan", "IdeaGraphProjection", From c6846d924020c44f1f769f4ff2b6773ae2848a58 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:35:29 -0400 Subject: [PATCH 09/32] chore: compact idea portfolio spec without changing semantics --- domains/idea-portfolio/spec.yaml | 247 ++++++------------------------- 1 file changed, 48 insertions(+), 199 deletions(-) diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml index 5bdb35cd..3a8c6a4b 100644 --- a/domains/idea-portfolio/spec.yaml +++ b/domains/idea-portfolio/spec.yaml @@ -11,10 +11,7 @@ domain: id: idea-portfolio name: IdeaOS Portfolio Graph - description: > - Cross-idea portfolio intelligence for IdeaOS. IdeaOS remains authoritative - for idea identity and lifecycle truth. CEG owns graph intersections, - leverage matching, portfolio ranking, and graph-derived signals. + description: Cross-idea portfolio intelligence. IdeaOS owns idea/lifecycle truth; CEG owns graph persistence, intersections, and ranking. version: 0.1.0 ontology: @@ -36,13 +33,10 @@ ontology: - {name: active, type: bool} - {name: hydrated_at, type: datetime} - {name: tombstoned_at, type: datetime} - - label: IdeaQuery managedby: api queryentity: true - properties: - - {name: idea_id, type: string} - + properties: [{name: idea_id, type: string}] - label: PortfolioFacet managedby: sync auxiliary: true @@ -51,7 +45,6 @@ ontology: - {name: kind, type: enum, values: [capability, substrate, proof_asset, data_asset, market, customer_type, dependency]} - {name: key, type: string, required: true} - {name: last_seen_revision, type: string} - - label: IdeaPortfolioHydrationState managedby: sync auxiliary: true @@ -62,9 +55,9 @@ ontology: - {name: source_snapshot_digest, type: string} - {name: batch_digest, type: string} - {name: completed_at, type: datetime} - edges: - - type: PRODUCES + - &source_edge + type: PRODUCES from: Idea to: PortfolioFacet direction: DIRECTED @@ -77,69 +70,20 @@ ontology: - {name: source_refs_json, type: string} - {name: projection_digest, type: string} - {name: graph_revision, type: string} - - - type: REQUIRES - from: Idea - to: PortfolioFacet - direction: DIRECTED - category: capability - managedby: sync - properties: - - {name: assertion_id, type: string, required: true} - - {name: kind, type: string} - - {name: evidence_state, type: string} - - {name: source_refs_json, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} - - - type: TARGETS - from: Idea - to: PortfolioFacet - direction: DIRECTED + - <<: *source_edge + type: REQUIRES + - <<: *source_edge + type: TARGETS category: market - managedby: sync - properties: - - {name: assertion_id, type: string, required: true} - - {name: kind, type: string} - - {name: evidence_state, type: string} - - {name: source_refs_json, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} - - - type: USES - from: Idea - to: PortfolioFacet - direction: DIRECTED - category: capability - managedby: sync - properties: - - {name: assertion_id, type: string, required: true} - - {name: kind, type: string} - - {name: evidence_state, type: string} - - {name: source_refs_json, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} - - - type: DEPENDS_ON - from: Idea - to: PortfolioFacet - direction: DIRECTED + - <<: *source_edge + type: USES + - <<: *source_edge + type: DEPENDS_ON category: context - managedby: sync - properties: - - {name: assertion_id, type: string, required: true} - - {name: kind, type: string} - - {name: evidence_state, type: string} - - {name: source_refs_json, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} matchentities: - candidate: - - {label: Idea, matchdirection: portfolio_context_for_idea} - queryentity: - - {label: IdeaQuery, matchdirection: portfolio_context_for_idea} - + candidate: [{label: Idea, matchdirection: portfolio_context_for_idea}] + queryentity: [{label: IdeaQuery, matchdirection: portfolio_context_for_idea}] queryschema: matchdirections: [portfolio_context_for_idea] fields: @@ -154,180 +98,85 @@ queryschema: - {name: targets_count, type: int, default: 0} - {name: depends_on_facets, type: string, default: ""} - {name: self_dependency_facet_id, type: string, default: ""} - -traversal: - steps: [] - +traversal: {steps: []} gates: - - name: active_only - type: boolean - candidateprop: active - nullbehavior: fail - matchdirections: [portfolio_context_for_idea] + - {name: active_only, type: boolean, candidateprop: active, nullbehavior: fail, matchdirections: [portfolio_context_for_idea]} + - {name: exclude_self, type: threshold, candidateprop: idea_id, queryparam: idea_id, operator: "!=", nullbehavior: fail, matchdirections: [portfolio_context_for_idea]} - - name: exclude_self - type: threshold - candidateprop: idea_id - queryparam: idea_id - operator: "!=" - nullbehavior: fail - matchdirections: [portfolio_context_for_idea] - -# Only source-backed VERIFIED and SUPPORTED_INFERENCE assertions may influence -# portfolio rank. HYPOTHESIS and UNKNOWN assertions remain in the graph for -# context, but cannot be silently laundered into numeric ranking evidence. +# VERIFIED and source-backed SUPPORTED_INFERENCE assertions can rank. +# HYPOTHESIS/UNKNOWN remain graph context and cannot become rank evidence. scoring: dimensions: - name: incoming_requirement_fit source: computed computation: customcypher - expression: > - CASE WHEN $requires_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[rel:PRODUCES]->(f:PortfolioFacet) - WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] - AND rel.source_refs_json <> '[]' - AND $requires_facets CONTAINS ('|' + f.facet_id + '|') | f])) / - toFloat($requires_count) END + expression: >- + CASE WHEN $requires_count <= 0 THEN 0.0 ELSE toFloat(size([(candidate)-[rel:PRODUCES]->(f:PortfolioFacet) WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] AND rel.source_refs_json <> '[]' AND $requires_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($requires_count) END weightkey: wincoming defaultweight: 0.25 matchdirections: [portfolio_context_for_idea] - - name: outgoing_requirement_fit source: computed computation: customcypher - expression: > - CASE WHEN $produces_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[rel:REQUIRES]->(f:PortfolioFacet) - WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] - AND rel.source_refs_json <> '[]' - AND $produces_facets CONTAINS ('|' + f.facet_id + '|') | f])) / - toFloat($produces_count) END + expression: >- + CASE WHEN $produces_count <= 0 THEN 0.0 ELSE toFloat(size([(candidate)-[rel:REQUIRES]->(f:PortfolioFacet) WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] AND rel.source_refs_json <> '[]' AND $produces_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($produces_count) END weightkey: woutgoing defaultweight: 0.25 matchdirections: [portfolio_context_for_idea] - - name: shared_usage source: computed computation: customcypher - expression: > - CASE WHEN $uses_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[rel:USES]->(f:PortfolioFacet) - WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] - AND rel.source_refs_json <> '[]' - AND $uses_facets CONTAINS ('|' + f.facet_id + '|') | f])) / - toFloat($uses_count) END + expression: >- + CASE WHEN $uses_count <= 0 THEN 0.0 ELSE toFloat(size([(candidate)-[rel:USES]->(f:PortfolioFacet) WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] AND rel.source_refs_json <> '[]' AND $uses_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($uses_count) END weightkey: wusage defaultweight: 0.20 matchdirections: [portfolio_context_for_idea] - - name: shared_target source: computed computation: customcypher - expression: > - CASE WHEN $targets_count <= 0 THEN 0.0 ELSE - toFloat(size([(candidate)-[rel:TARGETS]->(f:PortfolioFacet) - WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] - AND rel.source_refs_json <> '[]' - AND $targets_facets CONTAINS ('|' + f.facet_id + '|') | f])) / - toFloat($targets_count) END + expression: >- + CASE WHEN $targets_count <= 0 THEN 0.0 ELSE toFloat(size([(candidate)-[rel:TARGETS]->(f:PortfolioFacet) WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] AND rel.source_refs_json <> '[]' AND $targets_facets CONTAINS ('|' + f.facet_id + '|') | f])) / toFloat($targets_count) END weightkey: wtarget defaultweight: 0.15 matchdirections: [portfolio_context_for_idea] - - name: query_depends_on_candidate source: computed computation: customcypher - expression: > - CASE WHEN candidate.self_dependency_facet_id IS NOT NULL AND - $depends_on_facets CONTAINS ('|' + candidate.self_dependency_facet_id + '|') - THEN 1.0 ELSE 0.0 END + expression: >- + CASE WHEN candidate.self_dependency_facet_id IS NOT NULL AND $depends_on_facets CONTAINS ('|' + candidate.self_dependency_facet_id + '|') THEN 1.0 ELSE 0.0 END weightkey: wquerydependency defaultweight: 0.10 matchdirections: [portfolio_context_for_idea] - - name: candidate_depends_on_query source: computed computation: customcypher - expression: > - CASE WHEN $self_dependency_facet_id = '' THEN 0.0 ELSE - CASE WHEN size([(candidate)-[rel:DEPENDS_ON]->(f:PortfolioFacet) - WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] - AND rel.source_refs_json <> '[]' - AND f.facet_id = $self_dependency_facet_id | f]) > 0 - THEN 1.0 ELSE 0.0 END END + expression: >- + CASE WHEN $self_dependency_facet_id = '' THEN 0.0 ELSE CASE WHEN size([(candidate)-[rel:DEPENDS_ON]->(f:PortfolioFacet) WHERE rel.evidence_state IN ['VERIFIED', 'SUPPORTED_INFERENCE'] AND rel.source_refs_json <> '[]' AND f.facet_id = $self_dependency_facet_id | f]) > 0 THEN 1.0 ELSE 0.0 END END weightkey: wcandidatedependency defaultweight: 0.05 matchdirections: [portfolio_context_for_idea] -# The live generic sync handler is intentionally not advertised for this -# domain yet. Corpus hydration uses engine/sync/idea_portfolio.py through the -# explicit admin CLI until the Gate-facing sync contract is added and tested. -sync: - endpoints: [] - -# Centrality is intentionally not declared yet. The current CEG scheduler does -# not implement PageRank, and declaring a skipped algorithm would create false -# capability. Community and learned-weight jobs can be added only when a proof -# requirement justifies them. +# Generic SyncGenerator cannot preserve assertion provenance/revision semantics. +sync: {endpoints: []} gdsjobs: [] - compliance: enabled: true - audit: - enabled: true - logallmatches: true - logretentiondays: 365 - pii: - enabled: false - prohibitedfactors: - enabled: true - blockedfields: [] - enforcement: compiletime + audit: {enabled: true, logallmatches: true, logretentiondays: 365} + pii: {enabled: false} + prohibitedfactors: {enabled: true, blockedfields: [], enforcement: compiletime} regionalrules: [] counterfactualaudit: false - capabilities: - - name: portfolio_read - actions: [match:read] - allowed_subjects: ["*"] - -feedbackloop: - enabled: false - -causal: - enabled: false - -counterfactual: - enabled: false - -semantic_registry: - enabled: false - -decision_arbitration: - enabled: false - + - {name: portfolio_read, actions: [match:read], allowed_subjects: ["*"]} +feedbackloop: {enabled: false} +causal: {enabled: false} +counterfactual: {enabled: false} +semantic_registry: {enabled: false} +decision_arbitration: {enabled: false} feature_catalog: - - feature_id: incoming_requirement_fit - owner: ceg - scoring_dimension: incoming_requirement_fit - evidence_required: true - - feature_id: outgoing_requirement_fit - owner: ceg - scoring_dimension: outgoing_requirement_fit - evidence_required: true - - feature_id: shared_usage - owner: ceg - scoring_dimension: shared_usage - evidence_required: true - - feature_id: shared_target - owner: ceg - scoring_dimension: shared_target - evidence_required: true - - feature_id: query_depends_on_candidate - owner: ceg - scoring_dimension: query_depends_on_candidate - evidence_required: true - - feature_id: candidate_depends_on_query - owner: ceg - scoring_dimension: candidate_depends_on_query - evidence_required: true + - {feature_id: incoming_requirement_fit, owner: ceg, scoring_dimension: incoming_requirement_fit, evidence_required: true} + - {feature_id: outgoing_requirement_fit, owner: ceg, scoring_dimension: outgoing_requirement_fit, evidence_required: true} + - {feature_id: shared_usage, owner: ceg, scoring_dimension: shared_usage, evidence_required: true} + - {feature_id: shared_target, owner: ceg, scoring_dimension: shared_target, evidence_required: true} + - {feature_id: query_depends_on_candidate, owner: ceg, scoring_dimension: query_depends_on_candidate, evidence_required: true} + - {feature_id: candidate_depends_on_query, owner: ceg, scoring_dimension: candidate_depends_on_query, evidence_required: true} From eba20377dcd4aa2e62a8e4170eede68452494619 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:35:52 -0400 Subject: [PATCH 10/32] test: cover idea portfolio admission and epistemic ranking --- tests/unit/test_idea_portfolio.py | 269 +++++++++--------------------- 1 file changed, 82 insertions(+), 187 deletions(-) diff --git a/tests/unit/test_idea_portfolio.py b/tests/unit/test_idea_portfolio.py index c53b13b5..4e4d752d 100644 --- a/tests/unit/test_idea_portfolio.py +++ b/tests/unit/test_idea_portfolio.py @@ -5,7 +5,8 @@ import pytest -from engine.config.loader import DomainPackLoader +from engine.config.loader import DomainNotFoundError, DomainPackLoader +from engine.config.settings import settings from engine.gates.compiler import GateCompiler from engine.scoring.assembler import ScoringAssembler from engine.sync.idea_portfolio import ( @@ -26,59 +27,28 @@ def _digest(char: str = "a") -> str: def _projection(idea_id: str = "idea-alpha") -> dict[str, Any]: + def assertion(kind: str, relation: str, key: str, state: str) -> dict[str, Any]: + refs = [] if state == "UNKNOWN" else [f"Ideas/{idea_id}.md#{key}"] + return {"kind": kind, "relation": relation, "key": key, "evidence_state": state, "source_refs": refs} + return { "schema": "ideaos.idea-graph-projection/v1", "idea_id": idea_id, "source_refs": [f"Ideas/{idea_id}.md"], - "source_digest": _digest("a"), - "lifecycle": { - "stage": "expanded", - "decision": None, - "proof_state": "P1", - "execution_state": None, - }, + "source_digest": _digest(), + "lifecycle": {"stage": "expanded", "decision": None, "proof_state": "P1", "execution_state": None}, "assertions": [ - { - "kind": "capability", - "relation": "produces", - "key": "shared-capability", - "evidence_state": "VERIFIED", - "source_refs": [f"Ideas/{idea_id}.md#capability"], - }, - { - "kind": "capability", - "relation": "requires", - "key": "required-capability", - "evidence_state": "SUPPORTED_INFERENCE", - "source_refs": [f"Ideas/{idea_id}.md#requirement"], - }, - { - "kind": "substrate", - "relation": "uses", - "key": "shared-substrate", - "evidence_state": "VERIFIED", - "source_refs": [f"Ideas/{idea_id}.md#substrate"], - }, - { - "kind": "market", - "relation": "targets", - "key": "industrial-ai", - "evidence_state": "HYPOTHESIS", - "source_refs": [f"Ideas/{idea_id}.md#market"], - }, - { - "kind": "dependency", - "relation": "depends_on", - "key": "idea-foundation", - "evidence_state": "VERIFIED", - "source_refs": [f"Ideas/{idea_id}.md#dependency"], - }, + assertion("capability", "produces", "shared-capability", "VERIFIED"), + assertion("capability", "requires", "required-capability", "SUPPORTED_INFERENCE"), + assertion("substrate", "uses", "shared-substrate", "VERIFIED"), + assertion("market", "targets", "industrial-ai", "HYPOTHESIS"), + assertion("dependency", "depends_on", "idea-foundation", "VERIFIED"), ], "unknowns": ["external demand not yet proven"], } -def _envelope(*, expected: str | None = None) -> dict[str, Any]: +def _envelope(expected: str | None = None) -> dict[str, Any]: return { "schema": "ceg.idea-portfolio-hydration/v1", "source_snapshot_ref": "Quantum-L9/IdeaOS@deadbeef", @@ -95,187 +65,112 @@ def _envelope(*, expected: str | None = None) -> dict[str, Any]: @pytest.mark.unit -class TestIdeaPortfolioDomain: - def test_domain_loads_and_compilers_accept_it(self) -> None: - loader = DomainPackLoader(config_path=str(ROOT / "domains")) - spec = loader.load_domain("idea-portfolio") - - assert spec.domain.id == "idea-portfolio" - assert {node.label for node in spec.ontology.nodes} == { - "Idea", - "IdeaQuery", - "PortfolioFacet", - "IdeaPortfolioHydrationState", - } - assert {edge.type for edge in spec.ontology.edges} == { - "PRODUCES", - "REQUIRES", - "TARGETS", - "USES", - "DEPENDS_ON", - } - assert spec.sync.endpoints == [] - assert sum(d.defaultweight for d in spec.scoring.dimensions) == pytest.approx(1.0) - - gate_clause = GateCompiler(spec).compile_all_gates("portfolio_context_for_idea") - assert "candidate.active" in gate_clause - assert "candidate.idea_id != $idea_id" in gate_clause - - score_clause, _ = ScoringAssembler(spec).assemble_scoring_clause( - "portfolio_context_for_idea", {} - ) - assert "PRODUCES" in score_clause - assert "REQUIRES" in score_clause - assert "PortfolioFacet" in score_clause +def test_domain_is_dormant_until_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + loader = DomainPackLoader(config_path=str(ROOT / "domains")) + monkeypatch.setattr(settings, "idea_portfolio_enabled", False) + assert "idea-portfolio" not in loader.list_domains() + with pytest.raises(DomainNotFoundError, match="disabled"): + loader.load_domain("idea-portfolio") + + monkeypatch.setattr(settings, "idea_portfolio_enabled", True) + spec = loader.load_domain("idea-portfolio") + assert spec.domain.id == "idea-portfolio" + assert spec.sync.endpoints == [] + assert sum(d.defaultweight for d in spec.scoring.dimensions) == pytest.approx(1.0) + assert "candidate.active" in GateCompiler(spec).compile_all_gates("portfolio_context_for_idea") + scoring, _ = ScoringAssembler(spec).assemble_scoring_clause("portfolio_context_for_idea", {}) + assert "SUPPORTED_INFERENCE" in scoring + assert "rel.evidence_state" in scoring @pytest.mark.unit -class TestProjectionBoundary: - def test_projection_compiles_to_flat_match_query(self) -> None: - projection = IdeaGraphProjection.model_validate(_projection()) - query = build_portfolio_match_query(projection) - - assert query["idea_id"] == "idea-alpha" - assert query["requires_count"] == 1 - assert query["produces_count"] == 1 - assert query["uses_count"] == 1 - assert query["targets_count"] == 1 - assert query["depends_on_facets"].startswith("|facet:") - assert query["self_dependency_facet_id"].startswith("facet:") - - def test_invalid_kind_relation_is_rejected(self) -> None: - raw = _projection() - raw["assertions"][0]["kind"] = "market" - raw["assertions"][0]["relation"] = "produces" - - with pytest.raises(ValueError, match="not valid for assertion kind"): - IdeaGraphProjection.model_validate(raw) - - def test_non_unknown_assertion_requires_source_reference(self) -> None: - raw = _projection() - raw["assertions"][0]["source_refs"] = [] - - with pytest.raises(ValueError, match="require at least one source_ref"): - IdeaGraphProjection.model_validate(raw) - - def test_duplicate_semantic_assertion_is_rejected(self) -> None: - raw = _projection() - raw["assertions"].append(dict(raw["assertions"][0])) - - with pytest.raises(ValueError, match="duplicate semantic assertions"): - IdeaGraphProjection.model_validate(raw) - - def test_upsert_replaces_source_projection_edges(self) -> None: - projection = IdeaGraphProjection.model_validate(_projection()) - command = compile_upsert_command(projection, graph_revision=_digest("c")) +def test_projection_admission_and_rank_filtering() -> None: + model = IdeaGraphProjection.model_validate(_projection()) + assert model.model_dump(by_alias=True)["schema"] == "ideaos.idea-graph-projection/v1" + query = build_portfolio_match_query(model) + assert query["requires_count"] == query["produces_count"] == query["uses_count"] == 1 + assert query["targets_count"] == 0 + assert query["depends_on_facets"].startswith("|facet:") - assert "DELETE old" in command.cypher - assert "MERGE (idea)-[rel:PRODUCES]->(facet)" in command.cypher - assert "rel.evidence_state" in command.cypher - assert command.parameters["projection_digest"] == projection_digest(projection) - assert command.parameters["unknowns_json"] == '["external demand not yet proven"]' - assert command.parameters["tenant"] == "idea-portfolio" + raw = _projection() + raw["assertions"][0]["source_refs"] = [] + with pytest.raises(ValueError, match="source_ref"): + IdeaGraphProjection.model_validate(raw) @pytest.mark.unit -class TestHydrationRevision: - def test_revision_is_deterministic_and_parent_linked(self) -> None: - first = compile_hydration_plan(_envelope()) - again = compile_hydration_plan(_envelope()) - child = compile_hydration_plan(_envelope(expected=first.graph_revision)) +def test_upsert_preserves_all_assertions_and_wire_digest() -> None: + model = IdeaGraphProjection.model_validate(_projection()) + command = compile_upsert_command(model, graph_revision=_digest("c")) + assert "DELETE old" in command.cypher + assert command.parameters["projection_digest"] == projection_digest(model) + assert len(command.parameters["targets"]) == 1 + assert command.parameters["targets"][0]["evidence_state"] == "HYPOTHESIS" - assert first.batch_digest == again.batch_digest - assert first.graph_revision == again.graph_revision - assert child.graph_revision != first.graph_revision - -class _FakeResult: +class _Result: def __init__(self, rows: list[dict[str, Any]] | None = None) -> None: self.rows = rows or [] - self.consumed = False async def data(self) -> list[dict[str, Any]]: - return list(self.rows) + return self.rows async def consume(self) -> None: - self.consumed = True + return None -class _FakeTransaction: - def __init__(self, current_revision: str | None) -> None: - self.current_revision = current_revision - self.calls: list[dict[str, Any]] = [] +class _Tx: + def __init__(self, revision: str | None) -> None: + self.revision = revision + self.calls: list[str] = [] - async def run(self, cypher: str, parameters: dict[str, Any]) -> _FakeResult: - self.calls.append({"cypher": cypher, "parameters": parameters}) + async def run(self, cypher: str, parameters: dict[str, Any]) -> _Result: + self.calls.append(cypher) if "RETURN state.current_revision AS current_revision" in cypher: - return _FakeResult([{"current_revision": self.current_revision}]) - return _FakeResult() + return _Result([{"current_revision": self.revision}]) + return _Result() -class _FakeWriter: - def __init__(self, current_revision: str | None = None) -> None: - self.tx = _FakeTransaction(current_revision) - self.execute_write_calls = 0 +class _Writer: + def __init__(self, revision: str | None = None) -> None: + self.tx = _Tx(revision) + self.calls = 0 self.database: str | None = None async def execute_write( self, - transaction_function: Any = None, + fn: Any = None, *args: Any, - cypher: str | None = None, - parameters: dict[str, Any] | None = None, database: str | None = None, **kwargs: Any, ) -> dict[str, Any]: - self.execute_write_calls += 1 + self.calls += 1 self.database = database - if transaction_function is None: + if fn is None: raise AssertionError("hydrator must use one managed transaction") - return await transaction_function(self.tx, *args, **kwargs) + return await fn(self.tx, *args, **kwargs) @pytest.mark.unit @pytest.mark.asyncio -async def test_hydrator_applies_whole_envelope_in_one_transaction() -> None: - writer = _FakeWriter(current_revision=None) - hydrator = IdeaPortfolioHydrator(writer) - - receipt = await hydrator.apply(_envelope()) - +async def test_hydrator_feature_gate_and_atomic_revision_chain() -> None: + disabled = _Writer() + with pytest.raises(IdeaPortfolioHydrationError, match="disabled"): + await IdeaPortfolioHydrator(disabled).apply(_envelope()) + assert disabled.calls == 0 + + writer = _Writer() + receipt = await IdeaPortfolioHydrator(writer, enabled=True).apply(_envelope()) assert receipt["status"] == "applied" - assert receipt["applied"] == ["idea-alpha"] - assert writer.execute_write_calls == 1 - assert writer.database == "idea-portfolio" + assert writer.calls == 1 and writer.database == "idea-portfolio" assert len(writer.tx.calls) == 3 - assert "IdeaPortfolioHydrationState" in writer.tx.calls[0]["cypher"] - assert "MERGE (idea:Idea" in writer.tx.calls[1]["cypher"] - assert "state.current_revision" in writer.tx.calls[2]["cypher"] - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_hydrator_exact_revision_replay_is_noop() -> None: plan = compile_hydration_plan(_envelope()) - writer = _FakeWriter(current_revision=plan.graph_revision) - hydrator = IdeaPortfolioHydrator(writer) - - receipt = await hydrator.apply(_envelope()) - - assert receipt["status"] == "reused" - assert receipt["applied"] == [] - assert writer.execute_write_calls == 1 - assert len(writer.tx.calls) == 1 - - -@pytest.mark.unit -@pytest.mark.asyncio -async def test_hydrator_rejects_parent_revision_conflict_before_projection_write() -> None: - writer = _FakeWriter(current_revision=_digest("e")) - hydrator = IdeaPortfolioHydrator(writer) + replay = _Writer(plan.graph_revision) + receipt = await IdeaPortfolioHydrator(replay, enabled=True).apply(_envelope()) + assert receipt["status"] == "reused" and len(replay.tx.calls) == 1 + conflict = _Writer(_digest("e")) with pytest.raises(IdeaPortfolioHydrationError, match="expected parent"): - await hydrator.apply(_envelope()) - - assert writer.execute_write_calls == 1 - assert len(writer.tx.calls) == 1 + await IdeaPortfolioHydrator(conflict, enabled=True).apply(_envelope()) + assert len(conflict.tx.calls) == 1 From 2231eeae7e6af817b9f3668c49a2079c6ff0adf1 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:36:06 -0400 Subject: [PATCH 11/32] fix: enforce idea portfolio activation before graph writes --- tools/hydrate_idea_portfolio.py | 39 +++++++++++---------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index 190926a2..2e5aaf94 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -1,12 +1,5 @@ #!/usr/bin/env python3 -"""Validate or apply an IdeaOS portfolio hydration envelope to CEG. - -Dry-run is the default. ``--apply`` is required for graph mutation. - -Examples: - python tools/hydrate_idea_portfolio.py hydration.json - python tools/hydrate_idea_portfolio.py hydration.json --apply -""" +"""Validate or apply a revision-chained IdeaOS portfolio hydration envelope.""" from __future__ import annotations @@ -20,18 +13,19 @@ sys.path.insert(0, str(ROOT)) from engine.config.loader import DomainPackLoader +from engine.config.settings import settings from engine.graph.driver import GraphDriver from engine.sync.idea_portfolio import ( DOMAIN_ID, IdeaPortfolioHydrationEnvelope, + IdeaPortfolioHydrationError, IdeaPortfolioHydrator, compile_hydration_plan, ) -def _load_envelope(path: Path) -> IdeaPortfolioHydrationEnvelope: - raw = json.loads(path.read_text(encoding="utf-8")) - return IdeaPortfolioHydrationEnvelope.model_validate(raw) +def _load(path: Path) -> IdeaPortfolioHydrationEnvelope: + return IdeaPortfolioHydrationEnvelope.model_validate(json.loads(path.read_text(encoding="utf-8"))) def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: @@ -40,38 +34,31 @@ def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: "schema": "ceg.idea-portfolio-hydration-plan/v1", "status": "validated", "domain": DOMAIN_ID, - "source_snapshot_ref": envelope.source_snapshot_ref, - "source_snapshot_digest": envelope.source_snapshot_digest, "expected_graph_revision": envelope.expected_graph_revision, "batch_digest": plan.batch_digest, "graph_revision": plan.graph_revision, - "records": [ - {"idea_id": record.resolved_idea_id, "operation": record.operation} - for record in envelope.records - ], + "records": [{"idea_id": r.resolved_idea_id, "operation": r.operation} for r in envelope.records], } async def _apply(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: - # Loading through the production loader proves the folder-shaped domain pack - # is discoverable and validates against the current DomainSpec before writes. + if not settings.idea_portfolio_enabled: + raise IdeaPortfolioHydrationError("idea-portfolio hydration is disabled by configuration") DomainPackLoader(config_path=str(ROOT / "domains")).load_domain(DOMAIN_ID) - driver = GraphDriver() await driver.connect() try: - return await IdeaPortfolioHydrator(driver).apply(envelope) + return await IdeaPortfolioHydrator(driver, enabled=True).apply(envelope) finally: await driver.close() def main() -> None: - parser = argparse.ArgumentParser(description="Validate or apply an IdeaOS -> CEG portfolio hydration envelope") - parser.add_argument("envelope", type=Path, help="Path to ceg.idea-portfolio-hydration/v1 JSON") - parser.add_argument("--apply", action="store_true", help="Mutate the CEG idea-portfolio graph") + parser = argparse.ArgumentParser(description="Validate or apply IdeaOS -> CEG portfolio hydration") + parser.add_argument("envelope", type=Path) + parser.add_argument("--apply", action="store_true", help="mutate the CEG idea-portfolio graph") args = parser.parse_args() - - envelope = _load_envelope(args.envelope) + envelope = _load(args.envelope) result = asyncio.run(_apply(envelope)) if args.apply else _dry_run(envelope) print(json.dumps(result, indent=2, sort_keys=True)) From a48690b4eb5eae71dd51aa24f36c230f6d9f17e6 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:36:12 -0400 Subject: [PATCH 12/32] chore: keep idea portfolio change reviewable --- docs/idea-portfolio.md | 232 ----------------------------------------- 1 file changed, 232 deletions(-) delete mode 100644 docs/idea-portfolio.md diff --git a/docs/idea-portfolio.md b/docs/idea-portfolio.md deleted file mode 100644 index 53583247..00000000 --- a/docs/idea-portfolio.md +++ /dev/null @@ -1,232 +0,0 @@ -# IdeaOS Idea Portfolio Domain - -## Authority boundary - -`idea-portfolio` is the CEG-owned graph intelligence surface for the first-class IdeaOS corpus. - -- **IdeaOS owns:** idea identity, source lineage, lifecycle state, Dream / Invariant / Wedge / Proof, decisions, proof state, execution state, and semantic `IdeaGraphProjection` emission. -- **CEG owns:** graph persistence of that projection, cross-idea traversal, overlap/leverage matching, portfolio ranking, graph-derived relationships, communities, and learned graph signals. -- **Transport adapters own neither:** they may serialize, route, and translate only. - -CEG must not parse raw files from IdeaOS `Ideas/`, infer an idea from a filename, or become a second lifecycle source of truth. - -## Graph model - -The first implementation intentionally uses two semantic node classes plus one internal revision-state node: - -```text -Idea --PRODUCES----> PortfolioFacet - --REQUIRES----> PortfolioFacet - --TARGETS-----> PortfolioFacet - --USES--------> PortfolioFacet - --DEPENDS_ON--> PortfolioFacet - -IdeaPortfolioHydrationState - current_revision -> parent-linked corpus mutation chain -``` - -`PortfolioFacet` is globally content-addressed by the exact `(kind, key)` pair. The source assertion's epistemic state, source references, projection digest, and graph revision live on the relationship, because those facts belong to the assertion between an Idea and a facet, not to the shared facet itself. - -No inferred Idea-to-Idea edge is persisted by hydration. That keeps source projection separate from graph-derived intelligence. - -## Why there is a dedicated hydration compiler - -The generic `SyncGenerator` can MERGE nodes and fixed child/taxonomy edges, but it cannot express all required IdeaGraphProjection semantics: - -1. relationship type varies by assertion relation; -2. evidence/provenance belongs on each relationship; -3. replacing a projection must remove relationships no longer present; -4. corpus writes need parent-linked revision semantics; -5. deletions need explicit tombstones rather than silent absence. - -Those are current, evidenced responsibilities. `engine/sync/idea_portfolio.py` therefore owns the smallest domain-specific persistence compiler needed to bridge that expressiveness gap. It still uses the shared `GraphDriver`; it does not introduce a graph client, queue, database, or second scoring engine. - -The live generic `handle_sync` path is deliberately **not** advertised in the v0.1 domain pack because its current `SyncGenerator` cannot preserve this contract. Pretending otherwise would create a false operational path. Initial corpus hydration is an explicit CEG admin operation through `tools/hydrate_idea_portfolio.py`. A later Gate-facing binding should call the same hydrator rather than reimplement it. - -## Hydration contract - -CEG accepts `ceg.idea-portfolio-hydration/v1`: - -```json -{ - "schema": "ceg.idea-portfolio-hydration/v1", - "source_snapshot_ref": "Quantum-L9/IdeaOS@", - "source_snapshot_digest": "sha256:<64 hex>", - "expected_graph_revision": null, - "records": [ - { - "schema": "ceg.idea-portfolio-sync-record/v1", - "operation": "upsert", - "projection": { - "schema": "ideaos.idea-graph-projection/v1", - "idea_id": "example", - "source_refs": ["Ideas/..."], - "source_digest": "sha256:<64 hex>", - "lifecycle": { - "stage": "expanded", - "decision": null, - "proof_state": null, - "execution_state": null - }, - "assertions": [], - "unknowns": [] - } - } - ] -} -``` - -A tombstone record is explicit: - -```json -{ - "schema": "ceg.idea-portfolio-sync-record/v1", - "operation": "tombstone", - "idea_id": "example" -} -``` - -The envelope contains at most one record per `idea_id`. - -## Atomic revision chain - -Every hydration computes: - -```text -batch_digest = sha256(canonical(records)) -new_graph_revision = sha256( - protocol_version - + expected_graph_revision_or_GENESIS - + source_snapshot_digest - + batch_digest -) -``` - -The hydrator executes the entire envelope inside one managed Neo4j write transaction: - -```text -lock IdeaPortfolioHydrationState - -> compare current_revision with expected_graph_revision - -> apply every upsert/tombstone - -> advance current_revision - -> commit once -``` - -Consequences: - -- a wrong parent revision fails closed before graph mutation commits; -- a mid-envelope exception rolls back the whole hydration revision; -- readers never see a committed half-new portfolio snapshot; -- an exact retry after a lost response is idempotent because `current_revision == new_graph_revision` returns `reused`; -- no timestamp is promoted into ordering authority. - -`Idea`, `PortfolioFacet`, and `IdeaPortfolioHydrationState` all have required ID properties in the domain ontology so the existing `init_schema` path creates uniqueness constraints needed by the write transaction. - -## Epistemic rules - -Hydration is stricter than filename ingestion: - -- a hydrated projection must have at least one root `source_ref`; -- non-`UNKNOWN` assertions must have at least one assertion source reference; -- duplicate semantic assertions are rejected; -- relation/kind combinations are validated; -- keys are Unicode-NFC normalized and trimmed only, not lower-cased, synonym-expanded, or guessed; -- source relationships remain source relationships; -- graph-derived Idea-to-Idea relationships are not written by hydration. - -The v1 kind/relation matrix is: - -| Kind | Allowed relations | -| --- | --- | -| capability | produces, requires, uses | -| substrate | produces, requires, uses | -| proof_asset | produces, requires, uses | -| data_asset | produces, requires, uses | -| market | targets | -| customer_type | targets | -| dependency | depends_on | - -## Portfolio matching - -`build_portfolio_match_query()` compiles an IdeaGraphProjection into the flat query contract consumed by the current CEG match handler. It never calculates rank itself. - -The domain ranks candidate Ideas using six graph-native dimensions: - -1. candidate produces something the query idea requires; -2. candidate requires something the query idea produces; -3. shared `USES` facets; -4. shared `TARGETS` facets; -5. query idea explicitly depends on the candidate; -6. candidate explicitly depends on the query idea. - -Weights sum to 1.0. The candidate itself is excluded, and tombstoned/inactive ideas fail the admission gate. - -The result is **portfolio ranking evidence**, not IdeaOS lifecycle authorization. - -## Centrality and learned graph signals - -They deliberately do not exist in v0.1. The current CEG scheduler supports Louvain, co-occurrence, reinforcement, temporal recency, geo proximity, equipment sync, feedback recalculation, and causal chain scoring. It does not currently execute PageRank even though older documentation mentions it. - -Declaring a PageRank job now would advertise a capability the runtime skips. Centrality is therefore deferred until an executable current owner path exists. - -## Initial activation - -1. Validate the domain: - - ```bash - python tools/validate_domain.py domains/idea-portfolio/spec.yaml --strict - ``` - -2. Ensure the `idea-portfolio` Neo4j database exists according to deployment topology. -3. Initialize schema constraints through the existing admin `init_schema` path for domain `idea-portfolio`. -4. Produce a hydration envelope from source-bound IdeaOS projections. -5. Dry-run it first: - - ```bash - python tools/hydrate_idea_portfolio.py hydration.json - ``` - -6. Apply deliberately: - - ```bash - python tools/hydrate_idea_portfolio.py hydration.json --apply - ``` - -7. Persist the hydration receipt and its `graph_revision`. Use that revision as the `expected_graph_revision` parent of the next corpus delta. - -## Corpus rule - -Raw historical corpus hydration remains an IdeaOS semantic task: - -```text -Ideas/ artifact - -> deterministic source identity - -> IdeaOS semantic extraction / expansion - -> IdeaGraphProjection - -> CEG hydration envelope - -> atomic CEG idea-portfolio graph revision -``` - -A raw ZIP, Markdown filename, or directory name is never sufficient evidence for semantic Idea identity or graph assertions. - -## Failure boundaries - -- **Bad projection:** reject before opening the write transaction. -- **Wrong parent revision:** transaction aborts before portfolio mutation commits. -- **Write failure:** whole hydration transaction rolls back. -- **Exact replay:** returns `reused` without a second semantic mutation. -- **Tombstone:** marks Idea inactive and removes its source-projection edges; does not delete shared facet nodes. -- **Orphan facets:** tolerated as non-authoritative derived residue. Garbage collection is deferred rather than mixed into the critical write transaction. - -## Deferred seam - -The next transport unit is intentionally small: - -```text -IdeaOS IdeaGraphProvider adapter - -> Gate / TransportPacket - -> CEG owner-native portfolio context action - -> committed graph_revision + intersections + ranking evidence -``` - -That adapter must call CEG semantics. It must not copy matching, relationship classification, or ranking logic into IdeaOS. From b24c58b983c7913119ba93ff7bfaae2082b08f43 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:37:47 -0400 Subject: [PATCH 13/32] style: converge idea portfolio implementation before CI --- engine/sync/idea_portfolio.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 8c78453e..c862af51 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -240,7 +240,11 @@ def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertio def build_portfolio_match_query(projection: IdeaGraphProjection | dict[str, Any]) -> dict[str, Any]: """Compile only source-backed rank-eligible assertions into match input.""" - model = projection if isinstance(projection, IdeaGraphProjection) else IdeaGraphProjection.model_validate(projection) + model = ( + projection + if isinstance(projection, IdeaGraphProjection) + else IdeaGraphProjection.model_validate(projection) + ) by_relation = {relation.value: [] for relation in AssertionRelation} for raw, compiled in zip(model.assertions, compile_assertions(model), strict=True): if raw.evidence_state in _RANK_ELIGIBLE and raw.source_refs: @@ -269,11 +273,17 @@ def encoded(relation: AssertionRelation) -> str: def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, Any]) -> HydrationPlan: - model = envelope if isinstance(envelope, IdeaPortfolioHydrationEnvelope) else IdeaPortfolioHydrationEnvelope.model_validate(envelope) + model = ( + envelope + if isinstance(envelope, IdeaPortfolioHydrationEnvelope) + else IdeaPortfolioHydrationEnvelope.model_validate(envelope) + ) payload = [record.model_dump(mode="json", by_alias=True) for record in model.records] batch_digest = _sha256_text(_canonical_json(payload)) parent = model.expected_graph_revision or "GENESIS" - revision = _sha256_text(f"ceg.idea-portfolio-graph/v1\x00{parent}\x00{model.source_snapshot_digest}\x00{batch_digest}") + revision = _sha256_text( + f"ceg.idea-portfolio-graph/v1\x00{parent}\x00{model.source_snapshot_digest}\x00{batch_digest}" + ) return HydrationPlan(model, batch_digest, revision) @@ -394,7 +404,9 @@ async def apply_transaction(tx: Any) -> dict[str, Any]: if current == plan.graph_revision: return self._receipt(plan, "reused", [], []) if current != plan.envelope.expected_graph_revision: - raise IdeaPortfolioHydrationError("hydration revision conflict: expected parent does not match committed graph revision") + raise IdeaPortfolioHydrationError( + "hydration revision conflict: expected parent does not match committed graph revision" + ) applied: list[str] = [] tombstoned: list[str] = [] From e347bff03ebf33044100938b784ef061cb546c65 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:40:56 -0400 Subject: [PATCH 14/32] fix: preserve wire schemas without banned pydantic aliases --- engine/sync/idea_portfolio.py | 53 ++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index c862af51..a68d946b 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -7,14 +7,26 @@ import unicodedata from dataclasses import dataclass from enum import StrEnum -from typing import Any, Literal, Protocol, Self +from typing import Any, ClassVar, Literal, Protocol, Self from pydantic import BaseModel, ConfigDict, Field, model_validator DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" DOMAIN_ID = "idea-portfolio" _STATE_ID = "canonical" -_MODEL_CONFIG = ConfigDict(extra="forbid", populate_by_name=True) +_MODEL_CONFIG = ConfigDict(extra="forbid") +PROJECTION_SCHEMA = "ideaos.idea-graph-projection/v1" +SYNC_RECORD_SCHEMA = "ceg.idea-portfolio-sync-record/v1" +HYDRATION_SCHEMA = "ceg.idea-portfolio-hydration/v1" + + +def _admit_schema(value: Any, expected: str) -> Any: + if not isinstance(value, dict): + return value + data = dict(value) + if data.pop("schema", None) != expected: + raise ValueError(f"schema must equal {expected!r}") + return data class IdeaPortfolioHydrationError(ValueError): @@ -96,7 +108,7 @@ class IdeaGraphProjection(BaseModel): """CEG admission model for the IdeaOS idea-graph-projection/v1 wire contract.""" model_config = _MODEL_CONFIG - schema_id: Literal["ideaos.idea-graph-projection/v1"] = Field(alias="schema") + wire_schema: ClassVar[str] = PROJECTION_SCHEMA idea_id: str = Field(min_length=1) source_refs: list[str] source_digest: str = Field(pattern=DIGEST_PATTERN) @@ -104,6 +116,11 @@ class IdeaGraphProjection(BaseModel): assertions: list[IdeaAssertion] unknowns: list[str] + @model_validator(mode="before") + @classmethod + def validate_schema(cls, value: Any) -> Any: + return _admit_schema(value, cls.wire_schema) + @model_validator(mode="after") def validate_projection(self) -> Self: if not self.source_refs: @@ -120,11 +137,16 @@ def validate_projection(self) -> Self: class IdeaPortfolioSyncRecord(BaseModel): model_config = _MODEL_CONFIG - schema_id: Literal["ceg.idea-portfolio-sync-record/v1"] = Field(alias="schema") + wire_schema: ClassVar[str] = SYNC_RECORD_SCHEMA operation: Literal["upsert", "tombstone"] projection: IdeaGraphProjection | None = None idea_id: str | None = None + @model_validator(mode="before") + @classmethod + def validate_schema(cls, value: Any) -> Any: + return _admit_schema(value, cls.wire_schema) + @model_validator(mode="after") def validate_operation(self) -> Self: if self.operation == "upsert": @@ -147,12 +169,17 @@ def resolved_idea_id(self) -> str: class IdeaPortfolioHydrationEnvelope(BaseModel): model_config = _MODEL_CONFIG - schema_id: Literal["ceg.idea-portfolio-hydration/v1"] = Field(alias="schema") + wire_schema: ClassVar[str] = HYDRATION_SCHEMA source_snapshot_ref: str = Field(min_length=1) source_snapshot_digest: str = Field(pattern=DIGEST_PATTERN) expected_graph_revision: str | None = Field(default=None, pattern=DIGEST_PATTERN) records: list[IdeaPortfolioSyncRecord] = Field(min_length=1) + @model_validator(mode="before") + @classmethod + def validate_schema(cls, value: Any) -> Any: + return _admit_schema(value, cls.wire_schema) + @model_validator(mode="after") def validate_records(self) -> Self: idea_ids = [record.resolved_idea_id for record in self.records] @@ -219,8 +246,20 @@ def _assertion_id(idea_id: str, assertion: IdeaAssertion) -> str: return "assertion:" + hashlib.sha256(value.encode()).hexdigest() +def _projection_wire(projection: IdeaGraphProjection) -> dict[str, Any]: + return {"schema": PROJECTION_SCHEMA, **projection.model_dump(mode="json")} + + +def _record_wire(record: IdeaPortfolioSyncRecord) -> dict[str, Any]: + payload = record.model_dump(mode="json", exclude={"projection"}) + payload["schema"] = SYNC_RECORD_SCHEMA + if record.projection is not None: + payload["projection"] = _projection_wire(record.projection) + return payload + + def projection_digest(projection: IdeaGraphProjection) -> str: - return _sha256_text(_canonical_json(projection.model_dump(mode="json", by_alias=True))) + return _sha256_text(_canonical_json(_projection_wire(projection))) def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertion]: @@ -278,7 +317,7 @@ def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, if isinstance(envelope, IdeaPortfolioHydrationEnvelope) else IdeaPortfolioHydrationEnvelope.model_validate(envelope) ) - payload = [record.model_dump(mode="json", by_alias=True) for record in model.records] + payload = [_record_wire(record) for record in model.records] batch_digest = _sha256_text(_canonical_json(payload)) parent = model.expected_graph_revision or "GENESIS" revision = _sha256_text( From 4a77e7c0340897a090058e78ac87254568f153e9 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:41:24 -0400 Subject: [PATCH 15/32] test: align wire-schema and lint contracts --- tests/unit/test_idea_portfolio.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_idea_portfolio.py b/tests/unit/test_idea_portfolio.py index 4e4d752d..7595827e 100644 --- a/tests/unit/test_idea_portfolio.py +++ b/tests/unit/test_idea_portfolio.py @@ -86,7 +86,7 @@ def test_domain_is_dormant_until_enabled(monkeypatch: pytest.MonkeyPatch) -> Non @pytest.mark.unit def test_projection_admission_and_rank_filtering() -> None: model = IdeaGraphProjection.model_validate(_projection()) - assert model.model_dump(by_alias=True)["schema"] == "ideaos.idea-graph-projection/v1" + assert model.wire_schema == "ideaos.idea-graph-projection/v1" query = build_portfolio_match_query(model) assert query["requires_count"] == query["produces_count"] == query["uses_count"] == 1 assert query["targets_count"] == 0 @@ -97,6 +97,11 @@ def test_projection_admission_and_rank_filtering() -> None: with pytest.raises(ValueError, match="source_ref"): IdeaGraphProjection.model_validate(raw) + wrong_schema = _projection() + wrong_schema["schema"] = "ideaos.idea-graph-projection/v0" + with pytest.raises(ValueError, match="schema must equal"): + IdeaGraphProjection.model_validate(wrong_schema) + @pytest.mark.unit def test_upsert_preserves_all_assertions_and_wire_digest() -> None: @@ -162,13 +167,15 @@ async def test_hydrator_feature_gate_and_atomic_revision_chain() -> None: writer = _Writer() receipt = await IdeaPortfolioHydrator(writer, enabled=True).apply(_envelope()) assert receipt["status"] == "applied" - assert writer.calls == 1 and writer.database == "idea-portfolio" + assert writer.calls == 1 + assert writer.database == "idea-portfolio" assert len(writer.tx.calls) == 3 plan = compile_hydration_plan(_envelope()) replay = _Writer(plan.graph_revision) receipt = await IdeaPortfolioHydrator(replay, enabled=True).apply(_envelope()) - assert receipt["status"] == "reused" and len(replay.tx.calls) == 1 + assert receipt["status"] == "reused" + assert len(replay.tx.calls) == 1 conflict = _Writer(_digest("e")) with pytest.raises(IdeaPortfolioHydrationError, match="expected parent"): From e5d74634c8cfa7e44044f7652e35bef1b6d77ec7 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:49:48 -0400 Subject: [PATCH 16/32] test: align Gate fail-closed receipt with idempotency contract --- tests/unit/test_gate_egress.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_gate_egress.py b/tests/unit/test_gate_egress.py index 922080c7..55d68ae6 100644 --- a/tests/unit/test_gate_egress.py +++ b/tests/unit/test_gate_egress.py @@ -82,7 +82,12 @@ async def test_request_enrichment_fails_closed_without_gate_url(monkeypatch: pyt fake = _FakeClient(response=_response_packet()) monkeypatch.setattr(gate_egress, "get_gate_client", lambda: fake) result = await request_enrichment(tenant="acme", entity_id="ent-1", domain="plasticos", target_fields=["polymer"]) - assert result == {"status": "failed", "error": "gate_not_configured", "action": "enrich"} + assert result == { + "status": "failed", + "error": "gate_not_configured", + "action": "enrich", + "idempotency_key": enrichment_idempotency_key("acme", "ent-1", ["polymer"]), + } assert fake.calls == [], "no direct fallback: nothing may be sent when Gate is not configured" From 6052625abb36b24500fa52212df8933cdd8c682c Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:52:32 -0400 Subject: [PATCH 17/32] fix: converge idea portfolio sync contracts --- engine/sync/idea_portfolio.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index a68d946b..8cfc9f66 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -1,4 +1,16 @@ -"""IdeaOS projection hydration and portfolio-query compilation for CEG.""" +""" +--- L9_META --- +l9_schema: 1 +origin: engine-specific +engine: graph +layer: [sync] +tags: [ideaos, portfolio, hydration, graph] +owner: engine-team +status: active +--- /L9_META --- + +IdeaOS projection hydration and portfolio-query compilation for CEG. +""" from __future__ import annotations @@ -280,11 +292,9 @@ def compile_assertions(projection: IdeaGraphProjection) -> list[CompiledAssertio def build_portfolio_match_query(projection: IdeaGraphProjection | dict[str, Any]) -> dict[str, Any]: """Compile only source-backed rank-eligible assertions into match input.""" model = ( - projection - if isinstance(projection, IdeaGraphProjection) - else IdeaGraphProjection.model_validate(projection) + projection if isinstance(projection, IdeaGraphProjection) else IdeaGraphProjection.model_validate(projection) ) - by_relation = {relation.value: [] for relation in AssertionRelation} + by_relation: dict[str, list[str]] = {relation.value: [] for relation in AssertionRelation} for raw, compiled in zip(model.assertions, compile_assertions(model), strict=True): if raw.evidence_state in _RANK_ELIGIBLE and raw.source_refs: by_relation[compiled.relation].append(compiled.facet_id) @@ -373,7 +383,7 @@ def compile_hydration_plan(envelope: IdeaPortfolioHydrationEnvelope | dict[str, def compile_upsert_command(projection: IdeaGraphProjection, *, graph_revision: str) -> WriteCommand: - grouped = {relation.value: [] for relation in AssertionRelation} + grouped: dict[str, list[dict[str, Any]]] = {relation.value: [] for relation in AssertionRelation} for assertion in compile_assertions(projection): grouped[assertion.relation].append( { From b7343d48ed631e80b720ae65ebf1fea9d3852b40 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:23:04 -0400 Subject: [PATCH 18/32] docs: register idea portfolio feature gate --- docs/FEATURE_GATES.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/FEATURE_GATES.md b/docs/FEATURE_GATES.md index 30875ebe..8ca34300 100644 --- a/docs/FEATURE_GATES.md +++ b/docs/FEATURE_GATES.md @@ -61,6 +61,7 @@ independently of the code default. | Tenant Auth (JWT allowed_tenants) | `TENANT_AUTH_ENABLED` | `True` | `True` | active | | Capability Auth (domain-spec model) | `CAPABILITY_AUTH_ENABLED` | `True` | `True` | active | | PostgreSQL Audit Pool | `POSTGRES_DSN` | unset (`None`) | set | active (opt-in, soft dependency — see §7) | +| Idea Portfolio Graph | `IDEA_PORTFOLIO_ENABLED` / `idea_portfolio_enabled` | `False` | unset | dormant — enables IdeaOS portfolio reads and owner-native hydration | | Constellation Orchestration | — | — | — | accepted architectural gap — see §9 | --- @@ -324,6 +325,37 @@ intact. --- +## 13. Idea Portfolio Graph + +**State**: Dormant +**Flag**: `IDEA_PORTFOLIO_ENABLED=True` +**Settings**: `idea_portfolio_enabled=False` by default + +Enables the CEG-owned `idea-portfolio` domain used by IdeaOS for cross-idea +intersection and portfolio-context computation. The feature is intentionally dormant +by default. IdeaOS remains authoritative for idea identity and lifecycle truth; CEG +owns graph persistence, matching, ranking evidence, and derived portfolio context. + +### Activation Steps +1. Set `IDEA_PORTFOLIO_ENABLED=True` in the environment. +2. Validate `domains/idea-portfolio/spec.yaml` under strict domain validation. +3. Hydrate only normalized `IdeaGraphProjection` artifacts through the owner-native + hydration seam. Do not feed raw IdeaOS ZIP names or filenames into CEG. +4. Confirm portfolio reads and hydration both fail closed when the flag is disabled. + +### Validation +- `DomainPackLoader` exposes `idea-portfolio` only when the feature flag is enabled. +- `tools/hydrate_idea_portfolio.py --apply` refuses mutation while disabled. +- `IdeaPortfolioHydrator` performs a second fail-closed activation check. +- `HYPOTHESIS` and `UNKNOWN` assertions remain available as context but are excluded + from numeric ranking influence. + +### Rollback +- Set `IDEA_PORTFOLIO_ENABLED=False`. Portfolio reads and hydration fail closed. +- Existing graph data may remain stored but is not reachable through the gated domain. + +--- + ## Querying Feature Status Use the `feature_status` admin subaction to get current state of all gates: From e978ffdf21eae87211a01933ffcc9a773d6a6791 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:24:26 -0400 Subject: [PATCH 19/32] docs: minimize idea portfolio gate registration --- docs/FEATURE_GATES.md | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/docs/FEATURE_GATES.md b/docs/FEATURE_GATES.md index 8ca34300..75effa8d 100644 --- a/docs/FEATURE_GATES.md +++ b/docs/FEATURE_GATES.md @@ -61,7 +61,7 @@ independently of the code default. | Tenant Auth (JWT allowed_tenants) | `TENANT_AUTH_ENABLED` | `True` | `True` | active | | Capability Auth (domain-spec model) | `CAPABILITY_AUTH_ENABLED` | `True` | `True` | active | | PostgreSQL Audit Pool | `POSTGRES_DSN` | unset (`None`) | set | active (opt-in, soft dependency — see §7) | -| Idea Portfolio Graph | `IDEA_PORTFOLIO_ENABLED` / `idea_portfolio_enabled` | `False` | unset | dormant — enables IdeaOS portfolio reads and owner-native hydration | +| Idea Portfolio Graph | `IDEA_PORTFOLIO_ENABLED` (`idea_portfolio_enabled`) | `False` | unset | dormant; opt-in IdeaOS portfolio reads/hydration | | Constellation Orchestration | — | — | — | accepted architectural gap — see §9 | --- @@ -325,37 +325,6 @@ intact. --- -## 13. Idea Portfolio Graph - -**State**: Dormant -**Flag**: `IDEA_PORTFOLIO_ENABLED=True` -**Settings**: `idea_portfolio_enabled=False` by default - -Enables the CEG-owned `idea-portfolio` domain used by IdeaOS for cross-idea -intersection and portfolio-context computation. The feature is intentionally dormant -by default. IdeaOS remains authoritative for idea identity and lifecycle truth; CEG -owns graph persistence, matching, ranking evidence, and derived portfolio context. - -### Activation Steps -1. Set `IDEA_PORTFOLIO_ENABLED=True` in the environment. -2. Validate `domains/idea-portfolio/spec.yaml` under strict domain validation. -3. Hydrate only normalized `IdeaGraphProjection` artifacts through the owner-native - hydration seam. Do not feed raw IdeaOS ZIP names or filenames into CEG. -4. Confirm portfolio reads and hydration both fail closed when the flag is disabled. - -### Validation -- `DomainPackLoader` exposes `idea-portfolio` only when the feature flag is enabled. -- `tools/hydrate_idea_portfolio.py --apply` refuses mutation while disabled. -- `IdeaPortfolioHydrator` performs a second fail-closed activation check. -- `HYPOTHESIS` and `UNKNOWN` assertions remain available as context but are excluded - from numeric ranking influence. - -### Rollback -- Set `IDEA_PORTFOLIO_ENABLED=False`. Portfolio reads and hydration fail closed. -- Existing graph data may remain stored but is not reachable through the gated domain. - ---- - ## Querying Feature Status Use the `feature_status` admin subaction to get current state of all gates: From c72155819a7a896656b5b73ed43756a377b4a24e Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:28:01 -0400 Subject: [PATCH 20/32] ci: update OpenSSF Scorecard to v2.4.4 --- .github/workflows/supply-chain.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index 73cbd518..c2e2a227 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -64,7 +64,7 @@ jobs: persist-credentials: false - name: Run OpenSSF Scorecard Analysis - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 with: results_file: scorecard.sarif results_format: sarif From 481c0d0f1fe5383517ca5482b8616d3fac77c8da Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:30:49 -0400 Subject: [PATCH 21/32] ci: move Scorecard action to GHCR-backed v2.4.4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad5c73a..fa28a0f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -465,7 +465,7 @@ jobs: persist-credentials: false - name: Run OpenSSF Scorecard - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 with: results_file: scorecard.sarif results_format: sarif From 1aecb5f8f4d3ffe8ff0504f9e7fdf38c66ddca40 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:32:47 -0400 Subject: [PATCH 22/32] ci: sync L9 analysis caller to current core preset --- .github/workflows/l9-analysis.yml | 186 ++++-------------------------- 1 file changed, 25 insertions(+), 161 deletions(-) diff --git a/.github/workflows/l9-analysis.yml b/.github/workflows/l9-analysis.yml index a74002ba..b4f10291 100644 --- a/.github/workflows/l9-analysis.yml +++ b/.github/workflows/l9-analysis.yml @@ -3,184 +3,48 @@ # DO NOT EDIT — this file is managed by l9-ci-core presets/python. # To update, pull the latest preset from Quantum-L9/l9-ci-core. # -# This workflow runs the full L9 analysis pipeline: -# 1. Resolve governance config from .github/governance/ -# 2. Run semgrep with Python rulesets -# 3. Provision the SDK (immutable, pinned) -# 4. Normalize → Validate → Project → Route → Manifest → Upload -# 5. Publish results as GitHub Checks +# Thin caller into Core analyze-semgrep.yml (SDK semgrep run → gate → +# publish + SARIF). Language is locked to python. + name: L9 Analysis on: pull_request: push: branches: [main] workflow_dispatch: - env: - L9_CORE_REF: "f88116503430aa18992b70d8d31063e34ff97ef1" - L9_PROFILE: "pr_fast" - L9_MATRIX_ID: "pr-semgrep" - + # Pin to a full Core SHA. Bump after merging self-CI dogfood so consumers + # pick up analyze-semgrep SARIF + gate wiring from that merge commit. + # `uses:` cannot interpolate env — duplicate the SHA literally on the uses + # line (same as CORE_ACTIONS_PIN). Keep L9_CORE_REF here for humans/agents. + L9_CORE_REF: "1aa6c97b3a6f30b9e9d55e61575a172581ba8558" permissions: contents: read - checks: write - concurrency: group: l9-analysis-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - jobs: analyze: - name: Governed Semgrep Analysis - runs-on: ubuntu-latest - timeout-minutes: 15 - outputs: - enabled: ${{ steps.gov.outputs.enabled }} - mode: ${{ steps.gov.outputs.mode }} - governance-digest: ${{ steps.gov.outputs.governance-digest }} - artifact-name: ${{ steps.names.outputs.artifact-name }} - permissions: - contents: read - checks: write - steps: - - name: Checkout immutable event revision - env: - REPOSITORY: ${{ github.repository }} - REVISION: ${{ github.sha }} - TOKEN: ${{ github.token }} - run: | - set -euo pipefail - git init . - git remote add origin \ - "https://x-access-token:${TOKEN}@github.com/${REPOSITORY}.git" - git -c protocol.version=2 fetch --depth=1 origin "${REVISION}" - git checkout --detach FETCH_HEAD - git remote set-url origin "https://github.com/${REPOSITORY}.git" - - - id: gov - name: Resolve governance - uses: Quantum-L9/l9-ci-core/.github/actions/resolve-governance@555d577eb805851b624cf7b0b8fc4df75a225d9f - with: - profile: ${{ env.L9_PROFILE }} - provider: semgrep - event-name: ${{ github.event_name }} - repository: ${{ github.repository }} - ref: ${{ github.ref }} - - - id: names - name: Compute artifact names - env: - MATRIX_ID: ${{ env.L9_MATRIX_ID }} - run: | - set -euo pipefail - echo "artifact-name=l9-semgrep-${MATRIX_ID}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - - - name: Run semgrep - if: steps.gov.outputs.enabled == 'true' - run: | - set -euo pipefail - pip install --upgrade pip semgrep - mkdir -p "artifacts/raw/semgrep/${L9_MATRIX_ID}" - # No `|| true` (Baseline Ratchet rejects fail-open). Also no - # `--error`: findings must reach normalize/publish so governance - # can decide blocking vs advisory; `--error` exits 1 before that. - semgrep scan \ - --config p/python \ - --json \ - --output "artifacts/raw/semgrep/${L9_MATRIX_ID}/report.json" \ - --quiet - env: - L9_MATRIX_ID: ${{ env.L9_MATRIX_ID }} - - - id: sdk - name: Provision immutable SDK - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/provision-sdk@0d28395428426853c44825c4645c23ee8ace23b1 - - - name: Normalize provider report - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/invoke-sdk@f88116503430aa18992b70d8d31063e34ff97ef1 - with: - executable: ${{ steps.sdk.outputs.executable }} - operation: semgrep-normalize - input: artifacts/raw/semgrep/${{ env.L9_MATRIX_ID }}/report.json - output: .l9/runtime/${{ env.L9_MATRIX_ID }}/finding-bundle.json - root: . - snapshot-id: ${{ github.sha }} - revision: ${{ github.sha }} - strict: ${{ steps.gov.outputs.strict }} - required: ${{ steps.gov.outputs.required-provider }} - policy: ${{ steps.gov.outputs.sdk-policy }} - identity-map: .github/governance/semgrep-identity-map.yaml - - - name: Validate canonical bundle - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/validate-bundle@84375ed2bc9e005048dfb6f74076fc420b4bc01c - with: - executable: ${{ steps.sdk.outputs.executable }} - bundle: .l9/runtime/${{ env.L9_MATRIX_ID }}/finding-bundle.json - - - name: Project agent-review payload - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/invoke-sdk@f88116503430aa18992b70d8d31063e34ff97ef1 - with: - executable: ${{ steps.sdk.outputs.executable }} - operation: bundle-project-agent-payload - input: .l9/runtime/${{ env.L9_MATRIX_ID }}/finding-bundle.json - output: .l9/runtime/${{ env.L9_MATRIX_ID }}/agent-review-payload.json - strict: ${{ steps.gov.outputs.strict }} - - - id: route - name: Route artifacts - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/route-artifacts@84375ed2bc9e005048dfb6f74076fc420b4bc01c - with: - provider: semgrep - matrix-id: ${{ env.L9_MATRIX_ID }} - raw-report: artifacts/raw/semgrep/${{ env.L9_MATRIX_ID }}/report.json - bundle: .l9/runtime/${{ env.L9_MATRIX_ID }}/finding-bundle.json - agent-payload: .l9/runtime/${{ env.L9_MATRIX_ID }}/agent-review-payload.json - destination-root: artifacts - - - name: Build artifact manifest - if: steps.gov.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/actions/build-artifact-manifest@555d577eb805851b624cf7b0b8fc4df75a225d9f - with: - provider: semgrep - matrix-id: ${{ env.L9_MATRIX_ID }} - sdk-revision: ${{ steps.sdk.outputs.sdk-revision }} - bundle: ${{ steps.route.outputs.bundle }} - agent-payload: ${{ steps.route.outputs.agent-payload }} - raw-directory: ${{ steps.route.outputs.raw-directory }} - output: artifacts/metadata/${{ env.L9_MATRIX_ID }}/artifact-manifest.json - - - name: Upload analysis artifact set - if: steps.gov.outputs.enabled == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ steps.names.outputs.artifact-name }} - path: | - artifacts/raw/semgrep/${{ env.L9_MATRIX_ID }}/ - artifacts/l9/${{ env.L9_MATRIX_ID }}/ - artifacts/metadata/${{ env.L9_MATRIX_ID }}/ - if-no-files-found: error - retention-days: 14 - - publish: - name: Publish analysis (Core) - needs: analyze - if: needs.analyze.outputs.enabled == 'true' - uses: Quantum-L9/l9-ci-core/.github/workflows/publish-analysis.yml@0d28395428426853c44825c4645c23ee8ace23b1 + name: Analyze (semgrep) + uses: Quantum-L9/l9-ci-core/.github/workflows/analyze-semgrep.yml@1aa6c97b3a6f30b9e9d55e61575a172581ba8558 permissions: actions: read checks: write contents: read + security-events: write with: - artifact-name: ${{ needs.analyze.outputs.artifact-name }} - profile: pr_fast - mode: ${{ needs.analyze.outputs.mode }} - provider: semgrep + # `env` is not available in a reusable-workflow `with:` block (GitHub + # allows only github, needs, strategy, matrix, inputs, vars there), so + # the profile is selected from the github context. Each profile declares + # the event classes it admits — pr_fast only pull_request, merge push, + # nightly the class the kernel maps workflow_dispatch onto. A profile + # that does not admit the incoming class makes resolve-governance fail + # closed: "event '' is not allowed for profile ''". + profile: >- + ${{ github.event_name == 'push' && 'merge' + || github.event_name == 'workflow_dispatch' && 'nightly' + || 'pr_fast' }} matrix-id: pr-semgrep - governance-digest: ${{ needs.analyze.outputs.governance-digest }} - repository-revision: ${{ github.sha }} - workflow-result: ${{ needs.analyze.result }} + # SDK-owned Semgrep execution language: python or typescript. The SDK + # selects the packaged ruleset; this caller authors no --config list. + language: python From 736cb45a6e05f74f407697e4f6cc82171983a52f Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:34:15 -0400 Subject: [PATCH 23/32] chore: compact idea portfolio domain without semantic change --- domains/idea-portfolio/spec.yaml | 42 +++----------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/domains/idea-portfolio/spec.yaml b/domains/idea-portfolio/spec.yaml index 3a8c6a4b..8afbedc1 100644 --- a/domains/idea-portfolio/spec.yaml +++ b/domains/idea-portfolio/spec.yaml @@ -13,26 +13,12 @@ domain: name: IdeaOS Portfolio Graph description: Cross-idea portfolio intelligence. IdeaOS owns idea/lifecycle truth; CEG owns graph persistence, intersections, and ranking. version: 0.1.0 - ontology: nodes: - label: Idea managedby: sync candidate: true - properties: - - {name: idea_id, type: string, required: true} - - {name: source_digest, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} - - {name: lifecycle_stage, type: string} - - {name: decision, type: enum, values: [GO, CONDITIONAL_GO, HOLD, NO_GO]} - - {name: proof_state, type: string} - - {name: execution_state, type: string} - - {name: unknowns_json, type: string} - - {name: self_dependency_facet_id, type: string} - - {name: active, type: bool} - - {name: hydrated_at, type: datetime} - - {name: tombstoned_at, type: datetime} + properties: [{name: idea_id, type: string, required: true}, {name: source_digest, type: string}, {name: projection_digest, type: string}, {name: graph_revision, type: string}, {name: lifecycle_stage, type: string}, {name: decision, type: enum, values: [GO, CONDITIONAL_GO, HOLD, NO_GO]}, {name: proof_state, type: string}, {name: execution_state, type: string}, {name: unknowns_json, type: string}, {name: self_dependency_facet_id, type: string}, {name: active, type: bool}, {name: hydrated_at, type: datetime}, {name: tombstoned_at, type: datetime}] - label: IdeaQuery managedby: api queryentity: true @@ -40,21 +26,11 @@ ontology: - label: PortfolioFacet managedby: sync auxiliary: true - properties: - - {name: facet_id, type: string, required: true} - - {name: kind, type: enum, values: [capability, substrate, proof_asset, data_asset, market, customer_type, dependency]} - - {name: key, type: string, required: true} - - {name: last_seen_revision, type: string} + properties: [{name: facet_id, type: string, required: true}, {name: kind, type: enum, values: [capability, substrate, proof_asset, data_asset, market, customer_type, dependency]}, {name: key, type: string, required: true}, {name: last_seen_revision, type: string}] - label: IdeaPortfolioHydrationState managedby: sync auxiliary: true - properties: - - {name: state_id, type: string, required: true} - - {name: current_revision, type: string} - - {name: source_snapshot_ref, type: string} - - {name: source_snapshot_digest, type: string} - - {name: batch_digest, type: string} - - {name: completed_at, type: datetime} + properties: [{name: state_id, type: string, required: true}, {name: current_revision, type: string}, {name: source_snapshot_ref, type: string}, {name: source_snapshot_digest, type: string}, {name: batch_digest, type: string}, {name: completed_at, type: datetime}] edges: - &source_edge type: PRODUCES @@ -63,13 +39,7 @@ ontology: direction: DIRECTED category: capability managedby: sync - properties: - - {name: assertion_id, type: string, required: true} - - {name: kind, type: string} - - {name: evidence_state, type: string} - - {name: source_refs_json, type: string} - - {name: projection_digest, type: string} - - {name: graph_revision, type: string} + properties: [{name: assertion_id, type: string, required: true}, {name: kind, type: string}, {name: evidence_state, type: string}, {name: source_refs_json, type: string}, {name: projection_digest, type: string}, {name: graph_revision, type: string}] - <<: *source_edge type: REQUIRES - <<: *source_edge @@ -80,7 +50,6 @@ ontology: - <<: *source_edge type: DEPENDS_ON category: context - matchentities: candidate: [{label: Idea, matchdirection: portfolio_context_for_idea}] queryentity: [{label: IdeaQuery, matchdirection: portfolio_context_for_idea}] @@ -102,7 +71,6 @@ traversal: {steps: []} gates: - {name: active_only, type: boolean, candidateprop: active, nullbehavior: fail, matchdirections: [portfolio_context_for_idea]} - {name: exclude_self, type: threshold, candidateprop: idea_id, queryparam: idea_id, operator: "!=", nullbehavior: fail, matchdirections: [portfolio_context_for_idea]} - # VERIFIED and source-backed SUPPORTED_INFERENCE assertions can rank. # HYPOTHESIS/UNKNOWN remain graph context and cannot become rank evidence. scoring: @@ -155,8 +123,6 @@ scoring: weightkey: wcandidatedependency defaultweight: 0.05 matchdirections: [portfolio_context_for_idea] - -# Generic SyncGenerator cannot preserve assertion provenance/revision semantics. sync: {endpoints: []} gdsjobs: [] compliance: From e626680b56551fef63b6d2b022e0a635f195d9a7 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:39:48 -0400 Subject: [PATCH 24/32] ci: pin L9 analysis to current Core runtime --- .github/workflows/l9-analysis.yml | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/l9-analysis.yml b/.github/workflows/l9-analysis.yml index b4f10291..6a6038dd 100644 --- a/.github/workflows/l9-analysis.yml +++ b/.github/workflows/l9-analysis.yml @@ -1,8 +1,5 @@ # L9 Governed Analysis Pipeline — Python Preset (LOCKED) # -# DO NOT EDIT — this file is managed by l9-ci-core presets/python. -# To update, pull the latest preset from Quantum-L9/l9-ci-core. -# # Thin caller into Core analyze-semgrep.yml (SDK semgrep run → gate → # publish + SARIF). Language is locked to python. @@ -13,11 +10,10 @@ on: branches: [main] workflow_dispatch: env: - # Pin to a full Core SHA. Bump after merging self-CI dogfood so consumers - # pick up analyze-semgrep SARIF + gate wiring from that merge commit. - # `uses:` cannot interpolate env — duplicate the SHA literally on the uses - # line (same as CORE_ACTIONS_PIN). Keep L9_CORE_REF here for humans/agents. - L9_CORE_REF: "1aa6c97b3a6f30b9e9d55e61575a172581ba8558" + # Immutable current Core pin. Keep this and the reusable-workflow `uses:` + # revision identical so the caller cannot drift between human-visible and + # executable authority. + L9_CORE_REF: "ef138020013f077bcd14bc2e65a28f32a734400f" permissions: contents: read concurrency: @@ -26,25 +22,16 @@ concurrency: jobs: analyze: name: Analyze (semgrep) - uses: Quantum-L9/l9-ci-core/.github/workflows/analyze-semgrep.yml@1aa6c97b3a6f30b9e9d55e61575a172581ba8558 + uses: Quantum-L9/l9-ci-core/.github/workflows/analyze-semgrep.yml@ef138020013f077bcd14bc2e65a28f32a734400f permissions: actions: read checks: write contents: read security-events: write with: - # `env` is not available in a reusable-workflow `with:` block (GitHub - # allows only github, needs, strategy, matrix, inputs, vars there), so - # the profile is selected from the github context. Each profile declares - # the event classes it admits — pr_fast only pull_request, merge push, - # nightly the class the kernel maps workflow_dispatch onto. A profile - # that does not admit the incoming class makes resolve-governance fail - # closed: "event '' is not allowed for profile ''". profile: >- ${{ github.event_name == 'push' && 'merge' || github.event_name == 'workflow_dispatch' && 'nightly' || 'pr_fast' }} matrix-id: pr-semgrep - # SDK-owned Semgrep execution language: python or typescript. The SDK - # selects the packaged ruleset; this caller authors no --config list. language: python From 0c803962814e4ab317a1dc424dead68868617bb3 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:40:53 -0400 Subject: [PATCH 25/32] ci: align governance policy selection with current Core --- .github/governance/quality-thresholds.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/governance/quality-thresholds.yaml b/.github/governance/quality-thresholds.yaml index 4e4e1000..5d117db8 100644 --- a/.github/governance/quality-thresholds.yaml +++ b/.github/governance/quality-thresholds.yaml @@ -2,20 +2,20 @@ "schema": "l9.quality-threshold-selection/v1", "profiles": { "pr_fast": { - "sdk_policy": ".github/governance/semgrep-policy.yaml" + "sdk_policy": "semgrep-policy.yaml" }, "merge": { - "sdk_policy": ".github/governance/semgrep-policy.yaml" + "sdk_policy": "semgrep-policy.yaml" }, "nightly": { - "sdk_policy": ".github/governance/semgrep-policy.yaml" + "sdk_policy": "semgrep-policy.yaml" }, "release": { - "sdk_policy": ".github/governance/semgrep-policy.yaml" + "sdk_policy": "semgrep-policy.yaml" }, "supply_chain": { - "sdk_policy": ".github/governance/semgrep-policy.yaml" + "sdk_policy": "semgrep-policy.yaml" } }, - "note": "Core selects an SDK policy file but never evaluates finding thresholds itself. Point sdk_policy at a policy file the pinned SDK understands to raise/lower gates." + "note": "Core selects an SDK policy filename relative to governance-root but never evaluates finding thresholds itself. Point sdk_policy at a policy file the pinned SDK understands to raise/lower gates." } From 0058b0edf11999146ebb4ab44b5d7dd0d82cfb21 Mon Sep 17 00:00:00 2001 From: Igor Beylin <31744795+cryptoxdog@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:44:01 -0400 Subject: [PATCH 26/32] ci: remove superseded local L9 analysis caller --- .github/workflows/l9-analysis.yml | 37 ------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/l9-analysis.yml diff --git a/.github/workflows/l9-analysis.yml b/.github/workflows/l9-analysis.yml deleted file mode 100644 index 6a6038dd..00000000 --- a/.github/workflows/l9-analysis.yml +++ /dev/null @@ -1,37 +0,0 @@ -# L9 Governed Analysis Pipeline — Python Preset (LOCKED) -# -# Thin caller into Core analyze-semgrep.yml (SDK semgrep run → gate → -# publish + SARIF). Language is locked to python. - -name: L9 Analysis -on: - pull_request: - push: - branches: [main] - workflow_dispatch: -env: - # Immutable current Core pin. Keep this and the reusable-workflow `uses:` - # revision identical so the caller cannot drift between human-visible and - # executable authority. - L9_CORE_REF: "ef138020013f077bcd14bc2e65a28f32a734400f" -permissions: - contents: read -concurrency: - group: l9-analysis-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - analyze: - name: Analyze (semgrep) - uses: Quantum-L9/l9-ci-core/.github/workflows/analyze-semgrep.yml@ef138020013f077bcd14bc2e65a28f32a734400f - permissions: - actions: read - checks: write - contents: read - security-events: write - with: - profile: >- - ${{ github.event_name == 'push' && 'merge' - || github.event_name == 'workflow_dispatch' && 'nightly' - || 'pr_fast' }} - matrix-id: pr-semgrep - language: python From ef4c5f484b19bdccbd395ab030641b12c6df1e7f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 13:48:37 +0000 Subject: [PATCH 27/32] fix(idea-portfolio): gate hydration on state_id uniqueness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remediates CEG-262-002 (RELIABILITY, High) and CEG-262-003 (REVIEW, Low) from audit ceg-open-prs-2026-09-17, against PR #262 source head 0058b0edf11999146ebb4ab44b5d7dd0d82cfb21. CEG-262-002 — the --apply path checked the feature flag, loaded the domain, connected the driver and called IdeaPortfolioHydrator without ever running init_schema or verifying the state-id uniqueness constraint (E007). The hydrator serialises revisions with MERGE on IdeaPortfolioHydrationState {state_id} (E008), and MERGE alone is not single-node-safe under concurrent load without a uniqueness constraint on the merged property (E010). So two concurrent initial hydrations could each create a canonical state node, each read current_revision=null, and each commit a child revision. The apply path now runs the repository's existing schema-init contract — the same _init_schema the admin/init_schema subaction invokes, which provisions the constraint from the ontology's required state_id (E009) — and then verifies the constraint actually exists, failing closed if it does not. Verification is not redundant: _init_schema logs and swallows per-constraint failures, so calling it proves nothing on its own. The label, property and SHOW CONSTRAINTS query now have one definition in engine/sync/idea_portfolio.py, alongside a pure predicate over the returned rows. Neo4j 5 names node uniqueness NODE_PROPERTY_UNIQUENESS (4.x called it UNIQUENESS); NODE_KEY also satisfies the precondition. The check is deliberately outside the write transaction: the hydrator's transaction statement sequence is unchanged, so the existing atomicity and replay tests keep their exact tx.run counts. CEG-262-003 — GraphWriter.execute_write carried a bare ellipsis body, the subject of the one current unresolved review thread (E012). Replaced with the repository-accepted Protocol form used by engine/hoprag/indexer.py GraphStore: class docstring, method docstring with Args/Returns, ellipsis on its own line. No behaviour change. Scope: strictly the two write surfaces the handoff allows. The bundle grants no publish, merge or policy authority, and PRESERVE-L9-ANALYSIS-OWNER remains VIOLATED on this branch — that is CEG-262-001, CI_PIPELINE-owned, untouched here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- engine/sync/idea_portfolio.py | 58 +++++++++++++++++++++++++++++++-- tools/hydrate_idea_portfolio.py | 43 +++++++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 8cfc9f66..344b05b3 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -17,6 +17,7 @@ import hashlib import json import unicodedata +from collections.abc import Iterable, Mapping from dataclasses import dataclass from enum import StrEnum from typing import Any, ClassVar, Literal, Protocol, Self @@ -26,6 +27,17 @@ DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" DOMAIN_ID = "idea-portfolio" _STATE_ID = "canonical" +STATE_LABEL = "IdeaPortfolioHydrationState" +STATE_ID_PROPERTY = "state_id" + +# MERGE alone does not guarantee node uniqueness under concurrent load; Neo4j +# requires a uniqueness constraint on the merged identifying property for that. +# Without it two concurrent initial hydrations can each MERGE their own canonical +# state node, both read current_revision=null, and both commit a child revision. +SHOW_CONSTRAINTS_CYPHER = "SHOW CONSTRAINTS YIELD labelsOrTypes, properties, type, entityType" +# Neo4j 5 names node uniqueness NODE_PROPERTY_UNIQUENESS (4.x called it UNIQUENESS); +# NODE_KEY is uniqueness plus existence, so it satisfies the precondition too. +_UNIQUENESS_CONSTRAINT_TYPES = frozenset({"NODE_PROPERTY_UNIQUENESS", "NODE_KEY"}) _MODEL_CONFIG = ConfigDict(extra="forbid") PROJECTION_SCHEMA = "ideaos.idea-graph-projection/v1" SYNC_RECORD_SCHEMA = "ceg.idea-portfolio-sync-record/v1" @@ -201,6 +213,8 @@ def validate_records(self) -> Self: class GraphWriter(Protocol): + """Protocol for the managed-write surface hydration depends on.""" + async def execute_write( self, transaction_function: Any = None, @@ -209,7 +223,21 @@ async def execute_write( parameters: dict[str, Any] | None = None, database: str | None = None, **kwargs: Any, - ) -> dict[str, Any] | Any: ... + ) -> dict[str, Any] | Any: + """Run one managed write transaction. + + Args: + transaction_function: Async callable receiving the transaction. + *args: Positional arguments forwarded to the transaction function. + cypher: Single-statement form, used when no transaction function is given. + parameters: Query parameters for the single-statement form. + database: Target database name. + **kwargs: Keyword arguments forwarded to the transaction function. + + Returns: + The transaction function's result, or the driver's statement result. + """ + ... @dataclass(frozen=True) @@ -424,7 +452,29 @@ def compile_tombstone_command(idea_id: str, *, graph_revision: str) -> WriteComm ) -_LOCK_STATE_CYPHER = """MERGE (state:IdeaPortfolioHydrationState {state_id: $state_id}) +def state_uniqueness_constraint_present(rows: Iterable[Mapping[str, Any]]) -> bool: + """Decide whether SHOW CONSTRAINTS rows prove state_id uniqueness. + + Args: + rows: Rows yielded by ``SHOW_CONSTRAINTS_CYPHER``. + + Returns: + True when a node uniqueness or node key constraint covers exactly + ``STATE_LABEL.STATE_ID_PROPERTY``. + """ + for row in rows: + if row.get("entityType") not in (None, "NODE"): + continue + if row.get("type") not in _UNIQUENESS_CONSTRAINT_TYPES: + continue + labels = row.get("labelsOrTypes") or [] + properties = row.get("properties") or [] + if STATE_LABEL in labels and list(properties) == [STATE_ID_PROPERTY]: + return True + return False + + +_LOCK_STATE_CYPHER = f"""MERGE (state:{STATE_LABEL} {{{STATE_ID_PROPERTY}: $state_id}}) SET state._cas_lock=coalesce(state._cas_lock, 0)+1, state._tenant=$tenant RETURN state.current_revision AS current_revision""" _FINALIZE_STATE_CYPHER = """MATCH (state:IdeaPortfolioHydrationState {state_id: $state_id}) @@ -514,6 +564,9 @@ def _receipt( __all__ = [ "DOMAIN_ID", + "SHOW_CONSTRAINTS_CYPHER", + "STATE_ID_PROPERTY", + "STATE_LABEL", "AssertionKind", "AssertionRelation", "EvidenceState", @@ -530,4 +583,5 @@ def _receipt( "compile_tombstone_command", "compile_upsert_command", "projection_digest", + "state_uniqueness_constraint_present", ] diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index 2e5aaf94..be1d727f 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -13,14 +13,20 @@ sys.path.insert(0, str(ROOT)) from engine.config.loader import DomainPackLoader +from engine.config.schema import DomainSpec from engine.config.settings import settings from engine.graph.driver import GraphDriver +from engine.handlers import _init_schema from engine.sync.idea_portfolio import ( DOMAIN_ID, + SHOW_CONSTRAINTS_CYPHER, + STATE_ID_PROPERTY, + STATE_LABEL, IdeaPortfolioHydrationEnvelope, IdeaPortfolioHydrationError, IdeaPortfolioHydrator, compile_hydration_plan, + state_uniqueness_constraint_present, ) @@ -41,13 +47,48 @@ def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: } +async def _require_state_uniqueness(driver: GraphDriver, spec: DomainSpec) -> None: + """Fail closed unless the canonical state node is protected by a uniqueness constraint. + + The hydrator serialises revisions with `MERGE (state:...{state_id: ...})`, and + MERGE is only single-node-safe when a uniqueness constraint covers the merged + property. Without it two concurrent initial hydrations can each create a + canonical state node and each commit a child revision. + + Runs the repository's existing schema-init contract first — the same + `_init_schema` the `admin`/`init_schema` subaction invokes, which provisions + the constraint from the domain ontology's required `state_id` — and then + verifies the constraint really exists. Verification is not redundant: + `_init_schema` logs and swallows per-constraint failures, so calling it + proves nothing on its own. + + Args: + driver: Connected graph driver. + spec: Loaded idea-portfolio domain spec. + + Raises: + IdeaPortfolioHydrationError: The constraint is absent after schema init. + """ + await _init_schema(driver, spec) + rows = await driver.execute_query(SHOW_CONSTRAINTS_CYPHER, {}, database=DOMAIN_ID) + if not state_uniqueness_constraint_present(rows): + msg = ( + f"idea-portfolio schema precondition unmet: no uniqueness constraint on " + f"{STATE_LABEL}.{STATE_ID_PROPERTY} after schema init. Concurrent hydration " + f"could fork the canonical revision chain; refusing to mutate. Run the admin " + f"init_schema subaction for domain '{DOMAIN_ID}' against this database." + ) + raise IdeaPortfolioHydrationError(msg) + + async def _apply(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: if not settings.idea_portfolio_enabled: raise IdeaPortfolioHydrationError("idea-portfolio hydration is disabled by configuration") - DomainPackLoader(config_path=str(ROOT / "domains")).load_domain(DOMAIN_ID) + spec = DomainPackLoader(config_path=str(ROOT / "domains")).load_domain(DOMAIN_ID) driver = GraphDriver() await driver.connect() try: + await _require_state_uniqueness(driver, spec) return await IdeaPortfolioHydrator(driver, enabled=True).apply(envelope) finally: await driver.close() From 6404959117774bfcb322a85f2c69e89ce8b862d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 14:26:45 +0000 Subject: [PATCH 28/32] fix(idea-portfolio): raise NotImplementedError in the GraphWriter protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit github-code-quality re-flagged "Statement has no effect" on the ellipsis at engine/sync/idea_portfolio.py after the previous push, proposing `raise NotImplementedError`. The earlier fix matched the form used by engine/hoprag/indexer.py GraphStore (docstring plus a bare `...`), which the bot still counts as a no-op statement — so CEG-262-003's closure condition, that the review thread actually closes, was not met. Protocols are structural: IdeaPortfolioHydrator calls execute_write on the concrete object it is given, never on the protocol class, so raising here changes no behaviour and gives a clear error if it is ever invoked directly. The 1425-test unit suite passes unchanged, including the test doubles that implement this protocol. Also tightens prose that duplicated the previous commit message, cutting this remediation from +98 to +63 added lines. No logic changed: the constants, the SHOW CONSTRAINTS predicate, the schema-init call and the fail-closed check are byte-identical in behaviour, and the discriminating proof still shows the apply path reaching IdeaPortfolioHydrator iff state_id uniqueness is present. This does not clear `Enforce PR Policies`: #262 stood at +969 additions against a 1000 limit before any of this work, so the arithmetic is reported on the PR rather than solved by deleting the fix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- engine/sync/idea_portfolio.py | 34 ++++----------------------------- tools/hydrate_idea_portfolio.py | 21 ++++++-------------- 2 files changed, 10 insertions(+), 45 deletions(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 344b05b3..4ba9da20 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -29,14 +29,8 @@ _STATE_ID = "canonical" STATE_LABEL = "IdeaPortfolioHydrationState" STATE_ID_PROPERTY = "state_id" - -# MERGE alone does not guarantee node uniqueness under concurrent load; Neo4j -# requires a uniqueness constraint on the merged identifying property for that. -# Without it two concurrent initial hydrations can each MERGE their own canonical -# state node, both read current_revision=null, and both commit a child revision. SHOW_CONSTRAINTS_CYPHER = "SHOW CONSTRAINTS YIELD labelsOrTypes, properties, type, entityType" -# Neo4j 5 names node uniqueness NODE_PROPERTY_UNIQUENESS (4.x called it UNIQUENESS); -# NODE_KEY is uniqueness plus existence, so it satisfies the precondition too. +# Neo4j 5 spelling; 4.x called uniqueness UNIQUENESS. NODE_KEY implies it too. _UNIQUENESS_CONSTRAINT_TYPES = frozenset({"NODE_PROPERTY_UNIQUENESS", "NODE_KEY"}) _MODEL_CONFIG = ConfigDict(extra="forbid") PROJECTION_SCHEMA = "ideaos.idea-graph-projection/v1" @@ -224,20 +218,8 @@ async def execute_write( database: str | None = None, **kwargs: Any, ) -> dict[str, Any] | Any: - """Run one managed write transaction. - - Args: - transaction_function: Async callable receiving the transaction. - *args: Positional arguments forwarded to the transaction function. - cypher: Single-statement form, used when no transaction function is given. - parameters: Query parameters for the single-statement form. - database: Target database name. - **kwargs: Keyword arguments forwarded to the transaction function. - - Returns: - The transaction function's result, or the driver's statement result. - """ - ... + """Run one managed write transaction, via a transaction function or `cypher`.""" + raise NotImplementedError @dataclass(frozen=True) @@ -453,15 +435,7 @@ def compile_tombstone_command(idea_id: str, *, graph_revision: str) -> WriteComm def state_uniqueness_constraint_present(rows: Iterable[Mapping[str, Any]]) -> bool: - """Decide whether SHOW CONSTRAINTS rows prove state_id uniqueness. - - Args: - rows: Rows yielded by ``SHOW_CONSTRAINTS_CYPHER``. - - Returns: - True when a node uniqueness or node key constraint covers exactly - ``STATE_LABEL.STATE_ID_PROPERTY``. - """ + """True when SHOW CONSTRAINTS rows prove STATE_LABEL.STATE_ID_PROPERTY is unique.""" for row in rows: if row.get("entityType") not in (None, "NODE"): continue diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index be1d727f..1b4f48ab 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -48,23 +48,14 @@ def _dry_run(envelope: IdeaPortfolioHydrationEnvelope) -> dict[str, object]: async def _require_state_uniqueness(driver: GraphDriver, spec: DomainSpec) -> None: - """Fail closed unless the canonical state node is protected by a uniqueness constraint. + """Fail closed unless state_id uniqueness protects the canonical state node. - The hydrator serialises revisions with `MERGE (state:...{state_id: ...})`, and MERGE is only single-node-safe when a uniqueness constraint covers the merged - property. Without it two concurrent initial hydrations can each create a - canonical state node and each commit a child revision. - - Runs the repository's existing schema-init contract first — the same - `_init_schema` the `admin`/`init_schema` subaction invokes, which provisions - the constraint from the domain ontology's required `state_id` — and then - verifies the constraint really exists. Verification is not redundant: - `_init_schema` logs and swallows per-constraint failures, so calling it - proves nothing on its own. - - Args: - driver: Connected graph driver. - spec: Loaded idea-portfolio domain spec. + property, so without one two concurrent initial hydrations can each create a + canonical state node and each commit a child revision. Runs the existing + schema-init contract (the same `_init_schema` the `admin`/`init_schema` + subaction invokes), then verifies the constraint really exists — `_init_schema` + swallows per-constraint failures, so calling it proves nothing on its own. Raises: IdeaPortfolioHydrationError: The constraint is absent after schema init. From a4c5817ac469a38eaf9ad44810aa4fe27e22416a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 14:29:50 +0000 Subject: [PATCH 29/32] =?UTF-8?q?fix(idea-portfolio):=20restore=20the=20el?= =?UTF-8?q?lipsis=20protocol=20body=20=E2=80=94=20STUB-001=20bans=20the=20?= =?UTF-8?q?raise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the `raise NotImplementedError` from 6404959, which turned `Scan for Contract Violations` and `compliance` red on this PR. tools/contract_scanner.py rejects it: [STUB-001] engine/sync/idea_portfolio.py:222 (CRITICAL) Stub in engine/ - unimplemented code path The two bots want opposite things. github-code-quality reads the ellipsis as a no-op statement and proposes `raise NotImplementedError`; this repository's own contract scanner classifies that raise as a stub in engine/ and blocks the merge on it. The repository's contract is the authority — it is a blocking CI gate and repository law, where the bot comment is advisory. This is also what the audit's closure condition meant by "the repository-accepted no-op Protocol body": the form already used by engine/hoprag/indexer.py GraphStore, which is a docstring followed by an ellipsis, and which the scanner accepts. Verified before pushing this time, including the gates the previous push skipped: contract scanner clean (no violations), verify_contracts 27/27 present and wired, ruff, ruff format, mypy engine/ 134 files, 1425 unit tests, 2010 tests in CI shape. The github-code-quality thread cannot be satisfied as proposed; answering it on the PR rather than breaking a blocking contract to silence it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- engine/sync/idea_portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 4ba9da20..e8a48697 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -219,7 +219,7 @@ async def execute_write( **kwargs: Any, ) -> dict[str, Any] | Any: """Run one managed write transaction, via a transaction function or `cypher`.""" - raise NotImplementedError + ... @dataclass(frozen=True) From 3b16333e591ec72344448fd393c4934c3e22f7db Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 23:35:28 +0000 Subject: [PATCH 30/32] test(idea-portfolio): close CEG-262-002 concurrency validation on real Neo4j MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/integration/test_idea_portfolio_hydration.py, the closing validation audit ceg-open-prs-2026-09-17 names for CEG-262-002, and fixes a fail-open bug in the precondition that only a real server could reveal. The bug: state_uniqueness_constraint_present accepted NODE_PROPERTY_UNIQUENESS but not UNIQUENESS. Neo4j 5.18 — the server tests/conftest.py pins — reports `type: 'UNIQUENESS'`; the rename to NODE_PROPERTY_UNIQUENESS is later. So on the pinned server the predicate returned False even with the constraint correctly created by _init_schema, and _require_state_uniqueness would have refused every hydration permanently. Verified directly against 5.18.1: u1 type='UNIQUENESS' props=['a'] # IS UNIQUE k1 type='NODE_KEY' props=['b'] # IS NODE KEY e1 type='NODE_PROPERTY_EXISTENCE' props=['c'] # IS NOT NULL Both spellings are now accepted, so the check works on the pinned server and on later ones. The hand-written rows in the unit-level proof could not catch this — they encoded the same wrong assumption as the code. That is precisely why the audit required a real-Neo4j validation rather than a mocked one. The tests: - the predicate agrees with a real server before and after the constraint exists - two concurrent initial envelopes yield exactly one commit, one rejection, one canonical state node, and a committed revision matching the winner's receipt - replaying the committed envelope is reused, not a second revision - a child naming the wrong parent revision is rejected Run against neo4j:5.18-enterprise via the existing testcontainers fixtures: 4 passed. Contract scanner clean, ruff, ruff format, mypy engine/ 134 files, 1425 unit tests green. Note for reviewers: this adds ~193 lines and pushes #262 further past the 1000-addition size policy, which was already failing at 1032. That gate needs an owner decision either way; it is not a reason to omit the validation the audit requires. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- engine/sync/idea_portfolio.py | 7 +- .../test_idea_portfolio_hydration.py | 204 ++++++++++++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_idea_portfolio_hydration.py diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index e8a48697..32d8fdf4 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -30,8 +30,11 @@ STATE_LABEL = "IdeaPortfolioHydrationState" STATE_ID_PROPERTY = "state_id" SHOW_CONSTRAINTS_CYPHER = "SHOW CONSTRAINTS YIELD labelsOrTypes, properties, type, entityType" -# Neo4j 5 spelling; 4.x called uniqueness UNIQUENESS. NODE_KEY implies it too. -_UNIQUENESS_CONSTRAINT_TYPES = frozenset({"NODE_PROPERTY_UNIQUENESS", "NODE_KEY"}) +# Both spellings are load-bearing: 5.18 (the pinned test server) reports +# UNIQUENESS, later versions renamed it NODE_PROPERTY_UNIQUENESS. Accepting only +# the new name fails open — the constraint exists and we would not see it. +# NODE_KEY is uniqueness plus existence, so it satisfies the precondition too. +_UNIQUENESS_CONSTRAINT_TYPES = frozenset({"UNIQUENESS", "NODE_PROPERTY_UNIQUENESS", "NODE_KEY"}) _MODEL_CONFIG = ConfigDict(extra="forbid") PROJECTION_SCHEMA = "ideaos.idea-graph-projection/v1" SYNC_RECORD_SCHEMA = "ceg.idea-portfolio-sync-record/v1" diff --git a/tests/integration/test_idea_portfolio_hydration.py b/tests/integration/test_idea_portfolio_hydration.py new file mode 100644 index 00000000..f63a82ad --- /dev/null +++ b/tests/integration/test_idea_portfolio_hydration.py @@ -0,0 +1,204 @@ +"""Integration tests — idea-portfolio hydration against a real Neo4j. + +Closes the concurrency half of audit finding CEG-262-002. The hydrator +serialises revisions with `MERGE (state:IdeaPortfolioHydrationState {state_id})`, +and MERGE is only single-node-safe when a uniqueness constraint covers the merged +property. The unit suite exercises this against a mocked transaction, so it +cannot observe the constraint at all — these tests use a real database. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +import pytest_asyncio + +from engine.sync.idea_portfolio import ( + DOMAIN_ID, + SHOW_CONSTRAINTS_CYPHER, + STATE_ID_PROPERTY, + STATE_LABEL, + IdeaPortfolioHydrationError, + IdeaPortfolioHydrator, + compile_hydration_plan, + state_uniqueness_constraint_present, +) + +pytestmark = pytest.mark.integration + + +def _digest(char: str = "a") -> str: + return "sha256:" + char * 64 + + +def _envelope(idea_id: str, *, expected: str | None = None, snapshot: str = "b") -> dict[str, Any]: + return { + "schema": "ceg.idea-portfolio-hydration/v1", + "source_snapshot_ref": f"Quantum-L9/IdeaOS@{idea_id}", + "source_snapshot_digest": _digest(snapshot), + "expected_graph_revision": expected, + "records": [ + { + "schema": "ceg.idea-portfolio-sync-record/v1", + "operation": "upsert", + "projection": { + "schema": "ideaos.idea-graph-projection/v1", + "idea_id": idea_id, + "source_refs": [f"Ideas/{idea_id}.md"], + "source_digest": _digest(), + "lifecycle": {"stage": "expanded", "decision": None, "proof_state": "P1"}, + "assertions": [ + { + "kind": "capability", + "relation": "produces", + "key": "shared-capability", + "evidence_state": "VERIFIED", + "source_refs": [f"Ideas/{idea_id}.md#cap"], + } + ], + "unknowns": [], + }, + } + ], + } + + +@pytest_asyncio.fixture +async def idea_portfolio_db(graph_driver) -> Any: + """Create the idea-portfolio database and reset its hydration state per test.""" + await graph_driver.execute_query( + cypher=f"CREATE DATABASE `{DOMAIN_ID}` IF NOT EXISTS WAIT", + parameters={}, + database="system", + ) + await graph_driver.execute_write(cypher="MATCH (n) DETACH DELETE n", parameters={}, database=DOMAIN_ID) + yield graph_driver + await graph_driver.execute_write(cypher="MATCH (n) DETACH DELETE n", parameters={}, database=DOMAIN_ID) + + +async def _drop_state_constraint(driver: Any) -> None: + rows = await driver.execute_query(cypher=SHOW_CONSTRAINTS_CYPHER, parameters={}, database=DOMAIN_ID) + if not state_uniqueness_constraint_present(rows): + return + named = await driver.execute_query( + cypher="SHOW CONSTRAINTS YIELD name, labelsOrTypes, properties, type, entityType", + parameters={}, + database=DOMAIN_ID, + ) + for row in named: + if STATE_LABEL in (row.get("labelsOrTypes") or []) and list(row.get("properties") or []) == [STATE_ID_PROPERTY]: + await driver.execute_query( + cypher=f"DROP CONSTRAINT `{row['name']}` IF EXISTS", parameters={}, database=DOMAIN_ID + ) + + +async def _create_state_constraint(driver: Any) -> None: + await driver.execute_query( + cypher=( + f"CREATE CONSTRAINT idea_portfolio_state_id IF NOT EXISTS " + f"FOR (n:{STATE_LABEL}) REQUIRE n.{STATE_ID_PROPERTY} IS UNIQUE" + ), + parameters={}, + database=DOMAIN_ID, + ) + + +async def _count_state_nodes(driver: Any) -> int: + rows = await driver.execute_query( + cypher=f"MATCH (s:{STATE_LABEL}) RETURN count(s) AS n", parameters={}, database=DOMAIN_ID + ) + return int(rows[0]["n"]) if rows else 0 + + +@pytest.mark.asyncio +async def test_show_constraints_predicate_matches_a_real_constraint(idea_portfolio_db) -> None: + """The predicate agrees with a real server, not just with hand-written rows. + + Guards the uniqueness-constraint rename. Neo4j 5.18 — the server this suite + pins — reports `type: 'UNIQUENESS'`; later versions report + `NODE_PROPERTY_UNIQUENESS`. A predicate accepting only one spelling fails + open: the constraint exists and is not seen, so hydration refuses to run + forever. Hand-written rows cannot catch that; this test did. + """ + driver = idea_portfolio_db + await _drop_state_constraint(driver) + rows = await driver.execute_query(cypher=SHOW_CONSTRAINTS_CYPHER, parameters={}, database=DOMAIN_ID) + assert state_uniqueness_constraint_present(rows) is False + + await _create_state_constraint(driver) + rows = await driver.execute_query(cypher=SHOW_CONSTRAINTS_CYPHER, parameters={}, database=DOMAIN_ID) + assert state_uniqueness_constraint_present(rows) is True + + +@pytest.mark.asyncio +async def test_concurrent_initial_hydration_yields_one_state_and_one_revision(idea_portfolio_db) -> None: + """Two concurrent initial envelopes cannot fork the canonical revision chain. + + Both envelopes declare expected_graph_revision=None, so both are claiming to + be the first write. With the uniqueness constraint in place exactly one may + commit; the other must fail rather than create a second canonical state node + or a second committed child revision. + """ + driver = idea_portfolio_db + await _create_state_constraint(driver) + + hydrator = IdeaPortfolioHydrator(driver, enabled=True) + first = _envelope("idea-alpha", snapshot="b") + second = _envelope("idea-beta", snapshot="c") + + results = await asyncio.gather(hydrator.apply(first), hydrator.apply(second), return_exceptions=True) + + applied = [r for r in results if isinstance(r, dict)] + failed = [r for r in results if isinstance(r, BaseException)] + + assert len(applied) == 1, f"expected exactly one commit, got {len(applied)}: {results}" + assert len(failed) == 1, f"expected exactly one rejection, got {len(failed)}: {results}" + assert isinstance(failed[0], IdeaPortfolioHydrationError | Exception) + + assert await _count_state_nodes(driver) == 1, "a second canonical state node was created" + + rows = await driver.execute_query( + cypher=f"MATCH (s:{STATE_LABEL}) RETURN s.current_revision AS rev", + parameters={}, + database=DOMAIN_ID, + ) + committed = rows[0]["rev"] + winner = applied[0] + assert committed == winner["graph_revision"], "committed revision does not match the winning receipt" + assert committed in { + compile_hydration_plan(first).graph_revision, + compile_hydration_plan(second).graph_revision, + } + + +@pytest.mark.asyncio +async def test_replay_of_committed_revision_is_reused_not_reapplied(idea_portfolio_db) -> None: + """Re-applying the committed envelope is idempotent, not a second revision.""" + driver = idea_portfolio_db + await _create_state_constraint(driver) + hydrator = IdeaPortfolioHydrator(driver, enabled=True) + + envelope = _envelope("idea-alpha") + first = await hydrator.apply(envelope) + assert first["status"] == "applied" + + replay = await hydrator.apply(envelope) + assert replay["status"] == "reused" + assert replay["graph_revision"] == first["graph_revision"] + assert await _count_state_nodes(driver) == 1 + + +@pytest.mark.asyncio +async def test_wrong_parent_revision_is_rejected(idea_portfolio_db) -> None: + """A child naming the wrong parent revision is refused, keeping the chain linear.""" + driver = idea_portfolio_db + await _create_state_constraint(driver) + hydrator = IdeaPortfolioHydrator(driver, enabled=True) + + await hydrator.apply(_envelope("idea-alpha")) + with pytest.raises(IdeaPortfolioHydrationError, match="expected parent"): + await hydrator.apply(_envelope("idea-gamma", expected=_digest("f"))) + + assert await _count_state_nodes(driver) == 1 From f6ddcadfc09139960b281c24bf1fe73b5960a9fd Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 23:39:37 +0000 Subject: [PATCH 31/32] test(idea-portfolio): hoist envelope construction out of pytest.raises MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SonarCloud python:S5778 on the file added in 3b16333: the `pytest.raises` block held three calls that could throw — `_digest`, `_envelope` and the `apply` under test — so a failure in the fixture builders would have been indistinguishable from the rejection the test asserts. Only `apply` belongs inside the block. Same shape as the surrounding suite. Behaviour unchanged: 4 passed against neo4j:5.18-enterprise, ruff and format clean, contract scanner clean. This was the only finding SonarCloud raised against the new file; the other ten open issues on this PR are pre-existing and attributed in earlier comments. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- tests/integration/test_idea_portfolio_hydration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_idea_portfolio_hydration.py b/tests/integration/test_idea_portfolio_hydration.py index f63a82ad..0c00ba88 100644 --- a/tests/integration/test_idea_portfolio_hydration.py +++ b/tests/integration/test_idea_portfolio_hydration.py @@ -198,7 +198,8 @@ async def test_wrong_parent_revision_is_rejected(idea_portfolio_db) -> None: hydrator = IdeaPortfolioHydrator(driver, enabled=True) await hydrator.apply(_envelope("idea-alpha")) + wrong_parent = _envelope("idea-gamma", expected=_digest("f")) with pytest.raises(IdeaPortfolioHydrationError, match="expected parent"): - await hydrator.apply(_envelope("idea-gamma", expected=_digest("f"))) + await hydrator.apply(wrong_parent) assert await _count_state_nodes(driver) == 1 From cbaa90cf28060073214e8aa0e2a3c9dea43583f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 19 Sep 2026 00:29:35 +0000 Subject: [PATCH 32/32] fix(idea-portfolio): report an unreadable constraint list as a clean refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running the CLI end-to-end against a live Neo4j as a least-privilege user. A database user may be denied SHOW CONSTRAINTS itself: Neo.ClientError.Security.Forbidden: Show constraints on database 'idea-portfolio' is not allowed for user 'limited' The precondition already failed closed in that case — nothing was written, the graph stayed empty — but it surfaced as a raw neo4j driver traceback. An operator could not tell "the constraint is missing" from "this user may not read constraints", and those have different remedies: one is a schema init, the other is a grant. The driver error is now caught and re-raised as IdeaPortfolioHydrationError naming the cause and the grant needed. Unverifiable is treated exactly like absent — not provably safe, so it still refuses to mutate. Verified against neo4j:5.18-enterprise, all four paths of the real CLI: restricted user, cannot read constraints -> refuses, 0 nodes written admin, constraint absent -> _init_schema creates it, applies, 1 state node admin, replay same envelope -> "reused", still 1 state node no --apply -> "validated", no mutation Contract scanner clean, ruff, ruff format, mypy engine/ 134 files, and the full suite with a real database: 2039 passed, 10 skipped, 56 xfailed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BXqHneK2x7xKy1WkbBMumX --- tools/hydrate_idea_portfolio.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/hydrate_idea_portfolio.py b/tools/hydrate_idea_portfolio.py index 1b4f48ab..735a4a87 100755 --- a/tools/hydrate_idea_portfolio.py +++ b/tools/hydrate_idea_portfolio.py @@ -12,6 +12,8 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) +from neo4j.exceptions import Neo4jError + from engine.config.loader import DomainPackLoader from engine.config.schema import DomainSpec from engine.config.settings import settings @@ -58,10 +60,21 @@ async def _require_state_uniqueness(driver: GraphDriver, spec: DomainSpec) -> No swallows per-constraint failures, so calling it proves nothing on its own. Raises: - IdeaPortfolioHydrationError: The constraint is absent after schema init. + IdeaPortfolioHydrationError: The constraint is absent, or could not be read. """ await _init_schema(driver, spec) - rows = await driver.execute_query(SHOW_CONSTRAINTS_CYPHER, {}, database=DOMAIN_ID) + try: + rows = await driver.execute_query(SHOW_CONSTRAINTS_CYPHER, {}, database=DOMAIN_ID) + except Neo4jError as exc: + # A least-privilege database user may be denied SHOW CONSTRAINTS itself. + # Unreadable is not provably safe, so it fails closed like an absent + # constraint — but says so, since the remedy is a grant, not a schema init. + msg = ( + f"idea-portfolio schema precondition unverifiable: could not read constraints on " + f"'{DOMAIN_ID}' ({type(exc).__name__}: {exc}). Refusing to mutate. Grant this user " + f"SHOW CONSTRAINT on the database, or run hydration as a user that holds it." + ) + raise IdeaPortfolioHydrationError(msg) from exc if not state_uniqueness_constraint_present(rows): msg = ( f"idea-portfolio schema precondition unmet: no uniqueness constraint on "