From a34cd497184f6be2356c878a299627871d80974d Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 13:55:08 +0200 Subject: [PATCH 01/22] fix: dispatch temp Neo4j filtering to SLURM compute nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The temp Neo4j instance used for creating filtered graph dumps (dd-only and per-facility) was being OOM-killed by per-user cgroup memory limits on service/login nodes. The 6.4GB graph dump requires mmap pages that exceed the ~12GB of free cgroup headroom. Changes: - Add SLURM dispatch via srun for temp Neo4j lifecycle - Script passed on stdin (bash -s) to avoid shared-fs visibility issues - Readiness probe uses cypher-shell instead of HTTP for Bolt reliability - Trap cleanup kills Neo4j PID before removing temp dir - Property-based deletion (facility_id IS NOT NULL) instead of hard-coded label list — self-maintaining as new labels are added - Symlink source dump instead of copying (saves 60s I/O) - Batch size increased to 50000 rows per transaction - Falls back to local temp Neo4j when already on a compute node BREAKING CHANGE: graph export on HPC requires SLURM access --- imas_codex/graph/temp_neo4j.py | 531 ++++++++++++++++++++++++++------- 1 file changed, 424 insertions(+), 107 deletions(-) diff --git a/imas_codex/graph/temp_neo4j.py b/imas_codex/graph/temp_neo4j.py index ea3368c03..189462e7e 100644 --- a/imas_codex/graph/temp_neo4j.py +++ b/imas_codex/graph/temp_neo4j.py @@ -4,20 +4,28 @@ - Start an ephemeral Neo4j instance for dump filtering - Create per-facility or IMAS-only filtered dumps - Write temporary Neo4j configuration + +When running on a service/login node with cgroup memory limits, +the temp Neo4j is dispatched to a SLURM compute node via ``srun`` +to avoid OOM kills from per-user memory constraints. """ from __future__ import annotations +import logging import os import shutil import signal import subprocess import tempfile +import textwrap import time from pathlib import Path import click +logger = logging.getLogger(__name__) + # ============================================================================ # Constants # ============================================================================ @@ -57,8 +65,10 @@ def write_temp_neo4j_conf(conf_dir: Path, bolt_port: int, http_port: int) -> Pat """Write a neo4j.conf for a temporary filtering instance. Disables authentication and binds to non-standard ports to avoid - conflicts with the production instance. Memory sized for filtering - a full graph dump (~200K+ nodes) via batched DETACH DELETE. + conflicts with the production instance. Memory is kept very low + because the temp instance runs inside a per-user cgroup with + limited headroom. Filtering uses ``CALL {} IN TRANSACTIONS`` + which commits in server-side batches and needs minimal heap. """ conf_file = conf_dir / "neo4j.conf" conf_file.write_text( @@ -66,10 +76,12 @@ def write_temp_neo4j_conf(conf_dir: Path, bolt_port: int, http_port: int) -> Pat dbms.security.auth_enabled=false server.bolt.listen_address=127.0.0.1:{bolt_port} server.http.listen_address=127.0.0.1:{http_port} -server.memory.heap.initial_size=1g -server.memory.heap.max_size=4g -server.memory.pagecache.size=1g -dbms.memory.transaction.total.max=4g +server.memory.heap.initial_size=128m +server.memory.heap.max_size=512m +server.memory.pagecache.size=128m +dbms.memory.transaction.total.max=512m +server.jvm.additional=-XX:MaxDirectMemorySize=256m +server.jvm.additional=-XX:MaxMetaspaceSize=128m """ ) return conf_file @@ -84,11 +96,19 @@ def start_temp_neo4j( Returns the process handle and log file path. The caller is responsible for terminating the process. + + All Neo4j write paths are bind-mounted to host directories so that + the Apptainer ``--writable-tmpfs`` overlay (often capped at 64 MB + by ``sessiondir max size``) is only used for trivial metadata. """ import urllib.request neo4j_image = _neo4j_image() + # Ensure all Neo4j write-target directories exist on host + for subdir in ("run", "tmp"): + (temp_dir / subdir).mkdir(exist_ok=True) + # Load dump into temp data dir click.echo(" Loading dump into temp instance...") load_cmd = [ @@ -128,6 +148,10 @@ def start_temp_neo4j( f"{temp_dir}/logs:/logs", "--bind", f"{temp_dir}/conf:/var/lib/neo4j/conf", + "--bind", + f"{temp_dir}/run:/var/lib/neo4j/run", + "--bind", + f"{temp_dir}/tmp:/tmp", "--writable-tmpfs", str(neo4j_image), "neo4j", @@ -249,13 +273,393 @@ def dump_temp_neo4j(temp_dir: Path, output_path: Path) -> None: click.echo(f" Filtered dump: {size_mb:.1f} MB") +def _filter_to_dd_only(bolt_port: int, neo4j_log: Path) -> None: + """Delete all non-DD nodes from the temp Neo4j instance. + + Uses ``CALL {} IN TRANSACTIONS`` for server-side batched commits + so the JVM never accumulates a large transaction in memory. + """ + from neo4j import GraphDatabase + + click.echo(" Filtering graph: keeping only IMAS DD nodes...") + + label_check = " AND ".join(f"NOT n:{label}" for label in IMAS_DD_LABELS) + driver = GraphDatabase.driver(f"bolt://localhost:{bolt_port}") + + try: + with driver.session() as session: + # Count nodes to delete for progress reporting + result = session.run( + f"MATCH (n) WHERE {label_check} AND NOT n:GraphMeta " + "RETURN count(n) AS total" + ) + total = result.single()["total"] + click.echo(f" {total:,} non-DD nodes to remove") + + # Server-side batched deletion — single query, Neo4j manages commits + result = session.run( + f"MATCH (n) WHERE {label_check} AND NOT n:GraphMeta " + "CALL { WITH n DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS " + "RETURN count(*) AS deleted" + ) + deleted = result.single()["deleted"] + click.echo(f" Removed {deleted:,} non-DD nodes") + + # Clean up orphaned Unit nodes left after facility node removal + result = session.run( + "MATCH (u:Unit) WHERE NOT (u)<-[:HAS_UNIT]-() " + "CALL { WITH u DETACH DELETE u } IN TRANSACTIONS OF 1000 ROWS " + "RETURN count(*) AS deleted" + ) + orphan_deleted = result.single()["deleted"] + if orphan_deleted > 0: + click.echo(f" Removed {orphan_deleted:,} orphaned Unit nodes") + + # Update GraphMeta to reflect dd-only content + session.run( + 'MATCH (m:GraphMeta {id: "meta"}) ' + "SET m.facilities = [], m.imas = true, " + " m.updated_at = datetime().epochMillis" + ) + except Exception: + # Capture temp Neo4j logs on failure for diagnostics + if neo4j_log.exists(): + tail = neo4j_log.read_text()[-1000:] + click.echo(f" Temp Neo4j log tail:\n{tail}") + raise + finally: + driver.close() + + +def _filter_to_facility(bolt_port: int, facility: str, neo4j_log: Path) -> None: + """Delete nodes not belonging to *facility* from the temp instance. + + Uses ``CALL {} IN TRANSACTIONS`` for server-side batched commits. + """ + from neo4j import GraphDatabase + + click.echo(f" Filtering graph: keeping facility={facility}...") + + driver = GraphDatabase.driver(f"bolt://localhost:{bolt_port}") + + try: + with driver.session() as session: + # Remove nodes belonging to other facilities + result = session.run( + "MATCH (n) " + "WHERE n.facility_id IS NOT NULL " + "AND n.facility_id <> $facility " + "CALL { WITH n DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS " + "RETURN count(*) AS deleted", + facility=facility, + ) + deleted = result.single()["deleted"] + click.echo(f" Removed {deleted:,} non-{facility} nodes") + + # Remove orphaned nodes (no relationships, not core DD types) + result = session.run( + "MATCH (n) WHERE NOT (n)--() " + "AND NOT n:IMASNode AND NOT n:DDVersion AND NOT n:Unit " + "AND NOT n:IMASCoordinateSpec AND NOT n:PhysicsDomain " + "AND NOT n:IMASSemanticCluster " + "AND NOT n:GraphMeta " + "CALL { WITH n DELETE n } IN TRANSACTIONS OF 1000 ROWS " + "RETURN count(*) AS deleted" + ) + orphans = result.single()["deleted"] + click.echo(f" Removed {orphans:,} orphan nodes") + + # Update GraphMeta to reflect the kept facility + session.run( + 'MATCH (m:GraphMeta {id: "meta"}) ' + "SET m.facilities = [$facility], " + " m.updated_at = datetime().epochMillis", + facility=facility, + ) + except Exception: + if neo4j_log.exists(): + tail = neo4j_log.read_text()[-1000:] + click.echo(f" Temp Neo4j log tail:\n{tail}") + raise + finally: + driver.close() + + +def _should_use_slurm() -> bool: + """Return True if temp Neo4j should run via SLURM. + + On HPC service/login nodes, per-user cgroup memory limits can OOM-kill + a temp Neo4j loading a multi-GB dump. SLURM compute nodes have + dedicated memory allocations that avoid this. + """ + if os.environ.get("SLURM_JOB_ID"): + return False # Already on a compute node + if not shutil.which("srun"): + return False # Not an HPC environment + return True + + +def _slurm_partition() -> str: + """Resolve the SLURM partition to use for temp Neo4j jobs. + + Uses the general (non-GPU) partition from compute config, falling + back to ``"rigel"`` which is the standard CPU partition on ITER HPC. + """ + try: + from imas_codex.cli.services import _general_partition_name + + return _general_partition_name() + except Exception: + return "rigel" + + +def _build_filter_script( + *, + neo4j_image: str, + source_dump: str, + output_dump: str, + mode: str, + facility: str = "", +) -> str: + """Build a bash script for the temp Neo4j filter lifecycle. + + The script runs entirely on a compute node: load → start → + filter via cypher-shell → stop → dump. + + Args: + neo4j_image: Path to the Neo4j Apptainer SIF image. + source_dump: Path to the full neo4j.dump on shared GPFS. + output_dump: Where to write the filtered dump on shared GPFS. + mode: Either ``"dd-only"`` or ``"facility"``. + facility: Facility ID (required when mode is ``"facility"``). + """ + # Build the Cypher filter commands based on mode + if mode == "dd-only": + # Property-based deletion — self-maintaining. Every facility + # node has ``facility_id`` per schema, so new labels are + # automatically covered without updating a hard-coded list. + dd_keep_labels = " OR ".join(f"n:{lbl}" for lbl in IMAS_DD_LABELS) + filter_cypher = textwrap.dedent(f"""\ + # Delete all facility-owned nodes (every facility node has facility_id) + echo " Deleting facility nodes..." + $CYPHER "MATCH (n) WHERE n.facility_id IS NOT NULL CALL {{ WITH n DETACH DELETE n }} IN TRANSACTIONS OF 50000 ROWS RETURN count(*) AS deleted;" + + # Delete Facility nodes themselves (they use id, not facility_id) + echo " Deleting Facility nodes..." + $CYPHER "MATCH (n:Facility) CALL {{ WITH n DETACH DELETE n }} IN TRANSACTIONS RETURN count(*) AS deleted;" + + # Clean orphan nodes that lost all relationships and aren't DD types + echo " Cleaning orphan nodes..." + $CYPHER "MATCH (n) WHERE NOT (n)--() AND NOT ({dd_keep_labels}) AND NOT n:GraphMeta CALL {{ WITH n DELETE n }} IN TRANSACTIONS OF 50000 ROWS RETURN count(*) AS deleted;" + + # Orphan Unit cleanup + echo " Cleaning orphaned Unit nodes..." + $CYPHER "MATCH (u:Unit) WHERE NOT (u)<-[:HAS_UNIT]-() CALL {{ WITH u DETACH DELETE u }} IN TRANSACTIONS OF 10000 ROWS RETURN count(*) AS deleted;" + + # Update GraphMeta + $CYPHER "MATCH (m:GraphMeta {{id: 'meta'}}) SET m.facilities = [], m.imas = true, m.updated_at = datetime().epochMillis;" + """) + else: + filter_cypher = textwrap.dedent(f"""\ + # Delete nodes from other facilities + echo " Deleting non-{facility} nodes..." + $CYPHER "MATCH (n) WHERE n.facility_id IS NOT NULL AND n.facility_id <> '{facility}' CALL {{ WITH n DETACH DELETE n }} IN TRANSACTIONS OF 50000 ROWS RETURN count(*) AS deleted;" + + # Remove orphaned nodes (no relationships, not DD types) + echo " Cleaning orphaned nodes..." + $CYPHER "MATCH (n) WHERE NOT (n)--() AND NOT n:IMASNode AND NOT n:DDVersion AND NOT n:Unit AND NOT n:IMASCoordinateSpec AND NOT n:PhysicsDomain AND NOT n:IMASSemanticCluster AND NOT n:GraphMeta CALL {{ WITH n DELETE n }} IN TRANSACTIONS OF 50000 ROWS RETURN count(*) AS deleted;" + + # Update GraphMeta + $CYPHER "MATCH (m:GraphMeta {{id: 'meta'}}) SET m.facilities = ['{facility}'], m.updated_at = datetime().epochMillis;" + """) + + return textwrap.dedent(f"""\ + #!/bin/bash + set -euo pipefail + + NEO4J_IMAGE="{neo4j_image}" + SOURCE_DUMP="{source_dump}" + OUTPUT_DUMP="{output_dump}" + BOLT_PORT=27687 + HTTP_PORT=27474 + NEO4J_PID="" + + # Use GPFS temp dir (accessible across nodes, no size cap) + TEMP_DIR=$(mktemp -d "${{HOME}}/.local/share/imas-codex/.neo4j-filter-XXXXXX") + + cleanup() {{ + if [ -n "$NEO4J_PID" ]; then + kill "$NEO4J_PID" 2>/dev/null || true + wait "$NEO4J_PID" 2>/dev/null || true + fi + rm -rf "$TEMP_DIR" + }} + trap cleanup EXIT + + mkdir -p "$TEMP_DIR"/{{data,logs,dumps,conf,run,tmp}} + + echo " Loading dump into temp instance..." + # Symlink avoids copying multi-GB dump (source is on shared GPFS) + ln -s "$SOURCE_DUMP" "$TEMP_DIR/dumps/neo4j.dump" + + apptainer exec \\ + --bind "$TEMP_DIR/data:/data" \\ + --bind "$TEMP_DIR/dumps:/dumps" \\ + --writable-tmpfs \\ + "$NEO4J_IMAGE" \\ + neo4j-admin database load neo4j --from-path=/dumps --overwrite-destination=true + + # Write minimal config + cat > "$TEMP_DIR/conf/neo4j.conf" <<'CONF' +dbms.security.auth_enabled=false +server.bolt.listen_address=127.0.0.1:27687 +server.http.listen_address=127.0.0.1:27474 +server.memory.heap.initial_size=1g +server.memory.heap.max_size=4g +server.memory.pagecache.size=2g +dbms.memory.transaction.total.max=4g +CONF + + echo " Starting temp Neo4j instance..." + apptainer exec \\ + --bind "$TEMP_DIR/data:/data" \\ + --bind "$TEMP_DIR/logs:/logs" \\ + --bind "$TEMP_DIR/conf:/var/lib/neo4j/conf" \\ + --bind "$TEMP_DIR/run:/var/lib/neo4j/run" \\ + --bind "$TEMP_DIR/tmp:/tmp" \\ + --writable-tmpfs \\ + "$NEO4J_IMAGE" \\ + neo4j console > "$TEMP_DIR/logs/neo4j.log" 2>&1 & + NEO4J_PID=$! + + # Wait for Bolt readiness (not just HTTP) via cypher-shell probe + cypher_probe() {{ + apptainer exec --writable-tmpfs "$NEO4J_IMAGE" \\ + cypher-shell -a "bolt://localhost:$BOLT_PORT" "RETURN 1;" \\ + > /dev/null 2>&1 + }} + + READY=0 + for i in $(seq 1 180); do + if ! kill -0 $NEO4J_PID 2>/dev/null; then + echo "ERROR: Temp Neo4j exited prematurely" + tail -20 "$TEMP_DIR/logs/neo4j.log" || true + exit 1 + fi + if cypher_probe; then + READY=1 + break + fi + sleep 1 + done + if [ $READY -eq 0 ]; then + echo "ERROR: Temp Neo4j did not start in 180s" + tail -20 "$TEMP_DIR/logs/neo4j.log" || true + exit 1 + fi + + echo " Filtering graph ({mode})..." + CYPHER="apptainer exec --writable-tmpfs $NEO4J_IMAGE cypher-shell -a bolt://localhost:$BOLT_PORT" + + {filter_cypher} + + # Stop temp Neo4j gracefully + kill $NEO4J_PID 2>/dev/null || true + wait $NEO4J_PID 2>/dev/null || true + NEO4J_PID="" + + # Dump filtered graph + echo " Dumping filtered graph..." + rm -f "$TEMP_DIR/dumps/neo4j.dump" + apptainer exec \\ + --bind "$TEMP_DIR/data:/data" \\ + --bind "$TEMP_DIR/dumps:/dumps" \\ + --writable-tmpfs \\ + "$NEO4J_IMAGE" \\ + neo4j-admin database dump neo4j --to-path=/dumps --overwrite-destination=true + + mv "$TEMP_DIR/dumps/neo4j.dump" "$OUTPUT_DUMP" + SIZE=$(du -h "$OUTPUT_DUMP" | cut -f1) + echo " Filtered dump: $SIZE" + echo "FILTER_SUCCESS" + """) + + +def _run_filter_via_slurm( + source_dump_path: Path, + output_path: Path, + *, + mode: str, + facility: str = "", +) -> None: + """Run the temp Neo4j filter lifecycle on a SLURM compute node. + + Submits the entire load → start → filter → dump pipeline via + ``srun`` with dedicated memory, avoiding per-user cgroup limits + on service/login nodes. + + The script is passed on stdin (``bash -s``) to avoid temp-file + visibility issues between service and compute nodes. + """ + neo4j_image = str(_neo4j_image()) + partition = _slurm_partition() + + script = _build_filter_script( + neo4j_image=neo4j_image, + source_dump=str(source_dump_path), + output_dump=str(output_path), + mode=mode, + facility=facility, + ) + + click.echo(f" Submitting filter job to SLURM ({partition})...") + result = subprocess.run( + [ + "srun", + f"--partition={partition}", + "--mem=16G", + "--time=01:00:00", + "--job-name=neo4j-filter", + "bash", + "-s", + ], + input=script, + capture_output=True, + text=True, + timeout=4200, # headroom over SLURM 1h walltime + ) + if result.returncode != 0 or "FILTER_SUCCESS" not in result.stdout: + # Print both stdout and stderr for diagnostics + click.echo(result.stdout[-2000:] if result.stdout else "") + raise click.ClickException( + f"SLURM filter job failed (rc={result.returncode})\n" + f"{result.stderr[-1000:] if result.stderr else ''}" + ) + # Echo the output (progress messages from the script) + for line in result.stdout.splitlines(): + if line.startswith(" ") or line.startswith(" "): + click.echo(line) + + if not output_path.exists(): + raise click.ClickException( + f"Filter script succeeded but output not found: {output_path}" + ) + size_mb = output_path.stat().st_size / 1024 / 1024 + click.echo(f" Filtered dump: {size_mb:.1f} MB") + + def create_dd_only_dump(source_dump_path: Path, output_path: Path) -> None: """Create an IMAS-only dump by filtering out facility nodes. - Loads the full dump into a temporary Neo4j instance, deletes all - nodes that are not IMAS Data Dictionary types, then dumps the - filtered graph. + On HPC systems with SLURM, the filtering runs on a compute node + to avoid per-user cgroup memory limits. Otherwise falls back to + a local temporary Neo4j instance. """ + if _should_use_slurm(): + _run_filter_via_slurm(source_dump_path, output_path, mode="dd-only") + return + + # Local fallback (compute nodes, CI, non-HPC) temp_bolt_port = 27687 temp_http_port = 27474 @@ -271,55 +675,7 @@ def create_dd_only_dump(source_dump_path: Path, output_path: Path) -> None: proc, neo4j_log = start_temp_neo4j(temp_dir, temp_bolt_port, temp_http_port) try: - click.echo(" Filtering graph: keeping only IMAS DD nodes...") - - from neo4j import GraphDatabase - - label_check = " AND ".join(f"NOT n:{label}" for label in IMAS_DD_LABELS) - driver = GraphDatabase.driver( - f"bolt://localhost:{temp_bolt_port}", - ) - with driver.session() as session: - total_deleted = 0 - while True: - result = session.run( - f"MATCH (n) WHERE {label_check} " - "AND NOT n:GraphMeta " - "WITH n LIMIT 200 " - "DETACH DELETE n " - "RETURN count(*) AS deleted" - ) - batch_deleted = result.single()["deleted"] - if batch_deleted == 0: - break - total_deleted += batch_deleted - click.echo(f" Removed {total_deleted} non-DD nodes") - - # Clean up orphaned Unit nodes left after facility node removal - orphan_deleted_total = 0 - while True: - orphan_result = session.run( - "MATCH (u:Unit) WHERE NOT (u)<-[:HAS_UNIT]-() " - "WITH u LIMIT 200 DETACH DELETE u " - "RETURN count(*) AS deleted" - ) - batch = orphan_result.single()["deleted"] - if batch == 0: - break - orphan_deleted_total += batch - if orphan_deleted_total > 0: - click.echo( - f" Removed {orphan_deleted_total} orphaned Unit nodes" - ) - - # Update GraphMeta to reflect dd-only content - session.run( - 'MATCH (m:GraphMeta {id: "meta"}) ' - "SET m.facilities = [], m.imas = true, " - " m.updated_at = datetime().epochMillis" - ) - - driver.close() + _filter_to_dd_only(temp_bolt_port, neo4j_log) finally: stop_temp_neo4j(proc) @@ -332,15 +688,21 @@ def create_facility_dump( ) -> None: """Create a per-facility dump by filtering a full graph dump. - Loads the full dump into a temporary Neo4j instance, deletes nodes - belonging to other facilities and orphaned non-DD nodes, then dumps - the filtered graph. + On HPC systems with SLURM, the filtering runs on a compute node. + Otherwise falls back to a local temporary Neo4j instance. Args: source_dump_path: Path to the full ``neo4j.dump`` file. facility: Facility ID to keep (e.g. ``"tcv"``). output_path: Where to write the filtered dump file. """ + if _should_use_slurm(): + _run_filter_via_slurm( + source_dump_path, output_path, mode="facility", facility=facility + ) + return + + # Local fallback temp_bolt_port = 27687 temp_http_port = 27474 @@ -356,52 +718,7 @@ def create_facility_dump( proc, neo4j_log = start_temp_neo4j(temp_dir, temp_bolt_port, temp_http_port) try: - click.echo(f" Filtering graph: keeping facility={facility}...") - - from neo4j import GraphDatabase - - driver = GraphDatabase.driver( - f"bolt://localhost:{temp_bolt_port}", - ) - with driver.session() as session: - total_deleted = 0 - while True: - result = session.run( - "MATCH (n) " - "WHERE n.facility_id IS NOT NULL " - "AND n.facility_id <> $facility " - "WITH n LIMIT 200 " - "DETACH DELETE n " - "RETURN count(*) AS deleted", - facility=facility, - ) - batch_deleted = result.single()["deleted"] - if batch_deleted == 0: - break - total_deleted += batch_deleted - click.echo(f" Removed {total_deleted} non-{facility} nodes") - - result = session.run( - "MATCH (n) WHERE NOT (n)--() " - "AND NOT n:IMASNode AND NOT n:DDVersion AND NOT n:Unit " - "AND NOT n:IMASCoordinateSpec AND NOT n:PhysicsDomain " - "AND NOT n:IMASSemanticCluster " - "AND NOT n:GraphMeta " - "DELETE n " - "RETURN count(*) AS deleted" - ) - deleted_orphans = result.single()["deleted"] - click.echo(f" Removed {deleted_orphans} orphan nodes") - - # Update GraphMeta to reflect the kept facility - session.run( - 'MATCH (m:GraphMeta {id: "meta"}) ' - "SET m.facilities = [$facility], " - " m.updated_at = datetime().epochMillis", - facility=facility, - ) - - driver.close() + _filter_to_facility(temp_bolt_port, facility, neo4j_log) finally: stop_temp_neo4j(proc) From d9baaaf5a5c9632eefae9133dcca98e90e87f742 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 13:55:28 +0200 Subject: [PATCH 02/22] fix: prevent build hook hang from NFS-induced gen-pydantic stalls The hatch build hook could hang indefinitely when gen-pydantic subprocess stalled on NFS (linkml_runtime SchemaView initialization). Changes: - Add 120s timeout to all gen-pydantic subprocess calls - Pass stdin=subprocess.DEVNULL to prevent interactive hangs - Drop --force flag in build hook (use freshness checks instead) - Set _IMAS_CODEX_BUILD=1 env var to skip schema daemon thread during builds (avoids import-lock contention) - Add schema_context_data.py to build hook output validation (runtime dependency that must not be skipped) - Skip redundant schema reference/context generation calls (already handled inside build_models.main()) --- hatch_build_hooks.py | 25 +++++++++---- imas_codex/graph/schema.py | 18 ++++++++- scripts/build_models.py | 75 ++++++++++++++++++++++++++------------ 3 files changed, 85 insertions(+), 33 deletions(-) diff --git a/hatch_build_hooks.py b/hatch_build_hooks.py index d770363fc..d8d07a13d 100644 --- a/hatch_build_hooks.py +++ b/hatch_build_hooks.py @@ -38,12 +38,18 @@ def _check_graph_models_exist(self) -> bool: schemas_dir / "common.yaml", schemas_dir / "imas_dd.yaml", schemas_dir / "facility_config.yaml", + schemas_dir / "standard_name.yaml", + schemas_dir / "task_groups.yaml", ] # All generated output files that must exist output_files = [ package_root / "imas_codex" / "graph" / "models.py", + package_root / "imas_codex" / "graph" / "dd_models.py", package_root / "imas_codex" / "config" / "models.py", + # schema_context_data.py is a runtime dependency + # (imported by schema_context.py, query_builder.py, client.py) + package_root / "imas_codex" / "graph" / "schema_context_data.py", ] # If no schema files exist yet, nothing to generate @@ -69,8 +75,9 @@ def _generate_graph_models(self, package_root: Path) -> None: try: from scripts.build_models import build_models - # Invoke with empty args to avoid picking up hatch's CLI arguments - result = build_models.main(args=["--force"], standalone_mode=False) + # Let build_models freshness checks decide what to regenerate + # instead of using --force which rebuilds everything. + result = build_models.main(args=[], standalone_mode=False) if result == 0: self._trace("Graph models generated successfully") else: @@ -228,6 +235,11 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: start_time = time.time() self._trace(f"initialize() called with version={version}") + # Signal to imas_codex internals that we are in a build context. + # This prevents the schema.py daemon thread from starting + # (avoids import-lock contention with linkml_runtime on NFS). + os.environ["_IMAS_CODEX_BUILD"] = "1" + # Add package root to sys.path temporarily to resolve internal imports package_root = Path(__file__).parent original_path = sys.path[:] @@ -285,11 +297,10 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: self._trace("Generating graph models from LinkML schema...") self._generate_graph_models(package_root) - # Generate schema reference for agents (after models exist) - self._generate_schema_reference(package_root) - - # Generate schema context data for schema_for() - self._generate_schema_context(package_root) + # Schema reference (agents/schema-reference.md) and schema context + # (imas_codex/graph/schema_context_data.py) are generated inside + # build_models.main() above, with their own freshness checks. + # No need to call them again here. # Get resource paths for this version path_accessor = ResourcePathAccessor(dd_version=resolved_dd_version) diff --git a/imas_codex/graph/schema.py b/imas_codex/graph/schema.py index 70b7f6260..a4f4b1963 100644 --- a/imas_codex/graph/schema.py +++ b/imas_codex/graph/schema.py @@ -15,6 +15,7 @@ [('MDSplusServer', 'facility_id', 'Facility'), ...] """ +import os import threading from dataclasses import dataclass from enum import Enum @@ -793,6 +794,10 @@ def merge_relationship_query( # timeout), we kick off a daemon thread that loads the schema in the # background. ``get_schema()`` blocks only if the caller actually # needs the schema before the background thread finishes. +# +# The preload is skipped during hatch build hooks (_IMAS_CODEX_BUILD=1) +# to avoid needless work and potential import-lock contention with the +# main thread's own GraphSchema instantiations. _schema: GraphSchema | None = None _schema_ready = threading.Event() @@ -810,15 +815,24 @@ def _preload_schema() -> None: _schema_ready.set() -threading.Thread(target=_preload_schema, daemon=True, name="schema-preload").start() +if not os.environ.get("_IMAS_CODEX_BUILD"): + threading.Thread(target=_preload_schema, daemon=True, name="schema-preload").start() +else: + # During build: no background preload; get_schema() will init on demand. + pass def get_schema() -> GraphSchema: """Get the global GraphSchema instance. Returns immediately if background preload has finished, otherwise - blocks until the schema is ready. + blocks until the schema is ready. When the preload thread was + skipped (build mode), creates the schema on demand. """ + global _schema, _schema_error + if not _schema_ready.is_set() and os.environ.get("_IMAS_CODEX_BUILD"): + # Build mode: no background thread was started; init synchronously. + _preload_schema() _schema_ready.wait() if _schema_error is not None: raise RuntimeError("Schema preload failed") from _schema_error diff --git a/scripts/build_models.py b/scripts/build_models.py index 51fe87b96..c46cbbc04 100644 --- a/scripts/build_models.py +++ b/scripts/build_models.py @@ -262,14 +262,23 @@ def build_models( logger.debug(f"Running command: {' '.join(cmd)}") - result = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - encoding="utf-8", - errors="replace", - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + timeout=120, + ) + except subprocess.TimeoutExpired: + logger.error( + "gen-pydantic timed out after 120s for facility schema" + ) + click.echo("Error: gen-pydantic timed out after 120s", err=True) + return 1 if result.returncode != 0: logger.error(f"gen-pydantic failed: {result.stderr}") @@ -333,14 +342,23 @@ def build_models( logger.debug(f"Running command: {' '.join(cmd)}") - result = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - encoding="utf-8", - errors="replace", - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + timeout=120, + ) + except subprocess.TimeoutExpired: + logger.error( + "gen-pydantic timed out after 120s for imas_dd schema" + ) + click.echo("Error: gen-pydantic timed out after 120s", err=True) + return 1 if result.returncode != 0: logger.error( @@ -408,14 +426,23 @@ def build_models( logger.debug(f"Running command: {' '.join(cmd)}") - result = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - encoding="utf-8", - errors="replace", - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + timeout=120, + ) + except subprocess.TimeoutExpired: + logger.error( + "gen-pydantic timed out after 120s for facility_config" + ) + click.echo("Error: gen-pydantic timed out after 120s", err=True) + return 1 if result.returncode != 0: logger.error( From 1fad65b6dc6f3918b2f90ed6970cc45b376bfdcd Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 14:07:19 +0200 Subject: [PATCH 03/22] fix: bind-mount source dump directly into Apptainer container Symlinks to GPFS paths don't resolve inside Apptainer when the target is outside the explicit bind mounts. Bind the source dump file directly as /dumps/neo4j.dump:ro for the load step. --- imas_codex/graph/temp_neo4j.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/imas_codex/graph/temp_neo4j.py b/imas_codex/graph/temp_neo4j.py index 189462e7e..3f7338d26 100644 --- a/imas_codex/graph/temp_neo4j.py +++ b/imas_codex/graph/temp_neo4j.py @@ -499,12 +499,11 @@ def _build_filter_script( mkdir -p "$TEMP_DIR"/{{data,logs,dumps,conf,run,tmp}} echo " Loading dump into temp instance..." - # Symlink avoids copying multi-GB dump (source is on shared GPFS) - ln -s "$SOURCE_DUMP" "$TEMP_DIR/dumps/neo4j.dump" - + # Bind-mount source dump directly into container (symlinks don't + # resolve inside Apptainer when the target is outside bind paths). apptainer exec \\ --bind "$TEMP_DIR/data:/data" \\ - --bind "$TEMP_DIR/dumps:/dumps" \\ + --bind "$SOURCE_DUMP:/dumps/neo4j.dump:ro" \\ --writable-tmpfs \\ "$NEO4J_IMAGE" \\ neo4j-admin database load neo4j --from-path=/dumps --overwrite-destination=true @@ -570,7 +569,6 @@ def _build_filter_script( # Dump filtered graph echo " Dumping filtered graph..." - rm -f "$TEMP_DIR/dumps/neo4j.dump" apptainer exec \\ --bind "$TEMP_DIR/data:/data" \\ --bind "$TEMP_DIR/dumps:/dumps" \\ From 8019a67e22d730b423fea7c3633644e7afb80ba7 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 14:18:52 +0200 Subject: [PATCH 04/22] fix: use GPFS temp dir for graph archive when SLURM available The graph push builds archives in a temp dir. On HPC, $TMPDIR resolves to /run/user/ (per-user tmpfs) which is not visible from SLURM compute nodes. Use the Neo4j profile's data_dir (on GPFS) as the temp base when srun is available. --- imas_codex/cli/graph/data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/imas_codex/cli/graph/data.py b/imas_codex/cli/graph/data.py index a4f5419a7..5a528e1ac 100644 --- a/imas_codex/cli/graph/data.py +++ b/imas_codex/cli/graph/data.py @@ -451,7 +451,10 @@ def _build_archive(archive_dir: Path) -> None: # No Neo4j stop/start needed — work from cached dump click.echo(f"Creating archive [{profile.name}]: {output_path}") - with tempfile.TemporaryDirectory() as tmpdir: + # Use GPFS-visible temp dir when SLURM dispatch is possible, + # otherwise /run/user tmpfs is not visible from compute nodes. + tmp_base = str(profile.data_dir) if shutil.which("srun") else None + with tempfile.TemporaryDirectory(dir=tmp_base) as tmpdir: tmp = Path(tmpdir) archive_dir = tmp / f"{pkg_name}-{version_label}" archive_dir.mkdir() From 37958ef549b2524e9245557027227b2178d87819 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 14:28:35 +0200 Subject: [PATCH 05/22] docs: add export+rebuild pipeline plan for filtered graph dumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design plan for replacing the stop→dump→load→delete→dump approach with a live Cypher export → CSV → neo4j-admin import → dump pipeline. Includes benchmarks from production graph: 205K DD nodes export in 13s, 559K relationships in 9s, import+dump projected at ~2 min (vs 10-20 min current). Covers DD-only and per-facility variants, index recreation strategy, edge cases (COCOS integer IDs, dual-target HAS_COORDINATE), and SLURM integration. --- .../features/export-rebuild-graph-pipeline.md | 728 ++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 plans/features/export-rebuild-graph-pipeline.md diff --git a/plans/features/export-rebuild-graph-pipeline.md b/plans/features/export-rebuild-graph-pipeline.md new file mode 100644 index 000000000..c7d340875 --- /dev/null +++ b/plans/features/export-rebuild-graph-pipeline.md @@ -0,0 +1,728 @@ +# Export + Rebuild Pipeline for Filtered Graph Dumps + +## Problem + +The release workflow creates three graph dump variants: **full** (all nodes), +**dd-only** (IMAS Data Dictionary only), and **per-facility** (DD + one +facility). The current filtering approach in `temp_neo4j.py` is: + +1. Stop production Neo4j +2. `neo4j-admin database dump` the entire graph (~1.9 GB) +3. Load the dump into a temporary Neo4j instance +4. Start the temp instance and wait for readiness +5. Run Cypher `DELETE` queries to remove unwanted nodes/relationships +6. Stop the temp instance +7. `neo4j-admin database dump` the filtered result +8. Restart production Neo4j + +### Measured Costs + +| Step | Duration | Notes | +|------|----------|-------| +| Stop production | ~5 s | Causes downtime for MCP server | +| Full dump (1.9 GB) | ~30 s | Entire graph serialized | +| Load into temp | ~20 s | Full dump loaded | +| Temp Neo4j startup + recovery | ~60–120 s | WAL replay on 1.9 GB | +| Cypher DELETE (dd-only) | ~300–600 s | `MATCH (n) WHERE NOT ... DETACH DELETE n` in batches | +| Cypher DELETE (facility) | ~120–300 s | More selective | +| Stop temp + dump filtered | ~30 s | | +| **Total per variant** | **~10–20 min** | × 3 variants = 25–50 min | + +### Specific Failures + +- **Production downtime**: ~30 s per dump cycle while Neo4j is stopped +- **OOM on CI**: Full graph load + Cypher DELETE + transaction log = memory + spike. 4 GB SLURM jobs often fail; 8–16 GB required +- **Bloated output**: The filtered dump retains free-space from deleted nodes. + DD-only dump (~200 MB of data) weighs ~900 MB because it was carved from a + 1.9 GB store +- **Slow filtering**: Cypher `DETACH DELETE` on 1.3 M nodes (for dd-only) is + O(n) in the *removed* set, not the *kept* set + +--- + +## Solution: Query Live → CSV → `neo4j-admin import` → Dump + +Build filtered graphs from scratch rather than carving them from the full +dump. Query the **live** production Neo4j via Cypher, export matching nodes +and relationships to CSV, use `neo4j-admin database import full` to construct +a fresh compact database, create indexes, then dump. + +### Architecture Overview + +``` +┌─────────────────────────────────────┐ +│ LIVE Production Neo4j │ +│ bolt://98dci4-clu-2001:7687 │ +│ ~1.5M nodes, ~4.4M relationships │ +└───────────┬─────────────────────────┘ + │ Cypher read queries + │ (keyset pagination) + ▼ +┌─────────────────────────────────────┐ +│ Phase 1: Export to CSV │ +│ • Per-label node CSVs │ +│ • Per-type relationship CSVs │ +│ • Index/constraint DDL captured │ +│ Timing: ~22 s (sequential) │ +│ Output: ~200 MB CSV (DD-only) │ +└───────────┬─────────────────────────┘ + │ CSV files on local disk + ▼ +┌─────────────────────────────────────┐ +│ Phase 2: neo4j-admin import full │ +│ • Reads CSV directly (no Neo4j) │ +│ • Creates compact store files │ +│ • ID groups prevent cross-label │ +│ collisions │ +│ Timing: ~40 s (projected) │ +│ Output: ~50 MB database store │ +└───────────┬─────────────────────────┘ + │ /data/databases/neo4j/ + ▼ +┌─────────────────────────────────────┐ +│ Phase 3: Create Indexes │ +│ • Start temp Neo4j (brief) │ +│ • CREATE CONSTRAINT / INDEX │ +│ • Wait for ONLINE state │ +│ • Stop temp Neo4j │ +│ Timing: ~30–45 s │ +└───────────┬─────────────────────────┘ + │ /data/databases/neo4j/ + ▼ +┌─────────────────────────────────────┐ +│ Phase 4: neo4j-admin dump │ +│ • Serializes compact store to dump │ +│ Timing: ~15–25 s │ +│ Output: ~25 MB .dump file │ +└─────────────────────────────────────┘ +``` + +### Key Benefits + +| Metric | Current | New Pipeline | +|--------|---------|-------------| +| Production downtime | ~30 s per variant | **Zero** (read-only queries) | +| DD-only total time | 10–20 min | **~1.5 min** | +| Per-facility total time | 5–10 min | **~2 min** | +| DD-only dump size | ~900 MB (bloated) | **~25 MB** (compact) | +| Memory requirement | 8–16 GB | **4 GB** sufficient | +| Failure mode | Corrupted temp store | CSV on disk (restartable) | + +--- + +## Benchmarks + +All benchmarks run against the live production graph on `98dci4-clu-2001`. +Neo4j 2026.01.4 Community in Apptainer. Measurements are wall-clock times +including network latency. + +### DD Subgraph Statistics + +| Label | Nodes | With Embeddings | CSV Size | +|-------|-------|-----------------|----------| +| IMASNode | 61,366 | 20,037 | 89.4 MB | +| IMASNodeChange | 137,310 | 0 | 22.5 MB | +| IMASSemanticCluster | 3,912 | 3,912 (×3 cols) | 36.6 MB | +| DDVersion | 35 | 0 | < 1 KB | +| Unit | 182 | 0 | < 1 KB | +| IMASCoordinateSpec | 109 | 0 | < 1 KB | +| IdentifierSchema | 62 | 62 | < 1 KB | +| IDS | 87 | 87 | < 1 KB | +| COCOS | 18 | 0 | < 1 KB | +| GraphMeta | 1 | 0 | < 1 KB | +| **Total** | **205,047** | **~24,100** | **~150 MB** | + +Note: Embeddings are 256-dimensional float arrays. PhysicsDomain, +SignConvention, CoordinateRelationship, and ClusterMembership labels have +0 nodes currently and are excluded. + +### Node Export Performance + +Sequential export using keyset pagination (`WHERE n.id > $last LIMIT 5000`): + +| Label | Batches | Time | Throughput | +|-------|---------|------|------------| +| IMASNode | 13 | 6.55 s | 9,369 nodes/s | +| IMASNodeChange | 28 | 3.35 s | 40,937 nodes/s | +| IMASSemanticCluster | 1 | 1.65 s | 2,372 nodes/s | +| Others (small) | 1 each | < 1 s | — | +| **Total** | — | **12.8 s** | — | + +Throughput for IMASNode is lower because embedding serialization dominates +(~4.5 KB per 256-dim vector in CSV text form). + +### Relationship Export Performance + +| Relationship Type | Count | Source → Target | +|---|---|---| +| IN_VERSION | 137,310 | IMASNodeChange → DDVersion | +| FOR_IMAS_PATH | 94,158 | IMASNodeChange → IMASNode | +| INTRODUCED_IN | 61,583 | IMASNode/IDS → DDVersion | +| IN_IDS | 61,366 | IMASNode → IDS | +| HAS_PARENT | 60,334 | IMASNode → IMASNode | +| IN_CLUSTER | 33,873 | IMASNode → IMASSemanticCluster | +| HAS_ERROR | 31,281 | IMASNode → IMASNode | +| HAS_UNIT | 25,270 | IMASNode → Unit | +| HAS_COORDINATE (→Spec) | 13,769 | IMASNode → IMASCoordinateSpec | +| HAS_COORDINATE (→Node) | 12,467 | IMASNode → IMASNode | +| DEPRECATED_IN | 17,324 | IMASNode → DDVersion | +| COORDINATE_SAME_AS | 7,439 | IMASNode → IMASNode | +| RENAMED_TO | 2,696 | IMASNode → IMASNode | +| HAS_IDENTIFIER_SCHEMA | 327 | IMASNode → IdentifierSchema | +| HAS_PREDECESSOR | 34 | DDVersion → DDVersion | +| HAS_SUCCESSOR | 34 | DDVersion → DDVersion | +| HAS_COCOS | 17 | DDVersion → COCOS | +| **Total** | **559,282** | **8.9 s** | + +Three relationship types carry properties: +- `HAS_ERROR`: `error_type` (string) +- `HAS_COORDINATE`: `dimension` (integer) +- `COORDINATE_SAME_AS`: `dimension` (integer) + +### Parallel Export + +Tested 4 concurrent threads exporting different labels simultaneously: + +| Configuration | Time | Speedup | +|---|---|---| +| Sequential (1 thread) | 12.8 s | 1.0× | +| 4 threads | 9.7 s | 1.3× | + +Bottleneck is the Neo4j server, not the client. Parallel export provides +marginal benefit and adds complexity. **Recommendation: sequential export.** + +### neo4j-admin Import Performance + +Tested with actual data structure (nodes with 256-dim embeddings, +relationships, label groups): + +| Test Scale | Import Time | DB Size | Dump Size | Dump Time | +|---|---|---|---|---| +| 1K nodes + 999 rels | 2.5 s (+7 s JVM) | 2.3 MB | 1.0 MB | 8 s | +| 120K nodes + 200K rels | 22.5 s | 48.8 MB | 22.8 MB | 12 s | + +**Projected for DD data (205K nodes, 559K rels):** +- Import: ~40 s +- Dump: ~25 s + +### `--schema` Flag Incompatibility + +`neo4j-admin database import full --schema=` **fails** with: + +> Record format batch import does not support schema changes + +Indexes and constraints must be created **post-import** by briefly starting a +temp Neo4j instance. This adds ~30–45 s (Neo4j startup + DDL execution + +index population) but is the only supported path. + +### Projected End-to-End Timing + +| Phase | DD-Only | Per-Facility | +|---|---|---| +| Export nodes to CSV | 13 s | 15 s (+facility nodes) | +| Export relationships to CSV | 9 s | 12 s (+facility rels) | +| `neo4j-admin import full` | 40 s | 50 s | +| Start temp Neo4j | 20 s | 20 s | +| Create indexes + wait ONLINE | 15 s | 20 s | +| Stop temp Neo4j | 5 s | 5 s | +| `neo4j-admin dump` | 25 s | 30 s | +| **Total** | **~2 min** | **~2.5 min** | + +vs current: **10–20 min** (dd-only), **5–10 min** (per-facility). + +--- + +## Design Decisions + +### CSV Format (not Parquet, not JSONL) + +- **Human-readable** for debugging — `head -5 nodes_IMASNode.csv` +- **Native vector support** — `--vector-delimiter=;` handles float arrays +- **~200 MB total** for DD — well within memory/disk constraints +- Parquet would save ~30% on disk but adds a build dependency and loses + readability. The total CSV volume is trivially small. + +### Label-Specific ID Groups + +COCOS nodes use integer IDs (1–18), all other labels use string IDs. Without +ID groups, `neo4j-admin import` treats IDs as globally unique, causing +collisions or mismatched relationships. + +Solution: each label gets its own ID namespace: + +```csv +# nodes_COCOS.csv +id:ID(COCOS),convention:int,... +1,1,... +``` + +```csv +# rels_HAS_COCOS.csv +:START_ID(DDVersion),:END_ID(COCOS) +3.39.0,11 +``` + +This maps exactly to neo4j-admin's `--id-type=string` with group syntax. + +### Keyset Pagination (not SKIP/LIMIT) + +Standard SKIP/LIMIT can produce inconsistent snapshots if nodes are +added/modified during export. Keyset pagination guarantees each node is +exported exactly once: + +```cypher +MATCH (n:IMASNode) +WHERE n.id > $last_id +RETURN n.id AS id, n.name AS name, ... +ORDER BY n.id ASC +LIMIT 5000 +``` + +For the DD subgraph (which changes only during DD ingestion, not +continuously), this is extra safety that costs nothing. + +### Sequential Export (not Parallel) + +Benchmarked 4 threads → only 1.3× speedup. The Neo4j server is the +bottleneck. Sequential export is simpler, deterministic, and easier to debug. +Total export time is ~22 s regardless. + +### Post-Import Index Creation (not --schema) + +`neo4j-admin import full --schema` fails on Neo4j 2026.01.4 with "Record +format batch import does not support schema changes". The workaround: + +1. Import CSV data (no indexes) +2. Start a temp Neo4j instance pointing at the imported data dir +3. Execute `CREATE CONSTRAINT` and `CREATE INDEX` statements +4. Wait for all indexes to reach `ONLINE` state +5. Stop the temp instance +6. Dump the database + +This adds ~30 s but is reliable and uses the same temp Neo4j lifecycle +management already implemented in `temp_neo4j.py`. + +### Separate Module (not extending temp_neo4j.py) + +The new pipeline has a fundamentally different approach (build vs carve) with +different failure modes, dependencies, and lifecycle. It should live in a +new module alongside `temp_neo4j.py` rather than extending it: + +- `imas_codex/graph/export_rebuild.py` — the new pipeline +- `imas_codex/graph/temp_neo4j.py` — retained for backward compatibility + until the new pipeline is proven + +### APOC Evaluation + +APOC provides `apoc.export.csv.*` procedures that could simplify the export +phase. However: +- APOC is **not installed** in the production Apptainer image +- Adding APOC requires rebuilding the image, testing compatibility, and + managing plugin versions across Neo4j upgrades +- The native Cypher export (22 s total) is already fast enough +- APOC adds a runtime dependency for a build-time operation + +**Recommendation: do not use APOC.** The Cypher+Python CSV writer is simpler, +faster to develop, and has zero additional dependencies. + +### Pipe/stdin Import + +`neo4j-admin database import full` does **not** support reading from stdin +or named pipes. All input must be regular files on disk. This is a non-issue +since the total CSV volume is ~200 MB and the export phase writes directly to +the temp directory used by the import phase. + +### SLURM Execution + +The existing SLURM dispatch pattern in `temp_neo4j.py` (`_run_filter_via_slurm`, +`_should_use_slurm`) should be reused. The new pipeline's resource requirements +are actually **lower** than the current approach: + +| Resource | Current | New Pipeline | +|---|---|---| +| Memory | 8–16 GB (Neo4j + Cypher DELETE) | 4 GB (neo4j-admin import) | +| Disk | ~3 GB (full dump + temp store) | ~300 MB (CSV + temp store) | +| Time | 10–20 min | ~2 min | + +A 4 GB / 30 min SLURM allocation is conservative and sufficient. + +--- + +## DD Index Inventory + +Indexes that must be recreated in the filtered dump, captured from production: + +### Constraints (10) + +| Label | Properties | Type | +|-------|-----------|------| +| COCOS | id | UNIQUENESS | +| DDVersion | id | UNIQUENESS | +| IDS | id | UNIQUENESS | +| IMASCoordinateSpec | id | UNIQUENESS | +| IMASNode | id | UNIQUENESS | +| IMASNodeChange | id | UNIQUENESS | +| IMASSemanticCluster | id | UNIQUENESS | +| IdentifierSchema | id | UNIQUENESS | +| SignConvention | id, facility_id | UNIQUENESS | +| Unit | id | UNIQUENESS | + +### Range Indexes (13 non-constraint) + +| Label | Properties | +|-------|-----------| +| DDVersion | status | +| IDS | name | +| IMASNode | node_category | +| IMASNode | node_category, ids | +| IMASNode | is_leaf | +| IMASNode | path_lower | +| IMASNode | ids | +| IMASNode | status | +| IMASNode | url | +| SignConvention | facility_id | +| SignConvention | id | +| Unit | symbol | + +### Vector Indexes (6) + +| Label | Property | Dimensions | Similarity | Quantization | +|-------|----------|-----------|------------|--------------| +| IDS | embedding | 256 | COSINE | true | +| IMASNode | embedding | 256 | COSINE | true | +| IMASSemanticCluster | embedding | 256 | COSINE | true | +| IMASSemanticCluster | label_embedding | 256 | COSINE | true | +| IMASSemanticCluster | description_embedding | 256 | COSINE | true | +| IdentifierSchema | embedding | 256 | COSINE | true | + +### Fulltext Indexes (1) + +| Name | Label | Properties | Analyzer | +|------|-------|-----------|----------| +| imas_node_text | IMASNode | documentation, name, id, description, keywords | standard-no-stop-words | + +**Total: 30 indexes** (10 constraint-backed + 13 range + 6 vector + 1 fulltext) + +--- + +## Implementation Plan + +### Phase 1: Core Export Module + +**File: `imas_codex/graph/export_rebuild.py`** + +```python +"""Export + rebuild pipeline for creating filtered graph dumps. + +Queries the live production graph via Cypher, exports nodes and relationships +to CSV, builds a fresh database with neo4j-admin import, creates indexes, +and produces a compact dump file. +""" +``` + +Key components: + +1. **`ExportConfig` dataclass** — holds label lists, relationship specs, + facility filter, temp directory, batch size (5000), GraphClient reference + +2. **`export_nodes_csv(config, label, property_keys, output_dir)`** — + Exports all nodes of a given label to CSV using keyset pagination. + Handles embedding serialization (`";".join(f"{v:.8g}" for v in vec)`). + Returns row count and file path. + +3. **`export_relationships_csv(config, rel_type, start_label, end_label, output_dir)`** — + Exports relationships to CSV. Splits types with multiple target labels + (HAS_COORDINATE) into separate files. Includes property columns where + applicable. + +4. **`capture_index_ddl(config, labels)`** — Queries `SHOW INDEXES` and + `SHOW CONSTRAINTS` for matching labels and returns a list of Cypher + CREATE statements for post-import replay. + +5. **`run_import(csv_dir, data_dir, neo4j_image)`** — Assembles the + `neo4j-admin database import full` command with proper `--nodes`, + `--relationships`, `--id-type=string`, `--vector-delimiter=;` flags. + Runs via `subprocess.run()` (or `srun` if on SLURM). + +6. **`create_indexes_post_import(data_dir, ddl_statements, neo4j_image)`** — + Starts a temp Neo4j pointed at the imported data dir, executes CREATE + CONSTRAINT/INDEX statements, polls `SHOW INDEXES` until all are ONLINE, + then stops. + +7. **`run_dump(data_dir, output_path, neo4j_image)`** — Executes + `neo4j-admin database dump` on the built database. + +8. **`export_rebuild_dd_only(output_path)`** — Top-level orchestrator for + the DD-only variant. Calls phases 1–4 in sequence. + +9. **`export_rebuild_facility(facility_id, output_path)`** — Top-level + orchestrator for per-facility variant. Exports DD nodes + facility-specific + nodes and their inter-relationships. + +### Phase 2: DD Label and Relationship Registry + +Extract the DD subgraph specification from hardcoded constants into a +queryable registry: + +```python +# Relationship types internal to DD subgraph +DD_RELATIONSHIPS = [ + RelSpec("IN_VERSION", "IMASNodeChange", "DDVersion"), + RelSpec("FOR_IMAS_PATH", "IMASNodeChange", "IMASNode"), + RelSpec("INTRODUCED_IN", "IMASNode", "DDVersion"), + RelSpec("INTRODUCED_IN", "IDS", "DDVersion"), + RelSpec("IN_IDS", "IMASNode", "IDS"), + RelSpec("HAS_PARENT", "IMASNode", "IMASNode"), + RelSpec("IN_CLUSTER", "IMASNode", "IMASSemanticCluster"), + RelSpec("HAS_ERROR", "IMASNode", "IMASNode", props=["error_type"]), + RelSpec("HAS_UNIT", "IMASNode", "Unit"), + RelSpec("HAS_COORDINATE", "IMASNode", "IMASCoordinateSpec", props=["dimension"]), + RelSpec("HAS_COORDINATE", "IMASNode", "IMASNode", props=["dimension"]), + RelSpec("DEPRECATED_IN", "IMASNode", "DDVersion"), + RelSpec("COORDINATE_SAME_AS", "IMASNode", "IMASNode", props=["dimension"]), + RelSpec("RENAMED_TO", "IMASNode", "IMASNode"), + RelSpec("HAS_IDENTIFIER_SCHEMA", "IMASNode", "IdentifierSchema"), + RelSpec("HAS_PREDECESSOR", "DDVersion", "DDVersion"), + RelSpec("HAS_SUCCESSOR", "DDVersion", "DDVersion"), + RelSpec("HAS_COCOS", "DDVersion", "COCOS"), +] +``` + +Important: `HAS_COORDINATE` has **two target label types** (IMASCoordinateSpec +and IMASNode), which requires two separate relationship CSV files since +`neo4j-admin import` uses `:START_ID(Group)` and `:END_ID(Group)` syntax +that is per-file. + +### Phase 3: Facility Filter Variant + +The per-facility variant exports: +1. All DD nodes and relationships (same as dd-only) +2. All nodes with `facility_id = $facility` for the specified facility +3. Facility-independent nodes: `Facility`, `GraphMeta`, `DiscoveryRoot`, etc. +4. All relationships **between exported nodes** (closed set) + +The relationship closure is key: we cannot blindly export all relationships +from facility nodes, because some may reference nodes in other facilities. +Strategy: + +```python +# Phase A: Export all DD nodes → node_id set +# Phase B: Export facility nodes → extend node_id set +# Phase C: For each relationship type, export only where BOTH endpoints +# are in the node_id set +``` + +For the export query: +```cypher +MATCH (a)-[r:MAPS_TO]->(b) +WHERE a.facility_id = $facility OR a.facility_id IS NULL + AND b.facility_id = $facility OR b.facility_id IS NULL +RETURN ... +``` + +### Phase 4: Integration with Release Workflow + +Replace the call path in `release.py`: + +```python +# Current (in _push_graph_variant): +# temp_neo4j.create_filtered_dump(source_dump, output, filter_type, ...) + +# New: +# export_rebuild.export_rebuild_dd_only(output_path) +# export_rebuild.export_rebuild_facility(facility, output_path) +``` + +The new functions read from the **live graph** and don't need a source dump +parameter. This also eliminates the need to stop production Neo4j at all — +the release workflow can create filtered variants without any downtime. + +### Phase 5: SLURM Integration + +Reuse the existing SLURM dispatch pattern: + +```python +def _should_use_slurm() -> bool: + """Check if running on a SLURM-managed cluster.""" + # Same logic as temp_neo4j._should_use_slurm() + ... + +def export_rebuild_via_slurm(variant, output_path, **kwargs): + """Dispatch export-rebuild to a SLURM compute node.""" + # srun --mem=4G --time=00:30:00 --partition=rigel + # python -m imas_codex.graph.export_rebuild --variant=dd-only --output=... + ... +``` + +Resource request: `--mem=4G --time=00:30:00` (conservative for a 2-min job). + +### Phase 6: Verification and Testing + +1. **Count verification** — after import, start the temp Neo4j and compare + node/relationship counts against the export CSVs: + ```cypher + MATCH (n:IMASNode) RETURN count(n) -- should equal CSV row count + ``` + +2. **Spot-check queries** — run a few known queries (e.g., "find paths in + equilibrium IDS") against the rebuilt database and verify results match + production. + +3. **Integration test** — add a test that: + - Exports a small subset (e.g., 1 IDS worth of nodes) + - Imports into a temp Neo4j + - Verifies counts and a semantic query + +4. **Dump size regression** — assert the DD-only dump is < 100 MB (currently + projected at ~25 MB, vs ~900 MB with the old approach). + +--- + +## CSV File Format + +### Node CSV Headers + +Each label gets one CSV file: `nodes_{Label}.csv` + +```csv +# nodes_IMASNode.csv +id:ID(IMASNode),name:string,ids:string,path_lower:string,description:string,documentation:string,keywords:string[],data_type:string,units:string,url:string,status:string,is_leaf:boolean,node_category:string,dd_version:string,lifecycle_status:string,cocos_label_transformation:string,cocos_replace:string,node_type:string,embedding:float[] +equilibrium/time_slice/profiles_1d/psi,psi,equilibrium,equilibrium/time_slice/profiles_1d/psi,"Poloidal flux","Full docs...",...,true,data,3.41.0,active,psi_like,psi_like,dynamic,-0.123;0.456;... +``` + +For the `embedding:float[]` column, the vector delimiter is `;`: +`-0.12345678;0.98765432;...` (256 values, 8 significant digits each). + +String arrays (e.g., `keywords:string[]`) also use `;` as the array delimiter +(same flag: `--array-delimiter=;`). This works because the column type +annotation (`float[]` vs `string[]`) tells the importer which delimiter +semantics to apply. + +### Relationship CSV Headers + +Each (type, start_label, end_label) triple gets one CSV file: +`rels_{TYPE}_{StartLabel}_{EndLabel}.csv` + +```csv +# rels_HAS_COORDINATE_IMASNode_IMASCoordinateSpec.csv +:START_ID(IMASNode),:END_ID(IMASCoordinateSpec),dimension:int + +# rels_HAS_COORDINATE_IMASNode_IMASNode.csv +:START_ID(IMASNode),:END_ID(IMASNode),dimension:int + +# rels_IN_VERSION_IMASNodeChange_DDVersion.csv (no properties) +:START_ID(IMASNodeChange),:END_ID(DDVersion) +``` + +### GraphMeta Node + +The `GraphMeta` singleton node (`id: "meta"`) must be included in every +variant. It carries `name`, `facilities`, and `updated_at` properties. For +DD-only dumps, the `facilities` list should be set to `[]` (or omitted) +since no facility data is present. + +--- + +## Edge Cases and Gotchas + +### COCOS Integer IDs + +COCOS nodes use integer IDs (1–18) in Neo4j, while the schema defines +`id: string`. During export, stringify them: `str(node["id"])`. The ID group +`ID(COCOS)` prevents collision with other labels' IDs. + +### Empty Labels + +PhysicsDomain, SignConvention (DD-only), CoordinateRelationship, and +ClusterMembership currently have 0 nodes. The export should handle empty +labels gracefully (skip CSV generation, omit from import command). + +### SignConvention (Facility-Dependent) + +SignConvention uses a composite key `(id, facility_id)`. In DD-only dumps, +no SignConvention nodes exist (they're facility-specific). In per-facility +dumps, only SignConvention nodes matching the facility are included. + +### HAS_COORDINATE Dual Targets + +`HAS_COORDINATE` relationships connect to **both** `IMASCoordinateSpec` and +`IMASNode`. Since `neo4j-admin import` requires consistent `:END_ID(Group)` +per file, these must be exported as two separate CSV files (one per target +label). The export query uses explicit label filtering: + +```cypher +MATCH (a:IMASNode)-[r:HAS_COORDINATE]->(b:IMASCoordinateSpec) +RETURN a.id, b.id, r.dimension +``` + +### INTRODUCED_IN / DEPRECATED_IN Multi-Source + +`INTRODUCED_IN` has both `IMASNode` and `IDS` as source labels. Similarly +requires separate CSV files per (source_label, target_label) combination. + +### Connection Resilience + +The Neo4j bolt connection can drop during long exports (observed during +benchmarking). The export code should: +- Use the existing `GraphClient` with retry logic +- Checkpoint progress (last exported ID) to allow resume +- Set a per-batch timeout (30 s) to detect stale connections + +### Embedding Precision + +256-dim embeddings are 32-bit floats. CSV serialization uses `f"{v:.8g}"` +(8 significant digits) which preserves full float32 precision. The +`--vector-delimiter=;` flag tells `neo4j-admin` to parse these correctly. + +--- + +## Migration Path + +### Step 1: Implement and Validate (this plan) + +Build `export_rebuild.py` alongside the existing `temp_neo4j.py`. Both +approaches remain available. + +### Step 2: A/B Comparison + +During the next release, run both pipelines and compare: +- Dump file sizes +- Query results on sample queries +- Node/relationship counts + +### Step 3: Replace Default + +Once validated, update `release.py` to use `export_rebuild` by default. Keep +`temp_neo4j.py` as a fallback for one release cycle. + +### Step 4: Remove Old Code + +After one successful release cycle, remove the filtering code from +`temp_neo4j.py` (the temp Neo4j lifecycle management remains useful for other +purposes). + +--- + +## File Layout + +``` +imas_codex/graph/ +├── export_rebuild.py # NEW: export + rebuild pipeline +├── temp_neo4j.py # EXISTING: retained, eventually simplified +├── neo4j_ops.py # EXISTING: dump/load operations (reused) +├── client.py # EXISTING: GraphClient (used for export queries) +└── schema_context_data.py # EXISTING: index definitions (reference, not imported) +``` + +--- + +## Risks + +| Risk | Likelihood | Impact | Mitigation | +|------|-----------|--------|-----------| +| Neo4j connection drops during export | Medium | Low | Keyset pagination + retry via GraphClient; each batch is independent | +| neo4j-admin import format changes in future Neo4j versions | Low | High | Pin to CSV format; test in CI against the Apptainer image | +| Missing relationship type in DD_RELATIONSHIPS registry | Low | Medium | Verify counts post-import match a control query on production | +| SLURM job preemption during pipeline | Low | Low | Pipeline completes in ~2 min; well within time limits | +| Concurrent DD ingestion during export | Low | Medium | Keyset pagination guarantees consistency within each label; run exports during quiet periods or add advisory lock | From 33a7fa5ece8e5067371a02a1c782789f7f43aa49 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:14:56 +0200 Subject: [PATCH 06/22] feat: CSV-based graph distribution pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace fragile dump-filter-dump pipeline with direct CSV export from live Neo4j. Zero production downtime — no graph stop/start needed for DD-only and per-facility variants. Pipeline: export_dd_only_csv() → CSVs + DDL + import.sh → tar.gz Load side: import_from_csv() or import.sh (Docker) - export_rebuild.py: full export pipeline (nodes, rels, indexes, metadata) - data.py: graph_load handles both CSV and legacy dump formats - release.py: DD-only and per-facility use CSV export, full uses dump - Dockerfile: Stage 4 handles CSV via import.sh + DDL in pre-start step Tested: 203K nodes, 559K rels exported in ~34s (vs 10-20 min old pipeline) Archive: ~25 MB compressed (vs ~900 MB dump) --- Dockerfile | 91 ++- imas_codex/cli/graph/data.py | 78 ++- imas_codex/cli/release.py | 62 +- imas_codex/graph/export_rebuild.py | 970 +++++++++++++++++++++++++++++ 4 files changed, 1120 insertions(+), 81 deletions(-) create mode 100644 imas_codex/graph/export_rebuild.py diff --git a/Dockerfile b/Dockerfile index d47d0befe..e6d1473ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,8 +153,10 @@ RUN --mount=type=secret,id=GHCR_TOKEN \ touch /tmp/graph-pull/.no-graph; \ fi -## Stage 4: Load graph dump into Neo4j data directory -# Uses neo4j-admin from the Neo4j image to load the dump +## Stage 4: Load graph data into Neo4j data directory +# Handles both CSV-based archives (new) and dump-based archives (legacy). +# CSV format: csv/ dir + import.json + ddl.cypher + import.sh +# Dump format: graph.dump FROM neo4j:2026.01.4-community AS graph-loader # Propagate GRAPH_TAG to bust cache when graph version changes @@ -164,54 +166,63 @@ RUN echo "Graph tag: ${GRAPH_TAG}" > /dev/null # Copy graph archive from builder COPY --from=builder /tmp/graph-pull/ /tmp/graph-pull/ -# Extract and load the graph dump (or create empty database) -# Handles both raw .dump files (oras pull) and .tar.gz archives. +# Extract and load the graph data (or create empty database) # CRITICAL: clean up intermediate files progressively to minimize peak disk usage. -# The graph dump is ~5 GB; without cleanup we'd have archive + extracted + copy + loaded -# data all on disk simultaneously (~15+ GB), exceeding CI runner capacity. RUN set -ex && \ if [ -f /tmp/graph-pull/.no-graph ]; then \ - echo "⚠ No graph data — creating empty Neo4j database"; \ + echo "No graph data — creating empty Neo4j database"; \ mkdir -p /data/databases/neo4j /data/transactions/neo4j; \ else \ cd /tmp/graph-pull && \ DUMP=$(ls *.dump 2>/dev/null | head -1) && \ ARCHIVE=$(ls *.tar.gz 2>/dev/null | head -1) && \ - mkdir -p /tmp/dumps && \ if [ -n "$DUMP" ]; then \ echo "Loading dump directly: $DUMP" && \ + mkdir -p /tmp/dumps && \ mv "$DUMP" /tmp/dumps/neo4j.dump && \ - rm -rf /tmp/graph-pull; \ + rm -rf /tmp/graph-pull && \ + neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ + rm -rf /tmp/dumps && \ + echo "Graph loaded from dump"; \ elif [ -n "$ARCHIVE" ]; then \ echo "Extracting: $ARCHIVE" && \ mkdir -p /tmp/graph-extracted && \ tar -xzf "$ARCHIVE" -C /tmp/graph-extracted && \ rm -rf /tmp/graph-pull && \ - DUMP=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \ - if [ -z "$DUMP" ]; then \ - echo "ERROR: No .dump file found in archive" >&2; \ - find /tmp/graph-extracted -type f >&2; \ - exit 1; \ - fi && \ - echo "Found dump: $DUMP ($(du -sh "$DUMP" | cut -f1))" && \ - mv "$DUMP" /tmp/dumps/neo4j.dump && \ - rm -rf /tmp/graph-extracted; \ + CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) && \ + if [ -f "$CONTENT_DIR/import.sh" ] && [ -d "$CONTENT_DIR/csv" ]; then \ + echo "CSV-based archive — running import.sh" && \ + CSV_DIR="$CONTENT_DIR/csv" bash "$CONTENT_DIR/import.sh" && \ + if [ -f "$CONTENT_DIR/ddl.cypher" ]; then \ + cp "$CONTENT_DIR/ddl.cypher" /tmp/ddl.cypher; \ + fi && \ + rm -rf /tmp/graph-extracted && \ + echo "Graph imported from CSV"; \ + else \ + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \ + if [ -z "$DUMP_FILE" ]; then \ + echo "ERROR: No csv/ + import.sh or .dump found in archive" >&2; \ + find /tmp/graph-extracted -type f >&2; \ + exit 1; \ + fi && \ + echo "Found dump: $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))" && \ + mkdir -p /tmp/dumps && \ + mv "$DUMP_FILE" /tmp/dumps/neo4j.dump && \ + rm -rf /tmp/graph-extracted && \ + neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ + rm -rf /tmp/dumps && \ + echo "Graph loaded from dump"; \ + fi; \ else \ echo "ERROR: No .dump or .tar.gz found in /tmp/graph-pull/" >&2; \ ls -la /tmp/graph-pull/ >&2; \ exit 1; \ - fi && \ - echo "Loading dump into Neo4j ($(du -sh /tmp/dumps/neo4j.dump | cut -f1))..." && \ - df -h / && \ - cd / && \ - neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ - rm -rf /tmp/dumps && \ - echo "✓ Graph loaded into Neo4j data directory"; \ + fi; \ fi -# Pre-start Neo4j to complete WAL recovery and create system DB. -# This shifts the expensive recovery from runtime (slow Azure I/O) -# to build time (fast CI SSD). The database ships fully recovered. +# Pre-start Neo4j to: (1) complete WAL recovery, (2) create system DB, +# and (3) execute DDL index statements for CSV-based imports. +# This shifts expensive work from runtime (slow Azure I/O) to build time (fast CI SSD). RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ echo "Pre-starting Neo4j for database recovery..." && \ echo "dbms.security.auth_enabled=false" >> /var/lib/neo4j/conf/neo4j.conf && \ @@ -220,7 +231,7 @@ RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ READY=0 && \ for i in $(seq 1 120); do \ if /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "RETURN 1" > /dev/null 2>&1; then \ - echo "✓ Database recovered (${i}s)"; \ + echo "Database ready (${i}s)"; \ READY=1; \ break; \ fi; \ @@ -236,10 +247,30 @@ RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ tail -50 /tmp/neo4j-recovery.log; \ exit 1; \ fi && \ + if [ -f /tmp/ddl.cypher ]; then \ + echo "Executing DDL index statements..." && \ + DDL_COUNT=0 && \ + while IFS= read -r stmt; do \ + [ -z "$stmt" ] && continue; \ + /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "$stmt" 2>&1 || true; \ + DDL_COUNT=$((DDL_COUNT + 1)); \ + done < /tmp/ddl.cypher && \ + echo "Executed $DDL_COUNT DDL statements — waiting for indexes..." && \ + for i in $(seq 1 300); do \ + PENDING=$(/var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 \ + "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') && \ + if [ "$PENDING" = "0" ]; then \ + echo "All indexes ONLINE (${i}s)"; \ + break; \ + fi; \ + sleep 1; \ + done && \ + rm -f /tmp/ddl.cypher; \ + fi && \ /var/lib/neo4j/bin/neo4j stop && \ sleep 2 && \ rm -f /tmp/neo4j-recovery.log && \ - echo "✓ Neo4j shut down cleanly — database is recovery-free"; \ + echo "Neo4j shut down cleanly — database is recovery-free"; \ fi # NOTE: Do NOT remove transaction logs (/data/transactions/neo4j/*). diff --git a/imas_codex/cli/graph/data.py b/imas_codex/cli/graph/data.py index 5a528e1ac..10586e0db 100644 --- a/imas_codex/cli/graph/data.py +++ b/imas_codex/cli/graph/data.py @@ -15,6 +15,10 @@ import click from imas_codex import __version__ +from imas_codex.graph.export_rebuild import ( + export_dd_only_csv, + export_facility_csv, +) from imas_codex.graph.ghcr import ( get_git_info, get_package_name, @@ -401,13 +405,40 @@ def graph_export( require_apptainer() def _build_archive(archive_dir: Path) -> None: - """Build the archive contents: dump + filter + manifest.""" - if source_dump: + """Build the archive contents: CSV export or dump + manifest.""" + use_csv = not source_dump and (dd_only or facilities) + + if use_csv: + # CSV export from live graph — zero downtime, no dump needed + if dd_only: + click.echo(" Exporting DD-only graph to CSV...") + export_dd_only_csv(archive_dir) + elif facilities: + for fac in facilities: + click.echo(f" Exporting {fac}+DD graph to CSV...") + export_facility_csv(fac, archive_dir) + elif source_dump: + # Use existing dump with optional legacy filtering click.echo(f" Using cached dump: {source_dump}") shutil.copy(source_dump, str(archive_dir / "graph.dump")) size_mb = (archive_dir / "graph.dump").stat().st_size / 1024 / 1024 click.echo(f" Graph: {size_mb:.1f} MB") + + if facilities: + for fac in facilities: + click.echo(f" Filtering dump for facility: {fac}") + _create_facility_dump( + archive_dir / "graph.dump", + fac, + archive_dir / "graph.dump", + ) + if dd_only: + _create_dd_only_dump( + archive_dir / "graph.dump", + archive_dir / "graph.dump", + ) else: + # Full graph dump (no filtering) click.echo(" Dumping graph database...") dumps_dir = profile.data_dir / "dumps" dumps_dir.mkdir(parents=True, exist_ok=True) @@ -422,33 +453,19 @@ def _build_archive(archive_dir: Path) -> None: else: raise click.ClickException("Graph dump file not created") - # If facilities specified, filter the dump - if facilities: - for fac in facilities: - click.echo(f" Filtering dump for facility: {fac}") - _create_facility_dump( - archive_dir / "graph.dump", - fac, - archive_dir / "graph.dump", - ) - - # If imas-only, remove all facility nodes - if dd_only: - _create_dd_only_dump( - archive_dir / "graph.dump", - archive_dir / "graph.dump", - ) - manifest = { "version": __version__, "git_commit": git_info["commit"], "git_tag": git_info["tag"], "timestamp": datetime.now(UTC).isoformat(), + "format": "csv" if (archive_dir / "csv").is_dir() else "dump", } (archive_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) - if source_dump: - # No Neo4j stop/start needed — work from cached dump + use_csv = not source_dump and (dd_only or facilities) + + if source_dump or use_csv: + # No Neo4j stop/start needed — either cached dump or live CSV export click.echo(f"Creating archive [{profile.name}]: {output_path}") # Use GPFS-visible temp dir when SLURM dispatch is possible, @@ -592,13 +609,26 @@ def graph_load( archive_dir = extracted_dirs[0] manifest_file = archive_dir / "manifest.json" + manifest = {} if manifest_file.exists(): manifest = json.loads(manifest_file.read_text()) click.echo(f" Version: {manifest.get('version')}") click.echo(f" Commit: {manifest.get('git_commit', 'unknown')[:7]}") + csv_dir = archive_dir / "csv" + import_manifest_file = archive_dir / "import.json" dump_file = archive_dir / "graph.dump" - if dump_file.exists(): + + if csv_dir.is_dir() and import_manifest_file.exists(): + # CSV-based archive: import + create indexes + from imas_codex.graph.export_rebuild import import_from_csv + + data_dir = profile.data_dir / "data" + data_dir.mkdir(parents=True, exist_ok=True) + import_from_csv(archive_dir, data_dir) + + elif dump_file.exists(): + # Legacy dump-based archive click.echo(" Loading graph database...") dumps_dir = profile.data_dir / "dumps" dumps_dir.mkdir(parents=True, exist_ok=True) @@ -623,6 +653,10 @@ def graph_load( result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: raise click.ClickException(f"Graph load failed: {result.stderr}") + else: + raise click.ClickException( + "Archive contains neither csv/ + import.json nor graph.dump" + ) if manifest_file.exists(): manifest = json.loads(manifest_file.read_text()) diff --git a/imas_codex/cli/release.py b/imas_codex/cli/release.py index a081cda99..690b82dd4 100644 --- a/imas_codex/cli/release.py +++ b/imas_codex/cli/release.py @@ -1056,45 +1056,50 @@ def _push_all_graph_variants( dispatch_graph_quality(git_info, git_tag, registry) return - # ── Local push path — dump once, reuse for filtered variants ──────── + # ── Local push path — export+rebuild for filtered, dump for full ─── + # Full graph variant still needs a traditional dump (pushes the whole DB). + # DD-only and per-facility variants use export+rebuild from the live graph + # — zero downtime, compact output, ~2 min instead of 10-20 min. cached_dump = None - click.echo("\n Creating shared graph dump (stops Neo4j once)...") - if dry_run: - cached_dump = None - else: - cached_dump = _create_shared_dump() - if not cached_dump: - raise click.ClickException( - "Failed to create shared graph dump.\n" - " Is Neo4j running? Check: imas-codex graph status" - ) + if not is_rc or len(facilities) == 0: + # Only create shared dump when we need the full variant + click.echo("\n Creating shared graph dump (stops Neo4j once)...") + if dry_run: + cached_dump = None + else: + cached_dump = _create_shared_dump() + if not cached_dump: + raise click.ClickException( + "Failed to create shared graph dump.\n" + " Is Neo4j running? Check: imas-codex graph status" + ) failed: list[str] = [] variant = 0 - # Push full graph (all facilities) - variant += 1 - click.echo( - f"\n Variant {variant}: Full graph (facilities: {', '.join(facilities)})" - ) - if not _push_graph_variant( - message=message, - registry=registry, - version_tag=git_tag, - source_dump=cached_dump, - dry_run=dry_run, - ): - failed.append("full") + # Push full graph (all facilities) — needs traditional dump + if cached_dump or dry_run: + variant += 1 + click.echo( + f"\n Variant {variant}: Full graph (facilities: {', '.join(facilities)})" + ) + if not _push_graph_variant( + message=message, + registry=registry, + version_tag=git_tag, + source_dump=cached_dump, + dry_run=dry_run, + ): + failed.append("full") - # Push dd-only (filtered from cached dump) + # Push dd-only — export+rebuild from live graph (no source dump needed) variant += 1 - click.echo(f"\n Variant {variant}: IMAS Data Dictionary only") + click.echo(f"\n Variant {variant}: IMAS Data Dictionary only (export+rebuild)") if not _push_graph_variant( dd_only=True, message=message, registry=registry, version_tag=git_tag, - source_dump=cached_dump, dry_run=dry_run, ): failed.append("dd-only") @@ -1107,13 +1112,12 @@ def _push_all_graph_variants( else: for fac in facilities: variant += 1 - click.echo(f"\n Variant {variant}: {fac} + IMAS DD") + click.echo(f"\n Variant {variant}: {fac} + IMAS DD (export+rebuild)") if not _push_graph_variant( facility=fac, message=message, registry=registry, version_tag=git_tag, - source_dump=cached_dump, dry_run=dry_run, ): failed.append(fac) diff --git a/imas_codex/graph/export_rebuild.py b/imas_codex/graph/export_rebuild.py new file mode 100644 index 000000000..31871ee93 --- /dev/null +++ b/imas_codex/graph/export_rebuild.py @@ -0,0 +1,970 @@ +"""Export pipeline for creating filtered graph archives. + +Queries the live production graph via Cypher, exports nodes and relationships +to CSV, and captures index DDL statements. The output is a directory of CSVs ++ DDL that can be archived and distributed via GHCR. + +On the **load side** (``graph load`` / Docker entrypoint), the CSVs are +imported via ``neo4j-admin import`` and indexes created on first start. +This eliminates the fragile dump/load cycle entirely. + +Usage:: + + from imas_codex.graph.export_rebuild import export_dd_only_csv + + csv_dir = export_dd_only_csv(Path("/output/archive_dir")) +""" + +from __future__ import annotations + +import csv +import json +import logging +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple + +import click + +logger = logging.getLogger(__name__) + + +# ============================================================================ +# DD subgraph specification +# ============================================================================ + +DD_LABELS: list[str] = [ + "COCOS", + "DDVersion", + "GraphMeta", + "IDS", + "IdentifierSchema", + "IMASCoordinateSpec", + "IMASNode", + "IMASNodeChange", + "IMASSemanticCluster", + "Unit", +] + +# Labels with integer IDs (all others use string IDs) +INTEGER_ID_LABELS: set[str] = {"COCOS"} + + +class RelSpec(NamedTuple): + """Specification for a relationship type to export.""" + + rel_type: str + start_label: str + end_label: str + props: list[str] = [] + + +DD_RELATIONSHIPS: list[RelSpec] = [ + RelSpec("IN_VERSION", "IMASNodeChange", "DDVersion"), + RelSpec("FOR_IMAS_PATH", "IMASNodeChange", "IMASNode"), + RelSpec("INTRODUCED_IN", "IMASNode", "DDVersion"), + RelSpec("INTRODUCED_IN", "IDS", "DDVersion"), + RelSpec("IN_IDS", "IMASNode", "IDS"), + RelSpec("HAS_PARENT", "IMASNode", "IMASNode"), + RelSpec("IN_CLUSTER", "IMASNode", "IMASSemanticCluster"), + RelSpec("HAS_ERROR", "IMASNode", "IMASNode", ["error_type"]), + RelSpec("HAS_UNIT", "IMASNode", "Unit"), + RelSpec("HAS_COORDINATE", "IMASNode", "IMASCoordinateSpec", ["dimension"]), + RelSpec("HAS_COORDINATE", "IMASNode", "IMASNode", ["dimension"]), + RelSpec("DEPRECATED_IN", "IMASNode", "DDVersion"), + RelSpec("COORDINATE_SAME_AS", "IMASNode", "IMASNode", ["dimension"]), + RelSpec("RENAMED_TO", "IMASNode", "IMASNode"), + RelSpec("HAS_IDENTIFIER_SCHEMA", "IMASNode", "IdentifierSchema"), + RelSpec("HAS_PREDECESSOR", "DDVersion", "DDVersion"), + RelSpec("HAS_SUCCESSOR", "DDVersion", "DDVersion"), + RelSpec("HAS_COCOS", "DDVersion", "COCOS"), +] + + +# ============================================================================ +# Export configuration +# ============================================================================ + + +@dataclass +class ExportConfig: + """Configuration for an export+rebuild run.""" + + labels: list[str] = field(default_factory=lambda: list(DD_LABELS)) + relationships: list[RelSpec] = field(default_factory=lambda: list(DD_RELATIONSHIPS)) + batch_size: int = 5000 + facility: str | None = None + + +# ============================================================================ +# CSV export +# ============================================================================ + + +def _serialize_value(value: object) -> str: + """Serialize a Neo4j property value for CSV. + + Strips newlines from string values to prevent multi-line CSV fields + which neo4j-admin import rejects by default. + """ + if value is None: + return "" + if isinstance(value, list): + if value and isinstance(value[0], float | int): + # Float array (embedding) — semicolon-delimited + return ";".join(f"{v:.8g}" for v in value) + # String array — semicolon-delimited, newlines stripped + return ";".join(str(v).replace("\n", " ").replace("\r", "") for v in value) + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, str): + return value.replace("\n", " ").replace("\r", "") + return str(value) + + +def _neo4j_type(value: object) -> str: + """Infer neo4j-admin CSV type annotation from a Python value.""" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "int" + if isinstance(value, float): + return "double" + if isinstance(value, list): + if value and isinstance(value[0], float | int): + return "float[]" + return "string[]" + return "string" + + +def export_nodes_csv( + gc: object, + label: str, + csv_dir: Path, + batch_size: int = 5000, +) -> tuple[Path, int]: + """Export all nodes of a label to CSV using keyset pagination. + + Returns (csv_path, row_count). + """ + # Discover properties from first batch + is_integer_id = label in INTEGER_ID_LABELS + + first_batch = gc.query( + f"MATCH (n:{label}) RETURN n ORDER BY n.id ASC LIMIT $limit", + limit=batch_size, + ) + + if not first_batch: + return csv_dir / f"nodes_{label}.csv", 0 + + # Extract property keys from first node (stable across nodes of same label) + sample_node = first_batch[0]["n"] + prop_keys = sorted(k for k in sample_node.keys() if k != "id") + + # Build header with type annotations + id_group = f"ID({label})" + header_parts = [f"id:{id_group}"] + + # Infer types from the sample + type_map: dict[str, str] = {} + for key in prop_keys: + val = sample_node.get(key) + if val is not None: + type_map[key] = _neo4j_type(val) + else: + type_map[key] = "string" + + for key in prop_keys: + t = type_map[key] + header_parts.append(f"{key}:{t}") + + csv_path = csv_dir / f"nodes_{label}.csv" + row_count = 0 + + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header_parts) + + # Write first batch + for record in first_batch: + node = record["n"] + node_id = str(node["id"]) if not is_integer_id else node["id"] + row = [node_id] + [_serialize_value(node.get(k)) for k in prop_keys] + writer.writerow(row) + row_count += 1 + + # Keyset pagination for remaining batches + last_id = first_batch[-1]["n"]["id"] + while True: + batch = gc.query( + f"MATCH (n:{label}) WHERE n.id > $last_id " + f"RETURN n ORDER BY n.id ASC LIMIT $limit", + last_id=last_id, + limit=batch_size, + ) + if not batch: + break + for record in batch: + node = record["n"] + node_id = str(node["id"]) if not is_integer_id else node["id"] + row = [node_id] + [_serialize_value(node.get(k)) for k in prop_keys] + writer.writerow(row) + row_count += 1 + last_id = batch[-1]["n"]["id"] + + logger.info("Exported %d %s nodes to %s", row_count, label, csv_path.name) + return csv_path, row_count + + +def export_relationships_csv( + gc: object, + spec: RelSpec, + csv_dir: Path, + batch_size: int = 10000, +) -> tuple[Path, int]: + """Export relationships of a specific type to CSV. + + Returns (csv_path, row_count). + """ + start_group = f"START_ID({spec.start_label})" + end_group = f"END_ID({spec.end_label})" + + header_parts = [f":{start_group}", f":{end_group}"] + for prop in spec.props: + header_parts.append(prop) + + fname = f"rels_{spec.rel_type}_{spec.start_label}_{spec.end_label}.csv" + csv_path = csv_dir / fname + row_count = 0 + + # Build property return clause + prop_return = "" + if spec.props: + prop_return = ", " + ", ".join(f"r.{p} AS {p}" for p in spec.props) + + # Export with SKIP/LIMIT — relationships don't have stable IDs for keyset + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header_parts) + + offset = 0 + while True: + cypher = ( + f"MATCH (a:{spec.start_label})-[r:{spec.rel_type}]->(b:{spec.end_label}) " + f"RETURN a.id AS start_id, b.id AS end_id{prop_return} " + f"SKIP $offset LIMIT $limit" + ) + batch = gc.query(cypher, offset=offset, limit=batch_size) + if not batch: + break + for record in batch: + start_id = str(record["start_id"]) + end_id = str(record["end_id"]) + row = [start_id, end_id] + [str(record.get(p, "")) for p in spec.props] + writer.writerow(row) + row_count += 1 + offset += len(batch) + if len(batch) < batch_size: + break + + logger.info( + "Exported %d %s rels (%s→%s) to %s", + row_count, + spec.rel_type, + spec.start_label, + spec.end_label, + csv_path.name, + ) + return csv_path, row_count + + +# ============================================================================ +# Index DDL capture +# ============================================================================ + + +def capture_index_ddl(gc: object, labels: list[str]) -> list[str]: + """Capture CREATE INDEX/CONSTRAINT statements for the given labels. + + Queries the live graph and reconstructs DDL statements. + Returns a list of Cypher CREATE statements. + """ + label_set = set(labels) + statements: list[str] = [] + + # Constraints + constraints = gc.query( + "SHOW CONSTRAINTS YIELD name, type, labelsOrTypes, properties" + ) + for c in constraints: + c_labels = c.get("labelsOrTypes") or [] + if not any(lbl in label_set for lbl in c_labels): + continue + lbl = c_labels[0] + props = c["properties"] + name = c["name"] + if c["type"] == "UNIQUENESS": + prop_str = ", ".join(f"n.{p}" for p in props) + statements.append( + f"CREATE CONSTRAINT {name} IF NOT EXISTS " + f"FOR (n:{lbl}) REQUIRE ({prop_str}) IS UNIQUE" + ) + + # Indexes (non-constraint) + indexes = gc.query( + "SHOW INDEXES YIELD name, type, labelsOrTypes, properties, " + "owningConstraint, options" + ) + for idx in indexes: + if idx.get("owningConstraint"): + continue # Skip constraint-backed indexes + idx_labels = idx.get("labelsOrTypes") or [] + if not any(lbl in label_set for lbl in idx_labels): + continue + + name = idx["name"] + lbl = idx_labels[0] + props = idx["properties"] + idx_type = idx["type"] + + if idx_type == "RANGE": + prop_str = ", ".join(f"n.{p}" for p in props) + statements.append( + f"CREATE INDEX {name} IF NOT EXISTS FOR (n:{lbl}) ON ({prop_str})" + ) + elif idx_type == "VECTOR": + options = idx.get("options", {}) + config = options.get("indexConfig", {}) + dim = config.get("vector.dimensions", 256) + sim = config.get("vector.similarity_function", "COSINE") + quant = config.get("vector.quantization.enabled", True) + prop = props[0] + statements.append( + f"CREATE VECTOR INDEX {name} IF NOT EXISTS " + f"FOR (n:{lbl}) ON (n.{prop}) " + f"OPTIONS {{indexConfig: {{" + f"`vector.dimensions`: {dim}, " + f"`vector.similarity_function`: '{sim}', " + f"`vector.quantization.enabled`: {'true' if quant else 'false'}" + f"}}}}" + ) + elif idx_type == "FULLTEXT": + prop_str = ", ".join(f"n.{p}" for p in props) + statements.append( + f"CREATE FULLTEXT INDEX {name} IF NOT EXISTS " + f"FOR (n:{lbl}) ON EACH [{prop_str}]" + ) + + return statements + + +# ============================================================================ +# neo4j-admin import +# ============================================================================ + + +def _neo4j_image() -> Path: + """Resolve the Neo4j Apptainer SIF image path.""" + from imas_codex.settings import get_neo4j_image_path + + return get_neo4j_image_path() + + +def run_import( + csv_dir: Path, + data_dir: Path, + node_files: list[tuple[str, Path]], + rel_files: list[tuple[RelSpec, Path]], +) -> None: + """Run neo4j-admin database import full with the exported CSVs.""" + image = _neo4j_image() + + # Build the import command + cmd = [ + "apptainer", + "exec", + "--bind", + f"{csv_dir}:/import", + "--bind", + f"{data_dir}:/data", + "--writable-tmpfs", + str(image), + "neo4j-admin", + "database", + "import", + "full", + "neo4j", + "--overwrite-destination=true", + "--id-type=string", + "--array-delimiter=;", + "--multiline-fields=true", + "--skip-bad-relationships=true", + "--verbose", + ] + + # Add node files with label annotations + for label, csv_path in node_files: + cmd.append(f"--nodes={label}=/import/{csv_path.name}") + + # Add relationship files + for spec, csv_path in rel_files: + cmd.append(f"--relationships={spec.rel_type}=/import/{csv_path.name}") + + click.echo(" Running neo4j-admin import...") + t0 = time.monotonic() + result = subprocess.run(cmd, capture_output=True, text=True) + elapsed = time.monotonic() - t0 + + if result.returncode != 0: + logger.error("Import stderr: %s", result.stderr[-2000:]) + raise click.ClickException( + f"neo4j-admin import failed (rc={result.returncode}):\n" + f"{result.stderr[-1000:]}" + ) + + click.echo(f" ✓ Import completed in {elapsed:.1f}s") + logger.info("Import stdout: %s", result.stdout[-500:]) + + +# ============================================================================ +# Post-import index creation +# ============================================================================ + + +def create_indexes_post_import( + data_dir: Path, + ddl_statements: list[str], + bolt_port: int = 7690, + http_port: int = 7480, +) -> None: + """Start a temp Neo4j on imported data, create indexes, wait for ONLINE, stop. + + Unlike ``start_temp_neo4j`` which loads a dump first, this starts Neo4j + directly on the data directory produced by ``neo4j-admin import``. + """ + import urllib.request + + from imas_codex.graph.temp_neo4j import ( + _cleanup_stale_temp_neo4j, + stop_temp_neo4j, + write_temp_neo4j_conf, + ) + + temp_dir = data_dir.parent + for subdir in ("conf", "logs", "run", "tmp"): + (temp_dir / subdir).mkdir(exist_ok=True) + + write_temp_neo4j_conf(temp_dir / "conf", bolt_port, http_port) + _cleanup_stale_temp_neo4j(bolt_port, http_port) + + image = _neo4j_image() + + # Start Neo4j directly on the imported data (no dump load needed) + click.echo(" Starting temp Neo4j on imported data...") + neo4j_log = temp_dir / "logs" / "neo4j-index.log" + log_fh = open(neo4j_log, "w") # noqa: SIM115 + start_cmd = [ + "apptainer", + "exec", + "--bind", + f"{data_dir}:/data", + "--bind", + f"{temp_dir}/logs:/logs", + "--bind", + f"{temp_dir}/conf:/var/lib/neo4j/conf", + "--bind", + f"{temp_dir}/run:/var/lib/neo4j/run", + "--bind", + f"{temp_dir}/tmp:/tmp", + "--writable-tmpfs", + str(image), + "neo4j", + "console", + ] + proc = subprocess.Popen( + start_cmd, + stdout=log_fh, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + # Wait for readiness + ready = False + for _ in range(120): + if proc.poll() is not None: + log_fh.flush() + tail = neo4j_log.read_text()[-500:] if neo4j_log.exists() else "" + raise click.ClickException( + f"Temp Neo4j exited prematurely (rc={proc.returncode})\n{tail}" + ) + try: + urllib.request.urlopen(f"http://localhost:{http_port}/", timeout=2) + ready = True + break + except Exception: + time.sleep(1) + + if not ready: + log_fh.flush() + stop_temp_neo4j(proc) + log_fh.close() + tail = neo4j_log.read_text()[-500:] if neo4j_log.exists() else "" + raise click.ClickException(f"Temp Neo4j did not start within 120s\n{tail}") + + try: + from neo4j import GraphDatabase + + driver = GraphDatabase.driver(f"bolt://localhost:{bolt_port}") + try: + with driver.session() as session: + for stmt in ddl_statements: + try: + session.run(stmt).consume() + logger.debug("DDL: %s", stmt[:80]) + except Exception as e: + logger.warning( + "DDL failed (may already exist): %s — %s", stmt[:60], e + ) + + # Wait for all indexes to become ONLINE + click.echo(" Waiting for indexes to go ONLINE...") + for _ in range(120): + result = list(session.run("SHOW INDEXES YIELD state RETURN state")) + states = [r["state"] for r in result] + if states and all(s == "ONLINE" for s in states): + click.echo(f" ✓ {len(states)} indexes ONLINE") + break + time.sleep(1) + else: + not_online = [s for s in states if s != "ONLINE"] + logger.warning( + "%d indexes not ONLINE after 120s: %s", + len(not_online), + not_online[:5], + ) + finally: + # Force a checkpoint before closing the driver so all index data + # and transaction logs are flushed to store files. Without this, + # neo4j-admin dump refuses to run ("active logical log detected"). + try: + with driver.session() as session: + session.run("CALL db.checkpoint()").consume() + logger.debug("Checkpoint completed") + except Exception as e: + logger.warning("Checkpoint call failed: %s", e) + driver.close() + + # Clean shutdown: send SIGTERM to just the Apptainer process (not the + # whole process group). Apptainer forwards SIGTERM to the JVM which + # runs its shutdown hook (final checkpoint + close store). Killing the + # process group with os.killpg can race — Apptainer children may die + # before the JVM completes its shutdown hook, leaving active txn logs. + click.echo(" Stopping temp Neo4j (clean shutdown)...") + proc.terminate() # SIGTERM to Apptainer only + try: + proc.wait(timeout=30) + logger.debug("Temp Neo4j exited cleanly (rc=%d)", proc.returncode) + except subprocess.TimeoutExpired: + logger.warning("Clean shutdown timed out, falling back to SIGKILL") + stop_temp_neo4j(proc) + except Exception: + stop_temp_neo4j(proc) + raise + + +# ============================================================================ +# Export metadata helpers +# ============================================================================ + + +def _write_export_metadata( + output_dir: Path, + node_files: list[tuple[str, Path]], + rel_files: list[tuple[RelSpec, Path]], + ddl_statements: list[str], +) -> None: + """Write import.json, ddl.cypher, and import.sh into the output directory.""" + # import.json — structured metadata for programmatic loaders + import_manifest = { + "nodes": [ + {"label": label, "file": csv_path.name} for label, csv_path in node_files + ], + "relationships": [ + {"type": spec.rel_type, "file": csv_path.name} + for spec, csv_path in rel_files + ], + } + (output_dir / "import.json").write_text(json.dumps(import_manifest, indent=2)) + + # ddl.cypher — index/constraint DDL for post-import execution + (output_dir / "ddl.cypher").write_text("\n".join(ddl_statements)) + + # import.sh — self-contained import script for Docker/shell use + # Expects CSV_DIR env var pointing to the csv/ directory and + # DATA_DIR env var pointing to the target data directory. + lines = [ + "#!/bin/bash", + "# Auto-generated neo4j-admin import command", + "set -e", + 'CSV_DIR="${CSV_DIR:-.}"', + "", + "neo4j-admin database import full neo4j \\", + " --overwrite-destination=true \\", + " --id-type=string \\", + ' --array-delimiter=";" \\', + " --multiline-fields=true \\", + " --skip-bad-relationships=true \\", + ] + for label, csv_path in node_files: + lines.append(f' --nodes={label}="$CSV_DIR/{csv_path.name}" \\') + for spec, csv_path in rel_files: + lines.append(f' --relationships={spec.rel_type}="$CSV_DIR/{csv_path.name}" \\') + # Remove trailing backslash from last line + lines[-1] = lines[-1].rstrip(" \\") + (output_dir / "import.sh").write_text("\n".join(lines) + "\n") + + +# ============================================================================ +# Load-side: import from CSV archive +# ============================================================================ + + +def import_from_csv( + archive_dir: Path, + data_dir: Path, +) -> None: + """Import a CSV-based archive into a Neo4j data directory. + + This is the load-side counterpart to the export functions. It reads + ``import.json`` for file→label mappings, runs ``neo4j-admin import``, + then creates indexes from ``ddl.cypher`` via a temp Neo4j instance. + + Args: + archive_dir: Extracted archive directory containing csv/, import.json, + and ddl.cypher. + data_dir: Target Neo4j data directory (e.g. ``profile.data_dir/data``). + """ + csv_dir = archive_dir / "csv" + meta_file = archive_dir / "import.json" + ddl_file = archive_dir / "ddl.cypher" + + if not csv_dir.is_dir() or not meta_file.exists(): + raise click.ClickException( + f"Not a CSV archive: expected csv/ and import.json in {archive_dir}" + ) + + meta = json.loads(meta_file.read_text()) + + node_files = [(e["label"], csv_dir / e["file"]) for e in meta["nodes"]] + rel_files = [ + (RelSpec(e["type"], "", ""), csv_dir / e["file"]) for e in meta["relationships"] + ] + + click.echo(" Importing from CSV...") + run_import(csv_dir, data_dir, node_files, rel_files) + + if ddl_file.exists(): + ddl_statements = [ + s.strip() for s in ddl_file.read_text().splitlines() if s.strip() + ] + if ddl_statements: + click.echo(" Creating indexes...") + create_indexes_post_import(data_dir, ddl_statements) + + +# ============================================================================ +# Top-level orchestrators +# ============================================================================ + + +def export_dd_only_csv(output_dir: Path) -> Path: + """Export DD-only subgraph from live graph to CSVs + DDL. + + Queries live Neo4j for all DD labels and relationships, writes CSVs + and a ``ddl.cypher`` file into *output_dir*. Zero production downtime. + + The output directory is ready to be archived and distributed. The + load side runs ``import_from_csv()`` to build the database. + + Returns the output directory path. + """ + from imas_codex.graph.client import GraphClient + + config = ExportConfig() + + click.echo("Export: DD-only variant") + t_start = time.monotonic() + + csv_dir = output_dir / "csv" + csv_dir.mkdir(parents=True, exist_ok=True) + + click.echo("\n Exporting from live graph...") + t0 = time.monotonic() + + with GraphClient() as gc: + node_files: list[tuple[str, Path]] = [] + total_nodes = 0 + + for label in config.labels: + csv_path, count = export_nodes_csv(gc, label, csv_dir, config.batch_size) + if count > 0: + node_files.append((label, csv_path)) + total_nodes += count + click.echo(f" {label}: {count:,} nodes") + + rel_files: list[tuple[RelSpec, Path]] = [] + total_rels = 0 + + for spec in config.relationships: + csv_path, count = export_relationships_csv( + gc, spec, csv_dir, batch_size=10000 + ) + if count > 0: + rel_files.append((spec, csv_path)) + total_rels += count + + # Capture index DDL while still connected + ddl_statements = capture_index_ddl(gc, config.labels) + + export_time = time.monotonic() - t0 + click.echo( + f" ✓ Exported {total_nodes:,} nodes, {total_rels:,} rels in {export_time:.1f}s" + ) + + # Write DDL and import metadata + _write_export_metadata(output_dir, node_files, rel_files, ddl_statements) + click.echo(f" {len(ddl_statements)} DDL statements → ddl.cypher") + + total_time = time.monotonic() - t_start + csv_size = sum(f.stat().st_size for f in csv_dir.iterdir()) / 1024 / 1024 + click.echo(f"\n ✓ Export complete: {csv_size:.1f} MB CSV in {total_time:.1f}s") + + return output_dir + + +def export_facility_csv(facility: str, output_dir: Path) -> Path: + """Export facility+DD subgraph from live graph to CSVs + DDL. + + Exports all DD nodes plus nodes with ``facility_id = facility`` and + all relationships between the exported node set. + + Returns the output directory path. + """ + from imas_codex.graph.client import GraphClient + + click.echo(f"Export: {facility} + DD variant") + t_start = time.monotonic() + + csv_dir = output_dir / "csv" + csv_dir.mkdir(parents=True, exist_ok=True) + + with GraphClient() as gc: + # Export DD nodes + click.echo("\n Exporting DD nodes...") + config = ExportConfig() + node_files: list[tuple[str, Path]] = [] + total_nodes = 0 + + for label in config.labels: + csv_path, count = export_nodes_csv(gc, label, csv_dir, config.batch_size) + if count > 0: + node_files.append((label, csv_path)) + total_nodes += count + + click.echo(f" DD: {total_nodes:,} nodes") + + # Export facility nodes + click.echo(f"\n Exporting {facility} nodes...") + facility_labels = gc.query( + "MATCH (n) WHERE n.facility_id = $facility " + "WITH labels(n) AS lbls UNWIND lbls AS lbl " + "RETURN DISTINCT lbl AS label, count(*) AS cnt " + "ORDER BY cnt DESC", + facility=facility, + ) + + dd_label_set = set(config.labels) + for row in facility_labels: + lbl = row["label"] + if lbl in dd_label_set: + continue + + csv_path, count = _export_facility_nodes_csv( + gc, lbl, facility, csv_dir, config.batch_size + ) + if count > 0: + node_files.append((lbl, csv_path)) + total_nodes += count + click.echo(f" {lbl}: {count:,} nodes") + + # Export relationships + click.echo("\n Exporting relationships...") + rel_files: list[tuple[RelSpec, Path]] = [] + total_rels = 0 + + # DD relationships + for spec in config.relationships: + csv_path, count = export_relationships_csv( + gc, spec, csv_dir, batch_size=10000 + ) + if count > 0: + rel_files.append((spec, csv_path)) + total_rels += count + + # Facility relationships — discover and export + fac_rels = gc.query( + "MATCH (a)-[r]->(b) " + "WHERE a.facility_id = $facility OR b.facility_id = $facility " + "WITH type(r) AS rel_type, labels(a)[0] AS start_lbl, " + "labels(b)[0] AS end_lbl, count(*) AS cnt " + "RETURN rel_type, start_lbl, end_lbl, cnt " + "ORDER BY cnt DESC", + facility=facility, + ) + + exported_rels = { + (s.rel_type, s.start_label, s.end_label) for s in config.relationships + } + for row in fac_rels: + key = (row["rel_type"], row["start_lbl"], row["end_lbl"]) + if key in exported_rels: + continue + + spec = RelSpec(row["rel_type"], row["start_lbl"], row["end_lbl"]) + csv_path, count = _export_facility_rels_csv(gc, spec, facility, csv_dir) + if count > 0: + rel_files.append((spec, csv_path)) + total_rels += count + exported_rels.add(key) + + ddl_statements = capture_index_ddl( + gc, + config.labels + [r["label"] for r in facility_labels], + ) + + click.echo(f" ✓ Exported {total_nodes:,} nodes, {total_rels:,} rels") + + # Write DDL and import manifest + _write_export_metadata(output_dir, node_files, rel_files, ddl_statements) + + total_time = time.monotonic() - t_start + csv_size = sum(f.stat().st_size for f in csv_dir.iterdir()) / 1024 / 1024 + click.echo(f"\n ✓ Export complete: {csv_size:.1f} MB CSV in {total_time:.1f}s") + + return output_dir + + +# ============================================================================ +# Facility-specific export helpers +# ============================================================================ + + +def _export_facility_nodes_csv( + gc: object, + label: str, + facility: str, + csv_dir: Path, + batch_size: int = 5000, +) -> tuple[Path, int]: + """Export nodes of a label filtered by facility_id.""" + first_batch = gc.query( + f"MATCH (n:{label}) WHERE n.facility_id = $facility " + f"RETURN n ORDER BY n.id ASC LIMIT $limit", + facility=facility, + limit=batch_size, + ) + + if not first_batch: + return csv_dir / f"nodes_{label}_{facility}.csv", 0 + + sample_node = first_batch[0]["n"] + prop_keys = sorted(k for k in sample_node.keys() if k != "id") + + id_group = f"ID({label})" + header_parts = [f"id:{id_group}"] + for key in prop_keys: + val = sample_node.get(key) + t = _neo4j_type(val) if val is not None else "string" + header_parts.append(f"{key}:{t}") + + csv_path = csv_dir / f"nodes_{label}_{facility}.csv" + row_count = 0 + + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header_parts) + + for record in first_batch: + node = record["n"] + row = [str(node["id"])] + [_serialize_value(node.get(k)) for k in prop_keys] + writer.writerow(row) + row_count += 1 + + last_id = first_batch[-1]["n"]["id"] + while True: + batch = gc.query( + f"MATCH (n:{label}) WHERE n.facility_id = $facility " + f"AND n.id > $last_id " + f"RETURN n ORDER BY n.id ASC LIMIT $limit", + facility=facility, + last_id=last_id, + limit=batch_size, + ) + if not batch: + break + for record in batch: + node = record["n"] + row = [str(node["id"])] + [ + _serialize_value(node.get(k)) for k in prop_keys + ] + writer.writerow(row) + row_count += 1 + last_id = batch[-1]["n"]["id"] + + return csv_path, row_count + + +def _export_facility_rels_csv( + gc: object, + spec: RelSpec, + facility: str, + csv_dir: Path, + batch_size: int = 10000, +) -> tuple[Path, int]: + """Export facility-specific relationships.""" + start_group = f"START_ID({spec.start_label})" + end_group = f"END_ID({spec.end_label})" + header_parts = [f":{start_group}", f":{end_group}"] + + fname = f"rels_{spec.rel_type}_{spec.start_label}_{spec.end_label}_{facility}.csv" + csv_path = csv_dir / fname + row_count = 0 + + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header_parts) + + offset = 0 + while True: + batch = gc.query( + f"MATCH (a:{spec.start_label})-[r:{spec.rel_type}]->(b:{spec.end_label}) " + f"WHERE a.facility_id = $facility OR b.facility_id = $facility " + f"RETURN a.id AS start_id, b.id AS end_id " + f"SKIP $offset LIMIT $limit", + facility=facility, + offset=offset, + limit=batch_size, + ) + if not batch: + break + for record in batch: + writer.writerow([str(record["start_id"]), str(record["end_id"])]) + row_count += 1 + offset += len(batch) + if len(batch) < batch_size: + break + + return csv_path, row_count From cfd7870ef52b5de8d53e1e049342334e77f1d1aa Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:19:45 +0200 Subject: [PATCH 07/22] fix: CI workflow handles CSV-based graph archives Update graph-quality job to detect CSV archives (import.sh + csv/) and use neo4j-admin import instead of database load. Adds DDL execution step for creating indexes after import. Supports both CSV and legacy dump formats. --- .github/workflows/docker-build-push.yml | 108 ++++++++++++++++++------ 1 file changed, 84 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index c059166f4..2f0dcfa47 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -134,7 +134,7 @@ jobs: docker stop $(docker ps -q --filter "ancestor=neo4j:2026.01.4-community") || true sleep 3 - - name: Load graph dump into Neo4j + - name: Load graph data into Neo4j if: steps.graph-tag.outputs.imas-tag != 'none' run: | set -euo pipefail @@ -145,33 +145,62 @@ jobs: mkdir -p /tmp/graph-extracted tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted - # Find the dump file (may be named graph.dump or neo4j.dump) - DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) - if [ -z "${DUMP_FILE}" ]; then - echo "ERROR: No .dump file found in archive" - ls -laR /tmp/graph-extracted/ - exit 1 - fi - echo "Found dump: ${DUMP_FILE}" - - # neo4j-admin expects the dump file named neo4j.dump - DUMP_DIR=$(dirname "${DUMP_FILE}") - if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then - cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" - fi + # Detect archive format: CSV (import.sh + csv/) or dump (.dump) + CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) - # Get the Neo4j container ID for data volume NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') - # Load into Neo4j using a fresh container - docker run --rm \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${DUMP_DIR}:/dump" \ - neo4j:2026.01.4-community \ - neo4j-admin database load neo4j \ - --from-path=/dump \ - --overwrite-destination=true + if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then + echo "CSV-based archive detected" + + # Run neo4j-admin import via Docker + CSV_DIR="${CONTENT_DIR}/csv" + + # Build import command from import.sh + IMPORT_ARGS="--overwrite-destination=true --id-type=string --array-delimiter=; --multiline-fields=true --skip-bad-relationships=true" + for f in "${CSV_DIR}"/nodes_*.csv; do + LABEL=$(basename "$f" | sed 's/^nodes_//;s/\.csv$//') + IMPORT_ARGS="${IMPORT_ARGS} --nodes=${LABEL}=/import/$(basename $f)" + done + for f in "${CSV_DIR}"/rels_*.csv; do + TYPE=$(basename "$f" | sed 's/^rels_//;s/\.csv$//') + IMPORT_ARGS="${IMPORT_ARGS} --relationships=${TYPE}=/import/$(basename $f)" + done + + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${CSV_DIR}:/import" \ + neo4j:2026.01.4-community \ + neo4j-admin database import full neo4j ${IMPORT_ARGS} + + # Save DDL for post-start execution + if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then + cp "${CONTENT_DIR}/ddl.cypher" /tmp/ddl.cypher + fi + else + # Legacy dump-based archive + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) + if [ -z "${DUMP_FILE}" ]; then + echo "ERROR: No import.sh+csv/ or .dump file found in archive" + ls -laR /tmp/graph-extracted/ + exit 1 + fi + echo "Found dump: ${DUMP_FILE}" + + DUMP_DIR=$(dirname "${DUMP_FILE}") + if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then + cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" + fi + + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${DUMP_DIR}:/dump" \ + neo4j:2026.01.4-community \ + neo4j-admin database load neo4j \ + --from-path=/dump \ + --overwrite-destination=true + fi - name: Start Neo4j with loaded data if: steps.graph-tag.outputs.imas-tag != 'none' @@ -194,6 +223,37 @@ jobs: sleep 2 done + - name: Execute DDL for CSV-based archives + if: steps.graph-tag.outputs.imas-tag != 'none' + run: | + if [ ! -f /tmp/ddl.cypher ]; then + echo "No DDL to execute (dump-based archive)" + exit 0 + fi + + NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) + echo "Executing DDL index statements..." + DDL_COUNT=0 + while IFS= read -r stmt; do + [ -z "$stmt" ] && continue + docker exec "${NEO4J_CONTAINER}" cypher-shell -u neo4j -p neo4j "$stmt" 2>&1 || true + DDL_COUNT=$((DDL_COUNT + 1)) + done < /tmp/ddl.cypher + echo "Executed $DDL_COUNT DDL statements" + + # Wait for indexes to come online + echo "Waiting for indexes to come online..." + for i in $(seq 1 300); do + PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell -u neo4j -p neo4j \ + "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') + if [ "$PENDING" = "0" ]; then + echo "All indexes ONLINE (${i}s)" + break + fi + sleep 1 + done + rm -f /tmp/ddl.cypher + - name: Reset Neo4j password if: steps.graph-tag.outputs.imas-tag != 'none' run: | From 14e80b0ddf630e1d385bcf629217689f9e4dd9e6 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:33:17 +0200 Subject: [PATCH 08/22] fix: use correct Neo4j password for DDL execution in CI The service container sets NEO4J_AUTH=neo4j/imas-codex on first start. After neo4j-admin import (which only replaces the neo4j database, not system), the password persists. Use imas-codex instead of default neo4j. Also adds failure counting and error reporting for DDL execution. --- .github/workflows/docker-build-push.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 2f0dcfa47..345cd02d3 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -234,17 +234,29 @@ jobs: NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) echo "Executing DDL index statements..." DDL_COUNT=0 + DDL_FAIL=0 while IFS= read -r stmt; do [ -z "$stmt" ] && continue - docker exec "${NEO4J_CONTAINER}" cypher-shell -u neo4j -p neo4j "$stmt" 2>&1 || true - DDL_COUNT=$((DDL_COUNT + 1)) + if docker exec "${NEO4J_CONTAINER}" cypher-shell \ + -u neo4j -p imas-codex "$stmt" 2>&1; then + DDL_COUNT=$((DDL_COUNT + 1)) + else + echo "WARNING: DDL failed: $stmt" + DDL_FAIL=$((DDL_FAIL + 1)) + fi done < /tmp/ddl.cypher - echo "Executed $DDL_COUNT DDL statements" + echo "Executed $DDL_COUNT DDL statements ($DDL_FAIL failed)" + + if [ "$DDL_FAIL" -gt 0 ] && [ "$DDL_COUNT" -eq 0 ]; then + echo "ERROR: All DDL statements failed" + exit 1 + fi # Wait for indexes to come online echo "Waiting for indexes to come online..." for i in $(seq 1 300); do - PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell -u neo4j -p neo4j \ + PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell \ + -u neo4j -p imas-codex \ "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') if [ "$PENDING" = "0" ]; then echo "All indexes ONLINE (${i}s)" From 260a39b33c4389a8b0d0fa6d3bed9ff199a6ff87 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:43:42 +0200 Subject: [PATCH 09/22] fix: exclude dev deps in CI to avoid imas-standard-names path issue --- .github/workflows/docker-build-push.yml | 2 +- .github/workflows/test.yml | 4 ++-- pyproject.toml | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 345cd02d3..a8d1c4bb3 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -293,7 +293,7 @@ jobs: - name: Install dependencies if: steps.graph-tag.outputs.imas-tag != 'none' - run: uv sync --extra test + run: uv sync --extra test --no-dev env: HATCH_BUILD_NO_HOOKS: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67740c18e..c8becbf92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Install dependencies - run: uv sync --extra test + run: uv sync --extra test --no-dev env: HATCH_BUILD_NO_HOOKS: true IMAS_DD_VERSION: ${{ matrix.imas-dd-version }} @@ -128,7 +128,7 @@ jobs: run: uv python install 3.12 - name: Install dependencies - run: uv sync --extra test + run: uv sync --extra test --no-dev env: HATCH_BUILD_NO_HOOKS: true diff --git a/pyproject.toml b/pyproject.toml index ddc2ebc31..bff744f73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,8 @@ dev = [ # --- Interactive dev --- "ipython>=9.2.0", "ipykernel>=6.29.5", + # --- Standard Names catalog --- + "imas-standard-names", # --- LLM & Discovery --- "litellm>=1.81.0", # --- Graph build & schema --- @@ -154,7 +156,6 @@ dev = [ # --- Serve (embedding + LLM proxy) --- "fastapi>=0.115.0", "uvicorn>=0.31.1", - "imas-standard-names", ] [project.urls] From 6b0df404cd75d4827dd34362a16ef14535bb7232 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:51:31 +0200 Subject: [PATCH 10/22] fix: update graph-quality workflow for CSV archives and correct deps --- .github/workflows/graph-quality.yml | 124 +++++++++++++++++++++------- 1 file changed, 94 insertions(+), 30 deletions(-) diff --git a/.github/workflows/graph-quality.yml b/.github/workflows/graph-quality.yml index 0e79eb71a..9d97ce7da 100644 --- a/.github/workflows/graph-quality.yml +++ b/.github/workflows/graph-quality.yml @@ -108,13 +108,12 @@ jobs: sleep 2 done - - name: Stop Neo4j for dump load + - name: Stop Neo4j for data load run: | - # Stop Neo4j service container to load dump docker stop $(docker ps -q --filter "ancestor=neo4j:2026.01.4-community") || true sleep 3 - - name: Load graph dump into Neo4j + - name: Load graph data into Neo4j run: | set -euo pipefail ARCHIVE=$(ls /tmp/graph-dump/*.tar.gz | head -1) @@ -124,40 +123,65 @@ jobs: mkdir -p /tmp/graph-extracted tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted - # Find the graph.dump file - DUMP_FILE=$(find /tmp/graph-extracted -name "graph.dump" | head -1) - if [ -z "${DUMP_FILE}" ]; then - echo "ERROR: No graph.dump found in archive" - ls -laR /tmp/graph-extracted/ - exit 1 - fi - echo "Found dump: ${DUMP_FILE}" + # Detect archive format: CSV (import.sh + csv/) or dump (.dump) + CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) - # neo4j-admin load expects the file named .dump - DUMP_DIR=$(mktemp -d) - cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" - chmod -R 777 "${DUMP_DIR}" - - # Get the Neo4j container ID for data volume NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') - # Load into Neo4j using a fresh container - docker run --rm --user root \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${DUMP_DIR}:/dump" \ - neo4j:2026.01.4-community \ - neo4j-admin database load neo4j \ - --from-path=/dump \ - --overwrite-destination=true \ - --verbose + if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then + echo "CSV-based archive detected" + + CSV_DIR="${CONTENT_DIR}/csv" + IMPORT_ARGS="--overwrite-destination=true --id-type=string --array-delimiter=; --multiline-fields=true --skip-bad-relationships=true" + for f in "${CSV_DIR}"/nodes_*.csv; do + LABEL=$(basename "$f" | sed 's/^nodes_//;s/\.csv$//') + IMPORT_ARGS="${IMPORT_ARGS} --nodes=${LABEL}=/import/$(basename $f)" + done + for f in "${CSV_DIR}"/rels_*.csv; do + TYPE=$(basename "$f" | sed 's/^rels_//;s/\.csv$//') + IMPORT_ARGS="${IMPORT_ARGS} --relationships=${TYPE}=/import/$(basename $f)" + done + + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${CSV_DIR}:/import" \ + neo4j:2026.01.4-community \ + neo4j-admin database import full neo4j ${IMPORT_ARGS} + + # Save DDL for post-start execution + if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then + cp "${CONTENT_DIR}/ddl.cypher" /tmp/ddl.cypher + fi + else + # Legacy dump-based archive + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) + if [ -z "${DUMP_FILE}" ]; then + echo "ERROR: No import.sh+csv/ or .dump file found in archive" + ls -laR /tmp/graph-extracted/ + exit 1 + fi + echo "Found dump: ${DUMP_FILE}" + + DUMP_DIR=$(dirname "${DUMP_FILE}") + if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then + cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" + fi + + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${DUMP_DIR}:/dump" \ + neo4j:2026.01.4-community \ + neo4j-admin database load neo4j \ + --from-path=/dump \ + --overwrite-destination=true + fi - name: Start Neo4j with loaded data run: | NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) docker start "${NEO4J_CONTAINER}" - # Wait for Neo4j to come back up echo "Waiting for Neo4j to restart..." for i in $(seq 1 60); do if curl -sf http://localhost:7474/ > /dev/null 2>&1; then @@ -172,15 +196,55 @@ jobs: sleep 2 done + - name: Execute DDL for CSV-based archives + run: | + if [ ! -f /tmp/ddl.cypher ]; then + echo "No DDL to execute (dump-based archive)" + exit 0 + fi + + NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) + echo "Executing DDL index statements..." + DDL_COUNT=0 + DDL_FAIL=0 + while IFS= read -r stmt; do + [ -z "$stmt" ] && continue + if docker exec "${NEO4J_CONTAINER}" cypher-shell \ + -u neo4j -p imas-codex "$stmt" 2>&1; then + DDL_COUNT=$((DDL_COUNT + 1)) + else + echo "WARNING: DDL failed: $stmt" + DDL_FAIL=$((DDL_FAIL + 1)) + fi + done < /tmp/ddl.cypher + echo "Executed $DDL_COUNT DDL statements ($DDL_FAIL failed)" + + if [ "$DDL_FAIL" -gt 0 ] && [ "$DDL_COUNT" -eq 0 ]; then + echo "ERROR: All DDL statements failed" + exit 1 + fi + + # Wait for indexes to come online + echo "Waiting for indexes to come online..." + for i in $(seq 1 300); do + PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell \ + -u neo4j -p imas-codex \ + "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') + if [ "$PENDING" = "0" ]; then + echo "All indexes ONLINE (${i}s)" + break + fi + sleep 1 + done + rm -f /tmp/ddl.cypher + - name: Reset Neo4j password run: | - # After dump load, auth is reset. Set the password. NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) docker exec "${NEO4J_CONTAINER}" neo4j-admin dbms set-initial-password imas-codex 2>/dev/null || true - name: Verify graph is loaded run: | - # Quick sanity check with cypher-shell NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) docker exec "${NEO4J_CONTAINER}" cypher-shell \ -u neo4j -p imas-codex \ @@ -196,7 +260,7 @@ jobs: run: uv python install 3.12 - name: Install dependencies - run: uv sync --extra test + run: uv sync --extra test --no-dev env: HATCH_BUILD_NO_HOOKS: true From 502a8a4a65e1abb76379e0913122954c5d26ce03 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 15:54:50 +0200 Subject: [PATCH 11/22] fix: graph-quality pulls dd-only graph (only variant pushed for RCs) --- .github/workflows/graph-quality.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/graph-quality.yml b/.github/workflows/graph-quality.yml index 9d97ce7da..b8d4def89 100644 --- a/.github/workflows/graph-quality.yml +++ b/.github/workflows/graph-quality.yml @@ -83,18 +83,24 @@ jobs: fi echo "tag=${TAG:-latest}" >> $GITHUB_OUTPUT echo "registry=${REGISTRY:-ghcr.io/iterorganization}" >> $GITHUB_OUTPUT - echo "Graph: ${REGISTRY}/imas-codex-graph:${TAG}" + echo "Graph: ${REGISTRY}/imas-codex-graph-dd:${TAG}" - name: Login to GHCR run: | echo "${{ secrets.GHCR_TOKEN }}" | oras login ghcr.io -u token --password-stdin - - name: Pull graph dump from GHCR + - name: Pull graph from GHCR run: | - ARTIFACT="${{ steps.resolve-tag.outputs.registry }}/imas-codex-graph:${{ steps.resolve-tag.outputs.tag }}" + ARTIFACT="${{ steps.resolve-tag.outputs.registry }}/imas-codex-graph-dd:${{ steps.resolve-tag.outputs.tag }}" echo "Pulling: ${ARTIFACT}" mkdir -p /tmp/graph-dump - oras pull "${ARTIFACT}" -o /tmp/graph-dump + oras pull "${ARTIFACT}" -o /tmp/graph-dump --allow-path-traversal + # Handle oras path traversal: artifact may land outside -o dir + FOUND=$(find /tmp -maxdepth 3 -name "*.tar.gz" ! -path "/tmp/graph-dump/*" 2>/dev/null | head -1) + if [ -n "$FOUND" ] && [ ! -f /tmp/graph-dump/*.tar.gz ]; then + echo "Moving artifact from traversal path: ${FOUND}" + mv "$FOUND" /tmp/graph-dump/ + fi ls -la /tmp/graph-dump/ - name: Wait for Neo4j to be ready From e5f394df5f809affe061f9b5b3936f39f7a8194b Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 16:04:47 +0200 Subject: [PATCH 12/22] fix: remove imas-standard-names from deps until PyPI release The package is not yet published to PyPI, so uv cannot resolve it in CI environments. Remove from both dev deps and [tool.uv.sources]. Developers install manually: uv pip install -e ../imas-standard-names --- pyproject.toml | 4 +- uv.lock | 828 ------------------------------------------------- 2 files changed, 1 insertion(+), 831 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bff744f73..ee95fd2da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,8 +114,7 @@ dev = [ # --- Interactive dev --- "ipython>=9.2.0", "ipykernel>=6.29.5", - # --- Standard Names catalog --- - "imas-standard-names", + # --- Standard Names catalog (install manually: uv pip install -e ../imas-standard-names) --- # --- LLM & Discovery --- "litellm>=1.81.0", # --- Graph build & schema --- @@ -432,7 +431,6 @@ torch = [ { index = "pytorch-cpu", extra = "test" }, { index = "pytorch-gpu", extra = "gpu" }, ] -imas-standard-names = { path = "../imas-standard-names", editable = true } [tool.uv] # cpu, gpu, and test extras for torch are mutually exclusive with gpu diff --git a/uv.lock b/uv.lock index 7f470523a..ed77a5c55 100644 --- a/uv.lock +++ b/uv.lock @@ -37,18 +37,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/d2/c581486aa6c4fbd7394c23c47b83fa1a919d34194e16944241daf9e762dd/accelerate-1.12.0-py3-none-any.whl", hash = "sha256:3e2091cd341423207e2f084a6654b1efcd250dc326f2a37d6dde446e07cabb11", size = 380935, upload-time = "2025-11-21T11:27:44.522Z" }, ] -[[package]] -name = "ag-ui-protocol" -version = "0.1.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/71/96c21ae7e2fb9b610c1a90d38bd2de8b6e5b2900a63001f3882f43e519af/ag_ui_protocol-0.1.15.tar.gz", hash = "sha256:5e23c1042c7d4e364d685e68d2fb74d37c16bc83c66d270102d8eaedce56ad82", size = 6269, upload-time = "2026-04-01T15:44:33.136Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/a0/a73398d30bb0f9ad70cd70426151a4a19527a7296e48a3a16a50e1d5db05/ag_ui_protocol-0.1.15-py3-none-any.whl", hash = "sha256:85cde077023ccbc37b5ce2ad953537883c262d210320f201fc2ec4e85408b06a", size = 8661, upload-time = "2026-04-01T15:44:32.079Z" }, -] - [[package]] name = "aiofile" version = "3.9.0" @@ -144,25 +132,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "anthropic" -version = "0.92.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/2d/fc5c5a369db977efbaa646d77ba42b38a6de4e95789884032b0e2e3fc834/anthropic-0.92.0.tar.gz", hash = "sha256:d1e792ed0692379452a1af6b266df495e973c3695cd0aace2a108b838393cbc4", size = 652420, upload-time = "2026-04-08T16:55:35.37Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/21/bf5b5ab10b6932c5c43eaa66b6e3f256de569cf0323d89f9cc281a0d0f39/anthropic-0.92.0-py3-none-any.whl", hash = "sha256:f92a4bd065d5cab90a96b65bb44e473bf7c6fe731a743cd156e9ad1d245c381e", size = 621195, upload-time = "2026-04-08T16:55:33.639Z" }, -] - [[package]] name = "antlr4-python3-runtime" version = "4.9.3" @@ -191,15 +160,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] -[[package]] -name = "argcomplete" -version = "3.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, -] - [[package]] name = "arrow" version = "1.4.0" @@ -351,34 +311,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] -[[package]] -name = "boto3" -version = "1.42.85" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/9d/a9a7b5a9351e3ff0baae01136f71ba6fc4652fe0dc2da3b0a8ebdfc1be44/boto3-1.42.85.tar.gz", hash = "sha256:1cd3dcbfaba85c6071ba9397c1804b6a94a1a97031b8f1993fdba27c0c5d6eba", size = 112769, upload-time = "2026-04-07T19:40:53.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/ab/3167b8ec3cf1d87ad08d2ad5f15823a22945cae7870798274c283c3a18f1/boto3-1.42.85-py3-none-any.whl", hash = "sha256:4f6ac066e41d18ec33f532253fac0f35e0fdca373724458f983ce3d531340b7a", size = 140556, upload-time = "2026-04-07T19:40:52.186Z" }, -] - -[[package]] -name = "botocore" -version = "1.42.85" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0a/ac/7f14b05cf43e4baae99f4570b02e10b2aebf242dfd86245523340390c834/botocore-1.42.85.tar.gz", hash = "sha256:2ee61f80b7724a143e16d0a85408ef5fa20b99dce7a3c8ec5d25cc8dced164c1", size = 15159562, upload-time = "2026-04-07T19:40:43.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/f3/c1fbaff4c509c616fd01f44357283a8992f10b3a05d932b22e602aa3a221/botocore-1.42.85-py3-none-any.whl", hash = "sha256:828b67722caeb7e240eefedee74050e803d1fa102958ead9c4009101eefd5381", size = 14839741, upload-time = "2026-04-07T19:40:40.733Z" }, -] - [[package]] name = "build" version = "1.4.0" @@ -511,25 +443,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] -[[package]] -name = "cohere" -version = "5.21.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastavro" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "pydantic-core" }, - { name = "requests" }, - { name = "tokenizers" }, - { name = "types-requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d2/75/4c346f6e2322e545f8452692304bd4eca15a2a0209ab9af6a0d1a7810b67/cohere-5.21.1.tar.gz", hash = "sha256:e5ade4423b928b01ff2038980e1b62b2a5bb412c8ab83e30882753b810a5509f", size = 191272, upload-time = "2026-03-26T15:09:27.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/50/5538f02ec6d10fbb84f29c1b18c68ff2a03d7877926a80275efdf8755a9f/cohere-5.21.1-py3-none-any.whl", hash = "sha256:f15592ec60d8cf12f01563db94ec28c388c61269d9617f23c2d6d910e505344e", size = 334262, upload-time = "2026-03-26T15:09:26.284Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -718,17 +631,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "dotenv" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dotenv" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/b7/545d2c10c1fc15e48653c91efde329a790f2eecfbbf2bd16003b5db2bab0/dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9", size = 1892, upload-time = "2025-02-19T22:15:01.647Z" }, -] - [[package]] name = "email-validator" version = "2.3.0" @@ -751,15 +653,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] -[[package]] -name = "eval-type-backport" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fb/a3/cafafb4558fd638aadfe4121dc6cefb8d743368c085acb2f521df0f3d9d7/eval_type_backport-0.3.1.tar.gz", hash = "sha256:57e993f7b5b69d271e37482e62f74e76a0276c82490cf8e4f0dffeb6b332d5ed", size = 9445, upload-time = "2025-12-02T11:51:42.987Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/22/fdc2e30d43ff853720042fa15baa3e6122722be1a7950a98233ebb55cd71/eval_type_backport-0.3.1-py3-none-any.whl", hash = "sha256:279ab641905e9f11129f56a8a78f493518515b83402b860f6f06dd7c011fdfa8", size = 6063, upload-time = "2025-12-02T11:51:41.665Z" }, -] - [[package]] name = "exceptiongroup" version = "1.3.1" @@ -821,20 +714,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/dd/d0ee25348ac58245ee9f90b6f3cbb666bf01f69be7e0911f9851bddbda16/fastapi-0.129.0-py3-none-any.whl", hash = "sha256:b4946880e48f462692b31c083be0432275cbfb6e2274566b1be91479cc1a84ec", size = 102950, upload-time = "2026-02-12T13:54:54.528Z" }, ] -[[package]] -name = "fastavro" -version = "1.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/8b/fa2d3287fd2267be6261d0177c6809a7fa12c5600ddb33490c8dc29e77b2/fastavro-1.12.1.tar.gz", hash = "sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b", size = 1025661, upload-time = "2025-10-10T15:40:55.41Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/f0/10bd1a3d08667fa0739e2b451fe90e06df575ec8b8ba5d3135c70555c9bd/fastavro-1.12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167", size = 1009057, upload-time = "2025-10-10T15:41:24.556Z" }, - { url = "https://files.pythonhosted.org/packages/78/ad/0d985bc99e1fa9e74c636658000ba38a5cd7f5ab2708e9c62eaf736ecf1a/fastavro-1.12.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14", size = 3391866, upload-time = "2025-10-10T15:41:26.882Z" }, - { url = "https://files.pythonhosted.org/packages/0d/9e/b4951dc84ebc34aac69afcbfbb22ea4a91080422ec2bfd2c06076ff1d419/fastavro-1.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34", size = 3458005, upload-time = "2025-10-10T15:41:29.017Z" }, - { url = "https://files.pythonhosted.org/packages/af/f8/5a8df450a9f55ca8441f22ea0351d8c77809fc121498b6970daaaf667a21/fastavro-1.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b", size = 3295258, upload-time = "2025-10-10T15:41:31.564Z" }, - { url = "https://files.pythonhosted.org/packages/99/b2/40f25299111d737e58b85696e91138a66c25b7334f5357e7ac2b0e8966f8/fastavro-1.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c", size = 3430328, upload-time = "2025-10-10T15:41:33.689Z" }, - { url = "https://files.pythonhosted.org/packages/e0/07/85157a7c57c5f8b95507d7829b5946561e5ee656ff80e9dd9a757f53ddaf/fastavro-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f", size = 444140, upload-time = "2025-10-10T15:41:34.902Z" }, -] - [[package]] name = "fastjsonschema" version = "2.21.2" @@ -971,70 +850,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" }, ] -[[package]] -name = "genai-prices" -version = "0.0.56" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/6b/94b3018a672c7775edfb485f0fed8f6068fba75e49b067e8a1ac5eb96764/genai_prices-0.0.56.tar.gz", hash = "sha256:ac24b16a84d0ab97539bfa48dfa4649689de8e3ce71c12ebacef29efb1998045", size = 65872, upload-time = "2026-03-20T20:33:00.732Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/f6/8ef7e4c286deb2709d11ca96a5237caae3ef4876ab3c48095856cfd2df30/genai_prices-0.0.56-py3-none-any.whl", hash = "sha256:dbe86be8f3f556bed1b72209ed36851fec8b01793b3b220f42921a4e7da945f6", size = 68966, upload-time = "2026-03-20T20:33:02.555Z" }, -] - -[[package]] -name = "google-auth" -version = "2.49.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ea/80/6a696a07d3d3b0a92488933532f03dbefa4a24ab80fb231395b9a2a1be77/google_auth-2.49.1.tar.gz", hash = "sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64", size = 333825, upload-time = "2026-03-12T19:30:58.135Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/eb/c6c2478d8a8d633460be40e2a8a6f8f429171997a35a96f81d3b680dec83/google_auth-2.49.1-py3-none-any.whl", hash = "sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7", size = 240737, upload-time = "2026-03-12T19:30:53.159Z" }, -] - -[package.optional-dependencies] -requests = [ - { name = "requests" }, -] - -[[package]] -name = "google-genai" -version = "1.71.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "google-auth", extra = ["requests"] }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "sniffio" }, - { name = "tenacity" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/49/a13e9cf4d963691fc79d661f2d78f041bc1f2e7287d41ef0f831b82462f0/google_genai-1.71.0.tar.gz", hash = "sha256:044f7ac453437d5d380ec192f823dba64e001c478d7878c5a2d327432f4a28ac", size = 520044, upload-time = "2026-04-08T17:55:51.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/d4/63c97d487f0b4861a6f530628a9e56a380f9c9704d2e207f0bed9d16e31a/google_genai-1.71.0-py3-none-any.whl", hash = "sha256:6213ebfee7fc8e6a21692c2c340309e463322cc35c2c603d1ea59e8ea34ac240", size = 760561, upload-time = "2026-04-08T17:55:49.107Z" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.74.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, -] - [[package]] name = "graphviz" version = "0.21" @@ -1061,53 +876,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/ab/717c58343cf02c5265b531384b248787e04d8160b8afe53d9eec053d7b44/greenlet-3.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bfb2d1763d777de5ee495c85309460f6fd8146e50ec9d0ae0183dbf6f0a829d1", size = 226403, upload-time = "2026-01-23T15:31:39.372Z" }, ] -[[package]] -name = "griffelib" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461, upload-time = "2026-03-27T11:34:51.091Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" }, -] - -[[package]] -name = "groq" -version = "1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/c7/a2153b639062f59f9bc93a1b5507c0c4a6b654b8a9edbf432ec2f4a62d2d/groq-1.1.2.tar.gz", hash = "sha256:9ec2b5b6a1c4856a8c6c38741353c5ab37472a4e3fded02af783750d849cc988", size = 154033, upload-time = "2026-03-25T23:16:10.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/b0/83e3892a4597a4b8ebf8a662aeaf314765c4c2340516eb1d049b459b24fc/groq-1.1.2-py3-none-any.whl", hash = "sha256:348cb7a674b6aa7105719b533f6fc48fd32b503bc9256924aaed6dc186f778b5", size = 141700, upload-time = "2026-03-25T23:16:08.998Z" }, -] - -[[package]] -name = "grpcio" -version = "1.80.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/e8/a2b749265eb3415abc94f2e619bbd9e9707bebdda787e61c593004ec927a/grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0", size = 6015616, upload-time = "2026-03-30T08:47:13.428Z" }, - { url = "https://files.pythonhosted.org/packages/3e/97/b1282161a15d699d1e90c360df18d19165a045ce1c343c7f313f5e8a0b77/grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2", size = 12014204, upload-time = "2026-03-30T08:47:15.873Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/d319c6e997b50c155ac5a8cb12f5173d5b42677510e886d250d50264949d/grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de", size = 6563866, upload-time = "2026-03-30T08:47:18.588Z" }, - { url = "https://files.pythonhosted.org/packages/ae/f6/fdd975a2cb4d78eb67769a7b3b3830970bfa2e919f1decf724ae4445f42c/grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921", size = 7273060, upload-time = "2026-03-30T08:47:21.113Z" }, - { url = "https://files.pythonhosted.org/packages/db/f0/a3deb5feba60d9538a962913e37bd2e69a195f1c3376a3dd44fe0427e996/grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411", size = 6782121, upload-time = "2026-03-30T08:47:23.827Z" }, - { url = "https://files.pythonhosted.org/packages/ca/84/36c6dcfddc093e108141f757c407902a05085e0c328007cb090d56646cdf/grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd", size = 7383811, upload-time = "2026-03-30T08:47:26.517Z" }, - { url = "https://files.pythonhosted.org/packages/7c/ef/f3a77e3dc5b471a0ec86c564c98d6adfa3510d38f8ee99010410858d591e/grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f", size = 8393860, upload-time = "2026-03-30T08:47:29.439Z" }, - { url = "https://files.pythonhosted.org/packages/9b/8d/9d4d27ed7f33d109c50d6b5ce578a9914aa68edab75d65869a17e630a8d1/grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f", size = 7830132, upload-time = "2026-03-30T08:47:33.254Z" }, - { url = "https://files.pythonhosted.org/packages/14/e4/9990b41c6d7a44e1e9dee8ac11d7a9802ba1378b40d77468a7761d1ad288/grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193", size = 4140904, upload-time = "2026-03-30T08:47:35.319Z" }, - { url = "https://files.pythonhosted.org/packages/2f/2c/296f6138caca1f4b92a31ace4ae1b87dab692fc16a7a3417af3bb3c805bf/grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff", size = 4880944, upload-time = "2026-03-30T08:47:37.831Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -1310,7 +1078,6 @@ dev = [ { name = "fastapi" }, { name = "hdbscan" }, { name = "imas-python" }, - { name = "imas-standard-names" }, { name = "ipykernel" }, { name = "ipython" }, { name = "jellyfish" }, @@ -1399,7 +1166,6 @@ dev = [ { name = "fastapi", specifier = ">=0.115.0" }, { name = "hdbscan", specifier = ">=0.8.41" }, { name = "imas-python", specifier = ">=2.0.1" }, - { name = "imas-standard-names", editable = "../imas-standard-names" }, { name = "ipykernel", specifier = ">=6.29.5" }, { name = "ipython", specifier = ">=9.2.0" }, { name = "jellyfish", specifier = ">=1.2.1" }, @@ -1476,72 +1242,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/c3/51724c1ba79aa3f34566750de5a0ad41176a272b19e0585dc62aea3a987b/imas_python-2.2.0-py3-none-any.whl", hash = "sha256:52a16cd13d7756413ff918c0cf754d42ab9ac61ae2524ab7f72a9df00a70637c", size = 2405647, upload-time = "2026-02-12T15:32:16.657Z" }, ] -[[package]] -name = "imas-standard-names" -source = { editable = "../imas-standard-names" } -dependencies = [ - { name = "click" }, - { name = "dotenv" }, - { name = "fastmcp" }, - { name = "markdown" }, - { name = "pint" }, - { name = "pydantic" }, - { name = "pydantic-ai" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "strictyaml" }, - { name = "textual" }, -] - -[package.metadata] -requires-dist = [ - { name = "click", specifier = ">=8.1.8,<9.0.0" }, - { name = "dotenv", specifier = ">=0.9.9,<0.10.0" }, - { name = "en-core-web-sm", marker = "extra == 'quality'", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, - { name = "fastmcp", specifier = "~=3.2.0" }, - { name = "markdown", specifier = ">=3.8,<4.0" }, - { name = "mike", marker = "extra == 'docs'", specifier = ">=2.1.3,<3.0.0" }, - { name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1,<2.0.0" }, - { name = "mkdocs-data-plugin", marker = "extra == 'docs'", specifier = ">=0.2.0,<0.3.0" }, - { name = "mkdocs-include-markdown-plugin", marker = "extra == 'docs'", specifier = ">=7.0.0,<8.0.0" }, - { name = "mkdocs-macros-plugin", marker = "extra == 'docs'", specifier = ">=1.0.4,<2.0.0" }, - { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.6.5,<10.0.0" }, - { name = "mkdocs-table-reader-plugin", marker = "extra == 'docs'", specifier = ">=3.1.0,<4.0.0" }, - { name = "pint", specifier = ">=0.24.4,<0.25.0" }, - { name = "proselint", marker = "extra == 'quality'", specifier = ">=0.14.0,<0.15.0" }, - { name = "pydantic", specifier = ">=2.10.6,<3.0.0" }, - { name = "pydantic-ai", specifier = ">=1.56.0" }, - { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4,<9.0.0" }, - { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.1.0,<5.0.0" }, - { name = "pytest-html", marker = "extra == 'test'", specifier = ">=4.1.1,<5.0.0" }, - { name = "pyyaml", specifier = ">=6.0.2,<7.0.0" }, - { name = "requests", specifier = ">=2.33.0,<3.0.0" }, - { name = "ruff", marker = "extra == 'test'", specifier = ">=0.9.8,<1.0.0" }, - { name = "spacy", marker = "extra == 'quality'", specifier = ">=3.8.0,<4.0.0" }, - { name = "strictyaml", specifier = ">=1.7.3,<2.0.0" }, - { name = "textual", specifier = ">=6.1.0" }, -] -provides-extras = ["docs", "quality", "test"] - -[package.metadata.requires-dev] -dev = [ - { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, - { name = "ipykernel", specifier = ">=6.29.5,<7.0.0" }, - { name = "logfire", specifier = ">=4.16.0,<5.0.0" }, - { name = "mcp-cli", specifier = ">=0.1.0,<1.0.0" }, - { name = "pandas", specifier = ">=2.2.3,<3.0.0" }, - { name = "pandas-stubs", specifier = ">=2.2.3.250308,<3.0.0" }, - { name = "pre-commit", specifier = ">=4.1.0,<5.0.0" }, - { name = "proselint", specifier = ">=0.14.0,<0.15.0" }, - { name = "pytest", specifier = ">=8.3.4,<9.0.0" }, - { name = "pytest-cov", specifier = ">=4.1.0,<5.0.0" }, - { name = "pytest-html", specifier = ">=4.1.1,<5.0.0" }, - { name = "ruff", specifier = ">=0.11.10,<1.0.0" }, - { name = "spacy", specifier = ">=3.8.0,<4.0.0" }, - { name = "textual-dev", specifier = ">=1.7.0" }, - { name = "types-pyyaml", specifier = ">=6.0.12.20241230,<7.0.0" }, -] - [[package]] name = "importlib-metadata" version = "8.7.1" @@ -1768,15 +1468,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, ] -[[package]] -name = "jmespath" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, -] - [[package]] name = "joblib" version = "1.5.3" @@ -1829,15 +1520,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl", hash = "sha256:12e86f86324d54fcf60632db94ea74488d5314e3da554c994fe1e2c6f29acb79", size = 6324, upload-time = "2021-06-02T17:43:27.126Z" }, ] -[[package]] -name = "jsonpath-python" -version = "1.1.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/db/2f4ecc24da35c6142b39c353d5b7c16eef955cc94b35a48d3fa47996d7c3/jsonpath_python-1.1.5.tar.gz", hash = "sha256:ceea2efd9e56add09330a2c9631ea3d55297b9619348c1055e5bfb9cb0b8c538", size = 87352, upload-time = "2026-03-17T06:16:40.597Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/50/1a313fb700526b134c71eb8a225d8b83be0385dbb0204337b4379c698cef/jsonpath_python-1.1.5-py3-none-any.whl", hash = "sha256:a60315404d70a65e76c9a782c84e50600480221d94a58af47b7b4d437351cb4b", size = 14090, upload-time = "2026-03-17T06:16:39.152Z" }, -] - [[package]] name = "jsonpointer" version = "3.0.0" @@ -1977,18 +1659,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, ] -[[package]] -name = "linkify-it-py" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "uc-micro-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, -] - [[package]] name = "linkml" version = "1.9.3" @@ -2071,38 +1741,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/f3/fffb7932870163cea7addc392165647a9a8a5489967de486c854226f1141/litellm-1.81.13-py3-none-any.whl", hash = "sha256:ae4aea2a55e85993f5f6dd36d036519422d24812a1a3e8540d9e987f2d7a4304", size = 14587505, upload-time = "2026-02-17T02:00:44.22Z" }, ] -[[package]] -name = "logfire" -version = "4.31.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "executing" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-sdk" }, - { name = "protobuf" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/fc/21f923243d8c3ca2ebfa97de46970ced734e66ac634c1c35b6abb41300f1/logfire-4.31.0.tar.gz", hash = "sha256:361bfda17c9d70ada5d220211033bae06b871ddac9d5b06978bc0ceca6b8e658", size = 1080609, upload-time = "2026-03-27T19:00:46.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/1a/8c860e35bf847ac0d647d94bad89dccbb66cbcafdd61d8334f8cc7cfdd58/logfire-4.31.0-py3-none-any.whl", hash = "sha256:49fad38b5e6f199a98e9c8814e860c8a42595bb81479b52a20413e53ee475b72", size = 308896, upload-time = "2026-03-27T19:00:43.107Z" }, -] - -[package.optional-dependencies] -httpx = [ - { name = "opentelemetry-instrumentation-httpx" }, -] - -[[package]] -name = "logfire-api" -version = "4.31.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/a2/8d5a3c1c282d5f2bd9f5e9ddd5288d1414a53301ce389af9016b6d82bd50/logfire_api-4.31.0.tar.gz", hash = "sha256:fc4b01257ebd4ce297ad374ed201eb1a9213b999f6ae6df45cfca5bd0ef378f8", size = 77838, upload-time = "2026-03-27T19:00:47.545Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/27/9372b7492b3e146908d520f8599909311cd930175801ad219171fafc6f3e/logfire_api-4.31.0-py3-none-any.whl", hash = "sha256:3c1f502fd4eb8ef0996427a5cf275fd8f327f38600650a1f53071a8171c812db", size = 123402, upload-time = "2026-03-27T19:00:44.952Z" }, -] - [[package]] name = "lxml" version = "6.0.2" @@ -2129,15 +1767,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" }, ] -[[package]] -name = "markdown" -version = "3.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -2150,14 +1779,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] -[package.optional-dependencies] -linkify = [ - { name = "linkify-it-py" }, -] -plugins = [ - { name = "mdit-py-plugins" }, -] - [[package]] name = "markupsafe" version = "3.0.3" @@ -2214,18 +1835,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/d9/eaa1f80170d2b7c5ba23f3b59f766f3a0bb41155fbc32a69adfa1adaaef9/mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca", size = 233615, upload-time = "2026-01-24T19:40:30.652Z" }, ] -[[package]] -name = "mdit-py-plugins" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -2235,25 +1844,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "mistralai" -version = "2.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "eval-type-backport" }, - { name = "httpx" }, - { name = "jsonpath-python" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "pydantic" }, - { name = "python-dateutil" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/96/b8ab9bbdcefda9803cf3b51e11548730ac94303850028fb86c163472aac3/mistralai-2.3.1.tar.gz", hash = "sha256:02989e509124cb28aaffd92660bf7511b3f8f5c215e1de8d49d0c8276bacc72a", size = 390323, upload-time = "2026-04-07T14:49:18.38Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/1d/b0da235154e9c7039c27b91785ec81f01ba1b3c48092924c6770ba2da22a/mistralai-2.3.1-py3-none-any.whl", hash = "sha256:8f4f783cb7603f6060490105f55b16a5d0a7e854c05e96fed316efcc4b393fe3", size = 930912, upload-time = "2026-04-07T14:49:16.863Z" }, -] - [[package]] name = "more-itertools" version = "10.8.0" @@ -2374,18 +1964,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] -[[package]] -name = "nexus-rpc" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/35/d5/cd1ffb202b76ebc1b33c1332a3416e55a39929006982adc2b1eb069aaa9b/nexus_rpc-1.4.0.tar.gz", hash = "sha256:3b8b373d4865671789cc43623e3dc0bcbf192562e40e13727e17f1c149050fba", size = 82367, upload-time = "2026-02-25T22:01:34.053Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/52/6327a5f4fda01207205038a106a99848a41c83e933cd23ea2cab3d2ebc6c/nexus_rpc-1.4.0-py3-none-any.whl", hash = "sha256:14c953d3519113f8ccec533a9efdb6b10c28afef75d11cdd6d422640c40b3a49", size = 29645, upload-time = "2026-02-25T22:01:33.122Z" }, -] - [[package]] name = "nodeenv" version = "1.10.0" @@ -2589,115 +2167,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" }, ] -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation" -version = "0.60b1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "packaging" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/d2/6788e83c5c86a2690101681aeef27eeb2a6bf22df52d3f263a22cee20915/opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d", size = 33096, upload-time = "2025-12-11T13:35:33.067Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-httpx" -version = "0.60b1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/08/11208bcfcab4fc2023252c3f322aa397fd9ad948355fea60f5fc98648603/opentelemetry_instrumentation_httpx-0.60b1.tar.gz", hash = "sha256:a506ebaf28c60112cbe70ad4f0338f8603f148938cb7b6794ce1051cd2b270ae", size = 20611, upload-time = "2025-12-11T13:37:01.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/59/b98e84eebf745ffc75397eaad4763795bff8a30cbf2373a50ed4e70646c5/opentelemetry_instrumentation_httpx-0.60b1-py3-none-any.whl", hash = "sha256:f37636dd742ad2af83d896ba69601ed28da51fa4e25d1ab62fde89ce413e275b", size = 15701, upload-time = "2025-12-11T13:36:04.56Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.60b1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" }, -] - -[[package]] -name = "opentelemetry-util-http" -version = "0.60b1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/50/fc/c47bb04a1d8a941a4061307e1eddfa331ed4d0ab13d8a9781e6db256940a/opentelemetry_util_http-0.60b1.tar.gz", hash = "sha256:0d97152ca8c8a41ced7172d29d3622a219317f74ae6bb3027cfbdcf22c3cc0d6", size = 11053, upload-time = "2025-12-11T13:37:25.115Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" }, -] - [[package]] name = "packaging" version = "25.0" @@ -2902,21 +2371,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] -[[package]] -name = "protobuf" -version = "6.33.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, -] - [[package]] name = "psutil" version = "7.2.2" @@ -2985,27 +2439,6 @@ memory = [ { name = "cachetools" }, ] -[[package]] -name = "pyasn1" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - [[package]] name = "pycparser" version = "3.0" @@ -3035,106 +2468,6 @@ email = [ { name = "email-validator" }, ] -[[package]] -name = "pydantic-ai" -version = "1.78.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic-ai-slim", extra = ["ag-ui", "anthropic", "bedrock", "cli", "cohere", "evals", "fastmcp", "google", "groq", "huggingface", "logfire", "mcp", "mistral", "openai", "retries", "spec", "temporal", "ui", "vertexai", "xai"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/e1/ced6f04f60accb11deb1a8ca4fc576270022e646f011a9e1674695420710/pydantic_ai-1.78.0.tar.gz", hash = "sha256:dd3f56306c671f7785126e78d72924e5a80c30bca27460081941ad22b63fcc8d", size = 12645, upload-time = "2026-04-08T05:20:34.096Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/1a/497fdf8224aed69752559350e437395266ff0f3107ca3bff63b6925358cc/pydantic_ai-1.78.0-py3-none-any.whl", hash = "sha256:aa0fdacec813fa457243206a9dae4d5152e0814bf17fc7eee75045d3469f5ca8", size = 7551, upload-time = "2026-04-08T05:20:24.209Z" }, -] - -[[package]] -name = "pydantic-ai-slim" -version = "1.78.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "genai-prices" }, - { name = "griffelib" }, - { name = "httpx" }, - { name = "opentelemetry-api" }, - { name = "pydantic" }, - { name = "pydantic-graph" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/84/4cf98c41a2019a5c5ed6aa5d5fa2bbc8b70b152b527c56d27dabbeaeb75c/pydantic_ai_slim-1.78.0.tar.gz", hash = "sha256:97c6467a6bb09f61fd48cd828db066204ae77419d14a4edf47f90f72d06ab11f", size = 531385, upload-time = "2026-04-08T05:20:36.291Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/91/487992a441c03f16525885ee541083f66f579500edd7aa8b838f3296455f/pydantic_ai_slim-1.78.0-py3-none-any.whl", hash = "sha256:36f88bab6016186b958363ca1254741999df276322d7066dd69793dcb2134b66", size = 680002, upload-time = "2026-04-08T05:20:27.508Z" }, -] - -[package.optional-dependencies] -ag-ui = [ - { name = "ag-ui-protocol" }, - { name = "starlette" }, -] -anthropic = [ - { name = "anthropic" }, -] -bedrock = [ - { name = "boto3" }, -] -cli = [ - { name = "argcomplete" }, - { name = "prompt-toolkit" }, - { name = "pyperclip" }, - { name = "pyyaml" }, - { name = "rich" }, -] -cohere = [ - { name = "cohere", marker = "sys_platform != 'emscripten' or (extra == 'extra-10-imas-codex-cpu' and extra == 'extra-10-imas-codex-gpu') or (extra == 'extra-10-imas-codex-gpu' and extra == 'extra-10-imas-codex-test')" }, -] -evals = [ - { name = "pydantic-evals" }, -] -fastmcp = [ - { name = "fastmcp" }, -] -google = [ - { name = "google-genai" }, -] -groq = [ - { name = "groq" }, -] -huggingface = [ - { name = "huggingface-hub" }, -] -logfire = [ - { name = "logfire", extra = ["httpx"] }, -] -mcp = [ - { name = "mcp" }, -] -mistral = [ - { name = "mistralai" }, -] -openai = [ - { name = "openai" }, - { name = "tiktoken" }, -] -retries = [ - { name = "tenacity" }, -] -spec = [ - { name = "pydantic-handlebars" }, - { name = "pyyaml" }, -] -temporal = [ - { name = "temporalio" }, -] -ui = [ - { name = "starlette" }, -] -vertexai = [ - { name = "google-auth" }, - { name = "requests" }, -] -xai = [ - { name = "xai-sdk" }, -] - [[package]] name = "pydantic-core" version = "2.41.5" @@ -3164,50 +2497,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, ] -[[package]] -name = "pydantic-evals" -version = "1.78.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "logfire-api" }, - { name = "pydantic" }, - { name = "pydantic-ai-slim" }, - { name = "pyyaml" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/80/539238f284f6a4fdef5737a0cb7efa746e4259bb27526af271898784f2fa/pydantic_evals-1.78.0.tar.gz", hash = "sha256:8608068c2569a0169977526a93ddea45e924331115233eb1f297ab19653e14e0", size = 65818, upload-time = "2026-04-08T05:20:37.844Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/4c/e6bcf445ff3f15211c217975a11d401e78b3f2d1e989c50d859527e81050/pydantic_evals-1.78.0-py3-none-any.whl", hash = "sha256:b00cc22e2d24a0771f40fc7e3b2b4afeec7c99bda8cfce37e6bee58e7a29fe3c", size = 77739, upload-time = "2026-04-08T05:20:29.577Z" }, -] - -[[package]] -name = "pydantic-graph" -version = "1.78.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "logfire-api" }, - { name = "pydantic" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/e4/eb52021f43f2ac495955af19219c1ab261d707bbd15f4dc229079c2276d4/pydantic_graph-1.78.0.tar.gz", hash = "sha256:dd627e37cb3adaf8c95cca6a4b33e0d1b7fc9bed075dc3b8ad5df2c2a3cb432b", size = 58682, upload-time = "2026-04-08T05:20:39.288Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/97/e3dd1a1c6f6b9c104c844c1a2383361a831d1b027c99c0ac1a20544f0b13/pydantic_graph-1.78.0-py3-none-any.whl", hash = "sha256:0302835f46da3ee70ba3602a4d886c41a76fa8750f23f4257b968163ba4bb89f", size = 72500, upload-time = "2026-04-08T05:20:31.237Z" }, -] - -[[package]] -name = "pydantic-handlebars" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/16/d41768bd3fd77e6250c20be11a3e68fee5fff07c3356455e6708f6a60f2a/pydantic_handlebars-0.1.0.tar.gz", hash = "sha256:1931c54946add1b5e3796c9bf6a005ed7662cef0109bb05c352f0b3d031a1260", size = 159826, upload-time = "2026-03-01T20:00:17.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5f/86b1630be61bdebf253c2f953a6c3f073ec21bb0725565ea3896802e1ca3/pydantic_handlebars-0.1.0-py3-none-any.whl", hash = "sha256:8a436fe8bc607295eb04bec58bd6e2c9498c9e069c557ff0b505e3d568c783bc", size = 40890, upload-time = "2026-03-01T20:00:16.106Z" }, -] - [[package]] name = "pydantic-settings" version = "2.13.1" @@ -3881,18 +3170,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/07/5bda6a85b220c64c65686bc85bd0bbb23b29c62b3a9f9433fa55f17cda93/ruff-0.15.1-py3-none-win_arm64.whl", hash = "sha256:5ff7d5f0f88567850f45081fac8f4ec212be8d0b963e385c3f7d0d2eb4899416", size = 10874604, upload-time = "2026-02-12T23:09:05.515Z" }, ] -[[package]] -name = "s3transfer" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" }, -] - [[package]] name = "safetensors" version = "0.7.0" @@ -4238,18 +3515,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, ] -[[package]] -name = "strictyaml" -version = "1.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/08/efd28d49162ce89c2ad61a88bd80e11fb77bc9f6c145402589112d38f8af/strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407", size = 115206, upload-time = "2023-03-10T12:50:27.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/7c/a81ef5ef10978dd073a854e0fa93b5d8021d0594b639cc8f6453c3c78a1d/strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7", size = 123917, upload-time = "2023-03-10T12:50:17.242Z" }, -] - [[package]] name = "sympy" version = "1.13.1" @@ -4287,50 +3552,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] -[[package]] -name = "temporalio" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nexus-rpc" }, - { name = "protobuf" }, - { name = "types-protobuf" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/9c/3782bab0bf11a40b550147c19a5d1a476c17405391751982408902d9f138/temporalio-1.25.0.tar.gz", hash = "sha256:a3bbec1dcc904f674402cfa4faae480fda490b1c53ea5440c1f1996c562016fb", size = 2152534, upload-time = "2026-04-08T18:53:55.388Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/e3/5676dd10d1164b6d6ca8752314054097b89c5da931e936af402a7b15236c/temporalio-1.25.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6dc1bc8e1773b1a833d86a7ede2dd90ef4e031ced5b748b59e7f09a5bf9b327d", size = 13943906, upload-time = "2026-04-08T18:53:30.022Z" }, - { url = "https://files.pythonhosted.org/packages/89/50/7cbf7f845973be986ec165348f72f7a409750842a04d554965a39be5cb4f/temporalio-1.25.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3c8fdcf79ea5ae8ae2cf6f48072e4a86c3e0f4778f6a8a066c6ff1d336587db4", size = 13298719, upload-time = "2026-04-08T18:53:35.95Z" }, - { url = "https://files.pythonhosted.org/packages/d2/31/d474bab8535552add6ed289911bf1ffae5d7071823ece1069842190fcaed/temporalio-1.25.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:141f37aaafd7d090ba5c8776e4e9bc60df1fbc64b9f50c8f00e905a436588ddc", size = 13555435, upload-time = "2026-04-08T18:53:41.36Z" }, - { url = "https://files.pythonhosted.org/packages/2a/c8/e7dc053d6107bf2a037a3c9fe7b86639a25dcb888bde0e1ca366901ee47f/temporalio-1.25.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff7ca5bb80264976477d4dc7a839b3d22af8577ae92306526a061481db49bf92", size = 14052050, upload-time = "2026-04-08T18:53:46.44Z" }, - { url = "https://files.pythonhosted.org/packages/08/70/9340ed3a578321cbc153041d34834bb1ec3f1f3e3d9cded47cd1b7c3e403/temporalio-1.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:9411534279a2e64847231b6059c214bff4d57cfd1532bd09f333d0b1603daa7f", size = 14299684, upload-time = "2026-04-08T18:53:52.482Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - -[[package]] -name = "textual" -version = "6.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py", extra = ["linkify", "plugins"] }, - { name = "platformdirs" }, - { name = "pygments" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/30/38b615f7d4b16f6fdd73e4dcd8913e2d880bbb655e68a076e3d91181a7ee/textual-6.2.1.tar.gz", hash = "sha256:4699d8dfae43503b9c417bd2a6fb0da1c89e323fe91c4baa012f9298acaa83e1", size = 1570645, upload-time = "2025-10-01T16:11:24.467Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/93/02c7adec57a594af28388d85da9972703a4af94ae1399542555cd9581952/textual-6.2.1-py3-none-any.whl", hash = "sha256:3c7190633cd4d8bfe6049ae66808b98da91ded2edb85cef54e82bf77b03d2a54", size = 710702, upload-time = "2025-10-01T16:11:22.161Z" }, -] - [[package]] name = "threadpoolctl" version = "3.6.0" @@ -4652,27 +3873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, ] -[[package]] -name = "types-protobuf" -version = "6.32.1.20260221" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/e2/9aa4a3b2469508bd7b4e2ae11cbedaf419222a09a1b94daffcd5efca4023/types_protobuf-6.32.1.20260221.tar.gz", hash = "sha256:6d5fb060a616bfb076cbb61b4b3c3969f5fc8bec5810f9a2f7e648ee5cbcbf6e", size = 64408, upload-time = "2026-02-21T03:55:13.916Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/e8/1fd38926f9cf031188fbc5a96694203ea6f24b0e34bd64a225ec6f6291ba/types_protobuf-6.32.1.20260221-py3-none-any.whl", hash = "sha256:da7cdd947975964a93c30bfbcc2c6841ee646b318d3816b033adc2c4eb6448e4", size = 77956, upload-time = "2026-02-21T03:55:12.894Z" }, -] - -[[package]] -name = "types-requests" -version = "2.33.0.20260408" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/69/6a/749dc53a54a3f35842c1f8197b3ca6b54af6d7458a1bfc75f6629b6da666/types_requests-2.33.0.20260408.tar.gz", hash = "sha256:95b9a86376807a216b2fb412b47617b202091c3ea7c078f47cc358d5528ccb7b", size = 23882, upload-time = "2026-04-08T04:34:49.33Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/b8/78fd6c037de4788c040fdd323b3369804400351b7827473920f6c1d03c10/types_requests-2.33.0.20260408-py3-none-any.whl", hash = "sha256:81f31d5ea4acb39f03be7bc8bed569ba6d5a9c5d97e89f45ac43d819b68ca50f", size = 20739, upload-time = "2026-04-08T04:34:48.325Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" @@ -4703,15 +3903,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] -[[package]] -name = "uc-micro-py" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, -] - [[package]] name = "uncalled-for" version = "0.3.1" @@ -4880,25 +4071,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] -[[package]] -name = "xai-sdk" -version = "1.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "opentelemetry-sdk" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/32/bb8385f7a3b05ce406b689aa000c9a34289caa1526f1c093a1cefc0d9695/xai_sdk-1.11.0.tar.gz", hash = "sha256:ca87a830d310fb8e06fba44fb2a8c5cdf0d9f716b61126eddd51b7f416a63932", size = 404313, upload-time = "2026-03-27T18:23:10.091Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/76/86d9a3589c725ce825d2ed3e7cb3ecf7f956d3fd015353d52197bb341bcd/xai_sdk-1.11.0-py3-none-any.whl", hash = "sha256:fe58ce6d8f8115ae8bd57ded57bcd847d0bb7cb28bb7b236abefd4626df1ed8d", size = 251388, upload-time = "2026-03-27T18:23:08.573Z" }, -] - [[package]] name = "xlrd" version = "2.0.2" From 815d0911d56561705dd9f32b2f8a05ffd2c3d20b Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 16:14:47 +0200 Subject: [PATCH 13/22] fix: use generated import.sh for CSV graph loading in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI was reimplementing the import command by parsing CSV filenames, but extracted relationship types incorrectly — rels_IN_VERSION_IMASNodeChange_DDVersion.csv became type IN_VERSION_IMASNodeChange_DDVersion instead of IN_VERSION. Use the archive's import.sh which has correct types baked in. --- .github/workflows/docker-build-push.yml | 23 ++++++----------------- .github/workflows/graph-quality.yml | 20 ++++++-------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index a8d1c4bb3..01a0c667e 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -152,27 +152,16 @@ jobs: NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then - echo "CSV-based archive detected" - - # Run neo4j-admin import via Docker - CSV_DIR="${CONTENT_DIR}/csv" - - # Build import command from import.sh - IMPORT_ARGS="--overwrite-destination=true --id-type=string --array-delimiter=; --multiline-fields=true --skip-bad-relationships=true" - for f in "${CSV_DIR}"/nodes_*.csv; do - LABEL=$(basename "$f" | sed 's/^nodes_//;s/\.csv$//') - IMPORT_ARGS="${IMPORT_ARGS} --nodes=${LABEL}=/import/$(basename $f)" - done - for f in "${CSV_DIR}"/rels_*.csv; do - TYPE=$(basename "$f" | sed 's/^rels_//;s/\.csv$//') - IMPORT_ARGS="${IMPORT_ARGS} --relationships=${TYPE}=/import/$(basename $f)" - done + echo "CSV-based archive detected — using generated import.sh" + # Use the archive's import.sh which has correct relationship types docker run --rm \ -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${CSV_DIR}:/import" \ + -v "${CONTENT_DIR}/csv:/import" \ + -v "${CONTENT_DIR}/import.sh:/import.sh:ro" \ + -e CSV_DIR=/import \ neo4j:2026.01.4-community \ - neo4j-admin database import full neo4j ${IMPORT_ARGS} + bash /import.sh # Save DDL for post-start execution if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then diff --git a/.github/workflows/graph-quality.yml b/.github/workflows/graph-quality.yml index b8d4def89..620f96447 100644 --- a/.github/workflows/graph-quality.yml +++ b/.github/workflows/graph-quality.yml @@ -136,24 +136,16 @@ jobs: NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then - echo "CSV-based archive detected" - - CSV_DIR="${CONTENT_DIR}/csv" - IMPORT_ARGS="--overwrite-destination=true --id-type=string --array-delimiter=; --multiline-fields=true --skip-bad-relationships=true" - for f in "${CSV_DIR}"/nodes_*.csv; do - LABEL=$(basename "$f" | sed 's/^nodes_//;s/\.csv$//') - IMPORT_ARGS="${IMPORT_ARGS} --nodes=${LABEL}=/import/$(basename $f)" - done - for f in "${CSV_DIR}"/rels_*.csv; do - TYPE=$(basename "$f" | sed 's/^rels_//;s/\.csv$//') - IMPORT_ARGS="${IMPORT_ARGS} --relationships=${TYPE}=/import/$(basename $f)" - done + echo "CSV-based archive detected — using generated import.sh" + # Use the archive's import.sh which has correct relationship types docker run --rm \ -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${CSV_DIR}:/import" \ + -v "${CONTENT_DIR}/csv:/import" \ + -v "${CONTENT_DIR}/import.sh:/import.sh:ro" \ + -e CSV_DIR=/import \ neo4j:2026.01.4-community \ - neo4j-admin database import full neo4j ${IMPORT_ARGS} + bash /import.sh # Save DDL for post-start execution if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then From d06578706ce80453824ceb6e5923201286dae644 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 16:35:30 +0200 Subject: [PATCH 14/22] fix: CI tests against full graph dump, RC releases push full dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-quality.yml pulls imas-codex-graph (full dump) instead of imas-codex-graph-dd (CSV). Full dumps have zero edge cases — no property discovery, type inference, or filtering issues. RC releases now also push the full dump to GHCR so CI can pull it. DD-only CSV is still pushed for the container image. --- .github/workflows/graph-quality.yml | 4 ++-- imas_codex/cli/release.py | 29 +++++++++++++---------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/graph-quality.yml b/.github/workflows/graph-quality.yml index 620f96447..4053632ed 100644 --- a/.github/workflows/graph-quality.yml +++ b/.github/workflows/graph-quality.yml @@ -83,7 +83,7 @@ jobs: fi echo "tag=${TAG:-latest}" >> $GITHUB_OUTPUT echo "registry=${REGISTRY:-ghcr.io/iterorganization}" >> $GITHUB_OUTPUT - echo "Graph: ${REGISTRY}/imas-codex-graph-dd:${TAG}" + echo "Graph: ${REGISTRY}/imas-codex-graph:${TAG}" - name: Login to GHCR run: | @@ -91,7 +91,7 @@ jobs: - name: Pull graph from GHCR run: | - ARTIFACT="${{ steps.resolve-tag.outputs.registry }}/imas-codex-graph-dd:${{ steps.resolve-tag.outputs.tag }}" + ARTIFACT="${{ steps.resolve-tag.outputs.registry }}/imas-codex-graph:${{ steps.resolve-tag.outputs.tag }}" echo "Pulling: ${ARTIFACT}" mkdir -p /tmp/graph-dump oras pull "${ARTIFACT}" -o /tmp/graph-dump --allow-path-traversal diff --git a/imas_codex/cli/release.py b/imas_codex/cli/release.py index 690b82dd4..ea7589f86 100644 --- a/imas_codex/cli/release.py +++ b/imas_codex/cli/release.py @@ -1056,23 +1056,20 @@ def _push_all_graph_variants( dispatch_graph_quality(git_info, git_tag, registry) return - # ── Local push path — export+rebuild for filtered, dump for full ─── - # Full graph variant still needs a traditional dump (pushes the whole DB). - # DD-only and per-facility variants use export+rebuild from the live graph - # — zero downtime, compact output, ~2 min instead of 10-20 min. + # ── Local push path — dump for full, export+rebuild for filtered ─── + # Full graph is always pushed (RC and final) — CI tests against it. + # DD-only and per-facility variants use export+rebuild from the live graph. cached_dump = None - if not is_rc or len(facilities) == 0: - # Only create shared dump when we need the full variant - click.echo("\n Creating shared graph dump (stops Neo4j once)...") - if dry_run: - cached_dump = None - else: - cached_dump = _create_shared_dump() - if not cached_dump: - raise click.ClickException( - "Failed to create shared graph dump.\n" - " Is Neo4j running? Check: imas-codex graph status" - ) + click.echo("\n Creating shared graph dump (stops Neo4j once)...") + if dry_run: + cached_dump = None + else: + cached_dump = _create_shared_dump() + if not cached_dump: + raise click.ClickException( + "Failed to create shared graph dump.\n" + " Is Neo4j running? Check: imas-codex graph status" + ) failed: list[str] = [] variant = 0 From fa377d7165fe69b423a752cbeeaeb1cc88f44aea Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 17:34:30 +0200 Subject: [PATCH 15/22] fix: migrate from old node before tearing down SSH connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --set-default migration ran AFTER killing ControlMaster sockets, updating SSH config, and stopping tunnels — breaking connectivity to the old node. Move migration to run first while connections are warm. Also surface SSH stderr in the error message for easier diagnosis. --- imas_codex/cli/host.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/imas_codex/cli/host.py b/imas_codex/cli/host.py index 4e3c2a356..71562a748 100644 --- a/imas_codex/cli/host.py +++ b/imas_codex/cli/host.py @@ -905,9 +905,15 @@ def _migrate_from_node( f"({count} procs) — may re-spawn MCP servers" ) else: + stderr_hint = "" + if result.stderr: + # Show first line of stderr for diagnosis + first_line = result.stderr.strip().splitlines()[0] + stderr_hint = f" ({first_line})" click.echo( f" {click.style('⚠', fg='yellow')} " f"Could not reach {old_short} for process cleanup" + f"{stderr_hint}" ) except subprocess.TimeoutExpired: click.echo( @@ -1191,6 +1197,19 @@ def _discover_and_survey(): if current == target_fqdn: click.echo(f" Already set: {facility} → {target_fqdn}") else: + # Migrate FIRST — clean up processes on the old node + # while SSH connections are still warm. Must happen + # before we kill ControlMaster sockets, update config, + # or stop tunnels, all of which can break connectivity + # to the old node. + if current: + _migrate_from_node( + current, + gateway, + user, + timeout, + ) + # Kill ControlMaster sockets BEFORE config update so # ssh -O exit resolves to the old (current) HostName. sockets_to_kill = [facility] @@ -1223,16 +1242,6 @@ def _discover_and_survey(): + ")" ) - # Migrate: clean up processes and zellij sessions - # on the old node - if current: - _migrate_from_node( - current, - gateway, - user, - timeout, - ) - # LLM alias: --set-llm pins to explicit node, otherwise follows default if llm_node is not None: llm_fqdn = _resolve_node_fqdn(llm_node, sorted_nodes, results) From 753afda28908eaa5fd87e8d7cbca0e442341437b Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 19:17:16 +0200 Subject: [PATCH 16/22] refactor: remove CSV graph pipeline, use dump-filter-dump for all variants BREAKING CHANGE: CSV-based graph distribution removed after 7 RCs of cascading failures. All graph variants (full, dd-only, per-facility) now use the proven dump-filter-dump pipeline via temp Neo4j. - Delete export_rebuild.py (970 lines) and its feature plan - Remove CSV import branches from CI workflows and Dockerfile - CI tests against full graph dump (imas-codex-graph) - Container builds use dd-only dump (imas-codex-graph-dd) - Simplify Dockerfile: remove DDL execution step (dumps include indexes) --- .github/workflows/docker-build-push.yml | 105 +- .github/workflows/graph-quality.yml | 101 +- Dockerfile | 64 +- imas_codex/cli/graph/data.py | 76 +- imas_codex/cli/release.py | 16 +- imas_codex/graph/export_rebuild.py | 970 ------------------ .../features/export-rebuild-graph-pipeline.md | 728 ------------- 7 files changed, 87 insertions(+), 1973 deletions(-) delete mode 100644 imas_codex/graph/export_rebuild.py delete mode 100644 plans/features/export-rebuild-graph-pipeline.md diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 01a0c667e..beabc4281 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -69,7 +69,7 @@ jobs: - name: Resolve graph tag id: graph-tag run: | - IMAS_PACKAGE="imas-codex-graph-dd" + IMAS_PACKAGE="imas-codex-graph" OWNER_REGISTRY=$(echo "${{ env.GRAPH_REGISTRY }}" | tr '[:upper:]' '[:lower:]') FALLBACK_REGISTRY="${{ env.GRAPH_REGISTRY_FALLBACK }}" @@ -116,7 +116,7 @@ jobs: if: steps.graph-tag.outputs.imas-tag != 'none' run: | REGISTRY="${{ steps.graph-tag.outputs.graph-registry }}" - ARTIFACT="${REGISTRY}/imas-codex-graph-dd:${{ steps.graph-tag.outputs.imas-tag }}" + ARTIFACT="${REGISTRY}/imas-codex-graph:${{ steps.graph-tag.outputs.imas-tag }}" echo "Pulling: ${ARTIFACT}" mkdir -p /tmp/graph-dump oras pull "${ARTIFACT}" -o /tmp/graph-dump --allow-path-traversal @@ -145,52 +145,30 @@ jobs: mkdir -p /tmp/graph-extracted tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted - # Detect archive format: CSV (import.sh + csv/) or dump (.dump) - CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) - NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') - if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then - echo "CSV-based archive detected — using generated import.sh" - - # Use the archive's import.sh which has correct relationship types - docker run --rm \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${CONTENT_DIR}/csv:/import" \ - -v "${CONTENT_DIR}/import.sh:/import.sh:ro" \ - -e CSV_DIR=/import \ - neo4j:2026.01.4-community \ - bash /import.sh - - # Save DDL for post-start execution - if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then - cp "${CONTENT_DIR}/ddl.cypher" /tmp/ddl.cypher - fi - else - # Legacy dump-based archive - DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) - if [ -z "${DUMP_FILE}" ]; then - echo "ERROR: No import.sh+csv/ or .dump file found in archive" - ls -laR /tmp/graph-extracted/ - exit 1 - fi - echo "Found dump: ${DUMP_FILE}" - - DUMP_DIR=$(dirname "${DUMP_FILE}") - if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then - cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" - fi + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) + if [ -z "${DUMP_FILE}" ]; then + echo "ERROR: No .dump file found in archive" + ls -laR /tmp/graph-extracted/ + exit 1 + fi + echo "Found dump: ${DUMP_FILE}" - docker run --rm \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${DUMP_DIR}:/dump" \ - neo4j:2026.01.4-community \ - neo4j-admin database load neo4j \ - --from-path=/dump \ - --overwrite-destination=true + DUMP_DIR=$(dirname "${DUMP_FILE}") + if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then + cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" fi + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${DUMP_DIR}:/dump" \ + neo4j:2026.01.4-community \ + neo4j-admin database load neo4j \ + --from-path=/dump \ + --overwrite-destination=true + - name: Start Neo4j with loaded data if: steps.graph-tag.outputs.imas-tag != 'none' run: | @@ -212,49 +190,6 @@ jobs: sleep 2 done - - name: Execute DDL for CSV-based archives - if: steps.graph-tag.outputs.imas-tag != 'none' - run: | - if [ ! -f /tmp/ddl.cypher ]; then - echo "No DDL to execute (dump-based archive)" - exit 0 - fi - - NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) - echo "Executing DDL index statements..." - DDL_COUNT=0 - DDL_FAIL=0 - while IFS= read -r stmt; do - [ -z "$stmt" ] && continue - if docker exec "${NEO4J_CONTAINER}" cypher-shell \ - -u neo4j -p imas-codex "$stmt" 2>&1; then - DDL_COUNT=$((DDL_COUNT + 1)) - else - echo "WARNING: DDL failed: $stmt" - DDL_FAIL=$((DDL_FAIL + 1)) - fi - done < /tmp/ddl.cypher - echo "Executed $DDL_COUNT DDL statements ($DDL_FAIL failed)" - - if [ "$DDL_FAIL" -gt 0 ] && [ "$DDL_COUNT" -eq 0 ]; then - echo "ERROR: All DDL statements failed" - exit 1 - fi - - # Wait for indexes to come online - echo "Waiting for indexes to come online..." - for i in $(seq 1 300); do - PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell \ - -u neo4j -p imas-codex \ - "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') - if [ "$PENDING" = "0" ]; then - echo "All indexes ONLINE (${i}s)" - break - fi - sleep 1 - done - rm -f /tmp/ddl.cypher - - name: Reset Neo4j password if: steps.graph-tag.outputs.imas-tag != 'none' run: | diff --git a/.github/workflows/graph-quality.yml b/.github/workflows/graph-quality.yml index 4053632ed..e2954efca 100644 --- a/.github/workflows/graph-quality.yml +++ b/.github/workflows/graph-quality.yml @@ -129,52 +129,31 @@ jobs: mkdir -p /tmp/graph-extracted tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted - # Detect archive format: CSV (import.sh + csv/) or dump (.dump) - CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) - NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') - if [ -f "${CONTENT_DIR}/import.sh" ] && [ -d "${CONTENT_DIR}/csv" ]; then - echo "CSV-based archive detected — using generated import.sh" - - # Use the archive's import.sh which has correct relationship types - docker run --rm \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${CONTENT_DIR}/csv:/import" \ - -v "${CONTENT_DIR}/import.sh:/import.sh:ro" \ - -e CSV_DIR=/import \ - neo4j:2026.01.4-community \ - bash /import.sh - - # Save DDL for post-start execution - if [ -f "${CONTENT_DIR}/ddl.cypher" ]; then - cp "${CONTENT_DIR}/ddl.cypher" /tmp/ddl.cypher - fi - else - # Legacy dump-based archive - DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) - if [ -z "${DUMP_FILE}" ]; then - echo "ERROR: No import.sh+csv/ or .dump file found in archive" - ls -laR /tmp/graph-extracted/ - exit 1 - fi - echo "Found dump: ${DUMP_FILE}" - - DUMP_DIR=$(dirname "${DUMP_FILE}") - if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then - cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" - fi + # Dump-based archive + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) + if [ -z "${DUMP_FILE}" ]; then + echo "ERROR: No .dump file found in archive" + ls -laR /tmp/graph-extracted/ + exit 1 + fi + echo "Found dump: ${DUMP_FILE}" - docker run --rm \ - -v "${NEO4J_DATA_VOLUME}:/data" \ - -v "${DUMP_DIR}:/dump" \ - neo4j:2026.01.4-community \ - neo4j-admin database load neo4j \ - --from-path=/dump \ - --overwrite-destination=true + DUMP_DIR=$(dirname "${DUMP_FILE}") + if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then + cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" fi + docker run --rm \ + -v "${NEO4J_DATA_VOLUME}:/data" \ + -v "${DUMP_DIR}:/dump" \ + neo4j:2026.01.4-community \ + neo4j-admin database load neo4j \ + --from-path=/dump \ + --overwrite-destination=true + - name: Start Neo4j with loaded data run: | NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) @@ -194,48 +173,6 @@ jobs: sleep 2 done - - name: Execute DDL for CSV-based archives - run: | - if [ ! -f /tmp/ddl.cypher ]; then - echo "No DDL to execute (dump-based archive)" - exit 0 - fi - - NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) - echo "Executing DDL index statements..." - DDL_COUNT=0 - DDL_FAIL=0 - while IFS= read -r stmt; do - [ -z "$stmt" ] && continue - if docker exec "${NEO4J_CONTAINER}" cypher-shell \ - -u neo4j -p imas-codex "$stmt" 2>&1; then - DDL_COUNT=$((DDL_COUNT + 1)) - else - echo "WARNING: DDL failed: $stmt" - DDL_FAIL=$((DDL_FAIL + 1)) - fi - done < /tmp/ddl.cypher - echo "Executed $DDL_COUNT DDL statements ($DDL_FAIL failed)" - - if [ "$DDL_FAIL" -gt 0 ] && [ "$DDL_COUNT" -eq 0 ]; then - echo "ERROR: All DDL statements failed" - exit 1 - fi - - # Wait for indexes to come online - echo "Waiting for indexes to come online..." - for i in $(seq 1 300); do - PENDING=$(docker exec "${NEO4J_CONTAINER}" cypher-shell \ - -u neo4j -p imas-codex \ - "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') - if [ "$PENDING" = "0" ]; then - echo "All indexes ONLINE (${i}s)" - break - fi - sleep 1 - done - rm -f /tmp/ddl.cypher - - name: Reset Neo4j password run: | NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) diff --git a/Dockerfile b/Dockerfile index e6d1473ae..150e7dfd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,9 +154,7 @@ RUN --mount=type=secret,id=GHCR_TOKEN \ fi ## Stage 4: Load graph data into Neo4j data directory -# Handles both CSV-based archives (new) and dump-based archives (legacy). -# CSV format: csv/ dir + import.json + ddl.cypher + import.sh -# Dump format: graph.dump +# Graph archives contain a neo4j.dump file inside a tar.gz. FROM neo4j:2026.01.4-community AS graph-loader # Propagate GRAPH_TAG to bust cache when graph version changes @@ -189,30 +187,19 @@ RUN set -ex && \ mkdir -p /tmp/graph-extracted && \ tar -xzf "$ARCHIVE" -C /tmp/graph-extracted && \ rm -rf /tmp/graph-pull && \ - CONTENT_DIR=$(find /tmp/graph-extracted -maxdepth 1 -mindepth 1 -type d | head -1) && \ - if [ -f "$CONTENT_DIR/import.sh" ] && [ -d "$CONTENT_DIR/csv" ]; then \ - echo "CSV-based archive — running import.sh" && \ - CSV_DIR="$CONTENT_DIR/csv" bash "$CONTENT_DIR/import.sh" && \ - if [ -f "$CONTENT_DIR/ddl.cypher" ]; then \ - cp "$CONTENT_DIR/ddl.cypher" /tmp/ddl.cypher; \ - fi && \ - rm -rf /tmp/graph-extracted && \ - echo "Graph imported from CSV"; \ - else \ - DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \ - if [ -z "$DUMP_FILE" ]; then \ - echo "ERROR: No csv/ + import.sh or .dump found in archive" >&2; \ - find /tmp/graph-extracted -type f >&2; \ - exit 1; \ - fi && \ - echo "Found dump: $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))" && \ - mkdir -p /tmp/dumps && \ - mv "$DUMP_FILE" /tmp/dumps/neo4j.dump && \ - rm -rf /tmp/graph-extracted && \ - neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ - rm -rf /tmp/dumps && \ - echo "Graph loaded from dump"; \ - fi; \ + DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \ + if [ -z "$DUMP_FILE" ]; then \ + echo "ERROR: No .dump found in archive" >&2; \ + find /tmp/graph-extracted -type f >&2; \ + exit 1; \ + fi && \ + echo "Found dump: $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))" && \ + mkdir -p /tmp/dumps && \ + mv "$DUMP_FILE" /tmp/dumps/neo4j.dump && \ + rm -rf /tmp/graph-extracted && \ + neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ + rm -rf /tmp/dumps && \ + echo "Graph loaded from dump"; \ else \ echo "ERROR: No .dump or .tar.gz found in /tmp/graph-pull/" >&2; \ ls -la /tmp/graph-pull/ >&2; \ @@ -220,8 +207,7 @@ RUN set -ex && \ fi; \ fi -# Pre-start Neo4j to: (1) complete WAL recovery, (2) create system DB, -# and (3) execute DDL index statements for CSV-based imports. +# Pre-start Neo4j to complete WAL recovery and create system DB. # This shifts expensive work from runtime (slow Azure I/O) to build time (fast CI SSD). RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ echo "Pre-starting Neo4j for database recovery..." && \ @@ -247,26 +233,6 @@ RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ tail -50 /tmp/neo4j-recovery.log; \ exit 1; \ fi && \ - if [ -f /tmp/ddl.cypher ]; then \ - echo "Executing DDL index statements..." && \ - DDL_COUNT=0 && \ - while IFS= read -r stmt; do \ - [ -z "$stmt" ] && continue; \ - /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "$stmt" 2>&1 || true; \ - DDL_COUNT=$((DDL_COUNT + 1)); \ - done < /tmp/ddl.cypher && \ - echo "Executed $DDL_COUNT DDL statements — waiting for indexes..." && \ - for i in $(seq 1 300); do \ - PENDING=$(/var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 \ - "SHOW INDEXES YIELD state WHERE state <> 'ONLINE' RETURN count(*) AS c" 2>/dev/null | tail -1 | tr -d ' ') && \ - if [ "$PENDING" = "0" ]; then \ - echo "All indexes ONLINE (${i}s)"; \ - break; \ - fi; \ - sleep 1; \ - done && \ - rm -f /tmp/ddl.cypher; \ - fi && \ /var/lib/neo4j/bin/neo4j stop && \ sleep 2 && \ rm -f /tmp/neo4j-recovery.log && \ diff --git a/imas_codex/cli/graph/data.py b/imas_codex/cli/graph/data.py index 10586e0db..c646672b3 100644 --- a/imas_codex/cli/graph/data.py +++ b/imas_codex/cli/graph/data.py @@ -15,10 +15,6 @@ import click from imas_codex import __version__ -from imas_codex.graph.export_rebuild import ( - export_dd_only_csv, - export_facility_csv, -) from imas_codex.graph.ghcr import ( get_git_info, get_package_name, @@ -405,40 +401,12 @@ def graph_export( require_apptainer() def _build_archive(archive_dir: Path) -> None: - """Build the archive contents: CSV export or dump + manifest.""" - use_csv = not source_dump and (dd_only or facilities) - - if use_csv: - # CSV export from live graph — zero downtime, no dump needed - if dd_only: - click.echo(" Exporting DD-only graph to CSV...") - export_dd_only_csv(archive_dir) - elif facilities: - for fac in facilities: - click.echo(f" Exporting {fac}+DD graph to CSV...") - export_facility_csv(fac, archive_dir) - elif source_dump: - # Use existing dump with optional legacy filtering + """Build the archive contents: dump with optional filtering.""" + if source_dump: click.echo(f" Using cached dump: {source_dump}") shutil.copy(source_dump, str(archive_dir / "graph.dump")) - size_mb = (archive_dir / "graph.dump").stat().st_size / 1024 / 1024 - click.echo(f" Graph: {size_mb:.1f} MB") - - if facilities: - for fac in facilities: - click.echo(f" Filtering dump for facility: {fac}") - _create_facility_dump( - archive_dir / "graph.dump", - fac, - archive_dir / "graph.dump", - ) - if dd_only: - _create_dd_only_dump( - archive_dir / "graph.dump", - archive_dir / "graph.dump", - ) else: - # Full graph dump (no filtering) + # Create dump from live graph click.echo(" Dumping graph database...") dumps_dir = profile.data_dir / "dumps" dumps_dir.mkdir(parents=True, exist_ok=True) @@ -448,11 +416,28 @@ def _build_archive(archive_dir: Path) -> None: dump_file = dumps_dir / "neo4j.dump" if dump_file.exists(): shutil.move(str(dump_file), str(archive_dir / "graph.dump")) - size_mb = (archive_dir / "graph.dump").stat().st_size / 1024 / 1024 - click.echo(f" Graph: {size_mb:.1f} MB") else: raise click.ClickException("Graph dump file not created") + size_mb = (archive_dir / "graph.dump").stat().st_size / 1024 / 1024 + click.echo(f" Graph: {size_mb:.1f} MB") + + # Apply filtering to the dump + if facilities: + for fac in facilities: + click.echo(f" Filtering dump for facility: {fac}") + _create_facility_dump( + archive_dir / "graph.dump", + fac, + archive_dir / "graph.dump", + ) + if dd_only: + click.echo(" Filtering dump to DD-only...") + _create_dd_only_dump( + archive_dir / "graph.dump", + archive_dir / "graph.dump", + ) + manifest = { "version": __version__, "git_commit": git_info["commit"], @@ -615,20 +600,9 @@ def graph_load( click.echo(f" Version: {manifest.get('version')}") click.echo(f" Commit: {manifest.get('git_commit', 'unknown')[:7]}") - csv_dir = archive_dir / "csv" - import_manifest_file = archive_dir / "import.json" dump_file = archive_dir / "graph.dump" - if csv_dir.is_dir() and import_manifest_file.exists(): - # CSV-based archive: import + create indexes - from imas_codex.graph.export_rebuild import import_from_csv - - data_dir = profile.data_dir / "data" - data_dir.mkdir(parents=True, exist_ok=True) - import_from_csv(archive_dir, data_dir) - - elif dump_file.exists(): - # Legacy dump-based archive + if dump_file.exists(): click.echo(" Loading graph database...") dumps_dir = profile.data_dir / "dumps" dumps_dir.mkdir(parents=True, exist_ok=True) @@ -654,9 +628,7 @@ def graph_load( if result.returncode != 0: raise click.ClickException(f"Graph load failed: {result.stderr}") else: - raise click.ClickException( - "Archive contains neither csv/ + import.json nor graph.dump" - ) + raise click.ClickException("Archive does not contain graph.dump") if manifest_file.exists(): manifest = json.loads(manifest_file.read_text()) diff --git a/imas_codex/cli/release.py b/imas_codex/cli/release.py index ea7589f86..01a338d49 100644 --- a/imas_codex/cli/release.py +++ b/imas_codex/cli/release.py @@ -1056,9 +1056,9 @@ def _push_all_graph_variants( dispatch_graph_quality(git_info, git_tag, registry) return - # ── Local push path — dump for full, export+rebuild for filtered ─── - # Full graph is always pushed (RC and final) — CI tests against it. - # DD-only and per-facility variants use export+rebuild from the live graph. + # ── Local push path — dump-based for all variants ────────────────── + # Create a shared dump once, reuse for full + dd-only + per-facility. + # DD-only and per-facility variants filter the dump via temp Neo4j. cached_dump = None click.echo("\n Creating shared graph dump (stops Neo4j once)...") if dry_run: @@ -1074,7 +1074,7 @@ def _push_all_graph_variants( failed: list[str] = [] variant = 0 - # Push full graph (all facilities) — needs traditional dump + # Push full graph — CI tests against this if cached_dump or dry_run: variant += 1 click.echo( @@ -1089,14 +1089,15 @@ def _push_all_graph_variants( ): failed.append("full") - # Push dd-only — export+rebuild from live graph (no source dump needed) + # Push dd-only — filtered dump for container image variant += 1 - click.echo(f"\n Variant {variant}: IMAS Data Dictionary only (export+rebuild)") + click.echo(f"\n Variant {variant}: IMAS Data Dictionary only (filtered dump)") if not _push_graph_variant( dd_only=True, message=message, registry=registry, version_tag=git_tag, + source_dump=cached_dump, dry_run=dry_run, ): failed.append("dd-only") @@ -1109,12 +1110,13 @@ def _push_all_graph_variants( else: for fac in facilities: variant += 1 - click.echo(f"\n Variant {variant}: {fac} + IMAS DD (export+rebuild)") + click.echo(f"\n Variant {variant}: {fac} + IMAS DD (filtered dump)") if not _push_graph_variant( facility=fac, message=message, registry=registry, version_tag=git_tag, + source_dump=cached_dump, dry_run=dry_run, ): failed.append(fac) diff --git a/imas_codex/graph/export_rebuild.py b/imas_codex/graph/export_rebuild.py deleted file mode 100644 index 31871ee93..000000000 --- a/imas_codex/graph/export_rebuild.py +++ /dev/null @@ -1,970 +0,0 @@ -"""Export pipeline for creating filtered graph archives. - -Queries the live production graph via Cypher, exports nodes and relationships -to CSV, and captures index DDL statements. The output is a directory of CSVs -+ DDL that can be archived and distributed via GHCR. - -On the **load side** (``graph load`` / Docker entrypoint), the CSVs are -imported via ``neo4j-admin import`` and indexes created on first start. -This eliminates the fragile dump/load cycle entirely. - -Usage:: - - from imas_codex.graph.export_rebuild import export_dd_only_csv - - csv_dir = export_dd_only_csv(Path("/output/archive_dir")) -""" - -from __future__ import annotations - -import csv -import json -import logging -import subprocess -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import NamedTuple - -import click - -logger = logging.getLogger(__name__) - - -# ============================================================================ -# DD subgraph specification -# ============================================================================ - -DD_LABELS: list[str] = [ - "COCOS", - "DDVersion", - "GraphMeta", - "IDS", - "IdentifierSchema", - "IMASCoordinateSpec", - "IMASNode", - "IMASNodeChange", - "IMASSemanticCluster", - "Unit", -] - -# Labels with integer IDs (all others use string IDs) -INTEGER_ID_LABELS: set[str] = {"COCOS"} - - -class RelSpec(NamedTuple): - """Specification for a relationship type to export.""" - - rel_type: str - start_label: str - end_label: str - props: list[str] = [] - - -DD_RELATIONSHIPS: list[RelSpec] = [ - RelSpec("IN_VERSION", "IMASNodeChange", "DDVersion"), - RelSpec("FOR_IMAS_PATH", "IMASNodeChange", "IMASNode"), - RelSpec("INTRODUCED_IN", "IMASNode", "DDVersion"), - RelSpec("INTRODUCED_IN", "IDS", "DDVersion"), - RelSpec("IN_IDS", "IMASNode", "IDS"), - RelSpec("HAS_PARENT", "IMASNode", "IMASNode"), - RelSpec("IN_CLUSTER", "IMASNode", "IMASSemanticCluster"), - RelSpec("HAS_ERROR", "IMASNode", "IMASNode", ["error_type"]), - RelSpec("HAS_UNIT", "IMASNode", "Unit"), - RelSpec("HAS_COORDINATE", "IMASNode", "IMASCoordinateSpec", ["dimension"]), - RelSpec("HAS_COORDINATE", "IMASNode", "IMASNode", ["dimension"]), - RelSpec("DEPRECATED_IN", "IMASNode", "DDVersion"), - RelSpec("COORDINATE_SAME_AS", "IMASNode", "IMASNode", ["dimension"]), - RelSpec("RENAMED_TO", "IMASNode", "IMASNode"), - RelSpec("HAS_IDENTIFIER_SCHEMA", "IMASNode", "IdentifierSchema"), - RelSpec("HAS_PREDECESSOR", "DDVersion", "DDVersion"), - RelSpec("HAS_SUCCESSOR", "DDVersion", "DDVersion"), - RelSpec("HAS_COCOS", "DDVersion", "COCOS"), -] - - -# ============================================================================ -# Export configuration -# ============================================================================ - - -@dataclass -class ExportConfig: - """Configuration for an export+rebuild run.""" - - labels: list[str] = field(default_factory=lambda: list(DD_LABELS)) - relationships: list[RelSpec] = field(default_factory=lambda: list(DD_RELATIONSHIPS)) - batch_size: int = 5000 - facility: str | None = None - - -# ============================================================================ -# CSV export -# ============================================================================ - - -def _serialize_value(value: object) -> str: - """Serialize a Neo4j property value for CSV. - - Strips newlines from string values to prevent multi-line CSV fields - which neo4j-admin import rejects by default. - """ - if value is None: - return "" - if isinstance(value, list): - if value and isinstance(value[0], float | int): - # Float array (embedding) — semicolon-delimited - return ";".join(f"{v:.8g}" for v in value) - # String array — semicolon-delimited, newlines stripped - return ";".join(str(v).replace("\n", " ").replace("\r", "") for v in value) - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, str): - return value.replace("\n", " ").replace("\r", "") - return str(value) - - -def _neo4j_type(value: object) -> str: - """Infer neo4j-admin CSV type annotation from a Python value.""" - if isinstance(value, bool): - return "boolean" - if isinstance(value, int): - return "int" - if isinstance(value, float): - return "double" - if isinstance(value, list): - if value and isinstance(value[0], float | int): - return "float[]" - return "string[]" - return "string" - - -def export_nodes_csv( - gc: object, - label: str, - csv_dir: Path, - batch_size: int = 5000, -) -> tuple[Path, int]: - """Export all nodes of a label to CSV using keyset pagination. - - Returns (csv_path, row_count). - """ - # Discover properties from first batch - is_integer_id = label in INTEGER_ID_LABELS - - first_batch = gc.query( - f"MATCH (n:{label}) RETURN n ORDER BY n.id ASC LIMIT $limit", - limit=batch_size, - ) - - if not first_batch: - return csv_dir / f"nodes_{label}.csv", 0 - - # Extract property keys from first node (stable across nodes of same label) - sample_node = first_batch[0]["n"] - prop_keys = sorted(k for k in sample_node.keys() if k != "id") - - # Build header with type annotations - id_group = f"ID({label})" - header_parts = [f"id:{id_group}"] - - # Infer types from the sample - type_map: dict[str, str] = {} - for key in prop_keys: - val = sample_node.get(key) - if val is not None: - type_map[key] = _neo4j_type(val) - else: - type_map[key] = "string" - - for key in prop_keys: - t = type_map[key] - header_parts.append(f"{key}:{t}") - - csv_path = csv_dir / f"nodes_{label}.csv" - row_count = 0 - - with open(csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(header_parts) - - # Write first batch - for record in first_batch: - node = record["n"] - node_id = str(node["id"]) if not is_integer_id else node["id"] - row = [node_id] + [_serialize_value(node.get(k)) for k in prop_keys] - writer.writerow(row) - row_count += 1 - - # Keyset pagination for remaining batches - last_id = first_batch[-1]["n"]["id"] - while True: - batch = gc.query( - f"MATCH (n:{label}) WHERE n.id > $last_id " - f"RETURN n ORDER BY n.id ASC LIMIT $limit", - last_id=last_id, - limit=batch_size, - ) - if not batch: - break - for record in batch: - node = record["n"] - node_id = str(node["id"]) if not is_integer_id else node["id"] - row = [node_id] + [_serialize_value(node.get(k)) for k in prop_keys] - writer.writerow(row) - row_count += 1 - last_id = batch[-1]["n"]["id"] - - logger.info("Exported %d %s nodes to %s", row_count, label, csv_path.name) - return csv_path, row_count - - -def export_relationships_csv( - gc: object, - spec: RelSpec, - csv_dir: Path, - batch_size: int = 10000, -) -> tuple[Path, int]: - """Export relationships of a specific type to CSV. - - Returns (csv_path, row_count). - """ - start_group = f"START_ID({spec.start_label})" - end_group = f"END_ID({spec.end_label})" - - header_parts = [f":{start_group}", f":{end_group}"] - for prop in spec.props: - header_parts.append(prop) - - fname = f"rels_{spec.rel_type}_{spec.start_label}_{spec.end_label}.csv" - csv_path = csv_dir / fname - row_count = 0 - - # Build property return clause - prop_return = "" - if spec.props: - prop_return = ", " + ", ".join(f"r.{p} AS {p}" for p in spec.props) - - # Export with SKIP/LIMIT — relationships don't have stable IDs for keyset - with open(csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(header_parts) - - offset = 0 - while True: - cypher = ( - f"MATCH (a:{spec.start_label})-[r:{spec.rel_type}]->(b:{spec.end_label}) " - f"RETURN a.id AS start_id, b.id AS end_id{prop_return} " - f"SKIP $offset LIMIT $limit" - ) - batch = gc.query(cypher, offset=offset, limit=batch_size) - if not batch: - break - for record in batch: - start_id = str(record["start_id"]) - end_id = str(record["end_id"]) - row = [start_id, end_id] + [str(record.get(p, "")) for p in spec.props] - writer.writerow(row) - row_count += 1 - offset += len(batch) - if len(batch) < batch_size: - break - - logger.info( - "Exported %d %s rels (%s→%s) to %s", - row_count, - spec.rel_type, - spec.start_label, - spec.end_label, - csv_path.name, - ) - return csv_path, row_count - - -# ============================================================================ -# Index DDL capture -# ============================================================================ - - -def capture_index_ddl(gc: object, labels: list[str]) -> list[str]: - """Capture CREATE INDEX/CONSTRAINT statements for the given labels. - - Queries the live graph and reconstructs DDL statements. - Returns a list of Cypher CREATE statements. - """ - label_set = set(labels) - statements: list[str] = [] - - # Constraints - constraints = gc.query( - "SHOW CONSTRAINTS YIELD name, type, labelsOrTypes, properties" - ) - for c in constraints: - c_labels = c.get("labelsOrTypes") or [] - if not any(lbl in label_set for lbl in c_labels): - continue - lbl = c_labels[0] - props = c["properties"] - name = c["name"] - if c["type"] == "UNIQUENESS": - prop_str = ", ".join(f"n.{p}" for p in props) - statements.append( - f"CREATE CONSTRAINT {name} IF NOT EXISTS " - f"FOR (n:{lbl}) REQUIRE ({prop_str}) IS UNIQUE" - ) - - # Indexes (non-constraint) - indexes = gc.query( - "SHOW INDEXES YIELD name, type, labelsOrTypes, properties, " - "owningConstraint, options" - ) - for idx in indexes: - if idx.get("owningConstraint"): - continue # Skip constraint-backed indexes - idx_labels = idx.get("labelsOrTypes") or [] - if not any(lbl in label_set for lbl in idx_labels): - continue - - name = idx["name"] - lbl = idx_labels[0] - props = idx["properties"] - idx_type = idx["type"] - - if idx_type == "RANGE": - prop_str = ", ".join(f"n.{p}" for p in props) - statements.append( - f"CREATE INDEX {name} IF NOT EXISTS FOR (n:{lbl}) ON ({prop_str})" - ) - elif idx_type == "VECTOR": - options = idx.get("options", {}) - config = options.get("indexConfig", {}) - dim = config.get("vector.dimensions", 256) - sim = config.get("vector.similarity_function", "COSINE") - quant = config.get("vector.quantization.enabled", True) - prop = props[0] - statements.append( - f"CREATE VECTOR INDEX {name} IF NOT EXISTS " - f"FOR (n:{lbl}) ON (n.{prop}) " - f"OPTIONS {{indexConfig: {{" - f"`vector.dimensions`: {dim}, " - f"`vector.similarity_function`: '{sim}', " - f"`vector.quantization.enabled`: {'true' if quant else 'false'}" - f"}}}}" - ) - elif idx_type == "FULLTEXT": - prop_str = ", ".join(f"n.{p}" for p in props) - statements.append( - f"CREATE FULLTEXT INDEX {name} IF NOT EXISTS " - f"FOR (n:{lbl}) ON EACH [{prop_str}]" - ) - - return statements - - -# ============================================================================ -# neo4j-admin import -# ============================================================================ - - -def _neo4j_image() -> Path: - """Resolve the Neo4j Apptainer SIF image path.""" - from imas_codex.settings import get_neo4j_image_path - - return get_neo4j_image_path() - - -def run_import( - csv_dir: Path, - data_dir: Path, - node_files: list[tuple[str, Path]], - rel_files: list[tuple[RelSpec, Path]], -) -> None: - """Run neo4j-admin database import full with the exported CSVs.""" - image = _neo4j_image() - - # Build the import command - cmd = [ - "apptainer", - "exec", - "--bind", - f"{csv_dir}:/import", - "--bind", - f"{data_dir}:/data", - "--writable-tmpfs", - str(image), - "neo4j-admin", - "database", - "import", - "full", - "neo4j", - "--overwrite-destination=true", - "--id-type=string", - "--array-delimiter=;", - "--multiline-fields=true", - "--skip-bad-relationships=true", - "--verbose", - ] - - # Add node files with label annotations - for label, csv_path in node_files: - cmd.append(f"--nodes={label}=/import/{csv_path.name}") - - # Add relationship files - for spec, csv_path in rel_files: - cmd.append(f"--relationships={spec.rel_type}=/import/{csv_path.name}") - - click.echo(" Running neo4j-admin import...") - t0 = time.monotonic() - result = subprocess.run(cmd, capture_output=True, text=True) - elapsed = time.monotonic() - t0 - - if result.returncode != 0: - logger.error("Import stderr: %s", result.stderr[-2000:]) - raise click.ClickException( - f"neo4j-admin import failed (rc={result.returncode}):\n" - f"{result.stderr[-1000:]}" - ) - - click.echo(f" ✓ Import completed in {elapsed:.1f}s") - logger.info("Import stdout: %s", result.stdout[-500:]) - - -# ============================================================================ -# Post-import index creation -# ============================================================================ - - -def create_indexes_post_import( - data_dir: Path, - ddl_statements: list[str], - bolt_port: int = 7690, - http_port: int = 7480, -) -> None: - """Start a temp Neo4j on imported data, create indexes, wait for ONLINE, stop. - - Unlike ``start_temp_neo4j`` which loads a dump first, this starts Neo4j - directly on the data directory produced by ``neo4j-admin import``. - """ - import urllib.request - - from imas_codex.graph.temp_neo4j import ( - _cleanup_stale_temp_neo4j, - stop_temp_neo4j, - write_temp_neo4j_conf, - ) - - temp_dir = data_dir.parent - for subdir in ("conf", "logs", "run", "tmp"): - (temp_dir / subdir).mkdir(exist_ok=True) - - write_temp_neo4j_conf(temp_dir / "conf", bolt_port, http_port) - _cleanup_stale_temp_neo4j(bolt_port, http_port) - - image = _neo4j_image() - - # Start Neo4j directly on the imported data (no dump load needed) - click.echo(" Starting temp Neo4j on imported data...") - neo4j_log = temp_dir / "logs" / "neo4j-index.log" - log_fh = open(neo4j_log, "w") # noqa: SIM115 - start_cmd = [ - "apptainer", - "exec", - "--bind", - f"{data_dir}:/data", - "--bind", - f"{temp_dir}/logs:/logs", - "--bind", - f"{temp_dir}/conf:/var/lib/neo4j/conf", - "--bind", - f"{temp_dir}/run:/var/lib/neo4j/run", - "--bind", - f"{temp_dir}/tmp:/tmp", - "--writable-tmpfs", - str(image), - "neo4j", - "console", - ] - proc = subprocess.Popen( - start_cmd, - stdout=log_fh, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - - # Wait for readiness - ready = False - for _ in range(120): - if proc.poll() is not None: - log_fh.flush() - tail = neo4j_log.read_text()[-500:] if neo4j_log.exists() else "" - raise click.ClickException( - f"Temp Neo4j exited prematurely (rc={proc.returncode})\n{tail}" - ) - try: - urllib.request.urlopen(f"http://localhost:{http_port}/", timeout=2) - ready = True - break - except Exception: - time.sleep(1) - - if not ready: - log_fh.flush() - stop_temp_neo4j(proc) - log_fh.close() - tail = neo4j_log.read_text()[-500:] if neo4j_log.exists() else "" - raise click.ClickException(f"Temp Neo4j did not start within 120s\n{tail}") - - try: - from neo4j import GraphDatabase - - driver = GraphDatabase.driver(f"bolt://localhost:{bolt_port}") - try: - with driver.session() as session: - for stmt in ddl_statements: - try: - session.run(stmt).consume() - logger.debug("DDL: %s", stmt[:80]) - except Exception as e: - logger.warning( - "DDL failed (may already exist): %s — %s", stmt[:60], e - ) - - # Wait for all indexes to become ONLINE - click.echo(" Waiting for indexes to go ONLINE...") - for _ in range(120): - result = list(session.run("SHOW INDEXES YIELD state RETURN state")) - states = [r["state"] for r in result] - if states and all(s == "ONLINE" for s in states): - click.echo(f" ✓ {len(states)} indexes ONLINE") - break - time.sleep(1) - else: - not_online = [s for s in states if s != "ONLINE"] - logger.warning( - "%d indexes not ONLINE after 120s: %s", - len(not_online), - not_online[:5], - ) - finally: - # Force a checkpoint before closing the driver so all index data - # and transaction logs are flushed to store files. Without this, - # neo4j-admin dump refuses to run ("active logical log detected"). - try: - with driver.session() as session: - session.run("CALL db.checkpoint()").consume() - logger.debug("Checkpoint completed") - except Exception as e: - logger.warning("Checkpoint call failed: %s", e) - driver.close() - - # Clean shutdown: send SIGTERM to just the Apptainer process (not the - # whole process group). Apptainer forwards SIGTERM to the JVM which - # runs its shutdown hook (final checkpoint + close store). Killing the - # process group with os.killpg can race — Apptainer children may die - # before the JVM completes its shutdown hook, leaving active txn logs. - click.echo(" Stopping temp Neo4j (clean shutdown)...") - proc.terminate() # SIGTERM to Apptainer only - try: - proc.wait(timeout=30) - logger.debug("Temp Neo4j exited cleanly (rc=%d)", proc.returncode) - except subprocess.TimeoutExpired: - logger.warning("Clean shutdown timed out, falling back to SIGKILL") - stop_temp_neo4j(proc) - except Exception: - stop_temp_neo4j(proc) - raise - - -# ============================================================================ -# Export metadata helpers -# ============================================================================ - - -def _write_export_metadata( - output_dir: Path, - node_files: list[tuple[str, Path]], - rel_files: list[tuple[RelSpec, Path]], - ddl_statements: list[str], -) -> None: - """Write import.json, ddl.cypher, and import.sh into the output directory.""" - # import.json — structured metadata for programmatic loaders - import_manifest = { - "nodes": [ - {"label": label, "file": csv_path.name} for label, csv_path in node_files - ], - "relationships": [ - {"type": spec.rel_type, "file": csv_path.name} - for spec, csv_path in rel_files - ], - } - (output_dir / "import.json").write_text(json.dumps(import_manifest, indent=2)) - - # ddl.cypher — index/constraint DDL for post-import execution - (output_dir / "ddl.cypher").write_text("\n".join(ddl_statements)) - - # import.sh — self-contained import script for Docker/shell use - # Expects CSV_DIR env var pointing to the csv/ directory and - # DATA_DIR env var pointing to the target data directory. - lines = [ - "#!/bin/bash", - "# Auto-generated neo4j-admin import command", - "set -e", - 'CSV_DIR="${CSV_DIR:-.}"', - "", - "neo4j-admin database import full neo4j \\", - " --overwrite-destination=true \\", - " --id-type=string \\", - ' --array-delimiter=";" \\', - " --multiline-fields=true \\", - " --skip-bad-relationships=true \\", - ] - for label, csv_path in node_files: - lines.append(f' --nodes={label}="$CSV_DIR/{csv_path.name}" \\') - for spec, csv_path in rel_files: - lines.append(f' --relationships={spec.rel_type}="$CSV_DIR/{csv_path.name}" \\') - # Remove trailing backslash from last line - lines[-1] = lines[-1].rstrip(" \\") - (output_dir / "import.sh").write_text("\n".join(lines) + "\n") - - -# ============================================================================ -# Load-side: import from CSV archive -# ============================================================================ - - -def import_from_csv( - archive_dir: Path, - data_dir: Path, -) -> None: - """Import a CSV-based archive into a Neo4j data directory. - - This is the load-side counterpart to the export functions. It reads - ``import.json`` for file→label mappings, runs ``neo4j-admin import``, - then creates indexes from ``ddl.cypher`` via a temp Neo4j instance. - - Args: - archive_dir: Extracted archive directory containing csv/, import.json, - and ddl.cypher. - data_dir: Target Neo4j data directory (e.g. ``profile.data_dir/data``). - """ - csv_dir = archive_dir / "csv" - meta_file = archive_dir / "import.json" - ddl_file = archive_dir / "ddl.cypher" - - if not csv_dir.is_dir() or not meta_file.exists(): - raise click.ClickException( - f"Not a CSV archive: expected csv/ and import.json in {archive_dir}" - ) - - meta = json.loads(meta_file.read_text()) - - node_files = [(e["label"], csv_dir / e["file"]) for e in meta["nodes"]] - rel_files = [ - (RelSpec(e["type"], "", ""), csv_dir / e["file"]) for e in meta["relationships"] - ] - - click.echo(" Importing from CSV...") - run_import(csv_dir, data_dir, node_files, rel_files) - - if ddl_file.exists(): - ddl_statements = [ - s.strip() for s in ddl_file.read_text().splitlines() if s.strip() - ] - if ddl_statements: - click.echo(" Creating indexes...") - create_indexes_post_import(data_dir, ddl_statements) - - -# ============================================================================ -# Top-level orchestrators -# ============================================================================ - - -def export_dd_only_csv(output_dir: Path) -> Path: - """Export DD-only subgraph from live graph to CSVs + DDL. - - Queries live Neo4j for all DD labels and relationships, writes CSVs - and a ``ddl.cypher`` file into *output_dir*. Zero production downtime. - - The output directory is ready to be archived and distributed. The - load side runs ``import_from_csv()`` to build the database. - - Returns the output directory path. - """ - from imas_codex.graph.client import GraphClient - - config = ExportConfig() - - click.echo("Export: DD-only variant") - t_start = time.monotonic() - - csv_dir = output_dir / "csv" - csv_dir.mkdir(parents=True, exist_ok=True) - - click.echo("\n Exporting from live graph...") - t0 = time.monotonic() - - with GraphClient() as gc: - node_files: list[tuple[str, Path]] = [] - total_nodes = 0 - - for label in config.labels: - csv_path, count = export_nodes_csv(gc, label, csv_dir, config.batch_size) - if count > 0: - node_files.append((label, csv_path)) - total_nodes += count - click.echo(f" {label}: {count:,} nodes") - - rel_files: list[tuple[RelSpec, Path]] = [] - total_rels = 0 - - for spec in config.relationships: - csv_path, count = export_relationships_csv( - gc, spec, csv_dir, batch_size=10000 - ) - if count > 0: - rel_files.append((spec, csv_path)) - total_rels += count - - # Capture index DDL while still connected - ddl_statements = capture_index_ddl(gc, config.labels) - - export_time = time.monotonic() - t0 - click.echo( - f" ✓ Exported {total_nodes:,} nodes, {total_rels:,} rels in {export_time:.1f}s" - ) - - # Write DDL and import metadata - _write_export_metadata(output_dir, node_files, rel_files, ddl_statements) - click.echo(f" {len(ddl_statements)} DDL statements → ddl.cypher") - - total_time = time.monotonic() - t_start - csv_size = sum(f.stat().st_size for f in csv_dir.iterdir()) / 1024 / 1024 - click.echo(f"\n ✓ Export complete: {csv_size:.1f} MB CSV in {total_time:.1f}s") - - return output_dir - - -def export_facility_csv(facility: str, output_dir: Path) -> Path: - """Export facility+DD subgraph from live graph to CSVs + DDL. - - Exports all DD nodes plus nodes with ``facility_id = facility`` and - all relationships between the exported node set. - - Returns the output directory path. - """ - from imas_codex.graph.client import GraphClient - - click.echo(f"Export: {facility} + DD variant") - t_start = time.monotonic() - - csv_dir = output_dir / "csv" - csv_dir.mkdir(parents=True, exist_ok=True) - - with GraphClient() as gc: - # Export DD nodes - click.echo("\n Exporting DD nodes...") - config = ExportConfig() - node_files: list[tuple[str, Path]] = [] - total_nodes = 0 - - for label in config.labels: - csv_path, count = export_nodes_csv(gc, label, csv_dir, config.batch_size) - if count > 0: - node_files.append((label, csv_path)) - total_nodes += count - - click.echo(f" DD: {total_nodes:,} nodes") - - # Export facility nodes - click.echo(f"\n Exporting {facility} nodes...") - facility_labels = gc.query( - "MATCH (n) WHERE n.facility_id = $facility " - "WITH labels(n) AS lbls UNWIND lbls AS lbl " - "RETURN DISTINCT lbl AS label, count(*) AS cnt " - "ORDER BY cnt DESC", - facility=facility, - ) - - dd_label_set = set(config.labels) - for row in facility_labels: - lbl = row["label"] - if lbl in dd_label_set: - continue - - csv_path, count = _export_facility_nodes_csv( - gc, lbl, facility, csv_dir, config.batch_size - ) - if count > 0: - node_files.append((lbl, csv_path)) - total_nodes += count - click.echo(f" {lbl}: {count:,} nodes") - - # Export relationships - click.echo("\n Exporting relationships...") - rel_files: list[tuple[RelSpec, Path]] = [] - total_rels = 0 - - # DD relationships - for spec in config.relationships: - csv_path, count = export_relationships_csv( - gc, spec, csv_dir, batch_size=10000 - ) - if count > 0: - rel_files.append((spec, csv_path)) - total_rels += count - - # Facility relationships — discover and export - fac_rels = gc.query( - "MATCH (a)-[r]->(b) " - "WHERE a.facility_id = $facility OR b.facility_id = $facility " - "WITH type(r) AS rel_type, labels(a)[0] AS start_lbl, " - "labels(b)[0] AS end_lbl, count(*) AS cnt " - "RETURN rel_type, start_lbl, end_lbl, cnt " - "ORDER BY cnt DESC", - facility=facility, - ) - - exported_rels = { - (s.rel_type, s.start_label, s.end_label) for s in config.relationships - } - for row in fac_rels: - key = (row["rel_type"], row["start_lbl"], row["end_lbl"]) - if key in exported_rels: - continue - - spec = RelSpec(row["rel_type"], row["start_lbl"], row["end_lbl"]) - csv_path, count = _export_facility_rels_csv(gc, spec, facility, csv_dir) - if count > 0: - rel_files.append((spec, csv_path)) - total_rels += count - exported_rels.add(key) - - ddl_statements = capture_index_ddl( - gc, - config.labels + [r["label"] for r in facility_labels], - ) - - click.echo(f" ✓ Exported {total_nodes:,} nodes, {total_rels:,} rels") - - # Write DDL and import manifest - _write_export_metadata(output_dir, node_files, rel_files, ddl_statements) - - total_time = time.monotonic() - t_start - csv_size = sum(f.stat().st_size for f in csv_dir.iterdir()) / 1024 / 1024 - click.echo(f"\n ✓ Export complete: {csv_size:.1f} MB CSV in {total_time:.1f}s") - - return output_dir - - -# ============================================================================ -# Facility-specific export helpers -# ============================================================================ - - -def _export_facility_nodes_csv( - gc: object, - label: str, - facility: str, - csv_dir: Path, - batch_size: int = 5000, -) -> tuple[Path, int]: - """Export nodes of a label filtered by facility_id.""" - first_batch = gc.query( - f"MATCH (n:{label}) WHERE n.facility_id = $facility " - f"RETURN n ORDER BY n.id ASC LIMIT $limit", - facility=facility, - limit=batch_size, - ) - - if not first_batch: - return csv_dir / f"nodes_{label}_{facility}.csv", 0 - - sample_node = first_batch[0]["n"] - prop_keys = sorted(k for k in sample_node.keys() if k != "id") - - id_group = f"ID({label})" - header_parts = [f"id:{id_group}"] - for key in prop_keys: - val = sample_node.get(key) - t = _neo4j_type(val) if val is not None else "string" - header_parts.append(f"{key}:{t}") - - csv_path = csv_dir / f"nodes_{label}_{facility}.csv" - row_count = 0 - - with open(csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(header_parts) - - for record in first_batch: - node = record["n"] - row = [str(node["id"])] + [_serialize_value(node.get(k)) for k in prop_keys] - writer.writerow(row) - row_count += 1 - - last_id = first_batch[-1]["n"]["id"] - while True: - batch = gc.query( - f"MATCH (n:{label}) WHERE n.facility_id = $facility " - f"AND n.id > $last_id " - f"RETURN n ORDER BY n.id ASC LIMIT $limit", - facility=facility, - last_id=last_id, - limit=batch_size, - ) - if not batch: - break - for record in batch: - node = record["n"] - row = [str(node["id"])] + [ - _serialize_value(node.get(k)) for k in prop_keys - ] - writer.writerow(row) - row_count += 1 - last_id = batch[-1]["n"]["id"] - - return csv_path, row_count - - -def _export_facility_rels_csv( - gc: object, - spec: RelSpec, - facility: str, - csv_dir: Path, - batch_size: int = 10000, -) -> tuple[Path, int]: - """Export facility-specific relationships.""" - start_group = f"START_ID({spec.start_label})" - end_group = f"END_ID({spec.end_label})" - header_parts = [f":{start_group}", f":{end_group}"] - - fname = f"rels_{spec.rel_type}_{spec.start_label}_{spec.end_label}_{facility}.csv" - csv_path = csv_dir / fname - row_count = 0 - - with open(csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(header_parts) - - offset = 0 - while True: - batch = gc.query( - f"MATCH (a:{spec.start_label})-[r:{spec.rel_type}]->(b:{spec.end_label}) " - f"WHERE a.facility_id = $facility OR b.facility_id = $facility " - f"RETURN a.id AS start_id, b.id AS end_id " - f"SKIP $offset LIMIT $limit", - facility=facility, - offset=offset, - limit=batch_size, - ) - if not batch: - break - for record in batch: - writer.writerow([str(record["start_id"]), str(record["end_id"])]) - row_count += 1 - offset += len(batch) - if len(batch) < batch_size: - break - - return csv_path, row_count diff --git a/plans/features/export-rebuild-graph-pipeline.md b/plans/features/export-rebuild-graph-pipeline.md deleted file mode 100644 index c7d340875..000000000 --- a/plans/features/export-rebuild-graph-pipeline.md +++ /dev/null @@ -1,728 +0,0 @@ -# Export + Rebuild Pipeline for Filtered Graph Dumps - -## Problem - -The release workflow creates three graph dump variants: **full** (all nodes), -**dd-only** (IMAS Data Dictionary only), and **per-facility** (DD + one -facility). The current filtering approach in `temp_neo4j.py` is: - -1. Stop production Neo4j -2. `neo4j-admin database dump` the entire graph (~1.9 GB) -3. Load the dump into a temporary Neo4j instance -4. Start the temp instance and wait for readiness -5. Run Cypher `DELETE` queries to remove unwanted nodes/relationships -6. Stop the temp instance -7. `neo4j-admin database dump` the filtered result -8. Restart production Neo4j - -### Measured Costs - -| Step | Duration | Notes | -|------|----------|-------| -| Stop production | ~5 s | Causes downtime for MCP server | -| Full dump (1.9 GB) | ~30 s | Entire graph serialized | -| Load into temp | ~20 s | Full dump loaded | -| Temp Neo4j startup + recovery | ~60–120 s | WAL replay on 1.9 GB | -| Cypher DELETE (dd-only) | ~300–600 s | `MATCH (n) WHERE NOT ... DETACH DELETE n` in batches | -| Cypher DELETE (facility) | ~120–300 s | More selective | -| Stop temp + dump filtered | ~30 s | | -| **Total per variant** | **~10–20 min** | × 3 variants = 25–50 min | - -### Specific Failures - -- **Production downtime**: ~30 s per dump cycle while Neo4j is stopped -- **OOM on CI**: Full graph load + Cypher DELETE + transaction log = memory - spike. 4 GB SLURM jobs often fail; 8–16 GB required -- **Bloated output**: The filtered dump retains free-space from deleted nodes. - DD-only dump (~200 MB of data) weighs ~900 MB because it was carved from a - 1.9 GB store -- **Slow filtering**: Cypher `DETACH DELETE` on 1.3 M nodes (for dd-only) is - O(n) in the *removed* set, not the *kept* set - ---- - -## Solution: Query Live → CSV → `neo4j-admin import` → Dump - -Build filtered graphs from scratch rather than carving them from the full -dump. Query the **live** production Neo4j via Cypher, export matching nodes -and relationships to CSV, use `neo4j-admin database import full` to construct -a fresh compact database, create indexes, then dump. - -### Architecture Overview - -``` -┌─────────────────────────────────────┐ -│ LIVE Production Neo4j │ -│ bolt://98dci4-clu-2001:7687 │ -│ ~1.5M nodes, ~4.4M relationships │ -└───────────┬─────────────────────────┘ - │ Cypher read queries - │ (keyset pagination) - ▼ -┌─────────────────────────────────────┐ -│ Phase 1: Export to CSV │ -│ • Per-label node CSVs │ -│ • Per-type relationship CSVs │ -│ • Index/constraint DDL captured │ -│ Timing: ~22 s (sequential) │ -│ Output: ~200 MB CSV (DD-only) │ -└───────────┬─────────────────────────┘ - │ CSV files on local disk - ▼ -┌─────────────────────────────────────┐ -│ Phase 2: neo4j-admin import full │ -│ • Reads CSV directly (no Neo4j) │ -│ • Creates compact store files │ -│ • ID groups prevent cross-label │ -│ collisions │ -│ Timing: ~40 s (projected) │ -│ Output: ~50 MB database store │ -└───────────┬─────────────────────────┘ - │ /data/databases/neo4j/ - ▼ -┌─────────────────────────────────────┐ -│ Phase 3: Create Indexes │ -│ • Start temp Neo4j (brief) │ -│ • CREATE CONSTRAINT / INDEX │ -│ • Wait for ONLINE state │ -│ • Stop temp Neo4j │ -│ Timing: ~30–45 s │ -└───────────┬─────────────────────────┘ - │ /data/databases/neo4j/ - ▼ -┌─────────────────────────────────────┐ -│ Phase 4: neo4j-admin dump │ -│ • Serializes compact store to dump │ -│ Timing: ~15–25 s │ -│ Output: ~25 MB .dump file │ -└─────────────────────────────────────┘ -``` - -### Key Benefits - -| Metric | Current | New Pipeline | -|--------|---------|-------------| -| Production downtime | ~30 s per variant | **Zero** (read-only queries) | -| DD-only total time | 10–20 min | **~1.5 min** | -| Per-facility total time | 5–10 min | **~2 min** | -| DD-only dump size | ~900 MB (bloated) | **~25 MB** (compact) | -| Memory requirement | 8–16 GB | **4 GB** sufficient | -| Failure mode | Corrupted temp store | CSV on disk (restartable) | - ---- - -## Benchmarks - -All benchmarks run against the live production graph on `98dci4-clu-2001`. -Neo4j 2026.01.4 Community in Apptainer. Measurements are wall-clock times -including network latency. - -### DD Subgraph Statistics - -| Label | Nodes | With Embeddings | CSV Size | -|-------|-------|-----------------|----------| -| IMASNode | 61,366 | 20,037 | 89.4 MB | -| IMASNodeChange | 137,310 | 0 | 22.5 MB | -| IMASSemanticCluster | 3,912 | 3,912 (×3 cols) | 36.6 MB | -| DDVersion | 35 | 0 | < 1 KB | -| Unit | 182 | 0 | < 1 KB | -| IMASCoordinateSpec | 109 | 0 | < 1 KB | -| IdentifierSchema | 62 | 62 | < 1 KB | -| IDS | 87 | 87 | < 1 KB | -| COCOS | 18 | 0 | < 1 KB | -| GraphMeta | 1 | 0 | < 1 KB | -| **Total** | **205,047** | **~24,100** | **~150 MB** | - -Note: Embeddings are 256-dimensional float arrays. PhysicsDomain, -SignConvention, CoordinateRelationship, and ClusterMembership labels have -0 nodes currently and are excluded. - -### Node Export Performance - -Sequential export using keyset pagination (`WHERE n.id > $last LIMIT 5000`): - -| Label | Batches | Time | Throughput | -|-------|---------|------|------------| -| IMASNode | 13 | 6.55 s | 9,369 nodes/s | -| IMASNodeChange | 28 | 3.35 s | 40,937 nodes/s | -| IMASSemanticCluster | 1 | 1.65 s | 2,372 nodes/s | -| Others (small) | 1 each | < 1 s | — | -| **Total** | — | **12.8 s** | — | - -Throughput for IMASNode is lower because embedding serialization dominates -(~4.5 KB per 256-dim vector in CSV text form). - -### Relationship Export Performance - -| Relationship Type | Count | Source → Target | -|---|---|---| -| IN_VERSION | 137,310 | IMASNodeChange → DDVersion | -| FOR_IMAS_PATH | 94,158 | IMASNodeChange → IMASNode | -| INTRODUCED_IN | 61,583 | IMASNode/IDS → DDVersion | -| IN_IDS | 61,366 | IMASNode → IDS | -| HAS_PARENT | 60,334 | IMASNode → IMASNode | -| IN_CLUSTER | 33,873 | IMASNode → IMASSemanticCluster | -| HAS_ERROR | 31,281 | IMASNode → IMASNode | -| HAS_UNIT | 25,270 | IMASNode → Unit | -| HAS_COORDINATE (→Spec) | 13,769 | IMASNode → IMASCoordinateSpec | -| HAS_COORDINATE (→Node) | 12,467 | IMASNode → IMASNode | -| DEPRECATED_IN | 17,324 | IMASNode → DDVersion | -| COORDINATE_SAME_AS | 7,439 | IMASNode → IMASNode | -| RENAMED_TO | 2,696 | IMASNode → IMASNode | -| HAS_IDENTIFIER_SCHEMA | 327 | IMASNode → IdentifierSchema | -| HAS_PREDECESSOR | 34 | DDVersion → DDVersion | -| HAS_SUCCESSOR | 34 | DDVersion → DDVersion | -| HAS_COCOS | 17 | DDVersion → COCOS | -| **Total** | **559,282** | **8.9 s** | - -Three relationship types carry properties: -- `HAS_ERROR`: `error_type` (string) -- `HAS_COORDINATE`: `dimension` (integer) -- `COORDINATE_SAME_AS`: `dimension` (integer) - -### Parallel Export - -Tested 4 concurrent threads exporting different labels simultaneously: - -| Configuration | Time | Speedup | -|---|---|---| -| Sequential (1 thread) | 12.8 s | 1.0× | -| 4 threads | 9.7 s | 1.3× | - -Bottleneck is the Neo4j server, not the client. Parallel export provides -marginal benefit and adds complexity. **Recommendation: sequential export.** - -### neo4j-admin Import Performance - -Tested with actual data structure (nodes with 256-dim embeddings, -relationships, label groups): - -| Test Scale | Import Time | DB Size | Dump Size | Dump Time | -|---|---|---|---|---| -| 1K nodes + 999 rels | 2.5 s (+7 s JVM) | 2.3 MB | 1.0 MB | 8 s | -| 120K nodes + 200K rels | 22.5 s | 48.8 MB | 22.8 MB | 12 s | - -**Projected for DD data (205K nodes, 559K rels):** -- Import: ~40 s -- Dump: ~25 s - -### `--schema` Flag Incompatibility - -`neo4j-admin database import full --schema=` **fails** with: - -> Record format batch import does not support schema changes - -Indexes and constraints must be created **post-import** by briefly starting a -temp Neo4j instance. This adds ~30–45 s (Neo4j startup + DDL execution + -index population) but is the only supported path. - -### Projected End-to-End Timing - -| Phase | DD-Only | Per-Facility | -|---|---|---| -| Export nodes to CSV | 13 s | 15 s (+facility nodes) | -| Export relationships to CSV | 9 s | 12 s (+facility rels) | -| `neo4j-admin import full` | 40 s | 50 s | -| Start temp Neo4j | 20 s | 20 s | -| Create indexes + wait ONLINE | 15 s | 20 s | -| Stop temp Neo4j | 5 s | 5 s | -| `neo4j-admin dump` | 25 s | 30 s | -| **Total** | **~2 min** | **~2.5 min** | - -vs current: **10–20 min** (dd-only), **5–10 min** (per-facility). - ---- - -## Design Decisions - -### CSV Format (not Parquet, not JSONL) - -- **Human-readable** for debugging — `head -5 nodes_IMASNode.csv` -- **Native vector support** — `--vector-delimiter=;` handles float arrays -- **~200 MB total** for DD — well within memory/disk constraints -- Parquet would save ~30% on disk but adds a build dependency and loses - readability. The total CSV volume is trivially small. - -### Label-Specific ID Groups - -COCOS nodes use integer IDs (1–18), all other labels use string IDs. Without -ID groups, `neo4j-admin import` treats IDs as globally unique, causing -collisions or mismatched relationships. - -Solution: each label gets its own ID namespace: - -```csv -# nodes_COCOS.csv -id:ID(COCOS),convention:int,... -1,1,... -``` - -```csv -# rels_HAS_COCOS.csv -:START_ID(DDVersion),:END_ID(COCOS) -3.39.0,11 -``` - -This maps exactly to neo4j-admin's `--id-type=string` with group syntax. - -### Keyset Pagination (not SKIP/LIMIT) - -Standard SKIP/LIMIT can produce inconsistent snapshots if nodes are -added/modified during export. Keyset pagination guarantees each node is -exported exactly once: - -```cypher -MATCH (n:IMASNode) -WHERE n.id > $last_id -RETURN n.id AS id, n.name AS name, ... -ORDER BY n.id ASC -LIMIT 5000 -``` - -For the DD subgraph (which changes only during DD ingestion, not -continuously), this is extra safety that costs nothing. - -### Sequential Export (not Parallel) - -Benchmarked 4 threads → only 1.3× speedup. The Neo4j server is the -bottleneck. Sequential export is simpler, deterministic, and easier to debug. -Total export time is ~22 s regardless. - -### Post-Import Index Creation (not --schema) - -`neo4j-admin import full --schema` fails on Neo4j 2026.01.4 with "Record -format batch import does not support schema changes". The workaround: - -1. Import CSV data (no indexes) -2. Start a temp Neo4j instance pointing at the imported data dir -3. Execute `CREATE CONSTRAINT` and `CREATE INDEX` statements -4. Wait for all indexes to reach `ONLINE` state -5. Stop the temp instance -6. Dump the database - -This adds ~30 s but is reliable and uses the same temp Neo4j lifecycle -management already implemented in `temp_neo4j.py`. - -### Separate Module (not extending temp_neo4j.py) - -The new pipeline has a fundamentally different approach (build vs carve) with -different failure modes, dependencies, and lifecycle. It should live in a -new module alongside `temp_neo4j.py` rather than extending it: - -- `imas_codex/graph/export_rebuild.py` — the new pipeline -- `imas_codex/graph/temp_neo4j.py` — retained for backward compatibility - until the new pipeline is proven - -### APOC Evaluation - -APOC provides `apoc.export.csv.*` procedures that could simplify the export -phase. However: -- APOC is **not installed** in the production Apptainer image -- Adding APOC requires rebuilding the image, testing compatibility, and - managing plugin versions across Neo4j upgrades -- The native Cypher export (22 s total) is already fast enough -- APOC adds a runtime dependency for a build-time operation - -**Recommendation: do not use APOC.** The Cypher+Python CSV writer is simpler, -faster to develop, and has zero additional dependencies. - -### Pipe/stdin Import - -`neo4j-admin database import full` does **not** support reading from stdin -or named pipes. All input must be regular files on disk. This is a non-issue -since the total CSV volume is ~200 MB and the export phase writes directly to -the temp directory used by the import phase. - -### SLURM Execution - -The existing SLURM dispatch pattern in `temp_neo4j.py` (`_run_filter_via_slurm`, -`_should_use_slurm`) should be reused. The new pipeline's resource requirements -are actually **lower** than the current approach: - -| Resource | Current | New Pipeline | -|---|---|---| -| Memory | 8–16 GB (Neo4j + Cypher DELETE) | 4 GB (neo4j-admin import) | -| Disk | ~3 GB (full dump + temp store) | ~300 MB (CSV + temp store) | -| Time | 10–20 min | ~2 min | - -A 4 GB / 30 min SLURM allocation is conservative and sufficient. - ---- - -## DD Index Inventory - -Indexes that must be recreated in the filtered dump, captured from production: - -### Constraints (10) - -| Label | Properties | Type | -|-------|-----------|------| -| COCOS | id | UNIQUENESS | -| DDVersion | id | UNIQUENESS | -| IDS | id | UNIQUENESS | -| IMASCoordinateSpec | id | UNIQUENESS | -| IMASNode | id | UNIQUENESS | -| IMASNodeChange | id | UNIQUENESS | -| IMASSemanticCluster | id | UNIQUENESS | -| IdentifierSchema | id | UNIQUENESS | -| SignConvention | id, facility_id | UNIQUENESS | -| Unit | id | UNIQUENESS | - -### Range Indexes (13 non-constraint) - -| Label | Properties | -|-------|-----------| -| DDVersion | status | -| IDS | name | -| IMASNode | node_category | -| IMASNode | node_category, ids | -| IMASNode | is_leaf | -| IMASNode | path_lower | -| IMASNode | ids | -| IMASNode | status | -| IMASNode | url | -| SignConvention | facility_id | -| SignConvention | id | -| Unit | symbol | - -### Vector Indexes (6) - -| Label | Property | Dimensions | Similarity | Quantization | -|-------|----------|-----------|------------|--------------| -| IDS | embedding | 256 | COSINE | true | -| IMASNode | embedding | 256 | COSINE | true | -| IMASSemanticCluster | embedding | 256 | COSINE | true | -| IMASSemanticCluster | label_embedding | 256 | COSINE | true | -| IMASSemanticCluster | description_embedding | 256 | COSINE | true | -| IdentifierSchema | embedding | 256 | COSINE | true | - -### Fulltext Indexes (1) - -| Name | Label | Properties | Analyzer | -|------|-------|-----------|----------| -| imas_node_text | IMASNode | documentation, name, id, description, keywords | standard-no-stop-words | - -**Total: 30 indexes** (10 constraint-backed + 13 range + 6 vector + 1 fulltext) - ---- - -## Implementation Plan - -### Phase 1: Core Export Module - -**File: `imas_codex/graph/export_rebuild.py`** - -```python -"""Export + rebuild pipeline for creating filtered graph dumps. - -Queries the live production graph via Cypher, exports nodes and relationships -to CSV, builds a fresh database with neo4j-admin import, creates indexes, -and produces a compact dump file. -""" -``` - -Key components: - -1. **`ExportConfig` dataclass** — holds label lists, relationship specs, - facility filter, temp directory, batch size (5000), GraphClient reference - -2. **`export_nodes_csv(config, label, property_keys, output_dir)`** — - Exports all nodes of a given label to CSV using keyset pagination. - Handles embedding serialization (`";".join(f"{v:.8g}" for v in vec)`). - Returns row count and file path. - -3. **`export_relationships_csv(config, rel_type, start_label, end_label, output_dir)`** — - Exports relationships to CSV. Splits types with multiple target labels - (HAS_COORDINATE) into separate files. Includes property columns where - applicable. - -4. **`capture_index_ddl(config, labels)`** — Queries `SHOW INDEXES` and - `SHOW CONSTRAINTS` for matching labels and returns a list of Cypher - CREATE statements for post-import replay. - -5. **`run_import(csv_dir, data_dir, neo4j_image)`** — Assembles the - `neo4j-admin database import full` command with proper `--nodes`, - `--relationships`, `--id-type=string`, `--vector-delimiter=;` flags. - Runs via `subprocess.run()` (or `srun` if on SLURM). - -6. **`create_indexes_post_import(data_dir, ddl_statements, neo4j_image)`** — - Starts a temp Neo4j pointed at the imported data dir, executes CREATE - CONSTRAINT/INDEX statements, polls `SHOW INDEXES` until all are ONLINE, - then stops. - -7. **`run_dump(data_dir, output_path, neo4j_image)`** — Executes - `neo4j-admin database dump` on the built database. - -8. **`export_rebuild_dd_only(output_path)`** — Top-level orchestrator for - the DD-only variant. Calls phases 1–4 in sequence. - -9. **`export_rebuild_facility(facility_id, output_path)`** — Top-level - orchestrator for per-facility variant. Exports DD nodes + facility-specific - nodes and their inter-relationships. - -### Phase 2: DD Label and Relationship Registry - -Extract the DD subgraph specification from hardcoded constants into a -queryable registry: - -```python -# Relationship types internal to DD subgraph -DD_RELATIONSHIPS = [ - RelSpec("IN_VERSION", "IMASNodeChange", "DDVersion"), - RelSpec("FOR_IMAS_PATH", "IMASNodeChange", "IMASNode"), - RelSpec("INTRODUCED_IN", "IMASNode", "DDVersion"), - RelSpec("INTRODUCED_IN", "IDS", "DDVersion"), - RelSpec("IN_IDS", "IMASNode", "IDS"), - RelSpec("HAS_PARENT", "IMASNode", "IMASNode"), - RelSpec("IN_CLUSTER", "IMASNode", "IMASSemanticCluster"), - RelSpec("HAS_ERROR", "IMASNode", "IMASNode", props=["error_type"]), - RelSpec("HAS_UNIT", "IMASNode", "Unit"), - RelSpec("HAS_COORDINATE", "IMASNode", "IMASCoordinateSpec", props=["dimension"]), - RelSpec("HAS_COORDINATE", "IMASNode", "IMASNode", props=["dimension"]), - RelSpec("DEPRECATED_IN", "IMASNode", "DDVersion"), - RelSpec("COORDINATE_SAME_AS", "IMASNode", "IMASNode", props=["dimension"]), - RelSpec("RENAMED_TO", "IMASNode", "IMASNode"), - RelSpec("HAS_IDENTIFIER_SCHEMA", "IMASNode", "IdentifierSchema"), - RelSpec("HAS_PREDECESSOR", "DDVersion", "DDVersion"), - RelSpec("HAS_SUCCESSOR", "DDVersion", "DDVersion"), - RelSpec("HAS_COCOS", "DDVersion", "COCOS"), -] -``` - -Important: `HAS_COORDINATE` has **two target label types** (IMASCoordinateSpec -and IMASNode), which requires two separate relationship CSV files since -`neo4j-admin import` uses `:START_ID(Group)` and `:END_ID(Group)` syntax -that is per-file. - -### Phase 3: Facility Filter Variant - -The per-facility variant exports: -1. All DD nodes and relationships (same as dd-only) -2. All nodes with `facility_id = $facility` for the specified facility -3. Facility-independent nodes: `Facility`, `GraphMeta`, `DiscoveryRoot`, etc. -4. All relationships **between exported nodes** (closed set) - -The relationship closure is key: we cannot blindly export all relationships -from facility nodes, because some may reference nodes in other facilities. -Strategy: - -```python -# Phase A: Export all DD nodes → node_id set -# Phase B: Export facility nodes → extend node_id set -# Phase C: For each relationship type, export only where BOTH endpoints -# are in the node_id set -``` - -For the export query: -```cypher -MATCH (a)-[r:MAPS_TO]->(b) -WHERE a.facility_id = $facility OR a.facility_id IS NULL - AND b.facility_id = $facility OR b.facility_id IS NULL -RETURN ... -``` - -### Phase 4: Integration with Release Workflow - -Replace the call path in `release.py`: - -```python -# Current (in _push_graph_variant): -# temp_neo4j.create_filtered_dump(source_dump, output, filter_type, ...) - -# New: -# export_rebuild.export_rebuild_dd_only(output_path) -# export_rebuild.export_rebuild_facility(facility, output_path) -``` - -The new functions read from the **live graph** and don't need a source dump -parameter. This also eliminates the need to stop production Neo4j at all — -the release workflow can create filtered variants without any downtime. - -### Phase 5: SLURM Integration - -Reuse the existing SLURM dispatch pattern: - -```python -def _should_use_slurm() -> bool: - """Check if running on a SLURM-managed cluster.""" - # Same logic as temp_neo4j._should_use_slurm() - ... - -def export_rebuild_via_slurm(variant, output_path, **kwargs): - """Dispatch export-rebuild to a SLURM compute node.""" - # srun --mem=4G --time=00:30:00 --partition=rigel - # python -m imas_codex.graph.export_rebuild --variant=dd-only --output=... - ... -``` - -Resource request: `--mem=4G --time=00:30:00` (conservative for a 2-min job). - -### Phase 6: Verification and Testing - -1. **Count verification** — after import, start the temp Neo4j and compare - node/relationship counts against the export CSVs: - ```cypher - MATCH (n:IMASNode) RETURN count(n) -- should equal CSV row count - ``` - -2. **Spot-check queries** — run a few known queries (e.g., "find paths in - equilibrium IDS") against the rebuilt database and verify results match - production. - -3. **Integration test** — add a test that: - - Exports a small subset (e.g., 1 IDS worth of nodes) - - Imports into a temp Neo4j - - Verifies counts and a semantic query - -4. **Dump size regression** — assert the DD-only dump is < 100 MB (currently - projected at ~25 MB, vs ~900 MB with the old approach). - ---- - -## CSV File Format - -### Node CSV Headers - -Each label gets one CSV file: `nodes_{Label}.csv` - -```csv -# nodes_IMASNode.csv -id:ID(IMASNode),name:string,ids:string,path_lower:string,description:string,documentation:string,keywords:string[],data_type:string,units:string,url:string,status:string,is_leaf:boolean,node_category:string,dd_version:string,lifecycle_status:string,cocos_label_transformation:string,cocos_replace:string,node_type:string,embedding:float[] -equilibrium/time_slice/profiles_1d/psi,psi,equilibrium,equilibrium/time_slice/profiles_1d/psi,"Poloidal flux","Full docs...",...,true,data,3.41.0,active,psi_like,psi_like,dynamic,-0.123;0.456;... -``` - -For the `embedding:float[]` column, the vector delimiter is `;`: -`-0.12345678;0.98765432;...` (256 values, 8 significant digits each). - -String arrays (e.g., `keywords:string[]`) also use `;` as the array delimiter -(same flag: `--array-delimiter=;`). This works because the column type -annotation (`float[]` vs `string[]`) tells the importer which delimiter -semantics to apply. - -### Relationship CSV Headers - -Each (type, start_label, end_label) triple gets one CSV file: -`rels_{TYPE}_{StartLabel}_{EndLabel}.csv` - -```csv -# rels_HAS_COORDINATE_IMASNode_IMASCoordinateSpec.csv -:START_ID(IMASNode),:END_ID(IMASCoordinateSpec),dimension:int - -# rels_HAS_COORDINATE_IMASNode_IMASNode.csv -:START_ID(IMASNode),:END_ID(IMASNode),dimension:int - -# rels_IN_VERSION_IMASNodeChange_DDVersion.csv (no properties) -:START_ID(IMASNodeChange),:END_ID(DDVersion) -``` - -### GraphMeta Node - -The `GraphMeta` singleton node (`id: "meta"`) must be included in every -variant. It carries `name`, `facilities`, and `updated_at` properties. For -DD-only dumps, the `facilities` list should be set to `[]` (or omitted) -since no facility data is present. - ---- - -## Edge Cases and Gotchas - -### COCOS Integer IDs - -COCOS nodes use integer IDs (1–18) in Neo4j, while the schema defines -`id: string`. During export, stringify them: `str(node["id"])`. The ID group -`ID(COCOS)` prevents collision with other labels' IDs. - -### Empty Labels - -PhysicsDomain, SignConvention (DD-only), CoordinateRelationship, and -ClusterMembership currently have 0 nodes. The export should handle empty -labels gracefully (skip CSV generation, omit from import command). - -### SignConvention (Facility-Dependent) - -SignConvention uses a composite key `(id, facility_id)`. In DD-only dumps, -no SignConvention nodes exist (they're facility-specific). In per-facility -dumps, only SignConvention nodes matching the facility are included. - -### HAS_COORDINATE Dual Targets - -`HAS_COORDINATE` relationships connect to **both** `IMASCoordinateSpec` and -`IMASNode`. Since `neo4j-admin import` requires consistent `:END_ID(Group)` -per file, these must be exported as two separate CSV files (one per target -label). The export query uses explicit label filtering: - -```cypher -MATCH (a:IMASNode)-[r:HAS_COORDINATE]->(b:IMASCoordinateSpec) -RETURN a.id, b.id, r.dimension -``` - -### INTRODUCED_IN / DEPRECATED_IN Multi-Source - -`INTRODUCED_IN` has both `IMASNode` and `IDS` as source labels. Similarly -requires separate CSV files per (source_label, target_label) combination. - -### Connection Resilience - -The Neo4j bolt connection can drop during long exports (observed during -benchmarking). The export code should: -- Use the existing `GraphClient` with retry logic -- Checkpoint progress (last exported ID) to allow resume -- Set a per-batch timeout (30 s) to detect stale connections - -### Embedding Precision - -256-dim embeddings are 32-bit floats. CSV serialization uses `f"{v:.8g}"` -(8 significant digits) which preserves full float32 precision. The -`--vector-delimiter=;` flag tells `neo4j-admin` to parse these correctly. - ---- - -## Migration Path - -### Step 1: Implement and Validate (this plan) - -Build `export_rebuild.py` alongside the existing `temp_neo4j.py`. Both -approaches remain available. - -### Step 2: A/B Comparison - -During the next release, run both pipelines and compare: -- Dump file sizes -- Query results on sample queries -- Node/relationship counts - -### Step 3: Replace Default - -Once validated, update `release.py` to use `export_rebuild` by default. Keep -`temp_neo4j.py` as a fallback for one release cycle. - -### Step 4: Remove Old Code - -After one successful release cycle, remove the filtering code from -`temp_neo4j.py` (the temp Neo4j lifecycle management remains useful for other -purposes). - ---- - -## File Layout - -``` -imas_codex/graph/ -├── export_rebuild.py # NEW: export + rebuild pipeline -├── temp_neo4j.py # EXISTING: retained, eventually simplified -├── neo4j_ops.py # EXISTING: dump/load operations (reused) -├── client.py # EXISTING: GraphClient (used for export queries) -└── schema_context_data.py # EXISTING: index definitions (reference, not imported) -``` - ---- - -## Risks - -| Risk | Likelihood | Impact | Mitigation | -|------|-----------|--------|-----------| -| Neo4j connection drops during export | Medium | Low | Keyset pagination + retry via GraphClient; each batch is independent | -| neo4j-admin import format changes in future Neo4j versions | Low | High | Pin to CSV format; test in CI against the Apptainer image | -| Missing relationship type in DD_RELATIONSHIPS registry | Low | Medium | Verify counts post-import match a control query on production | -| SLURM job preemption during pipeline | Low | Low | Pipeline completes in ~2 min; well within time limits | -| Concurrent DD ingestion during export | Low | Medium | Keyset pagination guarantees consistency within each label; run exports during quiet periods or add advisory lock | From 54940f2d1f6aead0284b5b250adf723f07967a13 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 19:41:26 +0200 Subject: [PATCH 17/22] fix: GHCR tag resolution regex excludes stale test tags The '^[0-9v]' pattern matched 'v-direct-test' (a stale test artifact) and filtered out 'latest' (starts with 'l'). Replace with semver-aware pattern that only matches proper version tags or 'latest'. --- .github/workflows/docker-build-push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index beabc4281..5f58be33b 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -78,8 +78,9 @@ jobs: local REGISTRY="$1" local TAGS TAGS=$(oras repo tags "${REGISTRY}/${IMAS_PACKAGE}" 2>&1) || true + # Match only 'latest' or semver tags (v1.2.3, 5.2.0rc8, etc.) local CLEAN - CLEAN=$(echo "$TAGS" | grep -E '^[0-9v]' || true) + CLEAN=$(echo "$TAGS" | grep -E '^(latest|v?[0-9]+\.[0-9]+\.)' || true) if echo "$CLEAN" | grep -qx "latest"; then echo "latest" elif [ -n "$CLEAN" ]; then From 83a5834797117d06afb792d196dd29fa99efa3e2 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 19:55:56 +0200 Subject: [PATCH 18/22] fix: restore CWD before neo4j-admin in Docker build After 'cd /tmp/graph-pull' and 'rm -rf /tmp/graph-pull', the CWD no longer exists. Java VM cannot initialize: 'getcwd() failed'. Add 'cd /' before neo4j-admin database load to restore a valid CWD. --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 150e7dfd7..1241260b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -179,7 +179,8 @@ RUN set -ex && \ mkdir -p /tmp/dumps && \ mv "$DUMP" /tmp/dumps/neo4j.dump && \ rm -rf /tmp/graph-pull && \ - neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ + cd / && \ + neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination=true 2>&1 && \ rm -rf /tmp/dumps && \ echo "Graph loaded from dump"; \ elif [ -n "$ARCHIVE" ]; then \ @@ -197,7 +198,8 @@ RUN set -ex && \ mkdir -p /tmp/dumps && \ mv "$DUMP_FILE" /tmp/dumps/neo4j.dump && \ rm -rf /tmp/graph-extracted && \ - neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \ + cd / && \ + neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination=true 2>&1 && \ rm -rf /tmp/dumps && \ echo "Graph loaded from dump"; \ else \ From eff2cef92ebcafd9df831c86d04a0ccedfc9c0fe Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 20:15:36 +0200 Subject: [PATCH 19/22] fix: aggressive disk cleanup and single-layer graph load in Docker - Free ~15 GB more on CI runner by removing hostedtoolcache, swift, boost - Merge graph load + Neo4j recovery into single RUN to eliminate ~5 GB intermediate layer that persisted in buildx storage - Root cause: smoke-test failed with 'No space left on device' because 2 GB dump expands to ~7 GB (data + WAL) across multiple layers --- .github/workflows/docker-build-push.yml | 6 ++- Dockerfile | 62 ++++++++++++------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 5f58be33b..bb3644279 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -281,8 +281,10 @@ jobs: run: | echo "Before cleanup:" df -h / - # Remove packages we don't need for docker builds - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + # Aggressively free space — Docker build with Neo4j graph needs ~25 GB + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache /usr/share/swift /usr/local/share/boost \ + /usr/local/graalvm /usr/local/share/chromium /usr/local/lib/node_modules sudo docker image prune --all --force echo "After cleanup:" df -h / diff --git a/Dockerfile b/Dockerfile index 1241260b7..53e2b4f1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,8 +164,9 @@ RUN echo "Graph tag: ${GRAPH_TAG}" > /dev/null # Copy graph archive from builder COPY --from=builder /tmp/graph-pull/ /tmp/graph-pull/ -# Extract and load the graph data (or create empty database) +# Extract, load graph data, and pre-start Neo4j for recovery — all in one layer. # CRITICAL: clean up intermediate files progressively to minimize peak disk usage. +# Merging into a single RUN avoids a multi-GB intermediate layer from the load step. RUN set -ex && \ if [ -f /tmp/graph-pull/.no-graph ]; then \ echo "No graph data — creating empty Neo4j database"; \ @@ -207,39 +208,34 @@ RUN set -ex && \ ls -la /tmp/graph-pull/ >&2; \ exit 1; \ fi; \ - fi - -# Pre-start Neo4j to complete WAL recovery and create system DB. -# This shifts expensive work from runtime (slow Azure I/O) to build time (fast CI SSD). -RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \ - echo "Pre-starting Neo4j for database recovery..." && \ - echo "dbms.security.auth_enabled=false" >> /var/lib/neo4j/conf/neo4j.conf && \ - /var/lib/neo4j/bin/neo4j console > /tmp/neo4j-recovery.log 2>&1 & \ - NEO4J_PID=$! && \ - READY=0 && \ - for i in $(seq 1 120); do \ - if /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "RETURN 1" > /dev/null 2>&1; then \ - echo "Database ready (${i}s)"; \ - READY=1; \ - break; \ - fi; \ - if ! kill -0 $NEO4J_PID 2>/dev/null; then \ - echo "ERROR: Neo4j exited during recovery. Log:"; \ - cat /tmp/neo4j-recovery.log; \ + echo "Pre-starting Neo4j for database recovery..." && \ + echo "dbms.security.auth_enabled=false" >> /var/lib/neo4j/conf/neo4j.conf && \ + /var/lib/neo4j/bin/neo4j console > /tmp/neo4j-recovery.log 2>&1 & \ + NEO4J_PID=$! && \ + READY=0 && \ + for i in $(seq 1 120); do \ + if /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "RETURN 1" > /dev/null 2>&1; then \ + echo "Database ready (${i}s)"; \ + READY=1; \ + break; \ + fi; \ + if ! kill -0 $NEO4J_PID 2>/dev/null; then \ + echo "ERROR: Neo4j exited during recovery. Log:"; \ + cat /tmp/neo4j-recovery.log; \ + exit 1; \ + fi; \ + sleep 1; \ + done && \ + if [ "$READY" -eq 0 ]; then \ + echo "ERROR: Recovery did not complete in 120s. Log:"; \ + tail -50 /tmp/neo4j-recovery.log; \ exit 1; \ - fi; \ - sleep 1; \ - done && \ - if [ "$READY" -eq 0 ]; then \ - echo "ERROR: Recovery did not complete in 120s. Log:"; \ - tail -50 /tmp/neo4j-recovery.log; \ - exit 1; \ - fi && \ - /var/lib/neo4j/bin/neo4j stop && \ - sleep 2 && \ - rm -f /tmp/neo4j-recovery.log && \ - echo "Neo4j shut down cleanly — database is recovery-free"; \ -fi + fi && \ + /var/lib/neo4j/bin/neo4j stop && \ + sleep 2 && \ + rm -f /tmp/neo4j-recovery.log && \ + echo "Neo4j shut down cleanly — database is recovery-free"; \ + fi # NOTE: Do NOT remove transaction logs (/data/transactions/neo4j/*). # Neo4j 2026 requires valid WAL state to open the database after From 0f0abbb162ebae56a627284a056cb9b1bf7d9a43 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 20:36:45 +0200 Subject: [PATCH 20/22] fix: defer Neo4j recovery to container startup to fit CI disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove pre-start recovery from Docker build — the 2.3 GB WAL creation caused 'no space left on device' on CI runners (38 GB free, but buildx layers + 2 GB dump + 5 GB expanded data + 2.3 GB WAL exceeded limit). Recovery now happens at container startup (entrypoint.sh waits 180s). First startup takes ~10-30s longer but the image is ~2.3 GB smaller. --- Dockerfile | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53e2b4f1c..c425d9735 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,9 +164,10 @@ RUN echo "Graph tag: ${GRAPH_TAG}" > /dev/null # Copy graph archive from builder COPY --from=builder /tmp/graph-pull/ /tmp/graph-pull/ -# Extract, load graph data, and pre-start Neo4j for recovery — all in one layer. +# Extract and load graph data into Neo4j data directory. # CRITICAL: clean up intermediate files progressively to minimize peak disk usage. -# Merging into a single RUN avoids a multi-GB intermediate layer from the load step. +# NOTE: Neo4j pre-start recovery is deferred to container startup (entrypoint.sh) +# to avoid ~2.3 GB WAL creation that exceeds CI runner disk during Docker build. RUN set -ex && \ if [ -f /tmp/graph-pull/.no-graph ]; then \ echo "No graph data — creating empty Neo4j database"; \ @@ -208,40 +209,11 @@ RUN set -ex && \ ls -la /tmp/graph-pull/ >&2; \ exit 1; \ fi; \ - echo "Pre-starting Neo4j for database recovery..." && \ - echo "dbms.security.auth_enabled=false" >> /var/lib/neo4j/conf/neo4j.conf && \ - /var/lib/neo4j/bin/neo4j console > /tmp/neo4j-recovery.log 2>&1 & \ - NEO4J_PID=$! && \ - READY=0 && \ - for i in $(seq 1 120); do \ - if /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "RETURN 1" > /dev/null 2>&1; then \ - echo "Database ready (${i}s)"; \ - READY=1; \ - break; \ - fi; \ - if ! kill -0 $NEO4J_PID 2>/dev/null; then \ - echo "ERROR: Neo4j exited during recovery. Log:"; \ - cat /tmp/neo4j-recovery.log; \ - exit 1; \ - fi; \ - sleep 1; \ - done && \ - if [ "$READY" -eq 0 ]; then \ - echo "ERROR: Recovery did not complete in 120s. Log:"; \ - tail -50 /tmp/neo4j-recovery.log; \ - exit 1; \ - fi && \ - /var/lib/neo4j/bin/neo4j stop && \ - sleep 2 && \ - rm -f /tmp/neo4j-recovery.log && \ - echo "Neo4j shut down cleanly — database is recovery-free"; \ fi -# NOTE: Do NOT remove transaction logs (/data/transactions/neo4j/*). -# Neo4j 2026 requires valid WAL state to open the database after -# neo4j-admin load. Deleting tx logs causes Neo4j HTTP to start but -# the bolt database remains offline — all Cypher queries fail silently. -# The ~2.3 GB cost is acceptable for a working container. +# NOTE: Neo4j recovery happens at container startup (entrypoint.sh waits up to +# 180s for database readiness). This adds ~10-30s to first startup but avoids +# the ~2.3 GB WAL creation that would exceed CI runner disk during Docker build. ## Stage 5: Final runtime image (assemble from builder + Neo4j + graph data) FROM python:3.12-slim From 61cadeb348ebddbf9321015c3df286872d9db3d0 Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Thu, 9 Apr 2026 23:12:09 +0200 Subject: [PATCH 21/22] fix: version history in fetch_dd_paths and search_dd_paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed Cypher queries using non-existent IMASNodeChange properties: - change.semantic_change_type → change.change_type - change.version → v.id (via IN_VERSION relationship) - change.summary → old_value/new_value pair Updated formatters to render version changes correctly. Fixed _get_version_context to filter null OPTIONAL MATCH results. --- imas_codex/llm/search_formatters.py | 40 ++++++++++++++++++++++++++--- imas_codex/tools/graph_search.py | 31 ++++++++++++++-------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/imas_codex/llm/search_formatters.py b/imas_codex/llm/search_formatters.py index 6128a2080..b122b2c44 100644 --- a/imas_codex/llm/search_formatters.py +++ b/imas_codex/llm/search_formatters.py @@ -640,8 +640,16 @@ def format_imas_report( if isinstance(ch, dict): ver = ch.get("version", "?") ctype = ch.get("type", "") - summary = ch.get("summary", "") - parts.append(f" DD {ver} [{ctype}]: {summary}") + old_v = ch.get("old_value", "") + new_v = ch.get("new_value", "") + if old_v and new_v: + parts.append( + f" DD {ver} [{ctype}]: `{old_v}` → `{new_v}`" + ) + elif new_v: + parts.append(f" DD {ver} [{ctype}]: {new_v}") + else: + parts.append(f" DD {ver} [{ctype}]") parts.append("") @@ -865,6 +873,22 @@ def format_fetch_paths_report(result: Any) -> str: labels = _stringify_cluster_labels(getattr(node, "cluster_labels", None)) if labels: parts.append(f" Clusters: {', '.join(f'"{c}"' for c in labels)}") + + # Render version changes if present + if hasattr(node, "version_changes") and node.version_changes: + parts.append(" Version changes:") + for vc in node.version_changes: + ver = vc.get("version", "?") + ctype = vc.get("type", "unknown") + old_v = vc.get("old_value", "") + new_v = vc.get("new_value", "") + if old_v and new_v: + parts.append(f" - {ver} [{ctype}]: `{old_v}` → `{new_v}`") + elif new_v: + parts.append(f" - {ver} [{ctype}]: {new_v}") + else: + parts.append(f" - {ver} [{ctype}]") + parts.append("") for nf in _get_value(result, "not_found_paths", []) or []: @@ -1164,8 +1188,16 @@ def format_search_dd_report(result: Any, cluster_result: Any | None = None) -> s if isinstance(ch, dict): ver = ch.get("version", "?") ctype = ch.get("type", "") - summary = ch.get("summary", "") - parts.append(f" DD {ver} [{ctype}]: {summary}") + old_v = ch.get("old_value", "") + new_v = ch.get("new_value", "") + if old_v and new_v: + parts.append( + f" DD {ver} [{ctype}]: `{old_v}` → `{new_v}`" + ) + elif new_v: + parts.append(f" DD {ver} [{ctype}]: {new_v}") + else: + parts.append(f" DD {ver} [{ctype}]") parts.append("") diff --git a/imas_codex/tools/graph_search.py b/imas_codex/tools/graph_search.py index 4a136a48f..de6d6cec3 100644 --- a/imas_codex/tools/graph_search.py +++ b/imas_codex/tools/graph_search.py @@ -710,12 +710,12 @@ async def fetch_dd_paths( if include_version_history: version_clause = """ OPTIONAL MATCH (change:IMASNodeChange)-[:FOR_IMAS_PATH]->(p) - WHERE change.semantic_change_type IN - ['sign_convention', 'coordinate_convention', 'units', 'definition_clarification'] + OPTIONAL MATCH (change)-[:IN_VERSION]->(cv:DDVersion) WITH p, u, cluster_labels, coordinates, ident, iv, - collect(DISTINCT {version: change.version, - type: change.semantic_change_type, - summary: change.summary}) AS version_changes + collect(DISTINCT {version: cv.id, + type: change.change_type, + old_value: change.old_value, + new_value: change.new_value}) AS version_changes """ else: version_clause = """ @@ -2755,16 +2755,25 @@ def _get_version_context( UNWIND $path_ids AS pid MATCH (p:IMASNode {id: pid}) OPTIONAL MATCH (change:IMASNodeChange)-[:FOR_IMAS_PATH]->(p) - WHERE change.semantic_change_type IN - ['sign_convention', 'coordinate_convention', 'units', 'definition_clarification'] + OPTIONAL MATCH (change)-[:IN_VERSION]->(v:DDVersion) RETURN p.id AS id, count(change) AS change_count, - collect({version: change.version, - type: change.semantic_change_type, - summary: change.summary})[..5] AS notable_changes + collect({version: v.id, + type: change.change_type, + old_value: change.old_value, + new_value: change.new_value})[..5] AS notable_changes """ results = gc.query(cypher, path_ids=path_ids) - return {r["id"]: r for r in results} + out = {} + for r in results: + # Filter null entries from OPTIONAL MATCH no-match + raw = r.get("notable_changes") or [] + filtered = [c for c in raw if c.get("version") is not None] + out[r["id"]] = { + "change_count": len(filtered), + "notable_changes": filtered, + } + return out def _common_path_prefix(paths: list[str]) -> str: From 057cf9292132f73fd0487dbe33f8ef148e18de8c Mon Sep 17 00:00:00 2001 From: Simon McIntosh Date: Fri, 10 Apr 2026 00:22:10 +0200 Subject: [PATCH 22/22] fix: remove non-existent summary property from version_tool bulk query --- imas_codex/tools/version_tool.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imas_codex/tools/version_tool.py b/imas_codex/tools/version_tool.py index 8c52ae903..90b26ac51 100644 --- a/imas_codex/tools/version_tool.py +++ b/imas_codex/tools/version_tool.py @@ -343,8 +343,7 @@ async def _bulk_query( RETURN p.id AS path, p.ids AS ids, c.old_value AS old_value, c.new_value AS new_value, v.id AS version, c.change_type AS change_type, - coalesce(c.breaking_level, 'informational') AS severity, - c.summary AS summary + coalesce(c.breaking_level, 'informational') AS severity ORDER BY v.id, p.ids, p.id LIMIT 200 """