Update Python dependencies

.gitignore

@@ -93,3 +93,5 @@ databases/metadata_inventory.md
 bin/issundb-mcp
 bin/issundb-rest
 bin/issundb-cli
+*.cache
+databases/hf-dataset/

AGENTS.md

@@ -5,14 +5,15 @@ This file provides guidance to coding agents collaborating on this repository.
 ## Mission
 
 This project builds graph databases from the [Meta Kaggle](https://www.kaggle.com/datasets/kaggle/meta-kaggle) dataset and loads them
-into [IssunDB](https://github.com/habedi/issun-db), an embedded graph database.
-The pipeline stages source files with DuckDB, parses code imports with Polars, parses Python API calls with Tree-sitter, and bulk-loads nodes and edges through the IssunDB CLI.
-The project priorities include correct graph construction, reproducible builds, scoped subset testing, and idiomatic Python.
+into [IssunDB](https://github.com/habedi/issun-db), an embedded graph database. The pipeline stages source files with DuckDB, parses code imports with
+Polars, parses Python API calls with Tree-sitter, and bulk-loads nodes and edges through the IssunDB CLI. The project priorities include correct graph
+construction, reproducible builds, scoped subset testing, and idiomatic Python.
 
 ## Core Rules
 
 - Use English for code, comments, documentation, and tests.
-- Stage with DuckDB, parse code imports with Polars, parse Python API calls with Tree-sitter, and load through the IssunDB command line interface. Do not mix these roles.
+- Stage with DuckDB, parse code imports with Polars, parse Python API calls with Tree-sitter, and load through the IssunDB command line interface. Do
+  not mix these roles.
 - Bulk-load nodes and edges with command line tools instead of Cypher queries.
 - Keep the large local source data out of the repository.
 - Ensure staging is deterministic by using a fixed seed rule and fixed source data.
@@ -30,7 +31,7 @@ The project priorities include correct graph construction, reproducible builds,
 - Avoid em dashes by using semicolons or restructuring sentences.
 - Avoid colorful adjectives and adverbs.
 - Balance the use of noun phrases for checklist items and imperative verbs.
-- Apply title case to headings in Markdown files.
+- Apply the title case to headings in Markdown files.
 - Use correct and complete sentences.
 - Avoid made-up words, abbreviations, and colons in the middle of sentences.
 
@@ -49,6 +50,7 @@ The project priorities include correct graph construction, reproducible builds,
 - `scripts/load_competition_kg.py` loads the Kaggle knowledge graph.
 - `scripts/import_to_issundb.py` loads the kernel graph.
 - `scripts/issundb_load.py` holds the loader logic shared by both load scripts.
+- `scripts/package_hf_dataset.py` packages the staged competition graph as a Hugging Face dataset with a card and manifest.
 - `databases/` includes staged files and graph databases.
 - `bin/issundb-cli` includes the database command line tool binary.
 - `bin/issundb-mcp` includes the MCP server binary.
@@ -64,19 +66,23 @@ The project priorities include correct graph construction, reproducible builds,
 - `make graph-kernel` builds the kernel knowledge graph end to end.
 - `make kernel-cli` opens the kernel knowledge graph in the IssunDB CLI.
 - `make kernel-mcp` runs the IssunDB MCP server for the kernel knowledge graph.
+- `make hf-package HF_SNAPSHOT=YYYY-MM-DD` packages the staged competition graph Parquet files, a dataset card, and a manifest into `databases/hf-dataset`.
+- `make hf-upload` pushes the packaged dataset to the Hugging Face Hub and tags the release.
 - `make help` lists all targets.
 
 ## Pipeline Constraints
 
 - Node files are `Id`-first. The `Id` column is auto-indexed. Edge files contain source and destination `Id` keys.
 - Unresolved edge endpoints are dropped. Malformed rows cause errors.
+- Staging only seeds users and organizations that have a row in `Users.csv` or `Organizations.csv`, and it filters every user and organization edge through those seed tables, so no staged edge points to a missing node.
 - Bulk loading must use `:import-edges` instead of Cypher `UNWIND ... CREATE` queries.
 - Node property lookups use auto-indexed full-text indexes. Raw markup language bodies must be searched with `CONTAINS` scans.
 - Cypher query lines take string literals verbatim.
 
 ## Workflow
 
-The workflow includes stage identification before coding, schema checks in `databases/metadata_inventory.md`, scoped pipeline changes with tunable `make`
+The workflow includes stage identification before coding, schema checks in `databases/metadata_inventory.md`, scoped pipeline changes with tunable
+`make`
 variables, dry run staging on subsets, code formatting, and repository documentation updates when behavior changes.
 
 ## Testing Expectations

Makefile

@@ -14,11 +14,17 @@ COMP_DB ?= databases/comp-kg
 KERNELS_PER_COMPETITION ?= 50
 ISSUNDB_CLI ?= bin/issundb-cli
 ISSUNDB_MCP ?= bin/issundb-mcp
-MAP_SIZE_GB ?= 8
+MAP_SIZE_GB ?= 12
 
 # Kernel knowledge graph build settings
 KERNEL_DB ?= databases/kernel-kg
 
+# Hugging Face dataset release settings
+HF_OUTPUT ?= databases/hf-dataset
+HF_REPO_ID ?= habedi/kaggle-knowledge-graph
+HF_SNAPSHOT ?=
+HF_VERSION ?= $(HF_SNAPSHOT)
+
 # Directories and files to clean
 CACHE_DIRS  = .mypy_cache .pytest_cache .ruff_cache
 COVERAGE    = .coverage htmlcov coverage.xml
@@ -162,6 +168,17 @@ comp-cli: ## Open the Kaggle knowledge graph in the IssunDB CLI
 comp-mcp: ## Run the IssunDB MCP server for the Kaggle knowledge graph
 	$(ISSUNDB_MCP) --db-path $(COMP_DB) --map-size-gb $(MAP_SIZE_GB)
 
+.PHONY: hf-package
+hf-package: ## Package the staged competition graph as a Hugging Face dataset (needs HF_SNAPSHOT=YYYY-MM-DD)
+	@test -n "$(HF_SNAPSHOT)" || (echo "set HF_SNAPSHOT to the Meta Kaggle export date, e.g. make hf-package HF_SNAPSHOT=2026-08-01" && exit 1)
+	.venv/bin/python scripts/package_hf_dataset.py --stage-dir "$(COMP_STAGE_DIR)" --output "$(HF_OUTPUT)" --repo-id "$(HF_REPO_ID)" --snapshot "$(HF_SNAPSHOT)" --version "$(HF_VERSION)"
+
+.PHONY: hf-upload
+hf-upload: ## Upload the packaged dataset to the Hugging Face Hub (needs `hf auth login` and HF_VERSION)
+	@test -n "$(HF_VERSION)" || (echo "set HF_VERSION (or HF_SNAPSHOT) to tag the release" && exit 1)
+	hf upload "$(HF_REPO_ID)" "$(HF_OUTPUT)" . --repo-type dataset --commit-message "Release $(HF_VERSION)"
+	hf repo tag create "$(HF_REPO_ID)" "$(HF_VERSION)" --repo-type dataset
+
 .PHONY: neo4j-up
 neo4j-up: ## Start Neo4j with stage/ mounted as /import
 	docker compose -f $(NEO4J_COMPOSE) up -d

README.md

@@ -1,4 +1,4 @@
-## Kaggle Knowledge Graph
+# Kaggle Knowledge Graph
 
 [![Tests](https://img.shields.io/github/actions/workflow/status/IssunDB/kaggle-knowledge-graph/tests.yml?label=tests&style=flat&labelColor=333333&logo=github&logoColor=white)](https://github.com/IssunDB/kaggle-knowledge-graph/actions/workflows/tests.yml)
 [![Code Coverage](https://img.shields.io/codecov/c/github/IssunDB/kaggle-knowledge-graph?style=flat&label=coverage&labelColor=333333&logo=codecov&logoColor=white)](https://codecov.io/gh/IssunDB/kaggle-knowledge-graph)
@@ -8,38 +8,54 @@
 ---
 
 This repository contains code for building a knowledge graph from the [Meta Kaggle](https://www.kaggle.com/datasets/kaggle/meta-kaggle) dataset
-(and loading it into [IssunDB](https://github.com/IssunDB/issun-db) which provides CLI and MCP interfaces for querying the data).
+and loading it into [IssunDB](https://github.com/IssunDB/issun-db), which provides CLI and MCP interfaces for querying the data.
 
-### Quickstart
+## Quickstart
 
-#### Data
+### Data
 
-##### 1. Download Dataset
+#### 1. Download Dataset
 
 ```bash
-curl -L -o /path/to/meta-kaggle.zip\
-https://www.kaggle.com/api/v1/datasets/download/kaggle/meta-kaggle
+curl -L -o /path/to/meta-kaggle.zip \
+  https://www.kaggle.com/api/v1/datasets/download/kaggle/meta-kaggle
 ```
 
-##### 2. Configure Environment Variables
+#### 2. Extract Dataset
+
+```bash
+unzip /path/to/meta-kaggle.zip -d /path/to/meta-kaggle
+```
+
+#### 3. Configure Environment Variables
 
 ```bash
 export META_KAGGLE_DIR="/path/to/meta-kaggle"
+export META_KAGGLE_CODE_DIR="/path/to/meta-kaggle-code"
 ```
 
-#### Build and Launch
+### Build and Launch
 
 Build the Kaggle knowledge graph and launch the CLI or MCP server:
 
-- `make graph-kc` builds the competition-centered knowledge graph from the Meta Kaggle dataset (needs `META_KAGGLE_CODE_DIR` for import and API call parsing).
+- `make graph-kc` builds the competition-centered knowledge graph from the Meta Kaggle dataset (needs `META_KAGGLE_CODE_DIR` for import and API call
+  parsing).
 - `make comp-cli` opens the competition knowledge graph in the IssunDB CLI.
 - `make comp-mcp` runs the IssunDB MCP server for the competition knowledge graph.
-- `make graph-kernel` builds the kernel-centered knowledge graph from the Meta Kaggle dataset (needs `META_KAGGLE_CODE_DIR` for import and API call parsing).
+- `make graph-kernel` builds the kernel-centered knowledge graph from the Meta Kaggle dataset (needs `META_KAGGLE_CODE_DIR` for import and API call
+  parsing).
 - `make kernel-cli` opens the kernel knowledge graph in the IssunDB CLI.
 - `make kernel-mcp` runs the IssunDB MCP server for the kernel knowledge graph.
 - `make help` shows all available Makefile targets.
 
-#### MCP Server Configuration
+### Publish the Graph as a Dataset
+
+- `make hf-package HF_SNAPSHOT=YYYY-MM-DD` packages the staged competition graph as a Hugging Face dataset in `databases/hf-dataset`, with the Parquet
+  files under `data/`, a dataset card, and a manifest of row counts and checksums.
+- `make hf-upload HF_REPO_ID=<user>/<dataset> HF_VERSION=<version>` uploads that directory to the Hugging Face Hub and tags it with the release
+  version (log in first with `hf auth login`).
+
+### MCP Server Configuration
 
 To connect AI agents to the IssunDB MCP server, use the configuration template at [examples/mcp_config.json](examples/mcp_config.json):
 
@@ -61,7 +77,7 @@ To connect AI agents to the IssunDB MCP server, use the configuration template a
 
 Replace `/path/to/kaggle-knowledge-graph` with the absolute path to your repository root directory.
 
-#### Knowledge Graph Schema
+### Knowledge Graph Schema
 
 <div align="center">
   <picture>
@@ -71,11 +87,11 @@ Replace `/path/to/kaggle-knowledge-graph` with the absolute path to your reposit
 
 ---
 
-### Contributing
+## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to make a contribution.
 
-### License
+## License
 
-This project is licensed under the [MIT License](LICENSE) except the knowledge graph data (built by the code in this repository), which
+This project is licensed under the [MIT License](LICENSE) except for the knowledge graph data (built by the code in this repository), which
 is subject to the [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) license.

pyproject.toml

@@ -5,6 +5,7 @@ description = "The Python environment for the `kaggle-knowledge-graph` project"
 requires-python = ">=3.10,<4.0"
 dependencies = [
     "duckdb>=1.5.4",
+    "huggingface-hub>=1.30.0",
     "kaggle>=1.7.4.5",
     "polars>=1.42.0",
     "python-dotenv>=1.1.0",
@@ -68,7 +69,7 @@ check_untyped_defs = true
 warn_return_any = true
 strict_optional = true
 warn_redundant_casts = true
-exclude = "^(benches/|examples/|tests/)"
+exclude = "^(benches/|examples/|tests/|tmp/)"
 
 [tool.ruff]
 exclude = [

scripts/inspect_metadata.py

@@ -10,13 +10,13 @@
 
 DEFAULT_META_DIR = Path(
     os.environ.get("META_KAGGLE_DIR", str(Path.home() / "downloads" / "KW" / "meta-kaggle"))
-)
+).expanduser()
 DEFAULT_OUTPUT = Path("databases/metadata_inventory.md")
 
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--meta-dir", type=Path, default=DEFAULT_META_DIR)
+    parser.add_argument("--meta-dir", type=lambda v: Path(v).expanduser(), default=DEFAULT_META_DIR)
     parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
     return parser.parse_args()
 

scripts/issundb_load.py

@@ -72,6 +72,7 @@ def build_script(
         lines.append(f"CREATE INDEX FOR (n:{label}) ON (n.{prop})")
 
     lines.append("rebuild-csr")
+    lines.append("materialize-columns")
     lines.append("stats")
     lines.append("quit")
     return "\n".join(lines) + "\n", expected_nodes

scripts/load_competition_kg.py

@@ -1,7 +1,7 @@
 """Load the staged competition subset into an IssunDB database.
 
 This driver generates an IssunDB CLI script that bulk-imports the staged node and
-edge CSVs, declares uniqueness constraints and full-text indexes, rebuilds the CSR
+edge Parquet files, declares uniqueness constraints and full-text indexes, rebuilds the CSR
 snapshot, and prints statistics. It then runs the CLI, captures the log, and
 validates the load: every node file must import in full, no edge row may be
 malformed, and no command may error (a failed uniqueness constraint means the
@@ -23,7 +23,7 @@
 
 from issundb_load import load_and_validate
 
-# Node CSV file names and the label each file's rows carry. Loaded before edges
+# Node Parquet file names and the label each file's rows carry. Loaded before edges
 # so the edge importer can resolve endpoints by the auto-indexed `Id` property.
 NODE_FILES: list[tuple[str, str]] = [
     ("nodes_competition.parquet", "Competition"),
@@ -43,7 +43,7 @@
     ("nodes_organization.parquet", "Organization"),
 ]
 
-# Edge CSV file names with their source label, destination label, and type.
+# Edge Parquet file names with their source label, destination label, and type.
 EDGE_FILES: list[tuple[str, str, str, str]] = [
     ("edges_team_competed_in_competition.parquet", "Team", "Competition", "COMPETED_IN"),
     ("edges_user_member_of_team.parquet", "User", "Team", "MEMBER_OF_TEAM"),