feat(xsd): parser scaffolding (Phase 3b)

parseSchemaSet({ schemaDir, entrypoints }) loads a working set of XSDs,
follows xsd:import schemaLocation references recursively, and indexes
every top-level declaration (element/complexType/simpleType/group/
attributeGroup/attribute) by canonical Clark-style qname.

fast-xml-parser configured with preserveOrder so sibling order across
different tag names is retained, and no value coercion that would
mutate XSD attribute strings.

Returns a typed schema set:
  - documents: per-file metadata + raw schemaNode
  - namespaceByPrefix: per-document prefix -> URI maps
  - importGraph: per-document outgoing imports with resolved targets
  - declarationsByQName: canonical qname -> declarations[]

QName resolution is conservative: declaration qnames use the document's
target namespace; attribute qnames (ref/type/base) resolve through the
document's prefix map and surface as { resolved: false } when the prefix
or namespace is unknown rather than guessing.

No DB writes in this phase. Smoke command bun run xsd:smoke parses
wml.xsd from the cache and reports counts (820 complexTypes,
389 simpleTypes, 67 groups, 47 elements, etc).

Also tightens DB test isolation: an afterAll TRUNCATE leaves the dev
DB clean instead of carrying the last test's xsd_profiles row.

Author: caiopizzol

bun.lock

@@ -22,6 +22,7 @@
     "db:migrate": "bun scripts/db-migrate.ts",
     "db:sync-sources": "bun scripts/sync-sources.ts",
     "xsd:fetch": "bun scripts/fetch-xsd.ts",
+    "xsd:smoke": "bun scripts/ingest-xsd/smoke.ts",
     "test": "bun test tests/",
     "ingest": "bun scripts/ingest/pipeline.ts",
     "ingest:chunk": "bun scripts/ingest/chunk.ts",
@@ -35,6 +36,7 @@
     "@semantic-release/exec": "^7.1.0",
     "@semantic-release/git": "^10.0.1",
     "@semantic-release/github": "^12.0.2",
+    "fast-xml-parser": "^5.7.2",
     "lefthook": "^2.0.16",
     "semantic-release": "^25.0.2",
     "typescript": "~5.9.3"

package.json

@@ -0,0 +1,73 @@
+/**
+ * Helpers for navigating the preserveOrder AST emitted by fast-xml-parser.
+ *
+ * AST shape: every element is a single-key object { tagName: children[], ":@"?: { "@_attrName": value } }.
+ * Text nodes are { "#text": string }. Children always live in an array, so sibling
+ * order is preserved across different tag names.
+ */
+
+import type { PreserveOrderDocument, PreserveOrderNode } from "./types.ts";
+
+/** Strip an XML namespace prefix from a tag name: "xsd:element" → "element". */
+export function stripPrefix(tag: string): string {
+	const colon = tag.indexOf(":");
+	return colon < 0 ? tag : tag.slice(colon + 1);
+}
+
+/** Return the single tag name on a preserveOrder node, or null for non-element nodes. */
+export function nodeTag(node: PreserveOrderNode): string | null {
+	for (const k of Object.keys(node)) {
+		if (k !== ":@") return k;
+	}
+	return null;
+}
+
+/** Return the children array of a preserveOrder element. */
+export function nodeChildren(node: PreserveOrderNode): PreserveOrderNode[] {
+	const tag = nodeTag(node);
+	if (!tag) return [];
+	const v = node[tag];
+	return Array.isArray(v) ? (v as PreserveOrderNode[]) : [];
+}
+
+/** Return attributes on a preserveOrder element. fast-xml-parser nests them under ":@" with "@_" prefix. */
+export function nodeAttrs(node: PreserveOrderNode): Record<string, string> {
+	const raw = node[":@"];
+	if (!raw || typeof raw !== "object") return {};
+	const out: Record<string, string> = {};
+	for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
+		const name = k.startsWith("@_") ? k.slice(2) : k;
+		if (typeof v === "string") out[name] = v;
+		else if (v != null) out[name] = String(v);
+	}
+	return out;
+}
+
+/**
+ * Find the first element in `doc` (or under `parent`) whose stripped tag name
+ * matches one of the given local names. Used to locate the xsd:schema root
+ * regardless of whether the file uses `xsd:`, `xs:`, or no prefix.
+ */
+export function findFirstByLocalName(
+	nodes: PreserveOrderDocument | PreserveOrderNode[],
+	localNames: string[],
+): PreserveOrderNode | null {
+	for (const node of nodes) {
+		const tag = nodeTag(node);
+		if (tag && localNames.includes(stripPrefix(tag))) return node;
+	}
+	return null;
+}
+
+/**
+ * Iterate immediate children of an element whose stripped tag name matches `localName`.
+ */
+export function* eachChildByLocalName(
+	parent: PreserveOrderNode,
+	localName: string,
+): Generator<PreserveOrderNode> {
+	for (const child of nodeChildren(parent)) {
+		const tag = nodeTag(child);
+		if (tag && stripPrefix(tag) === localName) yield child;
+	}
+}

scripts/ingest-xsd/ast.ts

@@ -0,0 +1,187 @@
+/**
+ * Parse a working set of XSDs into an in-memory schema set.
+ *
+ * Walks xsd:import schemaLocation references recursively starting from
+ * `entrypoints`, and indexes every top-level declaration by canonical qname.
+ *
+ * No DB writes here. Subsequent phases (3c+) walk documents/declarations to
+ * produce xsd_symbols, edges, etc.
+ */
+
+import { readFile } from "node:fs/promises";
+import { isAbsolute, normalize, relative, resolve, sep } from "node:path";
+import { XMLParser } from "fast-xml-parser";
+import { eachChildByLocalName, findFirstByLocalName, nodeAttrs, stripPrefix } from "./ast.ts";
+import { declarationQNameKey } from "./qname.ts";
+import type {
+	Declaration,
+	DeclarationKind,
+	ImportEdge,
+	ParsedSchemaDocument,
+	ParsedSchemaSet,
+	PreserveOrderDocument,
+	PreserveOrderNode,
+} from "./types.ts";
+import { vocabularyForNamespace } from "./vocabulary.ts";
+
+const xmlParser = new XMLParser({
+	preserveOrder: true,
+	ignoreAttributes: false,
+	attributeNamePrefix: "@_",
+	parseAttributeValue: false,
+	parseTagValue: false,
+	trimValues: true,
+});
+
+const TOP_LEVEL_KINDS: Record<string, DeclarationKind> = {
+	element: "element",
+	complexType: "complexType",
+	simpleType: "simpleType",
+	group: "group",
+	attributeGroup: "attributeGroup",
+	attribute: "attribute",
+};
+
+export interface ParseSchemaSetOptions {
+	schemaDir: string;
+	entrypoints: string[];
+}
+
+export async function parseSchemaSet(opts: ParseSchemaSetOptions): Promise<ParsedSchemaSet> {
+	const schemaDir = isAbsolute(opts.schemaDir) ? opts.schemaDir : resolve(opts.schemaDir);
+
+	const documents = new Map<string, ParsedSchemaDocument>();
+	const namespaceByPrefix = new Map<string, Map<string, string>>();
+	const importGraph = new Map<string, ImportEdge[]>();
+	const declarationsByQName = new Map<string, Declaration[]>();
+
+	const queue: string[] = opts.entrypoints.map((p) => relPath(schemaDir, resolve(schemaDir, p)));
+
+	while (queue.length) {
+		const relPathInDir = queue.shift()!;
+		if (documents.has(relPathInDir)) continue;
+
+		const absolutePath = resolve(schemaDir, relPathInDir);
+		const text = await readFile(absolutePath, "utf-8");
+		const ast = xmlParser.parse(text) as PreserveOrderDocument;
+
+		const schemaNode = findFirstByLocalName(ast, ["schema"]);
+		if (!schemaNode) {
+			throw new Error(`No xsd:schema root in ${absolutePath}`);
+		}
+
+		const attrs = nodeAttrs(schemaNode);
+		const targetNamespace = attrs.targetNamespace;
+		if (!targetNamespace) {
+			throw new Error(`Schema in ${absolutePath} is missing targetNamespace`);
+		}
+
+		const prefixes = extractNamespacePrefixes(attrs);
+		const imports = extractImports(schemaNode, schemaDir, absolutePath);
+
+		const doc: ParsedSchemaDocument = {
+			path: relPathInDir,
+			absolutePath,
+			targetNamespace,
+			vocabularyId: vocabularyForNamespace(targetNamespace),
+			schemaNode,
+		};
+
+		documents.set(relPathInDir, doc);
+		namespaceByPrefix.set(relPathInDir, prefixes);
+		importGraph.set(relPathInDir, imports);
+
+		indexTopLevelDeclarations(doc, declarationsByQName);
+
+		for (const edge of imports) {
+			if (edge.target && !documents.has(edge.target)) {
+				queue.push(edge.target);
+			}
+		}
+	}
+
+	return { documents, namespaceByPrefix, importGraph, declarationsByQName };
+}
+
+function extractNamespacePrefixes(attrs: Record<string, string>): Map<string, string> {
+	const map = new Map<string, string>();
+	for (const [name, value] of Object.entries(attrs)) {
+		if (name === "xmlns") map.set("", value);
+		else if (name.startsWith("xmlns:")) map.set(name.slice("xmlns:".length), value);
+	}
+	return map;
+}
+
+function extractImports(
+	schemaNode: PreserveOrderNode,
+	schemaDir: string,
+	currentAbsPath: string,
+): ImportEdge[] {
+	const imports: ImportEdge[] = [];
+	for (const importNode of eachChildByLocalName(schemaNode, "import")) {
+		const a = nodeAttrs(importNode);
+		const schemaLocation = a.schemaLocation ?? null;
+		let target: string | null = null;
+		if (schemaLocation) {
+			const importedAbs = resolve(currentAbsPath, "..", schemaLocation);
+			target = relPath(schemaDir, importedAbs);
+		}
+		imports.push({
+			namespace: a.namespace ?? "",
+			schemaLocation,
+			target,
+		});
+	}
+	return imports;
+}
+
+function indexTopLevelDeclarations(
+	doc: ParsedSchemaDocument,
+	declarationsByQName: Map<string, Declaration[]>,
+): void {
+	for (const child of nodeChildrenLocal(doc.schemaNode)) {
+		const tag = nodeTagLocal(child);
+		if (!tag) continue;
+		const local = stripPrefix(tag);
+		const kind = TOP_LEVEL_KINDS[local];
+		if (!kind) continue;
+
+		const a = nodeAttrs(child);
+		const localName = a.name;
+		if (!localName) continue;
+
+		const decl: Declaration = {
+			kind,
+			namespace: doc.targetNamespace,
+			vocabularyId: doc.vocabularyId,
+			localName,
+			documentPath: doc.path,
+			node: child,
+		};
+		const key = declarationQNameKey(doc.targetNamespace, kind, localName);
+		const arr = declarationsByQName.get(key);
+		if (arr) arr.push(decl);
+		else declarationsByQName.set(key, [decl]);
+	}
+}
+
+// Local helpers (avoid pulling extra exports from ast.ts).
+function nodeTagLocal(node: PreserveOrderNode): string | null {
+	for (const k of Object.keys(node)) if (k !== ":@") return k;
+	return null;
+}
+function nodeChildrenLocal(node: PreserveOrderNode): PreserveOrderNode[] {
+	const tag = nodeTagLocal(node);
+	if (!tag) return [];
+	const v = node[tag];
+	return Array.isArray(v) ? (v as PreserveOrderNode[]) : [];
+}
+
+function relPath(base: string, abs: string): string {
+	const r = relative(base, normalize(abs));
+	// Guard against escapes outside schemaDir.
+	if (r.startsWith(`..${sep}`) || r === "..") {
+		throw new Error(`Resolved path escapes schemaDir: ${abs} (base ${base})`);
+	}
+	return r;
+}