fix(xsd): preserve element/attr type and group-ref compositor metadata

Three correctness gaps surfaced before Phase 4:

P1 - Local elements lost type and profile membership.
  WML uses <xsd:element name="p" type="CT_P"/> inside groups; before this
  change the local element symbol carried no @type and was never linked
  to xsd_symbol_profiles, so ooxml_lookup_element/ooxml_children would
  not find it in the transitional profile or follow it to CT_P.

P2 - Group refs in nested compositors lost context.
  <xsd:group ref> inside a nested sequence/choice was inserted with
  parent_symbol_id and order_index only. The compositor it lives inside
  and the ref's own minOccurs/maxOccurs were dropped, so later expansion
  could not preserve ordering or cardinality relative to siblings.

P2 - Referenced attributes lost type/default/fixed.
  <xsd:attribute ref="r:id"/> set attr_symbol_id only; the type and
  default declared on the top-level <xsd:attribute name="id"
  type="ST_RelationshipId"/> were not recovered into the edge.

Migration 0003_phase3_metadata adds:
  - xsd_symbols.type_ref TEXT (Clark-style {namespace}localName for
    elements and attributes that declare @type; NULL for the rest).
  - xsd_group_edges.compositor_id INT (FK with ON DELETE CASCADE),
    plus min_occurs / max_occurs.

ingest.ts:
  - upsertSymbol now accepts typeRef; ON CONFLICT preserves the existing
    value via COALESCE so a re-run never blanks it out.
  - Pass 1 captures @type for top-level element/attribute decls.
  - Pass 3 captures @type and links local elements to xsd_symbol_profiles.
  - Pass 3 group refs thread compositor_id and parse min/max occurs.
  - Pass 4 attribute refs copy type_ref / default / fixed from the
    top-level declaration; attr_use stays from the ref site (XSD lets
    refs override use only).

Real WML ingest after fix:
  - profile memberships: 1345 -> 2723 (1345 top-level + 1378 local
    elements now visible to ooxml_lookup_element).
  - 148 / 161 group refs carry compositor_id (rest are top-level).
  - Sample r:id attribute refs now expose
    type_ref={...relationships}ST_RelationshipId.

Fixtures gain a top-level <xsd:attribute name="space"
type="xsd:string" default="preserve"/> in shared.xsd and a CT_RefTest
in main.xsd that refs it; the new test checks all three fixes.

Author: caiopizzol

db/migrations/0003_phase3_metadata.sql

@@ -0,0 +1,10 @@
+-- Phase 3 review fix: preserve element/attribute @type and group-ref compositor context.
+-- Idempotent.
+
+ALTER TABLE xsd_symbols
+	ADD COLUMN IF NOT EXISTS type_ref TEXT;
+
+ALTER TABLE xsd_group_edges
+	ADD COLUMN IF NOT EXISTS compositor_id INT REFERENCES xsd_compositors(id) ON DELETE CASCADE,
+	ADD COLUMN IF NOT EXISTS min_occurs INT DEFAULT 1,
+	ADD COLUMN IF NOT EXISTS max_occurs INT;

db/schema.sql

@@ -64,11 +64,15 @@ CREATE TABLE xsd_namespaces (
   created_at TIMESTAMPTZ DEFAULT NOW()
 );
 
+-- type_ref holds the Clark-style {namespace}localName for elements and attributes
+-- that declare a @type. NULL for complexType/simpleType/group/attributeGroup.
+-- Phase 4 lookups follow type_ref to resolve element -> type when reading children.
 CREATE TABLE xsd_symbols (
   id SERIAL PRIMARY KEY,
   vocabulary_id TEXT NOT NULL,
   local_name TEXT NOT NULL,
   kind TEXT NOT NULL,
+  type_ref TEXT,
   payload JSONB DEFAULT '{}'::jsonb,
   created_at TIMESTAMPTZ DEFAULT NOW(),
   UNIQUE (vocabulary_id, local_name, kind)
@@ -121,13 +125,20 @@ CREATE TABLE xsd_attr_edges (
   order_index INT DEFAULT 0
 );
 
+-- compositor_id is the enclosing compositor when a <xsd:group ref> appears inside
+-- a sequence/choice/all (NULL for refs at the type's top level or for
+-- attributeGroup refs which don't live in a compositor).
+-- min/max_occurs capture the ref site's own cardinality.
 CREATE TABLE xsd_group_edges (
   id SERIAL PRIMARY KEY,
   parent_symbol_id INT NOT NULL REFERENCES xsd_symbols(id) ON DELETE CASCADE,
+  compositor_id INT REFERENCES xsd_compositors(id) ON DELETE CASCADE,
   group_symbol_id INT NOT NULL REFERENCES xsd_symbols(id),
   profile_id INT NOT NULL REFERENCES xsd_profiles(id) ON DELETE CASCADE,
   ref_kind TEXT NOT NULL CHECK (ref_kind IN ('group', 'attributeGroup')),
   resolved BOOLEAN DEFAULT FALSE,
+  min_occurs INT DEFAULT 1,
+  max_occurs INT,
   order_index INT DEFAULT 0
 );
 

scripts/ingest-xsd/ingest.ts

@@ -118,11 +118,16 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 			for (const decl of decls) {
 				const key = symbolKey(decl.vocabularyId, decl.localName, decl.kind);
 				if (symbolIds.has(key)) continue;
+
+				// Capture @type for elements and attributes; resolved Clark-style.
+				const typeRef = resolveDeclTypeRef(decl, parseResult);
+
 				const { id, inserted } = await upsertSymbol(
 					sql,
 					decl.vocabularyId,
 					decl.localName,
 					decl.kind,
+					typeRef,
 				);
 				symbolIds.set(key, id);
 				if (inserted) stats.symbolsInserted++;
@@ -229,10 +234,13 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 				const ctx: WalkCtx = {
 					sql,
 					profileId,
+					sourceId,
 					ownerSymbolId,
 					ownerDecl: decl,
 					prefixMap,
 					symbolIds,
+					namespaceIds,
+					parseResult,
 					stats,
 				};
 
@@ -245,7 +253,7 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 							await walkCompositor(child, tag, null, topOrder, ctx);
 							topOrder++;
 						} else if (tag === "group") {
-							await handleGroupRef(child, topOrder, ctx);
+							await handleGroupRef(child, null, topOrder, ctx);
 							topOrder++;
 						}
 					}
@@ -283,6 +291,7 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 									prefixMap,
 									decl.namespace,
 									symbolIds,
+									parseResult,
 									order,
 									stats,
 								);
@@ -296,18 +305,27 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 									continue;
 								}
 								const groupSymbolId = symbolIds.get(
-									symbolKey(resolved.qname.vocabularyId, resolved.qname.localName, "attributeGroup"),
+									symbolKey(
+										resolved.qname.vocabularyId,
+										resolved.qname.localName,
+										"attributeGroup",
+									),
 								);
 								if (groupSymbolId == null) {
 									stats.attrGroupRefsUnresolved++;
 									continue;
 								}
+								// attributeGroup refs don't live inside content compositors;
+								// compositor_id stays null and min/max default to 1.
 								await insertGroupEdge(
 									sql,
 									ownerSymbolId,
+									null,
 									groupSymbolId,
 									profileId,
 									"attributeGroup",
+									1,
+									1,
 									order,
 								);
 								stats.attrGroupRefsInserted++;
@@ -333,13 +351,30 @@ export async function ingestSchemaSet(opts: IngestSchemaSetOptions): Promise<Ing
 interface WalkCtx {
 	sql: Sql;
 	profileId: number;
+	sourceId: number;
 	ownerSymbolId: number;
 	ownerDecl: Declaration;
 	prefixMap: Map<string, string>;
 	symbolIds: Map<string, number>;
+	namespaceIds: Map<string, number>;
+	parseResult: ParsedSchemaSet;
 	stats: IngestStats;
 }
 
+/**
+ * Resolve a declaration's @type qname (for top-level element/attribute decls)
+ * to Clark-style {namespace}localName, or null if the declaration has no @type.
+ */
+function resolveDeclTypeRef(decl: Declaration, parseResult: ParsedSchemaSet): string | null {
+	if (decl.kind !== "element" && decl.kind !== "attribute") return null;
+	const a = nodeAttrs(decl.node);
+	if (!a.type) return null;
+	const prefixMap = parseResult.namespaceByPrefix.get(decl.documentPath);
+	if (!prefixMap) return a.type;
+	const r = resolveQNameAttr(a.type, prefixMap, decl.namespace);
+	return r.resolved ? `{${r.qname.namespace}}${r.qname.localName}` : a.type;
+}
+
 /**
  * For a complexType: yield the node(s) whose direct children are particles
  * (sequence/choice/all/group). That's the complexType itself, OR (for derived
@@ -401,7 +436,7 @@ async function walkCompositor(
 			await walkCompositor(child, tag, compositorId, childOrder, ctx);
 			childOrder++;
 		} else if (tag === "group") {
-			await handleGroupRef(child, childOrder, ctx, compositorId);
+			await handleGroupRef(child, compositorId, childOrder, ctx);
 			childOrder++;
 		}
 		// xsd:any: skipped for now.
@@ -432,18 +467,49 @@ async function handleElement(
 		}
 		childSymbolId = id;
 	} else if (a.name) {
+		// Resolve @type so ooxml_lookup_element / ooxml_children can follow it.
+		let typeRef: string | null = null;
+		if (a.type) {
+			const r = resolveQNameAttr(a.type, ctx.prefixMap, ctx.ownerDecl.namespace);
+			typeRef = r.resolved ? `{${r.qname.namespace}}${r.qname.localName}` : a.type;
+		}
 		const key = symbolKey(ctx.ownerDecl.vocabularyId, a.name, "element");
 		let id = ctx.symbolIds.get(key);
 		if (id == null) {
-			const res = await upsertSymbol(ctx.sql, ctx.ownerDecl.vocabularyId, a.name, "element");
+			const res = await upsertSymbol(
+				ctx.sql,
+				ctx.ownerDecl.vocabularyId,
+				a.name,
+				"element",
+				typeRef,
+			);
 			ctx.symbolIds.set(key, res.id);
 			if (res.inserted) {
 				ctx.stats.symbolsInserted++;
 				ctx.stats.localElementsCreated++;
 			} else {
 				ctx.stats.symbolsExisting++;
 			}
+			// Local elements need profile membership too, otherwise
+			// ooxml_lookup_element won't find them in the transitional profile.
+			const nsId = ctx.namespaceIds.get(ctx.ownerDecl.namespace);
+			if (nsId != null) {
+				const linked = await linkSymbolToProfile(
+					ctx.sql,
+					res.id,
+					ctx.profileId,
+					nsId,
+					ctx.sourceId,
+				);
+				if (linked) ctx.stats.profileMembershipsInserted++;
+			}
 			id = res.id;
+		} else if (typeRef) {
+			// Existing symbol; ensure type_ref is set if we have one.
+			await ctx.sql`
+				UPDATE xsd_symbols SET type_ref = ${typeRef}
+				WHERE id = ${id} AND type_ref IS NULL
+			`;
 		}
 		childSymbolId = id;
 	}
@@ -465,11 +531,10 @@ async function handleElement(
 
 async function handleGroupRef(
 	node: PreserveOrderNode,
+	compositorId: number | null,
 	orderIndex: number,
 	ctx: WalkCtx,
-	_compositorId: number | null = null,
 ): Promise<void> {
-	void _compositorId; // group_edges aren't compositor-scoped in our schema; refs hang off the parent symbol.
 	const a = nodeAttrs(node);
 	if (!a.ref) return;
 	const resolved = resolveQNameAttr(a.ref, ctx.prefixMap, ctx.ownerDecl.namespace);
@@ -487,9 +552,12 @@ async function handleGroupRef(
 	await insertGroupEdge(
 		ctx.sql,
 		ctx.ownerSymbolId,
+		compositorId,
 		groupSymbolId,
 		ctx.profileId,
 		"group",
+		parseMinOccurs(a.minOccurs),
+		parseMaxOccurs(a.maxOccurs),
 		orderIndex,
 	);
 	ctx.stats.groupRefsInserted++;
@@ -542,11 +610,13 @@ async function upsertSymbol(
 	vocabularyId: string,
 	localName: string,
 	kind: string,
+	typeRef: string | null = null,
 ): Promise<{ id: number; inserted: boolean }> {
 	const [row] = await sql`
-		INSERT INTO xsd_symbols (vocabulary_id, local_name, kind)
-		VALUES (${vocabularyId}, ${localName}, ${kind})
-		ON CONFLICT (vocabulary_id, local_name, kind) DO UPDATE SET kind = EXCLUDED.kind
+		INSERT INTO xsd_symbols (vocabulary_id, local_name, kind, type_ref)
+		VALUES (${vocabularyId}, ${localName}, ${kind}, ${typeRef})
+		ON CONFLICT (vocabulary_id, local_name, kind) DO UPDATE
+			SET type_ref = COALESCE(EXCLUDED.type_ref, xsd_symbols.type_ref)
 		RETURNING id, (xmax = 0) AS inserted
 	`;
 	return { id: row.id, inserted: row.inserted };
@@ -625,16 +695,19 @@ async function insertChildEdge(
 async function insertGroupEdge(
 	sql: Sql,
 	parentSymbolId: number,
+	compositorId: number | null,
 	groupSymbolId: number,
 	profileId: number,
 	refKind: "group" | "attributeGroup",
+	minOccurs: number,
+	maxOccurs: number | null,
 	orderIndex: number,
 ): Promise<void> {
 	await sql`
 		INSERT INTO xsd_group_edges
-			(parent_symbol_id, group_symbol_id, profile_id, ref_kind, order_index)
+			(parent_symbol_id, compositor_id, group_symbol_id, profile_id, ref_kind, min_occurs, max_occurs, order_index)
 		VALUES
-			(${parentSymbolId}, ${groupSymbolId}, ${profileId}, ${refKind}, ${orderIndex})
+			(${parentSymbolId}, ${compositorId}, ${groupSymbolId}, ${profileId}, ${refKind}, ${minOccurs}, ${maxOccurs}, ${orderIndex})
 	`;
 }
 
@@ -721,13 +794,16 @@ async function handleAttribute(
 	prefixMap: Map<string, string>,
 	defaultNamespace: string,
 	symbolIds: Map<string, number>,
+	parseResult: ParsedSchemaSet,
 	orderIndex: number,
 	stats: IngestStats,
 ): Promise<void> {
 	const a = nodeAttrs(node);
 	let localName: string | null = null;
 	let attrSymbolId: number | null = null;
 	let typeRef: string | null = null;
+	let defaultValue: string | null = a.default ?? null;
+	let fixedValue: string | null = a.fixed ?? null;
 
 	if (a.ref) {
 		const resolved = resolveQNameAttr(a.ref, prefixMap, defaultNamespace);
@@ -740,6 +816,27 @@ async function handleAttribute(
 			symbolKey(resolved.qname.vocabularyId, resolved.qname.localName, "attribute"),
 		);
 		if (id != null) attrSymbolId = id;
+
+		// Carry type/default/fixed from the top-level <xsd:attribute name="..."> declaration.
+		// XSD allows these only on the declaration, not the ref site, so look them up there.
+		const declKey = `{${resolved.qname.namespace}}attribute:${resolved.qname.localName}`;
+		const topDecl = parseResult.declarationsByQName.get(declKey)?.[0];
+		if (topDecl) {
+			const declAttrs = nodeAttrs(topDecl.node);
+			if (declAttrs.type) {
+				const declPrefixMap = parseResult.namespaceByPrefix.get(topDecl.documentPath);
+				if (declPrefixMap) {
+					const t = resolveQNameAttr(declAttrs.type, declPrefixMap, topDecl.namespace);
+					typeRef = t.resolved
+						? `{${t.qname.namespace}}${t.qname.localName}`
+						: declAttrs.type;
+				} else {
+					typeRef = declAttrs.type;
+				}
+			}
+			if (defaultValue == null) defaultValue = declAttrs.default ?? null;
+			if (fixedValue == null) fixedValue = declAttrs.fixed ?? null;
+		}
 	} else if (a.name) {
 		localName = a.name;
 		if (a.type) {
@@ -756,9 +853,7 @@ async function handleAttribute(
 
 	const rawUse = a.use;
 	const attrUse: "required" | "optional" | "prohibited" =
-		rawUse === "required" || rawUse === "optional" || rawUse === "prohibited"
-			? rawUse
-			: "optional";
+		rawUse === "required" || rawUse === "optional" || rawUse === "prohibited" ? rawUse : "optional";
 
 	await insertAttrEdge(
 		sql,
@@ -767,8 +862,8 @@ async function handleAttribute(
 		localName,
 		profileId,
 		attrUse,
-		a.default ?? null,
-		a.fixed ?? null,
+		defaultValue,
+		fixedValue,
 		typeRef,
 		orderIndex,
 	);

tests/ingest-xsd/fixtures/main.xsd

@@ -52,5 +52,8 @@
     <xsd:attributeGroup ref="AG_TableProps"/>
     <xsd:attribute name="caption" type="xsd:string" use="required"/>
   </xsd:complexType>
+  <xsd:complexType name="CT_RefTest">
+    <xsd:attribute ref="s:space" use="required"/>
+  </xsd:complexType>
   <xsd:element name="document" type="CT_Empty"/>
 </xsd:schema>

tests/ingest-xsd/fixtures/shared.xsd

@@ -8,4 +8,5 @@
   <xsd:simpleType name="ST_String">
     <xsd:restriction base="xsd:string"/>
   </xsd:simpleType>
+  <xsd:attribute name="space" type="xsd:string" default="preserve"/>
 </xsd:schema>