test(spec): walk every depth of hand-written form lists in the form ↔ Zod reconciliation gate (#14327)

packages/spec/src/system/metadata-form-zod-reconciliation.test.ts

@@ -57,6 +57,22 @@
  * `scripts/build-schemas.ts`, which asks the same question of the emitted JSON
  * Schema (`{ "not": {} }`, Zod's rendering of `z.never()`).
  *
+ * ## The walk is recursive (#14327)
+ *
+ * `nestedLists` once collected a hand-written list only for a **top-level**
+ * entry carrying `fields`, so a repeater or composite nested inside another
+ * nested list — the object designer's per-field `options` / roll-up lists and
+ * its four `lifecycle.*` blocks — was outside the population entirely, not
+ * reconciled loosely. That is how the options repeater offered an `icon` input
+ * `SelectOptionSchema` refuses, through three hand-retirements of the same
+ * offer-vs-door class, with this gate green throughout. The walk now descends
+ * `entry.fields[*].fields` at every depth, keys each list by its dotted path,
+ * and resolves the sub-schema by walking `subSchemaOf` down the same path
+ * (unions looked through, arrays and records peeled). The ledger vocabulary is
+ * unchanged — a `subset` / `omit` entry simply carries a dotted `path` — and
+ * the walk is pinned at the bottom against a synthetic fixture, so a gate that
+ * reaches nothing at depth two cannot report green.
+ *
  * @see control-flow-form-zod-ledger.test.ts — same pattern for the flow designer
  */
 
@@ -97,6 +113,34 @@ const LEDGER: ReadonlyArray<OmitEntry | SubsetEntry> = [
     path: 'fields',
     why: "the object editor's inline column grid is a QUICK-ADD surface covering the common authoring keys; the full per-field editor is `field.form.ts` (registered as the `field` metadata type), which is where the long tail of FieldSchema is authored",
   },
+  // ── Depth two (#14327): the lists nested inside the `fields` quick-add row ──
+  {
+    kind: 'subset',
+    type: 'object',
+    path: 'fields.options',
+    why: "one row of the `fields` quick-add grid (the subset entry above), so the same design applies one level down: an option is captured as label / value / color / description, and the long tail — `default`, the per-option `visibleWhen` CEL predicate — is authored in the full per-field editor (`field.form.ts`), whose `options` repeater is schema-derived and so offers every SelectOptionSchema key",
+  },
+  {
+    kind: 'subset',
+    type: 'object',
+    path: 'fields.summaryOperations',
+    why: "one row of the `fields` quick-add grid (the subset entry above): the roll-up is captured as object / field / function; `relationshipField` (auto-detected unless the child references this object twice) and the `filter` FilterCondition are authored in the full per-field editor (`field.form.ts`), whose own `summaryOperations` composite offers both with their dedicated widgets (`ref:object`, `filter-condition`)",
+  },
+  // ── Depth two (#14327): the lifecycle policy blocks ──
+  {
+    kind: 'omit',
+    type: 'object',
+    path: 'lifecycle.retention',
+    key: 'onlyWhen',
+    why: "a per-field row-filter map ({ field: value | { $in: [...] } | { $null: bool } }) with no scalar rendering among the block's text inputs; every writer of it today is a platform system object declared in code — sys_job_queue, sys_automation_run, the storage service's system_file / system_upload_session — where the interleaved live-vs-terminal rows it exists for live. A Studio-authored object gets the plain age window; offering the filter needs a structured control, a form-face addition rather than a reconciliation",
+  },
+  {
+    kind: 'omit',
+    type: 'object',
+    path: 'lifecycle.ttl',
+    key: 'onlyWhen',
+    why: "the mirror of `retention.onlyWhen` — one shape by design (`lifecycleOnlyWhenSchema`, object.zod.ts) — with the same boundary: a row-filter map with no scalar rendering among the ttl block's text inputs, and its one writer today is the code-declared sys_session object (`revoked_at: { $null: true }`). Offering it needs a structured control, a form-face addition rather than a reconciliation",
+  },
 ];
 
 // ────────────────────────────────────────────────────────────────────────────
@@ -249,34 +293,107 @@ function topLevelFields(form: any): string[] {
   return names.sort();
 }
 
+type NestedList = { path: string; depth: number; offered: string[] };
+
 /**
  * Every nested list a form spells out by hand — `{ field, fields: [...] }` under
- * a composite / repeater / record entry. These are the hand-copied lists; an
- * entry with no `fields` is derived from the schema by the renderer and cannot
- * drift.
+ * a composite / repeater / record entry — at **any** depth, keyed by the dotted
+ * path from the section root (`fields`, `fields.options`, `lifecycle.ttl`).
+ * These are the hand-copied lists; an entry with no `fields` is derived from
+ * the schema by the renderer and cannot drift.
+ *
+ * Recursive since #14327: the walk used to stop at the section's own entries,
+ * so a repeater inside a record editor — where the object designer keeps its
+ * per-field option and roll-up lists — was outside the population entirely.
  */
-function nestedLists(form: any): Array<{ path: string; offered: string[] }> {
-  const out: Array<{ path: string; offered: string[] }> = [];
-  for (const section of form.sections ?? []) {
-    for (const entry of (section.fields ?? []) as FormEntry[]) {
+function nestedLists(form: any): NestedList[] {
+  const out: NestedList[] = [];
+  const walk = (entries: FormEntry[], prefix: string, depth: number) => {
+    for (const entry of entries) {
       if (!entry?.field || !Array.isArray(entry.fields) || entry.fields.length === 0) continue;
+      const path = prefix ? `${prefix}.${entry.field}` : entry.field;
       const offered = entry.fields.map((f) => f?.field).filter((f): f is string => !!f);
       // A record editor authors its map key through `keyField`, so that name is
-      // offered even though it is not in the `fields` array.
+      // offered even though it is not in the `fields` array — at every depth.
       if (entry.keyField?.field) offered.push(entry.keyField.field);
-      out.push({ path: entry.field, offered: offered.sort() });
+      out.push({ path, depth, offered: offered.sort() });
+      walk(entry.fields, path, depth + 1);
     }
-  }
+  };
+  for (const section of form.sections ?? []) walk((section.fields ?? []) as FormEntry[], '', 1);
   return out;
 }
 
+/**
+ * The sub-schema a dotted form path lands on: one `subSchemaOf` step per
+ * segment, so every level looks through unions and peels the array / record
+ * wrapper a repeater or record editor sits under. `undefined` as soon as a
+ * segment is not declared — the caller reports that as an unanchored list.
+ */
+function subSchemaAt(root: unknown, path: string): unknown {
+  let node: unknown = root;
+  for (const segment of path.split('.')) {
+    node = subSchemaOf(node, segment);
+    if (node === undefined) return undefined;
+  }
+  return node;
+}
+
+type Ledger = ReadonlyArray<OmitEntry | SubsetEntry>;
 const TYPES = Object.keys(METADATA_FORM_REGISTRY);
-const ledgerFor = (type: string, path: string) =>
-  LEDGER.filter((e) => e.type === type && e.path === path);
-const isSubset = (type: string, path: string) =>
-  ledgerFor(type, path).some((e) => e.kind === 'subset');
-const omittedAt = (type: string, path: string) =>
-  ledgerFor(type, path).flatMap((e) => (e.kind === 'omit' ? [e.key] : []));
+const ledgerFor = (ledger: Ledger, type: string, path: string) =>
+  ledger.filter((e) => e.type === type && e.path === path);
+const isSubset = (ledger: Ledger, type: string, path: string) =>
+  ledgerFor(ledger, type, path).some((e) => e.kind === 'subset');
+const omittedAt = (ledger: Ledger, type: string, path: string) =>
+  ledgerFor(ledger, type, path).flatMap((e) => (e.kind === 'omit' ? [e.key] : []));
+
+/**
+ * One hand-written nested list, judged against the sub-schema its dotted path
+ * resolves to. Empty arrays are the passing state; `unanchored` means the path
+ * resolved to nothing key-bearing, so the three key sets could not be judged.
+ */
+type NestedVerdict = {
+  path: string;
+  depth: number;
+  unanchored: boolean;
+  /** offered by the form, not declared by the Zod — silently dropped on save */
+  formOnly: string[];
+  /** offered by the form, tombstoned in the Zod — hard-fails the save */
+  retired: string[];
+  /** authorable in the Zod, not offered by the form, not excused by the ledger */
+  zodOnly: string[];
+};
+
+/**
+ * The nested-list predicate: one form against one root schema and one ledger.
+ * The `it.each` below applies it to the registry; the self-test at the bottom
+ * applies the SAME function to a synthetic fixture, which is what makes "the
+ * gate reaches depth two" a measured fact rather than an assumption.
+ */
+function reconcileNestedLists(type: string, form: any, root: unknown, ledger: Ledger): NestedVerdict[] {
+  return nestedLists(form).map(({ path, depth, offered }) => {
+    const sub = subSchemaAt(root, path);
+    const subKeys = keysOf(sub);
+    if (!subKeys) return { path, depth, unanchored: true, formOnly: [], retired: [], zodOnly: [] };
+    const excused = omittedAt(ledger, type, path);
+    return {
+      path,
+      depth,
+      unanchored: false,
+      formOnly: offered.filter((k) => !subKeys.includes(k)),
+      retired: offered.filter((k) => isRetiredAt(sub, k)),
+      // A tombstoned key needs no ledger entry to excuse its absence — the
+      // *only* correct thing to do with it is not offer it. Demanding one back
+      // (or a ledger row for it) is this gate's blind spot inverted: before
+      // #5280 the sole thing keeping `object.fields.conditionalRequired` off
+      // this list was an unrelated `subset` entry, i.e. luck.
+      zodOnly: isSubset(ledger, type, path)
+        ? []
+        : subKeys.filter((k) => !offered.includes(k) && !excused.includes(k) && !isRetiredAt(sub, k)),
+    };
+  });
+}
 
 describe('metadata form ↔ Zod reconciliation (#3786)', () => {
   it('the registry is non-empty and every form resolves a schema', () => {
@@ -313,34 +430,29 @@ describe('metadata form ↔ Zod reconciliation (#3786)', () => {
   it.each(TYPES)('%s: every hand-written nested list matches its sub-schema', (type) => {
     const root = getMetadataTypeSchema(type);
 
-    for (const { path, offered } of nestedLists(METADATA_FORM_REGISTRY[type])) {
-      const sub = subSchemaOf(root, path);
-      const subKeys = keysOf(sub);
+    // `expect.soft`, so one run names EVERY list that drifted in this form
+    // rather than the first: a form carries several hand-written lists, and a
+    // gate that reports one per red build is the sequential-artifact failure
+    // mode AGENTS.md describes for `check:generated` — triage wants the table.
+    for (const v of reconcileNestedLists(type, METADATA_FORM_REGISTRY[type], root, LEDGER)) {
       // A non-key-bearing sub-schema (a plain array of scalars, say) has nothing
       // to reconcile against — but a hand-written list under it is then
       // unanchored, so say so rather than skipping silently.
-      expect(subKeys, `${type}.${path}: hand-written sub-list over a non-key-bearing schema`).toBeTruthy();
+      expect.soft(v.unanchored, `${type}.${v.path}: hand-written sub-list over a non-key-bearing schema`).toBe(false);
 
-      expect(
-        offered.filter((k) => !subKeys!.includes(k)),
-        `${type}.${path}: offered by the form but not declared by the Zod (saved value is dropped)`,
+      expect.soft(
+        v.formOnly,
+        `${type}.${v.path}: offered by the form but not declared by the Zod (saved value is dropped)`,
       ).toEqual([]);
 
-      expect(
-        offered.filter((k) => isRetiredAt(sub, k)),
-        `${type}.${path}: offered by the form but RETIRED in the Zod (retiredKey tombstone — filling the control hard-fails the save). Delete the form entry and leave a comment naming the retirement`,
+      expect.soft(
+        v.retired,
+        `${type}.${v.path}: offered by the form but RETIRED in the Zod (retiredKey tombstone — filling the control hard-fails the save). Delete the form entry and leave a comment naming the retirement`,
       ).toEqual([]);
 
-      if (isSubset(type, path)) continue;
-      const excused = omittedAt(type, path);
-      // A tombstoned key needs no ledger entry to excuse its absence — the
-      // *only* correct thing to do with it is not offer it. Demanding one back
-      // (or a ledger row for it) is this gate's blind spot inverted: before
-      // #5280 the sole thing keeping `object.fields.conditionalRequired` off
-      // this list was an unrelated `subset` entry, i.e. luck.
-      expect(
-        subKeys!.filter((k) => !offered.includes(k) && !excused.includes(k) && !isRetiredAt(sub, k)),
-        `${type}.${path}: accepted by the Zod but unauthorable in the form — offer it, or add a ledger entry`,
+      expect.soft(
+        v.zodOnly,
+        `${type}.${v.path}: accepted by the Zod but unauthorable in the form — offer it, or add a ledger entry`,
       ).toEqual([]);
     }
   });
@@ -356,7 +468,7 @@ describe('metadata form ↔ Zod reconciliation (#3786)', () => {
       const list = lists.find((l) => l.path === entry.path);
       expect(list, `${entry.type}.${entry.path}: no hand-written list at this path any more`).toBeDefined();
 
-      const sub = subSchemaOf(root, entry.path);
+      const sub = subSchemaAt(root, entry.path);
       const subKeys = keysOf(sub);
       expect(subKeys, `${entry.type}.${entry.path}: sub-schema is not key-bearing any more`).toBeTruthy();
 
@@ -452,3 +564,129 @@ describe('retiredKey tombstones are not authoring surface (#5280)', () => {
     expect(isRetiredAt(union, 'shared')).toBe(false);
   });
 });
+
+// ────────────────────────────────────────────────────────────────────────────
+// The nested walk itself (#14327).
+//
+// A gate observed only green is indistinguishable from a gate that matches
+// nothing — which is exactly what the depth-one walk was, at depth two, for as
+// long as it existed. Two kinds of pin: the live registry's deep lists, by
+// name, so the population cannot collapse back to depth one unnoticed; and a
+// SYNTHETIC form + schema run through the same `reconcileNestedLists`, so the
+// predicate is shown to go red on a depth-two form-only key (positive control)
+// and green on a depth-two designed subset carrying its ledger entry (negative
+// control).
+// ────────────────────────────────────────────────────────────────────────────
+
+describe('the nested walk reaches every depth (#14327)', () => {
+  it('the live registry has hand-written lists below depth one, and the walk reaches them', () => {
+    const deep = TYPES.flatMap((type) =>
+      nestedLists(METADATA_FORM_REGISTRY[type])
+        .filter((l) => l.depth >= 2)
+        .map((l) => `${type}.${l.path}`),
+    );
+    // The six the depth-one walk never reached, measured over all seventeen
+    // registered forms when this pin was written: the object designer keeps
+    // its per-field and lifecycle blocks one level down. Named, not counted —
+    // a count would pass over any six.
+    expect(deep).toEqual(
+      expect.arrayContaining([
+        'object.fields.options',
+        'object.fields.summaryOperations',
+        'object.lifecycle.retention',
+        'object.lifecycle.ttl',
+        'object.lifecycle.storage',
+        'object.lifecycle.archive',
+      ]),
+    );
+  });
+
+  // A record editor (keyed by `name`) whose rows carry a repeater — the object
+  // designer's shape in miniature, with one tombstone in the deep shape so the
+  // retired direction is exercised at depth two as well.
+  const schema = z.object({
+    items: z.record(
+      z.string(),
+      z.object({
+        name: z.string(),
+        label: z.string(),
+        options: z
+          .array(
+            z.object({
+              label: z.string(),
+              value: z.string(),
+              extra: z.string().optional(),
+              gone: retiredKey('`Probe.options.gone` was removed in @objectstack/spec 17.0.0. Delete the key.'),
+            }),
+          )
+          .optional(),
+      }),
+    ),
+  });
+  const form = (optionInputs: string[]) => ({
+    sections: [
+      {
+        fields: [
+          {
+            field: 'items',
+            type: 'record',
+            keyField: { field: 'name' },
+            fields: [
+              { field: 'label' },
+              { field: 'options', type: 'repeater', fields: optionInputs.map((field) => ({ field })) },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  const at = <T extends { path: string }>(xs: T[], path: string) => xs.find((x) => x.path === path);
+
+  it('keys the lists by dotted path and resolves each level through the record and the array', () => {
+    const lists = nestedLists(form(['label', 'value']));
+    expect(lists.map((l) => [l.path, l.depth])).toEqual([
+      ['items', 1],
+      ['items.options', 2],
+    ]);
+    expect(at(lists, 'items')?.offered).toEqual(['label', 'name', 'options']);
+    expect(keysOf(subSchemaAt(schema, 'items.options'))).toEqual(['extra', 'gone', 'label', 'value']);
+    expect(subSchemaAt(schema, 'items.nothing')).toBeUndefined();
+  });
+
+  it('positive control: a form-only key two levels down is reported at its dotted path', () => {
+    const verdicts = reconcileNestedLists('probe', form(['label', 'value', 'icon']), schema, []);
+    expect(at(verdicts, 'items')?.formOnly).toEqual([]);
+    expect(at(verdicts, 'items.options')?.formOnly).toEqual(['icon']);
+  });
+
+  it('positive control: a tombstoned key offered two levels down is reported as retired, not as form-only', () => {
+    const verdicts = reconcileNestedLists('probe', form(['label', 'value', 'gone']), schema, []);
+    expect(at(verdicts, 'items.options')?.retired).toEqual(['gone']);
+    expect(at(verdicts, 'items.options')?.formOnly).toEqual([]);
+  });
+
+  it('negative control: a designed depth-two subset is green with its ledger entry and red without', () => {
+    const offered = form(['label', 'value']);
+    const bare = reconcileNestedLists('probe', offered, schema, []);
+    // `gone` is a tombstone and is excused automatically; `extra` is the gap.
+    expect(at(bare, 'items.options')?.zodOnly).toEqual(['extra']);
+
+    const asSubset = reconcileNestedLists('probe', offered, schema, [
+      { kind: 'subset', type: 'probe', path: 'items.options', why: 'synthetic: the probe row is a quick-add subset' },
+    ]);
+    expect(at(asSubset, 'items.options')?.zodOnly).toEqual([]);
+
+    const asOmit = reconcileNestedLists('probe', offered, schema, [
+      { kind: 'omit', type: 'probe', path: 'items.options', key: 'extra', why: 'synthetic: extra is deliberately not offered' },
+    ]);
+    expect(at(asOmit, 'items.options')?.zodOnly).toEqual([]);
+
+    // An entry at the PARENT path excuses nothing one level down — the ledger
+    // is keyed by the full dotted path, so a subset row cannot cover its
+    // children by accident.
+    const misfiled = reconcileNestedLists('probe', offered, schema, [
+      { kind: 'subset', type: 'probe', path: 'items', why: 'synthetic: the parent list is a subset' },
+    ]);
+    expect(at(misfiled, 'items.options')?.zodOnly).toEqual(['extra']);
+  });
+});