diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b79c7ad0..aaf570d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -320,6 +320,20 @@ loading. A port that reports `1.2` does not know that vocabulary; upgrade every ### Fixed +- **TypeScript: Drizzle now binds a `field.object` with `@storage: flattened` to its member columns + (#524).** The generator used to emit one `jsonb` column named for the field, while `meta migrate` + creates one column per member of the value object and none for the field, so every generated + query failed with an undefined-column error. The Drizzle table now carries one column per member, + named by the same `resolveFlattenedColumnName` the migration uses, and the Zod schemas, the + Row/Insert types and the filter allowlist expose them as flat keys (`shipToStreet`), the same + surface Prisma already has. Generated queries, Hono and Fastify routes need no change. The client filter + type, the sort allowlist, the OpenAPI filter and sort parameters and the string-decimal wire + rewrite use the same flat keys. A flattened field on a write-through entity is refused with a + `CodegenError` instead of generating code that cannot work. A flattened field declared on a TPH + subtype keeps a single column named for the field, as `meta migrate` creates it, and a value + object that declares one keeps it as one nested key. The React form and the TanStack grid + columns use the same flat keys, so a form posts `shipToStreet`, not a nested `shipTo`. + - **TypeScript: every `meta` command now loads the providers in `metaobjects.config.ts`.** `meta export`, `meta deps`, `meta fmt`, `meta types` and `meta upgrade` loaded the core vocabulary only, so a project's own subtype or attribute failed there while `meta gen` accepted diff --git a/docs/features/api-contract.md b/docs/features/api-contract.md index a4d965895..66b35596e 100644 --- a/docs/features/api-contract.md +++ b/docs/features/api-contract.md @@ -405,7 +405,7 @@ for the one-line change. | `field.timestamp` (`@localTime: true`) | string | Naive wall clock — `YYYY-MM-DDTHH:MM:SS[.fff]`, **no `Z`**. | | `field.currency` | **integer minor units** | Cents for USD, yen for JPY. Float arithmetic is forbidden. Server never formats. | | `field.object` (`@storage: jsonb`) | object | Nested per the sub-object schema. A null member is written per [the wire setting](#the-wire-setting) (`nullMembers`). | -| `field.object` (`@storage: flattened`) | object | Same JSON shape — only the storage differs. | +| `field.object` (`@storage: flattened`) | object | Same JSON shape in the contract — only the storage differs. The TypeScript generator exposes the members as flat keys instead (`shipToStreet`); see [field-types.md](field-types.md#what-each-port-generates). | The currency invariant is the load-bearing one: every port emits and expects integer minor units on the wire, and the diff --git a/docs/features/field-types.md b/docs/features/field-types.md index 8c847d42b..d04de99e8 100644 --- a/docs/features/field-types.md +++ b/docs/features/field-types.md @@ -359,6 +359,15 @@ The physical names the strategy above resolved are emitted **once**, into generated file references that constant rather than respelling it. So the table binding below names no column directly — to read the resolved spelling, open the names artifact. +A `@storage: flattened` field binds one Drizzle column per member, keyed `` (for +the `address` example above, `addressStreet`, `addressCity`, …) and named by the same core +function `meta migrate` uses. The Row and Insert types, the Zod schemas, the filter and sort +allowlists, the OpenAPI parameters and the form and grid generators all expose those flat keys; +a client reads and writes `addressStreet`, not a nested `address` object. A flattened field +declared on a TPH subtype keeps one column named for the field, as `meta migrate` creates it. +A flattened field on a write-through entity is refused at `meta gen` +([source-kinds.md](source-kinds.md)). + ```ts // generated/acme/blog/Author.ts import { AuthorNames } from "./Author.names"; diff --git a/docs/features/source-kinds.md b/docs/features/source-kinds.md index 629118f5e..6840c71a1 100644 --- a/docs/features/source-kinds.md +++ b/docs/features/source-kinds.md @@ -71,8 +71,9 @@ emitter, two hosts. > generated **reads** to the replica view and carries the derived fields on the entity read > **type** — a create/update re-reads the row through the view by primary key, so the returned > value carries the derived columns (read-your-writes). Writes still target the table and exclude -> derived fields. (A flattened `field.object` on a write-through entity is a tracked -> sub-limitation.) +> derived fields. (A flattened `field.object` on a write-through entity is refused at `meta gen` +> with a `CodegenError` in TypeScript: the replica view has no flattened-member form yet. Use +> `@storage: jsonb` there. See [field-types.md](field-types.md).) ```json { diff --git a/examples/advanced-modeling/codegen/templates/drizzle-schema.ts b/examples/advanced-modeling/codegen/templates/drizzle-schema.ts index 74df06cb0..16457c7da 100644 --- a/examples/advanced-modeling/codegen/templates/drizzle-schema.ts +++ b/examples/advanced-modeling/codegen/templates/drizzle-schema.ts @@ -33,6 +33,7 @@ import { renderRelationsBlock } from "./relations-block.js"; import { renderDocsFor } from "./jsdoc.js"; import { collectTphSubtypeFields, tphConcreteSubtypes } from "./tph-discriminator.js"; import { tphStorageObject } from "@metaobjectsdev/codegen-ts/engine"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { effectivePackage } from "@metaobjectsdev/codegen-ts/engine"; /** @@ -215,12 +216,43 @@ export function renderDrizzleSchema( const enumIntTypes = new Map(); // The field.duration codec: one declaration serves every duration column of the table. let usesDurationCodec = false; + // One column per member of a flattened value object, keyed `` and named by + // core naming (`resolveFlattenedColumnName`), the name `meta migrate` creates; the + // `Names` constant when the artifact carries it. No CHECK constraint: migrate derives + // CHECKs from the entity's own fields only. + const pushFlattenedColumns = ( + owner: MetaField, + members: NonNullable>, + ): void => { + for (const m of members) { + const base = mapColumnType(m.flat, ctx.dialect, ctx.columnNamingStrategy, ctx.timestampMode); + const spec: ColumnSpec = { ...base, dbName: m.column }; + if (spec.enumIntCustomType !== undefined) { + enumIntTypes.set(spec.enumIntCustomType.fnConstName, spec.enumIntCustomType); + } + if (spec.durationCustomType !== undefined) usesDurationCodec = true; + const named = names?.resolved.fields[owner.name]?.members?.[m.member.name]; + const dbExpr: Code = names !== undefined && named !== undefined + ? code`${names.symbol}.fields.${owner.name}.members.${m.member.name}.column` + : code`${JSON.stringify(m.column)}`; + columnLines.push( + renderColumn(spec, dbExpr, m.member, ctx, false, undefined, undefined, isComposite, pkg, obj.name, false, m.key), + ); + } + }; for (const child of obj.fields()) { // #213 — a derived (origin-bearing) field is read-only, materialized on the // read (view) side, NOT a column on the entity's write table (FR-024 §7). // Emitting it here would declare a Drizzle column for a table column migrate // no longer creates. if (child.isDerived()) continue; + // `field.object @storage: flattened` holds one column per member of its value object and + // none of its own - exactly the columns `meta migrate` creates (flattened.ts). + const flat = flattenedMembersOf(child, ctx.columnNamingStrategy); + if (flat !== undefined) { + pushFlattenedColumns(child, flat); + continue; + } const isPk = pkFieldNames.has(child.name); const fkInfo = fkMap.get(child.name); // Compute the column spec once per field and reuse it for both the column @@ -519,6 +551,9 @@ function renderColumn( // FR-017 Tier 2 — TPH subtype-only column: force nullable (drop .notNull()) // and suppress any DB default (other-subtype rows must stay NULL here). forceNullable: boolean = false, + // `field.object @storage: flattened`: the Drizzle KEY of a member's column (`shipToStreet`); + // absent, the key is the field's own name. + keyName: string = field.name, ): Code { // An int-backed field.enum's column function is a LOCAL generated const (the // customType helper emitted into this same file), so it must not be imported @@ -739,7 +774,7 @@ function renderColumn( } } - const columnLine = code` ${field.name}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; + const columnLine = code` ${keyName}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; return spec.leadingComment !== undefined ? code` // ${spec.leadingComment}\n${columnLine}` : columnLine; diff --git a/examples/advanced-modeling/codegen/templates/entity-file.ts b/examples/advanced-modeling/codegen/templates/entity-file.ts index b448ae1a5..7e460dbc6 100644 --- a/examples/advanced-modeling/codegen/templates/entity-file.ts +++ b/examples/advanced-modeling/codegen/templates/entity-file.ts @@ -6,6 +6,8 @@ // !hasWritableRdbSource(entity) → renderValueObjectFile (in-memory / transit shape: interface + Zod schema) // vanilla / write-through entity → Drizzle table path +import { CodegenError } from "./errors.js"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { SOURCE_ROLE_REPLICA } from "@metaobjectsdev/metadata"; @@ -137,6 +139,16 @@ export function renderEntityFile( // compile error). A base+write-through combo keeps the TPH polymorphic read path (reads // the base table); routing its reads through a replica view is a documented non-goal. const writeThrough = isWriteThrough(entity) && !tphBase; + // A write-through entity reads through a replica view whose columns are declared per field; + // that declaration has no flattened-member form yet, and a view column bound under the wrong + // name fails at query time, so refuse rather than emit it. + if (writeThrough && entity.fields().some((f) => flattenedMembersOf(f, ctx.columnNamingStrategy) !== undefined)) { + throw new CodegenError( + `${entity.name}: a write-through entity (writable table plus a replica view) cannot hold a ` + + `field.object @storage: flattened field yet - the replica view declaration has no flattened-member form. ` + + `Use @storage: jsonb, or drop the replica view.`, + ); + } // Rendered BEFORE the replica view below, for two reasons that both matter: the table // DECLARES the int-backed-enum codecs (a `const` the view's declaration would duplicate, // and a duplicate module-scope const is a parse error), and those consts must precede the diff --git a/examples/advanced-modeling/codegen/templates/errors.ts b/examples/advanced-modeling/codegen/templates/errors.ts new file mode 100644 index 000000000..71b2b751b --- /dev/null +++ b/examples/advanced-modeling/codegen/templates/errors.ts @@ -0,0 +1,10 @@ +// Typed errors for codegen-ts. + +export class CodegenError extends Error { + readonly file?: string; + constructor(message: string, opts?: { file?: string }) { + super(message); + this.name = "CodegenError"; + if (opts?.file !== undefined) this.file = opts.file; + } +} diff --git a/examples/advanced-modeling/codegen/templates/filter-allowlist.ts b/examples/advanced-modeling/codegen/templates/filter-allowlist.ts index ec7c195b7..fd7406700 100644 --- a/examples/advanced-modeling/codegen/templates/filter-allowlist.ts +++ b/examples/advanced-modeling/codegen/templates/filter-allowlist.ts @@ -1,3 +1,4 @@ +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -114,8 +115,7 @@ function valueShape(f: MetaField): string { function filterableFields(entity: MetaObject, exclude?: string): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in allowlists. - return entity - .fields() + return flatSurfaceFields(entity) .filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true && c.name !== exclude); } diff --git a/examples/advanced-modeling/codegen/templates/filter-shared.ts b/examples/advanced-modeling/codegen/templates/filter-shared.ts index 1ddcb3162..c4de261bb 100644 --- a/examples/advanced-modeling/codegen/templates/filter-shared.ts +++ b/examples/advanced-modeling/codegen/templates/filter-shared.ts @@ -14,6 +14,7 @@ // renders clickable for a field the endpoint rejects, or a grid whose initial sort // runs the opposite way from the endpoint it queries. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject, type MetaData } from "@metaobjectsdev/metadata"; import { FIELD_ATTR_FILTERABLE, @@ -47,7 +48,7 @@ export function isSortableField(field: MetaField): boolean { */ export function sortableFields(entity: MetaObject): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in sort ops. - return entity.fields().filter(isSortableField); + return flatSurfaceFields(entity).filter(isSortableField); } /** @@ -110,6 +111,6 @@ export function resolveGridDefaultSort( const declared = layout.attr(LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_ORDER); if (declared === "asc" || declared === "desc") return { field: fieldName, order: declared }; - const field = entity.fields().find((f) => f.name === fieldName); + const field = flatSurfaceFields(entity).find((f) => f.name === fieldName); return { field: fieldName, order: field === undefined ? "asc" : sortDefaultOrder(field) }; } diff --git a/examples/advanced-modeling/codegen/templates/filter-type.ts b/examples/advanced-modeling/codegen/templates/filter-type.ts index e2fd030d3..d3a06b53f 100644 --- a/examples/advanced-modeling/codegen/templates/filter-type.ts +++ b/examples/advanced-modeling/codegen/templates/filter-type.ts @@ -2,6 +2,7 @@ // String fields get eq/ne/in/like; numbers get eq/ne/gt/gte/lt/lte/in; booleans get eq/isNull only; // datetimes get eq/ne/gt/gte/lt/lte/in. An enum's operand type is its member union. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -82,7 +83,7 @@ function renderFieldUnion(field: MetaField): string { export function renderFilterType(entity: MetaObject, exclude?: string): Code { const filterName = entityTypeNames(entity).filter; // fields() returns effective fields, so inherited fields (from extends:/super:) are included in filter types. - const allFields = entity.fields().filter((c) => c.name !== exclude); + const allFields = flatSurfaceFields(entity).filter((c) => c.name !== exclude); const filterableFieldsList = allFields.filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true); // Sort union uses isSortableField — same predicate as renderSortAllowlist to prevent // client/server mismatches (@filterable: true + @sortable: false must be excluded from both). diff --git a/examples/advanced-modeling/codegen/templates/inferred-types.ts b/examples/advanced-modeling/codegen/templates/inferred-types.ts index 4b87f2f9b..def5fb049 100644 --- a/examples/advanced-modeling/codegen/templates/inferred-types.ts +++ b/examples/advanced-modeling/codegen/templates/inferred-types.ts @@ -7,6 +7,7 @@ // optional fields (matching Zod's `.optional()` inference — `T | undefined` — // without the superfluous `| null` that Drizzle nullable columns introduce). +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { @@ -146,7 +147,7 @@ export function renderEnumTypeAliases(entity: MetaObject, ctx?: RenderContext): // and non-colliding value objects keep their bare name (byte-identical). const ownerName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { if (field.subType !== FIELD_SUBTYPE_ENUM) continue; const values = enumValues(field); @@ -402,7 +403,7 @@ export function renderValueObjectInterface(entity: MetaObject, ctx?: RenderConte const objName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; const lines: Code[] = []; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { const required = field.attr(FIELD_ATTR_REQUIRED) === true; const optional = required ? "" : "?"; const undefinedArm = required ? "" : " | undefined"; diff --git a/examples/advanced-modeling/codegen/templates/zod-validators.ts b/examples/advanced-modeling/codegen/templates/zod-validators.ts index 195fddfac..ec3a5557b 100644 --- a/examples/advanced-modeling/codegen/templates/zod-validators.ts +++ b/examples/advanced-modeling/codegen/templates/zod-validators.ts @@ -9,6 +9,7 @@ // object-array field collapsed to z.array(z.string()) and the JSON Schema sent // downstream (e.g. to LLM tool_use input_schema) lost the nested object shape. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { entityTypeNames } from "./entity-type-names.js"; import { code, joinCode, imp, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaObject, MetaField, type MetaRoot, stripPackage } from "@metaobjectsdev/metadata"; @@ -104,7 +105,7 @@ function durationHelperSince(since: number): string { * a plain entity has nothing to preserve, so both are omitted. Resolving * (ADR-0039): honors an @autoSet inherited via extends. */ export function hasAutoSetFields(obj: MetaObject): boolean { - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { const autoSet = child.attr(FIELD_ATTR_AUTO_SET); if (autoSet === AUTO_SET_ON_CREATE || autoSet === AUTO_SET_ON_UPDATE) return true; } @@ -121,7 +122,7 @@ export function hasAutoSetFields(obj: MetaObject): boolean { function isTphBaseOwnField(obj: MetaObject, field: MetaField): boolean { const base = tphDiscriminatorBase(obj); if (base === undefined) return false; - return base.fields().some((f) => f.name === field.name); + return flatSurfaceFields(base).some((f) => f.name === field.name); } /** @@ -169,7 +170,7 @@ export function renderTphSubtypeReadSchema(obj: MetaObject, ctx?: RenderContext) const tphPin = tphDiscriminatorPin(obj); const fieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (tphPin !== undefined && child.name === tphPin.fieldName) { fieldLines.push(code` ${child.name}: z.literal(${JSON.stringify(tphPin.value)})`); continue; @@ -319,7 +320,7 @@ function crossComparable(left: MetaField, right: MetaField, opName: string): boo function insertSchemaFieldNameSet(obj: MetaObject): Set { const autoGen = autoGenPkFieldNames(obj); const out = new Set(); - for (const f of obj.fields()) { + for (const f of flatSurfaceFields(obj)) { if (autoGen.has(f.name)) continue; if (isReadOnlyMutability(f) || f.isDerived()) continue; out.add(f.name); @@ -385,7 +386,7 @@ interface RefinementBuild { blocks: string[]; helpers: Set } function crossFieldRefinement(obj: MetaObject): RefinementBuild { const inSchema = insertSchemaFieldNameSet(obj); - const byName = new Map(obj.fields().map((f) => [f.name, f] as const)); + const byName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f] as const)); // A reference must name a field that exists AND is in the insert schema; otherwise skip // the validator (migrate skips on a missing field; a readOnly/derived/auto-PK field is // never in the body, so enforcing on it would reject every create). @@ -599,7 +600,7 @@ export function renderInsertSchemaOnly(obj: MetaObject, ctx?: RenderContext): Co const tphPin = tphDiscriminatorPin(obj); const insertFieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field is populated by DB / replication / // external owner; the application has no path to write it. Exclude from the @@ -684,7 +685,7 @@ export function insertSchemaFields(obj: MetaObject): SchemaFieldShape[] { const assignedPkFields = assignedPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -720,7 +721,7 @@ export function updateSchemaFields(obj: MetaObject): SchemaFieldShape[] { const autoGenPkFields = autoGenPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -767,7 +768,7 @@ export function renderZodValidators(obj: MetaObject, ctx?: RenderContext): Code // transform). Only emitted when the entity declares @autoSet fields. const preservingFieldLines: Code[] = []; const emitPreserving = hasAutoSetFields(obj); - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field appears in neither InsertSchema // nor UpdateSchema. The DB / trigger / replication owns the write path; the app diff --git a/examples/showcase/codegen/templates/drizzle-schema.ts b/examples/showcase/codegen/templates/drizzle-schema.ts index 74df06cb0..16457c7da 100644 --- a/examples/showcase/codegen/templates/drizzle-schema.ts +++ b/examples/showcase/codegen/templates/drizzle-schema.ts @@ -33,6 +33,7 @@ import { renderRelationsBlock } from "./relations-block.js"; import { renderDocsFor } from "./jsdoc.js"; import { collectTphSubtypeFields, tphConcreteSubtypes } from "./tph-discriminator.js"; import { tphStorageObject } from "@metaobjectsdev/codegen-ts/engine"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { effectivePackage } from "@metaobjectsdev/codegen-ts/engine"; /** @@ -215,12 +216,43 @@ export function renderDrizzleSchema( const enumIntTypes = new Map(); // The field.duration codec: one declaration serves every duration column of the table. let usesDurationCodec = false; + // One column per member of a flattened value object, keyed `` and named by + // core naming (`resolveFlattenedColumnName`), the name `meta migrate` creates; the + // `Names` constant when the artifact carries it. No CHECK constraint: migrate derives + // CHECKs from the entity's own fields only. + const pushFlattenedColumns = ( + owner: MetaField, + members: NonNullable>, + ): void => { + for (const m of members) { + const base = mapColumnType(m.flat, ctx.dialect, ctx.columnNamingStrategy, ctx.timestampMode); + const spec: ColumnSpec = { ...base, dbName: m.column }; + if (spec.enumIntCustomType !== undefined) { + enumIntTypes.set(spec.enumIntCustomType.fnConstName, spec.enumIntCustomType); + } + if (spec.durationCustomType !== undefined) usesDurationCodec = true; + const named = names?.resolved.fields[owner.name]?.members?.[m.member.name]; + const dbExpr: Code = names !== undefined && named !== undefined + ? code`${names.symbol}.fields.${owner.name}.members.${m.member.name}.column` + : code`${JSON.stringify(m.column)}`; + columnLines.push( + renderColumn(spec, dbExpr, m.member, ctx, false, undefined, undefined, isComposite, pkg, obj.name, false, m.key), + ); + } + }; for (const child of obj.fields()) { // #213 — a derived (origin-bearing) field is read-only, materialized on the // read (view) side, NOT a column on the entity's write table (FR-024 §7). // Emitting it here would declare a Drizzle column for a table column migrate // no longer creates. if (child.isDerived()) continue; + // `field.object @storage: flattened` holds one column per member of its value object and + // none of its own - exactly the columns `meta migrate` creates (flattened.ts). + const flat = flattenedMembersOf(child, ctx.columnNamingStrategy); + if (flat !== undefined) { + pushFlattenedColumns(child, flat); + continue; + } const isPk = pkFieldNames.has(child.name); const fkInfo = fkMap.get(child.name); // Compute the column spec once per field and reuse it for both the column @@ -519,6 +551,9 @@ function renderColumn( // FR-017 Tier 2 — TPH subtype-only column: force nullable (drop .notNull()) // and suppress any DB default (other-subtype rows must stay NULL here). forceNullable: boolean = false, + // `field.object @storage: flattened`: the Drizzle KEY of a member's column (`shipToStreet`); + // absent, the key is the field's own name. + keyName: string = field.name, ): Code { // An int-backed field.enum's column function is a LOCAL generated const (the // customType helper emitted into this same file), so it must not be imported @@ -739,7 +774,7 @@ function renderColumn( } } - const columnLine = code` ${field.name}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; + const columnLine = code` ${keyName}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; return spec.leadingComment !== undefined ? code` // ${spec.leadingComment}\n${columnLine}` : columnLine; diff --git a/examples/showcase/codegen/templates/entity-file.ts b/examples/showcase/codegen/templates/entity-file.ts index b448ae1a5..7e460dbc6 100644 --- a/examples/showcase/codegen/templates/entity-file.ts +++ b/examples/showcase/codegen/templates/entity-file.ts @@ -6,6 +6,8 @@ // !hasWritableRdbSource(entity) → renderValueObjectFile (in-memory / transit shape: interface + Zod schema) // vanilla / write-through entity → Drizzle table path +import { CodegenError } from "./errors.js"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { SOURCE_ROLE_REPLICA } from "@metaobjectsdev/metadata"; @@ -137,6 +139,16 @@ export function renderEntityFile( // compile error). A base+write-through combo keeps the TPH polymorphic read path (reads // the base table); routing its reads through a replica view is a documented non-goal. const writeThrough = isWriteThrough(entity) && !tphBase; + // A write-through entity reads through a replica view whose columns are declared per field; + // that declaration has no flattened-member form yet, and a view column bound under the wrong + // name fails at query time, so refuse rather than emit it. + if (writeThrough && entity.fields().some((f) => flattenedMembersOf(f, ctx.columnNamingStrategy) !== undefined)) { + throw new CodegenError( + `${entity.name}: a write-through entity (writable table plus a replica view) cannot hold a ` + + `field.object @storage: flattened field yet - the replica view declaration has no flattened-member form. ` + + `Use @storage: jsonb, or drop the replica view.`, + ); + } // Rendered BEFORE the replica view below, for two reasons that both matter: the table // DECLARES the int-backed-enum codecs (a `const` the view's declaration would duplicate, // and a duplicate module-scope const is a parse error), and those consts must precede the diff --git a/examples/showcase/codegen/templates/errors.ts b/examples/showcase/codegen/templates/errors.ts new file mode 100644 index 000000000..71b2b751b --- /dev/null +++ b/examples/showcase/codegen/templates/errors.ts @@ -0,0 +1,10 @@ +// Typed errors for codegen-ts. + +export class CodegenError extends Error { + readonly file?: string; + constructor(message: string, opts?: { file?: string }) { + super(message); + this.name = "CodegenError"; + if (opts?.file !== undefined) this.file = opts.file; + } +} diff --git a/examples/showcase/codegen/templates/filter-allowlist.ts b/examples/showcase/codegen/templates/filter-allowlist.ts index ec7c195b7..fd7406700 100644 --- a/examples/showcase/codegen/templates/filter-allowlist.ts +++ b/examples/showcase/codegen/templates/filter-allowlist.ts @@ -1,3 +1,4 @@ +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -114,8 +115,7 @@ function valueShape(f: MetaField): string { function filterableFields(entity: MetaObject, exclude?: string): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in allowlists. - return entity - .fields() + return flatSurfaceFields(entity) .filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true && c.name !== exclude); } diff --git a/examples/showcase/codegen/templates/filter-shared.ts b/examples/showcase/codegen/templates/filter-shared.ts index 1ddcb3162..c4de261bb 100644 --- a/examples/showcase/codegen/templates/filter-shared.ts +++ b/examples/showcase/codegen/templates/filter-shared.ts @@ -14,6 +14,7 @@ // renders clickable for a field the endpoint rejects, or a grid whose initial sort // runs the opposite way from the endpoint it queries. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject, type MetaData } from "@metaobjectsdev/metadata"; import { FIELD_ATTR_FILTERABLE, @@ -47,7 +48,7 @@ export function isSortableField(field: MetaField): boolean { */ export function sortableFields(entity: MetaObject): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in sort ops. - return entity.fields().filter(isSortableField); + return flatSurfaceFields(entity).filter(isSortableField); } /** @@ -110,6 +111,6 @@ export function resolveGridDefaultSort( const declared = layout.attr(LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_ORDER); if (declared === "asc" || declared === "desc") return { field: fieldName, order: declared }; - const field = entity.fields().find((f) => f.name === fieldName); + const field = flatSurfaceFields(entity).find((f) => f.name === fieldName); return { field: fieldName, order: field === undefined ? "asc" : sortDefaultOrder(field) }; } diff --git a/examples/showcase/codegen/templates/filter-type.ts b/examples/showcase/codegen/templates/filter-type.ts index e2fd030d3..d3a06b53f 100644 --- a/examples/showcase/codegen/templates/filter-type.ts +++ b/examples/showcase/codegen/templates/filter-type.ts @@ -2,6 +2,7 @@ // String fields get eq/ne/in/like; numbers get eq/ne/gt/gte/lt/lte/in; booleans get eq/isNull only; // datetimes get eq/ne/gt/gte/lt/lte/in. An enum's operand type is its member union. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -82,7 +83,7 @@ function renderFieldUnion(field: MetaField): string { export function renderFilterType(entity: MetaObject, exclude?: string): Code { const filterName = entityTypeNames(entity).filter; // fields() returns effective fields, so inherited fields (from extends:/super:) are included in filter types. - const allFields = entity.fields().filter((c) => c.name !== exclude); + const allFields = flatSurfaceFields(entity).filter((c) => c.name !== exclude); const filterableFieldsList = allFields.filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true); // Sort union uses isSortableField — same predicate as renderSortAllowlist to prevent // client/server mismatches (@filterable: true + @sortable: false must be excluded from both). diff --git a/examples/showcase/codegen/templates/inferred-types.ts b/examples/showcase/codegen/templates/inferred-types.ts index 4b87f2f9b..def5fb049 100644 --- a/examples/showcase/codegen/templates/inferred-types.ts +++ b/examples/showcase/codegen/templates/inferred-types.ts @@ -7,6 +7,7 @@ // optional fields (matching Zod's `.optional()` inference — `T | undefined` — // without the superfluous `| null` that Drizzle nullable columns introduce). +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { @@ -146,7 +147,7 @@ export function renderEnumTypeAliases(entity: MetaObject, ctx?: RenderContext): // and non-colliding value objects keep their bare name (byte-identical). const ownerName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { if (field.subType !== FIELD_SUBTYPE_ENUM) continue; const values = enumValues(field); @@ -402,7 +403,7 @@ export function renderValueObjectInterface(entity: MetaObject, ctx?: RenderConte const objName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; const lines: Code[] = []; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { const required = field.attr(FIELD_ATTR_REQUIRED) === true; const optional = required ? "" : "?"; const undefinedArm = required ? "" : " | undefined"; diff --git a/examples/showcase/codegen/templates/zod-validators.ts b/examples/showcase/codegen/templates/zod-validators.ts index 195fddfac..ec3a5557b 100644 --- a/examples/showcase/codegen/templates/zod-validators.ts +++ b/examples/showcase/codegen/templates/zod-validators.ts @@ -9,6 +9,7 @@ // object-array field collapsed to z.array(z.string()) and the JSON Schema sent // downstream (e.g. to LLM tool_use input_schema) lost the nested object shape. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { entityTypeNames } from "./entity-type-names.js"; import { code, joinCode, imp, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaObject, MetaField, type MetaRoot, stripPackage } from "@metaobjectsdev/metadata"; @@ -104,7 +105,7 @@ function durationHelperSince(since: number): string { * a plain entity has nothing to preserve, so both are omitted. Resolving * (ADR-0039): honors an @autoSet inherited via extends. */ export function hasAutoSetFields(obj: MetaObject): boolean { - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { const autoSet = child.attr(FIELD_ATTR_AUTO_SET); if (autoSet === AUTO_SET_ON_CREATE || autoSet === AUTO_SET_ON_UPDATE) return true; } @@ -121,7 +122,7 @@ export function hasAutoSetFields(obj: MetaObject): boolean { function isTphBaseOwnField(obj: MetaObject, field: MetaField): boolean { const base = tphDiscriminatorBase(obj); if (base === undefined) return false; - return base.fields().some((f) => f.name === field.name); + return flatSurfaceFields(base).some((f) => f.name === field.name); } /** @@ -169,7 +170,7 @@ export function renderTphSubtypeReadSchema(obj: MetaObject, ctx?: RenderContext) const tphPin = tphDiscriminatorPin(obj); const fieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (tphPin !== undefined && child.name === tphPin.fieldName) { fieldLines.push(code` ${child.name}: z.literal(${JSON.stringify(tphPin.value)})`); continue; @@ -319,7 +320,7 @@ function crossComparable(left: MetaField, right: MetaField, opName: string): boo function insertSchemaFieldNameSet(obj: MetaObject): Set { const autoGen = autoGenPkFieldNames(obj); const out = new Set(); - for (const f of obj.fields()) { + for (const f of flatSurfaceFields(obj)) { if (autoGen.has(f.name)) continue; if (isReadOnlyMutability(f) || f.isDerived()) continue; out.add(f.name); @@ -385,7 +386,7 @@ interface RefinementBuild { blocks: string[]; helpers: Set } function crossFieldRefinement(obj: MetaObject): RefinementBuild { const inSchema = insertSchemaFieldNameSet(obj); - const byName = new Map(obj.fields().map((f) => [f.name, f] as const)); + const byName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f] as const)); // A reference must name a field that exists AND is in the insert schema; otherwise skip // the validator (migrate skips on a missing field; a readOnly/derived/auto-PK field is // never in the body, so enforcing on it would reject every create). @@ -599,7 +600,7 @@ export function renderInsertSchemaOnly(obj: MetaObject, ctx?: RenderContext): Co const tphPin = tphDiscriminatorPin(obj); const insertFieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field is populated by DB / replication / // external owner; the application has no path to write it. Exclude from the @@ -684,7 +685,7 @@ export function insertSchemaFields(obj: MetaObject): SchemaFieldShape[] { const assignedPkFields = assignedPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -720,7 +721,7 @@ export function updateSchemaFields(obj: MetaObject): SchemaFieldShape[] { const autoGenPkFields = autoGenPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -767,7 +768,7 @@ export function renderZodValidators(obj: MetaObject, ctx?: RenderContext): Code // transform). Only emitted when the entity declares @autoSet fields. const preservingFieldLines: Code[] = []; const emitPreserving = hasAutoSetFields(obj); - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field appears in neither InsertSchema // nor UpdateSchema. The DB / trigger / replication owns the write path; the app diff --git a/fixtures/generator-registry-conformance/registry.json b/fixtures/generator-registry-conformance/registry.json index 44db6ebcf..bbbc65138 100644 --- a/fixtures/generator-registry-conformance/registry.json +++ b/fixtures/generator-registry-conformance/registry.json @@ -363,6 +363,7 @@ "codegen-ts/src/enum-import.ts", "codegen-ts/src/enum-meta.ts", "codegen-ts/src/enum-shared.ts", + "codegen-ts/src/flattened.ts", "codegen-ts/src/format.ts", "codegen-ts/src/generator.ts", "codegen-ts/src/generators/agent-schema-input.ts", diff --git a/server/typescript/packages/codegen-ts-react/src/templates/form-file.ts b/server/typescript/packages/codegen-ts-react/src/templates/form-file.ts index 4fd8c0dd9..bd49fe392 100644 --- a/server/typescript/packages/codegen-ts-react/src/templates/form-file.ts +++ b/server/typescript/packages/codegen-ts-react/src/templates/form-file.ts @@ -61,6 +61,7 @@ import { VIEW_CONTEXT_FORM, effectivePackage, isCredentialField, + flatSurfaceFields, } from "@metaobjectsdev/codegen-ts/engine"; import { humanize, valueObjectFor, numericInputControl } from "@metaobjectsdev/codegen-ts/templates/field-meta"; import { tphDiscriminatorPin } from "@metaobjectsdev/codegen-ts/engine"; @@ -96,7 +97,7 @@ function visibleFields(entity: MetaObject, discField?: string): MetaField[] { const pkNames = primaryFieldNames(entity); const out: MetaField[] = []; // fields() returns effective fields, so inherited fields (from extends:/super:) are included in forms. - for (const child of entity.fields()) { + for (const child of flatSurfaceFields(entity)) { // ADR-0039: resolving — a field may inherit @formExclude via extends. if (child.attr(FIELD_ATTR_FORM_EXCLUDE) === true) continue; // A credential is never edited through a generated form: the generated write route diff --git a/server/typescript/packages/codegen-ts-react/test/form-flattened-storage.test.ts b/server/typescript/packages/codegen-ts-react/test/form-flattened-storage.test.ts new file mode 100644 index 000000000..59820bb6f --- /dev/null +++ b/server/typescript/packages/codegen-ts-react/test/form-flattened-storage.test.ts @@ -0,0 +1,42 @@ +// #524: a `field.object @storage: flattened` is flat on the write surface (`shipToStreet`), so the +// generated form must bind the member keys the InsertSchema accepts, not a nested `shipTo` it strips. +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, type MetaObject } from "@metaobjectsdev/metadata"; +import { renderFormFile } from "../src/templates/form-file.js"; +import { makeRenderContext, buildPkMap, buildRelationMap } from "@metaobjectsdev/codegen-ts"; + +describe("generated form — flattened value object", () => { + test("binds each member under its flat key and no nested path", async () => { + const { root, errors } = await new MetaDataLoader().load([ + new InMemoryStringSource( + JSON.stringify({ + "metadata.root": { + package: "demo", + children: [ + { "object.value": { name: "Address", children: [{ "field.string": { name: "street" } }, { "field.string": { name: "city" } }] } }, + { "object.entity": { name: "Shipment", children: [ + { "source.rdb": { "@table": "shipments" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "id", "@fields": "id", "@generation": "increment" } }, + { "field.object": { name: "shipTo", "@objectRef": "Address", "@storage": "flattened" } }, + ] } }, + ], + }, + }), + { id: "shipment.json" }, + ), + ]); + expect(errors).toEqual([]); + const shipment = root.objects().find((o) => o.name === "Shipment")! as MetaObject; + const out = renderFormFile( + shipment, + makeRenderContext({ + dialect: "postgres", loadedRoot: root, outDir: "/x", dbImport: "../db", extStyle: "none", + apiPrefix: "/api", pkMap: buildPkMap(root), relationMap: buildRelationMap(root), + }), + ); + expect(out).toContain("form.input.shipToStreet"); + expect(out).toContain("form.input.shipToCity"); + expect(out).not.toMatch(/\bshipTo\b/); + }); +}); diff --git a/server/typescript/packages/codegen-ts-tanstack/src/templates/columns-file.ts b/server/typescript/packages/codegen-ts-tanstack/src/templates/columns-file.ts index 96d2d486a..39dfadc42 100644 --- a/server/typescript/packages/codegen-ts-tanstack/src/templates/columns-file.ts +++ b/server/typescript/packages/codegen-ts-tanstack/src/templates/columns-file.ts @@ -17,6 +17,7 @@ import { viewForContext, VIEW_CONTEXT_GRID, effectivePackage, + flatSurfaceFields, } from "@metaobjectsdev/codegen-ts/engine"; import { isSortableField, @@ -102,7 +103,7 @@ function extractGrids(entity: MetaObject, tph?: TphGridInfo): GridSpec[] { // For a TPH base, also index the subtype-only fields so they can be folded // into the polymorphic grid's column set. const fieldsByName = new Map( - [...entity.fields(), ...(tph?.subtypeFields ?? [])].map((f) => [f.name, f] as const), + [...flatSurfaceFields(entity), ...(tph?.subtypeFields ?? [])].map((f) => [f.name, f] as const), ); const grids: GridSpec[] = []; diff --git a/server/typescript/packages/codegen-ts/src/api-surface.ts b/server/typescript/packages/codegen-ts/src/api-surface.ts index e587b27f5..98c3c4695 100644 --- a/server/typescript/packages/codegen-ts/src/api-surface.ts +++ b/server/typescript/packages/codegen-ts/src/api-surface.ts @@ -42,6 +42,7 @@ import { type MetaObject, type MetaRoot, } from "@metaobjectsdev/metadata"; +import { flatSurfaceFields } from "./flattened.js"; import { isAbstract } from "./instance-artifacts.js"; import { isProjection } from "./projection/projection-detector.js"; import { hasAnyRdbSource, hasWritableRdbSource, isReport, servedReport } from "./source-detect.js"; @@ -145,7 +146,7 @@ export function servesClientTier(entity: MetaObject): boolean { export function decimalFieldNames(entity: MetaObject): readonly string[] { // ADR-0039: resolving. A decimal may be inherited via extends, and a report read model's // derived ratio, average or sum is a field of the report itself. - return entity.fields().filter((f) => f.subType === FIELD_SUBTYPE_DECIMAL).map((f) => f.name); + return flatSurfaceFields(entity).filter((f) => f.subType === FIELD_SUBTYPE_DECIMAL).map((f) => f.name); } /** @@ -182,8 +183,8 @@ function markedFieldNames( hierarchy: readonly MetaObject[], isMarked: (f: MetaField) => boolean, ): string[] { - const marked = new Set(hierarchy.flatMap((o) => o.fields().filter(isMarked).map((f) => f.name))); - return [...new Set(objects.flatMap((o) => o.fields().map((f) => f.name)))].filter((n) => marked.has(n)); + const marked = new Set(hierarchy.flatMap((o) => flatSurfaceFields(o).filter(isMarked).map((f) => f.name))); + return [...new Set(objects.flatMap((o) => flatSurfaceFields(o).map((f) => f.name)))].filter((n) => marked.has(n)); } /** The objects that share `obj`'s table: a TPH base and every subtype, else `obj` alone. */ @@ -244,7 +245,7 @@ function shapeOf(field: MetaData, vo: MetaData, path: readonly MetaData[]): Valu /** The jsonb columns of `entity` that hold a value object (or an array / map of them), with their declared shape. */ export function valueObjectShapes(entity: MetaObject): Record { const out: Record = {}; - for (const f of entity.fields()) { + for (const f of flatSurfaceFields(entity)) { const vo = valueObjectOf(f); if (vo !== undefined) out[f.name] = shapeOf(f, vo, [vo]); } diff --git a/server/typescript/packages/codegen-ts/src/contract/build-openapi.ts b/server/typescript/packages/codegen-ts/src/contract/build-openapi.ts index 0eb12235b..4afe28067 100644 --- a/server/typescript/packages/codegen-ts/src/contract/build-openapi.ts +++ b/server/typescript/packages/codegen-ts/src/contract/build-openapi.ts @@ -7,6 +7,7 @@ // are left out, and so are the `405` refusals on a read-only surface, which are not operations a // caller can use (the api docs omit them for the same reason). +import { flatSurfaceFields } from "../flattened.js"; import { type MetaField, type MetaObject, type MetaRoot, opsForField } from "@metaobjectsdev/metadata"; import { FIELD_ATTR_FILTERABLE, FIELD_SUBTYPE_ENUM, FIELD_SUBTYPE_TIMESTAMP, FILTER_OP_IN, FILTER_OP_IS_NULL, FILTER_OP_LIKE } from "@metaobjectsdev/metadata/constants"; import { hasItemRoute, itemRouteField, servedPath, servesReadApi } from "../api-surface.js"; @@ -119,7 +120,7 @@ function filterValueSchema(field: MetaField, op: string): JsonSchema { /** One parameter per filterable field and operator, so removing either is a visible change. */ function filterParameters(obj: MetaObject, excludeField: string | undefined): JsonSchema[] { const params: JsonSchema[] = []; - const filterable = obj.fields().filter((f) => f.attr(FIELD_ATTR_FILTERABLE) === true && f.name !== excludeField); + const filterable = flatSurfaceFields(obj).filter((f) => f.attr(FIELD_ATTR_FILTERABLE) === true && f.name !== excludeField); for (const field of filterable) { const ops = opsForField(field); if (ops.length === 0) continue; diff --git a/server/typescript/packages/codegen-ts/src/contract/object-schema.ts b/server/typescript/packages/codegen-ts/src/contract/object-schema.ts index de217b58c..82af42b25 100644 --- a/server/typescript/packages/codegen-ts/src/contract/object-schema.ts +++ b/server/typescript/packages/codegen-ts/src/contract/object-schema.ts @@ -20,6 +20,7 @@ import { } from "@metaobjectsdev/metadata"; import { DOC_ATTR_DESCRIPTION, OBJECT_SUBTYPE_VALUE } from "@metaobjectsdev/metadata/constants"; import { hiddenFieldNames, servesWriteApi, tphHierarchy, writeDeniedFieldNames } from "../api-surface.js"; +import { flatSurfaceFields } from "../flattened.js"; import { isAbstract } from "../instance-artifacts.js"; import { tphPlan } from "../templates/tph-discriminator.js"; import { insertSchemaFields, updateSchemaFields } from "../templates/zod-validators.js"; @@ -77,7 +78,7 @@ function readSchema(obj: MetaObject, env: ObjectSchemaEnv, wire: WireFormat): Js const properties: Record = {}; const required: string[] = []; // A credential is never on the wire: the key is absent, not null. - for (const field of obj.fields().filter((f) => !hidden.has(f.name))) { + for (const field of flatSurfaceFields(obj).filter((f) => !hidden.has(f.name))) { if (pin !== undefined && field.name === pin.fieldName) { properties[field.name] = { const: pin.value }; required.push(field.name); @@ -110,7 +111,7 @@ function isClientWritten(field: MetaField | undefined, serverSet: boolean | unde } function createSchema(obj: MetaObject, env: ObjectSchemaEnv, wire: WireFormat): JsonSchema { - const fieldByName = new Map(obj.fields().map((f) => [f.name, f])); + const fieldByName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f])); const denied = writeDenied(obj, env); const properties: Record = {}; const required: string[] = []; @@ -125,7 +126,7 @@ function createSchema(obj: MetaObject, env: ObjectSchemaEnv, wire: WireFormat): } function updateSchema(obj: MetaObject, env: ObjectSchemaEnv, wire: WireFormat): JsonSchema { - const fieldByName = new Map(obj.fields().map((f) => [f.name, f])); + const fieldByName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f])); const pk = new Set(obj.primaryIdentity()?.fields ?? []); const denied = writeDenied(obj, env); const properties: Record = {}; diff --git a/server/typescript/packages/codegen-ts/src/engine-surface.ts b/server/typescript/packages/codegen-ts/src/engine-surface.ts index 5d90f1c4c..9612e3183 100644 --- a/server/typescript/packages/codegen-ts/src/engine-surface.ts +++ b/server/typescript/packages/codegen-ts/src/engine-surface.ts @@ -35,6 +35,7 @@ export const ENGINE_MODULES: readonly string[] = [ "codegen-ts/src/enum-import.ts", "codegen-ts/src/enum-meta.ts", "codegen-ts/src/enum-shared.ts", + "codegen-ts/src/flattened.ts", "codegen-ts/src/format.ts", "codegen-ts/src/generator.ts", "codegen-ts/src/generators/agent-schema-input.ts", diff --git a/server/typescript/packages/codegen-ts/src/engine.ts b/server/typescript/packages/codegen-ts/src/engine.ts index 47ef2cb8a..edffc9954 100644 --- a/server/typescript/packages/codegen-ts/src/engine.ts +++ b/server/typescript/packages/codegen-ts/src/engine.ts @@ -22,6 +22,7 @@ export * from "./docs-run.js"; export * from "./enum-import.js"; export * from "./enum-meta.js"; export * from "./enum-shared.js"; +export * from "./flattened.js"; export * from "./format.js"; export * from "./generator.js"; export * from "./generators/agent-schema-input.js"; diff --git a/server/typescript/packages/codegen-ts/src/flattened.ts b/server/typescript/packages/codegen-ts/src/flattened.ts new file mode 100644 index 000000000..9c6ebe03b --- /dev/null +++ b/server/typescript/packages/codegen-ts/src/flattened.ts @@ -0,0 +1,109 @@ +// `field.object @storage: flattened` — the one place codegen-ts decides what a flattened value +// object looks like in the Drizzle table and in the schemas that validate its writes. +// +// The columns are the ones `meta migrate` creates: one per member of the value object, named +// by core naming (`resolveFlattenedColumnName`), and none for the field itself. The generated +// surface is FLAT, the same surface the Prisma generator exposes: each member is a key +// `` (`shipToStreet`) on the Drizzle table, the Row/Insert types, the Zod +// schemas and the filter allowlist. + +import { + isMetaObject, + refTarget, + resolveFlattenedColumnName, + physicalNamingOf, + FIELD_SUBTYPE_OBJECT, + FIELD_ATTR_OBJECT_REF, + FIELD_ATTR_STORAGE, + FIELD_ATTR_FORM_EXCLUDE, + FIELD_ATTR_MUTABILITY, + MUTABILITY_READ_ONLY, + MUTABILITY_WRITE_ONCE, + fieldMutability, + STORAGE_FLATTENED, + type MetaField, + type MetaObject, +} from "@metaobjectsdev/metadata"; +import type { ColumnNamingStrategy } from "./metaobjects-config.js"; +import { hasWritableRdbSource } from "./source-detect.js"; +import { tphDiscriminatorBase, tphStorageObject } from "./tph.js"; + +export interface FlattenedMember { + /** The value object's own field. */ + readonly member: MetaField; + /** The member presented under its flat key: what the schemas, types and codec names are derived from. */ + readonly flat: MetaField; + /** The key of the member on the flat surface: `shipToStreet`. */ + readonly key: string; + /** The member's column in the database: `ship_to_street`. */ + readonly column: string; +} + +function upperFirst(s: string): string { + return s.length === 0 ? s : s.charAt(0).toUpperCase() + s.slice(1); +} + +/** True for a `field.object` whose storage is `flattened` (ADR-0039: resolving read). */ +export function isFlattenedObjectField(field: MetaField): boolean { + return field.subType === FIELD_SUBTYPE_OBJECT && field.attr(FIELD_ATTR_STORAGE) === STORAGE_FLATTENED; +} + +/** The members of a flattened field, or undefined when it is not flattened (or names no value object). */ +export function flattenedMembersOf(field: MetaField, strategy: ColumnNamingStrategy): FlattenedMember[] | undefined { + if (!isFlattenedObjectField(field)) return undefined; + const valueObject = refTarget(field, FIELD_ATTR_OBJECT_REF); + if (valueObject === undefined || !isMetaObject(valueObject)) return undefined; + // ADR-0039: effective fields — members inherited via extends are columns too (as migrate does). + return valueObject.fields().map((member) => { + const key = field.name + upperFirst(member.name); + return { member, flat: asFlatKey(member, key, field), key, column: resolveFlattenedColumnName(field, member, strategy) }; + }); +} + +const MUTABILITY_TIGHTNESS = [MUTABILITY_READ_ONLY, MUTABILITY_WRITE_ONCE] as const; + +/** + * A member field presented under its flat key: it behaves as the value object's own field in + * every way (subtype, attributes, validators) except that its `name` is ``, + * which is what the Zod schema and the filter allowlist key it by, and that the write + * protection of the flattened `parent` (`@mutability`, `@formExclude`) applies to every member. + */ +function asFlatKey(member: MetaField, key: string, parent: MetaField): MetaField { + const attr = (name: string): unknown => { + if (name === FIELD_ATTR_FORM_EXCLUDE && parent.attr(FIELD_ATTR_FORM_EXCLUDE) === true) return true; + if (name === FIELD_ATTR_MUTABILITY) { + const modes = [fieldMutability(parent), fieldMutability(member)]; + return MUTABILITY_TIGHTNESS.find((mode) => modes.includes(mode)) ?? member.attr(name); + } + return member.attr(name); + }; + return new Proxy(member, { + get(target, prop) { + if (prop === "name") return key; + if (prop === "attr") return attr; + const value = Reflect.get(target, prop, target) as unknown; + return typeof value === "function" ? (value as (...a: unknown[]) => unknown).bind(target) : value; + }, + }); +} + +/** + * `obj.fields()` as the flat surface sees them: each flattened `field.object` replaced by its + * members, keyed ``. Identical to `obj.fields()` for an object with none. + * + * Only a table-backed object flattens: a value object has no table, so a flattened field it + * declares is nested jsonb when embedded. A field a TPH subtype declares beyond its base is + * folded into the shared table as one jsonb column (`meta migrate` does not expand it), so it + * stays a single key too. + */ +export function flatSurfaceFields(obj: MetaObject): MetaField[] { + if (!hasWritableRdbSource(tphStorageObject(obj))) return obj.fields(); + const strategy = physicalNamingOf(obj).columns; + const tphBase = tphDiscriminatorBase(obj); + const baseFieldNames = tphBase === undefined ? undefined : new Set(tphBase.fields().map((f) => f.name)); + return obj.fields().flatMap((f) => { + if (baseFieldNames !== undefined && !baseFieldNames.has(f.name)) return [f]; + const members = flattenedMembersOf(f, strategy); + return members === undefined ? [f] : members.map((m) => m.flat); + }); +} diff --git a/server/typescript/packages/codegen-ts/src/generators/agent-ui-page.ts b/server/typescript/packages/codegen-ts/src/generators/agent-ui-page.ts index 59558098a..410215ed4 100644 --- a/server/typescript/packages/codegen-ts/src/generators/agent-ui-page.ts +++ b/server/typescript/packages/codegen-ts/src/generators/agent-ui-page.ts @@ -40,6 +40,7 @@ import { TYPE_LAYOUT, } from "@metaobjectsdev/metadata"; import type { MetaData, MetaObject, MetaRoot } from "@metaobjectsdev/metadata"; +import { flatSurfaceFields } from "../flattened.js"; import { GENERATED_HEADER } from "../constants.js"; import { generatableObjects, isReport } from "../source-detect.js"; import { hasGeneratedForm, restPath, servedPath, servesClientHooks } from "../api-surface.js"; @@ -252,7 +253,7 @@ export function renderAgentUiPage(root: MetaRoot, apiPrefix = ""): string { if (descriptor.fields.length > 0) { // The field nodes, keyed by name, so the three non-descriptor columns can be read // off the field the descriptor row came from. - const byName = new Map(obj.fields().map((f) => [f.name, f])); + const byName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f])); out.push("| Field | Label | Control | HTML type | Rules | Excluded | Filter | Sort |"); out.push("|---|---|---|---|---|---|---|---|"); for (const f of descriptor.fields) { diff --git a/server/typescript/packages/codegen-ts/src/generators/api-field-shape.ts b/server/typescript/packages/codegen-ts/src/generators/api-field-shape.ts index aee2c50bf..9dce8489c 100644 --- a/server/typescript/packages/codegen-ts/src/generators/api-field-shape.ts +++ b/server/typescript/packages/codegen-ts/src/generators/api-field-shape.ts @@ -18,6 +18,7 @@ // `fieldTsTypeString` mapping the VO interface emitter uses. import { type MetaData, type MetaObject, type MetaField, isMetaObject, refTarget } from "@metaobjectsdev/metadata"; +import { flatSurfaceFields } from "../flattened.js"; import { fieldTsTypeString } from "../templates/inferred-types.js"; import { insertSchemaFields, @@ -49,7 +50,7 @@ export interface FieldShape { */ export function modelFieldShapes(obj: MetaObject): FieldShape[] { // A credential is not on the wire, so it is not in the documented model either. - return fieldShapesOf(obj, obj.fields().filter((f) => !isCredentialField(f))); + return fieldShapesOf(obj, flatSurfaceFields(obj).filter((f) => !isCredentialField(f))); } function fieldShapesOf(obj: MetaObject, fields: readonly MetaField[]): FieldShape[] { @@ -69,7 +70,7 @@ function fieldShapesOf(obj: MetaObject, fields: readonly MetaField[]): FieldShap * pairing each schema field with its TS type (same `fieldTsTypeString` map). A * TPH-pinned discriminator field documents the literal as its type. */ function shapesFromSchemaFields(obj: MetaObject, walk: SchemaFieldShape[]): FieldShape[] { - const fieldByName = new Map(obj.fields().map((f) => [f.name, f])); + const fieldByName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f])); // The generated routes strip a write-denied field from the body, so it is not accepted. const accepted = walk.filter((sf) => { const f = fieldByName.get(sf.name); diff --git a/server/typescript/packages/codegen-ts/src/templates/drizzle-schema.ts b/server/typescript/packages/codegen-ts/src/templates/drizzle-schema.ts index 0431389d7..bda925153 100644 --- a/server/typescript/packages/codegen-ts/src/templates/drizzle-schema.ts +++ b/server/typescript/packages/codegen-ts/src/templates/drizzle-schema.ts @@ -33,6 +33,7 @@ import { renderRelationsBlock } from "./relations-block.js"; import { renderDocsFor } from "./jsdoc.js"; import { collectTphSubtypeFields, tphConcreteSubtypes } from "./tph-discriminator.js"; import { tphStorageObject } from "../tph.js"; +import { flattenedMembersOf } from "../flattened.js"; import { effectivePackage } from "../docs-paths.js"; /** @@ -215,12 +216,43 @@ export function renderDrizzleSchema( const enumIntTypes = new Map(); // The field.duration codec: one declaration serves every duration column of the table. let usesDurationCodec = false; + // One column per member of a flattened value object, keyed `` and named by + // core naming (`resolveFlattenedColumnName`), the name `meta migrate` creates; the + // `Names` constant when the artifact carries it. No CHECK constraint: migrate derives + // CHECKs from the entity's own fields only. + const pushFlattenedColumns = ( + owner: MetaField, + members: NonNullable>, + ): void => { + for (const m of members) { + const base = mapColumnType(m.flat, ctx.dialect, ctx.columnNamingStrategy, ctx.timestampMode); + const spec: ColumnSpec = { ...base, dbName: m.column }; + if (spec.enumIntCustomType !== undefined) { + enumIntTypes.set(spec.enumIntCustomType.fnConstName, spec.enumIntCustomType); + } + if (spec.durationCustomType !== undefined) usesDurationCodec = true; + const named = names?.resolved.fields[owner.name]?.members?.[m.member.name]; + const dbExpr: Code = names !== undefined && named !== undefined + ? code`${names.symbol}.fields.${owner.name}.members.${m.member.name}.column` + : code`${JSON.stringify(m.column)}`; + columnLines.push( + renderColumn(spec, dbExpr, m.member, ctx, false, undefined, undefined, isComposite, pkg, obj.name, false, m.key), + ); + } + }; for (const child of obj.fields()) { // #213 — a derived (origin-bearing) field is read-only, materialized on the // read (view) side, NOT a column on the entity's write table (FR-024 §7). // Emitting it here would declare a Drizzle column for a table column migrate // no longer creates. if (child.isDerived()) continue; + // `field.object @storage: flattened` holds one column per member of its value object and + // none of its own - exactly the columns `meta migrate` creates (flattened.ts). + const flat = flattenedMembersOf(child, ctx.columnNamingStrategy); + if (flat !== undefined) { + pushFlattenedColumns(child, flat); + continue; + } const isPk = pkFieldNames.has(child.name); const fkInfo = fkMap.get(child.name); // Compute the column spec once per field and reuse it for both the column @@ -519,6 +551,9 @@ function renderColumn( // FR-017 Tier 2 — TPH subtype-only column: force nullable (drop .notNull()) // and suppress any DB default (other-subtype rows must stay NULL here). forceNullable: boolean = false, + // `field.object @storage: flattened`: the Drizzle KEY of a member's column (`shipToStreet`); + // absent, the key is the field's own name. + keyName: string = field.name, ): Code { // An int-backed field.enum's column function is a LOCAL generated const (the // customType helper emitted into this same file), so it must not be imported @@ -739,7 +774,7 @@ function renderColumn( } } - const columnLine = code` ${field.name}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; + const columnLine = code` ${keyName}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; return spec.leadingComment !== undefined ? code` // ${spec.leadingComment}\n${columnLine}` : columnLine; diff --git a/server/typescript/packages/codegen-ts/src/templates/entity-file.ts b/server/typescript/packages/codegen-ts/src/templates/entity-file.ts index fec28c46e..935964130 100644 --- a/server/typescript/packages/codegen-ts/src/templates/entity-file.ts +++ b/server/typescript/packages/codegen-ts/src/templates/entity-file.ts @@ -6,6 +6,8 @@ // !hasWritableRdbSource(entity) → renderValueObjectFile (in-memory / transit shape: interface + Zod schema) // vanilla / write-through entity → Drizzle table path +import { CodegenError } from "../errors.js"; +import { flattenedMembersOf } from "../flattened.js"; import { code, imp, joinCode, type Code } from "ts-poet"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { SOURCE_ROLE_REPLICA } from "@metaobjectsdev/metadata"; @@ -137,6 +139,16 @@ export function renderEntityFile( // compile error). A base+write-through combo keeps the TPH polymorphic read path (reads // the base table); routing its reads through a replica view is a documented non-goal. const writeThrough = isWriteThrough(entity) && !tphBase; + // A write-through entity reads through a replica view whose columns are declared per field; + // that declaration has no flattened-member form yet, and a view column bound under the wrong + // name fails at query time, so refuse rather than emit it. + if (writeThrough && entity.fields().some((f) => flattenedMembersOf(f, ctx.columnNamingStrategy) !== undefined)) { + throw new CodegenError( + `${entity.name}: a write-through entity (writable table plus a replica view) cannot hold a ` + + `field.object @storage: flattened field yet - the replica view declaration has no flattened-member form. ` + + `Use @storage: jsonb, or drop the replica view.`, + ); + } // Rendered BEFORE the replica view below, for two reasons that both matter: the table // DECLARES the int-backed-enum codecs (a `const` the view's declaration would duplicate, // and a duplicate module-scope const is a parse error), and those consts must precede the diff --git a/server/typescript/packages/codegen-ts/src/templates/entity-ui-descriptor.ts b/server/typescript/packages/codegen-ts/src/templates/entity-ui-descriptor.ts index 4fb4ced2e..b06e87b16 100644 --- a/server/typescript/packages/codegen-ts/src/templates/entity-ui-descriptor.ts +++ b/server/typescript/packages/codegen-ts/src/templates/entity-ui-descriptor.ts @@ -54,6 +54,7 @@ import { toSnakeCase, } from "@metaobjectsdev/metadata"; import { inferViewKind, currencyMetaFor, labelFor, humanize, valueObjectFor, numericInputControl } from "./field-meta.js"; +import { flatSurfaceFields } from "../flattened.js"; import { VIEW_CONTEXT_FORM } from "../view-context.js"; import { enumValues } from "../enum-meta.js"; // `restPath` lives HERE rather than in api-surface.ts, which is where it used to sit and @@ -334,6 +335,6 @@ export function buildEntityUiDescriptor(obj: MetaObject, root?: MetaRoot): Entit path: restPath(obj), // ADR-0039: resolving `fields()` so inherited fields appear, exactly as the emitted // descriptor has always done. - fields: obj.fields().map((f) => buildUiFieldDescriptor(f, root)), + fields: flatSurfaceFields(obj).map((f) => buildUiFieldDescriptor(f, root)), }; } diff --git a/server/typescript/packages/codegen-ts/src/templates/filter-allowlist.ts b/server/typescript/packages/codegen-ts/src/templates/filter-allowlist.ts index c103ed601..b3b17b1a0 100644 --- a/server/typescript/packages/codegen-ts/src/templates/filter-allowlist.ts +++ b/server/typescript/packages/codegen-ts/src/templates/filter-allowlist.ts @@ -1,3 +1,4 @@ +import { flatSurfaceFields } from "../flattened.js"; import { code, type Code } from "ts-poet"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -114,8 +115,7 @@ function valueShape(f: MetaField): string { function filterableFields(entity: MetaObject, exclude?: string): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in allowlists. - return entity - .fields() + return flatSurfaceFields(entity) .filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true && c.name !== exclude); } diff --git a/server/typescript/packages/codegen-ts/src/templates/filter-shared.ts b/server/typescript/packages/codegen-ts/src/templates/filter-shared.ts index 1ddcb3162..7ed8ef6da 100644 --- a/server/typescript/packages/codegen-ts/src/templates/filter-shared.ts +++ b/server/typescript/packages/codegen-ts/src/templates/filter-shared.ts @@ -14,6 +14,7 @@ // renders clickable for a field the endpoint rejects, or a grid whose initial sort // runs the opposite way from the endpoint it queries. +import { flatSurfaceFields } from "../flattened.js"; import { MetaField, MetaObject, type MetaData } from "@metaobjectsdev/metadata"; import { FIELD_ATTR_FILTERABLE, @@ -47,7 +48,7 @@ export function isSortableField(field: MetaField): boolean { */ export function sortableFields(entity: MetaObject): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in sort ops. - return entity.fields().filter(isSortableField); + return flatSurfaceFields(entity).filter(isSortableField); } /** @@ -110,6 +111,6 @@ export function resolveGridDefaultSort( const declared = layout.attr(LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_ORDER); if (declared === "asc" || declared === "desc") return { field: fieldName, order: declared }; - const field = entity.fields().find((f) => f.name === fieldName); + const field = flatSurfaceFields(entity).find((f) => f.name === fieldName); return { field: fieldName, order: field === undefined ? "asc" : sortDefaultOrder(field) }; } diff --git a/server/typescript/packages/codegen-ts/src/templates/filter-type.ts b/server/typescript/packages/codegen-ts/src/templates/filter-type.ts index 4cbc0624e..cce239fe8 100644 --- a/server/typescript/packages/codegen-ts/src/templates/filter-type.ts +++ b/server/typescript/packages/codegen-ts/src/templates/filter-type.ts @@ -2,6 +2,7 @@ // String fields get eq/ne/in/like; numbers get eq/ne/gt/gte/lt/lte/in; booleans get eq/isNull only; // datetimes get eq/ne/gt/gte/lt/lte/in. An enum's operand type is its member union. +import { flatSurfaceFields } from "../flattened.js"; import { code, type Code } from "ts-poet"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -82,7 +83,7 @@ function renderFieldUnion(field: MetaField): string { export function renderFilterType(entity: MetaObject, exclude?: string): Code { const filterName = entityTypeNames(entity).filter; // fields() returns effective fields, so inherited fields (from extends:/super:) are included in filter types. - const allFields = entity.fields().filter((c) => c.name !== exclude); + const allFields = flatSurfaceFields(entity).filter((c) => c.name !== exclude); const filterableFieldsList = allFields.filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true); // Sort union uses isSortableField — same predicate as renderSortAllowlist to prevent // client/server mismatches (@filterable: true + @sortable: false must be excluded from both). diff --git a/server/typescript/packages/codegen-ts/src/templates/inferred-types.ts b/server/typescript/packages/codegen-ts/src/templates/inferred-types.ts index d49ccc03c..25a56b86b 100644 --- a/server/typescript/packages/codegen-ts/src/templates/inferred-types.ts +++ b/server/typescript/packages/codegen-ts/src/templates/inferred-types.ts @@ -7,6 +7,7 @@ // optional fields (matching Zod's `.optional()` inference — `T | undefined` — // without the superfluous `| null` that Drizzle nullable columns introduce). +import { flatSurfaceFields } from "../flattened.js"; import { code, imp, joinCode, type Code } from "ts-poet"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { @@ -146,7 +147,7 @@ export function renderEnumTypeAliases(entity: MetaObject, ctx?: RenderContext): // and non-colliding value objects keep their bare name (byte-identical). const ownerName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { if (field.subType !== FIELD_SUBTYPE_ENUM) continue; const values = enumValues(field); @@ -402,7 +403,7 @@ export function renderValueObjectInterface(entity: MetaObject, ctx?: RenderConte const objName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; const lines: Code[] = []; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { const required = field.attr(FIELD_ATTR_REQUIRED) === true; const optional = required ? "" : "?"; const undefinedArm = required ? "" : " | undefined"; diff --git a/server/typescript/packages/codegen-ts/src/templates/zod-validators.ts b/server/typescript/packages/codegen-ts/src/templates/zod-validators.ts index 87edb972d..609942393 100644 --- a/server/typescript/packages/codegen-ts/src/templates/zod-validators.ts +++ b/server/typescript/packages/codegen-ts/src/templates/zod-validators.ts @@ -9,6 +9,7 @@ // object-array field collapsed to z.array(z.string()) and the JSON Schema sent // downstream (e.g. to LLM tool_use input_schema) lost the nested object shape. +import { flatSurfaceFields } from "../flattened.js"; import { entityTypeNames } from "../entity-type-names.js"; import { code, joinCode, imp, type Code } from "ts-poet"; import { MetaObject, MetaField, type MetaRoot, stripPackage } from "@metaobjectsdev/metadata"; @@ -104,7 +105,7 @@ function durationHelperSince(since: number): string { * a plain entity has nothing to preserve, so both are omitted. Resolving * (ADR-0039): honors an @autoSet inherited via extends. */ export function hasAutoSetFields(obj: MetaObject): boolean { - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { const autoSet = child.attr(FIELD_ATTR_AUTO_SET); if (autoSet === AUTO_SET_ON_CREATE || autoSet === AUTO_SET_ON_UPDATE) return true; } @@ -121,7 +122,7 @@ export function hasAutoSetFields(obj: MetaObject): boolean { function isTphBaseOwnField(obj: MetaObject, field: MetaField): boolean { const base = tphDiscriminatorBase(obj); if (base === undefined) return false; - return base.fields().some((f) => f.name === field.name); + return flatSurfaceFields(base).some((f) => f.name === field.name); } /** @@ -169,7 +170,7 @@ export function renderTphSubtypeReadSchema(obj: MetaObject, ctx?: RenderContext) const tphPin = tphDiscriminatorPin(obj); const fieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (tphPin !== undefined && child.name === tphPin.fieldName) { fieldLines.push(code` ${child.name}: z.literal(${JSON.stringify(tphPin.value)})`); continue; @@ -319,7 +320,7 @@ function crossComparable(left: MetaField, right: MetaField, opName: string): boo function insertSchemaFieldNameSet(obj: MetaObject): Set { const autoGen = autoGenPkFieldNames(obj); const out = new Set(); - for (const f of obj.fields()) { + for (const f of flatSurfaceFields(obj)) { if (autoGen.has(f.name)) continue; if (isReadOnlyMutability(f) || f.isDerived()) continue; out.add(f.name); @@ -385,7 +386,7 @@ interface RefinementBuild { blocks: string[]; helpers: Set } function crossFieldRefinement(obj: MetaObject): RefinementBuild { const inSchema = insertSchemaFieldNameSet(obj); - const byName = new Map(obj.fields().map((f) => [f.name, f] as const)); + const byName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f] as const)); // A reference must name a field that exists AND is in the insert schema; otherwise skip // the validator (migrate skips on a missing field; a readOnly/derived/auto-PK field is // never in the body, so enforcing on it would reject every create). @@ -599,7 +600,7 @@ export function renderInsertSchemaOnly(obj: MetaObject, ctx?: RenderContext): Co const tphPin = tphDiscriminatorPin(obj); const insertFieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field is populated by DB / replication / // external owner; the application has no path to write it. Exclude from the @@ -684,7 +685,7 @@ export function insertSchemaFields(obj: MetaObject): SchemaFieldShape[] { const assignedPkFields = assignedPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -720,7 +721,7 @@ export function updateSchemaFields(obj: MetaObject): SchemaFieldShape[] { const autoGenPkFields = autoGenPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -767,7 +768,7 @@ export function renderZodValidators(obj: MetaObject, ctx?: RenderContext): Code // transform). Only emitted when the entity declares @autoSet fields. const preservingFieldLines: Code[] = []; const emitPreserving = hasAutoSetFields(obj); - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field appears in neither InsertSchema // nor UpdateSchema. The DB / trigger / replication owns the write path; the app diff --git a/server/typescript/packages/codegen-ts/test/flattened-storage-e2e.test.ts b/server/typescript/packages/codegen-ts/test/flattened-storage-e2e.test.ts new file mode 100644 index 000000000..408125458 --- /dev/null +++ b/server/typescript/packages/codegen-ts/test/flattened-storage-e2e.test.ts @@ -0,0 +1,423 @@ +// `field.object @storage: "flattened"` (#524). The generated Drizzle table must bind the columns +// `meta migrate` creates (one per member of the value object, none for the field), and the +// generated queries and routes must expose those members as flat keys. +// +// 1. parity: the columns of the generated table, per dialect, are exactly migrate-ts's expected +// schema for the same model; +// 2. round trip: GENERATED entity, queries and routes executed over bun:sqlite. The surface is +// FLAT (as the Prisma generator's): each member is a key `` (`shipToStreet`) +// on the row, the write schemas and the filter allowlist. + +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { Database } from "bun:sqlite"; +import { Hono } from "hono"; +import { drizzle } from "drizzle-orm/bun-sqlite"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema } from "@metaobjectsdev/migrate-ts"; +import { runGen } from "../src/runner.js"; +import { defineConfig, type Dialect } from "../src/metaobjects-config.js"; +import { entityFile } from "../src/reference/entity.js"; +import { queriesFile } from "../src/reference/queries.js"; +import { routesFileHono } from "../src/reference/routes-hono.js"; +import { routesFile } from "../src/reference/routes.js"; +import { buildOpenApiDocument } from "../src/contract/build-openapi.js"; +import { createFieldShapes, modelFieldShapes, updateFieldShapes } from "../src/generators/api-field-shape.js"; +import { writeDeniedFieldNames } from "../src/api-surface.js"; +import { isTphReadNullTolerant } from "../src/templates/zod-validators.js"; +import { flatSurfaceFields } from "../src/flattened.js"; +import { renderAgentUiPage } from "../src/generators/agent-ui-page.js"; + +const META = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [ + { "field.string": { name: "street" } }, + { "field.string": { name: "city", "@filterable": true } }, + { "field.int": { name: "floor" } }, + ] } }, + { "object.entity": { name: "Shipment", children: [ + { "source.rdb": { "@table": "shipments" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "ref", "@required": true } }, + { "field.object": { name: "shipTo", "@objectRef": "Address", "@storage": "flattened" } }, + { "field.object": { name: "billTo", "@objectRef": "Address", "@storage": "flattened", "@required": true } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + ] }, +}); + +async function load(): Promise { + const { root, errors } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(META, { id: "acme.json" })]); + expect(errors).toEqual([]); + return root; +} + +const dirs: string[] = []; +afterAll(() => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); }); + +async function generate(root: MetaRoot, dialect: Dialect, generators: ReturnType[], parent: string): Promise { + const repo = mkdtempSync(join(parent, "flat-")); + dirs.push(repo); + const outDir = join(repo, "out"); + await runGen({ + config: defineConfig({ outDir, dialect, extStyle: "js", dbImport: "../db", generators }), + metadata: root, + projectRoot: repo, + }); + return outDir; +} + +const allTs = (dir: string): string[] => + readdirSync(dir, { recursive: true, encoding: "utf8" }).filter((f) => f.endsWith(".ts")).map((f) => join(dir, f)); + +describe("flattened storage: generated Drizzle columns match migrate's expected schema", () => { + const expectedColumns = async (): Promise => { + const expected = buildExpectedSchema(await load(), { dialect: "postgres" }); + return expected.tables.find((t) => t.name === "shipments")!.columns.map((c) => c.name).sort(); + }; + + test("migrate creates one column per member and none for the field", async () => { + expect(await expectedColumns()).toEqual([ + "bill_to_city", "bill_to_floor", "bill_to_street", "id", "ref", + "ship_to_city", "ship_to_floor", "ship_to_street", + ]); + }); + + for (const dialect of ["postgres", "sqlite", "mysql"] as const) { + test(`${dialect}: the Drizzle table binds exactly those columns`, async () => { + const outDir = await generate(await load(), dialect, [entityFile()], tmpdir()); + const file = allTs(outDir).find((f) => f.endsWith("Shipment.ts"))!; + const source = readFileSync(file, "utf8"); + const table = source.slice(source.indexOf("Table(\"shipments\""), source.indexOf("});", source.indexOf("Table(\"shipments\""))); + const bound = [...table.matchAll(/^\s+\w+: \w+\("([^"]+)"/gm)].map((m) => m[1]!).sort(); + expect(bound).toEqual(await expectedColumns()); + // The issue's defect: a single jsonb column named for the field. + expect(table).not.toContain('"ship_to"'); + expect(table).not.toMatch(/jsonb\(/); + }); + } +}); + +describe("flattened storage: the API contract and documented shapes use the flat keys", () => { + const MEMBER_KEYS = ["shipToCity", "shipToFloor", "shipToStreet"]; + + test("OpenAPI read, create and update schemas list the members, never the nested field", async () => { + const root = await load(); + const shipment = root.findObject("Shipment")!; + const doc = buildOpenApiDocument({ root, objects: [shipment], apiPrefix: "", relationMap: new Map(), reportExport: false }) as { + components: { schemas: Record }> }; + }; + for (const name of ["Shipment", "ShipmentCreate", "ShipmentUpdate"]) { + const keys = Object.keys(doc.components.schemas[name]!.properties); + expect(keys).toEqual(expect.arrayContaining([...MEMBER_KEYS, "billToStreet"])); + expect(keys).not.toContain("shipTo"); + expect(keys).not.toContain("billTo"); + } + }); + + test("documented model and create shapes carry the members with their real types", async () => { + const shipment = (await load()).findObject("Shipment")!; + for (const shapes of [modelFieldShapes(shipment), createFieldShapes(shipment)]) { + const byName = new Map(shapes.map((f) => [f.name, f.type])); + expect(byName.has("shipTo")).toBe(false); + expect(byName.get("shipToStreet")).toContain("string"); + expect(byName.get("shipToFloor")).toContain("number"); + } + }); +}); + +describe("flattened storage: write protection on the flattened field covers every member", () => { + const PROTECTED = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [ + { "field.string": { name: "street" } }, + { "field.string": { name: "city" } }, + ] } }, + { "object.entity": { name: "Account", children: [ + { "source.rdb": { "@table": "accounts" } }, + { "field.long": { name: "id" } }, + { "field.object": { name: "registered", "@objectRef": "Address", "@storage": "flattened", "@mutability": "readOnly" } }, + { "field.object": { name: "founding", "@objectRef": "Address", "@storage": "flattened", "@mutability": "writeOnce" } }, + { "field.object": { name: "internal", "@objectRef": "Address", "@storage": "flattened", "@formExclude": true } }, + { "field.object": { name: "open", "@objectRef": "Address", "@storage": "flattened" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + ] }, + }); + + test("readOnly leaves the members out of create and update, writeOnce out of update, formExclude out of the body", async () => { + const { root } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(PROTECTED, { id: "p.json" })]); + const account = root.findObject("Account")!; + const create = createFieldShapes(account).map((f) => f.name); + const update = updateFieldShapes(account).map((f) => f.name); + expect(create).not.toContain("registeredStreet"); + expect(update).not.toContain("registeredStreet"); + expect(create).toContain("foundingStreet"); + expect(update).not.toContain("foundingStreet"); + expect(create).toContain("openStreet"); + expect(update).toContain("openStreet"); + expect(writeDeniedFieldNames(account)).toEqual(["internalStreet", "internalCity"]); + }); +}); + +describe("flattened storage: a derived value-object member keeps its column", () => { + test("the Drizzle table binds every column migrate creates, derived members included", async () => { + const DERIVED = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Source", children: [ + { "source.rdb": { "@table": "sources" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "label" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + { "object.value": { name: "Tag", children: [ + { "field.string": { name: "text" } }, + { "field.string": { name: "mirror", children: [{ "origin.passthrough": { "@from": "Source.label" } }] } }, + ] } }, + { "object.entity": { name: "Note", children: [ + { "source.rdb": { "@table": "notes" } }, + { "field.long": { name: "id" } }, + { "field.object": { name: "tag", "@objectRef": "Tag", "@storage": "flattened" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + ] }, + }); + const { root, errors } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(DERIVED, { id: "d.json" })]); + expect(errors).toEqual([]); + const expected = buildExpectedSchema(root, { dialect: "postgres" }).tables.find((t) => t.name === "notes")!.columns.map((c) => c.name).sort(); + const outDir = await generate(root, "postgres", [entityFile()], tmpdir()); + const source = readFileSync(allTs(outDir).find((f) => f.endsWith("Note.ts"))!, "utf8"); + const table = source.slice(source.indexOf("Table(\"notes\""), source.indexOf("});", source.indexOf("Table(\"notes\""))); + const bound = [...table.matchAll(/^\s+\w+: \w+\("([^"]+)"/gm)].map((m) => m[1]!).sort(); + expect(bound).toEqual(expected); + }); +}); + +describe("flattened storage: agent/ui.md reports the flat keys with their flags", () => { + test("a filterable member's row is marked filterable and sortable", async () => { + const page = renderAgentUiPage(await load()); + const row = page.split("\n").find((l) => l.startsWith("| `shipToCity`")); + expect(row).toBeDefined(); + const cells = row!.split("|").map((c) => c.trim()); + expect(cells.slice(-4, -1)).toEqual(["", "yes", "yes"]); + }); +}); + +describe("flattened storage: int-backed enum members keep their own codec", () => { + test("a flattened member and a top-level field both named status round-trip with their own member maps", async () => { + const ENUMS = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [ + { "field.enum": { name: "status", "@values": ["near", "far"], "@intValueMap": { near: 10, far: 20 } } }, + ] } }, + { "object.entity": { name: "Order", children: [ + { "source.rdb": { "@table": "orders" } }, + { "field.long": { name: "id" } }, + { "field.enum": { name: "status", "@values": ["open", "closed"], "@intValueMap": { open: 1, closed: 2 } } }, + { "field.object": { name: "shipTo", "@objectRef": "Address", "@storage": "flattened" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + ] }, + }); + const { root, errors } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(ENUMS, { id: "e.json" })]); + expect(errors).toEqual([]); + const outDir = await generate(root, "sqlite", [entityFile(), queriesFile()], import.meta.dir); + const find = (suffix: string) => allTs(outDir).find((f) => f.endsWith(suffix))!; + const sqlite = new Database(":memory:"); + sqlite.run("CREATE TABLE orders (id INTEGER PRIMARY KEY AUTOINCREMENT, status INTEGER, ship_to_status INTEGER)"); + const db = drizzle(sqlite); + const queries = (await import(pathToFileURL(find("Order.queries.ts")).href)) as Record Promise>; + const row = await queries["createOrder"]!(db, { status: "closed", shipToStatus: "far" }); + expect(row).toMatchObject({ status: "closed", shipToStatus: "far" }); + expect(sqlite.query("SELECT status, ship_to_status FROM orders").get()).toEqual({ status: 2, ship_to_status: 20 }); + }); +}); + +describe("flattened storage: a TPH subtype's read nullability follows the column", () => { + test("a required member of a base-declared flattened field is not nullable; a subtype-only column is", async () => { + const TPH = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [{ "field.string": { name: "street", "@required": true } }] } }, + { "object.entity": { name: "Party", "@discriminator": "kind", children: [ + { "source.rdb": { "@table": "parties" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "kind", "@required": true } }, + { "field.object": { name: "hq", "@objectRef": "Address", "@storage": "flattened" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + { "object.entity": { name: "Org", extends: "Party", "@discriminatorValue": "org", children: [ + { "field.string": { name: "motto" } }, + ] } }, + ] }, + }); + const { root, errors } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(TPH, { id: "tph2.json" })]); + expect(errors).toEqual([]); + const org = root.findObject("Org")!; + const byName = new Map(flatSurfaceFields(org).map((f) => [f.name, isTphReadNullTolerant(org, f)])); + expect(byName.get("hqStreet")).toBe(false); + expect(byName.get("motto")).toBe(true); + }); +}); + +describe("flattened storage: a TPH subtype's own flattened field keeps migrate's single column", () => { + test("migrate folds a subtype's field.object as one column, so the table binds one column named for the field", async () => { + const TPH = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [{ "field.string": { name: "street" } }] } }, + { "object.entity": { name: "Party", "@discriminator": "kind", children: [ + { "source.rdb": { "@table": "parties" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "kind", "@required": true } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + { "object.entity": { name: "Org", extends: "Party", "@discriminatorValue": "org", children: [ + { "field.object": { name: "hq", "@objectRef": "Address", "@storage": "flattened" } }, + ] } }, + ] }, + }); + const { root } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(TPH, { id: "tph.json" })]); + const expected = buildExpectedSchema(root, { dialect: "postgres" }); + const expectedColumns = expected.tables.find((t) => t.name === "parties")!.columns.map((c) => c.name).sort(); + const outDir = await generate(root, "postgres", [entityFile()], tmpdir()); + const source = readFileSync(allTs(outDir).find((f) => f.endsWith("Party.ts"))!, "utf8"); + const table = source.slice(source.indexOf("Table(\"parties\""), source.indexOf("});", source.indexOf("Table(\"parties\""))); + const bound = [...table.matchAll(/^\s+\w+: \w+\("([^"]+)"/gm)].map((m) => m[1]!).sort(); + expect(bound).toEqual(expectedColumns); + }); +}); + +describe("flattened storage: a value object has no table, so it does not flatten", () => { + test("a flattened field on an object.value stays one key", async () => { + const VO = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [{ "field.string": { name: "street" } }] } }, + { "object.value": { name: "Contact", children: [ + { "field.object": { name: "home", "@objectRef": "Address", "@storage": "flattened" } }, + ] } }, + ] }, + }); + const { root } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(VO, { id: "vo.json" })]); + const outDir = await generate(root, "sqlite", [entityFile()], import.meta.dir); + const mod = (await import(pathToFileURL(allTs(outDir).find((f) => f.endsWith("Contact.ts"))!).href)) as Record; + const schema = Object.values(mod).find((v) => typeof v === "object" && v !== null && "shape" in v)!; + expect(Object.keys(schema.shape!)).toEqual(["home"]); + }); +}); + +describe("flattened storage: a write-through entity is refused, not emitted broken", () => { + test("the replica view has no flattened-member form", async () => { + const WT = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.value": { name: "Address", children: [{ "field.string": { name: "street" } }] } }, + { "object.entity": { name: "Site", children: [ + { "source.rdb": { "@table": "sites", "@role": "primary" } }, + { "source.rdb": { "@kind": "view", "@view": "v_sites", "@role": "replica" } }, + { "field.long": { name: "id" } }, + { "field.object": { name: "hq", "@objectRef": "Address", "@storage": "flattened" } }, + { "identity.primary": { name: "pk", "@fields": "id", "@generation": "increment" } }, + ] } }, + ] }, + }); + const { root } = await new MetaDataLoader({ strict: true }).load([new InMemoryStringSource(WT, { id: "wt.json" })]); + await expect(generate(root, "sqlite", [entityFile()], tmpdir())).rejects.toThrow(/flattened/); + }); +}); + +describe("flattened storage: generated queries and routes, executed on SQLite", () => { + let queries: Record Promise>; + let app: Hono; + let sqlite: Database; + let db: unknown; + + beforeAll(async () => { + const outDir = await generate( + await load(), "sqlite", + [entityFile({ runtimeImport: "@metaobjectsdev/runtime-ts" }), queriesFile(), routesFileHono({ runtimeImport: "@metaobjectsdev/runtime-ts" }), routesFile({ runtimeImport: "@metaobjectsdev/runtime-ts" })], + import.meta.dir, + ); + sqlite = new Database(":memory:"); + // The DDL `meta migrate` produces for this model (columns asserted equal above). + sqlite.run(`CREATE TABLE shipments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ref TEXT NOT NULL, + ship_to_street TEXT, ship_to_city TEXT, ship_to_floor INTEGER, + bill_to_street TEXT, bill_to_city TEXT, bill_to_floor INTEGER)`); + db = drizzle(sqlite); + const find = (suffix: string) => allTs(outDir).find((f) => f.endsWith(suffix))!; + queries = (await import(pathToFileURL(find("Shipment.queries.ts")).href)) as typeof queries; + const hono = (await import(pathToFileURL(find("Shipment.routes.hono.ts")).href)) as Record void>; + app = new Hono(); + hono["registerShipmentRoutes"]!(app, { db }); + }); + + const post = (body: unknown) => app.request("/shipments", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + + test("create stores the member columns and answers the flat keys", async () => { + const row = await queries["createShipment"]!(db, { ref: "a", shipToStreet: "1 Main", shipToCity: "Oslo", shipToFloor: 3, billToStreet: "2 Side" }); + expect(row).toMatchObject({ ref: "a", shipToStreet: "1 Main", shipToCity: "Oslo", shipToFloor: 3, billToStreet: "2 Side", billToCity: null }); + expect(row).not.toHaveProperty("shipTo"); + expect(sqlite.query("SELECT ship_to_street, ship_to_floor, bill_to_street FROM shipments WHERE ref = 'a'").get()) + .toEqual({ ship_to_street: "1 Main", ship_to_floor: 3, bill_to_street: "2 Side" }); + }); + + test("find, list and update read and write the flat keys", async () => { + const created = (await queries["createShipment"]!(db, { ref: "b", shipToCity: "Rome" })) as { id: number }; + expect(await queries["findShipmentById"]!(db, created.id)).toMatchObject({ shipToCity: "Rome", shipToStreet: null }); + const updated = await queries["updateShipment"]!(db, created.id, { shipToStreet: "new" }); + expect(updated).toMatchObject({ shipToStreet: "new", shipToCity: "Rome" }); + expect(((await queries["listShipments"]!(db)) as unknown[]).length).toBeGreaterThanOrEqual(2); + }); + + test("Hono routes: flat JSON in and out", async () => { + const res = await post({ ref: "r", shipToStreet: "1 Main", shipToFloor: 2 }); + expect(res.status).toBe(201); + const body = (await res.json()) as { id: number }; + expect(body).toMatchObject({ shipToStreet: "1 Main", shipToFloor: 2, shipToCity: null }); + const patched = await app.request(`/shipments/${body.id}`, { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify({ shipToCity: "Bergen" }) }); + expect(await patched.json()).toMatchObject({ shipToStreet: "1 Main", shipToCity: "Bergen" }); + }); + + test("a member of the wrong type is a 400", async () => { + expect((await post({ ref: "bad", billToFloor: "high" })).status).toBe(400); + }); + + test("a flattened member filters and sorts by its flat key", async () => { + await queries["createShipment"]!(db, { ref: "f1", shipToCity: "Zagreb" }); + await queries["createShipment"]!(db, { ref: "f2", shipToCity: "Athens" }); + const filtered = await app.request("/shipments?filter[shipToCity][eq]=Zagreb"); + expect(filtered.status).toBe(200); + expect(((await filtered.json()) as { ref: string }[]).map((r) => r.ref)).toEqual(["f1"]); + const sorted = await app.request("/shipments?filter[shipToCity][in]=Zagreb,Athens&sort=shipToCity:asc"); + expect(sorted.status).toBe(200); + expect(((await sorted.json()) as { ref: string }[]).map((r) => r.ref)).toEqual(["f2", "f1"]); + }); + + test("Fastify routes: flat JSON in and out", async () => { + const outDir = join(dirs[dirs.length - 1]!, "out"); + // fastify is runtime-ts's dev dependency, not codegen-ts's: resolve it from there. + const fastifyPath = Bun.resolveSync("fastify", join(import.meta.dir, "../../runtime-ts")); + const Fastify = ((await import(pathToFileURL(fastifyPath).href)) as { default: () => FastifyLike }).default; + // The Fastify routes import the app's database module; supply the same in-memory one. + const { writeFileSync } = await import("node:fs"); + (globalThis as { __flatDb?: unknown }).__flatDb = db; + writeFileSync(join(outDir, "..", "db.ts"), "export const db = (globalThis as { __flatDb?: unknown }).__flatDb as never;\nexport default db;\n"); + const routes = (await import(pathToFileURL(allTs(outDir).find((f) => f.endsWith("Shipment.routes.ts"))!).href)) as Record Promise>; + const f = Fastify(); + const register = Object.values(routes).find((v) => typeof v === "function")!; + await register(f); + const created = await f.inject({ method: "POST", url: "/shipments", payload: { ref: "fx", shipToStreet: "9 Elm", billToFloor: 4 } }); + expect(created.statusCode).toBe(201); + const body = created.json() as { id: number }; + expect(body).toMatchObject({ shipToStreet: "9 Elm", billToFloor: 4, shipToCity: null }); + const got = await f.inject({ method: "GET", url: `/shipments/${body.id}` }); + expect(got.json()).toEqual(body); + await f.close(); + }); +}); + +interface FastifyLike { + inject: (o: object) => Promise<{ statusCode: number; json: () => unknown }>; + close: () => Promise; +} diff --git a/server/typescript/packages/test-generators/src/templates/drizzle-schema.ts b/server/typescript/packages/test-generators/src/templates/drizzle-schema.ts index 74df06cb0..16457c7da 100644 --- a/server/typescript/packages/test-generators/src/templates/drizzle-schema.ts +++ b/server/typescript/packages/test-generators/src/templates/drizzle-schema.ts @@ -33,6 +33,7 @@ import { renderRelationsBlock } from "./relations-block.js"; import { renderDocsFor } from "./jsdoc.js"; import { collectTphSubtypeFields, tphConcreteSubtypes } from "./tph-discriminator.js"; import { tphStorageObject } from "@metaobjectsdev/codegen-ts/engine"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { effectivePackage } from "@metaobjectsdev/codegen-ts/engine"; /** @@ -215,12 +216,43 @@ export function renderDrizzleSchema( const enumIntTypes = new Map(); // The field.duration codec: one declaration serves every duration column of the table. let usesDurationCodec = false; + // One column per member of a flattened value object, keyed `` and named by + // core naming (`resolveFlattenedColumnName`), the name `meta migrate` creates; the + // `Names` constant when the artifact carries it. No CHECK constraint: migrate derives + // CHECKs from the entity's own fields only. + const pushFlattenedColumns = ( + owner: MetaField, + members: NonNullable>, + ): void => { + for (const m of members) { + const base = mapColumnType(m.flat, ctx.dialect, ctx.columnNamingStrategy, ctx.timestampMode); + const spec: ColumnSpec = { ...base, dbName: m.column }; + if (spec.enumIntCustomType !== undefined) { + enumIntTypes.set(spec.enumIntCustomType.fnConstName, spec.enumIntCustomType); + } + if (spec.durationCustomType !== undefined) usesDurationCodec = true; + const named = names?.resolved.fields[owner.name]?.members?.[m.member.name]; + const dbExpr: Code = names !== undefined && named !== undefined + ? code`${names.symbol}.fields.${owner.name}.members.${m.member.name}.column` + : code`${JSON.stringify(m.column)}`; + columnLines.push( + renderColumn(spec, dbExpr, m.member, ctx, false, undefined, undefined, isComposite, pkg, obj.name, false, m.key), + ); + } + }; for (const child of obj.fields()) { // #213 — a derived (origin-bearing) field is read-only, materialized on the // read (view) side, NOT a column on the entity's write table (FR-024 §7). // Emitting it here would declare a Drizzle column for a table column migrate // no longer creates. if (child.isDerived()) continue; + // `field.object @storage: flattened` holds one column per member of its value object and + // none of its own - exactly the columns `meta migrate` creates (flattened.ts). + const flat = flattenedMembersOf(child, ctx.columnNamingStrategy); + if (flat !== undefined) { + pushFlattenedColumns(child, flat); + continue; + } const isPk = pkFieldNames.has(child.name); const fkInfo = fkMap.get(child.name); // Compute the column spec once per field and reuse it for both the column @@ -519,6 +551,9 @@ function renderColumn( // FR-017 Tier 2 — TPH subtype-only column: force nullable (drop .notNull()) // and suppress any DB default (other-subtype rows must stay NULL here). forceNullable: boolean = false, + // `field.object @storage: flattened`: the Drizzle KEY of a member's column (`shipToStreet`); + // absent, the key is the field's own name. + keyName: string = field.name, ): Code { // An int-backed field.enum's column function is a LOCAL generated const (the // customType helper emitted into this same file), so it must not be imported @@ -739,7 +774,7 @@ function renderColumn( } } - const columnLine = code` ${field.name}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; + const columnLine = code` ${keyName}: ${baseCall}${dollarTypeSegment}${modifiersStr}${autoSetSuffix}${sqlDefaultSegment ?? ""}${fkRefSegment ?? ""}`; return spec.leadingComment !== undefined ? code` // ${spec.leadingComment}\n${columnLine}` : columnLine; diff --git a/server/typescript/packages/test-generators/src/templates/entity-file.ts b/server/typescript/packages/test-generators/src/templates/entity-file.ts index b448ae1a5..7e460dbc6 100644 --- a/server/typescript/packages/test-generators/src/templates/entity-file.ts +++ b/server/typescript/packages/test-generators/src/templates/entity-file.ts @@ -6,6 +6,8 @@ // !hasWritableRdbSource(entity) → renderValueObjectFile (in-memory / transit shape: interface + Zod schema) // vanilla / write-through entity → Drizzle table path +import { CodegenError } from "./errors.js"; +import { flattenedMembersOf } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { SOURCE_ROLE_REPLICA } from "@metaobjectsdev/metadata"; @@ -137,6 +139,16 @@ export function renderEntityFile( // compile error). A base+write-through combo keeps the TPH polymorphic read path (reads // the base table); routing its reads through a replica view is a documented non-goal. const writeThrough = isWriteThrough(entity) && !tphBase; + // A write-through entity reads through a replica view whose columns are declared per field; + // that declaration has no flattened-member form yet, and a view column bound under the wrong + // name fails at query time, so refuse rather than emit it. + if (writeThrough && entity.fields().some((f) => flattenedMembersOf(f, ctx.columnNamingStrategy) !== undefined)) { + throw new CodegenError( + `${entity.name}: a write-through entity (writable table plus a replica view) cannot hold a ` + + `field.object @storage: flattened field yet - the replica view declaration has no flattened-member form. ` + + `Use @storage: jsonb, or drop the replica view.`, + ); + } // Rendered BEFORE the replica view below, for two reasons that both matter: the table // DECLARES the int-backed-enum codecs (a `const` the view's declaration would duplicate, // and a duplicate module-scope const is a parse error), and those consts must precede the diff --git a/server/typescript/packages/test-generators/src/templates/errors.ts b/server/typescript/packages/test-generators/src/templates/errors.ts new file mode 100644 index 000000000..71b2b751b --- /dev/null +++ b/server/typescript/packages/test-generators/src/templates/errors.ts @@ -0,0 +1,10 @@ +// Typed errors for codegen-ts. + +export class CodegenError extends Error { + readonly file?: string; + constructor(message: string, opts?: { file?: string }) { + super(message); + this.name = "CodegenError"; + if (opts?.file !== undefined) this.file = opts.file; + } +} diff --git a/server/typescript/packages/test-generators/src/templates/filter-allowlist.ts b/server/typescript/packages/test-generators/src/templates/filter-allowlist.ts index ec7c195b7..fd7406700 100644 --- a/server/typescript/packages/test-generators/src/templates/filter-allowlist.ts +++ b/server/typescript/packages/test-generators/src/templates/filter-allowlist.ts @@ -1,3 +1,4 @@ +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -114,8 +115,7 @@ function valueShape(f: MetaField): string { function filterableFields(entity: MetaObject, exclude?: string): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in allowlists. - return entity - .fields() + return flatSurfaceFields(entity) .filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true && c.name !== exclude); } diff --git a/server/typescript/packages/test-generators/src/templates/filter-shared.ts b/server/typescript/packages/test-generators/src/templates/filter-shared.ts index 1ddcb3162..c4de261bb 100644 --- a/server/typescript/packages/test-generators/src/templates/filter-shared.ts +++ b/server/typescript/packages/test-generators/src/templates/filter-shared.ts @@ -14,6 +14,7 @@ // renders clickable for a field the endpoint rejects, or a grid whose initial sort // runs the opposite way from the endpoint it queries. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject, type MetaData } from "@metaobjectsdev/metadata"; import { FIELD_ATTR_FILTERABLE, @@ -47,7 +48,7 @@ export function isSortableField(field: MetaField): boolean { */ export function sortableFields(entity: MetaObject): MetaField[] { // fields() returns effective fields, so inherited fields (from extends:/super:) are included in sort ops. - return entity.fields().filter(isSortableField); + return flatSurfaceFields(entity).filter(isSortableField); } /** @@ -110,6 +111,6 @@ export function resolveGridDefaultSort( const declared = layout.attr(LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_ORDER); if (declared === "asc" || declared === "desc") return { field: fieldName, order: declared }; - const field = entity.fields().find((f) => f.name === fieldName); + const field = flatSurfaceFields(entity).find((f) => f.name === fieldName); return { field: fieldName, order: field === undefined ? "asc" : sortDefaultOrder(field) }; } diff --git a/server/typescript/packages/test-generators/src/templates/filter-type.ts b/server/typescript/packages/test-generators/src/templates/filter-type.ts index e2fd030d3..d3a06b53f 100644 --- a/server/typescript/packages/test-generators/src/templates/filter-type.ts +++ b/server/typescript/packages/test-generators/src/templates/filter-type.ts @@ -2,6 +2,7 @@ // String fields get eq/ne/in/like; numbers get eq/ne/gt/gte/lt/lte/in; booleans get eq/isNull only; // datetimes get eq/ne/gt/gte/lt/lte/in. An enum's operand type is its member union. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaField, MetaObject } from "@metaobjectsdev/metadata"; import { @@ -82,7 +83,7 @@ function renderFieldUnion(field: MetaField): string { export function renderFilterType(entity: MetaObject, exclude?: string): Code { const filterName = entityTypeNames(entity).filter; // fields() returns effective fields, so inherited fields (from extends:/super:) are included in filter types. - const allFields = entity.fields().filter((c) => c.name !== exclude); + const allFields = flatSurfaceFields(entity).filter((c) => c.name !== exclude); const filterableFieldsList = allFields.filter((c) => c.attr(FIELD_ATTR_FILTERABLE) === true); // Sort union uses isSortableField — same predicate as renderSortAllowlist to prevent // client/server mismatches (@filterable: true + @sortable: false must be excluded from both). diff --git a/server/typescript/packages/test-generators/src/templates/inferred-types.ts b/server/typescript/packages/test-generators/src/templates/inferred-types.ts index 4b87f2f9b..def5fb049 100644 --- a/server/typescript/packages/test-generators/src/templates/inferred-types.ts +++ b/server/typescript/packages/test-generators/src/templates/inferred-types.ts @@ -7,6 +7,7 @@ // optional fields (matching Zod's `.optional()` inference — `T | undefined` — // without the superfluous `| null` that Drizzle nullable columns introduce). +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { code, imp, joinCode, type Code } from "@metaobjectsdev/codegen-ts/engine"; import type { MetaObject, MetaField } from "@metaobjectsdev/metadata"; import { @@ -146,7 +147,7 @@ export function renderEnumTypeAliases(entity: MetaObject, ctx?: RenderContext): // and non-colliding value objects keep their bare name (byte-identical). const ownerName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { if (field.subType !== FIELD_SUBTYPE_ENUM) continue; const values = enumValues(field); @@ -402,7 +403,7 @@ export function renderValueObjectInterface(entity: MetaObject, ctx?: RenderConte const objName = ctx ? ctx.valueObjectEmittedName(entity) : entity.name; const lines: Code[] = []; - for (const field of entity.fields()) { + for (const field of flatSurfaceFields(entity)) { const required = field.attr(FIELD_ATTR_REQUIRED) === true; const optional = required ? "" : "?"; const undefinedArm = required ? "" : " | undefined"; diff --git a/server/typescript/packages/test-generators/src/templates/zod-validators.ts b/server/typescript/packages/test-generators/src/templates/zod-validators.ts index 195fddfac..ec3a5557b 100644 --- a/server/typescript/packages/test-generators/src/templates/zod-validators.ts +++ b/server/typescript/packages/test-generators/src/templates/zod-validators.ts @@ -9,6 +9,7 @@ // object-array field collapsed to z.array(z.string()) and the JSON Schema sent // downstream (e.g. to LLM tool_use input_schema) lost the nested object shape. +import { flatSurfaceFields } from "@metaobjectsdev/codegen-ts/engine"; import { entityTypeNames } from "./entity-type-names.js"; import { code, joinCode, imp, type Code } from "@metaobjectsdev/codegen-ts/engine"; import { MetaObject, MetaField, type MetaRoot, stripPackage } from "@metaobjectsdev/metadata"; @@ -104,7 +105,7 @@ function durationHelperSince(since: number): string { * a plain entity has nothing to preserve, so both are omitted. Resolving * (ADR-0039): honors an @autoSet inherited via extends. */ export function hasAutoSetFields(obj: MetaObject): boolean { - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { const autoSet = child.attr(FIELD_ATTR_AUTO_SET); if (autoSet === AUTO_SET_ON_CREATE || autoSet === AUTO_SET_ON_UPDATE) return true; } @@ -121,7 +122,7 @@ export function hasAutoSetFields(obj: MetaObject): boolean { function isTphBaseOwnField(obj: MetaObject, field: MetaField): boolean { const base = tphDiscriminatorBase(obj); if (base === undefined) return false; - return base.fields().some((f) => f.name === field.name); + return flatSurfaceFields(base).some((f) => f.name === field.name); } /** @@ -169,7 +170,7 @@ export function renderTphSubtypeReadSchema(obj: MetaObject, ctx?: RenderContext) const tphPin = tphDiscriminatorPin(obj); const fieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (tphPin !== undefined && child.name === tphPin.fieldName) { fieldLines.push(code` ${child.name}: z.literal(${JSON.stringify(tphPin.value)})`); continue; @@ -319,7 +320,7 @@ function crossComparable(left: MetaField, right: MetaField, opName: string): boo function insertSchemaFieldNameSet(obj: MetaObject): Set { const autoGen = autoGenPkFieldNames(obj); const out = new Set(); - for (const f of obj.fields()) { + for (const f of flatSurfaceFields(obj)) { if (autoGen.has(f.name)) continue; if (isReadOnlyMutability(f) || f.isDerived()) continue; out.add(f.name); @@ -385,7 +386,7 @@ interface RefinementBuild { blocks: string[]; helpers: Set } function crossFieldRefinement(obj: MetaObject): RefinementBuild { const inSchema = insertSchemaFieldNameSet(obj); - const byName = new Map(obj.fields().map((f) => [f.name, f] as const)); + const byName = new Map(flatSurfaceFields(obj).map((f) => [f.name, f] as const)); // A reference must name a field that exists AND is in the insert schema; otherwise skip // the validator (migrate skips on a missing field; a readOnly/derived/auto-PK field is // never in the body, so enforcing on it would reject every create). @@ -599,7 +600,7 @@ export function renderInsertSchemaOnly(obj: MetaObject, ctx?: RenderContext): Co const tphPin = tphDiscriminatorPin(obj); const insertFieldLines: Code[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field is populated by DB / replication / // external owner; the application has no path to write it. Exclude from the @@ -684,7 +685,7 @@ export function insertSchemaFields(obj: MetaObject): SchemaFieldShape[] { const assignedPkFields = assignedPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -720,7 +721,7 @@ export function updateSchemaFields(obj: MetaObject): SchemaFieldShape[] { const autoGenPkFields = autoGenPkFieldNames(obj); const tphPin = tphDiscriminatorPin(obj); const out: SchemaFieldShape[] = []; - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // #213 — a derived (origin-bearing) field is read-only (FR-024 §7 / ADR-0028): // excluded from the Insert/Update/preserving schemas, exactly like readOnly. @@ -767,7 +768,7 @@ export function renderZodValidators(obj: MetaObject, ctx?: RenderContext): Code // transform). Only emitted when the entity declares @autoSet fields. const preservingFieldLines: Code[] = []; const emitPreserving = hasAutoSetFields(obj); - for (const child of obj.fields()) { + for (const child of flatSurfaceFields(obj)) { if (autoGenPkFields.has(child.name)) continue; // FR-037 R1: an @mutability:"readOnly" field appears in neither InsertSchema // nor UpdateSchema. The DB / trigger / replication owns the write path; the app