Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ jobs:
POSTGRES_DB: metaobjects_test
ports:
- 5432:5432
# The data directory (the image's VOLUME) on a size-capped tmpfs: a throwaway database
# never touches the runner's drives.
options: >-
--tmpfs /var/lib/postgresql/data:rw,size=2g
--health-cmd "pg_isready -U metaobjects -d metaobjects_test"
--health-interval 10s
--health-timeout 5s
Expand All @@ -75,6 +78,16 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Postgres durability off (a throwaway database)
# A service container takes no server flags, so set them on the running server. Its data
# directory is already a tmpfs (options above); nothing in it needs to survive a crash.
run: >-
docker exec ${{ job.services.postgres.id }}
psql -U metaobjects -d metaobjects_test
-c "ALTER SYSTEM SET fsync = off"
-c "ALTER SYSTEM SET synchronous_commit = off"
-c "ALTER SYSTEM SET full_page_writes = off"
-c "SELECT pg_reload_conf()"

- name: Set up Bun (TS lanes)
if: matrix.lane == 'ts-slow' || matrix.lane == 'cube'
Expand Down
52 changes: 52 additions & 0 deletions .github/workflows/local-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,10 @@ jobs:
POSTGRES_DB: metaobjects_test
ports:
- 5432 # dynamic host port — no fixed-port collision
# The data directory (the image's VOLUME) on a size-capped tmpfs: a throwaway database
# never touches the runner's drives.
options: >-
--tmpfs /var/lib/postgresql/data:rw,size=2g
--health-cmd "pg_isready -U metaobjects -d metaobjects_test"
--health-interval 10s
--health-timeout 5s
Expand All @@ -156,6 +159,16 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Postgres durability off (a throwaway database)
# A service container takes no server flags, so set them on the running server. Its data
# directory is already a tmpfs (options above); nothing in it needs to survive a crash.
run: >-
docker exec ${{ job.services.postgres.id }}
psql -U metaobjects -d metaobjects_test
-c "ALTER SYSTEM SET fsync = off"
-c "ALTER SYSTEM SET synchronous_commit = off"
-c "ALTER SYSTEM SET full_page_writes = off"
-c "SELECT pg_reload_conf()"
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
# Arms migrate-ts's real-Postgres suites (apply / lifecycle / rollback /
Expand Down Expand Up @@ -206,7 +219,10 @@ jobs:
POSTGRES_DB: metaobjects_test
ports:
- 5432
# The data directory (the image's VOLUME) on a size-capped tmpfs: a throwaway database
# never touches the runner's drives.
options: >-
--tmpfs /var/lib/postgresql/data:rw,size=2g
--health-cmd "pg_isready -U metaobjects -d metaobjects_test"
--health-interval 10s
--health-timeout 5s
Expand All @@ -215,6 +231,16 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Postgres durability off (a throwaway database)
# A service container takes no server flags, so set them on the running server. Its data
# directory is already a tmpfs (options above); nothing in it needs to survive a crash.
run: >-
docker exec ${{ job.services.postgres.id }}
psql -U metaobjects -d metaobjects_test
-c "ALTER SYSTEM SET fsync = off"
-c "ALTER SYSTEM SET synchronous_commit = off"
-c "ALTER SYSTEM SET full_page_writes = off"
-c "SELECT pg_reload_conf()"
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only java-slow --strict-toolchains
Expand All @@ -233,7 +259,10 @@ jobs:
POSTGRES_DB: metaobjects_test
ports:
- 5432
# The data directory (the image's VOLUME) on a size-capped tmpfs: a throwaway database
# never touches the runner's drives.
options: >-
--tmpfs /var/lib/postgresql/data:rw,size=2g
--health-cmd "pg_isready -U metaobjects -d metaobjects_test"
--health-interval 10s
--health-timeout 5s
Expand All @@ -242,6 +271,16 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Postgres durability off (a throwaway database)
# A service container takes no server flags, so set them on the running server. Its data
# directory is already a tmpfs (options above); nothing in it needs to survive a crash.
run: >-
docker exec ${{ job.services.postgres.id }}
psql -U metaobjects -d metaobjects_test
-c "ALTER SYSTEM SET fsync = off"
-c "ALTER SYSTEM SET synchronous_commit = off"
-c "ALTER SYSTEM SET full_page_writes = off"
-c "SELECT pg_reload_conf()"
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only python --strict-toolchains
Expand All @@ -260,7 +299,10 @@ jobs:
POSTGRES_DB: metaobjects_test
ports:
- 5432
# The data directory (the image's VOLUME) on a size-capped tmpfs: a throwaway database
# never touches the runner's drives.
options: >-
--tmpfs /var/lib/postgresql/data:rw,size=2g
--health-cmd "pg_isready -U metaobjects -d metaobjects_test"
--health-interval 10s
--health-timeout 5s
Expand All @@ -269,6 +311,16 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Postgres durability off (a throwaway database)
# A service container takes no server flags, so set them on the running server. Its data
# directory is already a tmpfs (options above); nothing in it needs to survive a crash.
run: >-
docker exec ${{ job.services.postgres.id }}
psql -U metaobjects -d metaobjects_test
-c "ALTER SYSTEM SET fsync = off"
-c "ALTER SYSTEM SET synchronous_commit = off"
-c "ALTER SYSTEM SET full_page_writes = off"
-c "SELECT pg_reload_conf()"
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only csharp --strict-toolchains
Expand Down
24 changes: 24 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -499,3 +499,27 @@ inverse of metadata-is-the-spine. *OMDB engine debt* — FR-003 Plan 4 closed th
anti-patterns; the Spring Boot 3 starter and OMDB autoconfiguration shipped. *WARN
envelope-shape on `expected-warnings.json`* — every runner asserts it; the legacy string-list
path is retired.

## Disk I/O: keep throwaway data off the drives

Test databases, sandboxes and temp files must not wear out the developer's drives.

- **Throwaway data goes in RAM.** Put scratch files, mutation-testing sandboxes (Stryker
`tempDirName`) and test-only temp directories under `$DEV_SCRATCH` (fall back to the system temp
directory when it is unset). Delete what you create when the run ends.
- **Test databases run on a capped tmpfs at the image's `VOLUME` path** (`postgres:18*`:
`/var/lib/postgresql`; `postgres:16/17` and `pgvector:*-pg17`: `/var/lib/postgresql/data`;
`mysql:8*`: `/var/lib/mysql`; check others with `docker image inspect -f '{{json .Config.Volumes}}'`),
`size=2g`, with Postgres durability off: `-c fsync=off -c synchronous_commit=off -c full_page_writes=off`.
- **Every container you start, you remove.** Use `docker run --rm`, Testcontainers with Ryuk enabled,
or explicit teardown in a `finally`/`trap`. Never leave a hand-started database running after your
task. A database you mean to keep gets the label `dev.keep=true`.
- **Generators and tools never rewrite a file whose bytes did not change**, write any manifest once per
run (not once per file), and verify drift in memory rather than by rendering everything to a temp
directory.
- **Run the narrowest tests that prove the change** while iterating (one module, the affected test
classes); run the full suite once, before the PR.
- **No background processes outlive the task.** Kill test watchers, dev servers and sandboxes you
started before you finish.

Project specifics: test databases start in the TypeScript `integration-tests` package (`src/postgres-container.ts`, `src/mysql-container.ts`, `src/cube-container.ts`, sharing `src/throwaway.ts`), the Python `tests/integration/` launchers (`postgres_container.py`, `mysql_container.py`, `throwaway.py`), the C# `MetaObjects.IntegrationTests/Runner/PostgresContainer.cs`, the Java and Kotlin Testcontainers `PostgresContainer` classes under `server/java/integration-tests*`, and the services and sidecars in `.github/workflows/integration-tests.yml` and `local-ci.yml`.
38 changes: 38 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,16 @@ loaded before stops loading. This vocabulary ships in 1.3.0 with `metamodelVersi
`Cannot find module './dprint-node.linux-x64-gnu.node'` on every command that loads the
codegen engine, `meta migrate` and `meta verify --db` included. Those schema commands, the
binary's documented scope, now run; `meta gen` is still not supported inside the binary.
- **Test databases stay off the drives.** Every Docker-backed database launcher in every port
(the TypeScript `integration-tests` Postgres, MySQL and Cube containers; the Python MySQL and
Postgres launchers; the C# MySQL and Postgres runners; the Java and Kotlin Testcontainers
classes; the services and sidecars in the integration-test and local-CI workflows and
`scripts/ci-local.sh`) mounts a 2 GiB tmpfs at the image's `VOLUME` path, and Postgres runs
with `fsync`, `synchronous_commit` and `full_page_writes` off. Each container is removed when
its run ends: by `--rm`, by Ryuk when a killed process never reaches its own cleanup, and by an
exit or SIGTERM handler that also removes anonymous volumes. The Stryker mutation gate for the
metadata package builds its sandbox under `$DEV_SCRATCH` (the system temp directory when unset)
and stops its test runners with it. Contributor steps are in [CONTRIBUTING.md](CONTRIBUTING.md).

### Deprecated

Expand All @@ -297,6 +307,34 @@ loaded before stops loading. This vocabulary ships in 1.3.0 with `metamodelVersi

### Fixed

- **`gen <Entity>` no longer shrinks the files rendered from the whole model.** Narrowing a run
to some entities (`meta gen Post`, Python `metaobjects gen --entities Post`) handed every
generator only the named entities, so a whole-model file (the TypeScript barrel and Cube
model, Python's shared enums module, duration codec and report writers, and the object index
Python's entity and route generators resolve relations through) was rendered from just them. A narrowed run now writes only the named entities' own files,
and every whole-model file it writes is byte-identical to a full run's. A generator sees the
whole model in `ctx.entities` and the narrowing as `ctx.selected`; `perEntity` (`per_entity`)
applies it. A Python target's declared `entities` still bounds that target's model. C# and
Java take no entity filter and were not affected. Two consequences to expect: a narrowed run
checks the whole model as a full run does (a sourced object with no `dialect`, colliding
collection names), because the files it writes are rendered from it; and an owned generator
that writes one file per object by looping over `ctx.entities` itself, instead of through
`perEntity`, now sees every object, so it should set each file's `sourceNode` (the runner
then drops the unnamed objects' files) or apply `ctx.selected`.
- **`gen` writes `.hashes.json` once per run, and not at all when nothing changed.** TypeScript,
Python and C# re-read and re-wrote the whole manifest after every generated file, so a run of
N files wrote it N times; they now hold it in memory for the run and write it once, as Java
already did. Python's `metaobjects gen` output changes, which matters to a script that parses it:
it now lists only the files it wrote, and its summary line counts the files it left as they
were (unchanged or skipped) separately instead of reporting them as written. A run that
fails still records the files it wrote before failing, and a failed manifest write then no
longer hides the error that stopped the run.
- **`meta gen` loads the metadata once.** The unknown-attribute warning re-loaded the project
strictly on every run; it now reuses the generation load and parses strictly again only when
the lenient load reported an unknown bare key. `meta gen` also no longer imports a database
driver: the migration advice it prints reads `@metaobjectsdev/migrate-ts/offline`, a new
subpath with the schema builder and diff and none of the introspection or apply code.

- **A projection view keeps the columns a reverse `@via` hop feeds.** An origin `@via` that walks
an `identity.reference` backwards (`Project.timeEntries`, #560) loaded, but `meta migrate` left
its `origin.aggregate` / `origin.first` columns out of `CREATE VIEW` without a word. The view
Expand Down
8 changes: 8 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ cd server/python && pytest # Python (in its .venv)
Cross-language persistence / api-contract corpora (Docker + Testcontainers) run via
`scripts/integration-test.sh`.

Throwaway test databases stay off the drives. The Docker-backed launchers (Postgres, MySQL, the
Cube stack and the `scripts/ci-local.sh` Postgres sidecar) keep each database's data directory on
a size-capped tmpfs with Postgres durability off, and remove the container when the run ends.
The Stryker mutation gate (`bun run conformance:mutation` in
`server/typescript/packages/metadata`, via `scripts/run-stryker.sh`) builds its sandbox under
`$DEV_SCRATCH` when that is set, else the system temp directory, and removes it on exit. Point
`DEV_SCRATCH` at a RAM-backed directory to keep that scratch off disk.

### Local CI (the checks GitHub runs — run it before opening a PR)

GitHub Actions is enabled on this repository. The check workflows in
Expand Down
3 changes: 2 additions & 1 deletion docs/features/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,8 @@ targets:
- **`--config <path>`** picks the config file explicitly on either command; without it both
look for `./metaobjects.config.yaml` in the current directory. **No config is an error**:
`gen` and `verify --codegen` print `no metaobjects.config.yaml`, exit 2 and write nothing.
- **Run options still apply to the list:** `--entities` (narrows each target's entities),
- **Run options still apply to the list:** `--entities` (narrows the entities each target writes; a
target's own `entities` still bounds its model),
`--provider` (added to `providers`), `--templates` (`--prompts` on `verify`), `--baseline`.
Column naming is not a run option: it is `naming` in `.metaobjects/config.json`, which every
port reads, and `--column-naming` is refused
Expand Down
9 changes: 9 additions & 0 deletions docs/features/codegen-concepts.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,15 @@ custom iteration:
is kept as a soft-deprecated alias — "run" is ambiguous under multi-target output;
`perModel` names the data scope.)

**`meta gen <Entity>` narrows what is written, never what a whole-model file sees.** `ctx.entities`
is the whole model in scope (after the project's `scope`), even when the run names some entities.
`ctx.selected` is the predicate for the named ones; it is absent when every object is selected.
`perEntity` applies it. A file rendered from the whole model (a barrel, an index, a shared schema)
reads `ctx.entities`, so it is the same on a narrowed run as on a full one. A generator that loops
over `ctx.entities` itself must apply `ctx.selected` (TypeScript: `ctx.selected?.(o) ?? true`;
Python: `ctx.is_selected(o)`), or set `sourceNode` on each per-object file, which lets the runner
drop the files of objects the run did not name.

A generator picks its scope by which helper it composes — the same way it picks
object vs model today. Reference templates and the guidance call out the scope each
starting point is for, so "I need a per-package service layer" maps to a clear
Expand Down
3 changes: 2 additions & 1 deletion docs/recipes/typed-client-for-the-reference-routes.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,8 @@ export function fetchClientGenerator(): Generator {
const layout = ctx.config.outputLayout ?? "flat";
const prefix = ctx.renderContext?.apiPrefix ?? ""; // the same apiPrefix routesFile mounts under
const files: EmittedFile[] = [{ path: "fetch-client.runtime.ts", content: RUNTIME }];
for (const obj of ctx.entities.filter(ctx.matches)) {
// ctx.entities is the whole model; ctx.selected narrows a `meta gen <Entity>` run to the named ones.
for (const obj of ctx.entities.filter((o) => ctx.matches(o) && (ctx.selected?.(o) ?? true))) {
const n = obj.name, path = servedPath(obj, prefix);
// Types come from the entity module `entity` wrote next to this file (flat layout).
files.push({
Expand Down
6 changes: 5 additions & 1 deletion scripts/ci-local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1047,10 +1047,14 @@ start_pg_sidecar() {
fi
# Clear a stopped leftover of an MO_PG_SIDECAR_NAME override, then start fresh.
docker rm -f -v "$name" >/dev/null 2>&1 || true
# The data directory (the image's VOLUME) is a size-capped tmpfs and durability is off:
# a throwaway database never touches the drives, and leaves no anonymous volume behind.
if ! docker run -d --name "$name" --label "$MO_CI_SESSION_LABEL" \
--tmpfs /var/lib/postgresql/data:rw,size=2g \
-e POSTGRES_USER=metaobjects -e POSTGRES_PASSWORD=metaobjects \
-e POSTGRES_DB=metaobjects_test \
-p 127.0.0.1::5432 postgres:16 >/dev/null 2>&1; then
-p 127.0.0.1::5432 postgres:16 \
-c fsync=off -c synchronous_commit=off -c full_page_writes=off >/dev/null 2>&1; then
echo " ⚠ could not start the Postgres sidecar — falling back to per-scenario" >&2
echo " containers (slow). Integration results are unaffected." >&2
return 0
Expand Down
40 changes: 40 additions & 0 deletions scripts/run-stryker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# Run a package's Stryker mutation gate without wearing the drives or leaving anything behind.
#
# scripts/run-stryker.sh <package-label> [stryker run options] (run from the package directory)
#
# - The sandbox (a full copy of the package, rewritten once per mutant) goes under
# $DEV_SCRATCH/stryker/<package-label>-XXXXXX, a RAM-backed scratch area, falling back to the
# system temp directory when DEV_SCRATCH is unset. The package's stryker.conf.mjs reads the
# directory from METAOBJECTS_STRYKER_TEMP.
# - Stryker runs in its own process group, so its test-runner children go with it: on exit, on
# failure and on Ctrl-C or SIGTERM the whole group is stopped before the sandbox is removed.
set -uo pipefail

label="${1:?usage: run-stryker.sh <package-label>}"
scratch="${DEV_SCRATCH:-${TMPDIR:-/tmp}}"
mkdir -p "$scratch/stryker"
sandbox="$(mktemp -d "$scratch/stryker/${label}-XXXXXX")"
export METAOBJECTS_STRYKER_TEMP="$sandbox"

pid=""
cleanup() {
if [[ -n "$pid" ]]; then
kill -TERM -- "-$pid" 2>/dev/null
for _ in 1 2 3 4 5 6 7 8 9 10; do kill -0 -- "-$pid" 2>/dev/null || break; sleep 0.5; done
kill -KILL -- "-$pid" 2>/dev/null
fi
rm -rf "$sandbox"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

# A background job of a non-interactive shell is not a group leader, so setsid execs in place:
# $pid is Stryker's own pid and the id of its new process group.
setsid stryker run "${@:2}" &
pid=$!
wait "$pid"
status=$?
# Stryker has exited; the EXIT trap stops any runner left in its group and removes the sandbox.
exit "$status"
Loading
Loading