test(integration-tests): reap test containers with Ryuk when runs are killed

scripts/ci-local.sh

@@ -91,16 +91,16 @@
 #
 # POSTGRES SIDECAR (see ensure_pg_sidecar below). Any lane that runs integration
 # tests shares ONE Postgres container instead of booting a Testcontainers instance
-# per scenario. It is started (or reused) automatically and costs ~6s cold, ~1s warm.
+# per scenario. It is started automatically and costs ~4s.
 # Measured on this repo: python integration 6m54s -> 11s, kotlin integration 47s with
 # zero containers booted. Every port's helper creates its own database per scenario,
-# so lanes that share the container never share a database. Its name carries a hash of
-# this checkout's path, so two checkouts on one machine never share or stop each
-# other's. It is left running for the next run (that is the ~1s warm start), so each
-# checkout keeps one; `docker rm -f metaobjects-ci-sidecar-<n>` removes it. If
+# so lanes that share the container never share a database. Each run starts its own
+# and removes it on the way out, and Ryuk removes it when the run is killed first, so
+# no two runs share or stop each other's and none outlives its run. If
 # METAOBJECTS_TEST_PG_URL is already set — as local-ci.yml does from its
 # `services:` block — nothing here changes. Opt out with MO_CI_NO_PG_SIDECAR=1; rename
-# it with MO_PG_SIDECAR_NAME.
+# it with MO_PG_SIDECAR_NAME (a container of that name already running is used and left
+# alone).
 #
 # MO_CI_LEAK_BASE overrides the ref the leak scan diffs against (default origin/main,
 # else HEAD~1); hygiene.yml points it at the PR's base branch. MO_CI_JACOCO=1 keeps
@@ -830,32 +830,106 @@ run_cube_lane() {
 # CI lanes. Nothing warned that the slow path had been taken; the run just looked slow.
 #
 # So: if the variable is already set (CI, or a deliberate override) this does nothing at
-# all. Otherwise it starts — or REUSES — a container named below and exports the same URL
-# shape CI uses. Reuse is the point: the second local run pays nothing.
+# all. Otherwise it starts a container named below and exports the same URL shape CI uses.
 #
-# Two details are load-bearing, both learned the hard way:
-#   * the host port is DYNAMIC (`-p 5432`, read back with `docker port`), never 5432:5432
-#     — this box already runs several Postgres containers and a fixed port collides;
+# Three details are load-bearing, all learned the hard way:
+#   * the host port is DYNAMIC (`-p 127.0.0.1::5432`, read back with `docker port`), never
+#     5432:5432 — this box already runs several Postgres containers and a fixed port collides;
 #   * readiness is POLLED before the URL is exported. The original shared-sidecar mode had
 #     no readiness gate and that was the dominant flake: the first connect to a
 #     transiently-slow sidecar hung to the test timeout.
 #
-#   * the container name is PER CHECKOUT (a checksum of this checkout's path). Several
-#     checkouts of this repository run on one machine at once, and a shared name made them
-#     reuse one database: the migrate-ts and runtime-ts suites create fixed table names in
-#     it, so two checkouts' runs could drop each other's tables mid-test.
+#   * the container is PER RUN, never kept for the next one. Several checkouts of this
+#     repository run on one machine at once, and a shared name made them reuse one database:
+#     the migrate-ts and runtime-ts suites create fixed table names in it, so two runs could
+#     drop each other's tables mid-test. Keeping one per checkout instead saved ~3s a run,
+#     and its name came from the checkout's path, so deleting the checkout left it running
+#     for good. A gate run gets a fresh worktree every time, which made that the common case.
+#
+# A killed run cannot clean up after itself: SIGKILL, the kernel OOM killer and earlyoom all
+# skip the EXIT trap. So the sidecar is put under Ryuk, the reaper the Testcontainers libraries
+# use (start_ci_reaper below), which removes it from outside this process. Ryuk removes only
+# what carries this run's own session label, so it cannot touch another run's container.
 #
 # Opt out with MO_CI_NO_PG_SIDECAR=1 to get the old per-scenario-container behaviour.
-MO_PG_SIDECAR_NAME="${MO_PG_SIDECAR_NAME:-metaobjects-ci-sidecar-$(printf '%s' "$ROOT" | cksum | cut -d' ' -f1)}"
-MO_CI_PG_URL=""   # set by start_pg_sidecar; exported only by use_pg_sidecar
+MO_CI_RUN_ID="${CI_TMP##*.}"
+MO_PG_SIDECAR_NAME="${MO_PG_SIDECAR_NAME:-metaobjects-ci-sidecar-$MO_CI_RUN_ID}"
+MO_CI_RYUK_IMAGE="testcontainers/ryuk:0.14.0"
+MO_CI_SESSION_LABEL="org.testcontainers.session-id=metaobjects-ci-$MO_CI_RUN_ID"
+MO_CI_PG_URL=""       # set by start_pg_sidecar; exported only by use_pg_sidecar
+CI_SIDECAR_STARTED=0  # 1 once this run started the sidecar, so it is this run's to remove
+CI_REAPER_PID=""      # the background shell that holds this run's Ryuk connection
+
+# docker_host_port <container> <container-port> — the host port Docker published for it.
+docker_host_port() { docker port "$1" "$2" 2>/dev/null | head -1 | sed 's/.*://'; }
+
+# start_ci_reaper — start Ryuk and hold its connection for as long as this script lives.
+#
+# Ryuk prunes a session ten seconds after its connection closes. The bash client differs from
+# the libraries in one way: the socket lives in a small background shell rather than here,
+# because bash cannot mark a descriptor close-on-exec. Every test JVM, and every build server
+# it leaves running, would inherit it, and the connection would outlive the run. The holder
+# polls this script's pid and exits when it is gone, however it went. A Ryuk that will not answer
+# costs about 15 seconds each time a lane asks for the sidecar, then the fall-back applies.
+start_ci_reaper() {
+  [ -n "$CI_REAPER_PID" ] && return 0
+
+  local name="metaobjects-ci-ryuk-$MO_CI_RUN_ID"
+  connect_ci_reaper "$name" && return 0
+  docker rm -f "$name" >/dev/null 2>&1
+  return 1
+}
+
+connect_ci_reaper() {
+  local name="$1" port i
+  docker run -d --rm --name "$name" \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -p 127.0.0.1::8080 "$MO_CI_RYUK_IMAGE" >/dev/null 2>&1 || return 1
+  port="$(docker_host_port "$name" 8080/tcp)"
+  [ -n "$port" ] || return 1
 
-# ensure_pg_sidecar — start (or reuse) the sidecar and export its URL into THIS shell.
+  (
+    i=0 ack=""
+    while [ "$i" -lt 75 ]; do
+      if { exec 9<>"/dev/tcp/127.0.0.1/$port"; } 2>/dev/null; then
+        printf 'label=%s\n' "$MO_CI_SESSION_LABEL" >&9
+        IFS= read -r -t 5 ack <&9 || ack=""
+        [ "$ack" = "ACK" ] && break
+        exec 9>&-
+      fi
+      i=$((i + 1)); sleep 0.2
+    done
+    [ "$ack" = "ACK" ] || exit 1
+    : > "$CI_TMP/reaper.ready"
+    while kill -0 $$ 2>/dev/null; do sleep 2; done
+  ) </dev/null >/dev/null 2>&1 &
+  CI_REAPER_PID=$!
+
+  i=0
+  while [ "$i" -lt 100 ]; do
+    [ -e "$CI_TMP/reaper.ready" ] && return 0
+    kill -0 "$CI_REAPER_PID" 2>/dev/null || break
+    i=$((i + 1)); sleep 0.2
+  done
+  kill "$CI_REAPER_PID" 2>/dev/null
+  CI_REAPER_PID=""
+  return 1
+}
+
+# stop_pg_sidecar — remove what this run started. Letting go of Ryuk ends it too.
+stop_pg_sidecar() {
+  [ "$CI_SIDECAR_STARTED" -eq 1 ] && docker rm -f -v "$MO_PG_SIDECAR_NAME" >/dev/null 2>&1
+  [ -n "$CI_REAPER_PID" ] && kill "$CI_REAPER_PID" 2>/dev/null
+  return 0
+}
+
+# ensure_pg_sidecar — start the sidecar, if this run has none yet, and export its URL into THIS shell.
 ensure_pg_sidecar() {
   [ -n "${METAOBJECTS_TEST_PG_URL:-}" ] && return 0   # CI, or an explicit override
   start_pg_sidecar && use_pg_sidecar
 }
 
-# start_pg_sidecar — start (or reuse) the sidecar and record its URL in MO_CI_PG_URL, exporting
+# start_pg_sidecar — start the sidecar and record its URL in MO_CI_PG_URL, exporting
 # nothing. The full run calls it once before its lanes fork and lets each lane that runs an
 # integration suite export the URL for itself, so a lane with no database (ts unit suites)
 # never sees one: runtime-ts's suite runs its real-PG files whenever the URL is set, against
@@ -865,22 +939,28 @@ start_pg_sidecar() {
 
   local name="$MO_PG_SIDECAR_NAME" port
   if [ -z "$(docker ps -q -f "name=^${name}$" 2>/dev/null)" ]; then
-    # Clear a stopped leftover of the same name, then start fresh.
-    docker rm -f "$name" >/dev/null 2>&1 || true
-    if ! docker run -d --name "$name" \
+    if ! start_ci_reaper; then
+      echo "  ⚠ could not start Ryuk ($MO_CI_RYUK_IMAGE) — falling back to per-scenario" >&2
+      echo "    containers (slow). Integration results are unaffected." >&2
+      return 0
+    fi
+    # Clear a stopped leftover of an MO_PG_SIDECAR_NAME override, then start fresh.
+    docker rm -f -v "$name" >/dev/null 2>&1 || true
+    if ! docker run -d --name "$name" --label "$MO_CI_SESSION_LABEL" \
         -e POSTGRES_USER=metaobjects -e POSTGRES_PASSWORD=metaobjects \
         -e POSTGRES_DB=metaobjects_test \
-        -p 5432 postgres:16 >/dev/null 2>&1; then
+        -p 127.0.0.1::5432 postgres:16 >/dev/null 2>&1; then
       echo "  ⚠ could not start the Postgres sidecar — falling back to per-scenario" >&2
       echo "    containers (slow). Integration results are unaffected." >&2
       return 0
     fi
+    CI_SIDECAR_STARTED=1
     echo "  ▸ started Postgres sidecar '$name'"
   else
-    echo "  ▸ reusing Postgres sidecar '$name'"
+    echo "  ▸ using Postgres sidecar '$name'"
   fi
 
-  port="$(docker port "$name" 5432/tcp 2>/dev/null | head -1 | sed 's/.*://')"
+  port="$(docker_host_port "$name" 5432/tcp)"
   if [ -z "$port" ]; then
     echo "  ⚠ sidecar has no published port — falling back to per-scenario containers." >&2
     return 0
@@ -915,8 +995,8 @@ start_pg_sidecar() {
   # reported as ✓ into a loud failure. Hosted CI sets both from its `services:` block; the
   # local run had neither, which is why a local ✓ and a hosted ✖ could disagree about the
   # only real-engine gate on migration correctness.
-  MO_CI_PG_URL="postgres://metaobjects:metaobjects@localhost:${port}/metaobjects_test"
-  echo "    METAOBJECTS_TEST_PG_URL -> localhost:${port} (stop it: docker rm -f $name)"
+  MO_CI_PG_URL="postgres://metaobjects:metaobjects@127.0.0.1:${port}/metaobjects_test"
+  echo "    METAOBJECTS_TEST_PG_URL -> 127.0.0.1:${port}"
 }
 
 # use_pg_sidecar — export the URL start_pg_sidecar recorded (nothing when it fell back).
@@ -1022,6 +1102,7 @@ run_lanes() {
 kill_tree() { local c; for c in $(pgrep -P "$1" 2>/dev/null); do kill_tree "$c"; done; kill "$1" 2>/dev/null; }
 ci_cleanup() {
   local p; for p in "${LANE_PID[@]:-}"; do [ -n "$p" ] && kill -0 "$p" 2>/dev/null && kill_tree "$p"; done
+  stop_pg_sidecar
   rm -rf "$CI_TMP"
 }
 trap ci_cleanup EXIT

server/csharp/MetaObjects.IntegrationTests/MetaObjects.IntegrationTests.csproj

@@ -12,6 +12,8 @@
          `dotnet test path/to/MetaObjects.IntegrationTests.csproj` or
          scripts/integration-test.sh. -->
     <IsPackable>false</IsPackable>
+    <!-- ReaperKillTests.Main is this assembly's entry point: its killed child process. -->
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
   <!-- The generated minimal-API routes (RoutesGenerator output) are hosted on a
        real ASP.NET Core host (Kestrel) in the SP-F generated-server lane
@@ -31,7 +33,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
     <PackageReference Include="xunit" Version="2.6.4" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6" />
-    <PackageReference Include="Testcontainers.PostgreSql" Version="3.10.0" />
+    <PackageReference Include="Testcontainers.PostgreSql" Version="4.16.0" />
     <PackageReference Include="YamlDotNet" Version="18.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.10" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.10" />
@@ -40,6 +42,6 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.10" />
     <!-- ...and on MySQL (Docker): Pomelo is the EF Core provider, MySqlConnector the ADO.NET driver. -->
     <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
-    <PackageReference Include="Testcontainers.MySql" Version="3.10.0" />
+    <PackageReference Include="Testcontainers.MySql" Version="4.16.0" />
   </ItemGroup>
 </Project>

server/csharp/MetaObjects.IntegrationTests/ReaperKillTests.cs

@@ -0,0 +1,87 @@
+// A killed process removes its container.
+//
+// Public behaviour: once a process that started a helper container dies without running any
+// teardown, Docker stops listing that container. The oracle is Docker itself, queried from outside
+// the killed process. A helper that labels nothing, or a reaper that accepts the label and removes
+// nothing, such as the Ryuk below Testcontainers 4.0 on Docker 29, fails on the deadline.
+
+using System.Diagnostics;
+using System.Text.RegularExpressions;
+using MetaObjects.IntegrationTests.Runner;
+using Npgsql;
+using Xunit;
+
+namespace MetaObjects.IntegrationTests;
+
+public sealed class ReaperKillTests
+{
+    private static readonly TimeSpan ChildReady = TimeSpan.FromSeconds(60);
+    private static readonly TimeSpan RemovalBound = TimeSpan.FromSeconds(45);
+
+    // The killed process: it starts a real helper container, reports the port and waits. This is
+    // the test assembly's entry point, which GenerateProgramFile=false leaves free.
+    private static async Task Main()
+    {
+        var pg = await PostgresContainer.StartAsync();
+        Console.WriteLine($"READY {new NpgsqlConnectionStringBuilder(pg.ConnectionString).Port}");
+        await Task.Delay(TimeSpan.FromMinutes(10));
+    }
+
+    [Fact]
+    public async Task Killed_process_removes_its_container()
+    {
+        var start = new ProcessStartInfo("dotnet") { RedirectStandardOutput = true };
+        start.ArgumentList.Add(typeof(ReaperKillTests).Assembly.Location);
+        start.Environment.Remove("METAOBJECTS_TEST_PG_URL");
+        using var child = Process.Start(start)!;
+        var container = "";
+        try
+        {
+            var port = await AwaitPortAsync(child);
+            container = await DockerAsync("ps", "--filter", $"publish={port}/tcp", "--format", "{{.Names}}");
+            Assert.NotEqual("", container);
+
+            child.Kill();
+            Assert.True(child.WaitForExit(30_000), "the child outlived SIGKILL");
+
+            var deadline = DateTime.UtcNow + RemovalBound;
+            while (await ExistsAsync(container) && DateTime.UtcNow < deadline)
+            {
+                await Task.Delay(250);
+            }
+            Assert.False(await ExistsAsync(container), $"{container} outlived its killed process");
+        }
+        finally
+        {
+            child.Kill();
+            if (container != "") await DockerAsync("rm", "-f", "-v", container);
+        }
+    }
+
+    private static async Task<string> AwaitPortAsync(Process child)
+    {
+        using var timeout = new CancellationTokenSource(ChildReady);
+        while (await child.StandardOutput.ReadLineAsync(timeout.Token) is { } line)
+        {
+            var ready = Regex.Match(line, @"^READY (\d+)$");
+            if (ready.Success) return ready.Groups[1].Value;
+        }
+        throw new InvalidOperationException("reaper child exited without reporting its container");
+    }
+
+    private static async Task<bool> ExistsAsync(string container) =>
+        await DockerAsync("ps", "-a", "--filter", $"name=^{container}$", "--format", "{{.Names}}") != "";
+
+    private static async Task<string> DockerAsync(params string[] args)
+    {
+        var start = new ProcessStartInfo("docker") { RedirectStandardOutput = true, RedirectStandardError = true };
+        foreach (var arg in args) start.ArgumentList.Add(arg);
+        using var docker = Process.Start(start)!;
+        using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+        var output = docker.StandardOutput.ReadToEndAsync(timeout.Token);
+        var error = docker.StandardError.ReadToEndAsync(timeout.Token);
+        await docker.WaitForExitAsync(timeout.Token);
+        Assert.True(docker.ExitCode == 0, $"docker {string.Join(' ', args)}: {await error}");
+        return (await output).Trim();
+    }
+}

server/csharp/MetaObjects.IntegrationTests/Runner/MySqlDatabase.cs

@@ -31,7 +31,7 @@ private async Task<string> StartAsync()
         var existing = Environment.GetEnvironmentVariable(ExistingServerEnv);
         if (!string.IsNullOrWhiteSpace(existing)) return FromUrl(existing);
 
-        _container = new MySqlBuilder().WithImage("mysql:8.4").WithUsername("root")
+        _container = new MySqlBuilder("mysql:8.4").WithUsername("root")
             .WithCommand("--innodb-use-native-aio=0").Build();
         await _container.StartAsync();
         return _container.GetConnectionString();

server/csharp/MetaObjects.IntegrationTests/Runner/PostgresContainer.cs

@@ -47,9 +47,7 @@ public static async Task<PostgresContainer> StartAsync()
             return await StartOnSharedAsync(sharedUrl);
         }
 
-        var container = new PostgreSqlBuilder()
-            .WithImage("postgres:16-alpine")
-            .Build();
+        var container = new PostgreSqlBuilder("postgres:16-alpine").Build();
         await container.StartAsync();
         return new PostgresContainer(container);
     }

server/java/codegen-kotlin-exposed1x-check/pom.xml

@@ -65,6 +65,7 @@
     <exposed1x.kotlin.stdlib.version>2.3.20</exposed1x.kotlin.stdlib.version>
     <jackson.version>2.18.2</jackson.version>
     <postgres.jdbc.version>42.7.4</postgres.jdbc.version>
+    <testcontainers.version>2.0.5</testcontainers.version>
     <!-- Controller-tier compile pass (issue #390 review fix) — the SAME version
          integration-tests-kotlin's SP-F lane already compiles the 0.x generated controller
          against, so "compatible with Spring 6.2 / Boot 3.5" is proven by precedent, not guessed. -->
@@ -222,8 +223,16 @@
       <scope>test</scope>
     </dependency>
 
-    <!-- Round-trip test: a real Postgres (via PostgresContainer, docker CLI — see
-         integration-tests-kotlin's copy for why not testcontainers-java) + its JDBC driver. -->
+    <!-- Ryuk only: removes this suite's docker-CLI containers when a killed JVM never reaches
+         close() (see ReaperLabels). Needs 2.x: the 1.21 line cannot talk to Docker Engine 29+. -->
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>testcontainers</artifactId>
+      <version>${testcontainers.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+    <!-- Round-trip test: a real Postgres (via PostgresContainer, docker CLI) + its JDBC driver. -->
     <dependency>
       <groupId>org.postgresql</groupId>
       <artifactId>postgresql</artifactId>