Align runner lifecycle cancellation and prove lifecycle behavior across adapters

.github/workflows/build.yml

@@ -14,6 +14,7 @@ on:
     paths:
       - "Source/**"
       - "Integration/Consumers/**"
+      - "Integration/Lifecycle/**"
       - "Directory.Build.props"
       - "Directory.Packages.props"
       - "global.json"
@@ -24,6 +25,7 @@ on:
     paths:
       - "Source/**"
       - "Integration/Consumers/**"
+      - "Integration/Lifecycle/**"
       - "Directory.Build.props"
       - "Directory.Packages.props"
       - "global.json"
@@ -198,6 +200,28 @@ jobs:
           dotnet build Integration/Consumers/AssertionXml/First/First.csproj -c Release --no-restore -p:AssertionProbeAssemblyName=Probe.Renamed
           node Integration/Consumers/AssertionXml/verify-generation.mjs First Probe.Renamed
 
+      - name: Verify shared lifecycle failures and cancellation through all adapters
+        shell: bash
+        run: |
+          set -euo pipefail
+          for consumer in \
+            'XUnit XUnit/Consumer.XUnit.csproj' \
+            'XUnitV3 XUnit.v3/XUnit.v3.csproj' \
+            'NUnit4 NUnit/Consumer.NUnit.csproj' \
+            'NUnit5 NUnit5/NUnit5.csproj' \
+            'TUnit TUnit/Consumer.TUnit.csproj' \
+            'Runner Runner/Runner.csproj'; do
+            adapter="${consumer%% *}"
+            stop=false
+            if [ "$adapter" = Runner ]; then stop=true; fi
+            dotnet build "Integration/Consumers/${consumer#* }" --configuration Release --no-restore \
+              -p:LifecycleProof=true -p:PlatformStopProof="$stop"
+            node Integration/Lifecycle/verify.mjs "$adapter"
+          done
+
+      - name: Verify MTP server-session Stop with concurrent classes
+        run: node Integration/Lifecycle/stop.mjs
+
       - name: Remove packed packages from the NuGet cache
         if: always()
         run: rm -rf ~/.nuget/packages/cratis.specifications*/0.0.0-ci

Documentation/runner.md

@@ -81,16 +81,20 @@ starts previously failed classes first with `--last-failed-first`.
 Cancelling the run (for example Ctrl+C, or Stop in the IDE) and `--fail-fast` both stop scheduling:
 
 - No new class is started after the stop.
-- A class that is already running finishes: its facts, `Destroy` and disposal always run.
-- Facts of classes that never started are reported as skipped, so every selected fact ends in a terminal state.
+- A class that is already running completes its lifecycle; `Destroy` and disposal are attempted once. A cancelled
+  setup or act still prevents its fact bodies from running, just like any other setup or act failure.
+- The runner emits skipped results for facts of classes that never started. Hard IDE cancellation can prevent MTP
+  from delivering these results to the client; see [limits](#limits).
 - Running code can cooperate through `SpecificationContext.CancellationToken`, which flows across awaits and is
   isolated per class. Cancellation never interrupts running code forcibly.
 
 ## Failures in setup, act and teardown
 
 If `Establish`/`Given` fails, `Because`/`When` and the fact bodies do not run. If `Because`/`When` fails, the fact
 bodies do not run. In either case, the selected non-skipped facts fail with the original exception; `Destroy` and
-disposal are still attempted once when an instance was created.
+disposal are still attempted once when an instance was created. Lifecycle cancellation is wrapped in
+`SpecificationCancelledException`, with the lifecycle phase in its message and the original cancellation as its
+inner exception, matching the reflection-based adapters.
 
 A failure in `Destroy`, disposal or ending the specification scope produces **one separate failed result for the
 class**, displayed as `<Class> (teardown)`. Multiple cleanup errors are aggregated into that result. Its stable UID
@@ -105,6 +109,13 @@ adopt that engine-owned behavior.
 
 ## Limits
 
+- **Hard IDE Stop can suppress result delivery.** With Microsoft.Testing.Platform 2.5.1, server-mode
+  `$/cancelRequest` cancels the request and stops forwarding subsequent test updates. The runner still cancels
+  cooperative running code, attempts `Destroy` and disposal once per started class, and does not construct queued
+  classes, but the client may receive no terminal results, including skipped results for queued facts. The server
+  acknowledges the cancelled request with JSON-RPC error `-32800`; its process exits successfully when the client
+  subsequently sends `exit`. That process exit is not a passing test run. This is a platform transport limitation,
+  not a guarantee of terminal-result parity with an uncancelled run or `--fail-fast`.
 - **Methods must return `void`, `Task` or `ValueTask`.** A fact with another return type fails its class with a
   discovery error. An `async void` fact or lifecycle method is a compile error (`CRSPEC0010`), because its failures can
   not be awaited.

Integration/Consumers/Directory.Build.targets

@@ -0,0 +1,10 @@
+<Project>
+    <!-- Expected-failure fixtures run only in the lifecycle proof, never in the ordinary green consumers. -->
+    <ItemGroup Condition="'$(LifecycleProof)' == 'true'">
+        <Compile Remove="../when_using_given_and_when.cs;../when_running_with_the_package_runtime_defaults.cs" />
+        <Compile Include="../../Lifecycle/lifecycle_probes.cs;../../Lifecycle/Cases/*.cs" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(LifecycleProof)' == 'true' And '$(PlatformStopProof)' == 'true'">
+        <Compile Include="../../Lifecycle/stop_probes.cs;../../Lifecycle/Stop/*.cs" />
+    </ItemGroup>
+</Project>

Integration/Lifecycle/Cases/act_cancellation.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class act_cancellation : lifecycle_context
+{
+    void Because() => throw new OperationCanceledException("deliberate act cancellation");
+}

Integration/Lifecycle/Cases/act_failure.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class act_failure : lifecycle_context
+{
+    void Because() => throw new InvalidOperationException("deliberate act failure");
+}

Integration/Lifecycle/Cases/async_act_cancellation.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_act_cancellation : lifecycle_context
+{
+    async ValueTask Because()
+    {
+        await Task.Yield();
+        throw new TaskCanceledException("deliberate act cancellation");
+    }
+}

Integration/Lifecycle/Cases/async_act_failure.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_act_failure : lifecycle_context
+{
+    async ValueTask Because()
+    {
+        await Task.Yield();
+        throw new InvalidOperationException("deliberate act failure");
+    }
+}

Integration/Lifecycle/Cases/async_cleanup_cancellation.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_cleanup_cancellation : lifecycle_context
+{
+    async Task Destroy()
+    {
+        await Task.Yield();
+        throw new TaskCanceledException("deliberate cleanup cancellation");
+    }
+}

Integration/Lifecycle/Cases/async_cleanup_failure.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_cleanup_failure : lifecycle_context
+{
+    async Task Destroy()
+    {
+        await Task.Yield();
+        throw new InvalidOperationException("deliberate cleanup failure");
+    }
+}

Integration/Lifecycle/Cases/async_setup_cancellation.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_setup_cancellation : lifecycle_context
+{
+    async Task Establish()
+    {
+        await Task.Yield();
+        throw new TaskCanceledException("deliberate setup cancellation");
+    }
+}

Integration/Lifecycle/Cases/async_setup_failure.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class async_setup_failure : lifecycle_context
+{
+    async Task Establish()
+    {
+        await Task.Yield();
+        throw new InvalidOperationException("deliberate setup failure");
+    }
+}

Integration/Lifecycle/Cases/cleanup_cancellation.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class cleanup_cancellation : lifecycle_context
+{
+    void Destroy() => throw new OperationCanceledException("deliberate cleanup cancellation");
+}

Integration/Lifecycle/Cases/cleanup_failure.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class cleanup_failure : lifecycle_context
+{
+    void Destroy() => throw new InvalidOperationException("deliberate cleanup failure");
+}

Integration/Lifecycle/Cases/setup_cancellation.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class setup_cancellation : lifecycle_context
+{
+    void Establish() => throw new OperationCanceledException("deliberate setup cancellation");
+}

Integration/Lifecycle/Cases/setup_failure.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace LifecycleProbes;
+
+#if TUNIT
+[global::TUnit.Core.InheritsTests]
+#endif
+public class setup_failure : lifecycle_context
+{
+    void Establish() => throw new InvalidOperationException("deliberate setup failure");
+}

Integration/Lifecycle/Stop/A_first.cs

@@ -0,0 +1,6 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace StopProbes;
+
+public class A_first : running_context;

Integration/Lifecycle/Stop/B_second.cs

@@ -0,0 +1,6 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace StopProbes;
+
+public class B_second : running_context;

Integration/Lifecycle/Stop/Z_queued.cs

@@ -0,0 +1,6 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace StopProbes;
+
+public class Z_queued : running_context;

Integration/Lifecycle/lifecycle_probes.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Cratis.Specifications;
+
+namespace LifecycleProbes;
+
+public abstract class lifecycle_context : Specification, IDisposable
+{
+    static readonly Lock Gate = new();
+
+    protected void Record(string operation)
+    {
+        var path = Environment.GetEnvironmentVariable("CRATIS_LIFECYCLE_TRACE");
+        if (path is null) return;
+        lock (Gate) File.AppendAllText(path, $"{GetType().Name}:{operation}{Environment.NewLine}");
+    }
+
+    void Establish() => Record("Establish");
+    void Because() => Record("Because");
+    void Destroy() => Record("Destroy");
+    public void Dispose() => Record("Dispose");
+
+#if XUNIT
+    [Xunit.Fact]
+#elif NUNIT
+    [NUnit.Framework.Test]
+#elif TUNIT
+    [global::TUnit.Core.Test]
+#endif
+    public void should_run_first() => Record("fact1");
+
+#if XUNIT
+    [Xunit.Fact]
+#elif NUNIT
+    [NUnit.Framework.Test]
+#elif TUNIT
+    [global::TUnit.Core.Test]
+#endif
+    public void should_run_second() => Record("fact2");
+}

Integration/Lifecycle/rpc.mjs

@@ -0,0 +1,49 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import assert from 'node:assert/strict';
+
+// MTP server-mode JSON-RPC uses UTF-8 Content-Length framing, not newline-delimited JSON.
+export function connect(socket) {
+    let buffer = Buffer.alloc(0);
+    let sequence = 0;
+    const pending = new Map();
+    const updates = [];
+    const send = message => {
+        const body = Buffer.from(JSON.stringify({ jsonrpc: '2.0', ...message }));
+        socket.write(Buffer.concat([Buffer.from(`Content-Length: ${body.length}\r\n\r\n`), body]));
+    };
+    const dispatch = message => {
+        if (message.method) {
+            updates.push(message);
+            if (message.id !== undefined) send({ id: message.id, result: null });
+            return;
+        }
+        assert.ok(pending.has(message.id), `Unexpected RPC response ${message.id}`);
+        pending.get(message.id)(message);
+        pending.delete(message.id);
+    };
+    socket.on('data', chunk => {
+        buffer = Buffer.concat([buffer, chunk]);
+        while (true) {
+            const end = buffer.indexOf('\r\n\r\n');
+            if (end < 0) return;
+            const length = Number(buffer.subarray(0, end).toString().match(/Content-Length:\s*(\d+)/i)?.[1]);
+            assert.ok(Number.isSafeInteger(length) && length > 0, 'Valid MTP frame required');
+            if (buffer.length < end + 4 + length) return;
+            const message = JSON.parse(buffer.subarray(end + 4, end + 4 + length).toString());
+            buffer = buffer.subarray(end + 4 + length);
+            dispatch(message);
+        }
+    });
+    return {
+        updates,
+        notify: (method, params) => send({ method, params }),
+        request(method, params) {
+            const id = ++sequence;
+            const response = new Promise(resolve => pending.set(id, resolve));
+            send({ id, method, params });
+            return { id, response };
+        },
+    };
+}