From aa8b2bf313281691e6b3cec62be32fc8d16d603d Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 11:53:56 +0100 Subject: [PATCH 1/2] Align runner lifecycle cancellation and prove lifecycle parity Generated lifecycle calls now wrap cancellation the same way as the reflection path. CI runs twelve shared lifecycle cases through every adapter and a Microsoft.Testing.Platform server-session stop with concurrent classes, asserting cleanup runs once per started class. --- .github/workflows/build.yml | 22 +++++ Documentation/runner.md | 17 +++- Integration/Consumers/Directory.Build.targets | 10 +++ .../Lifecycle/Cases/act_cancellation.cs | 12 +++ Integration/Lifecycle/Cases/act_failure.cs | 12 +++ .../Lifecycle/Cases/async_act_cancellation.cs | 16 ++++ .../Lifecycle/Cases/async_act_failure.cs | 16 ++++ .../Cases/async_cleanup_cancellation.cs | 16 ++++ .../Lifecycle/Cases/async_cleanup_failure.cs | 16 ++++ .../Cases/async_setup_cancellation.cs | 16 ++++ .../Lifecycle/Cases/async_setup_failure.cs | 16 ++++ .../Lifecycle/Cases/cleanup_cancellation.cs | 12 +++ .../Lifecycle/Cases/cleanup_failure.cs | 12 +++ .../Lifecycle/Cases/setup_cancellation.cs | 12 +++ Integration/Lifecycle/Cases/setup_failure.cs | 12 +++ Integration/Lifecycle/Stop/A_first.cs | 6 ++ Integration/Lifecycle/Stop/B_second.cs | 6 ++ Integration/Lifecycle/Stop/Z_queued.cs | 6 ++ Integration/Lifecycle/lifecycle_probes.cs | 41 +++++++++ Integration/Lifecycle/rpc.mjs | 49 ++++++++++ Integration/Lifecycle/stop.mjs | 89 +++++++++++++++++++ Integration/Lifecycle/stop_probes.cs | 25 ++++++ Integration/Lifecycle/verify.mjs | 83 +++++++++++++++++ .../Runner.Generator/SpecificationEmitter.cs | 11 ++- .../Runner.Specs/Probes/ContractProbes.csproj | 1 + .../when_cancelling_lifecycle_methods.cs | 85 ++++++++++++++++++ 26 files changed, 614 insertions(+), 5 deletions(-) create mode 100644 Integration/Consumers/Directory.Build.targets create mode 100644 Integration/Lifecycle/Cases/act_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/act_failure.cs create mode 100644 Integration/Lifecycle/Cases/async_act_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/async_act_failure.cs create mode 100644 Integration/Lifecycle/Cases/async_cleanup_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/async_cleanup_failure.cs create mode 100644 Integration/Lifecycle/Cases/async_setup_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/async_setup_failure.cs create mode 100644 Integration/Lifecycle/Cases/cleanup_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/cleanup_failure.cs create mode 100644 Integration/Lifecycle/Cases/setup_cancellation.cs create mode 100644 Integration/Lifecycle/Cases/setup_failure.cs create mode 100644 Integration/Lifecycle/Stop/A_first.cs create mode 100644 Integration/Lifecycle/Stop/B_second.cs create mode 100644 Integration/Lifecycle/Stop/Z_queued.cs create mode 100644 Integration/Lifecycle/lifecycle_probes.cs create mode 100644 Integration/Lifecycle/rpc.mjs create mode 100644 Integration/Lifecycle/stop.mjs create mode 100644 Integration/Lifecycle/stop_probes.cs create mode 100644 Integration/Lifecycle/verify.mjs create mode 100644 Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8276fa..95e8c3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,6 +198,28 @@ jobs: dotnet build Integration/Consumers/AssertionXml/First/First.csproj -c Release --no-restore -p:AssertionProbeAssemblyName=Probe.Renamed node Integration/Consumers/AssertionXml/verify-generation.mjs First Probe.Renamed + - name: Verify shared lifecycle failures and cancellation through all adapters + shell: bash + run: | + set -euo pipefail + for consumer in \ + 'XUnit XUnit/Consumer.XUnit.csproj' \ + 'XUnitV3 XUnit.v3/XUnit.v3.csproj' \ + 'NUnit4 NUnit/Consumer.NUnit.csproj' \ + 'NUnit5 NUnit5/NUnit5.csproj' \ + 'TUnit TUnit/Consumer.TUnit.csproj' \ + 'Runner Runner/Runner.csproj'; do + adapter="${consumer%% *}" + stop=false + if [ "$adapter" = Runner ]; then stop=true; fi + dotnet build "Integration/Consumers/${consumer#* }" --configuration Release --no-restore \ + -p:LifecycleProof=true -p:PlatformStopProof="$stop" + node Integration/Lifecycle/verify.mjs "$adapter" + done + + - name: Verify MTP server-session Stop with concurrent classes + run: node Integration/Lifecycle/stop.mjs + - name: Remove packed packages from the NuGet cache if: always() run: rm -rf ~/.nuget/packages/cratis.specifications*/0.0.0-ci diff --git a/Documentation/runner.md b/Documentation/runner.md index 5b30667..aa27f4c 100644 --- a/Documentation/runner.md +++ b/Documentation/runner.md @@ -81,8 +81,10 @@ starts previously failed classes first with `--last-failed-first`. Cancelling the run (for example Ctrl+C, or Stop in the IDE) and `--fail-fast` both stop scheduling: - No new class is started after the stop. -- A class that is already running finishes: its facts, `Destroy` and disposal always run. -- Facts of classes that never started are reported as skipped, so every selected fact ends in a terminal state. +- A class that is already running completes its lifecycle; `Destroy` and disposal are attempted once. A cancelled + setup or act still prevents its fact bodies from running, just like any other setup or act failure. +- The runner emits skipped results for facts of classes that never started. Hard IDE cancellation can prevent MTP + from delivering these results to the client; see [limits](#limits). - Running code can cooperate through `SpecificationContext.CancellationToken`, which flows across awaits and is isolated per class. Cancellation never interrupts running code forcibly. @@ -90,7 +92,9 @@ Cancelling the run (for example Ctrl+C, or Stop in the IDE) and `--fail-fast` bo If `Establish`/`Given` fails, `Because`/`When` and the fact bodies do not run. If `Because`/`When` fails, the fact bodies do not run. In either case, the selected non-skipped facts fail with the original exception; `Destroy` and -disposal are still attempted once when an instance was created. +disposal are still attempted once when an instance was created. Lifecycle cancellation is wrapped in +`SpecificationCancelledException`, with the lifecycle phase in its message and the original cancellation as its +inner exception, matching the reflection-based adapters. A failure in `Destroy`, disposal or ending the specification scope produces **one separate failed result for the class**, displayed as ` (teardown)`. Multiple cleanup errors are aggregated into that result. Its stable UID @@ -105,6 +109,13 @@ adopt that engine-owned behavior. ## Limits +- **Hard IDE Stop can suppress result delivery.** With Microsoft.Testing.Platform 2.5.1, server-mode + `$/cancelRequest` cancels the request and stops forwarding subsequent test updates. The runner still cancels + cooperative running code, attempts `Destroy` and disposal once per started class, and does not construct queued + classes, but the client may receive no terminal results, including skipped results for queued facts. The server + acknowledges the cancelled request with JSON-RPC error `-32800`; its process exits successfully when the client + subsequently sends `exit`. That process exit is not a passing test run. This is a platform transport limitation, + not a guarantee of terminal-result parity with an uncancelled run or `--fail-fast`. - **Methods must return `void`, `Task` or `ValueTask`.** A fact with another return type fails its class with a discovery error. An `async void` fact or lifecycle method is a compile error (`CRSPEC0010`), because its failures can not be awaited. diff --git a/Integration/Consumers/Directory.Build.targets b/Integration/Consumers/Directory.Build.targets new file mode 100644 index 0000000..fc9b8a8 --- /dev/null +++ b/Integration/Consumers/Directory.Build.targets @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Integration/Lifecycle/Cases/act_cancellation.cs b/Integration/Lifecycle/Cases/act_cancellation.cs new file mode 100644 index 0000000..64feebb --- /dev/null +++ b/Integration/Lifecycle/Cases/act_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class act_cancellation : lifecycle_context +{ + void Because() => throw new OperationCanceledException("deliberate act cancellation"); +} diff --git a/Integration/Lifecycle/Cases/act_failure.cs b/Integration/Lifecycle/Cases/act_failure.cs new file mode 100644 index 0000000..c336361 --- /dev/null +++ b/Integration/Lifecycle/Cases/act_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class act_failure : lifecycle_context +{ + void Because() => throw new InvalidOperationException("deliberate act failure"); +} diff --git a/Integration/Lifecycle/Cases/async_act_cancellation.cs b/Integration/Lifecycle/Cases/async_act_cancellation.cs new file mode 100644 index 0000000..588db5a --- /dev/null +++ b/Integration/Lifecycle/Cases/async_act_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_act_cancellation : lifecycle_context +{ + async ValueTask Because() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate act cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_act_failure.cs b/Integration/Lifecycle/Cases/async_act_failure.cs new file mode 100644 index 0000000..bf3f285 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_act_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_act_failure : lifecycle_context +{ + async ValueTask Because() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate act failure"); + } +} diff --git a/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs b/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs new file mode 100644 index 0000000..4ca9f84 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_cleanup_cancellation : lifecycle_context +{ + async Task Destroy() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate cleanup cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_cleanup_failure.cs b/Integration/Lifecycle/Cases/async_cleanup_failure.cs new file mode 100644 index 0000000..f7334b5 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_cleanup_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_cleanup_failure : lifecycle_context +{ + async Task Destroy() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate cleanup failure"); + } +} diff --git a/Integration/Lifecycle/Cases/async_setup_cancellation.cs b/Integration/Lifecycle/Cases/async_setup_cancellation.cs new file mode 100644 index 0000000..a6745cc --- /dev/null +++ b/Integration/Lifecycle/Cases/async_setup_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_setup_cancellation : lifecycle_context +{ + async Task Establish() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate setup cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_setup_failure.cs b/Integration/Lifecycle/Cases/async_setup_failure.cs new file mode 100644 index 0000000..4cf83a3 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_setup_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_setup_failure : lifecycle_context +{ + async Task Establish() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate setup failure"); + } +} diff --git a/Integration/Lifecycle/Cases/cleanup_cancellation.cs b/Integration/Lifecycle/Cases/cleanup_cancellation.cs new file mode 100644 index 0000000..a9bd963 --- /dev/null +++ b/Integration/Lifecycle/Cases/cleanup_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class cleanup_cancellation : lifecycle_context +{ + void Destroy() => throw new OperationCanceledException("deliberate cleanup cancellation"); +} diff --git a/Integration/Lifecycle/Cases/cleanup_failure.cs b/Integration/Lifecycle/Cases/cleanup_failure.cs new file mode 100644 index 0000000..9579ad1 --- /dev/null +++ b/Integration/Lifecycle/Cases/cleanup_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class cleanup_failure : lifecycle_context +{ + void Destroy() => throw new InvalidOperationException("deliberate cleanup failure"); +} diff --git a/Integration/Lifecycle/Cases/setup_cancellation.cs b/Integration/Lifecycle/Cases/setup_cancellation.cs new file mode 100644 index 0000000..93a03d6 --- /dev/null +++ b/Integration/Lifecycle/Cases/setup_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class setup_cancellation : lifecycle_context +{ + void Establish() => throw new OperationCanceledException("deliberate setup cancellation"); +} diff --git a/Integration/Lifecycle/Cases/setup_failure.cs b/Integration/Lifecycle/Cases/setup_failure.cs new file mode 100644 index 0000000..ee3e09d --- /dev/null +++ b/Integration/Lifecycle/Cases/setup_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class setup_failure : lifecycle_context +{ + void Establish() => throw new InvalidOperationException("deliberate setup failure"); +} diff --git a/Integration/Lifecycle/Stop/A_first.cs b/Integration/Lifecycle/Stop/A_first.cs new file mode 100644 index 0000000..b301ede --- /dev/null +++ b/Integration/Lifecycle/Stop/A_first.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class A_first : running_context; diff --git a/Integration/Lifecycle/Stop/B_second.cs b/Integration/Lifecycle/Stop/B_second.cs new file mode 100644 index 0000000..8f71a37 --- /dev/null +++ b/Integration/Lifecycle/Stop/B_second.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class B_second : running_context; diff --git a/Integration/Lifecycle/Stop/Z_queued.cs b/Integration/Lifecycle/Stop/Z_queued.cs new file mode 100644 index 0000000..9f82dda --- /dev/null +++ b/Integration/Lifecycle/Stop/Z_queued.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class Z_queued : running_context; diff --git a/Integration/Lifecycle/lifecycle_probes.cs b/Integration/Lifecycle/lifecycle_probes.cs new file mode 100644 index 0000000..0722323 --- /dev/null +++ b/Integration/Lifecycle/lifecycle_probes.cs @@ -0,0 +1,41 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Specifications; + +namespace LifecycleProbes; + +public abstract class lifecycle_context : Specification, IDisposable +{ + static readonly Lock Gate = new(); + + protected void Record(string operation) + { + var path = Environment.GetEnvironmentVariable("CRATIS_LIFECYCLE_TRACE"); + if (path is null) return; + lock (Gate) File.AppendAllText(path, $"{GetType().Name}:{operation}{Environment.NewLine}"); + } + + void Establish() => Record("Establish"); + void Because() => Record("Because"); + void Destroy() => Record("Destroy"); + public void Dispose() => Record("Dispose"); + +#if XUNIT + [Xunit.Fact] +#elif NUNIT + [NUnit.Framework.Test] +#elif TUNIT + [global::TUnit.Core.Test] +#endif + public void should_run_first() => Record("fact1"); + +#if XUNIT + [Xunit.Fact] +#elif NUNIT + [NUnit.Framework.Test] +#elif TUNIT + [global::TUnit.Core.Test] +#endif + public void should_run_second() => Record("fact2"); +} diff --git a/Integration/Lifecycle/rpc.mjs b/Integration/Lifecycle/rpc.mjs new file mode 100644 index 0000000..ce9096a --- /dev/null +++ b/Integration/Lifecycle/rpc.mjs @@ -0,0 +1,49 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; + +// MTP server-mode JSON-RPC uses UTF-8 Content-Length framing, not newline-delimited JSON. +export function connect(socket) { + let buffer = Buffer.alloc(0); + let sequence = 0; + const pending = new Map(); + const updates = []; + const send = message => { + const body = Buffer.from(JSON.stringify({ jsonrpc: '2.0', ...message })); + socket.write(Buffer.concat([Buffer.from(`Content-Length: ${body.length}\r\n\r\n`), body])); + }; + const dispatch = message => { + if (message.method) { + updates.push(message); + if (message.id !== undefined) send({ id: message.id, result: null }); + return; + } + assert.ok(pending.has(message.id), `Unexpected RPC response ${message.id}`); + pending.get(message.id)(message); + pending.delete(message.id); + }; + socket.on('data', chunk => { + buffer = Buffer.concat([buffer, chunk]); + while (true) { + const end = buffer.indexOf('\r\n\r\n'); + if (end < 0) return; + const length = Number(buffer.subarray(0, end).toString().match(/Content-Length:\s*(\d+)/i)?.[1]); + assert.ok(Number.isSafeInteger(length) && length > 0, 'Valid MTP frame required'); + if (buffer.length < end + 4 + length) return; + const message = JSON.parse(buffer.subarray(end + 4, end + 4 + length).toString()); + buffer = buffer.subarray(end + 4 + length); + dispatch(message); + } + }); + return { + updates, + notify: (method, params) => send({ method, params }), + request(method, params) { + const id = ++sequence; + const response = new Promise(resolve => pending.set(id, resolve)); + send({ id, method, params }); + return { id, response }; + }, + }; +} diff --git a/Integration/Lifecycle/stop.mjs b/Integration/Lifecycle/stop.mjs new file mode 100644 index 0000000..729c50d --- /dev/null +++ b/Integration/Lifecycle/stop.mjs @@ -0,0 +1,89 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdirSync, mkdtempSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { once } from 'node:events'; +import { join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { connect } from './rpc.mjs'; + +const output = resolve(process.argv[2] ?? 'artifacts/lifecycle'); +mkdirSync(output, { recursive: true }); +const directory = mkdtempSync(join(output, 'Stop-')); +const trace = join(directory, 'trace.txt'); +const server = createServer(); +server.listen(0, '127.0.0.1'); +await once(server, 'listening'); +const connection = once(server, 'connection'); +const child = spawn('dotnet', [ + 'Integration/Consumers/Runner/bin/Release/net10.0/Runner.dll', + '--server', '--client-port', String(server.address().port), + '--specs-parallel', '2', '--specs-cache', 'none', +], { env: { ...process.env, CRATIS_LIFECYCLE_TRACE: trace }, stdio: ['ignore', 'pipe', 'pipe'] }); +let stdout = ''; +let stderr = ''; +child.stdout.on('data', chunk => { stdout += chunk; }); +child.stderr.on('data', chunk => { stderr += chunk; }); +const exit = once(child, 'exit'); +let socket; +const watchdog = setTimeout(() => child.kill('SIGKILL'), 30_000); +const bounded = promise => Promise.race([promise, delay(20_000, null, { ref: false }).then(() => { throw new Error('MTP protocol operation timed out'); })]); +const lines = () => existsSync(trace) ? readFileSync(trace, 'utf8').trim().split(/\r?\n/) : []; +async function until(predicate) { + const deadline = Date.now() + 20_000; + while (!predicate()) { + assert.equal(child.exitCode, null, `Host exited before the barrier: ${stderr}`); + assert.ok(Date.now() < deadline, `Timed out waiting for probe barrier: ${lines().join(', ')}`); + await delay(10); + } +} +try { + [socket] = await bounded(connection); + const rpc = connect(socket); + const initialized = await bounded(rpc.request('initialize', { + processId: process.pid, + clientInfo: { name: 'Cratis lifecycle proof', version: '1.0.0' }, + capabilities: { testing: { debuggerProvider: false, isStateful: false } }, + protocolVersions: ['1.0.0'], + }).response); + assert.equal(initialized.error, undefined, 'MTP initialization must succeed'); + const filter = '/*/StopProbes/*/*'; + const discovered = await bounded(rpc.request('testing/discoverTests', { runId: randomUUID(), filter }).response); + assert.equal(discovered.error, undefined, 'MTP discovery must succeed'); + const discoveryUpdates = rpc.updates.splice(0); + const selected = discoveryUpdates.filter(update => update.method === 'testing/testUpdates/tests').flatMap(update => update.params.changes ?? []); + assert.equal(selected.length, 6, 'Real MTP discovery must deliver all six selected facts before Stop'); + const run = rpc.request('testing/runTests', { runId: randomUUID(), filter }); + await until(() => ['A_first', 'B_second'].every(name => lines().includes(`${name}:Started`))); + assert.ok(!lines().some(line => line.startsWith('Z_queued:')), 'Third class must still be queued'); + rpc.notify('$/cancelRequest', { id: run.id }); + await until(() => ['A_first', 'B_second'].every(name => lines().includes(`${name}:Dispose`))); + const response = await bounded(run.response); + writeFileSync(join(directory, 'messages.json'), JSON.stringify({ discoveryUpdates, response, updates: rpc.updates }, null, 2)); + assert.equal(response.error?.code, -32800, 'MTP must acknowledge the request as cancelled'); + const changes = rpc.updates.filter(update => update.method === 'testing/testUpdates/tests').flatMap(update => update.params.changes ?? []); + assert.equal(changes.length, 0, 'MTP 2.5.1 hard cancellation suppresses subsequent result updates, including queued facts'); + rpc.notify('exit', null); + const [code, signal] = await bounded(exit); + assert.equal(signal, null, 'MTP host must exit normally after Stop'); + assert.equal(code, 0, 'Server-mode exit after a cancelled request'); + for (const name of ['A_first', 'B_second']) { + for (const operation of ['Create', 'Started', 'Cancelled', 'Destroy', 'Dispose']) { + assert.equal(lines().filter(line => line === `${name}:${operation}`).length, 1, `${name}: ${operation} exactly once`); + } + assert.ok(!lines().some(line => line.startsWith(`${name}:fact`)), `${name}: cancelled act must not run facts`); + } + assert.ok(!lines().some(line => line.startsWith('Z_queued:')), 'Cancelled scheduling must not construct queued class'); + console.log(`MTP hard Stop: two concurrent classes cancelled; Destroy/Dispose once each; queued class not constructed; six facts discovered, zero result updates after hard cancellation; exit ${code}. Messages: ${directory}`); +} finally { + clearTimeout(watchdog); + socket?.destroy(); + server.close(); + if (child.exitCode === null) child.kill('SIGKILL'); + writeFileSync(join(directory, 'stdout.txt'), stdout); + writeFileSync(join(directory, 'stderr.txt'), stderr); +} diff --git a/Integration/Lifecycle/stop_probes.cs b/Integration/Lifecycle/stop_probes.cs new file mode 100644 index 0000000..789f706 --- /dev/null +++ b/Integration/Lifecycle/stop_probes.cs @@ -0,0 +1,25 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Specifications; +using LifecycleProbes; + +namespace StopProbes; + +public abstract class running_context : lifecycle_context +{ + protected running_context() => Record("Create"); + + async Task Because() + { + Record("Started"); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, SpecificationContext.CancellationToken); + } + finally + { + Record("Cancelled"); + } + } +} diff --git a/Integration/Lifecycle/verify.mjs b/Integration/Lifecycle/verify.mjs new file mode 100644 index 0000000..2d572c1 --- /dev/null +++ b/Integration/Lifecycle/verify.mjs @@ -0,0 +1,83 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; + +export const projects = { + XUnit: 'XUnit/Consumer.XUnit.csproj', + XUnitV3: 'XUnit.v3/XUnit.v3.csproj', + NUnit4: 'NUnit/Consumer.NUnit.csproj', + NUnit5: 'NUnit5/NUnit5.csproj', + TUnit: 'TUnit/Consumer.TUnit.csproj', + Runner: 'Runner/Runner.csproj', +}; + +const adapter = process.argv[2]; +assert.ok(Object.hasOwn(projects, adapter), `Unknown adapter: ${adapter}`); +const output = resolve(process.argv[3] ?? 'artifacts/lifecycle'); +mkdirSync(output, { recursive: true }); +const directory = mkdtempSync(join(output, `${adapter}-`)); +const trace = join(directory, 'trace.txt'); +const project = `Integration/Consumers/${projects[adapter]}`; +const mtp = adapter === 'TUnit' || adapter === 'Runner'; +const args = mtp + ? ['run', '--project', project, '-c', 'Release', '--no-build', '--', '--report-trx', '--report-trx-filename', 'result.trx', '--results-directory', directory] + : ['test', project, '-c', 'Release', '--no-build', '--logger', 'trx;LogFileName=result.trx', '--results-directory', directory]; +if (adapter === 'Runner') args.push('--specs-cache', 'none', '--treenode-filter', '/*/LifecycleProbes/*/*'); +const run = spawnSync('dotnet', args, { + encoding: 'utf8', timeout: 90_000, + env: { ...process.env, CRATIS_LIFECYCLE_TRACE: trace }, +}); +process.stdout.write(run.stdout ?? ''); +process.stderr.write(run.stderr ?? ''); +assert.ifError(run.error); +assert.equal(run.signal, null, 'The probe must exit without being killed'); +assert.equal(run.status, mtp ? 2 : 1, 'Expected native test-failure exit code'); +const xml = readFileSync(join(directory, 'result.trx'), 'utf8'); +const calls = readFileSync(trace, 'utf8').trim().split(/\r?\n/); +const cases = ['setup', 'act', 'cleanup'].flatMap(phase => + ['failure', 'cancellation'].flatMap(kind => [`${phase}_${kind}`, `async_${phase}_${kind}`])); +const attributes = text => Object.fromEntries([...text.matchAll(/([\w:]+)="([^"]*)"/g)].map(match => [match[1], match[2]])); +const definitions = new Map([...xml.matchAll(/]*)>([\s\S]*?)<\/UnitTest>/g)].map(match => { + const method = match[2].match(/]*)/); + return [attributes(match[1]).id, method ? attributes(method[1]).className : '']; +})); +const results = [...xml.matchAll(/]*)/g)].map(match => attributes(match[1])); +assert.equal(results.length, adapter === 'Runner' ? 28 : 24, 'Every expected fact/teardown must have a terminal result'); +assert.equal(new Set(results.map(result => result.executionId)).size, results.length, 'No duplicate terminal results'); +for (const name of cases) { + const operations = calls.filter(call => call.startsWith(`${name}:`)).map(call => call.slice(name.length + 1)); + assert.equal(operations.filter(call => call === 'Establish').length, 1, `${name}: setup once`); + assert.equal(operations.filter(call => call === 'Because').length, name.includes('setup') ? 0 : 1, `${name}: act eligibility`); + assert.equal(operations.filter(call => call === 'Destroy').length, 1, `${name}: cleanup once`); + const facts = operations.filter(call => call.startsWith('fact')).sort(); + assert.deepEqual(facts, name.includes('cleanup') ? ['fact1', 'fact2'] : [], `${name}: fact eligibility`); + const factResults = results.filter(result => definitions.get(result.testId)?.split(',')[0] === `LifecycleProbes.${name}`); + const ordinary = factResults.filter(result => !result.testName.includes('teardown')); + assert.equal(ordinary.length, 2, `${name}: both facts discovered through ${adapter}`); + if (!name.includes('cleanup')) { + assert.ok(ordinary.every(result => result.outcome === 'Failed'), `${name}: both facts failed`); + } else { + const expected = adapter === 'TUnit' ? ['Failed', 'Passed'] : ['Passed', 'Passed']; + assert.deepEqual(ordinary.map(result => result.outcome).sort(), expected, `${name}: preserve framework-owned cleanup reporting`); + if (adapter === 'Runner') { + const teardown = factResults.filter(result => result.testName.includes('teardown')); + assert.equal(teardown.length, 1, `${name}: separate native teardown result`); + assert.equal(teardown[0].outcome, 'Failed', `${name}: teardown fails the run`); + } else if (adapter.startsWith('XUnit')) { + assert.ok(`${run.stdout}${run.stderr}`.includes(`Test Class Cleanup Failure (LifecycleProbes.${name})`), `${name}: xUnit class cleanup diagnostic`); + } else if (adapter.startsWith('NUnit')) { + assert.ok(`${run.stdout}${run.stderr}`.includes(`TearDown failed for test fixture LifecycleProbes.${name}`), `${name}: NUnit fixture cleanup diagnostic`); + } + } +} +for (const phase of ['Establish', 'Because', 'Destroy']) { + assert.ok(xml.includes(`${phase}() was cancelled`), `${phase}: framework must report lifecycle cancellation`); +} +for (const phase of ['setup', 'act', 'cleanup']) { + assert.ok(xml.includes(`deliberate ${phase} failure`), `${phase}: original failure must be reported`); +} +console.log(`${adapter}: 12 shared lifecycle cases, ${results.length} terminal results, once-only cleanup verified (${directory})`); diff --git a/Source/Runner.Generator/SpecificationEmitter.cs b/Source/Runner.Generator/SpecificationEmitter.cs index 78748b1..8579d09 100644 --- a/Source/Runner.Generator/SpecificationEmitter.cs +++ b/Source/Runner.Generator/SpecificationEmitter.cs @@ -79,14 +79,21 @@ public static void Emit(StringBuilder builder, SpecificationModel model, string body.Append(" static ").Append(isAsync ? "async " : string.Empty).Append(ValueTask).Append(' ').Append(name).AppendLine("(object instance)"); body.AppendLine(" {"); body.Append(" var specification = (").Append(model.FullyQualifiedName).AppendLine(")instance;"); + body.AppendLine(" try"); + body.AppendLine(" {"); foreach (var call in calls) { var expression = Call(accessors, call, ref counter); var awaited = call.Returns != ReturnKind.Void || call.How == CallKind.Reflection; - body.Append(" ").Append(awaited ? "await " : string.Empty).Append(expression).AppendLine(";"); + body.Append(" ").Append(awaited ? "await " : string.Empty).Append(expression).AppendLine(";"); } - if (!isAsync) body.AppendLine(" return default;"); + if (!isAsync) body.AppendLine(" return default;"); + body.AppendLine(" }"); + body.AppendLine(" catch (global::System.OperationCanceledException exception)"); + body.AppendLine(" {"); + body.Append(" throw new global::Cratis.Specifications.SpecificationCancelledException(").Append(Literal(name)).AppendLine(", exception);"); + body.AppendLine(" }"); body.AppendLine(" }"); return name; } diff --git a/Source/Runner.Specs/Probes/ContractProbes.csproj b/Source/Runner.Specs/Probes/ContractProbes.csproj index e794e78..7a951e3 100644 --- a/Source/Runner.Specs/Probes/ContractProbes.csproj +++ b/Source/Runner.Specs/Probes/ContractProbes.csproj @@ -11,6 +11,7 @@ + diff --git a/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs b/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs new file mode 100644 index 0000000..c860bb5 --- /dev/null +++ b/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs @@ -0,0 +1,85 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.CompilerServices; +using LifecycleProbes; + +namespace Cratis.Specifications.Runner.for_LifecycleContract; + +public class when_cancelling_lifecycle_methods : Specification +{ + readonly List<(Exception Generated, Exception Reflected)> results = []; + + async Task Because() + { + await Compare("Establish"); + await Compare("Establish"); + await Compare("Because"); + await Compare("Because"); + await Compare("Destroy"); + await Compare("Destroy"); + } + + async Task Compare(string phase) + where T : lifecycle_context, new() + { + RuntimeHelpers.RunModuleConstructor(typeof(T).Assembly.ManifestModule.ModuleHandle); + var descriptor = given.contract_probes.WorkFor().Specification; + var generated = phase switch + { + "Establish" => descriptor.Establish, + "Because" => descriptor.Because, + _ => descriptor.Destroy + }; + results.Add((await Capture(() => generated(new T()).AsTask()), await Capture(() => Reflected(phase)))); + } + + static Task Reflected(string phase) + where T : lifecycle_context, new() => phase switch + { + "Establish" => SpecificationLifecycle.Establish(new T(), typeof(T), typeof(Specification)), + "Because" => SpecificationLifecycle.Because(new T(), typeof(T), typeof(Specification)), + _ => SpecificationLifecycle.Destroy(new T(), typeof(T), typeof(Specification)) + }; + + static async Task Capture(Func action) + { + try + { + await action(); + throw new InvalidOperationException("The lifecycle cancellation did not throw."); + } + catch (Exception exception) + { + return exception; + } + } + + void should_reproduce_all_six_sync_and_async_cases() => results.Count.ShouldEqual(6); + + void should_match_synchronous_reflection_cancellation() + { + foreach (var result in results.Where((_, index) => index % 2 == 0)) result.Generated.GetType().ShouldEqual(result.Reflected.GetType()); + } + + void should_match_asynchronous_reflection_cancellation() + { + foreach (var result in results.Where((_, index) => index % 2 != 0)) result.Generated.GetType().ShouldEqual(result.Reflected.GetType()); + } + + void should_preserve_the_phase_and_existing_message() + { + foreach (var result in results) result.Generated.Message.ShouldEqual(result.Reflected.Message); + } + + void should_preserve_the_original_cancellation() + { + foreach (var result in results) + { + result.Reflected.ShouldBeOfExactType(); + result.Generated.InnerException.ShouldNotBeNull(); + result.Generated.InnerException.GetType().ShouldEqual(result.Reflected.InnerException.GetType()); + result.Generated.InnerException?.Message.ShouldEqual(result.Reflected.InnerException?.Message); + } + } +} From db5c46e4c5506d5767d0c22959b53406e80c60e1 Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 11:56:16 +0100 Subject: [PATCH 2/2] Run the build when the lifecycle proof changes --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95e8c3c..10fb22e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ on: paths: - "Source/**" - "Integration/Consumers/**" + - "Integration/Lifecycle/**" - "Directory.Build.props" - "Directory.Packages.props" - "global.json" @@ -24,6 +25,7 @@ on: paths: - "Source/**" - "Integration/Consumers/**" + - "Integration/Lifecycle/**" - "Directory.Build.props" - "Directory.Packages.props" - "global.json"