diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8276fa..10fb22e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ on: paths: - "Source/**" - "Integration/Consumers/**" + - "Integration/Lifecycle/**" - "Directory.Build.props" - "Directory.Packages.props" - "global.json" @@ -24,6 +25,7 @@ on: paths: - "Source/**" - "Integration/Consumers/**" + - "Integration/Lifecycle/**" - "Directory.Build.props" - "Directory.Packages.props" - "global.json" @@ -198,6 +200,28 @@ jobs: dotnet build Integration/Consumers/AssertionXml/First/First.csproj -c Release --no-restore -p:AssertionProbeAssemblyName=Probe.Renamed node Integration/Consumers/AssertionXml/verify-generation.mjs First Probe.Renamed + - name: Verify shared lifecycle failures and cancellation through all adapters + shell: bash + run: | + set -euo pipefail + for consumer in \ + 'XUnit XUnit/Consumer.XUnit.csproj' \ + 'XUnitV3 XUnit.v3/XUnit.v3.csproj' \ + 'NUnit4 NUnit/Consumer.NUnit.csproj' \ + 'NUnit5 NUnit5/NUnit5.csproj' \ + 'TUnit TUnit/Consumer.TUnit.csproj' \ + 'Runner Runner/Runner.csproj'; do + adapter="${consumer%% *}" + stop=false + if [ "$adapter" = Runner ]; then stop=true; fi + dotnet build "Integration/Consumers/${consumer#* }" --configuration Release --no-restore \ + -p:LifecycleProof=true -p:PlatformStopProof="$stop" + node Integration/Lifecycle/verify.mjs "$adapter" + done + + - name: Verify MTP server-session Stop with concurrent classes + run: node Integration/Lifecycle/stop.mjs + - name: Remove packed packages from the NuGet cache if: always() run: rm -rf ~/.nuget/packages/cratis.specifications*/0.0.0-ci diff --git a/Documentation/runner.md b/Documentation/runner.md index 5b30667..aa27f4c 100644 --- a/Documentation/runner.md +++ b/Documentation/runner.md @@ -81,8 +81,10 @@ starts previously failed classes first with `--last-failed-first`. Cancelling the run (for example Ctrl+C, or Stop in the IDE) and `--fail-fast` both stop scheduling: - No new class is started after the stop. -- A class that is already running finishes: its facts, `Destroy` and disposal always run. -- Facts of classes that never started are reported as skipped, so every selected fact ends in a terminal state. +- A class that is already running completes its lifecycle; `Destroy` and disposal are attempted once. A cancelled + setup or act still prevents its fact bodies from running, just like any other setup or act failure. +- The runner emits skipped results for facts of classes that never started. Hard IDE cancellation can prevent MTP + from delivering these results to the client; see [limits](#limits). - Running code can cooperate through `SpecificationContext.CancellationToken`, which flows across awaits and is isolated per class. Cancellation never interrupts running code forcibly. @@ -90,7 +92,9 @@ Cancelling the run (for example Ctrl+C, or Stop in the IDE) and `--fail-fast` bo If `Establish`/`Given` fails, `Because`/`When` and the fact bodies do not run. If `Because`/`When` fails, the fact bodies do not run. In either case, the selected non-skipped facts fail with the original exception; `Destroy` and -disposal are still attempted once when an instance was created. +disposal are still attempted once when an instance was created. Lifecycle cancellation is wrapped in +`SpecificationCancelledException`, with the lifecycle phase in its message and the original cancellation as its +inner exception, matching the reflection-based adapters. A failure in `Destroy`, disposal or ending the specification scope produces **one separate failed result for the class**, displayed as ` (teardown)`. Multiple cleanup errors are aggregated into that result. Its stable UID @@ -105,6 +109,13 @@ adopt that engine-owned behavior. ## Limits +- **Hard IDE Stop can suppress result delivery.** With Microsoft.Testing.Platform 2.5.1, server-mode + `$/cancelRequest` cancels the request and stops forwarding subsequent test updates. The runner still cancels + cooperative running code, attempts `Destroy` and disposal once per started class, and does not construct queued + classes, but the client may receive no terminal results, including skipped results for queued facts. The server + acknowledges the cancelled request with JSON-RPC error `-32800`; its process exits successfully when the client + subsequently sends `exit`. That process exit is not a passing test run. This is a platform transport limitation, + not a guarantee of terminal-result parity with an uncancelled run or `--fail-fast`. - **Methods must return `void`, `Task` or `ValueTask`.** A fact with another return type fails its class with a discovery error. An `async void` fact or lifecycle method is a compile error (`CRSPEC0010`), because its failures can not be awaited. diff --git a/Integration/Consumers/Directory.Build.targets b/Integration/Consumers/Directory.Build.targets new file mode 100644 index 0000000..fc9b8a8 --- /dev/null +++ b/Integration/Consumers/Directory.Build.targets @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Integration/Lifecycle/Cases/act_cancellation.cs b/Integration/Lifecycle/Cases/act_cancellation.cs new file mode 100644 index 0000000..64feebb --- /dev/null +++ b/Integration/Lifecycle/Cases/act_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class act_cancellation : lifecycle_context +{ + void Because() => throw new OperationCanceledException("deliberate act cancellation"); +} diff --git a/Integration/Lifecycle/Cases/act_failure.cs b/Integration/Lifecycle/Cases/act_failure.cs new file mode 100644 index 0000000..c336361 --- /dev/null +++ b/Integration/Lifecycle/Cases/act_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class act_failure : lifecycle_context +{ + void Because() => throw new InvalidOperationException("deliberate act failure"); +} diff --git a/Integration/Lifecycle/Cases/async_act_cancellation.cs b/Integration/Lifecycle/Cases/async_act_cancellation.cs new file mode 100644 index 0000000..588db5a --- /dev/null +++ b/Integration/Lifecycle/Cases/async_act_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_act_cancellation : lifecycle_context +{ + async ValueTask Because() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate act cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_act_failure.cs b/Integration/Lifecycle/Cases/async_act_failure.cs new file mode 100644 index 0000000..bf3f285 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_act_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_act_failure : lifecycle_context +{ + async ValueTask Because() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate act failure"); + } +} diff --git a/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs b/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs new file mode 100644 index 0000000..4ca9f84 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_cleanup_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_cleanup_cancellation : lifecycle_context +{ + async Task Destroy() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate cleanup cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_cleanup_failure.cs b/Integration/Lifecycle/Cases/async_cleanup_failure.cs new file mode 100644 index 0000000..f7334b5 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_cleanup_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_cleanup_failure : lifecycle_context +{ + async Task Destroy() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate cleanup failure"); + } +} diff --git a/Integration/Lifecycle/Cases/async_setup_cancellation.cs b/Integration/Lifecycle/Cases/async_setup_cancellation.cs new file mode 100644 index 0000000..a6745cc --- /dev/null +++ b/Integration/Lifecycle/Cases/async_setup_cancellation.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_setup_cancellation : lifecycle_context +{ + async Task Establish() + { + await Task.Yield(); + throw new TaskCanceledException("deliberate setup cancellation"); + } +} diff --git a/Integration/Lifecycle/Cases/async_setup_failure.cs b/Integration/Lifecycle/Cases/async_setup_failure.cs new file mode 100644 index 0000000..4cf83a3 --- /dev/null +++ b/Integration/Lifecycle/Cases/async_setup_failure.cs @@ -0,0 +1,16 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class async_setup_failure : lifecycle_context +{ + async Task Establish() + { + await Task.Yield(); + throw new InvalidOperationException("deliberate setup failure"); + } +} diff --git a/Integration/Lifecycle/Cases/cleanup_cancellation.cs b/Integration/Lifecycle/Cases/cleanup_cancellation.cs new file mode 100644 index 0000000..a9bd963 --- /dev/null +++ b/Integration/Lifecycle/Cases/cleanup_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class cleanup_cancellation : lifecycle_context +{ + void Destroy() => throw new OperationCanceledException("deliberate cleanup cancellation"); +} diff --git a/Integration/Lifecycle/Cases/cleanup_failure.cs b/Integration/Lifecycle/Cases/cleanup_failure.cs new file mode 100644 index 0000000..9579ad1 --- /dev/null +++ b/Integration/Lifecycle/Cases/cleanup_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class cleanup_failure : lifecycle_context +{ + void Destroy() => throw new InvalidOperationException("deliberate cleanup failure"); +} diff --git a/Integration/Lifecycle/Cases/setup_cancellation.cs b/Integration/Lifecycle/Cases/setup_cancellation.cs new file mode 100644 index 0000000..93a03d6 --- /dev/null +++ b/Integration/Lifecycle/Cases/setup_cancellation.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class setup_cancellation : lifecycle_context +{ + void Establish() => throw new OperationCanceledException("deliberate setup cancellation"); +} diff --git a/Integration/Lifecycle/Cases/setup_failure.cs b/Integration/Lifecycle/Cases/setup_failure.cs new file mode 100644 index 0000000..ee3e09d --- /dev/null +++ b/Integration/Lifecycle/Cases/setup_failure.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace LifecycleProbes; + +#if TUNIT +[global::TUnit.Core.InheritsTests] +#endif +public class setup_failure : lifecycle_context +{ + void Establish() => throw new InvalidOperationException("deliberate setup failure"); +} diff --git a/Integration/Lifecycle/Stop/A_first.cs b/Integration/Lifecycle/Stop/A_first.cs new file mode 100644 index 0000000..b301ede --- /dev/null +++ b/Integration/Lifecycle/Stop/A_first.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class A_first : running_context; diff --git a/Integration/Lifecycle/Stop/B_second.cs b/Integration/Lifecycle/Stop/B_second.cs new file mode 100644 index 0000000..8f71a37 --- /dev/null +++ b/Integration/Lifecycle/Stop/B_second.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class B_second : running_context; diff --git a/Integration/Lifecycle/Stop/Z_queued.cs b/Integration/Lifecycle/Stop/Z_queued.cs new file mode 100644 index 0000000..9f82dda --- /dev/null +++ b/Integration/Lifecycle/Stop/Z_queued.cs @@ -0,0 +1,6 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace StopProbes; + +public class Z_queued : running_context; diff --git a/Integration/Lifecycle/lifecycle_probes.cs b/Integration/Lifecycle/lifecycle_probes.cs new file mode 100644 index 0000000..0722323 --- /dev/null +++ b/Integration/Lifecycle/lifecycle_probes.cs @@ -0,0 +1,41 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Specifications; + +namespace LifecycleProbes; + +public abstract class lifecycle_context : Specification, IDisposable +{ + static readonly Lock Gate = new(); + + protected void Record(string operation) + { + var path = Environment.GetEnvironmentVariable("CRATIS_LIFECYCLE_TRACE"); + if (path is null) return; + lock (Gate) File.AppendAllText(path, $"{GetType().Name}:{operation}{Environment.NewLine}"); + } + + void Establish() => Record("Establish"); + void Because() => Record("Because"); + void Destroy() => Record("Destroy"); + public void Dispose() => Record("Dispose"); + +#if XUNIT + [Xunit.Fact] +#elif NUNIT + [NUnit.Framework.Test] +#elif TUNIT + [global::TUnit.Core.Test] +#endif + public void should_run_first() => Record("fact1"); + +#if XUNIT + [Xunit.Fact] +#elif NUNIT + [NUnit.Framework.Test] +#elif TUNIT + [global::TUnit.Core.Test] +#endif + public void should_run_second() => Record("fact2"); +} diff --git a/Integration/Lifecycle/rpc.mjs b/Integration/Lifecycle/rpc.mjs new file mode 100644 index 0000000..ce9096a --- /dev/null +++ b/Integration/Lifecycle/rpc.mjs @@ -0,0 +1,49 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; + +// MTP server-mode JSON-RPC uses UTF-8 Content-Length framing, not newline-delimited JSON. +export function connect(socket) { + let buffer = Buffer.alloc(0); + let sequence = 0; + const pending = new Map(); + const updates = []; + const send = message => { + const body = Buffer.from(JSON.stringify({ jsonrpc: '2.0', ...message })); + socket.write(Buffer.concat([Buffer.from(`Content-Length: ${body.length}\r\n\r\n`), body])); + }; + const dispatch = message => { + if (message.method) { + updates.push(message); + if (message.id !== undefined) send({ id: message.id, result: null }); + return; + } + assert.ok(pending.has(message.id), `Unexpected RPC response ${message.id}`); + pending.get(message.id)(message); + pending.delete(message.id); + }; + socket.on('data', chunk => { + buffer = Buffer.concat([buffer, chunk]); + while (true) { + const end = buffer.indexOf('\r\n\r\n'); + if (end < 0) return; + const length = Number(buffer.subarray(0, end).toString().match(/Content-Length:\s*(\d+)/i)?.[1]); + assert.ok(Number.isSafeInteger(length) && length > 0, 'Valid MTP frame required'); + if (buffer.length < end + 4 + length) return; + const message = JSON.parse(buffer.subarray(end + 4, end + 4 + length).toString()); + buffer = buffer.subarray(end + 4 + length); + dispatch(message); + } + }); + return { + updates, + notify: (method, params) => send({ method, params }), + request(method, params) { + const id = ++sequence; + const response = new Promise(resolve => pending.set(id, resolve)); + send({ id, method, params }); + return { id, response }; + }, + }; +} diff --git a/Integration/Lifecycle/stop.mjs b/Integration/Lifecycle/stop.mjs new file mode 100644 index 0000000..729c50d --- /dev/null +++ b/Integration/Lifecycle/stop.mjs @@ -0,0 +1,89 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdirSync, mkdtempSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { once } from 'node:events'; +import { join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { connect } from './rpc.mjs'; + +const output = resolve(process.argv[2] ?? 'artifacts/lifecycle'); +mkdirSync(output, { recursive: true }); +const directory = mkdtempSync(join(output, 'Stop-')); +const trace = join(directory, 'trace.txt'); +const server = createServer(); +server.listen(0, '127.0.0.1'); +await once(server, 'listening'); +const connection = once(server, 'connection'); +const child = spawn('dotnet', [ + 'Integration/Consumers/Runner/bin/Release/net10.0/Runner.dll', + '--server', '--client-port', String(server.address().port), + '--specs-parallel', '2', '--specs-cache', 'none', +], { env: { ...process.env, CRATIS_LIFECYCLE_TRACE: trace }, stdio: ['ignore', 'pipe', 'pipe'] }); +let stdout = ''; +let stderr = ''; +child.stdout.on('data', chunk => { stdout += chunk; }); +child.stderr.on('data', chunk => { stderr += chunk; }); +const exit = once(child, 'exit'); +let socket; +const watchdog = setTimeout(() => child.kill('SIGKILL'), 30_000); +const bounded = promise => Promise.race([promise, delay(20_000, null, { ref: false }).then(() => { throw new Error('MTP protocol operation timed out'); })]); +const lines = () => existsSync(trace) ? readFileSync(trace, 'utf8').trim().split(/\r?\n/) : []; +async function until(predicate) { + const deadline = Date.now() + 20_000; + while (!predicate()) { + assert.equal(child.exitCode, null, `Host exited before the barrier: ${stderr}`); + assert.ok(Date.now() < deadline, `Timed out waiting for probe barrier: ${lines().join(', ')}`); + await delay(10); + } +} +try { + [socket] = await bounded(connection); + const rpc = connect(socket); + const initialized = await bounded(rpc.request('initialize', { + processId: process.pid, + clientInfo: { name: 'Cratis lifecycle proof', version: '1.0.0' }, + capabilities: { testing: { debuggerProvider: false, isStateful: false } }, + protocolVersions: ['1.0.0'], + }).response); + assert.equal(initialized.error, undefined, 'MTP initialization must succeed'); + const filter = '/*/StopProbes/*/*'; + const discovered = await bounded(rpc.request('testing/discoverTests', { runId: randomUUID(), filter }).response); + assert.equal(discovered.error, undefined, 'MTP discovery must succeed'); + const discoveryUpdates = rpc.updates.splice(0); + const selected = discoveryUpdates.filter(update => update.method === 'testing/testUpdates/tests').flatMap(update => update.params.changes ?? []); + assert.equal(selected.length, 6, 'Real MTP discovery must deliver all six selected facts before Stop'); + const run = rpc.request('testing/runTests', { runId: randomUUID(), filter }); + await until(() => ['A_first', 'B_second'].every(name => lines().includes(`${name}:Started`))); + assert.ok(!lines().some(line => line.startsWith('Z_queued:')), 'Third class must still be queued'); + rpc.notify('$/cancelRequest', { id: run.id }); + await until(() => ['A_first', 'B_second'].every(name => lines().includes(`${name}:Dispose`))); + const response = await bounded(run.response); + writeFileSync(join(directory, 'messages.json'), JSON.stringify({ discoveryUpdates, response, updates: rpc.updates }, null, 2)); + assert.equal(response.error?.code, -32800, 'MTP must acknowledge the request as cancelled'); + const changes = rpc.updates.filter(update => update.method === 'testing/testUpdates/tests').flatMap(update => update.params.changes ?? []); + assert.equal(changes.length, 0, 'MTP 2.5.1 hard cancellation suppresses subsequent result updates, including queued facts'); + rpc.notify('exit', null); + const [code, signal] = await bounded(exit); + assert.equal(signal, null, 'MTP host must exit normally after Stop'); + assert.equal(code, 0, 'Server-mode exit after a cancelled request'); + for (const name of ['A_first', 'B_second']) { + for (const operation of ['Create', 'Started', 'Cancelled', 'Destroy', 'Dispose']) { + assert.equal(lines().filter(line => line === `${name}:${operation}`).length, 1, `${name}: ${operation} exactly once`); + } + assert.ok(!lines().some(line => line.startsWith(`${name}:fact`)), `${name}: cancelled act must not run facts`); + } + assert.ok(!lines().some(line => line.startsWith('Z_queued:')), 'Cancelled scheduling must not construct queued class'); + console.log(`MTP hard Stop: two concurrent classes cancelled; Destroy/Dispose once each; queued class not constructed; six facts discovered, zero result updates after hard cancellation; exit ${code}. Messages: ${directory}`); +} finally { + clearTimeout(watchdog); + socket?.destroy(); + server.close(); + if (child.exitCode === null) child.kill('SIGKILL'); + writeFileSync(join(directory, 'stdout.txt'), stdout); + writeFileSync(join(directory, 'stderr.txt'), stderr); +} diff --git a/Integration/Lifecycle/stop_probes.cs b/Integration/Lifecycle/stop_probes.cs new file mode 100644 index 0000000..789f706 --- /dev/null +++ b/Integration/Lifecycle/stop_probes.cs @@ -0,0 +1,25 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.Specifications; +using LifecycleProbes; + +namespace StopProbes; + +public abstract class running_context : lifecycle_context +{ + protected running_context() => Record("Create"); + + async Task Because() + { + Record("Started"); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, SpecificationContext.CancellationToken); + } + finally + { + Record("Cancelled"); + } + } +} diff --git a/Integration/Lifecycle/verify.mjs b/Integration/Lifecycle/verify.mjs new file mode 100644 index 0000000..2d572c1 --- /dev/null +++ b/Integration/Lifecycle/verify.mjs @@ -0,0 +1,83 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; + +export const projects = { + XUnit: 'XUnit/Consumer.XUnit.csproj', + XUnitV3: 'XUnit.v3/XUnit.v3.csproj', + NUnit4: 'NUnit/Consumer.NUnit.csproj', + NUnit5: 'NUnit5/NUnit5.csproj', + TUnit: 'TUnit/Consumer.TUnit.csproj', + Runner: 'Runner/Runner.csproj', +}; + +const adapter = process.argv[2]; +assert.ok(Object.hasOwn(projects, adapter), `Unknown adapter: ${adapter}`); +const output = resolve(process.argv[3] ?? 'artifacts/lifecycle'); +mkdirSync(output, { recursive: true }); +const directory = mkdtempSync(join(output, `${adapter}-`)); +const trace = join(directory, 'trace.txt'); +const project = `Integration/Consumers/${projects[adapter]}`; +const mtp = adapter === 'TUnit' || adapter === 'Runner'; +const args = mtp + ? ['run', '--project', project, '-c', 'Release', '--no-build', '--', '--report-trx', '--report-trx-filename', 'result.trx', '--results-directory', directory] + : ['test', project, '-c', 'Release', '--no-build', '--logger', 'trx;LogFileName=result.trx', '--results-directory', directory]; +if (adapter === 'Runner') args.push('--specs-cache', 'none', '--treenode-filter', '/*/LifecycleProbes/*/*'); +const run = spawnSync('dotnet', args, { + encoding: 'utf8', timeout: 90_000, + env: { ...process.env, CRATIS_LIFECYCLE_TRACE: trace }, +}); +process.stdout.write(run.stdout ?? ''); +process.stderr.write(run.stderr ?? ''); +assert.ifError(run.error); +assert.equal(run.signal, null, 'The probe must exit without being killed'); +assert.equal(run.status, mtp ? 2 : 1, 'Expected native test-failure exit code'); +const xml = readFileSync(join(directory, 'result.trx'), 'utf8'); +const calls = readFileSync(trace, 'utf8').trim().split(/\r?\n/); +const cases = ['setup', 'act', 'cleanup'].flatMap(phase => + ['failure', 'cancellation'].flatMap(kind => [`${phase}_${kind}`, `async_${phase}_${kind}`])); +const attributes = text => Object.fromEntries([...text.matchAll(/([\w:]+)="([^"]*)"/g)].map(match => [match[1], match[2]])); +const definitions = new Map([...xml.matchAll(/]*)>([\s\S]*?)<\/UnitTest>/g)].map(match => { + const method = match[2].match(/]*)/); + return [attributes(match[1]).id, method ? attributes(method[1]).className : '']; +})); +const results = [...xml.matchAll(/]*)/g)].map(match => attributes(match[1])); +assert.equal(results.length, adapter === 'Runner' ? 28 : 24, 'Every expected fact/teardown must have a terminal result'); +assert.equal(new Set(results.map(result => result.executionId)).size, results.length, 'No duplicate terminal results'); +for (const name of cases) { + const operations = calls.filter(call => call.startsWith(`${name}:`)).map(call => call.slice(name.length + 1)); + assert.equal(operations.filter(call => call === 'Establish').length, 1, `${name}: setup once`); + assert.equal(operations.filter(call => call === 'Because').length, name.includes('setup') ? 0 : 1, `${name}: act eligibility`); + assert.equal(operations.filter(call => call === 'Destroy').length, 1, `${name}: cleanup once`); + const facts = operations.filter(call => call.startsWith('fact')).sort(); + assert.deepEqual(facts, name.includes('cleanup') ? ['fact1', 'fact2'] : [], `${name}: fact eligibility`); + const factResults = results.filter(result => definitions.get(result.testId)?.split(',')[0] === `LifecycleProbes.${name}`); + const ordinary = factResults.filter(result => !result.testName.includes('teardown')); + assert.equal(ordinary.length, 2, `${name}: both facts discovered through ${adapter}`); + if (!name.includes('cleanup')) { + assert.ok(ordinary.every(result => result.outcome === 'Failed'), `${name}: both facts failed`); + } else { + const expected = adapter === 'TUnit' ? ['Failed', 'Passed'] : ['Passed', 'Passed']; + assert.deepEqual(ordinary.map(result => result.outcome).sort(), expected, `${name}: preserve framework-owned cleanup reporting`); + if (adapter === 'Runner') { + const teardown = factResults.filter(result => result.testName.includes('teardown')); + assert.equal(teardown.length, 1, `${name}: separate native teardown result`); + assert.equal(teardown[0].outcome, 'Failed', `${name}: teardown fails the run`); + } else if (adapter.startsWith('XUnit')) { + assert.ok(`${run.stdout}${run.stderr}`.includes(`Test Class Cleanup Failure (LifecycleProbes.${name})`), `${name}: xUnit class cleanup diagnostic`); + } else if (adapter.startsWith('NUnit')) { + assert.ok(`${run.stdout}${run.stderr}`.includes(`TearDown failed for test fixture LifecycleProbes.${name}`), `${name}: NUnit fixture cleanup diagnostic`); + } + } +} +for (const phase of ['Establish', 'Because', 'Destroy']) { + assert.ok(xml.includes(`${phase}() was cancelled`), `${phase}: framework must report lifecycle cancellation`); +} +for (const phase of ['setup', 'act', 'cleanup']) { + assert.ok(xml.includes(`deliberate ${phase} failure`), `${phase}: original failure must be reported`); +} +console.log(`${adapter}: 12 shared lifecycle cases, ${results.length} terminal results, once-only cleanup verified (${directory})`); diff --git a/Source/Runner.Generator/SpecificationEmitter.cs b/Source/Runner.Generator/SpecificationEmitter.cs index 78748b1..8579d09 100644 --- a/Source/Runner.Generator/SpecificationEmitter.cs +++ b/Source/Runner.Generator/SpecificationEmitter.cs @@ -79,14 +79,21 @@ public static void Emit(StringBuilder builder, SpecificationModel model, string body.Append(" static ").Append(isAsync ? "async " : string.Empty).Append(ValueTask).Append(' ').Append(name).AppendLine("(object instance)"); body.AppendLine(" {"); body.Append(" var specification = (").Append(model.FullyQualifiedName).AppendLine(")instance;"); + body.AppendLine(" try"); + body.AppendLine(" {"); foreach (var call in calls) { var expression = Call(accessors, call, ref counter); var awaited = call.Returns != ReturnKind.Void || call.How == CallKind.Reflection; - body.Append(" ").Append(awaited ? "await " : string.Empty).Append(expression).AppendLine(";"); + body.Append(" ").Append(awaited ? "await " : string.Empty).Append(expression).AppendLine(";"); } - if (!isAsync) body.AppendLine(" return default;"); + if (!isAsync) body.AppendLine(" return default;"); + body.AppendLine(" }"); + body.AppendLine(" catch (global::System.OperationCanceledException exception)"); + body.AppendLine(" {"); + body.Append(" throw new global::Cratis.Specifications.SpecificationCancelledException(").Append(Literal(name)).AppendLine(", exception);"); + body.AppendLine(" }"); body.AppendLine(" }"); return name; } diff --git a/Source/Runner.Specs/Probes/ContractProbes.csproj b/Source/Runner.Specs/Probes/ContractProbes.csproj index e794e78..7a951e3 100644 --- a/Source/Runner.Specs/Probes/ContractProbes.csproj +++ b/Source/Runner.Specs/Probes/ContractProbes.csproj @@ -11,6 +11,7 @@ + diff --git a/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs b/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs new file mode 100644 index 0000000..c860bb5 --- /dev/null +++ b/Source/Runner.Specs/for_LifecycleContract/when_cancelling_lifecycle_methods.cs @@ -0,0 +1,85 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.CompilerServices; +using LifecycleProbes; + +namespace Cratis.Specifications.Runner.for_LifecycleContract; + +public class when_cancelling_lifecycle_methods : Specification +{ + readonly List<(Exception Generated, Exception Reflected)> results = []; + + async Task Because() + { + await Compare("Establish"); + await Compare("Establish"); + await Compare("Because"); + await Compare("Because"); + await Compare("Destroy"); + await Compare("Destroy"); + } + + async Task Compare(string phase) + where T : lifecycle_context, new() + { + RuntimeHelpers.RunModuleConstructor(typeof(T).Assembly.ManifestModule.ModuleHandle); + var descriptor = given.contract_probes.WorkFor().Specification; + var generated = phase switch + { + "Establish" => descriptor.Establish, + "Because" => descriptor.Because, + _ => descriptor.Destroy + }; + results.Add((await Capture(() => generated(new T()).AsTask()), await Capture(() => Reflected(phase)))); + } + + static Task Reflected(string phase) + where T : lifecycle_context, new() => phase switch + { + "Establish" => SpecificationLifecycle.Establish(new T(), typeof(T), typeof(Specification)), + "Because" => SpecificationLifecycle.Because(new T(), typeof(T), typeof(Specification)), + _ => SpecificationLifecycle.Destroy(new T(), typeof(T), typeof(Specification)) + }; + + static async Task Capture(Func action) + { + try + { + await action(); + throw new InvalidOperationException("The lifecycle cancellation did not throw."); + } + catch (Exception exception) + { + return exception; + } + } + + void should_reproduce_all_six_sync_and_async_cases() => results.Count.ShouldEqual(6); + + void should_match_synchronous_reflection_cancellation() + { + foreach (var result in results.Where((_, index) => index % 2 == 0)) result.Generated.GetType().ShouldEqual(result.Reflected.GetType()); + } + + void should_match_asynchronous_reflection_cancellation() + { + foreach (var result in results.Where((_, index) => index % 2 != 0)) result.Generated.GetType().ShouldEqual(result.Reflected.GetType()); + } + + void should_preserve_the_phase_and_existing_message() + { + foreach (var result in results) result.Generated.Message.ShouldEqual(result.Reflected.Message); + } + + void should_preserve_the_original_cancellation() + { + foreach (var result in results) + { + result.Reflected.ShouldBeOfExactType(); + result.Generated.InnerException.ShouldNotBeNull(); + result.Generated.InnerException.GetType().ShouldEqual(result.Reflected.InnerException.GetType()); + result.Generated.InnerException?.Message.ShouldEqual(result.Reflected.InnerException?.Message); + } + } +}