chore: Update test projects to use xunit v3

.gitattributes

@@ -0,0 +1,4 @@
+# Verify snapshot files must keep LF line endings on every platform.
+# Verify.XunitV3 rejects a verified file that contains a CR with core.autocrlf.
+*.verified.* text eol=lf
+*.received.* text eol=lf

.github/workflows/generate-coverage-report.yaml

@@ -8,7 +8,7 @@ on:
         type: boolean
         description: Set `true` to skip integration tests.
         default: true
-        
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.inputs.skip_integration_tests }}-${{ github.head_ref || github.ref || github.run_id }}
   cancel-in-progress: true
@@ -49,13 +49,13 @@ jobs:
 
       - name: Collect Code Coverage
         run: |
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0"
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net10.0"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0 --no-ansi --output Detailed"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net10.0 --no-ansi --output Detailed"
 
       - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests
         if: ${{ github.event.inputs.skip_integration_tests == 'false'}}
         run: |
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0 --no-ansi --output Detailed"
 
       - name: Shutdown dotnet-coverage server.
         run: dotnet coverage shutdown bdn_coverage --timeout 60000
@@ -77,7 +77,7 @@ jobs:
       - name: Setup additional tools for Wasm/NativeAot tests
         if: ${{ github.event.inputs.skip_integration_tests == 'false'}}
         uses: ./.github/actions/setup-additional-tools
-        
+
       - name: Install dotnet-coverage
         run: dotnet tool install --global dotnet-coverage
 
@@ -89,13 +89,13 @@ jobs:
 
       - name: Collect Code Coverage
         run: |
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472"
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net472"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472 --no-ansi --output Detailed"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net472 --no-ansi --output Detailed"
 
-      - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests 
-        if: ${{ github.event.inputs.skip_integration_tests == 'false'}} 
+      - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests
+        if: ${{ github.event.inputs.skip_integration_tests == 'false'}}
         run: |
-          dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472'
+          dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --no-ansi --output Detailed"
 
       - name: Shutdown dotnet-coverage server.
         run: dotnet coverage shutdown bdn_coverage --timeout 60000

.github/workflows/run-tests-selected.yaml

@@ -89,12 +89,13 @@ jobs:
         run: |
           $PSNativeCommandUseErrorActionPreference = $true
           $iterationCount = ${{ inputs.iteration_count }}
+
+          Write-Host ("OSArchitecture" + [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)
 
           foreach($i in 1..$iterationCount) {
             Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i)
 
-            dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal"
-
+            dotnet test -c Release --framework ${{ inputs.framework }} --no-build --filter "${{ inputs.filter }}" --no-ansi --progress off --output Detailed
             Write-Output '##[endgroup]'
           }
 

.github/workflows/run-tests.yaml

@@ -57,7 +57,9 @@ jobs:
         if: always()
         with:
           name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-windows-full:
     strategy:
@@ -100,7 +102,9 @@ jobs:
         if: always()
         with:
           name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-linux:
     strategy:
@@ -136,7 +140,9 @@ jobs:
         if: always()
         with:
           name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-macos:
     name: test-macos (${{ matrix.os.arch }})
@@ -177,7 +183,9 @@ jobs:
         if: always()
         with:
           name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-pack:
     runs-on: ubuntu-latest

BenchmarkDotNet.slnx

@@ -29,6 +29,7 @@
     <Project Path="templates/BenchmarkDotNet.Templates.csproj" />
   </Folder>
   <Folder Name="/tests/">
+    <File Path="tests/.editorconfig" />
     <File Path="tests/Directory.Build.props" />
     <File Path="tests/Directory.Build.targets" />
     <Project Path="tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj" />

build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs

@@ -1,8 +1,10 @@
 using BenchmarkDotNet.Build.Helpers;
 using Cake.Common.Diagnostics;
 using Cake.Common.Tools.DotNet;
-using Cake.Common.Tools.DotNet.Test;
+using Cake.Common.Tools.DotNet.Run;
+using Cake.Core;
 using Cake.Core.IO;
+using System.Linq;
 using System.Runtime.InteropServices;
 
 namespace BenchmarkDotNet.Build.Runners;
@@ -32,19 +34,34 @@ public class UnitTestRunner(BuildContext context)
     private DirectoryPath TestOutputDirectory { get; } = context.RootDirectory
         .Combine("TestResults");
 
-    private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tfm)
+    private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm)
     {
-        var settings = new DotNetTestSettings
+        // Enabled `Trace` level logging when debug logging is enabled on GitHub Actions.
+        // https://docs.github.com/en/actions/how-tos/monitor-workflows/enable-debug-logging
+        var diagnosticVerbosity = context.Environment.GetEnvironmentVariable("ACTIONS_STEP_DEBUG") == "true"
+             ? "Trace"
+             : "Warning"; // Logging Warning/Error/Critical level logs by default.
+
+        var settings = new DotNetRunSettings
         {
             Configuration = context.BuildConfiguration,
             Framework = tfm,
             NoBuild = true,
             NoRestore = true,
-            Loggers = new[] { "trx", $"trx;LogFileName={logFile.FullPath}", "console;verbosity=detailed" },
             EnvironmentVariables =
             {
                 ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory
-            }
+            },
+            ArgumentCustomization = args
+                => args.Append("--report-xunit-trx")
+                       .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath))
+                       .Append("--no-ansi")
+                       .AppendSwitch("--progress", "off")
+                       .AppendSwitch("--output", "Detailed")
+                       .AppendSwitch("--show-stdout", "Failed")
+                       .Append("--diagnostic")
+                       .AppendSwitch("--diagnostic-verbosity", diagnosticVerbosity)
+                       .Append("--no-launch-profile"),
         };
         return settings;
     }
@@ -58,7 +75,7 @@ private void RunTests(FilePath projectFile, string alias, string tfm)
         var settings = GetTestSettingsParameters(trxFile, tfm);
 
         context.Information($"Run tests for {projectFile} ({tfm}), result file: '{trxFile}'");
-        context.DotNetTest(projectFile.FullPath, settings);
+        context.DotNetRun(projectFile.FullPath, settings);
     }
 
     private void RunUnitTests(string tfm)

global.json

@@ -0,0 +1,5 @@
+{
+  "test": {
+    "runner": "Microsoft.Testing.Platform"
+  }
+}

src/BenchmarkDotNet/Exporters/HtmlExporter.cs

@@ -9,10 +9,10 @@ public class HtmlExporter : ExporterBase
     {
         private const string CssDefinition = @"
 <style type=""text/css"">
-	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
-	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
-	tr { background-color: #fff; border-top: 1px solid #ccc; }
-	tr:nth-child(even) { background: #f8f8f8; }
+    table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+    td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+    tr { background-color: #fff; border-top: 1px solid #ccc; }
+    tr:nth-child(even) { background: #f8f8f8; }
 </style>";
 
         protected override string FileExtension => "html";

tests/.editorconfig

@@ -0,0 +1,11 @@
+root = false
+
+# C# files
+[*.cs]
+# xUnit1051: Calls to methods which accept CancellationToken should use TestContext.Current.CancellationToken to allow test cancellation to be more responsive.
+dotnet_diagnostic.xUnit1051.severity = suggestion
+
+# Verify settings
+[*.{received,verified}.{txt}]
+charset = utf-8-bom
+trim_trailing_whitespace = false

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs

@@ -633,7 +633,7 @@ public async Task Providing_expected_value_type_should_not_trigger_diagnostic(
         {
             var testCode = /* lang=c#-test */ $$"""
                 using DifferentNamespace;
-                
+
                 using BenchmarkDotNet.Attributes;
 
                 public class BenchmarkClass
@@ -1188,7 +1188,7 @@ public void BenchmarkMethod({{integerValueAndType.Value2}} a)
         }
 
         public static IEnumerable<string> DummyAttributeUsage
-            => DummyAttributeUsageTheoryData;
+            => DummyAttributeUsageTheoryData.Select(x => x.Data);
 
         public static TheoryData<string> EmptyArgumentsAttributeUsagesWithMismatchingValueCount()
         {

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs

@@ -1,4 +1,4 @@
-using BenchmarkDotNet.Analyzers;
+using BenchmarkDotNet.Analyzers;
 using BenchmarkDotNet.Analyzers.Attributes;
 using BenchmarkDotNet.Analyzers.Tests.Fixtures;
 using Microsoft.CodeAnalysis;
@@ -191,7 +191,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, int, int[]> DuplicateSameParameterAttributeUsages
             => DuplicateSameAttributeUsagesTheoryData;
@@ -353,7 +353,7 @@ public class BenchmarkClass
             await RunAsync();
         }
 
-        public static TheoryData<string> UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+        public static TheoryData<string> UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, int, int[]> DuplicateSameParameterAttributeUsages => DuplicateSameAttributeUsagesTheoryData;
 
@@ -512,21 +512,19 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicCla
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers);
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<string> NonPublicClassMemberAccessModifiers
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new NonPublicClassMemberAccessModifiersTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data);
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3));
 
         public static IEnumerable<int[]> DuplicateParameterAttributeUsageCounts
-            => DuplicateAttributeUsageCountsTheoryData;
+            => DuplicateAttributeUsageCountsTheoryData.Select(x => x.Data);
     }
 
     public class PropertyMustBePublic : AnalyzerTestFixture<GeneralParameterAttributesAnalyzer>
@@ -681,18 +679,16 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicCla
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers);
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item2 as string)!)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<string> NonPublicClassMemberAccessModifiers
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new NonPublicClassMemberAccessModifiersTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data);
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1!, tdr.Data.Item2, tdr.Data.Item3));
 
         public static TheoryData<int[]> DuplicateParameterAttributeUsageCounts
             => DuplicateAttributeUsageCountsTheoryData;
@@ -846,7 +842,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, string> UniqueParameterAttributes
             => UniqueParameterAttributesTheoryData;
@@ -984,7 +980,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, string> UniqueParameterAttributes
             => UniqueParameterAttributesTheoryData;
@@ -1149,20 +1145,20 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicPro
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicPropertySetters());
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3));
 
         public static TheoryData<int[]> DuplicateParameterAttributeUsageCounts
             => DuplicateAttributeUsageCountsTheoryData;
 
         public static IEnumerable<string> NonPublicPropertySetters()
-            => new NonPublicPropertySetterAccessModifiersTheoryData()
-            .Select<string, string>(m => $"{{ get; {m} set; }}")
+            => new NonPublicPropertySetterAccessModifiersTheoryData().Select(x => x.Data)
+            .Select(m => $"{{ get; {m} set; }}")
             .Concat(["{ get; }", "=> 0;"]);
 
         public static TheoryData<string> NonPublicPropertySettersTheoryData()
@@ -1904,7 +1900,7 @@ private static IEnumerable<int[]> GenerateDuplicateAttributeUsageCombinations(Th
     private static ReadOnlyCollection<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> GenerateDuplicateSameAttributeUsageCombinations(TheoryData<string, string> uniqueAttributeUsages)
     {
         var uniqueAttributeUsagesList = uniqueAttributeUsages
-            .Select(tdr => (tdr[1] as string)!)
+            .Select(tdr => tdr.Data.Item2)
             .ToList()
             .AsReadOnly();
 

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAllValuesAttributeAnalyzerTests.cs

@@ -32,9 +32,7 @@ public class BenchmarkClass
         }
 
         public static IEnumerable<string> FieldOrPropertyDeclarations
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new FieldOrPropertyDeclarationsTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data);
 
         public static IEnumerable<string> InvalidTypes =>
         [
@@ -149,9 +147,7 @@ public class BenchmarkClass
         }
 
         public static IEnumerable<string> FieldOrPropertyDeclarations
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new FieldOrPropertyDeclarationsTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data);
 
         public static IEnumerable<string> NonEnumStructs =>
         [
@@ -249,9 +245,7 @@ public class BenchmarkClass
         }
 
         public static IEnumerable<string> FieldOrPropertyDeclarations
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new FieldOrPropertyDeclarationsTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data);
 
         public static IEnumerable<string> NonEnumOrBoolStructs =>
         [