diff --git a/ExtensionTesting.sln b/ExtensionTesting.sln index 7a40fe10..b62e8dd0 100644 --- a/ExtensionTesting.sln +++ b/ExtensionTesting.sln @@ -65,25 +65,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.VisualStudio.Exte EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator.UnitTests", "src\Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator.UnitTests\Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator.UnitTests.csproj", "{DA9797F0-F158-42AB-BE2B-922588756941}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3", "src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3\Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.csproj", "{076FCA31-75AF-4DE5-BD54-A3D057D8F2BD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests", "src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests\Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests.csproj", "{073D4E58-1EA3-463B-BE40-F636B3F98593}" + ProjectSection(ProjectDependencies) = postProject + {25E61E07-E449-4ABB-A197-BD5C8EE3347F} = {25E61E07-E449-4ABB-A197-BD5C8EE3347F} + EndProjectSection +EndProject Global - GlobalSection(SharedMSBuildProjectFiles) = preSolution - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{346081a2-a088-4486-94b3-b586bd6fd888}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{4232a7ba-a7b7-4c65-b7a1-17fb5cded299}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{43cc037c-8ebc-424c-ab73-9dd4047924a1}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{56550d99-71d3-44fe-804c-944d82d639f6}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{5a516b07-7b9c-4ce1-9ddc-e59800919d64}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{629938e1-1c2c-4f01-8038-90a34a940ac8}*SharedItemsImports = 13 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{6a12cff7-2499-4671-8ebe-ec3f2ff2adf0}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{75edeb61-db9d-4f12-b922-ecef8081cef1}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{784cb5c8-6258-499d-8efb-7a601a6f7e46}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{8bda92b2-e2e0-4607-94f6-444f56ad0582}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{9b5759ad-4eff-4e2d-921b-d617bb8cfa22}*SharedItemsImports = 13 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{b3bed6cb-abfe-4bb8-8af7-901ff0c6f027}*SharedItemsImports = 13 - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{c4c38eae-b6b1-4ee5-9f01-03b4d9249bbd}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{ce32d863-e8c3-461f-80c7-32d0b0853217}*SharedItemsImports = 5 - src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{d7647395-129b-4f45-8c78-10e6f92fc8ad}*SharedItemsImports = 13 - src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{da8322bc-e135-41bb-838a-d87551092e1a}*SharedItemsImports = 5 - EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU @@ -145,6 +134,14 @@ Global {DA9797F0-F158-42AB-BE2B-922588756941}.Debug|Any CPU.Build.0 = Debug|Any CPU {DA9797F0-F158-42AB-BE2B-922588756941}.Release|Any CPU.ActiveCfg = Release|Any CPU {DA9797F0-F158-42AB-BE2B-922588756941}.Release|Any CPU.Build.0 = Release|Any CPU + {076FCA31-75AF-4DE5-BD54-A3D057D8F2BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {076FCA31-75AF-4DE5-BD54-A3D057D8F2BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {076FCA31-75AF-4DE5-BD54-A3D057D8F2BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {076FCA31-75AF-4DE5-BD54-A3D057D8F2BD}.Release|Any CPU.Build.0 = Release|Any CPU + {073D4E58-1EA3-463B-BE40-F636B3F98593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {073D4E58-1EA3-463B-BE40-F636B3F98593}.Debug|Any CPU.Build.0 = Debug|Any CPU + {073D4E58-1EA3-463B-BE40-F636B3F98593}.Release|Any CPU.ActiveCfg = Release|Any CPU + {073D4E58-1EA3-463B-BE40-F636B3F98593}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -168,8 +165,30 @@ Global {CE32D863-E8C3-461F-80C7-32D0B0853217} = {7A983C88-A0B8-4A6C-8837-E45ADD6C01EE} {25E61E07-E449-4ABB-A197-BD5C8EE3347F} = {D2869BF7-27E8-42B9-BF23-8790C0CE08C2} {DA9797F0-F158-42AB-BE2B-922588756941} = {D2869BF7-27E8-42B9-BF23-8790C0CE08C2} + {076FCA31-75AF-4DE5-BD54-A3D057D8F2BD} = {9ED7BE6A-BDEE-44CF-B5C8-EA5F6E212CFD} + {073D4E58-1EA3-463B-BE40-F636B3F98593} = {7A983C88-A0B8-4A6C-8837-E45ADD6C01EE} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BEAADCC8-C5EB-46B6-8AA8-0242304BE240} EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{073d4e58-1ea3-463b-be40-f636b3f98593}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{076fca31-75af-4de5-bd54-a3d057d8f2bd}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{346081a2-a088-4486-94b3-b586bd6fd888}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{4232a7ba-a7b7-4c65-b7a1-17fb5cded299}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{43cc037c-8ebc-424c-ab73-9dd4047924a1}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{56550d99-71d3-44fe-804c-944d82d639f6}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{5a516b07-7b9c-4ce1-9ddc-e59800919d64}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.IntegrationTestService.Shared\Microsoft.VisualStudio.IntegrationTestService.Shared.projitems*{629938e1-1c2c-4f01-8038-90a34a940ac8}*SharedItemsImports = 13 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{6a12cff7-2499-4671-8ebe-ec3f2ff2adf0}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{75edeb61-db9d-4f12-b922-ecef8081cef1}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{784cb5c8-6258-499d-8efb-7a601a6f7e46}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{8bda92b2-e2e0-4607-94f6-444f56ad0582}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{9b5759ad-4eff-4e2d-921b-d617bb8cfa22}*SharedItemsImports = 13 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{b3bed6cb-abfe-4bb8-8af7-901ff0c6f027}*SharedItemsImports = 13 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{c4c38eae-b6b1-4ee5-9f01-03b4d9249bbd}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.IntegrationTests.Shared.projitems*{ce32d863-e8c3-461f-80c7-32d0b0853217}*SharedItemsImports = 5 + src\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared\Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems*{d7647395-129b-4f45-8c78-10e6f92fc8ad}*SharedItemsImports = 13 + src\Microsoft.VisualStudio.VsixInstaller.Shared\Microsoft.VisualStudio.VsixInstaller.Shared.projitems*{da8322bc-e135-41bb-838a-d87551092e1a}*SharedItemsImports = 5 + EndGlobalSection EndGlobal diff --git a/NuGet.config b/NuGet.config index 53a9386c..dbe02062 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,6 +7,7 @@ + diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator/TestServicesSourceGenerator.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator/TestServicesSourceGenerator.cs index 6ee69d61..826ab49b 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator/TestServicesSourceGenerator.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.SourceGenerator/TestServicesSourceGenerator.cs @@ -1420,6 +1420,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) } } + var isXunit3 = compilation.ReferencedAssemblyNames.Any(assemblyName => assemblyName.Name.StartsWith("xunit.v3", StringComparison.Ordinal)); + return new ReferenceDataModel( hasSAsyncServiceProvider, hasThreadHelperJoinableTaskContext, @@ -1430,7 +1432,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) hasOperationProgress, hasOperationProgressStatusService, hasEditorConstants, - editorConstantsCommandIDMissingGuid); + editorConstantsCommandIDMissingGuid, + isXunit3); }); context.RegisterSourceOutput( @@ -1515,19 +1518,19 @@ protected InProcComponent(TestServices testServices) protected JoinableTaskFactory JoinableTaskFactory => TestServices.JoinableTaskFactory; - Task IAsyncLifetime.InitializeAsync() + {(referenceDataModel.IsXUnit3 ? "ValueTask" : "Task")} IAsyncLifetime.InitializeAsync() {{ return InitializeCoreAsync(); }} - Task IAsyncLifetime.DisposeAsync() + {(referenceDataModel.IsXUnit3 ? "ValueTask" : "Task")} {(referenceDataModel.IsXUnit3 ? "global::System.IAsyncDisposable" : "IAsyncLifetime")}.DisposeAsync() {{ - return Task.CompletedTask; + return {(referenceDataModel.IsXUnit3 ? "default" : "Task.CompletedTask")}; }} - protected virtual Task InitializeCoreAsync() + protected virtual {(referenceDataModel.IsXUnit3 ? "ValueTask" : "Task")} InitializeCoreAsync() {{ - return Task.CompletedTask; + return {(referenceDataModel.IsXUnit3 ? "default" : "Task.CompletedTask")}; }} protected async Task GetRequiredGlobalServiceAsync(CancellationToken cancellationToken) @@ -1744,6 +1747,10 @@ namespace Microsoft.VisualStudio.Extensibility.Testing } usings2.Add("global::Xunit"); + if (referenceDataModel.IsXUnit3) + { + usings2.Add("global::Xunit.v3"); + } if (!referenceDataModel.HasThreadHelperJoinableTaskContext) { @@ -1810,7 +1817,7 @@ namespace Microsoft.VisualStudio.Extensibility.Testing /// /// Test method /// - /// + /// /// /// /// @@ -1917,17 +1924,17 @@ private CancellationToken CleanupCancellationToken => _cleanupCancellationTokenSource.Token; /// - public virtual async Task InitializeAsync() + public virtual async {(referenceDataModel.IsXUnit3 ? "ValueTask" : "Task")} InitializeAsync() {{ TestServices = await CreateTestServicesAsync(); }} /// - /// This method implements , and is used for releasing resources + /// This method implements , and is used for releasing resources /// created by . This method is only called if /// completes successfully. /// - public virtual async Task DisposeAsync() + public virtual async {(referenceDataModel.IsXUnit3 ? "ValueTask" : "Task")} DisposeAsync() {{ _cleanupCancellationTokenSource.CancelAfter(CleanupHangMitigatingTimeout); @@ -2123,6 +2130,7 @@ private sealed record ReferenceDataModel( bool HasOperationProgress, bool HasOperationProgressStatusService, bool HasEditorConstants, - bool EditorConstantsCommandIDMissingGuid); + bool EditorConstantsCommandIDMissingGuid, + bool IsXUnit3); } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/DataCollectionService.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/DataCollectionService.cs index 27887161..a806cb07 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/DataCollectionService.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/DataCollectionService.cs @@ -10,7 +10,9 @@ namespace Xunit.Harness using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; public static class DataCollectionService @@ -96,13 +98,26 @@ public static void RegisterCustomLogger(Action callback, string logId, s internal static string GetTestName(ITestCase testCase) { +#if USES_XUNIT_3 + var testMethod = testCase.TestMethod; + var testClass = testMethod!.TestClass.TestClassName; + return $"{testClass}.{testMethod.MethodName}"; +#else var testMethod = testCase.TestMethod.Method; var testClass = testMethod.Type.Name; var lastDot = testClass.LastIndexOf('.'); testClass = testClass.Substring(lastDot + 1); return $"{testClass}.{testMethod.Name}"; +#endif } + internal static string GetTestName(ITestCaseStarting testCaseStarting) +#if USES_XUNIT_3 + => $"{testCaseStarting.TestClassName}.{testCaseStarting.TestMethodName}"; +#else + => GetTestName(testCaseStarting.TestCase); +#endif + internal static void InstallFirstChanceExceptionHandler() { if (!_firstChanceExceptionHandlerInstalled) diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs index 3ae99d2e..a0ae7936 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs @@ -6,15 +6,25 @@ namespace Xunit.Harness using System; using System.Collections.Generic; using System.Collections.Immutable; + using System.ComponentModel; using System.Diagnostics; using System.Linq; using System.Reflection; + using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using System.Windows.Threading; +#if USES_XUNIT_3 + using Xunit.Internal; +#endif +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; using Xunit.Threading; +#if USES_XUNIT_3 + using Xunit.v3; +#endif internal class IdeTestAssemblyRunner : XunitTestAssemblyRunner { @@ -25,30 +35,75 @@ internal class IdeTestAssemblyRunner : XunitTestAssemblyRunner private HashSet? _ideInstancesInTests; +#if USES_XUNIT_3 + public IdeTestAssemblyRunner(IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + { + ExecutionMessageSink = executionMessageSink; + ExecutionOptions = executionOptions; + } + + private IMessageSink ExecutionMessageSink { get; } + + private ITestFrameworkExecutionOptions ExecutionOptions { get; } +#else public IdeTestAssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) { } +#endif +#if USES_XUNIT_3 + protected override ValueTask OnTestAssemblyStarting(XunitTestAssemblyRunnerContext ctxt) + { + _ideInstancesInTests = new HashSet(); + return base.OnTestAssemblyStarting(ctxt); + } +#else protected override async Task AfterTestAssemblyStartingAsync() { await base.AfterTestAssemblyStartingAsync().ConfigureAwait(false); + + // Note: test collection ordering is now part of the XunitTestAssembly rather than the runner. + // The logic around it for xUnit 3 is in our own Xunit3TestAssembly. TestCollectionOrderer = new TestCollectionOrdererWrapper(TestCollectionOrderer); _ideInstancesInTests = new HashSet(); } +#endif + +#if USES_XUNIT_3 + protected override ValueTask OnTestAssemblyFinished(XunitTestAssemblyRunnerContext ctxt, RunSummary summary) + { + _ideInstancesInTests = null; + return base.OnTestAssemblyFinished(ctxt, summary); + } +#else protected override async Task BeforeTestAssemblyFinishedAsync() { _ideInstancesInTests = null; + TestCollectionOrderer = ((TestCollectionOrdererWrapper)TestCollectionOrderer).Underlying; await base.BeforeTestAssemblyFinishedAsync(); } +#endif +#if USES_XUNIT_3 + protected override async ValueTask RunTestCollection(XunitTestAssemblyRunnerContext ctxt, IXunitTestCollection testCollection, IReadOnlyCollection testCases) +#else protected override async Task RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) +#endif { +#pragma warning disable SA1129 // Do not use default value type constructor - Note: in xUnit 2 it's class, but in 3 it's struct! var result = new RunSummary(); - var testAssemblyFinishedMessages = new List(); +#pragma warning restore SA1129 // Do not use default value type constructor + +#if USES_XUNIT_3 + var messageBus = ctxt.MessageBus; + var cancellationTokenSource = ctxt.CancellationTokenSource; +#endif + var completedTestCaseIds = new HashSet(); +#pragma warning disable CS0168 // Variable is declared but never used try { // Handle [Fact], and also handle IdeSkippedDataRowTestCase that doesn't run inside Visual Studio @@ -56,8 +111,7 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes if (nonIdeTestCases.Any()) { var summary = await RunTestCollectionForUnspecifiedVersionAsync(completedTestCaseIds, messageBus, testCollection, nonIdeTestCases, cancellationTokenSource); - result.Aggregate(summary.Item1); - testAssemblyFinishedMessages.Add(summary.Item2); + result.Aggregate(summary); } var ideTestCases = testCases.OfType().Where(testCase => testCase is not IdeInstanceTestCase).ToArray(); @@ -75,8 +129,7 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes marshalledObjects.Add(visualStudioInstanceFactory); var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, currentAttempt, currentInstance, completedTestCaseIds, messageBus, testCollection, currentTests, cancellationTokenSource); - result.Aggregate(summary.Item1); - testAssemblyFinishedMessages.Add(summary.Item2); + result.Aggregate(summary); currentTests = currentTests.Where(test => !completedTestCaseIds.Contains(test.UniqueID)).ToArray(); if (currentTests.Length == 0) @@ -92,18 +145,70 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes { // Already had at least one test run in this version, so no need to launch it separately. // Report it as passed and continue. - ExecutionMessageSink.OnMessage(new TestClassStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass)); - ExecutionMessageSink.OnMessage(new TestMethodStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod)); - ExecutionMessageSink.OnMessage(new TestCaseStarting(ideInstanceTestCase)); - + // TODO: Ensure that the move to IMessageBus.QueueMessage doesn't break xUnit 2. Otherwise, introduce #if to be more safe. + messageBus.QueueMessage(CreateTestClassStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass)); + messageBus.QueueMessage(CreateTestMethodStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod)); + messageBus.QueueMessage(CreateTestCaseStarting(ideInstanceTestCase)); + +#if USES_XUNIT_3 + DateTimeOffset now = DateTimeOffset.UtcNow; + messageBus.QueueMessage(new TestStarting() + { + AssemblyUniqueID = ideInstanceTestCase.TestCollection.TestAssembly.UniqueID, + Explicit = false, + StartTime = now, + TestCaseUniqueID = ideInstanceTestCase.UniqueID, + TestClassUniqueID = ideInstanceTestCase.TestClass.UniqueID, + TestCollectionUniqueID = ideInstanceTestCase.TestCollection.UniqueID, + TestDisplayName = ideInstanceTestCase.TestCaseDisplayName, // Unsure if this is the right TestDisplayName + TestMethodUniqueID = ideInstanceTestCase.TestMethod.UniqueID, + TestUniqueID = ideInstanceTestCase.UniqueID, // Currently same as TestCaseUniqueID, but that's suspicious and is likely wrong. + Timeout = 0, + Traits = new Dictionary>(), // Unlikely to be correct. + + // General note: ITest is available on TestRunner context, not TestAssemblyRunner context. + // So, with the breaking changes of xUni 3, we may be forced to refactor things in the right place. + // It already feels weird for xUnit 2 implementation that TestAssemblyRunner is responsible for this. + }); + + messageBus.QueueMessage(new TestPassed() + { + AssemblyUniqueID = ideInstanceTestCase.TestCollection.TestAssembly.UniqueID, + ExecutionTime = 0, + FinishTime = now, + Output = string.Empty, + TestCaseUniqueID = ideInstanceTestCase.UniqueID, + TestClassUniqueID = ideInstanceTestCase.TestClass.UniqueID, + TestCollectionUniqueID = ideInstanceTestCase.TestCollection.UniqueID, + TestMethodUniqueID = ideInstanceTestCase.TestMethod.UniqueID, + TestUniqueID = ideInstanceTestCase.UniqueID, // Currently same as TestCaseUniqueID, but that's suspicious and is likely wrong. + Warnings = null, + }); + + messageBus.QueueMessage(new TestFinished() + { + AssemblyUniqueID = ideInstanceTestCase.TestCollection.TestAssembly.UniqueID, + Attachments = new Dictionary(), + ExecutionTime = 0, + FinishTime = now, + Output = string.Empty, + TestCaseUniqueID = ideInstanceTestCase.UniqueID, + TestClassUniqueID = ideInstanceTestCase.TestClass.UniqueID, + TestCollectionUniqueID = ideInstanceTestCase.TestCollection.UniqueID, + TestMethodUniqueID = ideInstanceTestCase.TestMethod.UniqueID, + TestUniqueID = ideInstanceTestCase.UniqueID, // Currently same as TestCaseUniqueID, but that's suspicious and is likely wrong. + Warnings = null, + }); +#else var test = new XunitTest(ideInstanceTestCase, ideInstanceTestCase.DisplayName); - ExecutionMessageSink.OnMessage(new TestStarting(test)); - ExecutionMessageSink.OnMessage(new TestPassed(test, 0, output: null)); - ExecutionMessageSink.OnMessage(new TestFinished(test, 0, output: null)); + messageBus.QueueMessage(new TestStarting(test)); + messageBus.QueueMessage(new TestPassed(test, 0, output: null)); + messageBus.QueueMessage(new TestFinished(test, 0, output: null)); +#endif - ExecutionMessageSink.OnMessage(new TestCaseFinished(ideInstanceTestCase, 0, 1, 0, 0)); - ExecutionMessageSink.OnMessage(new TestMethodFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod, 0, 1, 0, 0)); - ExecutionMessageSink.OnMessage(new TestClassFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass, 0, 1, 0, 0)); + messageBus.QueueMessage(CreateTestCaseFinished(ideInstanceTestCase)); + messageBus.QueueMessage(CreateTestMethodFinished(ideInstanceTestCase)); + messageBus.QueueMessage(CreateTestClassFinished(ideInstanceTestCase)); continue; } @@ -113,8 +218,7 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes { marshalledObjects.Add(visualStudioInstanceFactory); var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, currentAttempt: 0, ideInstanceTestCase.VisualStudioInstanceKey, completedTestCaseIds, messageBus, testCollection, new[] { ideInstanceTestCase }, cancellationTokenSource); - result.Aggregate(summary.Item1); - testAssemblyFinishedMessages.Add(summary.Item2); + result.Aggregate(summary); } } } @@ -124,38 +228,51 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes var remainingTestCases = testCases.Except(completedTestCases); foreach (var casesByTestClass in remainingTestCases.GroupBy(testCase => testCase.TestMethod.TestClass)) { - ExecutionMessageSink.OnMessage(new TestClassStarting(casesByTestClass.ToArray(), casesByTestClass.Key)); + messageBus.QueueMessage(CreateTestClassStarting(casesByTestClass.ToArray(), casesByTestClass.Key)); foreach (var casesByTestMethod in casesByTestClass.GroupBy(testCase => testCase.TestMethod)) { - ExecutionMessageSink.OnMessage(new TestMethodStarting(casesByTestMethod.ToArray(), casesByTestMethod.Key)); + messageBus.QueueMessage(CreateTestMethodStarting(casesByTestMethod.ToArray(), casesByTestMethod.Key)); foreach (var testCase in casesByTestMethod) { - ExecutionMessageSink.OnMessage(new TestCaseStarting(testCase)); + messageBus.QueueMessage(CreateTestCaseStarting(testCase)); +#if USES_XUNIT_3 + throw; // TODO +#else var test = new XunitTest(testCase, testCase.DisplayName); - ExecutionMessageSink.OnMessage(new TestStarting(test)); - ExecutionMessageSink.OnMessage(new TestFailed(test, 0, null, new InvalidOperationException("Test did not run due to a harness failure.", ex))); + messageBus.QueueMessage(new TestStarting(test)); + messageBus.QueueMessage(new TestFailed(test, 0, null, new InvalidOperationException("Test did not run due to a harness failure.", ex))); result.Failed++; - ExecutionMessageSink.OnMessage(new TestFinished(test, 0, null)); + messageBus.QueueMessage(new TestFinished(test, 0, null)); - ExecutionMessageSink.OnMessage(new TestCaseFinished(testCase, 0, 1, 1, 0)); + messageBus.QueueMessage(new TestCaseFinished(testCase, 0, 1, 1, 0)); +#endif } - ExecutionMessageSink.OnMessage(new TestMethodFinished(casesByTestMethod.ToArray(), casesByTestMethod.Key, 0, casesByTestMethod.Count(), casesByTestMethod.Count(), 0)); +#if USES_XUNIT_3 + throw; // TODO +#else + messageBus.QueueMessage(new TestMethodFinished(casesByTestMethod.ToArray(), casesByTestMethod.Key, 0, casesByTestMethod.Count(), casesByTestMethod.Count(), 0)); +#endif } - ExecutionMessageSink.OnMessage(new TestClassFinished(casesByTestClass.ToArray(), casesByTestClass.Key, 0, casesByTestClass.Count(), casesByTestClass.Count(), 0)); +#if USES_XUNIT_3 + throw; // TODO +#else + messageBus.QueueMessage(new TestClassFinished(casesByTestClass.ToArray(), casesByTestClass.Key, 0, casesByTestClass.Count(), casesByTestClass.Count(), 0)); +#endif } } +#pragma warning restore CS0168 // Variable is declared but never used return result; } /// The 0-based attempt number. If this value is /// - 1, a failed test will not be retried. - protected virtual Task> RunTestCollectionForVersionAsync(VisualStudioInstanceFactory visualStudioInstanceFactory, int currentAttempt, VisualStudioInstanceKey visualStudioInstanceKey, HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) + protected virtual Task RunTestCollectionForVersionAsync(VisualStudioInstanceFactory visualStudioInstanceFactory, int currentAttempt, VisualStudioInstanceKey visualStudioInstanceKey, HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) { if (visualStudioInstanceKey.Version == VisualStudioVersion.Unspecified || !IdeTestCaseBase.IsInstalled(visualStudioInstanceKey.Version)) @@ -234,23 +351,30 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes }); } - private async Task> RunTestCollectionForUnspecifiedVersionAsync(HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) + private async Task RunTestCollectionForUnspecifiedVersionAsync(HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) { +#if !USES_XUNIT_3 // These tests just run in the current process, but we still need to hook the assembly and collection events // to work correctly in mixed-testing scenarios. using var marshalledObjects = new MarshalledObjects(); - var executionMessageSinkFilter = new IpcMessageSink(ExecutionMessageSink, testCases.ToDictionary(testCase => testCase.UniqueID, testCase => testCase), finalAttempt: true, completedTestCaseIds, cancellationTokenSource.Token); + var executionMessageSinkFilter = new IpcMessageSink(ExecutionMessageSink, finalAttempt: true, completedTestCaseIds, cancellationTokenSource.Token); marshalledObjects.Add(executionMessageSinkFilter); +#endif + +#if USES_XUNIT_3 + return await XunitTestAssemblyRunner.Instance.Run((IXunitTestAssembly)testCollection.TestAssembly, testCases.CastOrToReadOnlyList(), ExecutionMessageSink, ExecutionOptions, cancellationTokenSource.Token); +#else using (var runner = new XunitTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSinkFilter, ExecutionOptions)) { var runSummary = await runner.RunAsync(); - return Tuple.Create(runSummary, executionMessageSinkFilter.TestAssemblyFinished); + return runSummary; } +#endif } /// The 0-based attempt number. If this value is /// - 1, a failed test will not be retried. - private Func>> CreateTestCollectionInvoker(VisualStudioInstanceFactory visualStudioInstanceFactory, int currentAttempt, VisualStudioInstanceKey visualStudioInstanceKey, HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) + private Func> CreateTestCollectionInvoker(VisualStudioInstanceFactory visualStudioInstanceFactory, int currentAttempt, VisualStudioInstanceKey visualStudioInstanceKey, HashSet completedTestCaseIds, IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) { return async () => { @@ -263,8 +387,7 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes try { var finalAttempt = currentAttempt == visualStudioInstanceKey.MaxAttempts - 1; - var knownTestCasesByUniqueId = testCases.ToDictionary(testCase => testCase.UniqueID, testCase => testCase); - executionMessageSinkFilter = new IpcMessageSink(ExecutionMessageSink, knownTestCasesByUniqueId, finalAttempt, completedTestCaseIds, cancellationTokenSource.Token); + executionMessageSinkFilter = new IpcMessageSink(ExecutionMessageSink, finalAttempt, completedTestCaseIds, cancellationTokenSource.Token); marshalledObjects.Add(executionMessageSinkFilter); // Use SetItems instead of ToImmutableDictionary to avoid exceptions in the case of value conflicts @@ -278,14 +401,25 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes using (var messageFilter = new MessageFilter()) using (var visualStudioContext = await visualStudioInstanceFactory.GetNewOrUsedInstanceAsync(GetVersion(visualStudioInstanceKey.Version), visualStudioInstanceKey.RootSuffix, environmentVariables, GetExtensionFiles(testCases), ImmutableHashSet.Create()).ConfigureAwait(true)) { - using (var runner = visualStudioContext.Instance.TestInvoker.CreateTestAssemblyRunner(new IpcTestAssembly(TestAssembly), testCases.ToArray(), new IpcMessageSink(DiagnosticMessageSink, knownTestCasesByUniqueId, finalAttempt, new HashSet(), cancellationTokenSource.Token), executionMessageSinkFilter, ExecutionOptions)) +#if USES_XUNIT_3 + var testAssembly = (IXunitTestAssembly)testCollection.TestAssembly; +#else + var testAssembly = TestAssembly; +#endif + +#if USES_XUNIT_3 + var runner = visualStudioContext.Instance.TestInvoker.CreateTestAssemblyRunner(new IpcTestAssembly(testAssembly), testCases.ToArray(), executionMessageSinkFilter, ExecutionOptions); +#else + using (var runner = visualStudioContext.Instance.TestInvoker.CreateTestAssemblyRunner(new IpcTestAssembly(testAssembly), testCases.ToArray(), new IpcMessageSink(DiagnosticMessageSink, finalAttempt, new HashSet(), cancellationTokenSource.Token), executionMessageSinkFilter, ExecutionOptions)) +#endif { marshalledObjects.Add(runner); - var ipcMessageBus = new IpcMessageBus(messageBus); - marshalledObjects.Add(ipcMessageBus); - - var result = runner.RunTestCollection(ipcMessageBus, testCollection, testCases.ToArray()); +#if USES_XUNIT_3 + var result = runner.RunTestCollection(new TestContextWrapper(TestContext.Current)); +#else + var result = runner.RunTestCollection(); +#endif var runSummary = new RunSummary { Total = result.Item1, @@ -294,7 +428,7 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes Time = result.Item4, }; - return Tuple.Create(runSummary, executionMessageSinkFilter.TestAssemblyFinished); + return runSummary; } } } @@ -318,20 +452,159 @@ protected override async Task RunTestCollectionAsync(IMessageBus mes }; } + private static TestClassStarting CreateTestClassStarting(IEnumerable testCases, ITestClass testClass) + { +#if USES_XUNIT_3 + return new TestClassStarting() + { + AssemblyUniqueID = testClass.TestCollection.TestAssembly.UniqueID, + TestClassName = testClass.TestClassName, + TestClassNamespace = testClass.TestClassNamespace, + TestClassSimpleName = testClass.TestClassSimpleName, + TestClassUniqueID = testClass.UniqueID, + TestCollectionUniqueID = testClass.TestCollection.UniqueID, + Traits = testClass.Traits, + }; +#else + return new TestClassStarting(testCases, testClass); +#endif + } + + private static TestMethodStarting CreateTestMethodStarting(IEnumerable testCases, ITestMethod testMethod) + { +#if USES_XUNIT_3 + return new TestMethodStarting() + { + AssemblyUniqueID = testMethod.TestClass.TestCollection.TestAssembly.UniqueID, + MethodName = testMethod.MethodName, + TestClassUniqueID = testMethod.TestClass.UniqueID, + TestCollectionUniqueID = testMethod.TestClass.TestCollection.UniqueID, + TestMethodUniqueID = testMethod.UniqueID, + Traits = testMethod.Traits, + }; +#else + return new TestMethodStarting(testCases, testMethod); +#endif + } + + private static TestCaseStarting CreateTestCaseStarting(ITestCase testCase) + { +#if USES_XUNIT_3 + return new TestCaseStarting() + { + AssemblyUniqueID = testCase.TestCollection.TestAssembly.UniqueID, + Explicit = testCase.Explicit, + SkipReason = testCase.SkipReason, + SourceFilePath = testCase.SourceFilePath, + SourceLineNumber = testCase.SourceLineNumber, + TestCaseDisplayName = testCase.TestCaseDisplayName, + TestCaseUniqueID = testCase.UniqueID, + TestClassMetadataToken = testCase.TestClassMetadataToken, + TestClassName = testCase.TestClassName, + TestClassNamespace = testCase.TestClassNamespace, + TestClassSimpleName = testCase.TestClassSimpleName, + TestClassUniqueID = testCase.TestClass?.UniqueID, + TestCollectionUniqueID = testCase.TestCollection.UniqueID, + TestMethodMetadataToken = testCase.TestMethodMetadataToken, + TestMethodName = testCase.TestMethodName, + TestMethodParameterTypesVSTest = testCase.TestMethodParameterTypesVSTest, + TestMethodReturnTypeVSTest = testCase.TestMethodReturnTypeVSTest, + TestMethodUniqueID = testCase.TestMethod?.UniqueID, + Traits = testCase.Traits, + }; +#else + return new TestCaseStarting(testCase); +#endif + } + + private static TestCaseFinished CreateTestCaseFinished(ITestCase testCase) + { +#if USES_XUNIT_3 + return new TestCaseFinished() + { + AssemblyUniqueID = testCase.TestCollection.TestAssembly.UniqueID, + ExecutionTime = 0m, + TestCaseUniqueID = testCase.UniqueID, + TestClassUniqueID = testCase.TestClass?.UniqueID, + TestCollectionUniqueID = testCase.TestCollection.UniqueID, + TestMethodUniqueID = testCase.TestMethod?.UniqueID, + TestsFailed = 0, + TestsNotRun = 0, + TestsSkipped = 0, + TestsTotal = 1, + }; +#else + return new TestCaseFinished(testCase, 0, 1, 0, 0); +#endif + } + + private static TestMethodFinished CreateTestMethodFinished(ITestCase testCase) + { +#if USES_XUNIT_3 + return new TestMethodFinished() + { + AssemblyUniqueID = testCase.TestCollection.TestAssembly.UniqueID, + ExecutionTime = 0m, + TestClassUniqueID = testCase.TestClass?.UniqueID, + TestCollectionUniqueID = testCase.TestCollection.UniqueID, + TestMethodUniqueID = testCase.TestMethod?.UniqueID, + TestsFailed = 0, + TestsNotRun = 0, + TestsSkipped = 0, + TestsTotal = 1, + }; +#else + return new TestMethodFinished(new[] { testCase }, testCase.TestMethod, 0, 1, 0, 0); +#endif + } + + private static TestClassFinished CreateTestClassFinished(ITestCase testCase) + { +#if USES_XUNIT_3 + return new TestClassFinished() + { + AssemblyUniqueID = testCase.TestCollection.TestAssembly.UniqueID, + ExecutionTime = 0m, + TestClassUniqueID = testCase.TestClass?.UniqueID, + TestCollectionUniqueID = testCase.TestCollection.UniqueID, + TestsFailed = 0, + TestsNotRun = 0, + TestsSkipped = 0, + TestsTotal = 1, + }; +#else + return new TestClassFinished(new[] { testCase }, testCase.TestMethod.TestClass, 0, 1, 0, 0); +#endif + } + private ImmutableList GetExtensionFiles(IEnumerable testCases) { var extensionFiles = ImmutableHashSet.Create(StringComparer.OrdinalIgnoreCase); + +#if USES_XUNIT_3 + var visited = new HashSet(); +#else var visited = new HashSet(); +#endif foreach (var testCase in testCases) { +#if USES_XUNIT_3 + var assemblyInfo = testCase.TestClass.Class.Assembly; +#else var assemblyInfo = testCase.Method.Type.Assembly; +#endif + if (!visited.Add(assemblyInfo)) { continue; } var requiredExtensions = assemblyInfo.GetCustomAttributes(typeof(RequireExtensionAttribute)); +#if USES_XUNIT_3 + extensionFiles = extensionFiles.Union(requiredExtensions.Select(attributeInfo => ((RequireExtensionAttribute)attributeInfo).ExtensionFile)); +#else extensionFiles = extensionFiles.Union(requiredExtensions.Select(attributeInfo => attributeInfo.GetConstructorArguments().First().ToString())); +#endif } return extensionFiles.ToImmutableList(); @@ -377,16 +650,14 @@ private VisualStudioInstanceKey GetVisualStudioVersionForTestCase(IXunitTestCase private class IpcMessageSink : MarshalByRefObject, IMessageSink { private readonly IMessageSink _messageSink; - private readonly IReadOnlyDictionary _knownTestCasesByUniqueId; private readonly CancellationToken _cancellationToken; private readonly bool _finalAttempt; private readonly HashSet _completedTestCaseIds; - public IpcMessageSink(IMessageSink messageSink, IReadOnlyDictionary knownTestCasesByUniqueId, bool finalAttempt, HashSet completedTestCaseIds, CancellationToken cancellationToken) + public IpcMessageSink(IMessageSink messageSink, bool finalAttempt, HashSet completedTestCaseIds, CancellationToken cancellationToken) { _messageSink = messageSink; - _knownTestCasesByUniqueId = knownTestCasesByUniqueId; _finalAttempt = finalAttempt; _completedTestCaseIds = completedTestCaseIds; _cancellationToken = cancellationToken; @@ -398,50 +669,15 @@ public string? CurrentTestCase private set; } - public ITestAssemblyFinished? TestAssemblyFinished - { - get; - private set; - } - public bool OnMessage(IMessageSinkMessage message) { if (message is ITestAssemblyFinished testAssemblyFinished) { - // The test cases in the ITestAssemblyFinished message are remote proxies, but the objects won't be - // used until after the remote process terminates. Recreate the objects in the current process (or - // map them to an equivalent object already in the current process) to avoid using objects that are - // no longer available. - var testCases = testAssemblyFinished.TestCases.Select(testCase => - { - if (_knownTestCasesByUniqueId.TryGetValue(testCase.UniqueID, out var knownTestCase)) - { - return knownTestCase; - } - else if (testCase is IdeTestCase ideTestCase) - { - return new IdeTestCase(this, ideTestCase.DefaultMethodDisplay, ideTestCase.DefaultMethodDisplayOptions, ideTestCase.TestMethod, ideTestCase.VisualStudioInstanceKey, ideTestCase.TestMethodArguments); - } - else if (testCase is IdeTheoryTestCase ideTheoryTestCase) - { - return new IdeTheoryTestCase(this, ideTheoryTestCase.DefaultMethodDisplay, ideTheoryTestCase.DefaultMethodDisplayOptions, ideTheoryTestCase.TestMethod, ideTheoryTestCase.VisualStudioInstanceKey, ideTheoryTestCase.TestMethodArguments); - } - else if (testCase is IdeInstanceTestCase ideInstanceTestCase) - { - return new IdeInstanceTestCase(this, ideInstanceTestCase.DefaultMethodDisplay, ideInstanceTestCase.DefaultMethodDisplayOptions, ideInstanceTestCase.TestMethod, ideInstanceTestCase.VisualStudioInstanceKey, ideInstanceTestCase.TestMethodArguments); - } - else - { - return new XunitTestCase(this, TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testCase.TestMethod, testCase.TestMethodArguments); - } - }); - - TestAssemblyFinished = new TestAssemblyFinished(testCases.ToArray(), testAssemblyFinished.TestAssembly, testAssemblyFinished.ExecutionTime, testAssemblyFinished.TestsRun, testAssemblyFinished.TestsFailed, testAssemblyFinished.TestsSkipped); return !_cancellationToken.IsCancellationRequested; } else if (message is ITestCaseStarting testCaseStarting) { - CurrentTestCase = DataCollectionService.GetTestName(testCaseStarting.TestCase); + CurrentTestCase = DataCollectionService.GetTestName(testCaseStarting); return _messageSink.OnMessage(message); } else if (message is ITestCaseFinished testCaseFinished) @@ -450,17 +686,37 @@ public bool OnMessage(IMessageSinkMessage message) if (_finalAttempt || testCaseFinished.TestsFailed == 0) { +#if USES_XUNIT_3 + _completedTestCaseIds.Add(testCaseFinished.TestCaseUniqueID); +#else _completedTestCaseIds.Add(testCaseFinished.TestCase.UniqueID); +#endif } else { // This test will run again; report the statistics as skipped instead of failed +#if USES_XUNIT_3 + message = new TestCaseFinished() + { + AssemblyUniqueID = testCaseFinished.AssemblyUniqueID, + ExecutionTime = testCaseFinished.ExecutionTime, + TestCaseUniqueID = testCaseFinished.TestCaseUniqueID, + TestClassUniqueID = testCaseFinished.TestClassUniqueID, + TestCollectionUniqueID = testCaseFinished.TestCollectionUniqueID, + TestMethodUniqueID = testCaseFinished.TestMethodUniqueID, + TestsFailed = 0, + TestsNotRun = testCaseFinished.TestsNotRun, + TestsSkipped = testCaseFinished.TestsSkipped + testCaseFinished.TestsFailed, + TestsTotal = testCaseFinished.TestsTotal, + }; +#else message = new TestCaseFinished( testCaseFinished.TestCase, testCaseFinished.ExecutionTime, testCaseFinished.TestsRun, testsFailed: 0, testCaseFinished.TestsSkipped + testCaseFinished.TestsFailed); +#endif } return !_cancellationToken.IsCancellationRequested; @@ -469,7 +725,24 @@ public bool OnMessage(IMessageSinkMessage message) { // This test will run again; report it as skipped instead of failed // TODO: What kind of additional logs should we include? +#if USES_XUNIT_3 + message = new TestSkipped() + { + AssemblyUniqueID = testFailed.AssemblyUniqueID, + ExecutionTime = testFailed.ExecutionTime, + FinishTime = testFailed.FinishTime, + Output = testFailed.Output, + Reason = "Test will automatically retry.", + TestCaseUniqueID = testFailed.TestCaseUniqueID, + TestClassUniqueID = testFailed.TestClassUniqueID, + TestCollectionUniqueID = testFailed.TestCollectionUniqueID, + TestMethodUniqueID = testFailed.TestMethodUniqueID, + TestUniqueID = testFailed.TestUniqueID, + Warnings = testFailed.Warnings, + }; +#else message = new TestSkipped(testFailed.Test, "Test will automatically retry."); +#endif } else if (message is ITestAssemblyStarting) { @@ -486,25 +759,53 @@ public bool OnMessage(IMessageSinkMessage message) } } - private class IpcMessageBus : MarshalByRefObject, IMessageBus + private class IpcTestAssembly : LongLivedMarshalByRefObject, +#if USES_XUNIT_3 + IXunitTestAssembly +#else + ITestAssembly +#endif { - private readonly IMessageBus _messageBus; +#if USES_XUNIT_3 + private readonly IXunitTestAssembly _testAssembly; - public IpcMessageBus(IMessageBus messageBus) + public IpcTestAssembly(IXunitTestAssembly testAssembly) { - _messageBus = messageBus; + _testAssembly = testAssembly; } - public void Dispose() => _messageBus.Dispose(); + public Assembly Assembly => _testAssembly.Assembly; - public bool QueueMessage(IMessageSinkMessage message) => _messageBus.QueueMessage(message); + public IReadOnlyCollection AssemblyFixtureTypes => _testAssembly.AssemblyFixtureTypes; - // The life of this object is managed explicitly - public override object? InitializeLifetimeService() => null; - } + public IReadOnlyCollection BeforeAfterTestAttributes => _testAssembly.BeforeAfterTestAttributes; - private class IpcTestAssembly : LongLivedMarshalByRefObject, ITestAssembly - { + public ICollectionBehaviorAttribute? CollectionBehavior => _testAssembly.CollectionBehavior; + +#pragma warning disable SA1316 // Tuple element names should use correct casing + public IReadOnlyDictionary CollectionDefinitions => _testAssembly.CollectionDefinitions; +#pragma warning restore SA1316 // Tuple element names should use correct casing + + public string TargetFramework => _testAssembly.TargetFramework; + + public ITestCaseOrderer? TestCaseOrderer => _testAssembly.TestCaseOrderer; + + public ITestCollectionOrderer? TestCollectionOrderer => _testAssembly.TestCollectionOrderer; + + public Version Version => _testAssembly.Version; + + public string AssemblyName => _testAssembly.AssemblyName; + + public string AssemblyPath => _testAssembly.AssemblyPath; + + public string? ConfigFilePath => _testAssembly.ConfigFilePath; + + public IReadOnlyDictionary> Traits => _testAssembly.Traits; + + public string UniqueID => _testAssembly.UniqueID; + + public Guid ModuleVersionID => _testAssembly.ModuleVersionID; +#else private readonly ITestAssembly _testAssembly; private readonly IAssemblyInfo _assembly; @@ -527,8 +828,10 @@ public void Serialize(IXunitSerializationInfo info) { _testAssembly.Serialize(info); } +#endif } +#if !USES_XUNIT_3 private class IpcAssemblyInfo : LongLivedMarshalByRefObject, IAssemblyInfo { private IAssemblyInfo _assemblyInfo; @@ -584,5 +887,117 @@ private static bool ContainsIdeInstanceCase(ITestCollection collection) return assemblyName.Name == "Microsoft.VisualStudio.Extensibility.Testing.Xunit"; } } +#endif + } + +#if USES_XUNIT_3 + // TestContext.Current is lost when going in-process. + // This class helps marshalling the TestContext from the original process to devenv to restore it there. +#pragma warning disable SA1402 // File may only contain a single type + public sealed class TestContextWrapper : LongLivedMarshalByRefObject, ITestContext +#pragma warning restore SA1402 // File may only contain a single type + { + private readonly ITestContext _testContext; + + public TestContextWrapper(ITestContext testContext) + => _testContext = testContext; + + public IReadOnlyDictionary? Attachments => _testContext.Attachments; + + public CancellationToken CancellationToken => _testContext.CancellationToken; + + public Dictionary KeyValueStorage => _testContext.KeyValueStorage; + + public TestPipelineStage PipelineStage => _testContext.PipelineStage; + + public ITest? Test => _testContext.Test; + + public ITestAssembly? TestAssembly => _testContext.TestAssembly; + + // It has to be public for remoting. + // But intended to be used internally. + [EditorBrowsable(EditorBrowsableState.Never)] + public Assembly? Assembly => (TestAssembly as XunitTestAssembly)?.Assembly; + + public ITestCollectionOrderer? TestCollectionOrderer => (TestAssembly as XunitTestAssembly)?.TestCollectionOrderer; + + public TestEngineStatus? TestAssemblyStatus => _testContext.TestAssemblyStatus; + + public ITestCase? TestCase => _testContext.TestCase; + + public TestEngineStatus? TestCaseStatus => _testContext.TestCaseStatus; + + public ITestClass? TestClass => _testContext.TestClass; + + public object? TestClassInstance => _testContext.TestClassInstance; + + public TestEngineStatus? TestClassStatus => _testContext.TestClassStatus; + + public ITestCollection? TestCollection => _testContext.TestCollection; + + public TestEngineStatus? TestCollectionStatus => _testContext.TestCollectionStatus; + + public ITestMethod? TestMethod => _testContext.TestMethod; + + public TestEngineStatus? TestMethodStatus => _testContext.TestMethodStatus; + + public ITestOutputHelper? TestOutputHelper => _testContext.TestOutputHelper; + + public TestResultState? TestState => _testContext.TestState; + + public TestEngineStatus? TestStatus => _testContext.TestStatus; + + public IReadOnlyList? Warnings => _testContext.Warnings; + + public void AddAttachment(string name, string value) + { + _testContext.AddAttachment(name, value); + } + + public void AddAttachment(string name, byte[] value, string mediaType = "application/octet-stream") + { + _testContext.AddAttachment(name, value, mediaType); + } + + public void AddWarning(string message) + { + _testContext.AddWarning(message); + } + + public void CancelCurrentTest() + { + _testContext.CancelCurrentTest(); + } + + public ValueTask GetFixture(Type fixtureType) + { + return _testContext.GetFixture(fixtureType); + } + + public void SendDiagnosticMessage(string message) + { + _testContext.SendDiagnosticMessage(message); + } + + public void SendDiagnosticMessage(string format, object? arg0) + { + _testContext.SendDiagnosticMessage(format, arg0); + } + + public void SendDiagnosticMessage(string format, object? arg0, object? arg1) + { + _testContext.SendDiagnosticMessage(format, arg0, arg1); + } + + public void SendDiagnosticMessage(string format, object? arg0, object? arg1, object? arg2) + { + _testContext.SendDiagnosticMessage(format, arg0, arg1, arg2); + } + + public void SendDiagnosticMessage(string format, params object?[] args) + { + _testContext.SendDiagnosticMessage(format, args); + } } +#endif } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFramework.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFramework.cs index ac897b48..6c25ca66 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFramework.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFramework.cs @@ -3,20 +3,80 @@ namespace Xunit.Harness { + using System; + using System.Collections.Concurrent; using System.Reflection; + using System.Runtime.Remoting; + using System.Security; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public class IdeTestFramework : XunitTestFramework { +#if !USES_XUNIT_3 public IdeTestFramework(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) { } +#endif - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + protected override ITestFrameworkExecutor CreateExecutor( +#if USES_XUNIT_3 + Assembly assembly) +#else + AssemblyName assemblyName) +#endif { +#if USES_XUNIT_3 + return new IdeTestFrameworkExecutor(assembly); +#else return new IdeTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink); +#endif + } + +#if USES_XUNIT_3 + public override async ValueTask DisposeAsync() + { + await base.DisposeAsync(); + LongLivedMarshalByRefObject.DisconnectAll(); + } +#endif + } + +#if USES_XUNIT_3 +#pragma warning disable SA1402 // File may only contain a single type + public abstract class LongLivedMarshalByRefObject : MarshalByRefObject +#pragma warning restore SA1402 // File may only contain a single type + { + private static ConcurrentBag _remoteObjects = new ConcurrentBag(); + + protected LongLivedMarshalByRefObject() + { + _remoteObjects.Add(this); + } + + [SecuritySafeCritical] + public static void DisconnectAll() + { + foreach (MarshalByRefObject remoteObject in _remoteObjects) + { + RemotingServices.Disconnect(remoteObject); + } + + _remoteObjects = new ConcurrentBag(); + } + + [SecurityCritical] + public sealed override object? InitializeLifetimeService() + { + return null; } } +#endif } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFrameworkExecutor.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFrameworkExecutor.cs index 42159fcb..22a7d36a 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFrameworkExecutor.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestFrameworkExecutor.cs @@ -3,32 +3,104 @@ namespace Xunit.Harness { + using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; + using System.Linq; using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using Xunit; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public class IdeTestFrameworkExecutor : XunitTestFrameworkExecutor { +#if USES_XUNIT_3 + public IdeTestFrameworkExecutor(Assembly assembly) +#pragma warning disable CA1062 // Validate arguments of public methods + : base(new XUnit3TestAssembly(assembly)) +#pragma warning restore CA1062 // Validate arguments of public methods + { + } +#else public IdeTestFrameworkExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) { } +#endif +#if USES_XUNIT_3 + public override async ValueTask RunTestCases(IReadOnlyCollection testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, CancellationToken cancellationToken) +#else [SuppressMessage("Usage", "VSTHRD100:Avoid async void methods", Justification = "Follows pattern expected by Xunit framework.")] protected override async void RunTestCases(IEnumerable testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) +#endif { try { +#if USES_XUNIT_3 + await new IdeTestAssemblyRunner(executionMessageSink, executionOptions).Run(TestAssembly, testCases, executionMessageSink, executionOptions, cancellationToken); +#else using (var assemblyRunner = new IdeTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) { await assemblyRunner.RunAsync(); } +#endif } catch { } } + +#if USES_XUNIT_3 + private sealed class XUnit3TestAssembly : XunitTestAssembly, IXunitTestAssembly + { + public XUnit3TestAssembly(Assembly assembly) + : base(assembly, configFileName: null, assembly.GetName().Version) + { + } + + ITestCollectionOrderer? IXunitTestAssembly.TestCollectionOrderer + { + get + { + var original = ExtensibilityPointFactory.GetAssemblyTestCollectionOrderer(Assembly) ?? DefaultTestCollectionOrderer.Instance; + return new TestCollectionOrdererWrapper(original); // TODO: Copy TestCollectionOrdererWrapper + } + } + } + + private sealed class TestCollectionOrdererWrapper : MarshalByRefObject, ITestCollectionOrderer + { + public TestCollectionOrdererWrapper(ITestCollectionOrderer underlying) + { + Underlying = underlying; + } + + public ITestCollectionOrderer Underlying { get; } + + public IReadOnlyCollection OrderTestCollections(IReadOnlyCollection testCollections) + where TTestCollection : ITestCollection + { + var collections = Underlying.OrderTestCollections(testCollections).ToArray(); + var collectionsWithoutIdeInstanceCases = collections.Where(collection => !ContainsIdeInstanceCase(collection)); + var collectionsWithIdeInstanceCases = collections.Where(collection => ContainsIdeInstanceCase(collection)); + return collectionsWithoutIdeInstanceCases.Concat(collectionsWithIdeInstanceCases).ToArray(); + } + + private static bool ContainsIdeInstanceCase(ITestCollection collection) + { + var assemblyName = new AssemblyName(collection.TestAssembly.AssemblyName); + return assemblyName.Name == "Microsoft.VisualStudio.Extensibility.Testing.Xunit"; + } + } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/InProcessIdeTestAssemblyRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/InProcessIdeTestAssemblyRunner.cs index 7d93a866..70845dcd 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/InProcessIdeTestAssemblyRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/InProcessIdeTestAssemblyRunner.cs @@ -6,54 +6,128 @@ namespace Xunit.Harness using System; using System.Collections.Generic; using System.Linq; + using System.Reflection; using System.Threading; + using System.Threading.Tasks; + using Xunit; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; using Xunit.Threading; +#if USES_XUNIT_3 + using Xunit.v3; +#endif - public class InProcessIdeTestAssemblyRunner : MarshalByRefObject, IDisposable + public class InProcessIdeTestAssemblyRunner : MarshalByRefObject +#if !USES_XUNIT_3 +#pragma warning disable SA1001 // Commas should be spaced correctly + , IDisposable +#pragma warning restore SA1001 // Commas should be spaced correctly +#endif { +#if USES_XUNIT_3 + private readonly IXunitTestAssembly _testAssembly; + private readonly IXunitTestCase[] _testCases; + private readonly IMessageSink _executionMessageSink; + private readonly ITestFrameworkExecutionOptions _executionOptions; +#else private readonly TestAssemblyRunner _testAssemblyRunner; +#endif - public InProcessIdeTestAssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + public InProcessIdeTestAssemblyRunner( +#if USES_XUNIT_3 + IXunitTestAssembly testAssembly, +#else + ITestAssembly testAssembly, +#endif + IEnumerable testCases, +#if !USES_XUNIT_3 + IMessageSink diagnosticMessageSink, +#endif + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) { var reconstructedTestCases = testCases.Select(testCase => { if (testCase is IdeTestCase ideTestCase) { +#if USES_XUNIT_3 + return new IdeTestCase(ideTestCase.TestMethod, ideTestCase.TestCaseDisplayName, ideTestCase.UniqueID, ideTestCase.Explicit, ideTestCase.VisualStudioInstanceKey, ideTestCase.SkipExceptions, ideTestCase.SkipReason, ideTestCase.SkipType, ideTestCase.SkipUnless, ideTestCase.SkipWhen, ideTestCase.Traits, ideTestCase.TestMethodArguments, ideTestCase.SourceFilePath, ideTestCase.SourceLineNumber, ideTestCase.Timeout); +#else return new IdeTestCase(diagnosticMessageSink, ideTestCase.DefaultMethodDisplay, ideTestCase.DefaultMethodDisplayOptions, ideTestCase.TestMethod, ideTestCase.VisualStudioInstanceKey, ideTestCase.TestMethodArguments); +#endif } else if (testCase is IdeTheoryTestCase ideTheoryTestCase) { +#if USES_XUNIT_3 + return new IdeTheoryTestCase(ideTheoryTestCase.TestMethod, ideTheoryTestCase.TestCaseDisplayName, ideTheoryTestCase.UniqueID, ideTheoryTestCase.Explicit, ideTheoryTestCase.VisualStudioInstanceKey, ideTheoryTestCase.SkipExceptions, ideTheoryTestCase.SkipReason, ideTheoryTestCase.SkipType, ideTheoryTestCase.SkipUnless, ideTheoryTestCase.SkipWhen, ideTheoryTestCase.Traits, ideTheoryTestCase.TestMethodArguments, ideTheoryTestCase.SourceFilePath, ideTheoryTestCase.SourceLineNumber, ideTheoryTestCase.Timeout); +#else return new IdeTheoryTestCase(diagnosticMessageSink, ideTheoryTestCase.DefaultMethodDisplay, ideTheoryTestCase.DefaultMethodDisplayOptions, ideTheoryTestCase.TestMethod, ideTheoryTestCase.VisualStudioInstanceKey, ideTheoryTestCase.TestMethodArguments); +#endif } else if (testCase is IdeInstanceTestCase ideInstanceTestCase) { +#if USES_XUNIT_3 + return new IdeInstanceTestCase(ideInstanceTestCase.TestMethod, ideInstanceTestCase.TestCaseDisplayName, ideInstanceTestCase.UniqueID, ideInstanceTestCase.Explicit, ideInstanceTestCase.VisualStudioInstanceKey, ideInstanceTestCase.SkipExceptions, ideInstanceTestCase.SkipReason, ideInstanceTestCase.SkipType, ideInstanceTestCase.SkipUnless, ideInstanceTestCase.SkipWhen, ideInstanceTestCase.Traits, ideInstanceTestCase.TestMethodArguments, ideInstanceTestCase.SourceFilePath, ideInstanceTestCase.SourceLineNumber, ideInstanceTestCase.Timeout); +#else return new IdeInstanceTestCase(diagnosticMessageSink, ideInstanceTestCase.DefaultMethodDisplay, ideInstanceTestCase.DefaultMethodDisplayOptions, ideInstanceTestCase.TestMethod, ideInstanceTestCase.VisualStudioInstanceKey, ideInstanceTestCase.TestMethodArguments); +#endif } return testCase; }); +#if USES_XUNIT_3 + _testAssembly = testAssembly; + _testCases = reconstructedTestCases.ToArray(); + _executionMessageSink = executionMessageSink; + _executionOptions = executionOptions; +#else _testAssemblyRunner = new XunitTestAssemblyRunner(testAssembly, reconstructedTestCases.ToArray(), diagnosticMessageSink, executionMessageSink, executionOptions); +#endif } +#if !USES_XUNIT_3 + // NOTE: These parameters are unused. + // However, for backward compatibility, we keep them as this method is public. public Tuple RunTestCollection(IMessageBus messageBus, ITestCollection testCollection, IXunitTestCase[] testCases) + => RunTestCollection(); +#endif + +#if USES_XUNIT_3 + public Tuple RunTestCollection(TestContextWrapper wrapper) +#else + public Tuple RunTestCollection() +#endif { +#if USES_XUNIT_3 +#pragma warning disable CA1062 // Validate arguments of public methods + RestoreFromWrapper(wrapper); +#pragma warning restore CA1062 // Validate arguments of public methods +#endif + using (var cancellationTokenSource = new CancellationTokenSource()) { #pragma warning disable VSTHRD002 // Avoid problematic synchronous waits +#if USES_XUNIT_3 + var result = XunitTestAssemblyRunner.Instance.Run(_testAssembly, _testCases, _executionMessageSink, _executionOptions, cancellationTokenSource.Token).GetAwaiter().GetResult(); +#else var result = _testAssemblyRunner.RunAsync().GetAwaiter().GetResult(); +#endif #pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + return Tuple.Create(result.Total, result.Failed, result.Skipped, result.Time); } } +#if !USES_XUNIT_3 public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } +#endif // The life of this object is managed explicitly public override object? InitializeLifetimeService() @@ -61,6 +135,7 @@ public void Dispose() return null; } +#if !USES_XUNIT_3 protected virtual void Dispose(bool disposing) { if (disposing) @@ -68,5 +143,52 @@ protected virtual void Dispose(bool disposing) _testAssemblyRunner.Dispose(); } } +#endif + +#if USES_XUNIT_3 + private static void RestoreFromWrapper(TestContextWrapper wrapper) + { + if (wrapper.PipelineStage == TestPipelineStage.TestAssemblyExecution) + { + // NOTE: We cannot simply call SetForTestAssembly here. + // SetForTestAssembly will attempt to call TestContext.CurrentInternal.KeyValueStorage which will throw. + // To hack around that, we call SetForInitialization first, which will create a TestContext with empty dictionary. + // Then, we copy from the existing dictionary to the new one (in most cases (maybe always?), _keyValueStorage is going to be empty). + // Then, we call SetForTestAssembly. At this point, KeyValueStorage will not throw. + TestContext.SetForInitialization(null, false, false); + foreach (var entry in wrapper.KeyValueStorage) + { + TestContext.Current.KeyValueStorage[entry.Key] = entry.Value; + } + + // If wrapper.Assembly is non-null, that means it's our own XUnit3TestAssembly. + // XUnit3TestAssembly is not serializable, so create InProcessTestAssembly that is equivalent. + var testAssembly = wrapper.Assembly is { } assembly + ? new InProcessTestAssembly(assembly, wrapper.TestCollectionOrderer) + : wrapper.TestAssembly; + + TestContext.SetForTestAssembly(testAssembly!, wrapper.TestAssemblyStatus!.Value, default(CancellationToken)); + } + else + { + // If we hit this, add more conditions for different stages as needed. + throw new InvalidOperationException($"Unexpected PipelineStage '{wrapper.PipelineStage}'."); + } + } + + private sealed class InProcessTestAssembly : XunitTestAssembly, IXunitTestAssembly + { + private readonly ITestCollectionOrderer? _testCollectionOrderer; + + public InProcessTestAssembly(Assembly assembly, ITestCollectionOrderer? testCollectionOrderer) + : base(assembly, configFileName: null, assembly.GetName().Version) + { + _testCollectionOrderer = testCollectionOrderer; + } + + ITestCollectionOrderer? IXunitTestAssembly.TestCollectionOrderer + => _testCollectionOrderer; + } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeFactAttribute.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeFactAttribute.cs index ab08e179..7ca875da 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeFactAttribute.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeFactAttribute.cs @@ -5,9 +5,17 @@ namespace Xunit { using System; using Xunit.Sdk; + using Xunit.Threading; +#if USES_XUNIT_3 + using Xunit.v3; +#endif [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +#if USES_XUNIT_3 + [XunitTestCaseDiscoverer(typeof(IdeFactDiscoverer))] +#else [XunitTestCaseDiscoverer("Xunit.Threading.IdeFactDiscoverer", "Microsoft.VisualStudio.Extensibility.Testing.Xunit")] +#endif public class IdeFactAttribute : FactAttribute, IIdeSettingsAttribute { public IdeFactAttribute() diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeTheoryAttribute.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeTheoryAttribute.cs index 949b6260..689f6d5e 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeTheoryAttribute.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/IdeTheoryAttribute.cs @@ -5,9 +5,17 @@ namespace Xunit { using System; using Xunit.Sdk; + using Xunit.Threading; +#if USES_XUNIT_3 + using Xunit.v3; +#endif [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +#if USES_XUNIT_3 + [XunitTestCaseDiscoverer(typeof(IdeTheoryDiscoverer))] +#else [XunitTestCaseDiscoverer("Xunit.Threading.IdeTheoryDiscoverer", "Microsoft.VisualStudio.Extensibility.Testing.Xunit")] +#endif public class IdeTheoryAttribute : TheoryAttribute, IIdeSettingsAttribute { public IdeTheoryAttribute() diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/TestInvoker_InProc.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/TestInvoker_InProc.cs index a715eefd..7a9a35b3 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/TestInvoker_InProc.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/TestInvoker_InProc.cs @@ -10,10 +10,15 @@ namespace Xunit.InProcess using System.Threading.Tasks; using System.Windows; using System.Windows.Threading; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; using Xunit.Threading; +#if USES_XUNIT_3 + using Xunit.v3; +#endif internal class TestInvoker_InProc : InProcComponent { @@ -30,13 +35,36 @@ public void LoadAssembly(string codeBase) var assembly = Assembly.LoadFrom(codeBase); } - public InProcessIdeTestAssemblyRunner CreateTestAssemblyRunner(ITestAssembly testAssembly, IXunitTestCase[] testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + public InProcessIdeTestAssemblyRunner CreateTestAssemblyRunner( +#if USES_XUNIT_3 + IXunitTestAssembly testAssembly, +#else + ITestAssembly testAssembly, +#endif + IXunitTestCase[] testCases, +#if !USES_XUNIT_3 + IMessageSink diagnosticMessageSink, +#endif + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) { - return new InProcessIdeTestAssemblyRunner(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions); + return new InProcessIdeTestAssemblyRunner( + testAssembly, + testCases, +#if !USES_XUNIT_3 + diagnosticMessageSink, +#endif + executionMessageSink, + executionOptions); } +#if !USES_XUNIT_3 // potentially dead code, even for xUnit 2? - https://github.com/microsoft/vs-extension-testing/pull/177 public Tuple InvokeTest( +#if USES_XUNIT_3 + IXunitTest test, +#else ITest test, +#endif IMessageBus messageBus, Type testClass, object?[]? constructorArguments, @@ -79,5 +107,6 @@ public Tuple InvokeTest( return Tuple.Create(result, aggregator.ToException()); } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/VisualStudio_InProc.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/VisualStudio_InProc.cs index 1d304ca2..6435e61d 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/VisualStudio_InProc.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/InProcess/VisualStudio_InProc.cs @@ -55,6 +55,16 @@ public void AddCodeBaseDirectory(string directory) return Assembly.LoadFrom(path); } +#if USES_XUNIT_3 + // For xUnit 3, InProcessIdeTestAssemblyRunner.RestoreFromWrapper will attempt to resolve the test assembly. + // This is a .NET Framework executable. + path = Path.Combine(directory, new AssemblyName(e.Name).Name + ".exe"); + if (File.Exists(path)) + { + return Assembly.LoadFrom(path); + } +#endif + return null; }; } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems index a30d25df..9cdbae2a 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared.projitems @@ -63,7 +63,7 @@ - + diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/OutOfProcess/TestInvoker_OutOfProc.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/OutOfProcess/TestInvoker_OutOfProc.cs index 69584746..c981d1cb 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/OutOfProcess/TestInvoker_OutOfProc.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/OutOfProcess/TestInvoker_OutOfProc.cs @@ -8,10 +8,146 @@ namespace Xunit.OutOfProcess using System.Linq; using System.Reflection; using System.Runtime.Remoting; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.InProcess; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif + +#if USES_XUNIT_3 + // xUnit v3 implementation of ITestFrameworkExecutionOptions isn't serializable. + // Wrapping in a MarshalByRefObject to allow it to be passed in process. +#pragma warning disable SA1402 // File may only contain a single type +#pragma warning disable SA1649 // File name should match first type name + internal sealed class ExecutionOptionsWrapper : LongLivedMarshalByRefObject, ITestFrameworkExecutionOptions +#pragma warning restore SA1649 // File name should match first type name +#pragma warning restore SA1402 // File may only contain a single type + { + private readonly ITestFrameworkExecutionOptions _executionOptions; + + public ExecutionOptionsWrapper(ITestFrameworkExecutionOptions executionOptions) + => _executionOptions = executionOptions; + + public TValue? GetValue(string name) + => _executionOptions.GetValue(name); + + public void SetValue(string name, TValue value) + => _executionOptions.SetValue(name, value); + + public string ToJson() + => _executionOptions.ToJson(); + } + +#pragma warning disable SA1402 // File may only contain a single type + // xUnit v3 implementation of XunitTestCase isn't serializable. + // Wrapping in a MarshalByRefObject to allow it to be passed in process. + internal sealed class XunitTestCaseWrapper : LongLivedMarshalByRefObject, IXunitTestCase +#pragma warning restore SA1402 // File may only contain a single type + { + private readonly IXunitTestCase _testCase; + + public XunitTestCaseWrapper(IXunitTestCase testCase) + => _testCase = testCase; + + public string? SkipReason + => _testCase.SkipReason; + + public Type? SkipType + => _testCase.SkipType; + + public string? SkipUnless + => _testCase.SkipUnless; + + public string? SkipWhen + => _testCase.SkipWhen; + + public IXunitTestClass TestClass + => _testCase.TestClass; + + public int TestClassMetadataToken + => _testCase.TestClassMetadataToken; + + public string TestClassName + => _testCase.TestClassName; + + public string TestClassSimpleName + => _testCase.TestClassSimpleName; + + public IXunitTestCollection TestCollection + => _testCase.TestCollection; + + public IXunitTestMethod TestMethod + => _testCase.TestMethod; + + public int TestMethodMetadataToken + => _testCase.TestMethodMetadataToken; + + public string TestMethodName + => _testCase.TestMethodName; + + public string[] TestMethodParameterTypesVSTest + => _testCase.TestMethodParameterTypesVSTest; + + public string TestMethodReturnTypeVSTest + => _testCase.TestMethodReturnTypeVSTest; + + public int Timeout + => _testCase.Timeout; + + public bool Explicit + => _testCase.Explicit; + + public string? SourceFilePath + => _testCase.SourceFilePath; + + public int? SourceLineNumber + => _testCase.SourceLineNumber; + + public string TestCaseDisplayName + => _testCase.TestCaseDisplayName; + + public string? TestClassNamespace + => _testCase.TestClassNamespace; + + public IReadOnlyDictionary> Traits + => _testCase.Traits; + + public string UniqueID + => _testCase.UniqueID; + + public Type[]? SkipExceptions + => _testCase.SkipExceptions; + + ITestClass? ITestCase.TestClass + => ((ITestCase)_testCase).TestClass; + + ITestCollection ITestCase.TestCollection + => ((ITestCase)_testCase).TestCollection; + + ITestMethod? ITestCase.TestMethod + => ((ITestCase)_testCase).TestMethod; + + int? ITestCaseMetadata.TestClassMetadataToken + => ((ITestCaseMetadata)_testCase).TestClassMetadataToken; + + int? ITestCaseMetadata.TestMethodMetadataToken + => ((ITestCaseMetadata)_testCase).TestMethodMetadataToken; + + public ValueTask> CreateTests() + => _testCase.CreateTests(); + + public void PostInvoke() + => _testCase.PostInvoke(); + + public void PreInvoke() + => _testCase.PreInvoke(); + } +#endif internal class TestInvoker_OutOfProc : OutOfProcComponent { @@ -31,13 +167,40 @@ public void LoadAssembly(string codeBase) TestInvokerInProc.LoadAssembly(codeBase); } - public InProcessIdeTestAssemblyRunner CreateTestAssemblyRunner(ITestAssembly testAssembly, IXunitTestCase[] testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) + public InProcessIdeTestAssemblyRunner CreateTestAssemblyRunner( +#if USES_XUNIT_3 + IXunitTestAssembly testAssembly, +#else + ITestAssembly testAssembly, +#endif + IXunitTestCase[] testCases, +#if !USES_XUNIT_3 + IMessageSink diagnosticMessageSink, +#endif + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) { - return TestInvokerInProc.CreateTestAssemblyRunner(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions); +#if USES_XUNIT_3 + executionOptions = new ExecutionOptionsWrapper(executionOptions); + testCases = testCases.Select(testCase => new XunitTestCaseWrapper(testCase)).ToArray(); +#endif + return TestInvokerInProc.CreateTestAssemblyRunner( + testAssembly, + testCases, +#if !USES_XUNIT_3 + diagnosticMessageSink, +#endif + executionMessageSink, + executionOptions); } +#if !USES_XUNIT_3 // potentially dead code, even for xUnit 2? - https://github.com/microsoft/vs-extension-testing/pull/177 public Tuple InvokeTest( +#if USES_XUNIT_3 + IXunitTest test, +#else ITest test, +#endif IMessageBus messageBus, Type testClass, object?[]? constructorArguments, @@ -70,6 +233,7 @@ public Tuple InvokeTest( testMethod, testMethodArguments); } +#endif private class TestOutputHelperWrapper : MarshalByRefObject, ITestOutputHelper { @@ -80,12 +244,30 @@ public TestOutputHelperWrapper(ITestOutputHelper testOutputHelper) _testOutputHelper = testOutputHelper; } +#if USES_XUNIT_3 + public string Output => _testOutputHelper.Output; + + public void Write(string message) + { + throw new NotImplementedException(); + } + + public void Write(string format, params object[] args) + { + throw new NotImplementedException(); + } +#endif + public void WriteLine(string message) { _testOutputHelper.WriteLine(message); } +#if USES_XUNIT_3 + public void WriteLine(string format, params object[] args) +#else public void WriteLine(string format, params object?[] args) +#endif { _testOutputHelper.WriteLine(format, args); } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/ErrorReportingIdeTestRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/ErrorReportingIdeTestRunner.cs index d677aa79..3be5de4b 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/ErrorReportingIdeTestRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/ErrorReportingIdeTestRunner.cs @@ -8,33 +8,77 @@ namespace Xunit.Threading using System.Reflection; using System.Threading; using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public class ErrorReportingIdeTestRunner : XunitTestRunner { private readonly Exception _exception; +#if USES_XUNIT_3 + public ErrorReportingIdeTestRunner(Exception exception) + { + _exception = exception; + } +#else public ErrorReportingIdeTestRunner(Exception exception, ITest test, IMessageBus messageBus, Type testClass, object?[] constructorArguments, MethodInfo testMethod, object?[]? testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) { _exception = exception; } +#endif +#if USES_XUNIT_3 + protected override ValueTask RunTest(XunitTestRunnerContext ctxt) +#else protected override Task InvokeTestMethodAsync(ExceptionAggregator aggregator) +#endif { +#if USES_XUNIT_3 +#pragma warning disable CA1062 // Validate arguments of public methods + var aggregator = ctxt.Aggregator; +#pragma warning restore CA1062 // Validate arguments of public methods +#endif + +#if !USES_XUNIT_3 if (aggregator is null) { throw new ArgumentNullException(nameof(aggregator)); } +#endif +#pragma warning disable SA1001 // Commas should be spaced correctly +#pragma warning disable SA1113 // Comma should be on the same line as previous parameter +#pragma warning disable SA1115 // Parameter should follow comma +#pragma warning disable SA1009 // Closing parenthesis should be spaced correctly +#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter return aggregator.RunAsync( () => { + var exception = new InvalidOperationException("Test execution was skipped due to a prior exception in the harness.", _exception); + +#if USES_XUNIT_3 + return new ValueTask(Task.FromException(exception)); +#else var tcs = new TaskCompletionSource(); - tcs.SetException(new InvalidOperationException("Test execution was skipped due to a prior exception in the harness.", _exception)); + tcs.SetException(exception); return tcs.Task; - }); +#endif + } +#if USES_XUNIT_3 + , default +#endif + ); +#pragma warning restore SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning restore SA1009 // Closing parenthesis should be spaced correctly +#pragma warning restore SA1115 // Parameter should follow comma +#pragma warning restore SA1113 // Comma should be on the same line as previous parameter +#pragma warning restore SA1001 // Commas should be spaced correctly } } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeFactDiscoverer.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeFactDiscoverer.cs index 66571765..2ae6c28e 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeFactDiscoverer.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeFactDiscoverer.cs @@ -6,61 +6,136 @@ namespace Xunit.Threading using System; using System.Collections.Generic; using System.Linq; + using System.Reflection; using System.Runtime.CompilerServices; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; +#if USES_XUNIT_3 + using Xunit.Internal; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif + +#if USES_XUNIT_3 + using IdeSettingsAttributeAbstractedType = Xunit.IdeSettingsAttribute; + using IFactAttributeType = Xunit.v3.IFactAttribute; + using ITestMethodType = Xunit.v3.IXunitTestMethod; +#else + using IdeSettingsAttributeAbstractedType = Xunit.Abstractions.IAttributeInfo; + using IFactAttributeType = Xunit.Abstractions.IAttributeInfo; + using ITestMethodType = Xunit.Abstractions.ITestMethod; +#endif public class IdeFactDiscoverer : IXunitTestCaseDiscoverer { +#if !USES_XUNIT_3 private readonly IMessageSink _diagnosticMessageSink; public IdeFactDiscoverer(IMessageSink diagnosticMessageSink) { _diagnosticMessageSink = diagnosticMessageSink; } +#endif - public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) + public +#if USES_XUNIT_3 + ValueTask> +#else + IEnumerable +#endif + Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethodType testMethod, IFactAttributeType factAttribute) { if (testMethod is null) { throw new ArgumentNullException(nameof(testMethod)); } + var testCases = new List(); + +#if USES_XUNIT_3 + var details = TestIntrospectionHelper.GetTestCaseDetails(discoveryOptions, testMethod, factAttribute); +#endif if (!testMethod.Method.GetParameters().Any()) { if (!testMethod.Method.IsGenericMethodDefinition) { foreach (var supportedInstance in GetSupportedInstances(testMethod, factAttribute)) { - yield return new IdeTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance); +#if USES_XUNIT_3 + testCases.Add(new IdeTestCase( + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + details.Explicit, + supportedInstance, + details.SkipExceptions, + details.SkipReason, + details.SkipType, + details.SkipUnless, + details.SkipWhen, + testMethod.Traits.ToReadWrite(StringComparer.OrdinalIgnoreCase), + timeout: details.Timeout)); +#else + testCases.Add(new IdeTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance)); +#endif +#if USES_XUNIT_3 + if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, supportedInstance) is { } instanceTestCase) +#else if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, _diagnosticMessageSink, supportedInstance) is { } instanceTestCase) +#endif { - yield return instanceTestCase; + testCases.Add(instanceTestCase); } } } else { - yield return new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[IdeFact] methods are not allowed to be generic."); +#if USES_XUNIT_3 + testCases.Add(new ExecutionErrorTestCase(testMethod, details.TestCaseDisplayName, details.UniqueID, "[IdeFact] methods are not allowed to be generic.")); +#else + testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[IdeFact] methods are not allowed to be generic.")); +#endif } } else { - yield return new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[IdeFact] methods are not allowed to have parameters. Did you mean to use [IdeTheory]?"); +#if USES_XUNIT_3 + testCases.Add(new ExecutionErrorTestCase(testMethod, details.TestCaseDisplayName, details.UniqueID, "[IdeFact] methods are not allowed to have parameters. Did you mean to use [IdeTheory]?")); +#else + testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[IdeFact] methods are not allowed to have parameters. Did you mean to use [IdeTheory]?")); +#endif } + +#if USES_XUNIT_3 + return new ValueTask>(testCases); +#else + return testCases; +#endif } - internal static ITestMethod CreateVisualStudioTestMethod(VisualStudioInstanceKey supportedInstance) + internal static ITestMethodType CreateVisualStudioTestMethod(VisualStudioInstanceKey supportedInstance) { +#if USES_XUNIT_3 + var testAssembly = new XunitTestAssembly(typeof(Instances).Assembly); + var testCollection = new XunitTestCollection(testAssembly, collectionDefinition: null, disableParallelization: true, nameof(Instances)); + var testClass = new XunitTestClass(typeof(Instances), testCollection); + var testMethod = testClass.Methods.Single(method => method.Name == nameof(Instances.VisualStudio)); + return new XunitTestMethod(testClass, testMethod, Array.Empty()); +#else var testAssembly = new TestAssembly(new ReflectionAssemblyInfo(typeof(Instances).Assembly)); var testCollection = new TestCollection(testAssembly, collectionDefinition: null, nameof(Instances)); var testClass = new TestClass(testCollection, new ReflectionTypeInfo(typeof(Instances))); var testMethod = testClass.Class.GetMethods(false).Single(method => method.Name == nameof(Instances.VisualStudio)); return new TestMethod(testClass, testMethod); +#endif } - internal static IEnumerable GetSupportedInstances(ITestMethod testMethod, IAttributeInfo factAttribute) + internal static IEnumerable GetSupportedInstances(ITestMethodType testMethod, IFactAttributeType factAttribute) { var rootSuffix = GetRootSuffix(testMethod, factAttribute); var maxAttempts = GetMaxAttempts(testMethod, factAttribute); @@ -69,35 +144,43 @@ internal static IEnumerable GetSupportedInstances(ITest .Select(version => new VisualStudioInstanceKey(version, rootSuffix, maxAttempts, environmentVariables)); } - private static string GetRootSuffix(ITestMethod testMethod, IAttributeInfo factAttribute) + private static string GetRootSuffix(ITestMethodType testMethod, IFactAttributeType factAttribute) { return GetRootSuffix(factAttribute, GetSettingsAttributes(testMethod).ToArray()); } - private static int GetMaxAttempts(ITestMethod testMethod, IAttributeInfo factAttribute) + private static int GetMaxAttempts(ITestMethodType testMethod, IFactAttributeType factAttribute) { return GetMaxAttempts(factAttribute, GetSettingsAttributes(testMethod).ToArray()); } - private static string[] GetEnvironmentVariables(ITestMethod testMethod, IAttributeInfo factAttribute) + private static string[] GetEnvironmentVariables(ITestMethodType testMethod, IFactAttributeType factAttribute) { return GetEnvironmentVariables(factAttribute, GetSettingsAttributes(testMethod).ToArray()); } - private static IEnumerable GetSettingsAttributes(ITestMethod testMethod) + private static IEnumerable GetSettingsAttributes(ITestMethodType testMethod) { +#if USES_XUNIT_3 + foreach (var attributeData in testMethod.Method.GetCustomAttributes(true).OfType()) +#else foreach (var attributeData in testMethod.Method.GetCustomAttributes(typeof(IdeSettingsAttribute))) +#endif { yield return attributeData; } +#if USES_XUNIT_3 + foreach (var attributeData in testMethod.TestClass.Class.GetCustomAttributes(true).OfType()) +#else foreach (var attributeData in testMethod.TestClass.Class.GetCustomAttributes(typeof(IdeSettingsAttribute))) +#endif { yield return attributeData; } } - private static IEnumerable GetSupportedVersions(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes) + private static IEnumerable GetSupportedVersions(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes) { var minVersion = GetNamedArgument( factAttribute, @@ -131,7 +214,7 @@ private static IEnumerable GetSupportedVersions(IAttributeI } } - private static string GetRootSuffix(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes) + private static string GetRootSuffix(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes) { return GetNamedArgument( factAttribute, @@ -141,7 +224,7 @@ private static string GetRootSuffix(IAttributeInfo factAttribute, IAttributeInfo defaultValue: "Exp"); } - private static int GetMaxAttempts(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes) + private static int GetMaxAttempts(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes) { return GetNamedArgument( factAttribute, @@ -151,7 +234,7 @@ private static int GetMaxAttempts(IAttributeInfo factAttribute, IAttributeInfo[] defaultValue: 1); } - private static string[] GetEnvironmentVariables(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes) + private static string[] GetEnvironmentVariables(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes) { return GetNamedArgument( factAttribute, @@ -187,7 +270,7 @@ private static string[] MergeEnvironmentVariables(string[] inherited, string[] c return set.ToArray(); } - private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func isValidValue, TValue defaultValue) + private static TValue GetNamedArgument(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes, string argumentName, Func isValidValue, TValue defaultValue) { return GetNamedArgument( factAttribute, @@ -198,10 +281,20 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt defaultValue); } - private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func isValidValue, Func? merge, TValue defaultValue) + private static TValue GetNamedArgument(IFactAttributeType factAttribute, IdeSettingsAttributeAbstractedType[] settingsAttributes, string argumentName, Func isValidValue, Func? merge, TValue defaultValue) { StrongBox? result = null; - if (TryGetNamedArgument(factAttribute, argumentName, isValidValue, out var value)) +#pragma warning disable SA1114 // Parameter list should follow declaration +#pragma warning disable SA1003 // Symbols should be spaced correctly + if (TryGetNamedArgument( +#if USES_XUNIT_3 + (Attribute)factAttribute, +#else + factAttribute, +#endif + argumentName, + isValidValue, + out var value)) { if (merge is null) { @@ -210,6 +303,8 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt result = new StrongBox(value); } +#pragma warning restore SA1003 // Symbols should be spaced correctly +#pragma warning restore SA1114 // Parameter list should follow declaration foreach (var attribute in settingsAttributes) { @@ -239,11 +334,36 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt return defaultValue; +#if USES_XUNIT_3 + static bool TryGetNamedArgument(Attribute attribute, string argumentName, Func isValidValue, out TValue value) + { + foreach (var propInfo in attribute.GetType().GetRuntimeProperties()) + { + if (propInfo.Name == argumentName) + { + value = (TValue)propInfo.GetValue(attribute); + return isValidValue(value); + } + } + + foreach (var fieldInfo in attribute.GetType().GetRuntimeFields()) + { + if (fieldInfo.Name == argumentName) + { + value = (TValue)fieldInfo.GetValue(attribute); + return isValidValue(value); + } + } + + throw new ArgumentException($"Could not find property or field named '{argumentName}' on instance of '{attribute.GetType().FullName}'", nameof(argumentName)); + } +#else static bool TryGetNamedArgument(IAttributeInfo attribute, string argumentName, Func isValidValue, out TValue value) { value = attribute.GetNamedArgument(argumentName); return isValidValue(value); } +#endif } private class KeyOnlyComparerIgnoreCase : IEqualityComparer diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeInstanceTestCase.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeInstanceTestCase.cs index bf09150c..e1e966e6 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeInstanceTestCase.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeInstanceTestCase.cs @@ -4,16 +4,31 @@ namespace Xunit.Threading { using System; + using System.Collections.Generic; using System.Collections.Immutable; using System.ComponentModel; + using System.Data; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; +#if USES_XUNIT_3 + using Xunit.Internal; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public sealed class IdeInstanceTestCase : IdeTestCaseBase +#if USES_XUNIT_3 +#pragma warning disable SA1001 // Commas should be spaced correctly + , ISelfExecutingXunitTestCase +#pragma warning restore SA1001 // Commas should be spaced correctly +#endif { /// /// Keep track of unique instances returned for a given discovery pass. The @@ -31,17 +46,66 @@ public IdeInstanceTestCase() { } +#if USES_XUNIT_3 + public IdeInstanceTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + bool @explicit, + VisualStudioInstanceKey visualStudioInstanceKey, + Type[]? skipExceptions = null, + string? skipReason = null, + Type? skipType = null, + string? skipUnless = null, + string? skipWhen = null, + Dictionary>? traits = null, + object?[]? testMethodArguments = null, + string? sourceFilePath = null, + int? sourceLineNumber = null, + int? timeout = null) + : base(testMethod, testCaseDisplayName, uniqueID, @explicit, visualStudioInstanceKey, includeRootSuffixInDisplayName: true, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFilePath, sourceLineNumber, timeout) + { + } +#else public IdeInstanceTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, VisualStudioInstanceKey visualStudioInstanceKey, object?[]? testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, visualStudioInstanceKey, testMethodArguments) { } +#endif +#if !USES_XUNIT_3 protected override bool IncludeRootSuffixInDisplayName => true; +#endif - public static IdeInstanceTestCase? TryCreateNewInstanceForFramework(ITestFrameworkDiscoveryOptions discoveryOptions, IMessageSink diagnosticMessageSink, VisualStudioInstanceKey visualStudioInstanceKey) + public static IdeInstanceTestCase? TryCreateNewInstanceForFramework( + ITestFrameworkDiscoveryOptions discoveryOptions, +#if !USES_XUNIT_3 + IMessageSink diagnosticMessageSink, +#endif + VisualStudioInstanceKey visualStudioInstanceKey) { var lazyInstances = _instances.GetValue(discoveryOptions, static _ => new StrongBox>(ImmutableDictionary.Empty)); - var candidateTestCase = new IdeInstanceTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), IdeFactDiscoverer.CreateVisualStudioTestMethod(visualStudioInstanceKey), visualStudioInstanceKey); + var testMethod = IdeFactDiscoverer.CreateVisualStudioTestMethod(visualStudioInstanceKey); +#if USES_XUNIT_3 + var details = TestIntrospectionHelper.GetTestCaseDetails(discoveryOptions, testMethod, new FactAttribute()); + var traits = TestIntrospectionHelper.GetTraits(testMethod, null); + + var candidateTestCase = new IdeInstanceTestCase( + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + details.Explicit, + visualStudioInstanceKey, + details.SkipExceptions, + details.SkipReason, + details.SkipType, + details.SkipUnless, + details.SkipWhen, + testMethod.Traits.ToReadWrite(StringComparer.OrdinalIgnoreCase), + timeout: details.Timeout); +#else + var candidateTestCase = new IdeInstanceTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, visualStudioInstanceKey); +#endif var testCase = ImmutableInterlocked.GetOrAdd(ref lazyInstances.Value, visualStudioInstanceKey, candidateTestCase); if (testCase != candidateTestCase) { @@ -52,20 +116,37 @@ public IdeInstanceTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay return candidateTestCase; } - public override Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#if USES_XUNIT_3 + public async ValueTask Run(ExplicitOption explicitOption, IMessageBus messageBus, object?[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#else + public override async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#endif { - TestCaseRunner runner; + string displayName = +#if USES_XUNIT_3 + TestCaseDisplayName; +#else + DisplayName; +#endif + if (!string.IsNullOrEmpty(SkipReason)) { // Use XunitTestCaseRunner so the skip gets reported without trying to open VS - runner = new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + var tests = await aggregator.RunAsync(CreateTests, Array.Empty()); + return await XunitTestCaseRunner.Instance.Run(this, tests, messageBus, aggregator, cancellationTokenSource, displayName, SkipReason, ExplicitOption.Off, constructorArguments); +#else + return await new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } else { - runner = new IdeTestCaseRunner(SharedData, VisualStudioInstanceKey, this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + throw new NotImplementedException("TODO"); +#else + return await new IdeTestCaseRunner(SharedData, VisualStudioInstanceKey, this, displayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } - - return runner.RunAsync(); } } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeSkippedDataRowTestCase.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeSkippedDataRowTestCase.cs index 69f73dee..480fe8e6 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeSkippedDataRowTestCase.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeSkippedDataRowTestCase.cs @@ -4,12 +4,23 @@ namespace Xunit.Threading { using System; + using System.Collections.Generic; using System.ComponentModel; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif - public sealed class IdeSkippedDataRowTestCase : XunitSkippedDataRowTestCase + public sealed class IdeSkippedDataRowTestCase +#if USES_XUNIT_3 + : XunitTestCase +#else + : XunitSkippedDataRowTestCase +#endif { [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the deserializer; should only be called by deriving classes for deserialization purposes", error: true)] @@ -17,11 +28,35 @@ public IdeSkippedDataRowTestCase() { } +#if USES_XUNIT_3 + public IdeSkippedDataRowTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + bool @explicit, + VisualStudioInstanceKey visualStudioInstanceKey, + Type[]? skipExceptions = null, + string? skipReason = null, + Type? skipType = null, + string? skipUnless = null, + string? skipWhen = null, + Dictionary>? traits = null, + object?[]? testMethodArguments = null, + string? sourceFilePath = null, + int? sourceLineNumber = null, + int? timeout = null) + : base(testMethod, $"{testCaseDisplayName} ({visualStudioInstanceKey.Version})", $"{uniqueID}_{visualStudioInstanceKey.Version}", @explicit, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFilePath, sourceLineNumber, timeout) + { + } +#else public IdeSkippedDataRowTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, VisualStudioInstanceKey visualStudioInstanceKey, string skipReason, object?[]? testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, skipReason, testMethodArguments) { VisualStudioInstanceKey = visualStudioInstanceKey; } +#endif + +#if !USES_XUNIT_3 public VisualStudioInstanceKey VisualStudioInstanceKey { @@ -61,5 +96,6 @@ public override void Deserialize(IXunitSerializationInfo data) VisualStudioInstanceKey = VisualStudioInstanceKey.DeserializeFromString(data.GetValue(nameof(VisualStudioInstanceKey))); base.Deserialize(data); } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCase.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCase.cs index bea50e7a..8ecb2197 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCase.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCase.cs @@ -4,14 +4,25 @@ namespace Xunit.Threading { using System; + using System.Collections.Generic; using System.ComponentModel; using System.Threading; using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public sealed class IdeTestCase : IdeTestCaseBase +#if USES_XUNIT_3 +#pragma warning disable SA1001 // Commas should be spaced correctly + , ISelfExecutingXunitTestCase +#pragma warning restore SA1001 // Commas should be spaced correctly +#endif { [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the deserializer; should only be called by deriving classes for deserialization purposes", error: true)] @@ -19,25 +30,64 @@ public IdeTestCase() { } +#if USES_XUNIT_3 + public IdeTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + bool @explicit, + VisualStudioInstanceKey visualStudioInstanceKey, + Type[]? skipExceptions = null, + string? skipReason = null, + Type? skipType = null, + string? skipUnless = null, + string? skipWhen = null, + Dictionary>? traits = null, + object?[]? testMethodArguments = null, + string? sourceFilePath = null, + int? sourceLineNumber = null, + int? timeout = null) + : base(testMethod, testCaseDisplayName, uniqueID, @explicit, visualStudioInstanceKey, includeRootSuffixInDisplayName: false, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFilePath, sourceLineNumber, timeout) + { + } +#else public IdeTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, VisualStudioInstanceKey visualStudioInstanceKey, object?[]? testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, visualStudioInstanceKey, testMethodArguments) { } +#endif - public override Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#if USES_XUNIT_3 + public async ValueTask Run(ExplicitOption explicitOption, IMessageBus messageBus, object?[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#else + public override async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#endif { - TestCaseRunner runner; + string displayName = +#if USES_XUNIT_3 + TestCaseDisplayName; +#else + DisplayName; +#endif + if (!string.IsNullOrEmpty(SkipReason)) { // Use XunitTestCaseRunner so the skip gets reported without trying to open VS - runner = new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + var tests = await aggregator.RunAsync(CreateTests, Array.Empty()); + return await XunitTestCaseRunner.Instance.Run(this, tests, messageBus, aggregator, cancellationTokenSource, displayName, SkipReason, ExplicitOption.Off, constructorArguments); +#else + return await new XunitTestCaseRunner(this, displayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } else { - runner = new IdeTestCaseRunner(SharedData, VisualStudioInstanceKey, this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + return await new IdeTestCaseRunner(SharedData, VisualStudioInstanceKey).Run(this, Array.Empty(), messageBus, aggregator, cancellationTokenSource, displayName, SkipReason, ExplicitOption.Off, constructorArguments); +#else + return await new IdeTestCaseRunner(SharedData, VisualStudioInstanceKey, this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } - - return runner.RunAsync(); } } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseBase.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseBase.cs index 4b605835..fe9137a1 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseBase.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseBase.cs @@ -4,11 +4,17 @@ namespace Xunit.Threading { using System; + using System.Collections.Generic; using System.ComponentModel; using Microsoft.Win32; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public abstract class IdeTestCaseBase : XunitTestCase { @@ -20,6 +26,35 @@ protected IdeTestCaseBase() { } +#if USES_XUNIT_3 + protected IdeTestCaseBase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + bool @explicit, + VisualStudioInstanceKey visualStudioInstanceKey, + bool includeRootSuffixInDisplayName, + Type[]? skipExceptions = null, + string? skipReason = null, + Type? skipType = null, + string? skipUnless = null, + string? skipWhen = null, + Dictionary>? traits = null, + object?[]? testMethodArguments = null, + string? sourceFilePath = null, + int? sourceLineNumber = null, + int? timeout = null) + : base(testMethod, AdjustDisplayName(testCaseDisplayName, visualStudioInstanceKey, includeRootSuffixInDisplayName), AdjustUniqueID(uniqueID, visualStudioInstanceKey), @explicit, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFilePath, sourceLineNumber, timeout) + { + SharedData = WpfTestSharedData.Instance; + VisualStudioInstanceKey = visualStudioInstanceKey; + + if (!IsInstalled(visualStudioInstanceKey.Version)) + { + SkipReason = $"{visualStudioInstanceKey.Version} is not installed"; + } + } +#else protected IdeTestCaseBase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, VisualStudioInstanceKey visualStudioInstanceKey, object?[]? testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) { @@ -31,6 +66,7 @@ protected IdeTestCaseBase(IMessageSink diagnosticMessageSink, TestMethodDisplay SkipReason = $"{visualStudioInstanceKey.Version} is not installed"; } } +#endif public VisualStudioInstanceKey VisualStudioInstanceKey { @@ -38,9 +74,11 @@ public VisualStudioInstanceKey VisualStudioInstanceKey private set; } +#if !USES_XUNIT_3 public new TestMethodDisplay DefaultMethodDisplay => base.DefaultMethodDisplay; public new TestMethodDisplayOptions DefaultMethodDisplayOptions => base.DefaultMethodDisplayOptions; +#endif public WpfTestSharedData SharedData { @@ -48,34 +86,53 @@ public WpfTestSharedData SharedData private set; } +#if !USES_XUNIT_3 protected virtual bool IncludeRootSuffixInDisplayName => false; protected override string GetDisplayName(IAttributeInfo factAttribute, string displayName) { var baseName = base.GetDisplayName(factAttribute, displayName); - if (!IncludeRootSuffixInDisplayName || string.IsNullOrEmpty(VisualStudioInstanceKey.RootSuffix)) + return AdjustDisplayName(baseName, VisualStudioInstanceKey, IncludeRootSuffixInDisplayName); + } +#endif + + private static string AdjustDisplayName(string baseName, VisualStudioInstanceKey visualStudioInstanceKey, bool includeRootSuffixInDisplayName) + { + if (!includeRootSuffixInDisplayName || string.IsNullOrEmpty(visualStudioInstanceKey.RootSuffix)) { - return $"{baseName} ({VisualStudioInstanceKey.Version})"; + return $"{baseName} ({visualStudioInstanceKey.Version})"; } else { - return $"{baseName} ({VisualStudioInstanceKey.Version}, {VisualStudioInstanceKey.RootSuffix})"; + return $"{baseName} ({visualStudioInstanceKey.Version}, {visualStudioInstanceKey.RootSuffix})"; } } +#if !USES_XUNIT_3 protected override string GetUniqueID() { - if (string.IsNullOrEmpty(VisualStudioInstanceKey.RootSuffix)) + return AdjustUniqueID(base.GetUniqueID(), VisualStudioInstanceKey); + } +#endif + + private static string AdjustUniqueID(string baseUniqueID, VisualStudioInstanceKey visualStudioInstanceKey) + { + if (string.IsNullOrEmpty(visualStudioInstanceKey.RootSuffix)) { - return $"{base.GetUniqueID()}_{VisualStudioInstanceKey.Version}"; + return $"{baseUniqueID}_{visualStudioInstanceKey.Version}"; } else { - return $"{base.GetUniqueID()}_{VisualStudioInstanceKey.RootSuffix}_{VisualStudioInstanceKey.Version}"; + return $"{baseUniqueID}_{visualStudioInstanceKey.RootSuffix}_{visualStudioInstanceKey.Version}"; } } - public override void Serialize(IXunitSerializationInfo data) +#if USES_XUNIT_3 + protected +#else + public +#endif + override void Serialize(IXunitSerializationInfo data) { if (data is null) { @@ -87,14 +144,19 @@ public override void Serialize(IXunitSerializationInfo data) data.AddValue(nameof(SkipReason), SkipReason); } - public override void Deserialize(IXunitSerializationInfo data) +#if USES_XUNIT_3 + protected +#else + public +#endif + override void Deserialize(IXunitSerializationInfo data) { if (data is null) { throw new ArgumentNullException(nameof(data)); } - VisualStudioInstanceKey = VisualStudioInstanceKey.DeserializeFromString(data.GetValue(nameof(VisualStudioInstanceKey))); + VisualStudioInstanceKey = VisualStudioInstanceKey.DeserializeFromString(data.GetValue(nameof(VisualStudioInstanceKey))!); base.Deserialize(data); SkipReason = data.GetValue(nameof(SkipReason)); SharedData = WpfTestSharedData.Instance; diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseRunner.cs index 1cdcf900..b1113c3a 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTestCaseRunner.cs @@ -8,12 +8,25 @@ namespace Xunit.Threading using System.Diagnostics; using System.Reflection; using System.Threading; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public sealed class IdeTestCaseRunner : XunitTestCaseRunner { +#if USES_XUNIT_3 + public IdeTestCaseRunner(WpfTestSharedData sharedData, VisualStudioInstanceKey visualStudioInstanceKey) + { + SharedData = sharedData; + VisualStudioInstanceKey = visualStudioInstanceKey; + } +#else public IdeTestCaseRunner( WpfTestSharedData sharedData, VisualStudioInstanceKey visualStudioInstanceKey, @@ -30,6 +43,7 @@ public IdeTestCaseRunner( SharedData = sharedData; VisualStudioInstanceKey = visualStudioInstanceKey; } +#endif public WpfTestSharedData SharedData { @@ -41,6 +55,27 @@ public VisualStudioInstanceKey VisualStudioInstanceKey get; } +#if USES_XUNIT_3 + protected override async ValueTask RunTest(XunitTestCaseRunnerContext ctxt, IXunitTest test) + { + if (Process.GetCurrentProcess().ProcessName == "devenv") + { + // We are already running inside Visual Studio + // TODO: Verify version under test +#pragma warning disable CA1062 // Validate arguments of public methods + return await new InProcessIdeTestRunner().Run(test, ctxt.MessageBus, ctxt.ConstructorArguments, ExplicitOption.Off, ctxt.Aggregator, ctxt.CancellationTokenSource, ctxt.BeforeAfterTestAttributes); +#pragma warning restore CA1062 // Validate arguments of public methods + } + else if (SharedData.Exception is not null) + { + return await new ErrorReportingIdeTestRunner(SharedData.Exception).Run(test, ctxt.MessageBus, ctxt.ConstructorArguments, ExplicitOption.Off, ctxt.Aggregator, ctxt.CancellationTokenSource, ctxt.BeforeAfterTestAttributes); + } + else + { + throw new NotSupportedException($"{nameof(IdeFactAttribute)} can only be used with the {nameof(IdeTestFramework)} test framework"); + } + } +#else protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object?[] constructorArguments, MethodInfo testMethod, object?[]? testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { if (Process.GetCurrentProcess().ProcessName == "devenv") @@ -58,5 +93,6 @@ protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus mess throw new NotSupportedException($"{nameof(IdeFactAttribute)} can only be used with the {nameof(IdeTestFramework)} test framework"); } } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryDiscoverer.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryDiscoverer.cs index 0f7737ed..5cf1a6ac 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryDiscoverer.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryDiscoverer.cs @@ -3,17 +3,31 @@ namespace Xunit.Threading { + using System; using System.Collections.Generic; + using System.Data; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif +#if USES_XUNIT_3 + using Xunit.Internal; +#endif using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public class IdeTheoryDiscoverer : TheoryDiscoverer { +#if !USES_XUNIT_3 public IdeTheoryDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) { } +#endif +#if !USES_XUNIT_3 protected override IEnumerable CreateTestCasesForSkip(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, string skipReason) { foreach (var supportedInstance in IdeFactDiscoverer.GetSupportedInstances(testMethod, theoryAttribute)) @@ -33,29 +47,99 @@ protected override IEnumerable CreateTestCasesForSkippedDataRow( yield return new IdeSkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance, skipReason, dataRow); } } +#endif +#if USES_XUNIT_3 + protected override ValueTask> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments) +#else protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object?[] dataRow) +#endif { + var testCases = new List(); foreach (var supportedInstance in IdeFactDiscoverer.GetSupportedInstances(testMethod, theoryAttribute)) { - yield return new IdeTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance, dataRow); +#if USES_XUNIT_3 + var details = TestIntrospectionHelper.GetTestCaseDetailsForTheoryDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments); + var traits = TestIntrospectionHelper.GetTraits(testMethod, dataRow); + testCases.Add(new IdeTestCase( + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + details.Explicit, + supportedInstance, + details.SkipExceptions, + details.SkipReason, + details.SkipType, + details.SkipUnless, + details.SkipWhen, + traits, + testMethodArguments, + timeout: details.Timeout)); +#else + testCases.Add(new IdeTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance, dataRow)); +#endif +#if USES_XUNIT_3 + if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, supportedInstance) is { } instanceTestCase) +#else if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, DiagnosticMessageSink, supportedInstance) is { } instanceTestCase) +#endif { - yield return instanceTestCase; + testCases.Add(instanceTestCase); } } + +#if USES_XUNIT_3 + return new ValueTask>(testCases); +#else + return testCases; +#endif } +#if USES_XUNIT_3 + protected override ValueTask> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute) +#else protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) +#endif { + var testCases = new List(); foreach (var supportedInstance in IdeFactDiscoverer.GetSupportedInstances(testMethod, theoryAttribute)) { - yield return new IdeTheoryTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance); +#if USES_XUNIT_3 + var details = TestIntrospectionHelper.GetTestCaseDetails(discoveryOptions, testMethod, theoryAttribute); + +#pragma warning disable CA1062 // Validate arguments of public methods + testCases.Add(new IdeTheoryTestCase( + details.ResolvedTestMethod, + details.TestCaseDisplayName, + details.UniqueID, + details.Explicit, + supportedInstance, + details.SkipExceptions, + details.SkipReason, + details.SkipType, + details.SkipUnless, + details.SkipWhen, + testMethod.Traits.ToReadWrite(StringComparer.OrdinalIgnoreCase), + timeout: details.Timeout)); +#pragma warning restore CA1062 // Validate arguments of public methods +#else + testCases.Add(new IdeTheoryTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, supportedInstance)); +#endif +#if USES_XUNIT_3 + if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, supportedInstance) is { } instanceTestCase) +#else if (IdeInstanceTestCase.TryCreateNewInstanceForFramework(discoveryOptions, DiagnosticMessageSink, supportedInstance) is { } instanceTestCase) +#endif { - yield return instanceTestCase; + testCases.Add(instanceTestCase); } } + +#if USES_XUNIT_3 + return new ValueTask>(testCases); +#else + return testCases; +#endif } } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCase.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCase.cs index 2f667b88..72b8ead6 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCase.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCase.cs @@ -4,14 +4,25 @@ namespace Xunit.Threading { using System; + using System.Collections.Generic; using System.ComponentModel; using System.Threading; using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public sealed class IdeTheoryTestCase : IdeTestCaseBase +#if USES_XUNIT_3 +#pragma warning disable SA1001 // Commas should be spaced correctly + , ISelfExecutingXunitTestCase +#pragma warning restore SA1001 // Commas should be spaced correctly +#endif { [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the deserializer; should only be called by deriving classes for deserialization purposes", error: true)] @@ -19,25 +30,64 @@ public IdeTheoryTestCase() { } +#if USES_XUNIT_3 + public IdeTheoryTestCase( + IXunitTestMethod testMethod, + string testCaseDisplayName, + string uniqueID, + bool @explicit, + VisualStudioInstanceKey visualStudioInstanceKey, + Type[]? skipExceptions = null, + string? skipReason = null, + Type? skipType = null, + string? skipUnless = null, + string? skipWhen = null, + Dictionary>? traits = null, + object?[]? testMethodArguments = null, + string? sourceFilePath = null, + int? sourceLineNumber = null, + int? timeout = null) + : base(testMethod, testCaseDisplayName, uniqueID, @explicit, visualStudioInstanceKey, includeRootSuffixInDisplayName: false, skipExceptions, skipReason, skipType, skipUnless, skipWhen, traits, testMethodArguments, sourceFilePath, sourceLineNumber, timeout) + { + } +#else public IdeTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, VisualStudioInstanceKey visualStudioInstanceKey, object?[]? testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, visualStudioInstanceKey, testMethodArguments) { } +#endif - public override Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#if USES_XUNIT_3 + public async ValueTask Run(ExplicitOption explicitOption, IMessageBus messageBus, object?[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#else + public override async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) +#endif { - TestCaseRunner runner; + string displayName = +#if USES_XUNIT_3 + TestCaseDisplayName; +#else + DisplayName; +#endif + if (!string.IsNullOrEmpty(SkipReason)) { // Use XunitTheoryTestCaseRunner so the skip gets reported without trying to open VS - runner = new XunitTheoryTestCaseRunner(this, DisplayName, SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + var tests = await aggregator.RunAsync(CreateTests, Array.Empty()); + return await XunitTestCaseRunner.Instance.Run(this, tests, messageBus, aggregator, cancellationTokenSource, displayName, SkipReason, ExplicitOption.Off, constructorArguments); +#else + return await new XunitTheoryTestCaseRunner(this, displayName, SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } else { - runner = new IdeTheoryTestCaseRunner(SharedData, VisualStudioInstanceKey, this, DisplayName, SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource); +#if USES_XUNIT_3 + return await IdeTheoryTestCaseRunner.RunAsync(SharedData, null!/*TODO*/, messageBus, constructorArguments, null! /*TODO: beforeAfterAttributes*/, aggregator, cancellationTokenSource); +#else + return await new IdeTheoryTestCaseRunner(SharedData, VisualStudioInstanceKey, this, displayName, SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource).RunAsync(); +#endif } - - return runner.RunAsync(); } } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCaseRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCaseRunner.cs index 746aecb0..302e4a08 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCaseRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/IdeTheoryTestCaseRunner.cs @@ -8,18 +8,28 @@ namespace Xunit.Threading using System.Diagnostics; using System.Reflection; using System.Threading; + using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif - public sealed class IdeTheoryTestCaseRunner : XunitTheoryTestCaseRunner + public sealed class IdeTheoryTestCaseRunner +#if !USES_XUNIT_3 + : XunitTheoryTestCaseRunner +#endif { +#if !USES_XUNIT_3 public IdeTheoryTestCaseRunner( WpfTestSharedData sharedData, VisualStudioInstanceKey visualStudioInstanceKey, IXunitTestCase testCase, string displayName, - string skipReason, + string? skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, @@ -58,5 +68,26 @@ protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus mess throw new NotSupportedException($"{nameof(IdeTheoryAttribute)} can only be used with the {nameof(IdeTestFramework)} test framework"); } } +#endif + +#if USES_XUNIT_3 + internal static async Task RunAsync(WpfTestSharedData sharedData, IXunitTest test, IMessageBus messageBus, object?[] constructorArguments, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + if (Process.GetCurrentProcess().ProcessName == "devenv") + { + // We are already running inside Visual Studio + // TODO: Verify version under test + return await new InProcessIdeTestRunner().Run(test, messageBus, constructorArguments, ExplicitOption.Off, aggregator, cancellationTokenSource, beforeAfterAttributes); + } + else if (sharedData.Exception is not null) + { + return await new ErrorReportingIdeTestRunner(sharedData.Exception).Run(test, messageBus, constructorArguments, ExplicitOption.Off, aggregator, cancellationTokenSource, beforeAfterAttributes); + } + else + { + throw new NotSupportedException($"{nameof(IdeTheoryAttribute)} can only be used with the {nameof(IdeTestFramework)} test framework"); + } + } +#endif } } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestInvoker.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestInvoker.cs index d5778088..6f2f583b 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestInvoker.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestInvoker.cs @@ -8,24 +8,68 @@ namespace Xunit.Threading using System.Reflection; using System.Threading; using System.Threading.Tasks; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif + +#if USES_XUNIT_3 + using BeforeAfterTestAttributeType = Xunit.v3.IBeforeAfterTestAttribute; +#else + using BeforeAfterTestAttributeType = Xunit.Sdk.BeforeAfterTestAttribute; +#endif public class InProcessIdeTestInvoker : XunitTestInvoker { - private readonly Stack _beforeAfterAttributesRun = new(); - private readonly IReadOnlyList _beforeAfterAttributes; + private readonly Stack _beforeAfterAttributesRun = new(); + private readonly IReadOnlyCollection _beforeAfterAttributes; - public InProcessIdeTestInvoker(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + public InProcessIdeTestInvoker( +#if USES_XUNIT_3 + IXunitTest test, +#else + ITest test, +#endif + IMessageBus messageBus, + Type testClass, + object?[] constructorArguments, + MethodInfo testMethod, + object?[] testMethodArguments, +#if USES_XUNIT_3 + IReadOnlyCollection beforeAfterAttributes, +#else + IReadOnlyList beforeAfterAttributes, +#endif + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cancellationTokenSource) { _beforeAfterAttributes = beforeAfterAttributes; } - public new Task RunAsync() + public +#if !USES_XUNIT_3 + new +#endif + async +#if USES_XUNIT_3 + Task +#else + Task +#endif + RunAsync() { - return Aggregator.RunAsync(async delegate +#pragma warning disable SA1009 // Closing parenthesis should be spaced correctly +#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning disable SA1001 // Commas should be spaced correctly +#pragma warning disable SA1113 // Comma should be on the same line as previous parameter +#pragma warning disable SA1115 // Parameter should follow comma +#pragma warning disable SA1116 // Split parameters should start on line after declaration + return await Aggregator.RunAsync(async delegate { if (!CancellationTokenSource.IsCancellationRequested) { @@ -75,13 +119,32 @@ await Aggregator.RunAsync(async () => { Aggregator.Run(delegate { +#if USES_XUNIT_3 + Test.DisposeTestClass(testClassInstance, MessageBus, CancellationTokenSource); +#else Test.DisposeTestClass(testClassInstance, MessageBus, Timer, CancellationTokenSource); +#endif }); } } +#if USES_XUNIT_3 + // TODO: Measure time correctly + return TimeSpan.Zero; +#else return Timer.Total; - }); +#endif + } +#pragma warning restore SA1116 // Split parameters should start on line after declaration +#if USES_XUNIT_3 + , TimeSpan.Zero +#endif + ); +#pragma warning restore SA1115 // Parameter should follow comma +#pragma warning restore SA1113 // Comma should be on the same line as previous parameter +#pragma warning restore SA1001 // Commas should be spaced correctly +#pragma warning restore SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning restore SA1009 // Closing parenthesis should be spaced correctly } protected override object CreateTestClass() @@ -96,12 +159,88 @@ protected override object CreateTestClass() } } - protected override Task BeforeTestMethodInvokedAsync() + private static BeforeTestStarting CreateBeforeTestStarting(ITest test, string attributeName) + { +#if USES_XUNIT_3 + return new BeforeTestStarting() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + AttributeName = attributeName, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + }; +#else + return new BeforeTestStarting(test, attributeName); +#endif + } + + private static AfterTestStarting CreateAfterTestStarting(ITest test, string attributeName) + { +#if USES_XUNIT_3 + return new AfterTestStarting() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + AttributeName = attributeName, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + }; +#else + return new AfterTestStarting(test, attributeName); +#endif + } + + private static BeforeTestFinished CreateBeforeTestFinished(ITest test, string attributeName) + { +#if USES_XUNIT_3 + return new BeforeTestFinished() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + AttributeName = attributeName, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + }; +#else + return new BeforeTestFinished(test, attributeName); +#endif + } + + private static AfterTestFinished CreateAfterTestFinished(ITest test, string attributeName) + { +#if USES_XUNIT_3 + return new AfterTestFinished() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + AttributeName = attributeName, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + }; +#else + return new AfterTestFinished(test, attributeName); +#endif + } + + protected +#if !USES_XUNIT_3 + override +#endif + Task BeforeTestMethodInvokedAsync() { foreach (var beforeAfterAttribute in _beforeAfterAttributes) { var attributeName = beforeAfterAttribute.GetType().Name; - if (!MessageBus.QueueMessage(new BeforeTestStarting(Test, attributeName))) + if (!MessageBus.QueueMessage(CreateBeforeTestStarting(Test, attributeName))) { CancellationTokenSource.Cancel(); } @@ -109,7 +248,11 @@ protected override Task BeforeTestMethodInvokedAsync() { try { - Timer.Aggregate(() => beforeAfterAttribute.Before(TestMethod)); +#if USES_XUNIT_3 + beforeAfterAttribute.Before(TestMethod, Test); +#else + beforeAfterAttribute.Before(TestMethod); +#endif _beforeAfterAttributesRun.Push(beforeAfterAttribute); } catch (Exception ex) when (DataCollectionService.LogAndCatch(ex)) @@ -119,7 +262,7 @@ protected override Task BeforeTestMethodInvokedAsync() } finally { - if (!MessageBus.QueueMessage(new BeforeTestFinished(Test, attributeName))) + if (!MessageBus.QueueMessage(CreateBeforeTestFinished(Test, attributeName))) { CancellationTokenSource.Cancel(); } @@ -141,7 +284,17 @@ protected override Task BeforeTestMethodInvokedAsync() #endif } - protected override async Task InvokeTestMethodAsync(object testClassInstance) + protected +#if !USES_XUNIT_3 + override +#endif + async +#if USES_XUNIT_3 + Task +#else + Task +#endif + InvokeTestMethodAsync(object testClassInstance) { var oldSyncContext = SynchronizationContext.Current; @@ -150,8 +303,13 @@ protected override async Task InvokeTestMethodAsync(object testClassIns var asyncSyncContext = new AsyncTestSyncContext(oldSyncContext); SynchronizationContext.SetSynchronizationContext(asyncSyncContext); +#pragma warning disable SA1114 // Parameter list should follow declaration +#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning disable SA1009 // Closing parenthesis should be spaced correctly await Aggregator.RunAsync( - () => Timer.AggregateAsync( +#if !USES_XUNIT_3 + async () => await Timer.AggregateAsync( +#endif async () => { var parameterCount = TestMethod.GetParameters().Length; @@ -192,14 +350,23 @@ await Aggregator.RunAsync( } } } - })); + } +#if !USES_XUNIT_3 + ) +#endif + ); +#pragma warning restore SA1009 // Closing parenthesis should be spaced correctly +#pragma warning restore SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning restore SA1114 // Parameter list should follow declaration } finally { SynchronizationContext.SetSynchronizationContext(oldSyncContext); } +#if !USES_XUNIT_3 return Timer.Total; +#endif } protected override object CallTestMethod(object testClassInstance) @@ -214,32 +381,49 @@ protected override object CallTestMethod(object testClassInstance) } } - protected override Task AfterTestMethodInvokedAsync() + protected +#if !USES_XUNIT_3 + override +#endif + Task AfterTestMethodInvokedAsync() { foreach (var beforeAfterAttribute in _beforeAfterAttributesRun) { var attributeName = beforeAfterAttribute.GetType().Name; - if (!MessageBus.QueueMessage(new AfterTestStarting(Test, attributeName))) + if (!MessageBus.QueueMessage(CreateAfterTestStarting(Test, attributeName))) { CancellationTokenSource.Cancel(); } Aggregator.Run(() => { +#if !USES_XUNIT_3 +#pragma warning disable SA1009 // Closing parenthesis should be spaced correctly +#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter Timer.Aggregate(() => +#endif { try { +#if USES_XUNIT_3 + beforeAfterAttribute.After(TestMethod, Test); +#else beforeAfterAttribute.After(TestMethod); +#endif } catch (Exception ex) when (DataCollectionService.LogAndPropagate(ex)) { throw ExceptionUtilities.Unreachable; } - }); + } +#if !USES_XUNIT_3 + ); +#pragma warning restore SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning restore SA1009 // Closing parenthesis should be spaced correctly +#endif }); - if (!MessageBus.QueueMessage(new AfterTestFinished(Test, attributeName))) + if (!MessageBus.QueueMessage(CreateAfterTestFinished(Test, attributeName))) { CancellationTokenSource.Cancel(); } diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestRunner.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestRunner.cs index b7892d84..d52a25de 100644 --- a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestRunner.cs +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Threading/InProcessIdeTestRunner.cs @@ -5,25 +5,60 @@ namespace Xunit.Threading { using System; using System.Collections.Generic; + using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; using System.Windows; using System.Windows.Threading; +#if !USES_XUNIT_3 using Xunit.Abstractions; +#endif using Xunit.Harness; using Xunit.InProcess; using Xunit.Sdk; +#if USES_XUNIT_3 + using Xunit.v3; +#endif public class InProcessIdeTestRunner : XunitTestRunner { +#if !USES_XUNIT_3 public InProcessIdeTestRunner(ITest test, IMessageBus messageBus, Type testClass, object?[] constructorArguments, MethodInfo testMethod, object?[]? testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) { } +#endif +#if USES_XUNIT_3 + protected override async ValueTask RunTest(XunitTestRunnerContext ctxt) +#else protected override async Task InvokeTestMethodAsync(ExceptionAggregator aggregator) +#endif { +#if USES_XUNIT_3 +#pragma warning disable CA1062 // Validate arguments of public methods + var test = ctxt.Test; +#pragma warning restore CA1062 // Validate arguments of public methods + var messageBus = ctxt.MessageBus; + var testClass = ctxt.TestMethod.DeclaringType; + var constructorArguments = ctxt.ConstructorArguments; + var testMethod = ctxt.TestMethod; + var testMethodArguments = ctxt.TestMethodArguments; + var beforeAfterAttributes = ctxt.BeforeAfterTestAttributes; + var aggregator = ctxt.Aggregator; + var cts = ctxt.CancellationTokenSource; +#else + var test = Test; + var messageBus = MessageBus; + var testClass = TestClass; + var constructorArguments = ConstructorArguments; + var testMethod = TestMethod; + var testMethodArguments = TestMethodArguments; + var beforeAfterAttributes = BeforeAfterAttributes; + var cts = CancellationTokenSource; +#endif + DataCollectionService.InstallFirstChanceExceptionHandler(); VisualStudio_InProc.Create().ActivateMainWindow(); @@ -31,9 +66,9 @@ protected override async Task InvokeTestMethodAsync(ExceptionAggregator var taskScheduler = new SynchronizationContextTaskScheduler(synchronizationContext); try { - DataCollectionService.CurrentTest = Test; + DataCollectionService.CurrentTest = test; return await Task.Factory.StartNew( - () => new InProcessIdeTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource).RunAsync(), + () => new InProcessIdeTestInvoker(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cts).RunAsync(), CancellationToken.None, TaskCreationOptions.None, taskScheduler).Unwrap(); diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests.csproj b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests.csproj new file mode 100644 index 00000000..b0c1855b --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests.csproj @@ -0,0 +1,52 @@ + + + + + net472 + true + Exe + + + + + + + + + + + + + + + + + + + + + + + + + <_Parameter1>Xunit.Harness.IdeTestFramework + <_Parameter1_TypeName>System.Type + + + + + + PreserveNewest + + + + + + diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Properties/AssemblyInfo.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..4c72974d --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: CLSCompliant(false)] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/xunit.runner.json b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/xunit.runner.json new file mode 100644 index 00000000..1c72a421 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.IntegrationTests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "shadowCopy": false +} diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncManualResetEvent.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncManualResetEvent.cs new file mode 100644 index 00000000..1d86a07c --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncManualResetEvent.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +#if USES_XUNIT_3 + +namespace Xunit.Sdk +{ + using System.Threading.Tasks; + + internal class AsyncManualResetEvent + { + private volatile TaskCompletionSource _taskCompletionSource = new TaskCompletionSource(); + + public AsyncManualResetEvent(bool signaled = false) + { + if (signaled) + { + _taskCompletionSource.TrySetResult(true); + } + } + + public bool IsSet + { + get { return _taskCompletionSource.Task.IsCompleted; } + } + + public Task WaitAsync() + { + return _taskCompletionSource.Task; + } + + public void Set() + { + _taskCompletionSource.TrySetResult(true); + } + + public void Reset() + { + if (IsSet) + { + _taskCompletionSource = new TaskCompletionSource(); + } + } + } +} +#endif diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncTestSyncContext.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncTestSyncContext.cs new file mode 100644 index 00000000..61ce5df3 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/AsyncTestSyncContext.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +#if USES_XUNIT_3 +namespace Xunit.Sdk +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// This implementation of allows the developer to track the count + /// of outstanding "async void" operations, and wait for them all to complete. + /// + public class AsyncTestSyncContext : SynchronizationContext + { + private static readonly TaskFactory TaskFactory = new TaskFactory(); + + private readonly AsyncManualResetEvent _event = new AsyncManualResetEvent(true); + private readonly SynchronizationContext _innerContext; + private Exception? _exception; + private int _operationCount; + + /// + /// Initializes a new instance of the class. + /// + /// The existing synchronization context (may be null). + public AsyncTestSyncContext(SynchronizationContext innerContext) + { + _innerContext = innerContext; + } + + /// + public override void OperationCompleted() + { + var result = Interlocked.Decrement(ref _operationCount); + if (result == 0) + { + _event.Set(); + } + } + + /// + public override void OperationStarted() + { + Interlocked.Increment(ref _operationCount); + _event.Reset(); + } + + /// + public override void Post(SendOrPostCallback d, object state) + { + // The call to Post() may be the state machine signaling that an exception is + // about to be thrown, so we make sure the operation count gets incremented + // before the Task.Run, and then decrement the count when the operation is done. + OperationStarted(); + + try + { + if (_innerContext == null) + { + QueueUserWorkItem(() => + { + try + { + d(state); + } + catch (Exception ex) + { + _exception = ex; + } + finally + { + OperationCompleted(); + } + }); + } + else + { +#pragma warning disable VSTHRD001 // Avoid legacy thread switching APIs + _innerContext.Post( + _ => + { + try + { + d(state); + } + catch (Exception ex) + { + _exception = ex; + } + finally + { + OperationCompleted(); + } + }, + null); +#pragma warning restore VSTHRD001 // Avoid legacy thread switching APIs + } + } + catch + { + } + } + + /// + public override void Send(SendOrPostCallback d, object state) + { + try + { + if (_innerContext != null) + { +#pragma warning disable VSTHRD001 // Avoid legacy thread switching APIs + _innerContext.Send(d, state); +#pragma warning restore VSTHRD001 // Avoid legacy thread switching APIs + } + else + { +#pragma warning disable CA1062 // Validate arguments of public methods + d(state); +#pragma warning restore CA1062 // Validate arguments of public methods + } + } + catch (Exception ex) + { + _exception = ex; + } + } + + /// + /// Returns a task which is signaled when all outstanding operations are complete. + /// + public async Task WaitForCompletionAsync() + { + await _event.WaitAsync(); + + return _exception; + } + + public static void QueueUserWorkItem(Action backgroundTask, EventWaitHandle? finished = null) + { +#pragma warning disable VSTHRD110 // Observe result of async calls + TaskFactory.StartNew( + _ => + { + var state = (State)_; + + try + { + state.BackgroundTask(); + } + finally + { + if (state.Finished != null) + { + state.Finished.Set(); + } + } + }, + new State { BackgroundTask = backgroundTask, Finished = finished }, + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); +#pragma warning restore VSTHRD110 // Observe result of async calls + } + + private class State + { +#pragma warning disable SA1401 // Fields should be private + public Action BackgroundTask = null!; + public EventWaitHandle? Finished; +#pragma warning restore SA1401 // Fields should be private + } + } +} +#endif diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.csproj b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.csproj new file mode 100644 index 00000000..28ca5e82 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.csproj @@ -0,0 +1,53 @@ + + + + + net472 + Xunit + true + $(DefineConstants);MERGED_PIA;USES_XUNIT_3 + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/PolyfillExtensions.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/PolyfillExtensions.cs new file mode 100644 index 00000000..88e1fd42 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/PolyfillExtensions.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +#if USES_XUNIT_3 + +namespace Xunit.Sdk +{ + using System; + using System.Threading; + using Xunit.v3; + + internal static class PolyfillExtensions + { + public static object? CreateTestClass( + this ITest test, + Type testClassType, + object?[]? constructorArguments, + IMessageBus messageBus, + CancellationTokenSource cancellationTokenSource) + { + object? testClass = null; + + if (!messageBus.QueueMessage(new TestClassConstructionStarting() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + })) + { + cancellationTokenSource.Cancel(); + } + else + { + try + { + if (!cancellationTokenSource.IsCancellationRequested) + { + testClass = Activator.CreateInstance(testClassType, constructorArguments); + } + } + finally + { + if (!messageBus.QueueMessage(new TestClassConstructionFinished() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + })) + { + cancellationTokenSource.Cancel(); + } + } + } + + return testClass; + } + + public static void DisposeTestClass(this ITest test, object testClass, IMessageBus messageBus, CancellationTokenSource cancellationTokenSource) + { + if (!(testClass is IDisposable disposable)) + { + return; + } + + if (!messageBus.QueueMessage(new TestClassDisposeStarting() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + })) + { + cancellationTokenSource.Cancel(); + return; + } + + try + { + disposable.Dispose(); + } + finally + { + if (!messageBus.QueueMessage(new TestClassDisposeFinished() + { + AssemblyUniqueID = test.TestCase.TestCollection.TestAssembly.UniqueID, + TestCaseUniqueID = test.TestCase.UniqueID, + TestClassUniqueID = test.TestCase.TestClass?.UniqueID, + TestCollectionUniqueID = test.TestCase.TestCollection.UniqueID, + TestMethodUniqueID = test.TestCase.TestMethod?.UniqueID, + TestUniqueID = test.UniqueID, + })) + { + cancellationTokenSource.Cancel(); + } + } + } + } +} +#endif diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Properties/AssemblyInfo.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..4c72974d --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: CLSCompliant(false)] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/XunitTestInvoker.cs b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/XunitTestInvoker.cs new file mode 100644 index 00000000..131cb8df --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/XunitTestInvoker.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for more information. + +#if USES_XUNIT_3 + +namespace Xunit.Sdk +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using Xunit.v3; + +#if USES_XUNIT_3 + using BeforeAfterTestAttributeType = Xunit.v3.IBeforeAfterTestAttribute; +#else + using BeforeAfterTestAttributeType = Xunit.Sdk.BeforeAfterTestAttribute; +#endif + + public class XunitTestInvoker + { + private static MethodInfo? _startAsTaskOpenGenericMethod; + + public XunitTestInvoker(IXunitTest test, IMessageBus messageBus, Type testClass, object?[]? constructorArguments, MethodInfo testMethod, object?[]? testMethodArguments, IReadOnlyCollection beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) + { + Test = test; + MessageBus = messageBus; + TestClass = testClass; + ConstructorArguments = constructorArguments; + TestMethod = testMethod; + TestMethodArguments = testMethodArguments; + BeforeAfterAttributes = beforeAfterAttributes; + Aggregator = aggregator; + CancellationTokenSource = cancellationTokenSource; + } + + public IXunitTest Test { get; } + + public IMessageBus MessageBus { get; } + + public Type TestClass { get; } + + public object?[]? ConstructorArguments { get; } + + public MethodInfo TestMethod { get; } + + public object?[]? TestMethodArguments { get; } + + public IReadOnlyCollection BeforeAfterAttributes { get; } + + public ExceptionAggregator Aggregator { get; } + + public CancellationTokenSource CancellationTokenSource { get; } + + protected virtual object CreateTestClass() + { + object? testClass = null; + + if (!TestMethod.IsStatic && !Aggregator.HasExceptions) + { + testClass = Test.CreateTestClass(TestClass, ConstructorArguments, MessageBus, CancellationTokenSource); + } + + return testClass!; + } + + protected virtual object CallTestMethod(object testClassInstance) + => TestMethod.Invoke(testClassInstance, TestMethodArguments); + +#pragma warning disable VSTHRD200 // Use "Async" suffix for async methods + public static Task? GetTaskFromResult(object obj) +#pragma warning restore VSTHRD200 // Use "Async" suffix for async methods + { + if (obj == null) + { + return null; + } + + var task = obj as Task; + if (task != null) + { + return task; + } + + var type = obj.GetType(); + if (type.IsGenericType && type.GetGenericTypeDefinition().FullName == "Microsoft.FSharp.Control.FSharpAsync`1") + { + if (_startAsTaskOpenGenericMethod == null) + { + _startAsTaskOpenGenericMethod = type.Assembly.GetType("Microsoft.FSharp.Control.FSharpAsync") + .GetRuntimeMethods() + .FirstOrDefault(m => m.Name == "StartAsTask"); + } + + return _startAsTaskOpenGenericMethod.MakeGenericMethod(type.GetGenericArguments()[0]) + .Invoke(null, new[] { obj, null, null }) as Task; + } + + return null; + } + } +} + +#endif diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.props b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.props new file mode 100644 index 00000000..2dbc8c22 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.targets b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.targets new file mode 100644 index 00000000..6f0412b5 --- /dev/null +++ b/src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3/build/Microsoft.VisualStudio.Extensibility.Testing.Xunit.v3.targets @@ -0,0 +1,16 @@ + + + + + + true + + + + + <_Parameter1>Xunit.Harness.IdeTestFramework + <_Parameter1_TypeName>System.Type + + + +