From 147cb87310811cfee4df7080d37e4de3a8db53c4 Mon Sep 17 00:00:00 2001 From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:38:29 -0700 Subject: [PATCH] Port crash dump collection options to release/10.0 (#16716) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 7 +- eng/Versions.props | 2 +- .../Microsoft.DotNet.RemoteExecutor.csproj | 25 +++ .../src/RemoteExecutor.cs | 23 +++ .../src/RemoteInvokeHandle.cs | 163 ++++++++++-------- .../src/RemoteInvokeOptions.cs | 42 +++++ ...crosoft.DotNet.RemoteExecutor.Tests.csproj | 15 ++ .../tests/RemoteExecutorTests.cs | 93 ++++++++++ 8 files changed, 291 insertions(+), 79 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 9e2d6effc94..dc079f197f2 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -55,7 +55,7 @@ - + @@ -63,7 +63,7 @@ - + @@ -90,7 +90,8 @@ - + + diff --git a/eng/Versions.props b/eng/Versions.props index 50fe1932967..59218293b38 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -17,7 +17,7 @@ 1.1.1 - 4.5.5 + 4.6.3 6.0.1 diff --git a/src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj b/src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj index 8a76b50f650..2d8d8004bf9 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj +++ b/src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj @@ -12,10 +12,35 @@ + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteExecutor.cs b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteExecutor.cs index 17e38b1008d..11382b96a4b 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteExecutor.cs +++ b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteExecutor.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -453,6 +454,28 @@ private static RemoteInvokeHandle Invoke(MethodInfo method, string[] args, psi.Environment.Remove("CoreClr_Enable_Profiling"); } + if (options.CrashDumpCollectionType is CrashDumpCollectionType.None) + { + psi.Environment.Remove("DOTNET_DbgEnableMiniDump"); + psi.Environment.Remove("DOTNET_DbgMiniDumpType"); + psi.Environment.Remove("DOTNET_DbgMiniDumpName"); + } + else if (options.CrashDumpCollectionType.HasValue) + { + psi.Environment["DOTNET_DbgEnableMiniDump"] = "1"; + psi.Environment["DOTNET_DbgMiniDumpType"] = ((int)options.CrashDumpCollectionType.Value).ToString(CultureInfo.InvariantCulture); + if (!string.IsNullOrWhiteSpace(options.CrashDumpPath)) + { + psi.Environment["DOTNET_DbgMiniDumpName"] = options.CrashDumpPath; + } + else + { + string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); + string dumpDir = !string.IsNullOrWhiteSpace(uploadPath) ? uploadPath : IOPath.GetTempPath(); + psi.Environment["DOTNET_DbgMiniDumpName"] = IOPath.Combine(dumpDir, "%e.%p.%t.dmp"); + } + } + // If we need the host (if it exists), use it, otherwise target the console app directly. string metadataArgs = PasteArguments.Paste(new string[] { a.FullName, t.FullName, method.Name, options.ExceptionFile }, pasteFirstArgumentUsingArgV0Rules: false); string passedArgs = pasteArguments ? PasteArguments.Paste(args, pasteFirstArgumentUsingArgV0Rules: false) : string.Join(" ", args); diff --git a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeHandle.cs b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeHandle.cs index 70254c07553..c50576d68eb 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeHandle.cs +++ b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeHandle.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.Diagnostics.Runtime; using System; using System.Collections.Generic; using System.Diagnostics; @@ -10,6 +9,10 @@ using System.Runtime.InteropServices; using System.Text; using System.Threading; +#if NETCOREAPP +using Microsoft.Diagnostics.NETCore.Client; +#endif +using Microsoft.Diagnostics.Runtime; namespace Microsoft.DotNet.RemoteExecutor { @@ -146,87 +149,16 @@ private void Dispose(bool disposing) { description.AppendLine($"Timed out at {DateTime.Now} after {Options.TimeOut}ms waiting for remote process."); - // Create a dump if possible - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); - if (!string.IsNullOrWhiteSpace(uploadPath)) - { - try - { - string miniDmpPath = Path.Combine(uploadPath, $"{Process.Id}.{Path.GetRandomFileName()}.dmp"); - MiniDump.Create(Process, miniDmpPath); - description.AppendLine($"Wrote mini dump to: {miniDmpPath}"); - } - catch (Exception exc) - { - description.AppendLine($"Failed to create mini dump: {exc.Message}"); - } - } - } - - // Gather additional details about the process if possible - try + if (Options.EnableTimeoutDumpCollection) { - description.AppendLine($"\tProcess ID: {Process.Id}"); - description.AppendLine($"\tHandle: {Process.Handle}"); - description.AppendLine($"\tName: {Process.ProcessName}"); - description.AppendLine($"\tMainModule: {Process.MainModule?.FileName}"); - description.AppendLine($"\tStartTime: {Process.StartTime}"); - description.AppendLine($"\tTotalProcessorTime: {Process.TotalProcessorTime}"); - - // Attach ClrMD to gather some additional details. - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && // As of Microsoft.Diagnostics.Runtime v1.0.5, process attach only works on Windows. - Interlocked.CompareExchange(ref s_clrMdLock, 1, 0) == 0) // Make sure we only attach to one process at a time. - { - try - { - using (DataTarget dt = DataTarget.AttachToProcess(Process.Id, msecTimeout: 20_000)) // arbitrary timeout - { - ClrRuntime runtime = dt.ClrVersions.FirstOrDefault()?.CreateRuntime(); - if (runtime != null) - { - // Dump the threads in the remote process. - description.AppendLine("\tThreads:"); - foreach (ClrThread thread in runtime.Threads.Where(t => t.IsAlive)) - { - string threadKind = - thread.IsThreadpoolCompletionPort ? "[Thread pool completion port]" : - thread.IsThreadpoolGate ? "[Thread pool gate]" : - thread.IsThreadpoolTimer ? "[Thread pool timer]" : - thread.IsThreadpoolWait ? "[Thread pool wait]" : - thread.IsThreadpoolWorker ? "[Thread pool worker]" : - thread.IsFinalizer ? "[Finalizer]" : - thread.IsGC ? "[GC]" : - ""; - - string isBackground = thread.IsBackground ? "[Background]" : ""; - string apartmentModel = thread.IsMTA ? "[MTA]" : - thread.IsSTA ? "[STA]" : - ""; - - description.AppendLine($"\t\tThread #{thread.ManagedThreadId} (OS 0x{thread.OSThreadId:X}) {threadKind} {isBackground} {apartmentModel}"); - foreach (ClrStackFrame frame in thread.StackTrace) - { - description.AppendLine($"\t\t\t{frame}"); - } - } - } - } - } - finally - { - Interlocked.Exchange(ref s_clrMdLock, 0); - } - } + CollectTimeoutDiagnostics(description); } - catch { } throw new RemoteExecutionException(description.ToString()); } } - FileInfo exceptionFileInfo = new FileInfo(Options.ExceptionFile); + FileInfo exceptionFileInfo = new(Options.ExceptionFile); if (exceptionFileInfo.Exists && exceptionFileInfo.Length != 0) { throw new RemoteExecutionException("Remote process failed with an unhandled exception.", File.ReadAllText(Options.ExceptionFile)); @@ -271,6 +203,87 @@ private void Dispose(bool disposing) } } + /// + /// Collects diagnostic information (dump + thread stacks) from the timed-out remote process. + /// + private void CollectTimeoutDiagnostics(StringBuilder description) + { + string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); + if (!string.IsNullOrWhiteSpace(uploadPath)) + { + try + { + string dumpPath = Path.Combine(uploadPath, $"{Process.Id}.{Path.GetRandomFileName()}.dmp"); +#if NETCOREAPP + // These define guards assume that harness running on .NET Framework implies test process runs on .NET Framework. + var client = new DiagnosticsClient(Process.Id); + client.WriteDump(DumpType.WithHeap, dumpPath, logDumpGeneration: false); +#else + MiniDump.Create(Process, dumpPath); +#endif + description.AppendLine($"Wrote dump to: {dumpPath}"); + } + catch (Exception exc) + { + description.AppendLine($"Failed to create dump: {exc.Message}"); + } + } + + // Gather additional details about the process if possible + try + { + description.AppendLine($"\tProcess ID: {Process.Id}"); + description.AppendLine($"\tHandle: {Process.Handle}"); + description.AppendLine($"\tName: {Process.ProcessName}"); + description.AppendLine($"\tMainModule: {Process.MainModule?.FileName}"); + description.AppendLine($"\tStartTime: {Process.StartTime}"); + description.AppendLine($"\tTotalProcessorTime: {Process.TotalProcessorTime}"); + + // Attach ClrMD to gather some additional details. + if (Interlocked.CompareExchange(ref s_clrMdLock, 1, 0) == 0) // Make sure we only attach to one process at a time. + { + try + { + using DataTarget dt = DataTarget.CreateSnapshotAndAttach(Process.Id); + ClrRuntime runtime = dt.ClrVersions.FirstOrDefault()?.CreateRuntime(); + if (runtime is not null) + { + // Dump the threads in the remote process. + description.AppendLine("\tThreads:"); + foreach (ClrThread thread in runtime.Threads.Where(t => t.IsAlive)) + { + ClrThreadState state = thread.State; + // ClrMD v4 no longer exposes gate/timer/wait thread pool flags, so only the + // completion port and worker kinds remain detectable via ClrThreadState. + string threadKind = + (state & ClrThreadState.TS_CompletionPortThread) != 0 ? "[Thread pool completion port]" : + (state & ClrThreadState.TS_TPWorkerThread) != 0 ? "[Thread pool worker]" : + thread.IsFinalizer ? "[Finalizer]" : + thread.IsGc ? "[GC]" : + ""; + + string isBackground = (state & ClrThreadState.TS_Background) != 0 ? "[Background]" : ""; + string apartmentModel = (state & ClrThreadState.TS_InMTA) != 0 ? "[MTA]" : + (state & ClrThreadState.TS_InSTA) != 0 ? "[STA]" : + ""; + + description.AppendLine($"\t\tThread #{thread.ManagedThreadId} (OS 0x{thread.OSThreadId:X}) {threadKind} {isBackground} {apartmentModel}"); + foreach (ClrStackFrame frame in thread.EnumerateStackTrace(includeContext: false)) + { + description.AppendLine($"\t\t\t{frame}"); + } + } + } + } + finally + { + Interlocked.Exchange(ref s_clrMdLock, 0); + } + } + } + catch { } + } + ~RemoteInvokeHandle() { // Finalizer flags tests that omitted the explicit Dispose() call; they must have it, or they aren't diff --git a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeOptions.cs b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeOptions.cs index 932b7c807d9..59d19183d76 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeOptions.cs +++ b/src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeOptions.cs @@ -9,6 +9,23 @@ namespace Microsoft.DotNet.RemoteExecutor { + /// + /// The type of crash dump to collect. Maps to DOTNET_DbgMiniDumpType values + /// as documented in the docs. Only applies to .NET Core subprocesses. + /// + public enum CrashDumpCollectionType + { + /// + /// Explicitly disables crash dump collection, removing any inherited DOTNET_DbgEnableMiniDump, + /// DOTNET_DbgMiniDumpType, and DOTNET_DbgMiniDumpName environment variables from the subprocess. + /// + None = 0, + Mini = 1, + Heap = 2, + Triage = 3, + Full = 4 + } + /// /// Options used with RemoteInvoke. /// @@ -22,6 +39,8 @@ public sealed class RemoteInvokeOptions public bool EnableProfiling { get; set; } = true; + public bool EnableTimeoutDumpCollection { get; set; } = true; + public bool CheckExitCode { get; set; } = true; /// @@ -62,5 +81,28 @@ public bool RunAsSudo /// Specifies the roll-forward policy for dotnet cli to use. Only applies when running .NET Core /// public string RollForward { get; set; } + + /// + /// Gets or sets the type of crash dump to collect on the subprocess via + /// DOTNET_DbgEnableMiniDump / DOTNET_DbgMiniDumpType / DOTNET_DbgMiniDumpName. + /// When set to a value other than , + /// crash dump collection is enabled with that dump type. + /// When set to , crash dump collection is + /// explicitly disabled (removing any inherited env vars). + /// When null (default), the environment variables are left as-is. + /// + /// + /// Only applies to .NET Core subprocesses. + /// + public CrashDumpCollectionType? CrashDumpCollectionType { get; set; } + + /// + /// Gets or sets the path template for crash dump files. When is set, + /// this value is used for DOTNET_DbgMiniDumpName. Supports the same placeholders as createdump: + /// %p (PID), %e (process name), %t (timestamp), etc. + /// When null (default), defaults to HELIX_WORKITEM_UPLOAD_ROOT/%e.%p.%t.dmp if running in Helix, + /// or the system temp directory otherwise. + /// + public string CrashDumpPath { get; set; } } } diff --git a/src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj b/src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj index 0db92001071..df3f2cd1389 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj +++ b/src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj @@ -8,5 +8,20 @@ + + + + + + + + + + diff --git a/src/Microsoft.DotNet.RemoteExecutor/tests/RemoteExecutorTests.cs b/src/Microsoft.DotNet.RemoteExecutor/tests/RemoteExecutorTests.cs index 466c1807290..896506db60f 100644 --- a/src/Microsoft.DotNet.RemoteExecutor/tests/RemoteExecutorTests.cs +++ b/src/Microsoft.DotNet.RemoteExecutor/tests/RemoteExecutorTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.IO; using System.Threading.Tasks; using Xunit; using Xunit.Sdk; @@ -158,5 +159,97 @@ public static void IgnoreExitCode() Assert.Equal(exitCode, h.ExitCode); } } + + [Theory] + [InlineData(CrashDumpCollectionType.Mini, "1")] + [InlineData(CrashDumpCollectionType.Heap, "2")] + [InlineData(CrashDumpCollectionType.Triage, "3")] + [InlineData(CrashDumpCollectionType.Full, "4")] + public void CrashDumpCollection_SetsEnvVars(CrashDumpCollectionType dumpType, string expectedTypeValue) + { + using RemoteInvokeHandle h = RemoteExecutor.Invoke(expectedType => + { + Assert.Equal("1", Environment.GetEnvironmentVariable("DOTNET_DbgEnableMiniDump")); + Assert.Equal(expectedType, Environment.GetEnvironmentVariable("DOTNET_DbgMiniDumpType")); + return RemoteExecutor.SuccessExitCode; + }, expectedTypeValue, new RemoteInvokeOptions + { + RollForward = "Major", + CrashDumpCollectionType = dumpType + }); + } + + [Fact] + public void DisableCrashDumpCollection_RemovesEnvVars() + { + // Pre-set the env vars on the StartInfo to simulate inherited values + var options = new RemoteInvokeOptions + { + RollForward = "Major", + CrashDumpCollectionType = CrashDumpCollectionType.None + }; + options.StartInfo.Environment["DOTNET_DbgEnableMiniDump"] = "1"; + options.StartInfo.Environment["DOTNET_DbgMiniDumpType"] = "4"; + options.StartInfo.Environment["DOTNET_DbgMiniDumpName"] = "/tmp/test.dmp"; + + using RemoteInvokeHandle h = RemoteExecutor.Invoke(() => + { + Assert.Null(Environment.GetEnvironmentVariable("DOTNET_DbgEnableMiniDump")); + Assert.Null(Environment.GetEnvironmentVariable("DOTNET_DbgMiniDumpType")); + Assert.Null(Environment.GetEnvironmentVariable("DOTNET_DbgMiniDumpName")); + }, options); + } + + [Fact] + public void CrashDumpCollection_DefaultLeavesEnvVarsUntouched() + { + // When CrashDumpCollectionType is not set, RemoteExecutor should neither add nor remove + // the DOTNET_Dbg* env vars: whatever is on the StartInfo passes through to the child unchanged. + // Set known values explicitly so the assertion is deterministic regardless of the parent's environment. + var options = new RemoteInvokeOptions { RollForward = "Major" }; + options.StartInfo.Environment["DOTNET_DbgEnableMiniDump"] = "1"; + options.StartInfo.Environment["DOTNET_DbgMiniDumpType"] = "4"; + options.StartInfo.Environment["DOTNET_DbgMiniDumpName"] = "/tmp/passthrough.dmp"; + + using RemoteInvokeHandle h = RemoteExecutor.Invoke(() => + { + // The child should observe exactly the values set on the parent's StartInfo. + Assert.Equal("1", Environment.GetEnvironmentVariable("DOTNET_DbgEnableMiniDump")); + Assert.Equal("4", Environment.GetEnvironmentVariable("DOTNET_DbgMiniDumpType")); + Assert.Equal("/tmp/passthrough.dmp", Environment.GetEnvironmentVariable("DOTNET_DbgMiniDumpName")); + }, options); + } + + [Fact] + public static unsafe void CrashDumpCollection_CreatesDumpOnCrash() + { + string dumpDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(dumpDir); + try + { + var options = new RemoteInvokeOptions + { + RollForward = "Major", + CrashDumpCollectionType = CrashDumpCollectionType.Mini, + CheckExitCode = false, + // Point the dump path to our temp directory so we can verify the file is created. + // Use %p so the filename includes the PID and is unique. + CrashDumpPath = Path.Combine(dumpDir, "crashdump.%p.dmp") + }; + + RemoteExecutor.Invoke(() => + { + // Trigger an access violation to crash the process + *(int*)0x10000 = 0; + }, options).Dispose(); + + string[] dumpFiles = Directory.GetFiles(dumpDir, "*.dmp"); + Assert.NotEmpty(dumpFiles); + } + finally + { + Directory.Delete(dumpDir, recursive: true); + } + } } }