Skip to content

Enhance crash dump collection options in RemoteExecutor (release/10.0) #17060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
hoyosjs wants to merge 1 commit into
base: release/10.0
Choose a base branch
from
Draft

Enhance crash dump collection options in RemoteExecutor (release/10.0) #17060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,15 @@
<!-- roslyn -->
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisCSharpVersion)" />
<!-- runtime -->
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="$(MicrosoftBclAsyncInterfacesVersion)" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.8" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsDependencyInjectionAbstractionsVersion)" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="$(MicrosoftExtensionsDependencyInjectionVersion)" />
<PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="$(MicrosoftExtensionsDependencyModelVersion)" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpVersion)" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsFileProvidersAbstractionsVersion)" />
<PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
<PackageVersion Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
<PackageVersion Include="System.Collections.Immutable" Version="10.0.7" />
<PackageVersion Include="System.Composition" Version="$(SystemCompositionVersion)" />
<PackageVersion Include="System.IO.Packaging" Version="$(SystemIOPackagingVersion)" />
<PackageVersion Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
Expand All @@ -90,7 +90,8 @@
<PackageVersion Include="Microsoft.ApplicationInsights" Version="2.23.0" />
<PackageVersion Include="Microsoft.Data.OData" Version="5.8.4" />
<PackageVersion Include="Microsoft.DataServices.Client" Version="$(MicrosoftDataServicesClientVersion)" />
<PackageVersion Include="Microsoft.Diagnostics.Runtime" Version="1.0.5" />
<PackageVersion Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.721401" />
<PackageVersion Include="Microsoft.Diagnostics.Runtime" Version="4.0.732101" />
<PackageVersion Include="Microsoft.Identity.Client" Version="4.73.1" />
<PackageVersion Include="Microsoft.OpenApi" Version="1.3.2" />
<PackageVersion Include="Microsoft.OpenApi.Readers" Version="1.3.2" />
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
</PropertyGroup>
<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
<MicrosoftBclHashCodeVersion>1.1.1</MicrosoftBclHashCodeVersion>
<SystemMemoryVersion>4.5.5</SystemMemoryVersion>
<SystemMemoryVersion>4.6.3</SystemMemoryVersion>
<MicrosoftIORedistVersion>6.0.1</MicrosoftIORedistVersion>
</PropertyGroup>
<PropertyGroup>
Expand Down
25 changes: 25 additions & 0 deletions src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,35 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
<PackageReference Include="Microsoft.Diagnostics.Runtime" />
<PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'">
<Compile Remove="MiniDump.cs" />
</ItemGroup>

<!--
Microsoft.Diagnostics.Runtime 4.x pulls Azure.Identity >= 1.21.0 -> Azure.Core >= 1.53.0, which in turn
requires the .NET 10 (10.0.x) versions of System.Text.Json/Encodings.Web/Bcl.AsyncInterfaces/Extensions.*
and Microsoft.Identity.Client. On release/10.0 the central versions of those packages are intentionally
pinned lower (servicing / build-task binding-redirect constraints), so we override them locally for this
project only via VersionOverride instead of bumping the central versions repo-wide. This keeps the upgrade
isolated to RemoteExecutor and leaves all other consumers (Feed, Helix, SymbolHelper, ArcadeAzureIntegration,
netfx build tasks) untouched.
-->
<ItemGroup>
<PackageReference Include="Azure.Identity" VersionOverride="1.21.0" />
<PackageReference Include="Azure.Core" VersionOverride="1.53.0" />
<PackageReference Include="Microsoft.Identity.Client" VersionOverride="4.83.1" />
<PackageReference Include="System.Text.Json" VersionOverride="10.0.3" />
<PackageReference Include="System.Text.Encodings.Web" VersionOverride="10.0.3" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" VersionOverride="10.0.3" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" VersionOverride="10.0.3" />
<PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" VersionOverride="10.0.3" />
</ItemGroup>

<ItemGroup>
<None Include="build\$(MSBuildProjectName).targets" PackagePath="build\$(MSBuildProjectName).targets" Pack="true" />
<None Include="$(RepoRoot)LICENSE.txt" PackagePath="LICENSE.txt" Pack="true" />
Expand Down
23 changes: 23 additions & 0 deletions src/Microsoft.DotNet.RemoteExecutor/src/RemoteExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
Expand Down Expand Up @@ -453,6 +454,28 @@ private static RemoteInvokeHandle Invoke(MethodInfo method, string[] args,
psi.Environment.Remove("CoreClr_Enable_Profiling");
}

if (options.CrashDumpCollectionType is CrashDumpCollectionType.None)
{
psi.Environment.Remove("DOTNET_DbgEnableMiniDump");
psi.Environment.Remove("DOTNET_DbgMiniDumpType");
psi.Environment.Remove("DOTNET_DbgMiniDumpName");
}
else if (options.CrashDumpCollectionType.HasValue)
{
psi.Environment["DOTNET_DbgEnableMiniDump"] = "1";
psi.Environment["DOTNET_DbgMiniDumpType"] = ((int)options.CrashDumpCollectionType.Value).ToString(CultureInfo.InvariantCulture);
if (!string.IsNullOrWhiteSpace(options.CrashDumpPath))
{
psi.Environment["DOTNET_DbgMiniDumpName"] = options.CrashDumpPath;
}
else
{
string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
string dumpDir = !string.IsNullOrWhiteSpace(uploadPath) ? uploadPath : IOPath.GetTempPath();
psi.Environment["DOTNET_DbgMiniDumpName"] = IOPath.Combine(dumpDir, "%e.%p.%t.dmp");
}
}

// If we need the host (if it exists), use it, otherwise target the console app directly.
string metadataArgs = PasteArguments.Paste(new string[] { a.FullName, t.FullName, method.Name, options.ExceptionFile }, pasteFirstArgumentUsingArgV0Rules: false);
string passedArgs = pasteArguments ? PasteArguments.Paste(args, pasteFirstArgumentUsingArgV0Rules: false) : string.Join(" ", args);
Expand Down
163 changes: 88 additions & 75 deletions src/Microsoft.DotNet.RemoteExecutor/src/RemoteInvokeHandle.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.Diagnostics.Runtime;
using System;
using System.Collections.Generic;
using System.Diagnostics;
Expand All @@ -10,6 +9,10 @@
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
#if NETCOREAPP
using Microsoft.Diagnostics.NETCore.Client;
#endif
using Microsoft.Diagnostics.Runtime;

namespace Microsoft.DotNet.RemoteExecutor
{
Expand Down Expand Up @@ -146,87 +149,16 @@ private void Dispose(bool disposing)
{
description.AppendLine($"Timed out at {DateTime.Now} after {Options.TimeOut}ms waiting for remote process.");

// Create a dump if possible
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
if (!string.IsNullOrWhiteSpace(uploadPath))
{
try
{
string miniDmpPath = Path.Combine(uploadPath, $"{Process.Id}.{Path.GetRandomFileName()}.dmp");
MiniDump.Create(Process, miniDmpPath);
description.AppendLine($"Wrote mini dump to: {miniDmpPath}");
}
catch (Exception exc)
{
description.AppendLine($"Failed to create mini dump: {exc.Message}");
}
}
}

// Gather additional details about the process if possible
try
if (Options.EnableTimeoutDumpCollection)
{
description.AppendLine($"\tProcess ID: {Process.Id}");
description.AppendLine($"\tHandle: {Process.Handle}");
description.AppendLine($"\tName: {Process.ProcessName}");
description.AppendLine($"\tMainModule: {Process.MainModule?.FileName}");
description.AppendLine($"\tStartTime: {Process.StartTime}");
description.AppendLine($"\tTotalProcessorTime: {Process.TotalProcessorTime}");

// Attach ClrMD to gather some additional details.
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && // As of Microsoft.Diagnostics.Runtime v1.0.5, process attach only works on Windows.
Interlocked.CompareExchange(ref s_clrMdLock, 1, 0) == 0) // Make sure we only attach to one process at a time.
{
try
{
using (DataTarget dt = DataTarget.AttachToProcess(Process.Id, msecTimeout: 20_000)) // arbitrary timeout
{
ClrRuntime runtime = dt.ClrVersions.FirstOrDefault()?.CreateRuntime();
if (runtime != null)
{
// Dump the threads in the remote process.
description.AppendLine("\tThreads:");
foreach (ClrThread thread in runtime.Threads.Where(t => t.IsAlive))
{
string threadKind =
thread.IsThreadpoolCompletionPort ? "[Thread pool completion port]" :
thread.IsThreadpoolGate ? "[Thread pool gate]" :
thread.IsThreadpoolTimer ? "[Thread pool timer]" :
thread.IsThreadpoolWait ? "[Thread pool wait]" :
thread.IsThreadpoolWorker ? "[Thread pool worker]" :
thread.IsFinalizer ? "[Finalizer]" :
thread.IsGC ? "[GC]" :
"";

string isBackground = thread.IsBackground ? "[Background]" : "";
string apartmentModel = thread.IsMTA ? "[MTA]" :
thread.IsSTA ? "[STA]" :
"";

description.AppendLine($"\t\tThread #{thread.ManagedThreadId} (OS 0x{thread.OSThreadId:X}) {threadKind} {isBackground} {apartmentModel}");
foreach (ClrStackFrame frame in thread.StackTrace)
{
description.AppendLine($"\t\t\t{frame}");
}
}
}
}
}
finally
{
Interlocked.Exchange(ref s_clrMdLock, 0);
}
}
CollectTimeoutDiagnostics(description);
}
catch { }

throw new RemoteExecutionException(description.ToString());
}
}

FileInfo exceptionFileInfo = new FileInfo(Options.ExceptionFile);
FileInfo exceptionFileInfo = new(Options.ExceptionFile);
if (exceptionFileInfo.Exists && exceptionFileInfo.Length != 0)
{
throw new RemoteExecutionException("Remote process failed with an unhandled exception.", File.ReadAllText(Options.ExceptionFile));
Expand Down Expand Up @@ -271,6 +203,87 @@ private void Dispose(bool disposing)
}
}

/// <summary>
/// Collects diagnostic information (dump + thread stacks) from the timed-out remote process.
/// </summary>
private void CollectTimeoutDiagnostics(StringBuilder description)
{
string uploadPath = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
if (!string.IsNullOrWhiteSpace(uploadPath))
{
try
{
string dumpPath = Path.Combine(uploadPath, $"{Process.Id}.{Path.GetRandomFileName()}.dmp");
#if NETCOREAPP
// These define guards assume that harness running on .NET Framework implies test process runs on .NET Framework.
var client = new DiagnosticsClient(Process.Id);
client.WriteDump(DumpType.WithHeap, dumpPath, logDumpGeneration: false);
Comment on lines +214 to +220
#else
MiniDump.Create(Process, dumpPath);
#endif
description.AppendLine($"Wrote dump to: {dumpPath}");
}
catch (Exception exc)
{
description.AppendLine($"Failed to create dump: {exc.Message}");
}
}

// Gather additional details about the process if possible
try
{
description.AppendLine($"\tProcess ID: {Process.Id}");
description.AppendLine($"\tHandle: {Process.Handle}");
description.AppendLine($"\tName: {Process.ProcessName}");
description.AppendLine($"\tMainModule: {Process.MainModule?.FileName}");
description.AppendLine($"\tStartTime: {Process.StartTime}");
description.AppendLine($"\tTotalProcessorTime: {Process.TotalProcessorTime}");

// Attach ClrMD to gather some additional details.
if (Interlocked.CompareExchange(ref s_clrMdLock, 1, 0) == 0) // Make sure we only attach to one process at a time.
{
try
{
using DataTarget dt = DataTarget.CreateSnapshotAndAttach(Process.Id);
ClrRuntime runtime = dt.ClrVersions.FirstOrDefault()?.CreateRuntime();
if (runtime is not null)
{
// Dump the threads in the remote process.
description.AppendLine("\tThreads:");
foreach (ClrThread thread in runtime.Threads.Where(t => t.IsAlive))
{
ClrThreadState state = thread.State;
// ClrMD v4 no longer exposes gate/timer/wait thread pool flags, so only the
// completion port and worker kinds remain detectable via ClrThreadState.
string threadKind =
(state & ClrThreadState.TS_CompletionPortThread) != 0 ? "[Thread pool completion port]" :
(state & ClrThreadState.TS_TPWorkerThread) != 0 ? "[Thread pool worker]" :
thread.IsFinalizer ? "[Finalizer]" :
thread.IsGc ? "[GC]" :
"";

string isBackground = (state & ClrThreadState.TS_Background) != 0 ? "[Background]" : "";
string apartmentModel = (state & ClrThreadState.TS_InMTA) != 0 ? "[MTA]" :
(state & ClrThreadState.TS_InSTA) != 0 ? "[STA]" :
"";

description.AppendLine($"\t\tThread #{thread.ManagedThreadId} (OS 0x{thread.OSThreadId:X}) {threadKind} {isBackground} {apartmentModel}");
foreach (ClrStackFrame frame in thread.EnumerateStackTrace(includeContext: false))
{
description.AppendLine($"\t\t\t{frame}");
}
}
}
}
finally
{
Interlocked.Exchange(ref s_clrMdLock, 0);
}
}
}
catch { }
}

~RemoteInvokeHandle()
{
// Finalizer flags tests that omitted the explicit Dispose() call; they must have it, or they aren't
Expand Down
Loading
Loading