diff --git a/CHANGELOG.md b/CHANGELOG.md index e15031ec..3a0f6d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Notable changes to Wisp are recorded here. - End the process at once when a shutdown wait expires instead of finishing the shutdown under a worker that is still inside a driver call; the next run reports an incomplete shutdown rather than a crash. - Show the update repair instructions once and stop automatic update attempts for the session when the update helper is missing. Install update remains available from the dashboard. - Report the drift gauge setting in environment snapshots. +- Crash reports name the faulting module, its version and offset, and include the call stack Wisp captured at the fault. ## 2.6.12 - 2026-10-16 diff --git a/docs/releases/Wisp-2.6.13-release-notes.md b/docs/releases/Wisp-2.6.13-release-notes.md index b912e6e8..5f372cd1 100644 --- a/docs/releases/Wisp-2.6.13-release-notes.md +++ b/docs/releases/Wisp-2.6.13-release-notes.md @@ -11,3 +11,4 @@ ## Diagnostics - Wisp reports whether the drift gauge is switched on, so Wisp Tools shows feature use accurately. +- Crash reports name the faulting module, its version and offset, and include the call stack Wisp captured at the fault. diff --git a/src/Wisp.App/App.xaml.cs b/src/Wisp.App/App.xaml.cs index e8fd81f6..969e6c3b 100644 --- a/src/Wisp.App/App.xaml.cs +++ b/src/Wisp.App/App.xaml.cs @@ -79,6 +79,9 @@ protected override async void OnStartup(StartupEventArgs e) } _runExitMarker = RunExitMarkerStore.BeginCurrent(_runId); + // The native handler records the faulting address and call stack of a hardware fault + // for this run; the next run reports it with the crash. + NativeCrashCapture.TryInstall(_crashReports.StoreDirectory, _runId); _activationEvent = new EventWaitHandle( initialState: false, EventResetMode.AutoReset, diff --git a/src/Wisp.App/CrashDiagnostics/CrashReport.cs b/src/Wisp.App/CrashDiagnostics/CrashReport.cs index c5a005ab..39108f12 100644 --- a/src/Wisp.App/CrashDiagnostics/CrashReport.cs +++ b/src/Wisp.App/CrashDiagnostics/CrashReport.cs @@ -116,6 +116,11 @@ internal string Format() report.AppendLine("The faulting module alone does not establish the root cause."); } else report.AppendLine("The previous run did not finish normal shutdown. The cause is not established; termination or power loss can also leave this marker."); + if (exit.Native is { } native) + { + report.AppendLine($"Faulting address: {native.Module}+0x{native.Offset.ToString("X", CultureInfo.InvariantCulture)} (code 0x{native.ExceptionCode.ToString("X8", CultureInfo.InvariantCulture)}, thread {native.ThreadId})"); + foreach (var frame in native.Frames) report.AppendLine($"Frame: {frame}"); + } } for (var i = 0; i < safe.Exceptions.Length; i++) { diff --git a/src/Wisp.App/CrashDiagnostics/CrashReportStore.cs b/src/Wisp.App/CrashDiagnostics/CrashReportStore.cs index 1e5e7934..0859d518 100644 --- a/src/Wisp.App/CrashDiagnostics/CrashReportStore.cs +++ b/src/Wisp.App/CrashDiagnostics/CrashReportStore.cs @@ -23,6 +23,7 @@ internal sealed class CrashReportStore(string directory, Action? private long _recoveryGeneration; internal long RecoveryGeneration { get { lock (_gate) return _recoveryGeneration; } } private readonly string _path = Path.Combine(directory, "reports.json"); + internal string StoreDirectory => directory; private sealed record Entry(CrashReport Report, bool Acknowledged); private sealed record Document(int Version, Entry[] Reports); diff --git a/src/Wisp.App/CrashDiagnostics/NativeCrashCapture.cs b/src/Wisp.App/CrashDiagnostics/NativeCrashCapture.cs new file mode 100644 index 00000000..a6ab38fb --- /dev/null +++ b/src/Wisp.App/CrashDiagnostics/NativeCrashCapture.cs @@ -0,0 +1,145 @@ +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; + +namespace Wisp.App.CrashDiagnostics; + +// What the native renderer's vectored exception handler wrote when a hardware fault hit the +// process: the faulting address and a call stack, each as module+offset. Windows' own event +// only keeps the faulting module and offset; this is what locates the fault. +internal sealed record NativeFaultRecord(uint ExceptionCode, string Module, ulong Offset, string? Access, + ulong? AccessAddress, uint ThreadId, string[] Frames) +{ + internal const int MaximumFrames = 48; + internal const int MaximumRecordBytes = 64 * 1024; + + internal NativeFaultRecord? Sanitize() + { + var module = WindowsFaultEventReader.SafeModule(Module); + if (module is null || Frames is null) return null; + if (Access is not (null or "read" or "write" or "execute")) return null; + var frames = Frames.Where(ValidFrame).Take(MaximumFrames).ToArray(); + return this with { Module = module, Frames = frames }; + } + + internal static bool ValidFrame(string? frame) + { + if (frame is not { Length: <= 96 }) return false; + var plus = frame.IndexOf("+0x", StringComparison.Ordinal); + if (plus <= 0) return false; + var module = frame[..plus]; + var offset = frame[(plus + 3)..]; + return WindowsFaultEventReader.SafeModule(module) == module && offset.Length is >= 1 and <= 16 && + offset.All(character => character is >= '0' and <= '9' or >= 'a' and <= 'f'); + } + + // Parses the last complete record in the file. A record is a block of key=value lines + // between "wisp-native-fault 1" and "end". + internal static NativeFaultRecord? Parse(string text) + { + if (string.IsNullOrEmpty(text)) return null; + var blocks = text.Split("wisp-native-fault 1\n", StringSplitOptions.RemoveEmptyEntries); + for (var index = blocks.Length - 1; index >= 0; index--) + { + var block = blocks[index]; + var end = block.IndexOf("\nend\n", StringComparison.Ordinal); + if (end < 0) continue; + uint code = 0, thread = 0; + string? module = null, access = null; + ulong offset = 0, accessAddress = 0; + var hasAccessAddress = false; + var frames = new List(); + foreach (var line in block[..end].Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var separator = line.IndexOf('='); + if (separator <= 0) continue; + var key = line[..separator]; + var value = line[(separator + 1)..].Trim(); + switch (key) + { + case "code": if (!Hex(value, out var parsedCode) || parsedCode > uint.MaxValue) return null; code = (uint)parsedCode; break; + case "thread": if (!uint.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out thread)) return null; break; + case "address": + if (!SplitAddress(value, out module, out offset)) return null; + break; + case "access": + var parts = value.Split(' ', 2, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2 || !Hex(parts[1], out accessAddress)) return null; + access = parts[0]; + hasAccessAddress = true; + break; + case "frame": + if (frames.Count < MaximumFrames && ValidFrame(value)) frames.Add(value); + break; + } + } + if (module is null) return null; + return new NativeFaultRecord(code, module, offset, access, hasAccessAddress ? accessAddress : null, thread, frames.ToArray()).Sanitize(); + } + return null; + } + + private static bool SplitAddress(string value, out string? module, out ulong offset) + { + module = null; + offset = 0; + var plus = value.IndexOf("+0x", StringComparison.Ordinal); + if (plus <= 0 || !Hex(value[(plus + 3)..], out offset)) return false; + module = WindowsFaultEventReader.SafeModule(value[..plus]); + return module is not null; + } + + private static bool Hex(string value, out ulong number) + { + number = 0; + if (value.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) value = value[2..]; + return value is { Length: > 0 and <= 16 } && value.All(char.IsAsciiHexDigit) && + ulong.TryParse(value, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out number); + } +} + +internal static class NativeCrashCapture +{ + private static class Native + { + [DllImport("Wisp.NativeRenderer.dll", EntryPoint = "WispCrashCaptureInstall", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)] + internal static extern int Install(string path); + } + + internal static string PathFor(string directory, Guid runId) => + Path.Combine(directory, "native-" + runId.ToString("N", CultureInfo.InvariantCulture) + ".txt"); + + // Installs the handler for this run. Failure only means a future fault stays unlocated. + internal static bool TryInstall(string directory, Guid runId) + { + try + { + Directory.CreateDirectory(directory); + return Native.Install(PathFor(directory, runId)) >= 0; + } + catch (Exception error) when (error is DllNotFoundException or EntryPointNotFoundException or BadImageFormatException + or IOException or UnauthorizedAccessException or ArgumentException) + { + return false; + } + } + + // Reads and removes the previous run's record. Absent or unreadable records are null. + internal static NativeFaultRecord? TryRead(string directory, Guid runId) + { + var path = PathFor(directory, runId); + try + { + if (!File.Exists(path)) return null; + NativeFaultRecord? record = null; + var info = new FileInfo(path); + if (info.Length is > 0 and <= NativeFaultRecord.MaximumRecordBytes) record = NativeFaultRecord.Parse(File.ReadAllText(path)); + try { File.Delete(path); } catch (Exception) { } + return record; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + return null; + } + } +} diff --git a/src/Wisp.App/CrashDiagnostics/PreviousRunRecovery.cs b/src/Wisp.App/CrashDiagnostics/PreviousRunRecovery.cs index bb21e85d..0180acdd 100644 --- a/src/Wisp.App/CrashDiagnostics/PreviousRunRecovery.cs +++ b/src/Wisp.App/CrashDiagnostics/PreviousRunRecovery.cs @@ -26,6 +26,7 @@ internal sealed class PreviousRunRecovery(CrashReportStore reports, IWindowsFaul cancellationToken.ThrowIfCancellationRequested(); var exit = new RecoveredExitInfo(safe.StartedAtUtc, nextRunStartedAtUtc, lookup.Status, lookup.Fault).Sanitize(); if (exit is null) exit = new(safe.StartedAtUtc, nextRunStartedAtUtc, WindowsFaultLookupStatus.ReadFailed, null); + exit = exit with { Native = NativeCrashCapture.TryRead(reports.StoreDirectory, safe.RunId) }; // A recorded close request is not a crash. A correlated Windows fault still wins. if (safe.ShutdownIntent != ShutdownIntent.None && exit.Fault is null) return null; var report = safe.ToReport(exit.Fault?.TimeUtc ?? nextRunStartedAtUtc) with diff --git a/src/Wisp.App/CrashDiagnostics/WindowsFaultEventReader.cs b/src/Wisp.App/CrashDiagnostics/WindowsFaultEventReader.cs index 2a9595c0..e11a7a9f 100644 --- a/src/Wisp.App/CrashDiagnostics/WindowsFaultEventReader.cs +++ b/src/Wisp.App/CrashDiagnostics/WindowsFaultEventReader.cs @@ -9,11 +9,18 @@ namespace Wisp.App.CrashDiagnostics; internal enum WindowsFaultLookupStatus { NoMatchingEvent, Matched, AccessDenied, TimedOut, ReadFailed, ScanLimitReached } internal sealed record WindowsFaultEvidence(DateTimeOffset TimeUtc, uint ExceptionCode, ulong? FaultOffset, - string? Module, bool CorrelatedWerReport); + string? Module, bool CorrelatedWerReport) +{ + // The faulting module's file version, as Windows recorded it. + public string? ModuleVersion { get; init; } +} internal sealed record WindowsFaultLookup(WindowsFaultLookupStatus Status, WindowsFaultEvidence? Fault = null); internal sealed record RecoveredExitInfo(DateTimeOffset RunStartedAtUtc, DateTimeOffset DetectedAtUtc, WindowsFaultLookupStatus LookupStatus, WindowsFaultEvidence? Fault) { + // The in-process capture of the fault, when the native handler wrote one for that run. + public NativeFaultRecord? Native { get; init; } + internal RecoveredExitInfo? Sanitize() { if (RunStartedAtUtc.Offset != TimeSpan.Zero || RunStartedAtUtc.Ticks <= 0 || @@ -22,9 +29,13 @@ internal sealed record RecoveredExitInfo(DateTimeOffset RunStartedAtUtc, DateTim { if (Fault is not { } fault || fault.TimeUtc.Offset != TimeSpan.Zero || fault.TimeUtc < RunStartedAtUtc || fault.TimeUtc > DetectedAtUtc) return null; - return this with { Fault = fault with { Module = WindowsFaultEventReader.SafeModule(fault.Module) } }; + return this with + { + Fault = fault with { Module = WindowsFaultEventReader.SafeModule(fault.Module), ModuleVersion = WindowsFaultEventReader.SafeVersion(fault.ModuleVersion) }, + Native = Native?.Sanitize() + }; } - return this with { Fault = null }; + return this with { Fault = null, Native = Native?.Sanitize() }; } } internal interface IWindowsFaultEventSource @@ -36,12 +47,6 @@ internal sealed class WindowsFaultEventReader : IWindowsFaultEventSource { internal const int MaximumEvents = 64, MaximumXmlCharacters = 65536; private static readonly XNamespace Namespace = "http://schemas.microsoft.com/win/2004/08/events/event"; - private static readonly HashSet Modules = new(StringComparer.OrdinalIgnoreCase) - { - "wisp.exe", "wisp.dll", "wisp.nativerenderer.dll", "libmpv-2.dll", "libvlc.dll", "libvlccore.dll", - "ntdll.dll", "kernel32.dll", "kernelbase.dll", "ucrtbase.dll", "vcruntime140.dll", "vcruntime140_1.dll", - "msvcp140.dll", "coreclr.dll", "clrjit.dll", "d3d11.dll", "dxgi.dll", "dcomp.dll" - }; public WindowsFaultLookup Read(RunExitMarker previous, DateTimeOffset nextRunStartedAtUtc, CancellationToken cancellationToken) { @@ -125,7 +130,8 @@ internal static WindowsFaultLookup Correlate(IEnumerable events, RunExit !Hex(Value("ExceptionCode"), out var code) || code > uint.MaxValue) continue; if (fault is not null && fault.TimeUtc >= time) continue; fault = new(time, (uint)code, Hex(Value("FaultingOffset"), out var offset) ? offset : null, - SafeModule(Value("ModuleName")), false); + SafeModule(Value("ModuleName")), false) + { ModuleVersion = SafeVersion(Value("ModuleVersion")) }; faultReport = Guid.TryParse(Value("IntegratorReportId"), out var reportId) && reportId != Guid.Empty ? reportId : null; } catch (Exception error) when (error is XmlException or ArgumentException or InvalidOperationException) { } @@ -134,7 +140,22 @@ internal static WindowsFaultLookup Correlate(IEnumerable events, RunExit fault with { CorrelatedWerReport = faultReport is { } report && werReports.Contains(report) }); } - internal static string? SafeModule(string? module) => module is not null && Modules.Contains(module) ? module.ToLowerInvariant() : null; + // Any module can fault, including hooks other software injects, so every bare file name is + // kept: lower case, at most 64 characters of letters, digits, '.', '_' and '-', with a + // library, executable or driver extension. Paths and anything else are dropped. + internal static string? SafeModule(string? module) + { + if (module is null) return null; + var name = module.Trim().ToLowerInvariant(); + if (name == "unknown") return name; + if (name.Length is 0 or > 64 || !char.IsAsciiLetterOrDigit(name[0]) || + !name.All(character => char.IsAsciiLetterOrDigit(character) || character is '.' or '_' or '-')) return null; + return name.EndsWith(".dll", StringComparison.Ordinal) || name.EndsWith(".exe", StringComparison.Ordinal) || + name.EndsWith(".sys", StringComparison.Ordinal) || name.EndsWith(".drv", StringComparison.Ordinal) ? name : null; + } + internal static string? SafeVersion(string? version) => version is { Length: > 0 and <= 23 } && VersionPattern.IsMatch(version) ? version : null; + private static readonly System.Text.RegularExpressions.Regex VersionPattern = + new(@"^[0-9]{1,5}(\.[0-9]{1,5}){3}$", System.Text.RegularExpressions.RegexOptions.CultureInvariant); private static bool Unsigned(string? value, out ulong number) => value?.StartsWith("0x", StringComparison.OrdinalIgnoreCase) == true ? Hex(value, out number) : ulong.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out number); private static bool Hex(string? value, out ulong number) diff --git a/src/Wisp.App/DebugLogging/SupplementaryIncidentCapture.cs b/src/Wisp.App/DebugLogging/SupplementaryIncidentCapture.cs index 24d7f724..ad80313b 100644 --- a/src/Wisp.App/DebugLogging/SupplementaryIncidentCapture.cs +++ b/src/Wisp.App/DebugLogging/SupplementaryIncidentCapture.cs @@ -38,7 +38,10 @@ internal static SupplementaryIncident Crash(CrashReport report) failureStage: "crash-" + origin, code: safe?.RecoveredExit?.Fault?.ExceptionCode ?? (safe?.Exceptions.FirstOrDefault() is { } first ? unchecked((uint)first.HResult) : null), exceptions: Exceptions(safe?.Exceptions), context: Context(safe?.Context, report.TimeUtc, - recovered: safe?.RecoveredExit is not null), crash: new(origin, lookup, safe?.RecoveredExit?.Fault?.Module)); + recovered: safe?.RecoveredExit is not null), crash: new(origin, lookup, safe?.RecoveredExit?.Fault?.Module, + safe?.RecoveredExit?.Fault?.FaultOffset, safe?.RecoveredExit?.Fault?.ModuleVersion, + safe?.RecoveredExit?.Native?.Module, safe?.RecoveredExit?.Native?.Offset, + safe?.RecoveredExit?.Native is { } native ? native.Frames.ToImmutableArray() : null)); } internal static SupplementaryIncident? Export(Exception error, DateTimeOffset now) diff --git a/src/Wisp.App/ReleaseNotesCatalog.cs b/src/Wisp.App/ReleaseNotesCatalog.cs index d078321f..9e10d3ae 100644 --- a/src/Wisp.App/ReleaseNotesCatalog.cs +++ b/src/Wisp.App/ReleaseNotesCatalog.cs @@ -26,7 +26,8 @@ public static class ReleaseNotesCatalog "Closing Wisp while a renderer thread is stuck in a driver call ends the app cleanly instead of crashing.", "When the update helper is missing, Wisp shows the repair instructions once and stops automatic update attempts for the session; Install update stays available on the dashboard."), Group("Diagnostics", - "Wisp reports whether the drift gauge is switched on, so Wisp Tools shows feature use accurately.") + "Wisp reports whether the drift gauge is switched on, so Wisp Tools shows feature use accurately.", + "Crash reports name the faulting module, its version and offset, and include the call stack Wisp captured at the fault.") ]), new( "2.6.12", diff --git a/src/Wisp.App/Supplementary/SupplementaryIncident.cs b/src/Wisp.App/Supplementary/SupplementaryIncident.cs index 5696fc0c..f7e14c2d 100644 --- a/src/Wisp.App/Supplementary/SupplementaryIncident.cs +++ b/src/Wisp.App/Supplementary/SupplementaryIncident.cs @@ -13,7 +13,9 @@ internal sealed record SupplementaryIncidentContext(int AgeMs, string Connection string? GameBuild = null, string? GameVerification = null, string? RecorderState = null, string? RenderMode = null, bool? GameActive = null, bool? NativeExpected = null, bool? OverlayExpectedVisible = null); internal sealed record SupplementaryUpdateEvidence(string Recovery, uint? InstallerExitCode = null); -internal sealed record SupplementaryCrashEvidence(string Origin, string? Lookup = null, string? Module = null); +internal sealed record SupplementaryCrashEvidence(string Origin, string? Lookup = null, string? Module = null, + ulong? Offset = null, string? ModuleVersion = null, string? FaultModule = null, ulong? FaultOffset = null, + ImmutableArray? Frames = null); internal sealed record SupplementaryRecorderComponent(string Component, string Reason, uint Code); internal sealed record SupplementaryRecorderEvidence(long? VideoPackets = null, long? AudioPackets = null, long? SubmittedFrames = null, double? SchedulerLagMs = null, double? SourceAgeMs = null, double? LocalFrameAgeMs = null, @@ -109,7 +111,7 @@ internal static SupplementaryIncident Create(string category, string reason = "u internal static bool Valid(SupplementaryIncident value) => value.SchemaVersion is 1 or 2 or 3 && (value.SchemaVersion >= 2 || value.Recorder?.Components is null) && Categories.Contains(value.Category) && (value.Crash is null || value.SchemaVersion == 3 && ValidCrashOrigin(value.Category, value.Crash.Origin) && value.Crash.Lookup is null or "no-match" or "matched" or "access-denied" or "timeout" or "read-failed" or "scan-limit" && - (value.Crash.Module is null || CrashDiagnostics.WindowsFaultEventReader.SafeModule(value.Crash.Module) == value.Crash.Module)) && + ValidCrashDetails(value.Crash)) && (value.Update is null || value.SchemaVersion == 3 && value.Category == "update" && UpdateRecoveries.Contains(value.Update.Recovery)) && (Reasons.Contains(value.Reason) || value.Category == "update" && UpdateFailureReasons.Contains(value.Reason)) && (value.FailureStage is null || FailureStages.Contains(value.FailureStage)) && (value.Exceptions is null || value.Exceptions is { IsDefault: false, Length: > 0 and <= 3 } exceptions && exceptions.All(e => @@ -134,6 +136,12 @@ c.Foreground is null or "unknown" or "game" or "wisp" or "other" or "none" && c.GameVerification is null or "verified" or "unverified" or "unavailable" or "not-running" && c.RecorderState is null or "disabled" or "waiting" or "preparing" or "buffering" or "saving" or "stopping" or "reconnecting" or "unavailable" or "error" or "paused" && c.RenderMode is null or "cpu" or "gpu"; + private static bool ValidCrashDetails(SupplementaryCrashEvidence crash) => + (crash.Module is null || CrashDiagnostics.WindowsFaultEventReader.SafeModule(crash.Module) == crash.Module) && + (crash.ModuleVersion is null || CrashDiagnostics.WindowsFaultEventReader.SafeVersion(crash.ModuleVersion) == crash.ModuleVersion) && + (crash.FaultModule is null || CrashDiagnostics.WindowsFaultEventReader.SafeModule(crash.FaultModule) == crash.FaultModule) && + (crash.Frames is null || crash.Frames is { IsDefault: false, Length: <= CrashDiagnostics.NativeFaultRecord.MaximumFrames } frames && + frames.All(CrashDiagnostics.NativeFaultRecord.ValidFrame)); private static bool ValidCrashOrigin(string category, string origin) => category switch { "managed-exception" => origin is "ui" or "background" or "task", diff --git a/src/Wisp.NativeRenderer/NativeContractTests.cpp b/src/Wisp.NativeRenderer/NativeContractTests.cpp index 81a7e518..4076d025 100644 --- a/src/Wisp.NativeRenderer/NativeContractTests.cpp +++ b/src/Wisp.NativeRenderer/NativeContractTests.cpp @@ -4,6 +4,7 @@ #include "TearingPolicy.h" #include #include +#include #include #include #include @@ -21,6 +22,31 @@ static void Require(bool passed, const char* name) { if (!passed) { ++failures; std::printf("FAIL: %s\n", name); } } +static void CheckCrashCapture() +{ + char buffer[8192]; + const auto address = reinterpret_cast(reinterpret_cast(&GetTickCount64)); + const uint32_t length = WispCrashCaptureFormatForTest(0xC0000005u, address, 0, 0x10, buffer, sizeof(buffer)); + Require(length > 0 && length < sizeof(buffer), "crash record is written within its buffer"); + const std::string record(buffer, length); + Require(record.rfind("wisp-native-fault 1\n", 0) == 0, "crash record header"); + Require(record.find("\ncode=0xc0000005\n") != std::string::npos, "crash record exception code"); + Require(record.find("\naddress=kernel32.dll+0x") != std::string::npos, "crash record faulting address names the module"); + Require(record.find("\naccess=read 0x10\n") != std::string::npos, "crash record access violation details"); + Require(record.find("\nframe=") != std::string::npos, "crash record carries a call stack"); + Require(record.size() >= 5 && record.compare(record.size() - 5, 5, "\nend\n") == 0, "crash record terminator"); + Require(WispCrashCaptureFormatForTest(0xC0000005u, address, 0, 0x10, buffer, 16) == 0, "crash record refuses a tiny buffer"); + wchar_t path[MAX_PATH]; + const DWORD temp = GetTempPathW(MAX_PATH, path); + if (temp > 0 && temp < MAX_PATH - 40) + { + std::wstring file(path); + file += L"wisp-crash-capture-contract.txt"; + Require(WispCrashCaptureInstall(file.c_str()) == S_OK, "crash capture installs once"); + Require(WispCrashCaptureInstall(file.c_str()) == S_FALSE, "crash capture reports a second install"); + } +} + static void CheckTearingPolicy() { constexpr UINT baseline = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; @@ -1342,6 +1368,7 @@ int main(int argc, char** argv) } CheckFramePacer(); CheckTearingPolicy(); + CheckCrashCapture(); bool cpuRendering = false, hiddenOnly = false; for (int index = 1; index < argc; ++index) { diff --git a/src/Wisp.NativeRenderer/Wisp.NativeRenderer.cpp b/src/Wisp.NativeRenderer/Wisp.NativeRenderer.cpp index 9c97b4f4..72a69efa 100644 --- a/src/Wisp.NativeRenderer/Wisp.NativeRenderer.cpp +++ b/src/Wisp.NativeRenderer/Wisp.NativeRenderer.cpp @@ -1498,3 +1498,164 @@ HRESULT __cdecl WispRendererCapture(void* renderer, uint8_t* pixels, uint32_t by { return Invoke(renderer, [&](Renderer& r) { return r.Capture(pixels,bytes,stride); }); } HRESULT __cdecl WispRendererDeviceRemovedReason(void* renderer) noexcept { return Invoke(renderer, [&](Renderer& r) { CHECK_HR(r.CheckThread()); return r.device->GetDeviceRemovedReason(); }); } + +// ---- Native crash capture --------------------------------------------------------------- +// A vectored exception handler that records where a hardware fault happened, as +// module+offset for the faulting address and a module+offset call stack, into a file the +// app opens at startup. Windows' own event log only keeps the faulting module and offset, +// and nothing of the stack. The handler never allocates, never throws and never handles the +// exception: it writes one bounded record and lets the fault continue to whoever owns it. +namespace +{ + HANDLE crashCaptureFile = INVALID_HANDLE_VALUE; + void* crashCaptureHandler = nullptr; + LONG crashCaptureRecords = 0; + constexpr LONG kCrashCaptureMaximumRecords = 3; + constexpr ULONG kCrashCaptureMaximumFrames = 48; + + struct CrashText { char* cursor; char* end; }; + void CrashPut(CrashText& text, const char* value) noexcept + { + while (*value && text.cursor < text.end) *text.cursor++ = *value++; + } + void CrashPutHex(CrashText& text, unsigned long long value) noexcept + { + char digits[16]; + int count = 0; + do { digits[count++] = "0123456789abcdef"[value & 0xf]; value >>= 4; } while (value && count < 16); + while (count && text.cursor < text.end) *text.cursor++ = digits[--count]; + } + void CrashPutDecimal(CrashText& text, unsigned long long value) noexcept + { + char digits[20]; + int count = 0; + do { digits[count++] = static_cast('0' + value % 10); value /= 10; } while (value && count < 20); + while (count && text.cursor < text.end) *text.cursor++ = digits[--count]; + } + // "module.dll+0x1a2b": the module's file name in lower case, other characters replaced. + void CrashPutAddress(CrashText& text, const void* address) noexcept + { + void* base = nullptr; + if (RtlPcToFileHeader(const_cast(address), &base) && base) + { + wchar_t path[MAX_PATH]; + const DWORD length = GetModuleFileNameW(static_cast(base), path, MAX_PATH); + if (length > 0 && length < MAX_PATH) + { + const wchar_t* name = path; + for (const wchar_t* p = path; *p; ++p) if (*p == L'\\' || *p == L'/') name = p + 1; + for (const wchar_t* p = name; *p && text.cursor < text.end; ++p) + { + wchar_t c = *p; + if (c >= L'A' && c <= L'Z') c = static_cast(c - L'A' + L'a'); + const bool plain = (c >= L'a' && c <= L'z') || (c >= L'0' && c <= L'9') || c == L'.' || c == L'_' || c == L'-'; + *text.cursor++ = plain ? static_cast(c) : '_'; + } + } + else CrashPut(text, "unknown"); + CrashPut(text, "+0x"); + CrashPutHex(text, reinterpret_cast(address) - reinterpret_cast(base)); + } + else + { + CrashPut(text, "unknown+0x"); + CrashPutHex(text, reinterpret_cast(address)); + } + } + bool CrashCapturable(const EXCEPTION_RECORD& record) noexcept + { + switch (record.ExceptionCode) + { + case EXCEPTION_ACCESS_VIOLATION: + // Faults on the null page are the runtime's own null checks, which become managed exceptions. + return record.NumberParameters >= 2 && record.ExceptionInformation[1] >= 0x10000; + case EXCEPTION_IN_PAGE_ERROR: + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_PRIV_INSTRUCTION: + case EXCEPTION_DATATYPE_MISALIGNMENT: + case 0xC0000409: // STATUS_STACK_BUFFER_OVERRUN + case 0xC0000374: // STATUS_HEAP_CORRUPTION + return true; + default: + return false; + } + } + size_t CrashFormat(const EXCEPTION_RECORD& record, bool includeFrames, char* buffer, size_t bytes) noexcept + { + if (!buffer || bytes < 64) return 0; + CrashText text{ buffer, buffer + bytes - 1 }; + FILETIME now; + GetSystemTimeAsFileTime(&now); + CrashPut(text, "wisp-native-fault 1\ntime=0x"); + CrashPutHex(text, (static_cast(now.dwHighDateTime) << 32) | now.dwLowDateTime); + CrashPut(text, "\ncode=0x"); + CrashPutHex(text, record.ExceptionCode); + CrashPut(text, "\nthread="); + CrashPutDecimal(text, GetCurrentThreadId()); + CrashPut(text, "\naddress="); + CrashPutAddress(text, record.ExceptionAddress); + if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION && record.NumberParameters >= 2) + { + CrashPut(text, "\naccess="); + CrashPut(text, record.ExceptionInformation[0] == 0 ? "read" : record.ExceptionInformation[0] == 1 ? "write" : "execute"); + CrashPut(text, " 0x"); + CrashPutHex(text, record.ExceptionInformation[1]); + } + if (includeFrames) + { + void* frames[kCrashCaptureMaximumFrames]; + const USHORT count = RtlCaptureStackBackTrace(0, kCrashCaptureMaximumFrames, frames, nullptr); + for (USHORT index = 0; index < count; ++index) + { + CrashPut(text, "\nframe="); + CrashPutAddress(text, frames[index]); + } + } + CrashPut(text, "\nend\n"); + *text.cursor = '\0'; + return static_cast(text.cursor - buffer); + } + LONG CALLBACK CrashCaptureHandler(EXCEPTION_POINTERS* pointers) noexcept + { + if (!pointers || !pointers->ExceptionRecord || crashCaptureFile == INVALID_HANDLE_VALUE) return EXCEPTION_CONTINUE_SEARCH; + const EXCEPTION_RECORD& record = *pointers->ExceptionRecord; + if (!CrashCapturable(record)) return EXCEPTION_CONTINUE_SEARCH; + if (InterlockedIncrement(&crashCaptureRecords) > kCrashCaptureMaximumRecords) return EXCEPTION_CONTINUE_SEARCH; + char buffer[6144]; + const size_t length = CrashFormat(record, true, buffer, sizeof(buffer)); + DWORD written = 0; + if (length) WriteFile(crashCaptureFile, buffer, static_cast(length), &written, nullptr); + FlushFileBuffers(crashCaptureFile); + return EXCEPTION_CONTINUE_SEARCH; + } +} +HRESULT __cdecl WispCrashCaptureInstall(const wchar_t* path) noexcept +{ + if (!path) return E_POINTER; + if (crashCaptureHandler) return S_FALSE; + const HANDLE file = CreateFileW(path, GENERIC_WRITE, FILE_SHARE_READ, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH, nullptr); + if (file == INVALID_HANDLE_VALUE) return HRESULT_FROM_WIN32(GetLastError()); + crashCaptureFile = file; + crashCaptureHandler = AddVectoredExceptionHandler(1, CrashCaptureHandler); + if (!crashCaptureHandler) + { + crashCaptureFile = INVALID_HANDLE_VALUE; + CloseHandle(file); + return E_FAIL; + } + return S_OK; +} +// Formats a record for a synthetic fault so the contract tests can check the format without +// raising an exception. Returns the number of bytes written, zero when the buffer is too small. +uint32_t __cdecl WispCrashCaptureFormatForTest(uint32_t code, uint64_t address, uint32_t accessKind, uint64_t accessAddress, + char* buffer, uint32_t bytes) noexcept +{ + EXCEPTION_RECORD record{}; + record.ExceptionCode = code; + record.ExceptionAddress = reinterpret_cast(static_cast(address)); + record.NumberParameters = 2; + record.ExceptionInformation[0] = accessKind; + record.ExceptionInformation[1] = static_cast(accessAddress); + return static_cast(CrashFormat(record, true, buffer, bytes)); +} diff --git a/src/Wisp.NativeRenderer/Wisp.NativeRenderer.h b/src/Wisp.NativeRenderer/Wisp.NativeRenderer.h index 1b251e9e..5eb9a426 100644 --- a/src/Wisp.NativeRenderer/Wisp.NativeRenderer.h +++ b/src/Wisp.NativeRenderer/Wisp.NativeRenderer.h @@ -164,3 +164,8 @@ WISP_API HRESULT __cdecl WispRendererWaitForFrameMeasuredV2(void* renderer, uint HANDLE cancellation, int measure, uint32_t* result, WispWaitMetricsV2* metrics) noexcept; WISP_API HRESULT __cdecl WispRendererCapture(void* renderer, uint8_t* pixels, uint32_t byteCount, uint32_t stride) noexcept; WISP_API HRESULT __cdecl WispRendererDeviceRemovedReason(void* renderer) noexcept; +// Records hardware faults (faulting address and call stack as module+offset) to the file +// at path for the life of the process. S_FALSE when already installed. +WISP_API HRESULT __cdecl WispCrashCaptureInstall(const wchar_t* path) noexcept; +WISP_API uint32_t __cdecl WispCrashCaptureFormatForTest(uint32_t code, uint64_t address, uint32_t accessKind, uint64_t accessAddress, + char* buffer, uint32_t bytes) noexcept; diff --git a/tests/Wisp.App.Tests/NativeCrashCaptureTests.cs b/tests/Wisp.App.Tests/NativeCrashCaptureTests.cs new file mode 100644 index 00000000..fce365e6 --- /dev/null +++ b/tests/Wisp.App.Tests/NativeCrashCaptureTests.cs @@ -0,0 +1,82 @@ +using System.IO; +using Wisp.App.CrashDiagnostics; +using Xunit; + +namespace Wisp.App.Tests; + +public sealed class NativeCrashCaptureTests +{ + private const string Record = + "wisp-native-fault 1\n" + + "time=0x1dc2a3f4b5c6d7e\n" + + "code=0xc0000005\n" + + "thread=4321\n" + + "address=RTSSHooks64.dll+0x1a2b\n" + + "access=read 0x10\n" + + "frame=rtsshooks64.dll+0x1a2b\n" + + "frame=wisp.nativerenderer.dll+0x4f20\n" + + "frame=C:\\PRIVATE\\odd.dll+0x1\n" + + "frame=unknown+0x7ff6a0001000\n" + + "end\n"; + + [Fact] + public void TheLastCompleteRecordIsParsedAndSanitized() + { + var partial = "wisp-native-fault 1\ncode=0xc0000005\naddress=ntdll.dll+0x99\n"; + var record = NativeFaultRecord.Parse(Record + partial); + Assert.NotNull(record); + Assert.Equal(0xc0000005u, record.ExceptionCode); + Assert.Equal("rtsshooks64.dll", record.Module); + Assert.Equal(0x1a2bul, record.Offset); + Assert.Equal("read", record.Access); + Assert.Equal(0x10ul, record.AccessAddress); + Assert.Equal(4321u, record.ThreadId); + Assert.Equal(["rtsshooks64.dll+0x1a2b", "wisp.nativerenderer.dll+0x4f20", "unknown+0x7ff6a0001000"], record.Frames); + } + + [Theory] + [InlineData("")] + [InlineData("wisp-native-fault 1\ncode=0xc0000005\nend\n")] + [InlineData("wisp-native-fault 1\ncode=0xc0000005\naddress=C:\\PRIVATE\\x.dll+0x1\nend\n")] + [InlineData("wisp-native-fault 1\ncode=zz\naddress=ntdll.dll+0x1\nend\n")] + public void IncompleteOrUnsafeRecordsAreIgnored(string text) + { + Assert.Null(NativeFaultRecord.Parse(text)); + } + + [Fact] + public void FramesAreBoundedAndEachNamesAModuleAndOffset() + { + var frames = string.Concat(Enumerable.Range(0, 60).Select(index => $"frame=a.dll+0x{index:x}\n")); + var record = NativeFaultRecord.Parse("wisp-native-fault 1\ncode=0xc0000005\naddress=a.dll+0x1\n" + frames + "end\n"); + Assert.NotNull(record); + Assert.Equal(NativeFaultRecord.MaximumFrames, record.Frames.Length); + Assert.True(NativeFaultRecord.ValidFrame("wisp.exe+0xabcdef0123456789")); + Assert.False(NativeFaultRecord.ValidFrame("wisp.exe")); + Assert.False(NativeFaultRecord.ValidFrame("wisp.exe+0x")); + Assert.False(NativeFaultRecord.ValidFrame("C:\\x\\wisp.exe+0x1")); + Assert.False(NativeFaultRecord.ValidFrame("wisp.exe+0x" + new string('f', 17))); + } + + [Fact] + public void TheRecordFileIsReadOnceAndRemoved() + { + var directory = Path.Combine(Path.GetTempPath(), "wisp-native-capture-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + var runId = Guid.NewGuid(); + Assert.Null(NativeCrashCapture.TryRead(directory, runId)); + File.WriteAllText(NativeCrashCapture.PathFor(directory, runId), Record); + var record = NativeCrashCapture.TryRead(directory, runId); + Assert.NotNull(record); + Assert.Equal("rtsshooks64.dll", record.Module); + Assert.False(File.Exists(NativeCrashCapture.PathFor(directory, runId))); + Assert.Null(NativeCrashCapture.TryRead(directory, runId)); + } + finally + { + Directory.Delete(directory, true); + } + } +} diff --git a/tests/Wisp.App.Tests/WindowsFaultEventReaderTests.cs b/tests/Wisp.App.Tests/WindowsFaultEventReaderTests.cs index 8e3fa11d..38fe0ae5 100644 --- a/tests/Wisp.App.Tests/WindowsFaultEventReaderTests.cs +++ b/tests/Wisp.App.Tests/WindowsFaultEventReaderTests.cs @@ -42,17 +42,28 @@ public void MissingOrMismatchedIdentityNeverConfirmsACrash(string field, string } [Theory] - [InlineData("PRIVATE_PERSON.dll")] - [InlineData("C:\\PRIVATE_PATH\\libmpv-2.dll")] - [InlineData("libmpv-2.dll/PRIVATE_PATH")] - public void OnlyKnownModuleBasenamesCanBeCopied(string module) + [InlineData("RTSSHooks64.dll", "rtsshooks64.dll")] + [InlineData("nvwgf2umx.dll", "nvwgf2umx.dll")] + [InlineData("unknown", "unknown")] + [InlineData("C:\\PRIVATE_PATH\\libmpv-2.dll", null)] + [InlineData("libmpv-2.dll/PRIVATE_PATH", null)] + [InlineData("not a module", null)] + public void AnyModuleFileNameIsCopiedButPathsAreNot(string module, string? expected) { var marker = Marker(); var result = WindowsFaultEventReader.Correlate([Fault(marker, "ModuleName", module)], marker, Started.AddMinutes(5)); - Assert.NotNull(result.Fault); Assert.Null(result.Fault.Module); + Assert.NotNull(result.Fault); Assert.Equal(expected, result.Fault.Module); Assert.Equal(0xc0000005u, result.Fault.ExceptionCode); } + [Fact] + public void TheFaultingModuleVersionIsKeptWhenWellFormed() + { + var marker = Marker(); + Assert.Equal("7.3.6.28204", WindowsFaultEventReader.Correlate([Fault(marker, "ModuleVersion", "7.3.6.28204")], marker, Started.AddMinutes(5)).Fault?.ModuleVersion); + Assert.Null(WindowsFaultEventReader.Correlate([Fault(marker, "ModuleVersion", "PRIVATE 1.2")], marker, Started.AddMinutes(5)).Fault?.ModuleVersion); + } + [Fact] public void WrongProviderOutOfWindowWerAloneAndOversizedXmlAreNotEvidence() {