From c6975c541b35dd6aa47ff6def0bd2de58d82cd11 Mon Sep 17 00:00:00 2001 From: Armando Fernandez Date: Sun, 6 Sep 2026 23:07:49 -0700 Subject: [PATCH] Keep screenshots honest when the window is not on screen Capture reads the window's rendered surface, so it depends on the app having one. Probing a real WebView2 across window states found two failures behind the previous "returns PNG bytes" check: a minimized window makes the browser never answer at all, and a capture taken before the page paints returns valid bytes showing nothing. Capture is now bounded at six seconds and reports that the window needs restoring, instead of consuming the whole operation deadline and surfacing as a vague timeout. Hidden, transparent, and occluded windows were measured and capture normally, so they are left alone. A nearly uniform capture cannot be told apart from a genuinely blank page, so it is flagged rather than refused and the model is told to say the image looks blank instead of describing detail it cannot see. The threshold sits an order of magnitude below a measured painted page. Capturing from the renderer instead of the surface was the obvious candidate fix and is wrong: measured, it throws when minimized and returns a blank image when the window is hidden, which is worse than what it replaces. The smoke test now drives the window through those states rather than asserting only that some bytes came back, and pins the blank threshold against a real painted capture. It also caught that reading a viewport number threw depending on how the value was boxed. Co-Authored-By: Claude Opus 5 (1M context) --- docs/browser-tools.md | 10 ++++ .../Program.cs | 46 ++++++++++++++++++- .../DesktopPreviewImageAndServerTests.cs | 20 ++++++++ .../Controls/ChatTabView.BrowserTools.cs | 24 +++++++++- .../Services/DesktopPreviewTools.cs | 31 +++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) diff --git a/docs/browser-tools.md b/docs/browser-tools.md index bffd8db..1299291 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -111,6 +111,16 @@ An image is evidence for the turn that captured it and is retracted afterward, s screenshot does not re-upload on every later message. The model's written conclusion is what persists. +Capture reads the window's rendered surface, so it depends on the app actually having one. +A minimized window has none and the browser never answers at all, so capture is bounded at +six seconds and reports that the window needs restoring rather than consuming the whole +operation deadline. Hidden, transparent, and occluded windows still capture normally. + +A capture taken before the page painted returns valid image bytes showing nothing. That +cannot be told apart from a genuinely blank page, so it is flagged as `possiblyBlank` +rather than refused, and the model is told to say the image looks blank instead of +describing detail it cannot see. + ## Development servers `open_local_server_desktop_preview` opens a server already running on this machine, so the diff --git a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs index 85275fb..9d5de2b 100644 --- a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs +++ b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs @@ -24,7 +24,7 @@ private static int Main() await CheckBrowserAsync(browser.CoreWebView2).WaitAsync(TimeSpan.FromSeconds(45)); exitCode = 0; Console.WriteLine("PASS: real WebView2 DOM, pointer, keyboard, repeated clicks, focused observations, " + - "forms, scrolling, navigation, fresh assets on reload, screenshots, origin scoping, diagnostics, and argument escaping."); + "forms, scrolling, navigation, fresh assets on reload, screenshots under changing window visibility, origin scoping, diagnostics, and argument escaping."); } catch (Exception ex) { Console.Error.WriteLine(ex); } finally { browser.Dispose(); form.Close(); } @@ -54,6 +54,13 @@ async Task RunAs(string? origin, string operation, string? selector } Task Run(string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) => RunAs(previewOrigin, operation, selector, value, offset, deltaY, observe); + async Task CaptureFrom(bool fromSurface) + { + var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", + JsonSerializer.Serialize(new { format = "png", fromSurface, captureBeyondViewport = false })); + var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue(); + return string.IsNullOrEmpty(data) ? [] : Convert.FromBase64String(data); + } async Task Capture(JsonObject? clip) { object parameters = clip == null ? new { format = "png" } : new @@ -176,6 +183,40 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0) Assert(clipped.Length < full.Length, $"Clipping captured no less than the full page ({clipped.Length} vs {full.Length})"); Assert(!(await Run("bounds", "#hidden"))["ok"]!.GetValue(), "A hidden element was accepted for capture"); + // Capture depends on the window having a compositor surface. A minimized window has none, + // and the browser never answers at all, so the production path bounds this rather than + // letting one call eat the whole operation deadline. + var host = Application.OpenForms[0]!; + async Task<(string Outcome, int Bytes)> TimedCapture() + { + try + { + var bytes = await CaptureFrom(true).WaitAsync(TimeSpan.FromSeconds(4)); + return ("captured", bytes.Length); + } + catch (TimeoutException) { return ("hung", 0); } + } + host.Opacity = 1; + await Task.Delay(500); + var shown = await TimedCapture(); + Assert(shown.Outcome == "captured" && shown.Bytes > 20000, + $"A visible preview did not capture a painted page ({shown.Outcome}, {shown.Bytes} bytes)"); + + host.WindowState = FormWindowState.Minimized; + await Task.Delay(500); + var minimized = await TimedCapture(); + Assert(minimized.Outcome == "hung", + $"Minimized capture no longer hangs ({minimized.Outcome}, {minimized.Bytes} bytes) — the production " + + "deadline and its guidance message may now be unnecessary; re-check before removing them."); + host.WindowState = FormWindowState.Normal; + host.Opacity = 0; + await Task.Delay(500); + + // The blank heuristic has to separate these two in the real thing, not just in theory. + var painted = await CaptureFrom(true); + Assert(!DesktopPreviewTools.LooksBlank(painted.Length, Viewport(900, 700)), "A painted page was flagged blank"); + Assert(DesktopPreviewTools.LooksBlank(3160, Viewport(900, 700)), "A blank-sized capture was not flagged"); + // An edited script must never come back from cache; a stale asset is what pushes people // into adding ?v=2 cache-busting query strings to their own project files. var assetRoot = Path.Combine(root, "cache"); @@ -209,6 +250,9 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0) GC.KeepAlive(receiver); } + private static JsonObject Viewport(int width, int height) => + new() { ["viewport"] = new JsonObject { ["width"] = width, ["height"] = height } }; + private static async Task ReloadAsync(CoreWebView2 core, bool ignoreCache) { var completed = new TaskCompletionSource(); diff --git a/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs index 8e714d3..6ced011 100644 --- a/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs +++ b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs @@ -67,6 +67,26 @@ public async Task RefusedDeliveryIsReportedRatherThanClaimed() Assert.Contains("over the", result); } + [Fact] + public async Task NearBlankCapturesAreFlaggedRatherThanDescribed() + { + var tools = Tools(); + tools.ImageSink = new Sink(); + // 3160 bytes over a 900x700 viewport is what an unpainted preview actually returns. + tools.ExecuteAsync = (_, _) => Task.FromResult( + "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" + + Convert.ToBase64String(new byte[3160]) + "\"}"); + var blank = await tools.ScreenshotDesktopPreview(); + Assert.True(Ok(blank)); // a blank page is an observation, not an error + Assert.Contains("\"possiblyBlank\":true", blank); + Assert.Contains("appears blank", blank); + + tools.ExecuteAsync = (_, _) => Task.FromResult( + "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" + + Convert.ToBase64String(new byte[32000]) + "\"}"); + Assert.DoesNotContain("possiblyBlank", await tools.ScreenshotDesktopPreview()); + } + [Theory] [InlineData("http://localhost:5173/")] [InlineData("http://127.0.0.1:3000/about")] diff --git a/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs b/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs index 7e22802..ebf16a7 100644 --- a/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs +++ b/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs @@ -192,7 +192,21 @@ void CheckProject() state["captured"] = request.Selector; } CheckProject(); - var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters)); + string captured; + try + { + // A minimized window has no compositor surface to read, and the browser never + // answers rather than failing. Bound it so that costs seconds and a clear + // explanation instead of the whole operation deadline and a vague timeout. + captured = await CaptureWithDeadlineAsync(core, parameters, token); + } + catch (TimeoutException) + { + return DesktopPreviewTools.Failure( + "The preview could not be captured, which usually means the app window is minimized. " + + "Ask the user to restore the window, or continue with inspect and observe and say that " + + "visual layout could not be checked."); + } var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue(); if (string.IsNullOrEmpty(data)) return DesktopPreviewTools.Failure("The browser did not return a screenshot. The preview may be hidden or still loading."); @@ -255,6 +269,14 @@ private async Task ExecutePreviewKeyAsync(CoreWebView2 core, DesktopPrev return AddPreviewDiagnostics(state); } + /// Screenshot capture, bounded. See the call site for why the browser can never answer. + private static async Task CaptureWithDeadlineAsync(CoreWebView2 core, object parameters, CancellationToken token) + { + var operation = core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters)); + async Task Awaited() => await operation; + return await Awaited().WaitAsync(TimeSpan.FromSeconds(6), token); + } + private static async Task DispatchPreviewKeyAsync(CoreWebView2 core, BrowserKey key, int modifiers, bool down) => await core.CallDevToolsProtocolMethodAsync("Input.dispatchKeyEvent", DesktopPreviewKeys.BuildKeyEvent(key, modifiers, down)); diff --git a/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs b/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs index d8a8e2a..4736ce7 100644 --- a/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs +++ b/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs @@ -178,6 +178,17 @@ public async Task ScreenshotDesktopPreview( try { bytes = Convert.FromBase64String(encoded); } catch (FormatException) { return Failure("The preview returned an unreadable screenshot."); } + // A capture taken before the page painted comes back nearly uniform: valid PNG bytes that + // show nothing. It cannot be told apart from a genuinely blank page, so it is flagged + // rather than refused, and the model is told not to describe what it cannot see. + if (LooksBlank(bytes.Length, state)) + { + state["possiblyBlank"] = true; + state["blankWarning"] = "This capture is nearly uniform. It may be a genuinely blank page, or the " + + "preview may not have painted yet. Say the image appears blank rather than describing detail; " + + "refresh or wait, then capture again if content was expected."; + } + var caption = string.IsNullOrWhiteSpace(note) ? "Screenshot of the project preview." : "Screenshot of the project preview: " + note.Trim(); @@ -201,6 +212,26 @@ public Task OpenLocalServerDesktopPreview( return RunAsync(new("open", _projectRoot.ProjectRoot, Url: resolved), cancellationToken); } + /// + /// A rendered page carries far more compressed detail per pixel than an unpainted one. Measured + /// against a real preview, a painted page runs about 0.05 bytes per pixel and a blank one about + /// 0.005, so this sits an order of magnitude below the painted case and only trips the flat ones. + /// + internal static bool LooksBlank(int byteCount, JsonObject state) + { + var viewport = state["viewport"]; + var area = Number(viewport?["width"]) * Number(viewport?["height"]); + return area >= 10_000 && byteCount / area < 0.01; + + // A number parsed from the page arrives boxed differently than one built in code, and + // asking for the wrong one throws rather than converting. + static double Number(JsonNode? node) => + node is not JsonValue value ? 0 + : value.TryGetValue(out var d) ? d + : value.TryGetValue(out var i) ? i + : 0; + } + /// /// Loopback only, with an explicit port and no embedded credentials. A development server is a /// deliberate widening of what the preview may load; it must not become a way to reach the network.