enhancement: make preview screenshots fail honestly instead of returning blank images

docs/browser-tools.md

@@ -111,6 +111,16 @@ An image is evidence for the turn that captured it and is retracted afterward, s
 screenshot does not re-upload on every later message. The model's written conclusion is
 what persists.
 
+Capture reads the window's rendered surface, so it depends on the app actually having one.
+A minimized window has none and the browser never answers at all, so capture is bounded at
+six seconds and reports that the window needs restoring rather than consuming the whole
+operation deadline. Hidden, transparent, and occluded windows still capture normally.
+
+A capture taken before the page painted returns valid image bytes showing nothing. That
+cannot be told apart from a genuinely blank page, so it is flagged as `possiblyBlank`
+rather than refused, and the model is told to say the image looks blank instead of
+describing detail it cannot see.
+
 ## Development servers
 
 `open_local_server_desktop_preview` opens a server already running on this machine, so the

src/MandoCode.Desktop.BrowserSmokeTests/Program.cs

@@ -24,7 +24,7 @@ private static int Main()
                 await CheckBrowserAsync(browser.CoreWebView2).WaitAsync(TimeSpan.FromSeconds(45));
                 exitCode = 0;
                 Console.WriteLine("PASS: real WebView2 DOM, pointer, keyboard, repeated clicks, focused observations, " +
-                    "forms, scrolling, navigation, fresh assets on reload, screenshots, origin scoping, diagnostics, and argument escaping.");
+                    "forms, scrolling, navigation, fresh assets on reload, screenshots under changing window visibility, origin scoping, diagnostics, and argument escaping.");
             }
             catch (Exception ex) { Console.Error.WriteLine(ex); }
             finally { browser.Dispose(); form.Close(); }
@@ -54,6 +54,13 @@ async Task<JsonObject> RunAs(string? origin, string operation, string? selector
         }
         Task<JsonObject> Run(string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) =>
             RunAs(previewOrigin, operation, selector, value, offset, deltaY, observe);
+        async Task<byte[]> CaptureFrom(bool fromSurface)
+        {
+            var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot",
+                JsonSerializer.Serialize(new { format = "png", fromSurface, captureBeyondViewport = false }));
+            var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue<string>();
+            return string.IsNullOrEmpty(data) ? [] : Convert.FromBase64String(data);
+        }
         async Task<byte[]> Capture(JsonObject? clip)
         {
             object parameters = clip == null ? new { format = "png" } : new
@@ -176,6 +183,40 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0)
         Assert(clipped.Length < full.Length, $"Clipping captured no less than the full page ({clipped.Length} vs {full.Length})");
         Assert(!(await Run("bounds", "#hidden"))["ok"]!.GetValue<bool>(), "A hidden element was accepted for capture");
 
+        // Capture depends on the window having a compositor surface. A minimized window has none,
+        // and the browser never answers at all, so the production path bounds this rather than
+        // letting one call eat the whole operation deadline.
+        var host = Application.OpenForms[0]!;
+        async Task<(string Outcome, int Bytes)> TimedCapture()
+        {
+            try
+            {
+                var bytes = await CaptureFrom(true).WaitAsync(TimeSpan.FromSeconds(4));
+                return ("captured", bytes.Length);
+            }
+            catch (TimeoutException) { return ("hung", 0); }
+        }
+        host.Opacity = 1;
+        await Task.Delay(500);
+        var shown = await TimedCapture();
+        Assert(shown.Outcome == "captured" && shown.Bytes > 20000,
+            $"A visible preview did not capture a painted page ({shown.Outcome}, {shown.Bytes} bytes)");
+
+        host.WindowState = FormWindowState.Minimized;
+        await Task.Delay(500);
+        var minimized = await TimedCapture();
+        Assert(minimized.Outcome == "hung",
+            $"Minimized capture no longer hangs ({minimized.Outcome}, {minimized.Bytes} bytes) — the production " +
+            "deadline and its guidance message may now be unnecessary; re-check before removing them.");
+        host.WindowState = FormWindowState.Normal;
+        host.Opacity = 0;
+        await Task.Delay(500);
+
+        // The blank heuristic has to separate these two in the real thing, not just in theory.
+        var painted = await CaptureFrom(true);
+        Assert(!DesktopPreviewTools.LooksBlank(painted.Length, Viewport(900, 700)), "A painted page was flagged blank");
+        Assert(DesktopPreviewTools.LooksBlank(3160, Viewport(900, 700)), "A blank-sized capture was not flagged");
+
         // An edited script must never come back from cache; a stale asset is what pushes people
         // into adding ?v=2 cache-busting query strings to their own project files.
         var assetRoot = Path.Combine(root, "cache");
@@ -209,6 +250,9 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0)
         GC.KeepAlive(receiver);
     }
 
+    private static JsonObject Viewport(int width, int height) =>
+        new() { ["viewport"] = new JsonObject { ["width"] = width, ["height"] = height } };
+
     private static async Task ReloadAsync(CoreWebView2 core, bool ignoreCache)
     {
         var completed = new TaskCompletionSource<bool>();

src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs

@@ -67,6 +67,26 @@ public async Task RefusedDeliveryIsReportedRatherThanClaimed()
         Assert.Contains("over the", result);
     }
 
+    [Fact]
+    public async Task NearBlankCapturesAreFlaggedRatherThanDescribed()
+    {
+        var tools = Tools();
+        tools.ImageSink = new Sink();
+        // 3160 bytes over a 900x700 viewport is what an unpainted preview actually returns.
+        tools.ExecuteAsync = (_, _) => Task.FromResult(
+            "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" +
+            Convert.ToBase64String(new byte[3160]) + "\"}");
+        var blank = await tools.ScreenshotDesktopPreview();
+        Assert.True(Ok(blank));                       // a blank page is an observation, not an error
+        Assert.Contains("\"possiblyBlank\":true", blank);
+        Assert.Contains("appears blank", blank);
+
+        tools.ExecuteAsync = (_, _) => Task.FromResult(
+            "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" +
+            Convert.ToBase64String(new byte[32000]) + "\"}");
+        Assert.DoesNotContain("possiblyBlank", await tools.ScreenshotDesktopPreview());
+    }
+
     [Theory]
     [InlineData("http://localhost:5173/")]
     [InlineData("http://127.0.0.1:3000/about")]

src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs

@@ -192,7 +192,21 @@ void CheckProject()
                     state["captured"] = request.Selector;
                 }
                 CheckProject();
-                var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters));
+                string captured;
+                try
+                {
+                    // A minimized window has no compositor surface to read, and the browser never
+                    // answers rather than failing. Bound it so that costs seconds and a clear
+                    // explanation instead of the whole operation deadline and a vague timeout.
+                    captured = await CaptureWithDeadlineAsync(core, parameters, token);
+                }
+                catch (TimeoutException)
+                {
+                    return DesktopPreviewTools.Failure(
+                        "The preview could not be captured, which usually means the app window is minimized. " +
+                        "Ask the user to restore the window, or continue with inspect and observe and say that " +
+                        "visual layout could not be checked.");
+                }
                 var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue<string>();
                 if (string.IsNullOrEmpty(data))
                     return DesktopPreviewTools.Failure("The browser did not return a screenshot. The preview may be hidden or still loading.");
@@ -255,6 +269,14 @@ private async Task<string> ExecutePreviewKeyAsync(CoreWebView2 core, DesktopPrev
         return AddPreviewDiagnostics(state);
     }
 
+    /// <summary>Screenshot capture, bounded. See the call site for why the browser can never answer.</summary>
+    private static async Task<string> CaptureWithDeadlineAsync(CoreWebView2 core, object parameters, CancellationToken token)
+    {
+        var operation = core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters));
+        async Task<string> Awaited() => await operation;
+        return await Awaited().WaitAsync(TimeSpan.FromSeconds(6), token);
+    }
+
     private static async Task DispatchPreviewKeyAsync(CoreWebView2 core, BrowserKey key, int modifiers, bool down) =>
         await core.CallDevToolsProtocolMethodAsync("Input.dispatchKeyEvent", DesktopPreviewKeys.BuildKeyEvent(key, modifiers, down));
 

src/MandoCode.Desktop/Services/DesktopPreviewTools.cs

@@ -178,6 +178,17 @@ public async Task<string> ScreenshotDesktopPreview(
         try { bytes = Convert.FromBase64String(encoded); }
         catch (FormatException) { return Failure("The preview returned an unreadable screenshot."); }
 
+        // A capture taken before the page painted comes back nearly uniform: valid PNG bytes that
+        // show nothing. It cannot be told apart from a genuinely blank page, so it is flagged
+        // rather than refused, and the model is told not to describe what it cannot see.
+        if (LooksBlank(bytes.Length, state))
+        {
+            state["possiblyBlank"] = true;
+            state["blankWarning"] = "This capture is nearly uniform. It may be a genuinely blank page, or the " +
+                "preview may not have painted yet. Say the image appears blank rather than describing detail; " +
+                "refresh or wait, then capture again if content was expected.";
+        }
+
         var caption = string.IsNullOrWhiteSpace(note)
             ? "Screenshot of the project preview."
             : "Screenshot of the project preview: " + note.Trim();
@@ -201,6 +212,26 @@ public Task<string> OpenLocalServerDesktopPreview(
         return RunAsync(new("open", _projectRoot.ProjectRoot, Url: resolved), cancellationToken);
     }
 
+    /// <summary>
+    /// A rendered page carries far more compressed detail per pixel than an unpainted one. Measured
+    /// against a real preview, a painted page runs about 0.05 bytes per pixel and a blank one about
+    /// 0.005, so this sits an order of magnitude below the painted case and only trips the flat ones.
+    /// </summary>
+    internal static bool LooksBlank(int byteCount, JsonObject state)
+    {
+        var viewport = state["viewport"];
+        var area = Number(viewport?["width"]) * Number(viewport?["height"]);
+        return area >= 10_000 && byteCount / area < 0.01;
+
+        // A number parsed from the page arrives boxed differently than one built in code, and
+        // asking for the wrong one throws rather than converting.
+        static double Number(JsonNode? node) =>
+            node is not JsonValue value ? 0
+            : value.TryGetValue<double>(out var d) ? d
+            : value.TryGetValue<int>(out var i) ? i
+            : 0;
+    }
+
     /// <summary>
     /// Loopback only, with an explicit port and no embedded credentials. A development server is a
     /// deliberate widening of what the preview may load; it must not become a way to reach the network.