diff --git a/docs/browser-tools.md b/docs/browser-tools.md index bffd8db..1299291 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -111,6 +111,16 @@ An image is evidence for the turn that captured it and is retracted afterward, s screenshot does not re-upload on every later message. The model's written conclusion is what persists. +Capture reads the window's rendered surface, so it depends on the app actually having one. +A minimized window has none and the browser never answers at all, so capture is bounded at +six seconds and reports that the window needs restoring rather than consuming the whole +operation deadline. Hidden, transparent, and occluded windows still capture normally. + +A capture taken before the page painted returns valid image bytes showing nothing. That +cannot be told apart from a genuinely blank page, so it is flagged as `possiblyBlank` +rather than refused, and the model is told to say the image looks blank instead of +describing detail it cannot see. + ## Development servers `open_local_server_desktop_preview` opens a server already running on this machine, so the diff --git a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs index 85275fb..9d5de2b 100644 --- a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs +++ b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs @@ -24,7 +24,7 @@ private static int Main() await CheckBrowserAsync(browser.CoreWebView2).WaitAsync(TimeSpan.FromSeconds(45)); exitCode = 0; Console.WriteLine("PASS: real WebView2 DOM, pointer, keyboard, repeated clicks, focused observations, " + - "forms, scrolling, navigation, fresh assets on reload, screenshots, origin scoping, diagnostics, and argument escaping."); + "forms, scrolling, navigation, fresh assets on reload, screenshots under changing window visibility, origin scoping, diagnostics, and argument escaping."); } catch (Exception ex) { Console.Error.WriteLine(ex); } finally { browser.Dispose(); form.Close(); } @@ -54,6 +54,13 @@ async Task RunAs(string? origin, string operation, string? selector } Task Run(string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) => RunAs(previewOrigin, operation, selector, value, offset, deltaY, observe); + async Task CaptureFrom(bool fromSurface) + { + var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", + JsonSerializer.Serialize(new { format = "png", fromSurface, captureBeyondViewport = false })); + var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue(); + return string.IsNullOrEmpty(data) ? [] : Convert.FromBase64String(data); + } async Task Capture(JsonObject? clip) { object parameters = clip == null ? new { format = "png" } : new @@ -176,6 +183,40 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0) Assert(clipped.Length < full.Length, $"Clipping captured no less than the full page ({clipped.Length} vs {full.Length})"); Assert(!(await Run("bounds", "#hidden"))["ok"]!.GetValue(), "A hidden element was accepted for capture"); + // Capture depends on the window having a compositor surface. A minimized window has none, + // and the browser never answers at all, so the production path bounds this rather than + // letting one call eat the whole operation deadline. + var host = Application.OpenForms[0]!; + async Task<(string Outcome, int Bytes)> TimedCapture() + { + try + { + var bytes = await CaptureFrom(true).WaitAsync(TimeSpan.FromSeconds(4)); + return ("captured", bytes.Length); + } + catch (TimeoutException) { return ("hung", 0); } + } + host.Opacity = 1; + await Task.Delay(500); + var shown = await TimedCapture(); + Assert(shown.Outcome == "captured" && shown.Bytes > 20000, + $"A visible preview did not capture a painted page ({shown.Outcome}, {shown.Bytes} bytes)"); + + host.WindowState = FormWindowState.Minimized; + await Task.Delay(500); + var minimized = await TimedCapture(); + Assert(minimized.Outcome == "hung", + $"Minimized capture no longer hangs ({minimized.Outcome}, {minimized.Bytes} bytes) — the production " + + "deadline and its guidance message may now be unnecessary; re-check before removing them."); + host.WindowState = FormWindowState.Normal; + host.Opacity = 0; + await Task.Delay(500); + + // The blank heuristic has to separate these two in the real thing, not just in theory. + var painted = await CaptureFrom(true); + Assert(!DesktopPreviewTools.LooksBlank(painted.Length, Viewport(900, 700)), "A painted page was flagged blank"); + Assert(DesktopPreviewTools.LooksBlank(3160, Viewport(900, 700)), "A blank-sized capture was not flagged"); + // An edited script must never come back from cache; a stale asset is what pushes people // into adding ?v=2 cache-busting query strings to their own project files. var assetRoot = Path.Combine(root, "cache"); @@ -209,6 +250,9 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0) GC.KeepAlive(receiver); } + private static JsonObject Viewport(int width, int height) => + new() { ["viewport"] = new JsonObject { ["width"] = width, ["height"] = height } }; + private static async Task ReloadAsync(CoreWebView2 core, bool ignoreCache) { var completed = new TaskCompletionSource(); diff --git a/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs index 8e714d3..6ced011 100644 --- a/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs +++ b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs @@ -67,6 +67,26 @@ public async Task RefusedDeliveryIsReportedRatherThanClaimed() Assert.Contains("over the", result); } + [Fact] + public async Task NearBlankCapturesAreFlaggedRatherThanDescribed() + { + var tools = Tools(); + tools.ImageSink = new Sink(); + // 3160 bytes over a 900x700 viewport is what an unpainted preview actually returns. + tools.ExecuteAsync = (_, _) => Task.FromResult( + "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" + + Convert.ToBase64String(new byte[3160]) + "\"}"); + var blank = await tools.ScreenshotDesktopPreview(); + Assert.True(Ok(blank)); // a blank page is an observation, not an error + Assert.Contains("\"possiblyBlank\":true", blank); + Assert.Contains("appears blank", blank); + + tools.ExecuteAsync = (_, _) => Task.FromResult( + "{\"ok\":true,\"readyState\":\"complete\",\"viewport\":{\"width\":900,\"height\":700},\"image\":\"" + + Convert.ToBase64String(new byte[32000]) + "\"}"); + Assert.DoesNotContain("possiblyBlank", await tools.ScreenshotDesktopPreview()); + } + [Theory] [InlineData("http://localhost:5173/")] [InlineData("http://127.0.0.1:3000/about")] diff --git a/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs b/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs index 7e22802..ebf16a7 100644 --- a/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs +++ b/src/MandoCode.Desktop/Controls/ChatTabView.BrowserTools.cs @@ -192,7 +192,21 @@ void CheckProject() state["captured"] = request.Selector; } CheckProject(); - var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters)); + string captured; + try + { + // A minimized window has no compositor surface to read, and the browser never + // answers rather than failing. Bound it so that costs seconds and a clear + // explanation instead of the whole operation deadline and a vague timeout. + captured = await CaptureWithDeadlineAsync(core, parameters, token); + } + catch (TimeoutException) + { + return DesktopPreviewTools.Failure( + "The preview could not be captured, which usually means the app window is minimized. " + + "Ask the user to restore the window, or continue with inspect and observe and say that " + + "visual layout could not be checked."); + } var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue(); if (string.IsNullOrEmpty(data)) return DesktopPreviewTools.Failure("The browser did not return a screenshot. The preview may be hidden or still loading."); @@ -255,6 +269,14 @@ private async Task ExecutePreviewKeyAsync(CoreWebView2 core, DesktopPrev return AddPreviewDiagnostics(state); } + /// Screenshot capture, bounded. See the call site for why the browser can never answer. + private static async Task CaptureWithDeadlineAsync(CoreWebView2 core, object parameters, CancellationToken token) + { + var operation = core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters)); + async Task Awaited() => await operation; + return await Awaited().WaitAsync(TimeSpan.FromSeconds(6), token); + } + private static async Task DispatchPreviewKeyAsync(CoreWebView2 core, BrowserKey key, int modifiers, bool down) => await core.CallDevToolsProtocolMethodAsync("Input.dispatchKeyEvent", DesktopPreviewKeys.BuildKeyEvent(key, modifiers, down)); diff --git a/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs b/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs index d8a8e2a..4736ce7 100644 --- a/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs +++ b/src/MandoCode.Desktop/Services/DesktopPreviewTools.cs @@ -178,6 +178,17 @@ public async Task ScreenshotDesktopPreview( try { bytes = Convert.FromBase64String(encoded); } catch (FormatException) { return Failure("The preview returned an unreadable screenshot."); } + // A capture taken before the page painted comes back nearly uniform: valid PNG bytes that + // show nothing. It cannot be told apart from a genuinely blank page, so it is flagged + // rather than refused, and the model is told not to describe what it cannot see. + if (LooksBlank(bytes.Length, state)) + { + state["possiblyBlank"] = true; + state["blankWarning"] = "This capture is nearly uniform. It may be a genuinely blank page, or the " + + "preview may not have painted yet. Say the image appears blank rather than describing detail; " + + "refresh or wait, then capture again if content was expected."; + } + var caption = string.IsNullOrWhiteSpace(note) ? "Screenshot of the project preview." : "Screenshot of the project preview: " + note.Trim(); @@ -201,6 +212,26 @@ public Task OpenLocalServerDesktopPreview( return RunAsync(new("open", _projectRoot.ProjectRoot, Url: resolved), cancellationToken); } + /// + /// A rendered page carries far more compressed detail per pixel than an unpainted one. Measured + /// against a real preview, a painted page runs about 0.05 bytes per pixel and a blank one about + /// 0.005, so this sits an order of magnitude below the painted case and only trips the flat ones. + /// + internal static bool LooksBlank(int byteCount, JsonObject state) + { + var viewport = state["viewport"]; + var area = Number(viewport?["width"]) * Number(viewport?["height"]); + return area >= 10_000 && byteCount / area < 0.01; + + // A number parsed from the page arrives boxed differently than one built in code, and + // asking for the wrong one throws rather than converting. + static double Number(JsonNode? node) => + node is not JsonValue value ? 0 + : value.TryGetValue(out var d) ? d + : value.TryGetValue(out var i) ? i + : 0; + } + /// /// Loopback only, with an explicit port and no embedded credentials. A development server is a /// deliberate widening of what the preview may load; it must not become a way to reach the network.