diff --git a/AGENTS.md b/AGENTS.md index 5151a7ea5..67ea919a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,7 +130,8 @@ contextual-orchestrator owns model discovery and selection. flatten transparent pixels onto white for the derived analysis image while retaining the original asset and provenance. Recognize image DOM/visual regions before OCR, descriptions, Keyman extraction, or embeddings. Store - region-level evidence; never show an internal LLM instruction such as + region-level evidence and show each region's bounding range beside its + caption and OCR (ADR 0155); never show an internal LLM instruction such as `This post is an image` to a buyer. ## Source parsing and semantic units diff --git a/CHANGELOG.d/2.12.27-image-region-locations.md b/CHANGELOG.d/2.12.27-image-region-locations.md new file mode 100644 index 000000000..f199b939d --- /dev/null +++ b/CHANGELOG.d/2.12.27-image-region-locations.md @@ -0,0 +1,9 @@ +## 2.12.27 — Reader-visible image region locations + +- Show persisted image-region bounding ranges beside captions, OCR, and tags. +- Keep the existing buyer-safe image evidence boundary and hide internal LLM + guidance. +- Treat whitespace-only persisted image captions as missing so informative + source images keep the localized `Embedded image` text alternative instead + of a blank accessible name. +- Add the location label to the existing five-locale catalog. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7630dad..3ac674ec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ All notable changes to this project are documented here. Format follows ### Added +- Opening a post with persisted image-region evidence now shows each region's + bounding range beside its caption, OCR, and tags (ADR 0155). After + `make seed`, a synthetic process-diagram region reads **Region location: + 0%, 0% – 100%, 100%**. Internal LLM instructions stay hidden. Click the + region list to compare that box with the caption, then read the source + image. - Ontology neighborhoods can continue beyond the bounded SQL source window with a versioned opaque HMAC cursor and keyset pagination (ADR 0124 / #363). A missing process secret keeps the truncated-without-cursor contract. diff --git a/docs/adr/0110-buyer-image-evidence-rendering.md b/docs/adr/0110-buyer-image-evidence-rendering.md index e6aa26aea..76b21d91c 100644 --- a/docs/adr/0110-buyer-image-evidence-rendering.md +++ b/docs/adr/0110-buyer-image-evidence-rendering.md @@ -33,4 +33,5 @@ placeholder and is not buyer content. Buyers see useful image evidence without seeing instructions intended for an LLM. Search and embedding artifacts remain backed by the existing normalized -parent/region tables and contextual-orchestrator boundary. +parent/region tables and contextual-orchestrator boundary. Buyer-visible +region bounding ranges are specified in [0155](0155-buyer-image-region-locations.md). diff --git a/docs/adr/0155-buyer-image-region-locations.md b/docs/adr/0155-buyer-image-region-locations.md new file mode 100644 index 000000000..80198f914 --- /dev/null +++ b/docs/adr/0155-buyer-image-region-locations.md @@ -0,0 +1,56 @@ +# ADR 0155: Show persisted image-region locations to buyers + +- Status: Accepted +- Date: 2026-08-22 +- Depends on: [0067](0067-visual-region-vision-agent.md), [0091](0091-visual-region-embedding-persistence.md), [0110](0110-buyer-image-evidence-rendering.md) + +## Context + +Normalized visual-region coordinates (`x_ratio`, `y_ratio`, `width_ratio`, +`height_ratio`) already persist beside captions, OCR, and tags. The buyer +popup listed those regions as unlabeled evidence, so an operator could not +tell where on the source image a caption or OCR excerpt came from. Cai, Yu, +Wen, and Ma (2003) treat a visual block's bounding box as first-class +evidence; hiding the box after extraction made the stored coordinates +unusable at the product boundary. + +Stacked PR work already rendered these locations on a non-main branch. This +record lands the same buyer action on protected `main` without mixing into +unrelated stacks. + +## Decision + +- Render each persisted region's axis-aligned box as a percent range next to + its caption or OCR excerpt: `Region location: left%, top% – right%, bottom%`. +- Translate the label through the five-locale UI catalog. +- Omit the location row when any coordinate is non-finite, negative, or greater + than one, or when `x_ratio + width_ratio` or `y_ratio + height_ratio` exceeds + the normalized image boundary. Do not invent a box, a pixel overlay, or an + internal LLM instruction. +- Treat an all-whitespace persisted image caption as absent at the rendering + boundary. Keep the existing localized `Embedded image` short text + alternative rather than exposing a whitespace-only accessible name, and do + not add an empty visual `figcaption` merely because whitespace was stored. +- Treat all-whitespace region captions and OCR excerpts as absent too, so they + cannot suppress the localized `Unknown` evidence fallback. +- Keep image tags, OCR tables, and the source raster on their existing + contracts. This is a presentation boundary, not a new VISION call. + +## Consequences + +After opening a post whose image analysis produced regions, a buyer can read +where each caption or OCR excerpt sits. Informative source images remain +programmatically identifiable even if the persisted caption is blank, while +pure whitespace is no longer surfaced as buyer-visible or assistive-text +evidence. Internal prompts such as `This post is an image` remain hidden. +Complete overlay rendering and authorized-corpus quality remain later slices. + +## References + +Cai, D., Yu, S., Wen, J.-R., & Ma, W.-Y. (2003). *VIPS: A vision-based page +segmentation algorithm* (Microsoft Research Technical Report MSR-TR-2003-79). +Microsoft Research. https://www.microsoft.com/en-us/research/publication/vips-a-vision-based-page-segmentation-algorithm/ + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines +(WCAG) 2.2* (W3C Recommendation, December 12, 2024). +https://www.w3.org/TR/WCAG22/ diff --git a/docs/storybook-inventory.md b/docs/storybook-inventory.md index 3c56ca63d..e5812cc17 100644 --- a/docs/storybook-inventory.md +++ b/docs/storybook-inventory.md @@ -6,6 +6,7 @@ buyer-facing control you can click before changing product CSS. | Story | Buyer next action | Token / module | |---|---|---| | `Evidence/CitationChip` | Click a cited title to open that source post. | `--color-chip-border`, `--radius-chip`, `CitationChip` | +| `Evidence/PostBody` | Open Image regions and read each bounding range beside its caption; inspect the whitespace-caption state to confirm the image keeps a usable fallback name. | `--text-muted`, `PostBody` | | `Evidence/AskEvidenceLayerPopup` | Inspect one citation without leaving the answer; close to continue the answer or open the complete source post. Stories cover text/image evidence, no-evidence, missing OCR, null caption, and blank-caption fallback states. | shared popup tokens through `App.css`, `PopupCloseButton`, `AskEvidenceLayerPopup` | | `AnalysisRun/CutoffKnownBody` | Read the cutoff-known sentence, then compare it with the live body below. | `--color-accent-border`, `--space-panel-block`, `--radius-panel`, `CutoffKnownBody` | | `Analysis/LineageEntityPicker` | Choose which corp to reconstruct, then click Request a lineage reconstruction. | `--space-control-gap`, `--size-control-min`, `--radius-control`, `LineageEntityPicker` | diff --git a/frontend/src/App.css b/frontend/src/App.css index 84c802797..96e9a83a5 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -389,6 +389,22 @@ color: var(--text); } +.post-image-regions li { + margin: 0.45rem 0; +} + +.post-image-regions li > span, +.post-image-regions li > small { + display: block; +} + +.post-image-region-location { + display: block; + margin-top: 0.2rem; + font-size: 0.8rem; + color: var(--text-muted); +} + .popup-placeholder { margin-top: 1.5rem; padding: 1rem; diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 560ede9eb..fc7e03373 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -29,12 +29,14 @@ beforeEach(() => { afterEach(() => { vi.unstubAllGlobals(); + window.history.replaceState({}, "", "/"); window.sessionStorage.clear(); window.localStorage.clear(); }); describe("App, unauthenticated", () => { it("shows a login button that starts the real OIDC redirect", async () => { + window.history.replaceState({}, "", "/?post=abc#evidence"); render(); expect(screen.queryByRole("heading", { name: /admin settings/i })).toBeNull(); const button = screen.getByRole("button", { name: /log in/i }); @@ -42,7 +44,7 @@ describe("App, unauthenticated", () => { expect(signinRedirect).toHaveBeenCalledTimes(1); expect(signinRedirect).toHaveBeenCalledWith( expect.objectContaining({ - state: expect.objectContaining({ returnUrl: expect.stringMatching(/^\//) }), + state: { returnUrl: "/?post=abc#evidence" }, }), ); // Persisted as a fallback in case the OIDC state round-trip is dropped diff --git a/frontend/src/PostBody.blank-caption.test.tsx b/frontend/src/PostBody.blank-caption.test.tsx new file mode 100644 index 000000000..29b2e14b4 --- /dev/null +++ b/frontend/src/PostBody.blank-caption.test.tsx @@ -0,0 +1,30 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { PostBody } from "./PostBody"; + +const SAFE_PIXEL = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("PostBody image accessibility fallback", () => { + it("treats a whitespace-only persisted caption as missing", () => { + const { container } = render( + `} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: " \n ", + caption: " ", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByRole("img", { name: "Embedded image" })).toBeInTheDocument(); + expect(container.querySelector("figcaption")).toBeNull(); + expect(container.querySelector(".post-image-text")).toBeNull(); + }); +}); diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx new file mode 100644 index 000000000..eec12ffba --- /dev/null +++ b/frontend/src/PostBody.stories.tsx @@ -0,0 +1,79 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { PostBody } from "./PostBody"; +import diagramSrc from "./fixtures/synthetic-process-diagram.png?inline"; + +const meta = { + title: "Evidence/PostBody", + component: PostBody, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const ImageRegionLocations: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "Demo Corp process diagram", + caption: "A process diagram", + tags: ["diagram", "process"], + regions: [ + { + region_index: 0, + x_ratio: 0.1, + y_ratio: 0.2, + width_ratio: 0.3, + height_ratio: 0.4, + status_code: "described", + extracted_text: "Title block", + caption: "Title block", + tags: ["title"], + }, + { + region_index: 1, + x_ratio: 0.5, + y_ratio: 0.18, + width_ratio: 0.4, + height_ratio: 0.28, + status_code: "described", + extracted_text: "1. Capture 2. Fuse 3. Land", + caption: "Process steps", + tags: ["steps"], + }, + { + region_index: 2, + x_ratio: 0.5, + y_ratio: 0.55, + width_ratio: 0.4, + height_ratio: 0.3, + status_code: "described", + extracted_text: "Synthetic fixture only", + caption: "Notes", + tags: ["notes"], + }, + ], + }, + ], + }, +}; + +export const WhitespaceCaptionFallsBack: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: null, + caption: " ", + tags: [], + }, + ], + }, +}; diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 3eaba4970..b0b768195 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -347,10 +347,10 @@ describe("PostBody", () => { regions: [ { region_index: 0, - x_ratio: 0, - y_ratio: 0, - width_ratio: 1, - height_ratio: 1, + x_ratio: 0.1, + y_ratio: 0.1, + width_ratio: 0.9000000000000001, + height_ratio: 0.9000000000000001, status_code: "described", extracted_text: "Region OCR", caption: "Main panel", @@ -365,6 +365,7 @@ describe("PostBody", () => { expect(screen.getByText("A process diagram")).toBeInTheDocument(); expect(screen.getByText("diagram, process")).toBeInTheDocument(); expect(screen.getByText("Main panel")).toBeInTheDocument(); + expect(screen.getByText("Region location: 10%, 10% – 100%, 100%")).toBeInTheDocument(); expect(screen.queryByText(/This post is an image/)).not.toBeInTheDocument(); }); @@ -390,6 +391,115 @@ describe("PostBody", () => { expect(screen.getByText("Panel")).toBeInTheDocument(); }); + it("shows persisted image-region bounding ranges beside captions", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "Visible OCR", + caption: "A process diagram", + tags: [], + regions: [ + { + region_index: 0, + x_ratio: 0.1, + y_ratio: 0.2, + width_ratio: 0.3, + height_ratio: 0.4, + status_code: "described", + extracted_text: "Region OCR", + caption: "Title block", + tags: ["title"], + }, + ], + }, + ]} + />, + ); + + expect(screen.getByText("Title block")).toBeInTheDocument(); + expect(screen.getByText("Text detected in image: Region OCR")).toBeInTheDocument(); + expect(screen.getByText("Region location: 10%, 20% – 40%, 60%")).toBeInTheDocument(); + expect(screen.getByText("Image regions").closest("details")).toHaveAttribute("open"); + expect(screen.queryByText(/This post is an image/)).not.toBeInTheDocument(); + }); + + it("treats whitespace-only region caption and OCR as missing evidence", () => { + render( + '} + imageContent={[{ + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: null, + caption: "A process diagram", + tags: [], + regions: [{ + region_index: 0, + x_ratio: 0.1, + y_ratio: 0.2, + width_ratio: 0.3, + height_ratio: 0.4, + status_code: "described", + extracted_text: "\n", + caption: " ", + tags: [], + }], + }]} + />, + ); + + expect(screen.getByText("Unknown")).toBeInTheDocument(); + expect(screen.queryByText("Text detected in image:")).not.toBeInTheDocument(); + }); + + it.each([ + [Number.NaN, 0.2, 0.3, 0.4], + [-0.1, 0.2, 0.3, 0.4], + [0.9, 0.2, 0.2, 0.4], + ])( + "omits the location row for invalid region bounds %s, %s, %s, %s", + (xRatio, yRatio, widthRatio, heightRatio) => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: null, + caption: "A process diagram", + tags: [], + regions: [ + { + region_index: 0, + x_ratio: xRatio, + y_ratio: yRatio, + width_ratio: widthRatio, + height_ratio: heightRatio, + status_code: "described", + extracted_text: "Region OCR", + caption: "Broken box", + tags: [], + }, + ], + }, + ]} + />, + ); + + expect(screen.getByText("Broken box")).toBeInTheDocument(); + expect(screen.getByText("Text detected in image: Region OCR")).toBeInTheDocument(); + expect(screen.queryByText(/Region location/)).not.toBeInTheDocument(); + }, + ); + it("keeps source-image placement while showing persisted OCR and caption evidence", () => { render( !Number.isFinite(value) || value < 0 || value > 1) || + right > 1 + Number.EPSILON * 4 || + bottom > 1 + Number.EPSILON * 4 + ) { + return null; + } + const percent = (value: number) => `${Math.round(value * 100)}%`; + return `${t("Region location")}: ${percent(region.x_ratio)}, ${percent(region.y_ratio)} – ${percent(right)}, ${percent(bottom)}`; +} + function renderImageEvidence( index: number, imageContent?: PostImageContent, @@ -65,39 +80,53 @@ function renderImageEvidence( ) { const sourceImageSrc = sourceImage && SAFE_EMBEDDED_IMAGE_SOURCE.test(sourceImage.src) ? sourceImage.src : undefined; + const imageCaption = imageContent?.caption?.trim() || ""; + const imageExtractedText = imageContent?.extracted_text?.trim() || ""; return (
{sourceImageSrc ? ( - {imageContent?.caption + {imageCaption ) : null} - {imageContent?.caption || !sourceImageSrc ? ( -
{imageContent?.caption || t("Embedded image")}
+ {imageCaption || !sourceImageSrc ? ( +
{imageCaption || t("Embedded image")}
) : null} {imageContent?.tags.length ? (

{t("Image tags")}: {imageContent.tags.join(", ")}

) : null} - {imageContent?.extracted_text ? ( + {imageExtractedText ? (
{t("Text detected in image")} - {renderImageText(imageContent.extracted_text)} + {renderImageText(imageExtractedText)}
) : null} {imageContent?.regions?.length ? ( -
+
{t("Image regions")}
    - {imageContent.regions.map((region) => ( -
  1. - {region.caption || region.extracted_text || t("Unknown")} - {region.tags.length ? ( - - {t("Image tags")}: {region.tags.join(", ")} - - ) : null} -
    2. - ))} + {imageContent.regions.map((region) => { + const location = formatImageRegionLocation(region); + const caption = region.caption?.trim() || ""; + const extractedText = region.extracted_text?.trim() || ""; + return ( +
  2. + {caption ? {caption} : null} + {extractedText && extractedText !== caption ? ( + {t("Text detected in image")}: {extractedText} + ) : null} + {!caption && !extractedText ? {t("Unknown")} : null} + {region.tags.length ? ( + + {t("Image tags")}: {region.tags.join(", ")} + + ) : null} + {location ? ( + {location} + ) : null} +
    3. + ); + })}
) : null} diff --git a/frontend/src/fixtures/synthetic-process-diagram.png b/frontend/src/fixtures/synthetic-process-diagram.png new file mode 100644 index 000000000..28ea9acc5 Binary files /dev/null and b/frontend/src/fixtures/synthetic-process-diagram.png differ diff --git a/frontend/src/i18n.test.ts b/frontend/src/i18n.test.ts index 6469a5847..713217391 100644 --- a/frontend/src/i18n.test.ts +++ b/frontend/src/i18n.test.ts @@ -23,6 +23,8 @@ describe("i18n", () => { "Keymen", "Unknown", "Image tags", + "Image regions", + "Region location", "Counterparties", "due", "Activity", @@ -108,6 +110,16 @@ describe("i18n", () => { expect(tf("{post} is current in Event Lineage. Read Keyman and evaluation next.", { post: "DEMO" })).toBe(expected); }); + it.each([ + ["ko", "영역 위치"], + ["zh", "区域位置"], + ["ja", "領域の位置"], + ["vi", "Vị trí vùng"], + ] as const)("translates image region location in %s", (locale, expected) => { + setLocale(locale); + expect(t("Region location")).toBe(expected); + }); + it.each([ ["ko", "IRT 주효과 이후 잔여 R +0.40. sales-lead 기준을 읽으려면 이 글을 여세요."], ["zh", "IRT 主效应后的残余 R +0.40。打开这篇帖子阅读 sales-lead。"], diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 109420724..4779611b8 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -293,6 +293,7 @@ const TRANSLATIONS: Partial>> = { "Embedded image": "삽입 이미지", "Text detected in image": "이미지에서 인식된 텍스트", "Image regions": "이미지 영역", + "Region location": "영역 위치", "Image tags": "이미지 태그", "Embedded image could not be decoded. Re-export the source post and open it again.": "첨부 이미지를 해독할 수 없습니다. 원문을 다시 내보내고 다시 여세요.", @@ -735,6 +736,7 @@ const TRANSLATIONS: Partial>> = { "Embedded image": "嵌入图像", "Text detected in image": "图像中识别的文字", "Image regions": "图像区域", + "Region location": "区域位置", "Image tags": "图像标签", "Embedded image could not be decoded. Re-export the source post and open it again.": "无法解码嵌入图像。请重新导出原始文章后再打开。", @@ -1191,6 +1193,7 @@ const TRANSLATIONS: Partial>> = { "Embedded image": "埋め込み画像", "Text detected in image": "画像から認識されたテキスト", "Image regions": "画像領域", + "Region location": "領域の位置", "Image tags": "画像タグ", "Embedded image could not be decoded. Re-export the source post and open it again.": "埋め込み画像をデコードできませんでした。原文を再エクスポートして、もう一度開いてください。", @@ -1633,6 +1636,7 @@ const TRANSLATIONS: Partial>> = { "Embedded image": "Hình ảnh nhúng", "Text detected in image": "Văn bản nhận dạng trong hình ảnh", "Image regions": "Các vùng trong hình ảnh", + "Region location": "Vị trí vùng", "Image tags": "Thẻ hình ảnh", "Embedded image could not be decoded. Re-export the source post and open it again.": "Không thể giải mã hình ảnh nhúng. Hãy xuất lại bài viết gốc rồi mở lại.", diff --git a/frontend/src/oidcReturnUrl.test.ts b/frontend/src/oidcReturnUrl.test.ts index de6e95502..fca382921 100644 --- a/frontend/src/oidcReturnUrl.test.ts +++ b/frontend/src/oidcReturnUrl.test.ts @@ -12,8 +12,8 @@ describe("OIDC return URL handling", () => { }); it("keeps a post deep link and rejects external destinations", () => { - expect(returnUrlFromLocation({ pathname: "/", search: "?post=abc", hash: "" })).toBe( - "/?post=abc", + expect(returnUrlFromLocation({ pathname: "/", search: "?post=abc", hash: "#evidence" })).toBe( + "/?post=abc#evidence", ); expect(returnUrlFromLocation({ pathname: "//evil.example", search: "", hash: "" })).toBe("/"); }); diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts index bb17fb571..b218e965f 100644 --- a/frontend/src/vite-env.d.ts +++ b/frontend/src/vite-env.d.ts @@ -1,5 +1,10 @@ /// +declare module "*.png?inline" { + const src: string; + export default src; +} + interface ImportMetaEnv { readonly VITE_KEYVERSE_ISSUER?: string; readonly VITE_KEYVERSE_CLIENT_ID?: string; diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 3122f99dd..3f1db9918 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -55,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "2.12.18" +__version__ = "2.12.19"