From 19a613125e2bb20bfe8f5d8139c20b06d1f40ce6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:11:43 +0000 Subject: [PATCH] feat(ui): show embedded post images instead of raw base64 Open a post or evidence panel and see each data-URI picture in document order. The popup no longer dumps the base64 wall. Remote http(s) image URLs stay unloaded. Extract Keyman or Ask still runs OCR on those images. Rebased onto live #74 head ca9bd82 after #128 squash-merged. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 11 +++++ docs/image-content-schema.md | 26 ++++++++++ frontend/package.json | 2 +- frontend/src/App.css | 27 ++++++++++ frontend/src/App.test.tsx | 20 +++++++- frontend/src/App.tsx | 5 +- frontend/src/PostBody.tsx | 33 +++++++++++++ frontend/src/index.css | 5 ++ frontend/src/postBodyDisplay.test.ts | 74 ++++++++++++++++++++++++++++ frontend/src/postBodyDisplay.ts | 72 +++++++++++++++++++++++++++ lineageweave/__init__.py | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 14 files changed, 275 insertions(+), 8 deletions(-) create mode 100644 frontend/src/PostBody.tsx create mode 100644 frontend/src/postBodyDisplay.test.ts create mode 100644 frontend/src/postBodyDisplay.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 26c817cde..a8d082f0b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl) | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) renders each `data:image` payload in document order so the buyer sees the picture, not the base64 string; GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | | `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf4dc62a..fd8717513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.86.1] - 2026-08-16 + +### Changed + +- Opening a post or its evidence panel now shows each embedded + `data:image` picture in document order, with the surrounding sentences + as text. The raw base64 string is no longer dumped into the popup. + Remote `http(s)` image URLs stay unloaded. After `make seed`, a post + whose body includes a data-URI image shows the picture; Extract Keyman + or Ask still runs OCR on that image for search. + ## [0.86.0] - 2026-08-16 ### Added diff --git a/docs/image-content-schema.md b/docs/image-content-schema.md index f023d56a0..4dae03dcf 100644 --- a/docs/image-content-schema.md +++ b/docs/image-content-schema.md @@ -82,6 +82,18 @@ picture sat relative to the surrounding paragraphs." | `chunk_position` | `integer not null` | 0-based index among ALL of this document's chunks (text and image together) -- matches `Chunk.index` from `chunk_by_dom` | | primary key | `(source_document_id, chunk_position)` | one image slot per position per document | +## Viewer contract (before persistence exists) + +The demo popup does not yet read these tables. It splits the live +`post_body` the same way `extract_base64_images` does: each +`data:image/...;base64,...` payload becomes an `` at its original +character offset, and the surrounding HTML is shown as text. A buyer who +opens the post sees the picture that sat between the paragraphs, not the +base64 wall. Remote `src="https://..."` tags are stripped, never fetched. +OCR, caption, and tag search still require the vision client on extract / +Ask (Li et al., 2023; Radford et al., 2021) and, in a real deployment, +the tables below. + ## Query shapes this supports - **"Find images whose extracted text or tags match a search query, then @@ -105,3 +117,17 @@ picture sat relative to the surrounding paragraphs." ON CONFLICT DO NOTHING` before the provider call, or a short-lived lease row) to close that race; this schema documents the storage guarantee, not that concurrency control. + +## References + +Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, Z., +& Wei, F. (2023). TrOCR: Transformer-based optical character recognition +with pre-trained models. *Proceedings of the AAAI Conference on Artificial +Intelligence, 37*(11), 13094–13102. https://doi.org/10.1609/aaai.v37i11.26538 + +Radford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., +Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., & Sutskever, I. +(2021). Learning transferable visual models from natural language +supervision. In M. Meila & T. Zhang (Eds.), *Proceedings of the 38th +International Conference on Machine Learning* (pp. 8748–8763). PMLR. +https://proceedings.mlr.press/v139/radford21a.html diff --git a/frontend/package.json b/frontend/package.json index dac241738..803acd91b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.86.0", + "version": "0.86.1", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index 8f38b4dd0..b3fab25d1 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -108,9 +108,36 @@ } .post-body { + display: flex; + flex-direction: column; + gap: var(--post-body-gap); +} + +.post-body-text { + margin: 0; white-space: pre-wrap; } +.post-embedded-image { + margin: 0; + padding: var(--post-image-padding); + border: 1px solid var(--post-image-border); + border-radius: var(--post-image-radius); + background: var(--post-image-bg); +} + +.post-embedded-image img { + display: block; + max-width: 100%; + height: auto; +} + +.post-embedded-image figcaption { + margin-top: 0.4rem; + font-size: 0.85rem; + color: var(--text); +} + .popup-placeholder { margin-top: 1.5rem; padding: 1rem; diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 8618fcfbf..65d25259b 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -63,6 +63,7 @@ describe("App, authenticated", () => { failedReportRun?: boolean; succeededTeppRun?: boolean; pendingTeppRun?: boolean; + postBody?: string; }) { const statusLabel: Record = { open: "Open", @@ -668,7 +669,7 @@ describe("App, authenticated", () => { jsonResponse({ post_id: "post-1", post_title: "Public post", - post_body: "The full body text.", + post_body: options?.postBody ?? "The full body text.", voc_type_code: "voc", voc_type_label: "Voice of Customer", visibility_code: "public", @@ -1088,6 +1089,23 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); }); + it("shows an embedded invoice image instead of the raw base64 string", async () => { + const tinyPng = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + stubBackend({ + postBody: `

Quote attached.

Please confirm.

`, + }); + render(); + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + + const image = await screen.findByRole("img", { name: /embedded image at character offset/i }); + expect(image).toHaveAttribute("src", `data:image/png;base64,${tinyPng}`); + expect(screen.getByText("Quote attached.")).toBeInTheDocument(); + expect(screen.getByText("Please confirm.")).toBeInTheDocument(); + expect(screen.getByText(/Extract Keyman or ask a question/)).toBeInTheDocument(); + expect(screen.queryByText(new RegExp(tinyPng))).not.toBeInTheDocument(); + }); + it("fetches and renders the post list, then opens a detail popup on click", async () => { const fetchMock = stubBackend(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 35a080aa2..a511f713d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -59,6 +59,7 @@ import { type VocEvidence, } from "./api"; import { LineageDag } from "./LineageDag"; +import { PostBody } from "./PostBody"; import { subgraphForPost } from "./lineageLayout"; import "./App.css"; @@ -119,7 +120,7 @@ function EvidencePanel({ {post && ( <>

{post.post_title}

-

{post.post_body}

+ )} @@ -1245,7 +1246,7 @@ function PostDetailPopup({ {post.visibility_label ?? post.visibility_code} ·{" "} {new Date(post.created_at).toLocaleString()}

-

{post.post_body}

+

요약 (Summary)

diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx new file mode 100644 index 000000000..3ff77b537 --- /dev/null +++ b/frontend/src/PostBody.tsx @@ -0,0 +1,33 @@ +import { splitPostBody, type PostBodySegment } from "./postBodyDisplay"; + +function renderSegment(segment: PostBodySegment, index: number) { + switch (segment.kind) { + case "text": + return ( +

+ {segment.text} +

+ ); + case "image": + return ( +
+ {`Embedded +
+ Image from this post. Extract Keyman or ask a question to read text + inside it. +
+
+ ); + default: { + const _exhaustive: never = segment; + throw new Error(`unexpected post body segment: ${JSON.stringify(_exhaustive)}`); + } + } +} + +export function PostBody({ body }: { body: string }) { + return
{splitPostBody(body).map(renderSegment)}
; +} diff --git a/frontend/src/index.css b/frontend/src/index.css index 5fb331302..53f4db2ac 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -8,6 +8,11 @@ --accent-bg: rgba(170, 59, 255, 0.1); --accent-border: rgba(170, 59, 255, 0.5); --social-bg: rgba(244, 243, 236, 0.5); + --post-body-gap: 0.75rem; + --post-image-padding: 0.75rem; + --post-image-radius: 8px; + --post-image-border: var(--border); + --post-image-bg: var(--code-bg); --shadow: rgba(0, 0, 0, 0.1) 0 10px 15px -3px, rgba(0, 0, 0, 0.05) 0 4px 6px -2px; diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts new file mode 100644 index 000000000..f3092cea6 --- /dev/null +++ b/frontend/src/postBodyDisplay.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { splitPostBody } from "./postBodyDisplay"; + +/** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ +const TINY_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("splitPostBody", () => { + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { + expect(splitPostBody("The full body text.")).toEqual([ + { kind: "text", text: "The full body text." }, + ]); + }); + + it("keeps comparison operators that look like broken HTML", () => { + expect(splitPostBody("qty < 50 and price > 10")).toEqual([ + { kind: "text", text: "qty < 50 and price > 10" }, + ]); + }); + + it("renders a data-URI image as its own segment and never leaks the raw base64 into text", () => { + const html = + `

Quote attached.

Please confirm.

`; + const segments = splitPostBody(html); + + expect(segments).toEqual([ + { kind: "text", text: "Quote attached." }, + { + kind: "image", + src: `data:image/png;base64,${TINY_PNG_B64}`, + mimeType: "image/png", + position: html.indexOf(" { + const html = + `

between

` + + ``; + const segments = splitPostBody(html); + expect(segments.map((segment) => segment.kind)).toEqual(["image", "text", "image"]); + expect(segments[1]).toEqual({ kind: "text", text: "between" }); + expect(segments[0]?.kind === "image" && segments[0].position).toBe(0); + expect(segments[2]?.kind === "image" && segments[2].position).toBeGreaterThan(0); + }); + + it("tells the operator to re-export when the base64 payload is not decodable", () => { + const html = ''; + expect(splitPostBody(html)).toEqual([ + { + kind: "text", + text: "Embedded image could not be decoded. Re-export the source post and open it again.", + }, + ]); + }); + + it("does not turn a remote http img into a loaded image", () => { + const html = '

See

end

'; + const segments = splitPostBody(html); + expect(segments.every((segment) => segment.kind === "text")).toBe(true); + expect(segments.map((segment) => (segment.kind === "text" ? segment.text : "")).join(" ")).toContain( + "See", + ); + expect(JSON.stringify(segments)).not.toContain("https://example.test"); + }); +}); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts new file mode 100644 index 000000000..c6ea29fdd --- /dev/null +++ b/frontend/src/postBodyDisplay.ts @@ -0,0 +1,72 @@ +/** + * Split a raw `post_body` into text and in-place data-URI images. + * + * The popup used to dump the source string, so a buyer who opened a post + * with an embedded invoice saw a base64 wall instead of the picture. + * Only `data:image/...;base64,...` payloads are turned into images — + * remote `http(s)` img tags are stripped, never fetched. + */ + +export type PostBodySegment = + | { kind: "text"; text: string } + | { kind: "image"; src: string; mimeType: string; position: number }; + +const DATA_URI_IMG = + /]*\bsrc\s*=\s*["']data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["'][^>]*>/gi; + +const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; + +const UNDECODEABLE_IMAGE = + "Embedded image could not be decoded. Re-export the source post and open it again."; + +function stripHtmlTags(text: string): string { + return text.replace(HTML_TAG, " ").replace(/\s+/g, " ").trim(); +} + +function isDecodableBase64(raw: string): boolean { + if (raw.length === 0) { + return false; + } + try { + atob(raw); + return true; + } catch { + return false; + } +} + +function pushText(segments: PostBodySegment[], raw: string): void { + const text = stripHtmlTags(raw); + if (text) { + segments.push({ kind: "text", text }); + } +} + +export function splitPostBody(body: string): PostBodySegment[] { + const segments: PostBodySegment[] = []; + const pattern = new RegExp(DATA_URI_IMG.source, "gi"); + let lastIndex = 0; + let match = pattern.exec(body); + while (match !== null) { + pushText(segments, body.slice(lastIndex, match.index)); + const mimeType = match[1]; + const rawB64 = match[2].replace(/\s+/g, ""); + if (isDecodableBase64(rawB64)) { + segments.push({ + kind: "image", + src: `data:${mimeType};base64,${rawB64}`, + mimeType, + position: match.index, + }); + } else { + segments.push({ kind: "text", text: UNDECODEABLE_IMAGE }); + } + lastIndex = match.index + match[0].length; + match = pattern.exec(body); + } + pushText(segments, body.slice(lastIndex)); + if (segments.length === 0) { + return [{ kind: "text", text: body }]; + } + return segments; +} diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 5f70c6064..48bf6e481 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -55,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "0.86.0" +__version__ = "0.86.1" diff --git a/pyproject.toml b/pyproject.toml index 0393b774a..eb2e26318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "0.86.0" +version = "0.86.1" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/uv.lock b/uv.lock index b302b1250..c759df267 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "0.86.0" +version = "0.86.1" source = { virtual = "." } dependencies = [ { name = "certifi" },