diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 26c817cde..a8d082f0b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl) | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) renders each `data:image` payload in document order so the buyer sees the picture, not the base64 string; GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | | `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf4dc62a..fd8717513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.86.1] - 2026-08-16 + +### Changed + +- Opening a post or its evidence panel now shows each embedded + `data:image` picture in document order, with the surrounding sentences + as text. The raw base64 string is no longer dumped into the popup. + Remote `http(s)` image URLs stay unloaded. After `make seed`, a post + whose body includes a data-URI image shows the picture; Extract Keyman + or Ask still runs OCR on that image for search. + ## [0.86.0] - 2026-08-16 ### Added diff --git a/docs/image-content-schema.md b/docs/image-content-schema.md index f023d56a0..4dae03dcf 100644 --- a/docs/image-content-schema.md +++ b/docs/image-content-schema.md @@ -82,6 +82,18 @@ picture sat relative to the surrounding paragraphs." | `chunk_position` | `integer not null` | 0-based index among ALL of this document's chunks (text and image together) -- matches `Chunk.index` from `chunk_by_dom` | | primary key | `(source_document_id, chunk_position)` | one image slot per position per document | +## Viewer contract (before persistence exists) + +The demo popup does not yet read these tables. It splits the live +`post_body` the same way `extract_base64_images` does: each +`data:image/...;base64,...` payload becomes an `` at its original +character offset, and the surrounding HTML is shown as text. A buyer who +opens the post sees the picture that sat between the paragraphs, not the +base64 wall. Remote `src="https://..."` tags are stripped, never fetched. +OCR, caption, and tag search still require the vision client on extract / +Ask (Li et al., 2023; Radford et al., 2021) and, in a real deployment, +the tables below. + ## Query shapes this supports - **"Find images whose extracted text or tags match a search query, then @@ -105,3 +117,17 @@ picture sat relative to the surrounding paragraphs." ON CONFLICT DO NOTHING` before the provider call, or a short-lived lease row) to close that race; this schema documents the storage guarantee, not that concurrency control. + +## References + +Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, Z., +& Wei, F. (2023). TrOCR: Transformer-based optical character recognition +with pre-trained models. *Proceedings of the AAAI Conference on Artificial +Intelligence, 37*(11), 13094–13102. https://doi.org/10.1609/aaai.v37i11.26538 + +Radford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., +Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., & Sutskever, I. +(2021). Learning transferable visual models from natural language +supervision. In M. Meila & T. Zhang (Eds.), *Proceedings of the 38th +International Conference on Machine Learning* (pp. 8748–8763). PMLR. +https://proceedings.mlr.press/v139/radford21a.html diff --git a/frontend/package.json b/frontend/package.json index dac241738..803acd91b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.86.0", + "version": "0.86.1", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index 8f38b4dd0..b3fab25d1 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -108,9 +108,36 @@ } .post-body { + display: flex; + flex-direction: column; + gap: var(--post-body-gap); +} + +.post-body-text { + margin: 0; white-space: pre-wrap; } +.post-embedded-image { + margin: 0; + padding: var(--post-image-padding); + border: 1px solid var(--post-image-border); + border-radius: var(--post-image-radius); + background: var(--post-image-bg); +} + +.post-embedded-image img { + display: block; + max-width: 100%; + height: auto; +} + +.post-embedded-image figcaption { + margin-top: 0.4rem; + font-size: 0.85rem; + color: var(--text); +} + .popup-placeholder { margin-top: 1.5rem; padding: 1rem; diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 8618fcfbf..65d25259b 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -63,6 +63,7 @@ describe("App, authenticated", () => { failedReportRun?: boolean; succeededTeppRun?: boolean; pendingTeppRun?: boolean; + postBody?: string; }) { const statusLabel: Record = { open: "Open", @@ -668,7 +669,7 @@ describe("App, authenticated", () => { jsonResponse({ post_id: "post-1", post_title: "Public post", - post_body: "The full body text.", + post_body: options?.postBody ?? "The full body text.", voc_type_code: "voc", voc_type_label: "Voice of Customer", visibility_code: "public", @@ -1088,6 +1089,23 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); }); + it("shows an embedded invoice image instead of the raw base64 string", async () => { + const tinyPng = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + stubBackend({ + postBody: `

Quote attached.

Please confirm.

`, + }); + render(); + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + + const image = await screen.findByRole("img", { name: /embedded image at character offset/i }); + expect(image).toHaveAttribute("src", `data:image/png;base64,${tinyPng}`); + expect(screen.getByText("Quote attached.")).toBeInTheDocument(); + expect(screen.getByText("Please confirm.")).toBeInTheDocument(); + expect(screen.getByText(/Extract Keyman or ask a question/)).toBeInTheDocument(); + expect(screen.queryByText(new RegExp(tinyPng))).not.toBeInTheDocument(); + }); + it("fetches and renders the post list, then opens a detail popup on click", async () => { const fetchMock = stubBackend(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 35a080aa2..a511f713d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -59,6 +59,7 @@ import { type VocEvidence, } from "./api"; import { LineageDag } from "./LineageDag"; +import { PostBody } from "./PostBody"; import { subgraphForPost } from "./lineageLayout"; import "./App.css"; @@ -119,7 +120,7 @@ function EvidencePanel({ {post && ( <>

{post.post_title}

-

{post.post_body}

+ )} @@ -1245,7 +1246,7 @@ function PostDetailPopup({ {post.visibility_label ?? post.visibility_code} ·{" "} {new Date(post.created_at).toLocaleString()}

-

{post.post_body}

+

요약 (Summary)

diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx new file mode 100644 index 000000000..3ff77b537 --- /dev/null +++ b/frontend/src/PostBody.tsx @@ -0,0 +1,33 @@ +import { splitPostBody, type PostBodySegment } from "./postBodyDisplay"; + +function renderSegment(segment: PostBodySegment, index: number) { + switch (segment.kind) { + case "text": + return ( +

+ {segment.text} +

+ ); + case "image": + return ( +
+ {`Embedded +
+ Image from this post. Extract Keyman or ask a question to read text + inside it. +
+
+ ); + default: { + const _exhaustive: never = segment; + throw new Error(`unexpected post body segment: ${JSON.stringify(_exhaustive)}`); + } + } +} + +export function PostBody({ body }: { body: string }) { + return
{splitPostBody(body).map(renderSegment)}
; +} diff --git a/frontend/src/index.css b/frontend/src/index.css index 5fb331302..53f4db2ac 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -8,6 +8,11 @@ --accent-bg: rgba(170, 59, 255, 0.1); --accent-border: rgba(170, 59, 255, 0.5); --social-bg: rgba(244, 243, 236, 0.5); + --post-body-gap: 0.75rem; + --post-image-padding: 0.75rem; + --post-image-radius: 8px; + --post-image-border: var(--border); + --post-image-bg: var(--code-bg); --shadow: rgba(0, 0, 0, 0.1) 0 10px 15px -3px, rgba(0, 0, 0, 0.05) 0 4px 6px -2px; diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts new file mode 100644 index 000000000..f3092cea6 --- /dev/null +++ b/frontend/src/postBodyDisplay.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { splitPostBody } from "./postBodyDisplay"; + +/** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ +const TINY_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("splitPostBody", () => { + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { + expect(splitPostBody("The full body text.")).toEqual([ + { kind: "text", text: "The full body text." }, + ]); + }); + + it("keeps comparison operators that look like broken HTML", () => { + expect(splitPostBody("qty < 50 and price > 10")).toEqual([ + { kind: "text", text: "qty < 50 and price > 10" }, + ]); + }); + + it("renders a data-URI image as its own segment and never leaks the raw base64 into text", () => { + const html = + `

Quote attached.

Please confirm.

`; + const segments = splitPostBody(html); + + expect(segments).toEqual([ + { kind: "text", text: "Quote attached." }, + { + kind: "image", + src: `data:image/png;base64,${TINY_PNG_B64}`, + mimeType: "image/png", + position: html.indexOf(" { + const html = + `

between

` + + ``; + const segments = splitPostBody(html); + expect(segments.map((segment) => segment.kind)).toEqual(["image", "text", "image"]); + expect(segments[1]).toEqual({ kind: "text", text: "between" }); + expect(segments[0]?.kind === "image" && segments[0].position).toBe(0); + expect(segments[2]?.kind === "image" && segments[2].position).toBeGreaterThan(0); + }); + + it("tells the operator to re-export when the base64 payload is not decodable", () => { + const html = ''; + expect(splitPostBody(html)).toEqual([ + { + kind: "text", + text: "Embedded image could not be decoded. Re-export the source post and open it again.", + }, + ]); + }); + + it("does not turn a remote http img into a loaded image", () => { + const html = '

See

end

'; + const segments = splitPostBody(html); + expect(segments.every((segment) => segment.kind === "text")).toBe(true); + expect(segments.map((segment) => (segment.kind === "text" ? segment.text : "")).join(" ")).toContain( + "See", + ); + expect(JSON.stringify(segments)).not.toContain("https://example.test"); + }); +}); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts new file mode 100644 index 000000000..c6ea29fdd --- /dev/null +++ b/frontend/src/postBodyDisplay.ts @@ -0,0 +1,72 @@ +/** + * Split a raw `post_body` into text and in-place data-URI images. + * + * The popup used to dump the source string, so a buyer who opened a post + * with an embedded invoice saw a base64 wall instead of the picture. + * Only `data:image/...;base64,...` payloads are turned into images — + * remote `http(s)` img tags are stripped, never fetched. + */ + +export type PostBodySegment = + | { kind: "text"; text: string } + | { kind: "image"; src: string; mimeType: string; position: number }; + +const DATA_URI_IMG = + /]*\bsrc\s*=\s*["']data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["'][^>]*>/gi; + +const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; + +const UNDECODEABLE_IMAGE = + "Embedded image could not be decoded. Re-export the source post and open it again."; + +function stripHtmlTags(text: string): string { + return text.replace(HTML_TAG, " ").replace(/\s+/g, " ").trim(); +} + +function isDecodableBase64(raw: string): boolean { + if (raw.length === 0) { + return false; + } + try { + atob(raw); + return true; + } catch { + return false; + } +} + +function pushText(segments: PostBodySegment[], raw: string): void { + const text = stripHtmlTags(raw); + if (text) { + segments.push({ kind: "text", text }); + } +} + +export function splitPostBody(body: string): PostBodySegment[] { + const segments: PostBodySegment[] = []; + const pattern = new RegExp(DATA_URI_IMG.source, "gi"); + let lastIndex = 0; + let match = pattern.exec(body); + while (match !== null) { + pushText(segments, body.slice(lastIndex, match.index)); + const mimeType = match[1]; + const rawB64 = match[2].replace(/\s+/g, ""); + if (isDecodableBase64(rawB64)) { + segments.push({ + kind: "image", + src: `data:${mimeType};base64,${rawB64}`, + mimeType, + position: match.index, + }); + } else { + segments.push({ kind: "text", text: UNDECODEABLE_IMAGE }); + } + lastIndex = match.index + match[0].length; + match = pattern.exec(body); + } + pushText(segments, body.slice(lastIndex)); + if (segments.length === 0) { + return [{ kind: "text", text: body }]; + } + return segments; +} diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 5f70c6064..48bf6e481 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -55,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "0.86.0" +__version__ = "0.86.1" diff --git a/pyproject.toml b/pyproject.toml index 0393b774a..eb2e26318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "0.86.0" +version = "0.86.1" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/uv.lock b/uv.lock index b302b1250..c759df267 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "0.86.0" +version = "0.86.1" source = { virtual = "." } dependencies = [ { name = "certifi" },