From 79e064c034ee25834e13726905f307cc9c0571fe Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:28:48 +0000 Subject: [PATCH] fix(ui): parse embedded post images with an HTML parser Invoice-like HTML with alt="Invoice > 1000" no longer dumps the base64 wall. The popup, extract_base64_images, and chunk_by_dom share a raster allowlist and one synthetic invoice fixture (ADR 0019). Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 11 + docs/adr/0020-embedded-image-html-parser.md | 76 ++++++ docs/image-content-schema.md | 25 +- frontend/package.json | 2 +- frontend/src/PostBody.tsx | 5 +- frontend/src/postBodyDisplay.test.ts | 59 +++++ frontend/src/postBodyDisplay.ts | 224 +++++++++++++++--- lineageweave/__init__.py | 2 +- lineageweave/chunking.py | 31 ++- lineageweave/embedded_image_payload.py | 95 ++++++++ lineageweave/image_content.py | 71 ++++-- pyproject.toml | 2 +- .../synthetic_invoice_embedded_image.html | 10 + tests/test_embedded_image_payload.py | 44 ++++ tests/test_image_content.py | 42 ++++ uv.lock | 2 +- 17 files changed, 616 insertions(+), 87 deletions(-) create mode 100644 docs/adr/0020-embedded-image-html-parser.md create mode 100644 lineageweave/embedded_image_payload.py create mode 100644 tests/fixtures/synthetic_invoice_embedded_image.html create mode 100644 tests/test_embedded_image_payload.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3a4d0ac4a..1ae3e4a80 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) renders each `data:image` payload in document order so the buyer sees the picture, not the base64 string; GET does not call the vision client. | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) and `extract_base64_images` parse with the same HTML rules as `chunk_by_dom` (ADR 0020) so invoice-like `alt` values still show the picture; GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | | `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | diff --git a/CHANGELOG.md b/CHANGELOG.md index d30ae14aa..362b240b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.86.3] - 2026-08-16 + +### Fixed + +- Opening a post whose embedded picture uses invoice-like HTML + (`alt="Invoice > 1000"`, unquoted `width`, newlines in the base64) + now shows the picture. The raw payload no longer returns when a + remote-only or SVG tag is the whole body. Re-export as PNG or JPEG + if the type is rejected. The popup, `extract_base64_images`, and + `chunk_by_dom` share one raster allowlist (ADR 0020). + ## [0.86.2] - 2026-08-16 ### Fixed diff --git a/docs/adr/0020-embedded-image-html-parser.md b/docs/adr/0020-embedded-image-html-parser.md new file mode 100644 index 000000000..aaa8bedc6 --- /dev/null +++ b/docs/adr/0020-embedded-image-html-parser.md @@ -0,0 +1,76 @@ +# ADR 0020 — Embedded images use an HTML parser and a raster allowlist + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +PR #140 stopped the product popup from dumping a well-formed +`data:image/png;base64,...` invoice as a base64 wall. The splitter and +`extract_base64_images` still used a `[^>]*` regex. Real invoice HTML +puts `>` inside `alt` or `title` *before* `src`. That shape is legal +HTML (WHATWG, n.d.) and is what `chunk_by_dom` already parses. The regex +missed the picture and put the payload back into the text node. + +The same open MIME class `image/[a-zA-Z0-9.+-]+` accepted +`image/svg+xml`. SVG-as-`` does not run script in current browsers, +but the regex also fed the vision channel. `atob` and +`b64decode(validate=True)` already disagreed on padding. + +ADR 0019 is the R&R catalog-identity decision. This decision is the +viewer/extractor parse contract. + +Persistence of OCR under the figure (Li et al., 2023; Radford et al., +2021) is still the next buyer slice. It must not land on a splitter that +fails the HTML the buyer actually opens. + +## Decision + +The popup (`splitPostBody`), `extract_base64_images`, and `chunk_by_dom` +share one decode helper (`lineageweave.embedded_image_payload`): + +1. Parse with an HTML parser (`DOMParser` in the browser, `html.parser` + in Python). Comments, ` +

Please confirm.

diff --git a/tests/test_embedded_image_payload.py b/tests/test_embedded_image_payload.py new file mode 100644 index 000000000..375cdfbea --- /dev/null +++ b/tests/test_embedded_image_payload.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import base64 + +from lineageweave.embedded_image_payload import ( + decode_data_uri_image, + looks_like_raster_image, + source_offset, +) + +_TINY_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" +) +_TINY_PNG = base64.b64decode(_TINY_PNG_B64) + + +def test_looks_like_raster_image_accepts_png_signature() -> None: + assert looks_like_raster_image("image/png", _TINY_PNG) is True + + +def test_looks_like_raster_image_rejects_ascii_labeled_as_png() -> None: + assert looks_like_raster_image("image/png", b"Hello") is False + + +def test_decode_data_uri_image_rejects_svg_and_remote_src() -> None: + assert decode_data_uri_image("https://example.test/invoice.png") is None + assert ( + decode_data_uri_image( + "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==" + ) + is None + ) + + +def test_decode_data_uri_image_accepts_newlines_inside_png_payload() -> None: + wrapped = f"data:image/png;base64,{_TINY_PNG_B64[:24]}\n{_TINY_PNG_B64[24:]}" + decoded = decode_data_uri_image(wrapped) + assert decoded == ("image/png", _TINY_PNG) + + +def test_source_offset_maps_htmlparser_getpos() -> None: + source = "ab\ncd" + assert source_offset(source, 1, 0) == 0 + assert source_offset(source, 2, 1) == 4 diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 033202be6..26fdd727c 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -1,9 +1,12 @@ from __future__ import annotations import base64 +from pathlib import Path import pytest +from lineageweave.chunking import chunk_by_dom +from lineageweave.embedded_image_payload import decode_data_uri_image from lineageweave.image_content import ( ImageContentClient, ImageDescriptionParseError, @@ -55,6 +58,45 @@ def test_extract_base64_images_empty_document_yields_no_images() -> None: assert extract_base64_images("

No images here.

") == [] +def test_extract_base64_images_skips_svg_and_unpadded_payloads() -> None: + svg = ( + '' + ) + assert extract_base64_images(svg) == [] + assert extract_base64_images('') == [] + assert decode_data_uri_image(f"data:image/png;base64,{_TINY_PNG_B64}") is not None + + +def test_invoice_fixture_is_one_visible_png_for_every_extractor() -> None: + """Outlook-style invoice HTML must not resurrect the base64 wall. + + The same file is read by the TypeScript popup splitter. All three + extractors must see one raster PNG, ignore the commented copy, the + remote URL, the SVG, and the CSS background, and keep surrounding + sentences readable. + """ + html = (Path(__file__).parent / "fixtures" / "synthetic_invoice_embedded_image.html").read_text( + encoding="utf-8" + ) + images = extract_base64_images(html) + chunks = chunk_by_dom(html) + image_chunks = [chunk for chunk in chunks if chunk.unit_type == "image"] + text = " ".join(chunk.text for chunk in chunks if chunk.unit_type == "dom") + + assert len(images) == 1 + assert len(image_chunks) == 1 + assert images[0].mime_type == "image/png" + assert images[0].data == base64.b64decode(_TINY_PNG_B64) + assert images[0].position == html.find(") + assert None: content = "TEXT: Quarterly Budget Report\nCAPTION: A printed report cover page.\nTAGS: document, report, text" description = _parse_description(content) diff --git a/uv.lock b/uv.lock index e1d2860cf..179628fdd 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "0.86.2" +version = "0.86.3" source = { virtual = "." } dependencies = [ { name = "certifi" },