From 8bfdc6cdb923f96648f1a4d5cc1abfe3ca22ccfe Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 27 Aug 2026 19:56:21 +0200 Subject: [PATCH] Draw what a damaged stream did give MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reader v0.6.0 stopped flateDecode from lying: a damaged stream's inflated prefix used to come back with a nil error, so a caller could not tell a clean decode from a fragment. Fixing that makes the strict Decode this package used return nothing where it used to return the fragment — so taking the new reader without changing anything here is a REGRESSION, and the measurement says so: plots/P12positive01.pdf 188 770 inked pixels -> 0 (a blank page) uk-govuk/apply-for-help-... 48 455 -> 38 695 uk-govuk/form-n1 (x3) 42 347 -> 40 771, and two like it Every stream is now read through DecodeStreamRecovering, and every one of those pages comes back exactly as it was: 188 770 -> 0 -> 188 770. The recovery was already happening; it was happening by accident, inside a function that claimed success. Now it is asked for. WHY THIS IS THE RIGHT BEHAVIOUR AND NOT MERELY THE OLD ONE 263 streams in 212 of the 1 633 real forms — 13.0% of them — cannot be decoded cleanly. This package already draws a page as far as it got when its time runs out; a damaged stream is that same situation arriving another way, and half a figure is worth more to somebody reading a form than none of it. WHAT MAKES IT SAFE Bytes that no filter decoded are never painted. reader.Decoded keeps them in Undecoded, set INSTEAD of Data rather than beside it, so reading Data cannot reach them — a type doing the work rather than a flag anyone has to remember. That guarantee is why this change is possible at all: the same reader release found that a fax with a bad /Columns had been putting its encoded bytes where a caller would paint them, which is precisely the defect #20 had just fixed here from the other side. Two cases are still refused rather than salvaged, and the tests say so: a page whose /Contents is filtered as an image, which would run a JPEG as operators; and a stream whose filter nothing can apply, which yields no decoded bytes at all. MEASURED, THE THREE STATES SEPARATED reader v0.5.0, then v0.6.0 alone, then v0.6.0 with this change, over the first page of all 1 633 real forms and 1 334 arXiv files, each configuration run twice and identical to itself both times: real forms v0.5.0 -> v0.6.0: 6 pages change, 5 of them with less ink v0.6.0 -> +this: 4 change, all 4 back to what v0.5.0 drew v0.5.0 -> +this: 2 change, both by under thirty pixels arXiv v0.5.0 -> v0.6.0: 4 change, one of them to a blank page v0.6.0 -> +this: 3 change, the blank page back to 188 770 v0.5.0 -> +this: 2 change One arXiv figure disagrees in the other direction — 2512.03312/Fig3.pdf draws 24 913 pixels under v0.5.0, 38 525 under v0.6.0 alone, and 24 913 again with this. Which of the two is right has not been established, and it is written down here rather than left out. 100% statement coverage, go vet and -race clean, nine cross-compile targets. Co-Authored-By: Claude Opus 5 --- annots.go | 2 +- calculator.go | 2 +- funckinds.go | 2 +- go.mod | 2 +- go.sum | 4 +-- image.go | 8 +++-- mesh.go | 2 +- pattern.go | 2 +- render.go | 23 +++++++++++-- salvage_test.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ softmask.go | 2 +- space.go | 2 +- state.go | 21 ++++++++++++ text.go | 2 +- xobject.go | 2 +- 15 files changed, 144 insertions(+), 17 deletions(-) create mode 100644 salvage_test.go diff --git a/annots.go b/annots.go index 4d79812..688ffd1 100644 --- a/annots.go +++ b/annots.go @@ -119,7 +119,7 @@ func (r *renderer) appearanceOf(dict reader.Dict) (*reader.Stream, bool) { // rectangle — which is what makes a stamp put down at an angle land inside the // box that was drawn for it. func (r *renderer) drawAppearance(base gstate, stream *reader.Stream, rect [4]float64) { - content, img, err := r.doc.DecodeStream(stream) + content, img, err := r.salvaged(stream) if err != nil || img != "" { return } diff --git a/calculator.go b/calculator.go index 6799ca8..83db92b 100644 --- a/calculator.go +++ b/calculator.go @@ -37,7 +37,7 @@ func (r *renderer) readCalculator(base functionBase, dict reader.Dict, stream *r if stream == nil || len(base.rng) < 2 { return nil } - data, img, err := r.doc.DecodeStream(stream) + data, img, err := r.salvaged(stream) if err != nil || img != "" { return nil } diff --git a/funckinds.go b/funckinds.go index ee79a3b..6f7f308 100644 --- a/funckinds.go +++ b/funckinds.go @@ -26,7 +26,7 @@ func (r *renderer) readSampled(base functionBase, dict reader.Dict, stream *read if stream == nil || len(base.rng) < 2 { return nil } - data, img, err := r.doc.DecodeStream(stream) + data, img, err := r.salvaged(stream) if err != nil || img != "" { return nil } diff --git a/go.mod b/go.mod index 97793b5..85891a6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/go-gfx/gfx v0.10.0 github.com/go-opentype/fonts v0.9.0 github.com/go-opentype/opentype v0.10.0 - github.com/go-pdfkit/reader v0.5.0 + github.com/go-pdfkit/reader v0.6.0 ) require github.com/go-pdfkit/pdffont v0.3.0 diff --git a/go.sum b/go.sum index 39b7e36..323b803 100644 --- a/go.sum +++ b/go.sum @@ -6,5 +6,5 @@ github.com/go-opentype/opentype v0.10.0 h1:cZVMZ3RVkcijXmxlmpqyVkjlNc33aSB2ugKVW github.com/go-opentype/opentype v0.10.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= github.com/go-pdfkit/pdffont v0.3.0 h1:G5DKcAmsZJ0e17QhSrcUaL7PKEXKjUx4P/iGMl7bAbI= github.com/go-pdfkit/pdffont v0.3.0/go.mod h1:bfmNLna1l1CljNX/Utg55YzFylovfmI7sJnvgA3bzKI= -github.com/go-pdfkit/reader v0.5.0 h1:DaJ5C6eKXPRG9Cdu+olA/0KLmsQgyHEC1coteQQuBxU= -github.com/go-pdfkit/reader v0.5.0/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8= +github.com/go-pdfkit/reader v0.6.0 h1:KAabNOYUcTlZlNBTbG9bEhWP1NiZhjuhzUdavdTdfes= +github.com/go-pdfkit/reader v0.6.0/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8= diff --git a/image.go b/image.go index f23cb69..ba065f1 100644 --- a/image.go +++ b/image.go @@ -104,8 +104,12 @@ func (r *renderer) decodeImage(dict reader.Dict, raw []byte, resources reader.Di if w <= 0 || h <= 0 || w*h > maxImagePixels { return nil } - data, imageFilter, err := reader.Decode(dict, raw, r.doc.Resolver()) - if err != nil { + // An image whose filter chain broke part way gives the rows it managed, + // which is what a viewer shows for a truncated scan. Bytes no filter + // decoded are in a field this cannot reach. + dec := reader.DecodeRecovering(dict, raw, r.doc.Resolver()) + data, imageFilter := dec.Data, dec.Image + if len(data) == 0 { return nil } if mask, ok := reader.ToBool(resolve(r.doc, dict.Get("ImageMask"))); ok && mask { diff --git a/mesh.go b/mesh.go index 2001729..93492a2 100644 --- a/mesh.go +++ b/mesh.go @@ -47,7 +47,7 @@ const patchSteps = 16 // readMesh reads the vertices or patches a mesh shading's stream holds. func (r *renderer) readMesh(sh *shading, stream *reader.Stream) *mesh { - data, img, err := r.doc.DecodeStream(stream) + data, img, err := r.salvaged(stream) if err != nil || img != "" { return nil } diff --git a/pattern.go b/pattern.go index a5a9a9d..9d5ba6f 100644 --- a/pattern.go +++ b/pattern.go @@ -61,7 +61,7 @@ func (r *renderer) readPattern(name reader.Name, resources reader.Dict) *pattern if !isStream { return nil } - data, img, err := r.doc.DecodeStream(stream) + data, img, err := r.salvaged(stream) if err != nil || img != "" { return nil } diff --git a/render.go b/render.go index bcbc154..c2cdb85 100644 --- a/render.go +++ b/render.go @@ -127,10 +127,27 @@ func Page(d *reader.Document, i int, opt Options) (*raster.Image, error) { img := raster.New(w, h) fill(img, opt.background()) - content, err := d.PageContent(i) - if err != nil { - return nil, err + // A page's content is drawn as far as it decoded. 263 streams in 212 of + // the 1 633 real forms cannot be decoded cleanly, and a strict read hands + // back nothing for all of them; this package already draws a page as far as + // it got when its time runs out, and a damaged stream is the same + // situation arriving another way. + // + // Nothing that no filter decoded is ever run: the reader keeps those bytes + // in Undecoded, set instead of Data rather than beside it, so taking Data + // cannot reach them. A page whose /Contents is filtered as an image — a + // JPEG where operators should be — arrives here with Data empty and a + // cause, and is refused rather than run. + // The error is dropped: PageContentDecoded returns one only when the page + // itself cannot be read, and it was read a few lines above. A /Contents + // that points at an object the file does not hold resolves to nothing + // rather than to an error, and a page with no content is a blank page — + // which is what this drew before and what other readers draw. + dec, _ := d.PageContentDecoded(i) + if len(dec.Data) == 0 && dec.Cause != nil { + return nil, dec.Cause } + content := dec.Data resources, _ := d.GetDict(page, "Resources") r := &renderer{doc: d, img: img, fonts: map[int]*pdfFont{}, softMasks: map[softMaskKey][]uint8{}} if !opt.AllLayers { diff --git a/salvage_test.go b/salvage_test.go new file mode 100644 index 0000000..323d643 --- /dev/null +++ b/salvage_test.go @@ -0,0 +1,85 @@ +package render + +import ( + "bytes" + "compress/zlib" + "testing" + + "github.com/go-pdfkit/reader" +) + +// truncatedFlate compresses s and then cuts the result, so inflating it yields +// a prefix of s and then fails — which is what a damaged stream in a real file +// looks like. +func truncatedFlate(t *testing.T, s string, keep int) []byte { + t.Helper() + var buf bytes.Buffer + zw := zlib.NewWriter(&buf) + if _, err := zw.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + b := buf.Bytes() + if keep >= len(b) { + t.Fatalf("keeping %d of %d bytes cuts nothing", keep, len(b)) + } + return b[:keep] +} + +func TestAPageWhoseContentIsDamagedIsDrawnAsFarAsItDecoded(t *testing.T) { + // A page of many squares, compressed and then cut. The first squares are + // in the part that inflates; the rest are not. Drawing nothing at all + // would be the old behaviour and is the wrong one: 263 streams in 212 of + // the 1 633 real forms cannot be decoded cleanly. + content := "0 g" + for i := 0; i < 200; i++ { + content += " 0 0 10 10 re f 10 0 10 10 re f 20 0 10 10 re f" + } + raw := truncatedFlate(t, content, 40) + + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + page := w.Add(reader.Dict{"Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": nums(0, 0, 40, 20), + "Contents": w.Add(&reader.Stream{ + Dict: reader.Dict{"Filter": reader.Name("FlateDecode")}, Raw: raw})}) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + out, err := w.Finish(reader.Dict{"Root": w.Add(reader.Dict{ + "Type": reader.Name("Catalog"), "Pages": pagesRef})}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + img, err := Page(d, 1, Options{Scale: 1}) + if err != nil { + t.Fatalf("a damaged page came back as an error: %v", err) + } + if inked(img) == 0 { + t.Error("a damaged page drew nothing at all") + } +} + +func TestAFormWhoseFilterIsUnknownIsNotDrawn(t *testing.T) { + // A filter nothing can apply leaves no bytes any filter decoded, so there + // is nothing to draw — and the encoded bytes must not be run as operators. + d := layered(t, nil, func(w *reader.Writer, g []reader.Object) (string, reader.Array, reader.Dict) { + form := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "BBox": nums(0, 0, 100, 100), "Filter": reader.Name("NoSuchDecode"), + }, Raw: []byte("0 g 0 0 100 100 re f")}) + return "/F Do", nil, reader.Dict{"XObject": reader.Dict{"F": form}} + }, 0) + img, err := Page(d, 1, Options{Scale: 1}) + if err != nil { + t.Fatal(err) + } + if ink := inked(img); ink != 0 { + t.Errorf("%d pixels drawn from a stream no filter could decode", ink) + } +} diff --git a/softmask.go b/softmask.go index 111d6e3..3428fcf 100644 --- a/softmask.go +++ b/softmask.go @@ -72,7 +72,7 @@ func (r *renderer) readSoftMask(entry reader.Object, g *gstate, resources reader // drawSoftMask draws the mask's form on paper of its own and reads the mask // off what comes out. func (r *renderer) drawSoftMask(dict reader.Dict, form *reader.Stream, kind reader.Name, g *gstate, resources reader.Dict) []uint8 { - content, img, err := r.doc.DecodeStream(form) + content, img, err := r.salvaged(form) if err != nil || img != "" { return nil } diff --git a/space.go b/space.go index 4b98d83..0541877 100644 --- a/space.go +++ b/space.go @@ -223,7 +223,7 @@ func (r *renderer) lookupTable(o reader.Object) []byte { return s } if stream, ok := reader.ToStream(resolved); ok { - data, img, err := r.doc.DecodeStream(stream) + data, img, err := r.salvaged(stream) if err == nil && img == "" { return data } diff --git a/state.go b/state.go index 83919d3..9a6cd6b 100644 --- a/state.go +++ b/state.go @@ -278,3 +278,24 @@ func (r *renderer) narrow(g *gstate, cov []float64, ox, oy, w, h int, ok bool) { } g.clip = next } + +// salvaged decodes a stream and keeps whatever the filter chain managed to +// produce, which is not the same as keeping whatever bytes are left. +// +// 263 streams in 212 of the 1 633 real forms — 13.0% of them — cannot be +// decoded cleanly, and a strict decode hands back nothing for all 263. This +// package already draws a page as far as it got when its time runs out, and a +// damaged stream is the same situation arriving a different way: half a figure +// is worth more to somebody reading a form than none of it. +// +// What must never happen is bytes no filter decoded being painted as though +// they were content. reader.Decoded keeps those in a separate field, set +// instead of Data rather than beside it, so this reads Data and cannot reach +// them. That is a type doing the work, not a flag anyone has to remember. +func (r *renderer) salvaged(stream *reader.Stream) ([]byte, reader.Name, error) { + dec := r.doc.DecodeStreamRecovering(stream) + if len(dec.Data) == 0 && dec.Cause != nil { + return nil, "", dec.Cause + } + return dec.Data, dec.Image, nil +} diff --git a/text.go b/text.go index e3b39d2..47f1dfe 100644 --- a/text.go +++ b/text.go @@ -263,7 +263,7 @@ func (r *renderer) drawType3Glyph(g *gstate, f *pdfFont, code int, resources rea if !ok { return } - content, img, err := r.doc.DecodeStream(stream) + content, img, err := r.salvaged(stream) if err != nil || img != "" { return } diff --git a/xobject.go b/xobject.go index c4218bb..8b91599 100644 --- a/xobject.go +++ b/xobject.go @@ -45,7 +45,7 @@ func (r *renderer) drawXObject(g *gstate, operands []reader.Object, resources re // drawForm runs a form's own content inside the state that drew it, under its // own matrix and its own bounding box. func (r *renderer) drawForm(g *gstate, stream *reader.Stream, parent reader.Dict) { - content, img, err := r.doc.DecodeStream(stream) + content, img, err := r.salvaged(stream) if err != nil || img != "" { return }