From 090685d8dbba87b38d11e59c004ea7472198dfac Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 27 Aug 2026 20:32:16 +0200 Subject: [PATCH] deps: read a page as far as it decoded, and still refuse one that did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reader v0.6.0 keeps what a broken filter chain did produce and says so, instead of returning a fragment with a nil error. Taking it without deciding what this package wants would have changed the contract by accident in both directions, so the decision is made here and written down. A page whose content decoded part of the way is now read that far. 263 streams in 212 of the 1 633 real forms — 13.0% of them — cannot be decoded cleanly, and half a page of text is worth more to somebody searching than none of it. A page that decoded no bytes at all stays an error. That is not symmetry for its own sake: go-pdfkit/latex turns a document into LaTeX, and a conversion that silently produces an empty document from an unreadable page is worse than one that says it could not read it. Its own test caught this the moment the dependency moved, which is the test doing its job. The pair of tests states both halves. A stream of 108 bytes holding 301 pieces of text, cut to 80, gives 147 of them and no error; the same stream declared as Flate and holding no Flate at all still gives an error. Also on the fixes this brings: the /Crypt filter, which 209 files — 12.8% of the real forms — carry and which used to make 209 of them decode to nothing readable; a second map-order non-determinism, in indexObjectStreams; and files whose crypt filters say /Identity, which need no password and now open. Co-Authored-By: Claude Opus 5 --- extract.go | 14 ++++++++- extract_test.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/extract.go b/extract.go index 070dfe5..5adb2ac 100644 --- a/extract.go +++ b/extract.go @@ -61,11 +61,23 @@ func Runs(d *reader.Document, page int) ([]Run, error) { } // walk reads a page's content stream once, keeping what it finds. +// +// A page whose content decoded only part of the way is read as far as it went: +// 263 streams in 212 of the 1 633 real forms in the corpus cannot be decoded +// cleanly, and half a page of text is worth more to somebody searching than +// none of it. A page that decoded no bytes at all is a different answer and +// stays an error — a conversion that silently produces an empty document from +// an unreadable page is worse than one that says it could not read it, and +// go-pdfkit/latex relies on being told. func walk(d *reader.Document, page int) (*extractor, error) { - content, err := d.PageContent(page) + dec, err := d.PageContentDecoded(page) if err != nil { return nil, err } + if len(dec.Data) == 0 && dec.Cause != nil { + return nil, dec.Cause + } + content := dec.Data p, _ := d.Page(page) resources, _ := d.GetDict(p, "Resources") e := &extractor{doc: d, fonts: map[int]*pdffont.Font{}} diff --git a/extract_test.go b/extract_test.go index 89539a1..de72a78 100644 --- a/extract_test.go +++ b/extract_test.go @@ -1,6 +1,8 @@ package extract import ( + "bytes" + "compress/zlib" "strings" "testing" @@ -390,3 +392,77 @@ func TestAWordGapInAFontWithNoSpaceOfItsOwn(t *testing.T) { t.Errorf("the page says %q", text) } } + +// filteredPage builds a page whose content stream carries the filter and bytes +// the test names, which pageWith does not allow: it writes the content plain. +func filteredPage(t *testing.T, filter reader.Name, raw []byte) *reader.Document { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + page := w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), + reader.Integer(200), reader.Integer(200)}, + "Contents": w.Add(&reader.Stream{ + Dict: reader.Dict{"Filter": filter}, Raw: raw}), + "Resources": reader.Dict{"Font": reader.Dict{"F1": w.Add(simpleFont(w))}}, + }) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + out, err := w.Finish(reader.Dict{"Root": w.Add(reader.Dict{ + "Type": reader.Name("Catalog"), "Pages": pagesRef})}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestAPageThatDecodedPartOfTheWayIsReadThatFar(t *testing.T) { + // reader v0.6.0 keeps what a broken filter chain did produce. A page whose + // content inflates part of the way is read that far: 263 streams in 212 of + // the 1 633 real forms cannot be decoded cleanly, and half a page of text + // is worth more to somebody searching than none of it. + var buf bytes.Buffer + zw := zlib.NewWriter(&buf) + text := "BT /F1 12 Tf 10 100 Td (hello) Tj ET " + for i := 0; i < 300; i++ { + text += "BT /F1 12 Tf 10 100 Td (padding) Tj ET " + } + if _, err := zw.Write([]byte(text)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + // The whole stream is 108 bytes and holds 301 pieces of text. Keeping 80 + // of them inflates part of the way. + whole, err := Runs(filteredPage(t, "FlateDecode", buf.Bytes()), 1) + if err != nil { + t.Fatal(err) + } + runs, err := Runs(filteredPage(t, "FlateDecode", buf.Bytes()[:80]), 1) + if err != nil { + t.Fatalf("a page that decoded part of the way came back as an error: %v", err) + } + switch { + case len(runs) == 0: + t.Error("nothing was read from a page that decoded part of the way") + case len(runs) >= len(whole): + t.Errorf("the cut stream gave %d runs and the whole one %d: nothing was cut", + len(runs), len(whole)) + } +} + +func TestAPageThatDecodedNothingIsStillAnError(t *testing.T) { + // The other side of the same rule, and the one go-pdfkit/latex relies on: + // a page whose content yields no bytes at all is reported rather than + // turned into an empty document. + d := filteredPage(t, "FlateDecode", []byte("not compressed at all")) + if _, err := Runs(d, 1); err == nil { + t.Error("a page whose content will not decode read without error") + } +} diff --git a/go.mod b/go.mod index e70e3ed..2544fd8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.4 require ( github.com/go-opentype/opentype v0.9.0 github.com/go-pdfkit/pdffont v0.3.0 - github.com/go-pdfkit/reader v0.5.0 + github.com/go-pdfkit/reader v0.6.0 ) require github.com/go-opentype/fonts v0.8.0 diff --git a/go.sum b/go.sum index bbd4fd0..ec9a233 100644 --- a/go.sum +++ b/go.sum @@ -4,5 +4,5 @@ github.com/go-opentype/opentype v0.9.0 h1:GFgcJ3nwTDp4NJr5O+Paw7lhZx5Jv/R+noZwvh github.com/go-opentype/opentype v0.9.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= github.com/go-pdfkit/pdffont v0.3.0 h1:G5DKcAmsZJ0e17QhSrcUaL7PKEXKjUx4P/iGMl7bAbI= github.com/go-pdfkit/pdffont v0.3.0/go.mod h1:bfmNLna1l1CljNX/Utg55YzFylovfmI7sJnvgA3bzKI= -github.com/go-pdfkit/reader v0.5.0 h1:DaJ5C6eKXPRG9Cdu+olA/0KLmsQgyHEC1coteQQuBxU= -github.com/go-pdfkit/reader v0.5.0/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8= +github.com/go-pdfkit/reader v0.6.0 h1:KAabNOYUcTlZlNBTbG9bEhWP1NiZhjuhzUdavdTdfes= +github.com/go-pdfkit/reader v0.6.0/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8=