Read a page back as text somebody can actually search

extract.go

@@ -53,7 +53,7 @@ func Text(d *reader.Document, page int) (string, error) {
 
 // Runs is every piece of text on the page, in the order it was drawn.
 func Runs(d *reader.Document, page int) ([]Run, error) {
-	e, err := walk(d, page)
+	e, err := walk(d, page, false)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +69,15 @@ func Runs(d *reader.Document, page int) ([]Run, error) {
 // stays an error — a conversion that silently produces an empty document from
 // an unreadable page is worse than one that says it could not read it, and
 // go-pdfkit/latex relies on being told.
-func walk(d *reader.Document, page int) (*extractor, error) {
+//
+// wantPictures says whether the caller is going to look at the pictures. A
+// page's images have nothing to do with its text, and undoing their filters is
+// the most expensive thing on the page: one arXiv figure holds 378 MB of image
+// once decompressed, so asking that page what it *said* used to cost 1 094 MB
+// and 1.3 seconds spent inflating pictures the answer throws away. Where they
+// land is still noted either way, since that costs a matrix multiply; only the
+// bytes are left alone.
+func walk(d *reader.Document, page int, wantPictures bool) (*extractor, error) {
 	dec, err := d.PageContentDecoded(page)
 	if err != nil {
 		return nil, err
@@ -80,7 +88,7 @@ func walk(d *reader.Document, page int) (*extractor, error) {
 	content := dec.Data
 	p, _ := d.Page(page)
 	resources, _ := d.GetDict(p, "Resources")
-	e := &extractor{doc: d, fonts: map[int]*pdffont.Font{}}
+	e := &extractor{doc: d, fonts: map[int]*pdffont.Font{}, wantPictures: wantPictures}
 	e.run(content, resources, initialState(d, p), 0)
 	return e, nil
 }

fuzz_test.go

@@ -0,0 +1,76 @@
+package extract_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/go-pdfkit/extract"
+	"github.com/go-pdfkit/reader"
+)
+
+// seedDir can be pointed at an adversarial corpus — PDF_SEEDS — such as
+// mozilla's pdf.js test suite, every file of which is there because it broke a
+// reader once. Without it the built-in seeds and anything under testdata still
+// run.
+var seedDir = os.Getenv("PDF_SEEDS")
+
+func addSeeds(f *testing.F, max, cap int) {
+	f.Add([]byte("%PDF-1.7\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n" +
+		"2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj\n" +
+		"3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 99 99]>>endobj\n" +
+		"trailer<</Root 1 0 R>>"))
+	if seedDir == "" {
+		return
+	}
+	ents, err := os.ReadDir(seedDir)
+	if err != nil {
+		return
+	}
+	n := 0
+	for _, e := range ents {
+		info, err := e.Info()
+		if err != nil || info.Size() > int64(cap) || filepath.Ext(e.Name()) != ".pdf" {
+			continue
+		}
+		b, err := os.ReadFile(filepath.Join(seedDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		f.Add(b)
+		if n++; n >= max {
+			return
+		}
+	}
+}
+
+// FuzzExtract reads a page back three ways. The budget is asserted as well as
+// the absence of a panic: what a page says is worked out by running a little
+// program the document wrote, over numbers the document chose, and a small
+// file that costs a large amount of time raises nothing on its own.
+func FuzzExtract(f *testing.F) {
+	addSeeds(f, 300, 40*1024)
+	f.Fuzz(func(t *testing.T, b []byte) {
+		start := time.Now()
+		d, err := reader.Open(b)
+		if err != nil {
+			return
+		}
+		n := d.PageCount()
+		if n > 3 {
+			n = 3
+		}
+		for i := 1; i <= n; i++ {
+			runs, err := extract.Runs(d, i)
+			if err == nil {
+				extract.Assemble(runs)
+			}
+			_, _ = extract.Text(d, i)
+			_, _ = extract.Images(d, i)
+		}
+		if el := time.Since(start); el > 5*time.Second {
+			t.Fatalf("%d bytes took %s over %d pages", len(b), el, n)
+		}
+	})
+}

image.go

@@ -42,7 +42,7 @@ type Image struct {
 // same picture placed twice comes back twice, since where it lands is part of
 // what is being asked for.
 func Images(d *reader.Document, page int) ([]Image, error) {
-	e, err := walk(d, page)
+	e, err := walk(d, page, true)
 	if err != nil {
 		return nil, err
 	}
@@ -117,11 +117,16 @@ func (e *extractor) noteImage(g *state, name string, dict reader.Dict, raw []byt
 		Dict:        dict,
 		Inline:      inline,
 	}
-	data, filter, err := reader.Decode(dict, raw, e.doc.Resolver())
-	if err == nil {
-		img.Data, img.Filter = data, filter
-	} else {
-		img.Data = raw
+	// A caller after the text is not after the pixels, and undoing a
+	// picture's filters to hand it nothing is the most expensive thing a page
+	// can be made to do.
+	if e.wantPictures {
+		data, filter, err := reader.Decode(dict, raw, e.doc.Resolver())
+		if err == nil {
+			img.Data, img.Filter = data, filter
+		} else {
+			img.Data = raw
+		}
 	}
 	e.images = append(e.images, img)
 }

run.go

@@ -21,16 +21,29 @@ type extractor struct {
 	// belong to the page rather than to the graphics state.
 	tm, tlm matrix
 	images  []Image
+	// wantPictures says whether anybody is going to read Image.Data. When
+	// nobody is, the filters are left undone.
+	wantPictures bool
 }
 
 // run walks a content stream.
 func (e *extractor) run(content []byte, resources reader.Dict, g state, depth int) {
 	if depth > maxFormDepth {
 		return
 	}
-	ops, _ := reader.Operations(content)
+	// The operations are taken one at a time rather than all at once. The
+	// walk only ever goes forwards, so nothing needs the list; keeping it
+	// costs a struct and a slice header for every operator on the page, and
+	// pages are not always small. One arXiv figure holds a single page of
+	// 84.8 MB of content stream and 4 579 973 operations: holding them cost
+	// 2 166 MB where reading them one by one costs 935 MB.
+	scan := reader.NewContentScanner(content)
 	var stack []state
-	for _, op := range ops {
+	for {
+		op, more := scan.Next()
+		if !more {
+			break
+		}
 		n := numbers(op.Operands)
 		switch op.Operator {
 		case "q":
@@ -222,7 +235,25 @@ func (e *extractor) show(g *state, s []byte) {
 	if text.Len() == 0 && !unreadable {
 		return
 	}
-	scale := g.ctm.scale()
+	// How far a length in text space reaches on the page: the text matrix and
+	// the page's transform together, which is the same matrix the run's own
+	// position came through.
+	//
+	// Only the page's transform used to be counted here, and the text matrix
+	// was left out. A document is free to put the whole of its scale there —
+	// "/F1 1 Tf" and then a text matrix of ten is how TeX writes every PDF it
+	// has ever produced — and such a document reported its text as one point
+	// tall with letters half a point wide, while saying quite correctly where
+	// on the page each run began.
+	//
+	// Those are the two numbers a word break is decided by. A gap measured in
+	// points was being weighed against a space width ten times too small, so
+	// every ordinary kern between two letters looked like a space: "Original
+	// Domain" came back as "Or ig inal D omain". Across four thousand arXiv
+	// figures, 57% of the words a reader got back were one or two letters
+	// long. Nothing failed, and nothing was slow; the text was simply no
+	// longer text.
+	scale := start.scale()
 	e.runs = append(e.runs, Run{
 		Text:       text.String(),
 		X:          x,

stalereader_test.go

@@ -0,0 +1,67 @@
+package extract_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/go-pdfkit/extract"
+	"github.com/go-pdfkit/reader"
+)
+
+// hugeObjectNumber is a whole PDF in 219 bytes. It has no trailer and no
+// startxref, so it can only be read by repairing it, and the last object it
+// declares is numbered 2 147 483 647.
+//
+// reader v0.4.0 answered "which objects call themselves a catalogue?" by
+// counting from zero to the largest object number the file mentioned, one map
+// lookup each. For this file that is two thousand million lookups for four
+// objects: twenty-one seconds, and not one byte allocated, which is why no
+// memory limit anywhere caught it.
+const hugeObjectNumber = "%PDF-1.7\n" +
+	"1 0 obj <</Type /Catalog /Pages 2 0 R>>\nendobj\n" +
+	"2 0 obj <</Type /Pages /Kids [3 0 R] /Count 1>>\nendobj\n" +
+	"3 0 obj <</Type /Page /Parent 2 0 R /MediaBox [0 0 10 10]>>\nendobj\n\n" +
+	"2147483647 0 obj <</Root 1 0 R>>\nendobj\n"
+
+// TestATinyFileWithAHugeObjectNumber guards the version of the reader this
+// package is built against, not this package's own code.
+//
+// The defect was fixed in reader v0.4.1, and merging is not shipping: a
+// consumer still asking for v0.4.0 still hands its callers a file that takes
+// twenty-one seconds to open. The budget is two seconds because the answer is
+// either a fraction of a millisecond or twenty-one seconds, and nothing in
+// between; there is no threshold here to tune.
+func TestATinyFileWithAHugeObjectNumber(t *testing.T) {
+	b := []byte(hugeObjectNumber)
+	if len(b) > 300 {
+		t.Fatalf("the file is %d bytes; it is meant to be small enough that its cost cannot come from its size", len(b))
+	}
+
+	start := time.Now()
+	d, err := reader.Open(b)
+	opened := time.Since(start)
+	if err != nil {
+		t.Fatalf("opening %d bytes: %v", len(b), err)
+	}
+	if opened > 2*time.Second {
+		t.Fatalf("opening %d bytes took %s: the reader this is built against walks every object "+
+			"number up to the largest one named, so it is older than v0.4.1", len(b), opened)
+	}
+
+	// And the whole way through this package, since that is what callers use.
+	start = time.Now()
+	for i := 1; i <= d.PageCount(); i++ {
+		if _, err := extract.Text(d, i); err != nil {
+			t.Fatalf("page %d: %v", i, err)
+		}
+		if _, err := extract.Images(d, i); err != nil {
+			t.Fatalf("page %d images: %v", i, err)
+		}
+	}
+	if read := time.Since(start); read > 2*time.Second {
+		t.Fatalf("reading %d pages of a %d-byte file took %s", d.PageCount(), len(b), read)
+	}
+	if d.PageCount() != 1 {
+		t.Errorf("the file has %d pages, want 1 — the measurement means nothing if it was not read", d.PageCount())
+	}
+}