diff --git a/README.md b/README.md index f2ec9bb..adf596b 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,35 @@ form submission, the annotation types that exist to play or embed something, and files associated with a page. A link to the web is not executable and stays. +### What a screen reader follows + +A tagged document says which run of marks on which page is a heading, a +paragraph, a table cell, the label of a form field. That is the reading order a +screen reader follows, and for a government form it is often what the law +requires — RGAA in France, Section 508 in the United States, EN 301 549 in the +EU. It cannot be copied across a rebuild: every part of it points into the +document, and the number tree that indexes it is keyed by a number each page +carries. So it is rebuilt, element by element, around the pages that survived, +and those numbers are handed out afresh. A page that kept the number it had in +a file it is no longer part of is a page a reader would look up and be told, +with every confidence, about somebody else's. + +An element whose every page has gone is removed and its parent pruned; one with +some pages gone keeps the children that are left. An empty table cell is kept, +because the shape of a table is part of what it says. A page written twice +carries the structure once, on the first copy, since an element names one page. +Merging two files carries no tree at all: two files have two role maps in which +the same name may mean two different things, and there is no honest way to +choose between them. + +Of **1 633 real government forms** from eighteen issuers, 1 021 carry a +structure tree. After a rotate **1 014 still do** — the seven that do not are +the seven whose tree was empty in the file we were given — and **991 of the +1 021 are identical down to the last element, mark and annotation reference**. +The thirty that differ are accounted for one by one in the commit that added +this. What cannot be carried is named in `catalogue.go` with the reason, rather +than disappearing quietly. + Text is drawn in the four faces every viewer already has — Helvetica, Helvetica-Bold, Courier and Courier-Bold — so nothing is embedded and a watermark costs about a kilobyte. Stamp text may say `{page}`, `{pages}` diff --git a/annots.go b/annots.go index c651e7b..12f78e9 100644 --- a/annots.go +++ b/annots.go @@ -45,6 +45,10 @@ var annotKeysRebuilt = map[reader.Name]bool{ "Dest": true, // remapped "A": true, // remapped, and filtered when sanitising "AA": true, // an annotation's own actions run without anyone asking + + // The number under which an annotation is filed in the structure tree's + // parent tree is handed out afresh, for the same reason a page's is. + "StructParent": true, } // RemoveAnnotations drops every annotation: links, comments, form fields and @@ -124,10 +128,16 @@ func newKeptAnnots() *keptAnnots { return &keptAnnots{at: map[annotKey]reader.Ref{}, dict: map[reader.Ref]reader.Dict{}} } -// add records one annotation that survived. +// add records one annotation that survived. When a page is written twice — a +// selection may ask for the same page more than once — the first copy is the +// one anything pointing at that annotation is pointed at, which is the same +// choice the destination map makes for the page itself and the copy the +// structure tree describes. func (k *keptAnnots) add(src *reader.Document, was reader.Object, ref reader.Ref, dict reader.Dict) { if old, ok := was.(reader.Ref); ok { - k.at[annotKey{src, old.Num}] = ref + if _, already := k.at[annotKey{src, old.Num}]; !already { + k.at[annotKey{src, old.Num}] = ref + } } k.dict[ref] = dict k.order = append(k.order, ref) diff --git a/catalogue.go b/catalogue.go index d8dcaa3..e5befaf 100644 --- a/catalogue.go +++ b/catalogue.go @@ -38,13 +38,6 @@ var sensitiveKeys = map[reader.Name]bool{"Metadata": true} // rather than describing it, so copying one across a rebuild would leave it // naming objects that are no longer there. // -// - /StructTreeRoot, the marked-up structure a screen reader follows. Its -// elements name the page each belongs to and the numbered marks inside -// that page's content, and its parent tree is indexed by a number the page -// carries. Carrying it means rebuilding all three, and a structure tree -// that points at the wrong pages is worse than none: a reader would read -// the document aloud in the wrong order rather than fall back on the text. -// This is the one worth doing next. // - /Names, the name trees: named destinations point at pages, embedded // files travel with the document, and one of the trees is where a file // keeps its JavaScript. @@ -52,9 +45,30 @@ var sensitiveKeys = map[reader.Name]bool{"Metadata": true} // the bytes the signature was taken over, so the signature is void and the // permission it granted with it. // - /OpenAction and /AA, which run when the document is opened. +// +// /StructTreeRoot, the marked-up structure a screen reader follows, is +// rebuilt: see structtree.go. Three parts of it are left out, and each is left +// out because it cannot be placed rather than because it is awkward. +// +// - A mark inside a stream that no surviving page draws. Such a mark is +// numbered within its own stream and filed under a key that stream +// carries, so it can only be carried when the stream is still drawn — and +// 109 of the corpus's 215 such marks named a stream that no page of the +// source drew either. +// - A structure element's /Ref, which names other structure elements. It +// cannot be answered while the rebuild is still deciding which of them +// survive, and copied as it stands it would drag a second copy of the +// source's tree — and of the source's pages behind it — into the file. No +// file in the corpus has one; it is PDF 2.0. +// - The structure of pages from more than one file. Two files have two +// trees, and two /RoleMap and /ClassMap dictionaries in which the same +// name may stand for two different things; a merged tree read through +// either one of them would describe the other file's pages wrongly, and +// there is no honest way to choose. Such a document keeps its pages and +// nothing above them, as it already did for the catalogue and the form. // keepCatalogue carries across what the source document said about itself. -func (d *Doc) keepCatalogue(w *reader.Writer, catalog reader.Dict, kept *keptAnnots) { +func (d *Doc) keepCatalogue(w *reader.Writer, catalog reader.Dict, kept *keptAnnots, built []builtPage) { src, ok := d.singleSource() if !ok { // Pages from several files have several catalogues, and there is no @@ -77,6 +91,9 @@ func (d *Doc) keepCatalogue(w *reader.Writer, catalog reader.Dict, kept *keptAnn if form := d.keepForm(w, src, source, kept); form != nil { catalog["AcroForm"] = w.Add(form) } + if tree := d.keepStructure(w, src, source, kept, built); tree != nil { + catalog["StructTreeRoot"] = tree + } } // singleSource is the one document every page was borrowed from, when there is diff --git a/structtree.go b/structtree.go new file mode 100644 index 0000000..e24ab69 --- /dev/null +++ b/structtree.go @@ -0,0 +1,606 @@ +package ops + +import ( + "bytes" + "sort" + + "github.com/go-pdfkit/reader" +) + +// The structure tree is the document's reading order: which run of marks on +// which page is a heading, a paragraph, a table cell, the label of a form +// field. It is what a screen reader follows, and for a government form it is +// often what the law requires. Of 1 633 real forms in the corpus, 1 021 carry +// one and 1 012 of those carry the number tree that indexes it. +// +// It is also the one thing in a catalogue that cannot be copied across. Every +// part of it points into the document: an element names the page it is on, the +// numbered marks inside that page's content, and the annotations it stands +// for, and the number tree is indexed by a key the page itself carries. So it +// is rebuilt here, element by element, around the pages that survived — and +// where a piece of it cannot be placed honestly it is left out rather than +// pointed somewhere plausible, because a reader that finds no structure falls +// back on the text, and one that finds the wrong structure does not. + +// maxStructDepth bounds the walk of a structure tree. Deeper than this is a +// file playing games rather than a document with a shape. +const maxStructDepth = 64 + +// maxMarksPerPage bounds the array that maps a page's marks back to the +// elements that own them. The array is indexed by mark number and real files +// leave gaps in it — the corpus has two million holes and its longest array is +// 7 566 entries — so its size is decided by the largest number, and a file +// naming mark two billion would otherwise ask for the memory to match. +const maxMarksPerPage = 1 << 20 + +// structKeysRebuilt are the entries of a structure element this package +// decides for itself rather than copying across. +var structKeysRebuilt = map[reader.Name]bool{ + "K": true, // its children: the ones that survived, renumbered + "P": true, // its parent in the tree being written + "Pg": true, // the page it is on, which is a page of this document now + + // /Ref names other structure elements, which is a thing this rebuild + // cannot answer while it is still deciding which of them survive; copied + // as it stands it would drag a second copy of the source's own tree — + // and, through it, of the source's own pages — into the file behind it. + "Ref": true, +} + +// structRootKeys are the entries of the tree's root that describe how to read +// it rather than pointing into the document, and so travel unchanged. +var structRootKeys = []reader.Name{ + "RoleMap", // what a document's own element names stand for + "ClassMap", // the attribute classes its elements refer to +} + +// pageGone is the page of an element whose page is not in this document. It is +// told apart from naming no page at all, because a child that inherits a page +// that has gone has gone with it, while one that inherits nothing never had a +// page to lose. +const pageGone = -1 + +// A structRebuild is one rebuild of one structure tree. +type structRebuild struct { + w *reader.Writer + src *reader.Document + kept *keptAnnots + // root is the number the tree's root is given before its elements are + // built, since each of the top ones has to point back at it. + root reader.Ref + + // pageOf says which page of the source an object number is, counting from + // one, so that an element's /Pg is recognised without searching. + pageOf map[int]int + // at is where the first output copy of each source page went, with its + // dictionary, still open to be told which number its structure is filed + // under. + at map[int]builtPage + // order is the source page numbers in the order they were written. + order []int + + // marks[page][mark] is the element that owns one mark on one page. + marks map[int]map[int64]reader.Ref + // streams[key][mark] is the element that owns one mark inside a stream the + // page draws rather than inside the page's own content, under the key that + // stream carries. + streams map[int64]map[int64]reader.Ref + // floor is the first number free to be handed out to a page or an + // annotation: see floorAbove. + floor int64 + // reach[page] is the objects one page draws, worked out only when a mark + // inside one of them has to be placed. + reach map[int]map[int]bool + // owner[annot] is the element that stands for one surviving annotation. + owner map[reader.Ref]reader.Ref + // ids are the identifiers of the elements that survived. + ids []structID + // seen is the elements already visited, since a file may point back. + seen map[int]bool +} + +// A structID is one element's identifier and where the element ended up. +type structID struct { + id []byte + ref reader.Ref +} + +// keepStructure rebuilds the structure a screen reader follows, and reports +// where the tree's root went, or nil when nothing of it survived. +func (d *Doc) keepStructure(w *reader.Writer, src *reader.Document, catalog reader.Dict, kept *keptAnnots, built []builtPage) reader.Object { + root, ok := src.GetDict(catalog, "StructTreeRoot") + if !ok { + return nil + } + s := &structRebuild{ + w: w, src: src, kept: kept, root: w.Reserve(), + pageOf: map[int]int{}, + at: map[int]builtPage{}, + marks: map[int]map[int64]reader.Ref{}, + streams: map[int64]map[int64]reader.Ref{}, + reach: map[int]map[int]bool{}, + owner: map[reader.Ref]reader.Ref{}, + seen: map[int]bool{}, + } + for i := 1; i <= src.PageCount(); i++ { + ref, _ := src.PageRef(i) + s.pageOf[ref.Num] = i + } + for _, p := range built { + if _, already := s.at[p.num]; already { + // The same page of the source, written twice. An element says + // which single page it is on, so the first copy is the one the + // structure describes and the others carry none: a page a reader + // finds no structure on is read as it stands, which is what an + // unmarked page has always been. + continue + } + s.at[p.num] = p + s.order = append(s.order, p.num) + } + s.floorAbove() + kids := s.top(root) + if len(kids) == 0 { + return nil + } + out := reader.Dict{"Type": reader.Name("StructTreeRoot"), "K": kids} + for _, key := range structRootKeys { + if v, named := root[key]; named { + out[key] = w.Copy(src, v) + } + } + if nums, next := s.parentTree(); len(nums) > 0 { + out["ParentTree"] = w.Add(reader.Dict{"Nums": nums}) + // Where an editor adding to this tree should carry on numbering. + out["ParentTreeNextKey"] = reader.Integer(next) + } + if _, named := root["IDTree"]; named { + if names := s.idTree(); len(names) > 0 { + out["IDTree"] = w.Add(reader.Dict{"Names": names}) + } + } + w.Put(s.root, out) + return s.root +} + +// top rebuilds the children of the tree's root, which are elements and +// nothing else. +func (s *structRebuild) top(root reader.Dict) reader.Array { + entry := root.Get("K") + list, ok := resolveArray(s.src, entry) + if !ok { + list = reader.Array{entry} + } + var out reader.Array + for _, kid := range list { + if ref, _, ok := s.element(kid, s.root, 0, 0); ok { + out = append(out, ref) + } + } + return out +} + +// element rebuilds one structure element and reports whether anything of it +// survived, and which page of the source what is left of it is on. page is the +// page it is on coming in, which it may have inherited from an element above +// it. +func (s *structRebuild) element(o reader.Object, parent reader.Ref, page, depth int) (reader.Ref, int, bool) { + if depth > maxStructDepth { + return reader.Ref{}, 0, false + } + if ref, ok := o.(reader.Ref); ok { + if s.seen[ref.Num] { + return reader.Ref{}, 0, false + } + s.seen[ref.Num] = true + } + elem, ok := resolveDict(s.src, o) + if !ok { + return reader.Ref{}, 0, false + } + switch kind, _ := reader.ToName(resolve(s.src, elem.Get("Type"))); kind { + case "MCR", "OBJR": + // A mark, or a reference to an annotation, at the top of a tree: there + // is no element there for it to belong to. + return reader.Ref{}, 0, false + } + own, named := s.pageAt(elem) + if named { + page = own + } + // The element is given its number before its children are rebuilt, since + // each of them has to point back at it. + ref := s.w.Reserve() + kids, had, at := s.children(elem, ref, page, depth) + if len(kids) == 0 && (had > 0 || page == pageGone) { + // An element whose children have all gone describes nothing. One that + // never had any is the shape of the document rather than a claim about + // its content — an empty table cell, of which this corpus has 30 345 — + // and is kept, as long as the page it sits on is still here. + return reader.Ref{}, 0, false + } + out := reader.Dict{} + for k, v := range elem { + if structKeysRebuilt[k] { + continue + } + out[k] = s.w.Copy(s.src, v) + } + out["P"] = parent + if len(kids) > 0 { + out["K"] = kids + } + switch { + case named && page > 0: + out["Pg"] = s.at[page].ref + case named && at > 0: + // Its own page has gone but some of its content is still here, on + // another one. Saying nothing would leave it inheriting the page of + // whatever it sits under — a page it is not on, stated as confidently + // as the right one would have been. + out["Pg"] = s.at[at].ref + } + if id, ok := reader.ToString(resolve(s.src, elem.Get("ID"))); ok { + s.ids = append(s.ids, structID{id, ref}) + } + s.w.Put(ref, out) + if page > 0 { + return ref, page, true + } + return ref, at, true +} + +// pageAt reports which page of the source an element names, and whether it +// named one at all. +func (s *structRebuild) pageAt(elem reader.Dict) (int, bool) { + entry := elem.Get("Pg") + if entry.Kind() == reader.KindNull { + return 0, false + } + ref, ok := entry.(reader.Ref) + if !ok { + // A page written inside the element rather than referred to is not a + // page of the document: nothing else could point at it. + return pageGone, true + } + num, ok := s.pageOf[ref.Num] + if !ok { + return pageGone, true + } + if _, kept := s.at[num]; !kept { + return pageGone, true + } + return num, true +} + +// children rebuilds an element's children. It reports how many the source gave +// it, so that an element that never had any can be told from one whose own have +// all gone, and the first page any of what is left is on. +func (s *structRebuild) children(elem reader.Dict, ref reader.Ref, page, depth int) (reader.Array, int, int) { + entry := elem.Get("K") + list, ok := resolveArray(s.src, entry) + if !ok { + if resolve(s.src, entry).Kind() == reader.KindNull { + return nil, 0, 0 + } + // One child, written on its own rather than in an array, which is how + // 803 of the corpus's 1 021 trees write the root's. + list = reader.Array{entry} + } + var out reader.Array + at := 0 + for _, kid := range list { + got, on, ok := s.child(kid, ref, page, depth) + if !ok { + continue + } + out = append(out, got) + if at == 0 { + at = on + } + } + return out, len(list), at +} + +// child rebuilds one child of a structure element: another element, an integer +// naming a mark in the page's own content, a marked-content reference, or a +// reference to an annotation. +func (s *structRebuild) child(o reader.Object, parent reader.Ref, page, depth int) (reader.Object, int, bool) { + resolved := resolve(s.src, o) + if n, ok := reader.ToInt(resolved); ok { + return s.mark(n, parent, page) + } + kid, ok := reader.ToDict(resolved) + if !ok { + return nil, 0, false + } + switch kind, _ := reader.ToName(resolve(s.src, kid.Get("Type"))); kind { + case "MCR": + return s.markRef(kid, parent, page) + case "OBJR": + return s.objectRef(kid, parent, page) + } + ref, on, ok := s.element(o, parent, page, depth+1) + return ref, on, ok +} + +// mark records one mark of one page as belonging to an element, and reports +// the child to write in its place. The number is left exactly as it was: a +// page's content is copied byte for byte, so the marks inside it still carry +// the numbers they carried, and renumbering them here would be inventing a +// disagreement with the content. +func (s *structRebuild) mark(n int64, parent reader.Ref, page int) (reader.Object, int, bool) { + if page <= 0 || n < 0 || n >= maxMarksPerPage { + return nil, 0, false + } + at, ok := s.marks[page] + if !ok { + at = map[int64]reader.Ref{} + s.marks[page] = at + } + // Two elements claiming one mark is a file contradicting itself, and the + // number tree can only name one of them; the later one is taken, which is + // what a reader reading the file in order would have been left with. + at[n] = parent + return reader.Integer(n), page, true +} + +// markRef rebuilds a marked-content reference, which is the long way of naming +// a mark: it may say which page the mark is on rather than leave it to be +// inherited, and it may say the mark is inside a stream the page draws rather +// than inside the page's own content. +func (s *structRebuild) markRef(kid reader.Dict, parent reader.Ref, page int) (reader.Object, int, bool) { + n, ok := reader.ToInt(resolve(s.src, kid.Get("MCID"))) + if !ok { + return nil, 0, false + } + own, named := s.pageAt(kid) + if named { + page = own + } + if stm := kid.Get("Stm"); stm.Kind() != reader.KindNull { + return s.streamMark(stm, n, parent, page, named) + } + if _, _, ok := s.mark(n, parent, page); !ok { + return nil, 0, false + } + out := reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(n)} + if named { + out["Pg"] = s.at[page].ref + } + return out, page, true +} + +// streamMark carries a mark that lives inside a stream the page draws rather +// than inside the page's own content — a form XObject, in every one of the 215 +// the corpus has. Such a mark is numbered within that stream, and filed under +// a key the stream itself carries: the four UK power-of-attorney forms keep +// their whole content this way, on pages that carry no key at all. +// +// The stream is followed only when the page it belongs to still draws it. A +// mark in a stream no surviving page reaches is a mark nobody sees, and 109 of +// the corpus's 215 were already in that state in the file they came from. +func (s *structRebuild) streamMark(stm reader.Object, n int64, parent reader.Ref, page int, named bool) (reader.Object, int, bool) { + if page <= 0 || n < 0 || n >= maxMarksPerPage { + return nil, 0, false + } + ref, ok := stm.(reader.Ref) + if !ok || !s.reaches(page, ref.Num) { + return nil, 0, false + } + stream, ok := reader.ToStream(resolve(s.src, ref)) + if !ok { + return nil, 0, false + } + key, ok := reader.ToInt(resolve(s.src, stream.Dict.Get("StructParents"))) + if !ok || key < 0 { + // The stream does not say where its marks are filed, and this package + // cannot tell it: the copy of it has already been written. + return nil, 0, false + } + at, ok := s.streams[key] + if !ok { + at = map[int64]reader.Ref{} + s.streams[key] = at + } + at[n] = parent + out := reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(n), + "Stm": s.w.Copy(s.src, ref)} + if named { + out["Pg"] = s.at[page].ref + } + return out, page, true +} + +// reaches reports whether a page draws the object with the given number: its +// content, and everything the resources it draws with name. +func (s *structRebuild) reaches(page, num int) bool { + set, ok := s.reach[page] + if !ok { + set = map[int]bool{} + src, _ := s.src.Page(page) + s.follow(src.Get("Contents"), set, 0) + s.follow(src.Get("Resources"), set, 0) + s.reach[page] = set + } + return set[num] +} + +// floorAbove settles the first number free to be handed out to a page or an +// annotation here. +// +// A stream the page draws may hold marks of its own, filed under a key the +// stream carries rather than under the page's. That is not a guess: of the +// 1 021 tagged forms in the corpus, 991 file every mark under a page, 14 file +// some under a page and some under a form XObject drawn on it, and 3 — the UK +// power-of-attorney forms — carry no key on any page at all and file +// everything under the XObjects, whose stream dictionaries say +// /StructParents 0, 1, 2 and 3 while every page of the file says nothing. +// +// The copy of such a stream in this file keeps the key, because a stream is +// written before the structure above it is rebuilt and cannot be given a new +// one. So the numbers handed out below start above every key any object a +// surviving page draws already carries. Numbering from zero instead would +// eventually hand a page the number a form XObject on it is filed under, and a +// reader looking up a mark in that XObject would be told, with every +// confidence, about the page's own elements. +func (s *structRebuild) floorAbove() { + walked := map[int]bool{} + for _, num := range s.order { + page, _ := s.src.Page(num) + for _, key := range structDrawn { + s.follow(page.Get(key), walked, 0) + } + } +} + +// structDrawn are the entries of a page that lead to what it draws. +var structDrawn = []reader.Name{"Contents", "Resources", "Annots"} + +// structKeyed are the entries under which an object says where its own marks +// are filed in the parent tree. +var structKeyed = []reader.Name{"StructParents", "StructParent"} + +// follow walks what a page draws, collecting the objects it reaches and +// raising the floor above every parent-tree key they carry. +func (s *structRebuild) follow(o reader.Object, into map[int]bool, depth int) { + if depth > maxStructDepth { + return + } + if ref, ok := o.(reader.Ref); ok { + if into[ref.Num] { + return + } + into[ref.Num] = true + } + switch v := resolve(s.src, o).(type) { + case reader.Array: + for _, e := range v { + s.follow(e, into, depth+1) + } + case reader.Dict: + s.raise(v) + for _, e := range v { + s.follow(e, into, depth+1) + } + case *reader.Stream: + s.raise(v.Dict) + for _, e := range v.Dict { + s.follow(e, into, depth+1) + } + } +} + +// raise lifts the floor above the parent-tree keys one object carries. +func (s *structRebuild) raise(d reader.Dict) { + for _, key := range structKeyed { + if n, ok := reader.ToInt(resolve(s.src, d.Get(key))); ok && n >= s.floor { + s.floor = n + 1 + } + } +} + +// objectRef rebuilds a reference to something outside the content, which in +// practice is an annotation: a link, or the widget through which a form field +// is filled in. It survives exactly as long as the annotation does — 104 379 +// of the corpus's 104 521 point at one, and the other 142 pointed at nothing +// on a page in the source either. +// +// No page is written on it: the annotation says which page it is on, and so +// does the element above it. +func (s *structRebuild) objectRef(kid reader.Dict, parent reader.Ref, page int) (reader.Object, int, bool) { + to, ok := s.kept.find(s.src, kid.Get("Obj")) + if !ok { + return nil, 0, false + } + s.owner[to] = parent + if own, named := s.pageAt(kid); named { + page = own + } + if page < 0 { + page = 0 + } + return reader.Dict{"Type": reader.Name("OBJR"), "Obj": to}, page, true +} + +// parentTree maps each page, each drawn stream and each annotation the +// structure points at back to the elements on it, under the number that page, +// stream or annotation carries, and reports the next number free after it. +// +// A stream keeps the number it had, since the copy of it in this file has +// already been written carrying that number. A page and an annotation are +// given theirs afresh, in the order they were written: a page that kept the +// number it had in a file it is no longer part of is a page a reader would look +// up and be told about somebody else's. The entries come out in order of their +// number, which is what makes a number tree a tree. +func (s *structRebuild) parentTree() (reader.Array, int64) { + var nums reader.Array + filed := make([]int64, 0, len(s.streams)) + for key := range s.streams { + filed = append(filed, key) + } + sort.Slice(filed, func(i, j int) bool { return filed[i] < filed[j] }) + for _, key := range filed { + nums = append(nums, reader.Integer(key), s.w.Add(markArray(s.streams[key]))) + } + key := s.floor + for _, num := range s.order { + at, ok := s.marks[num] + if !ok { + continue + } + nums = append(nums, reader.Integer(key), s.w.Add(markArray(at))) + s.at[num].dict["StructParents"] = reader.Integer(key) + key++ + } + // An annotation is filed under a number of its own, and its entry is the + // one element that stands for it rather than an array. + for _, ref := range s.kept.order { + owner, ok := s.owner[ref] + if !ok { + continue + } + nums = append(nums, reader.Integer(key), owner) + s.kept.dict[ref]["StructParent"] = reader.Integer(key) + key++ + } + return nums, key +} + +// markArray lays a page's marks out as an array indexed by mark number, with +// the gaps a real file leaves in it written as null. +func markArray(at map[int64]reader.Ref) reader.Array { + high := int64(0) + for n := range at { + if n > high { + high = n + } + } + out := make(reader.Array, high+1) + for i := range out { + out[i] = reader.Null{} + } + for n, ref := range at { + out[n] = ref + } + return out +} + +// idTree lists the identifiers of the elements that survived. A name tree is +// its keys in order, so that a reader can find one by halving; the identifier +// of an element that has gone is not carried, since it would name nothing. +func (s *structRebuild) idTree() reader.Array { + sort.SliceStable(s.ids, func(i, j int) bool { return bytes.Compare(s.ids[i].id, s.ids[j].id) < 0 }) + var out reader.Array + seen := map[string]bool{} + for _, e := range s.ids { + if seen[string(e.id)] { + // One identifier naming two elements is a file contradicting + // itself, and a name tree has one entry per key. + continue + } + seen[string(e.id)] = true + out = append(out, reader.String(e.id), e.ref) + } + return out +} diff --git a/structtree_test.go b/structtree_test.go new file mode 100644 index 0000000..4d578ea --- /dev/null +++ b/structtree_test.go @@ -0,0 +1,793 @@ +package ops + +import ( + "testing" + + "github.com/go-pdfkit/reader" +) + +// A tagged is a document being written whose pages carry marked content and +// whose catalogue carries a structure tree over them. +type tagged struct { + w *reader.Writer + pages reader.Ref + page []reader.Ref + // stream is a form XObject drawn on the first page, holding a mark of its + // own and saying under what number that mark is filed. + stream reader.Ref + // drawn is what the first page's resources name. + drawn reader.Dict + // loose is a form XObject no page draws. + loose reader.Ref + // annot is one annotation per page, in page order. + annot []reader.Ref + // notAPage is an object an element can wrongly claim to be on. + notAPage reader.Ref +} + +// marked writes a content stream with the given mark numbers in it. +func marked(w *reader.Writer, nums ...int) reader.Object { + var raw []byte + for _, n := range nums { + raw = append(raw, "/P <> BDC BT ET EMC\n"...) + } + return w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: raw}) +} + +// newTagged lays out three pages, each with one mark of its own, the first of +// them also drawing a form XObject with a mark inside it, and each with one +// annotation on it. The pages are written by finish, so that a test can add +// something else for the first page to draw first. +func newTagged() *tagged { + w := reader.NewWriter("1.7") + g := &tagged{w: w, pages: w.Reserve(), drawn: reader.Dict{}} + g.notAPage = w.Add(reader.Dict{"Type": reader.Name("Whatever")}) + g.stream = w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "StructParents": reader.Integer(7), + "BBox": reader.Array{reader.Integer(0), reader.Integer(0), + reader.Integer(9), reader.Integer(9)}, + }, Raw: []byte("/P <> BDC BT ET EMC\n")}) + g.loose = w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "StructParents": reader.Integer(3), + }, Raw: []byte("/P <> BDC BT ET EMC\n")}) + g.drawn["Fm0"] = g.stream + for i := 0; i < 3; i++ { + g.page = append(g.page, w.Reserve()) + } + for i, ref := range g.page { + g.annot = append(g.annot, w.Add(reader.Dict{"Type": reader.Name("Annot"), + "Subtype": reader.Name("Link"), "P": ref, + "StructParent": reader.Integer(int64(11 + i)), + "Rect": reader.Array{reader.Integer(0), reader.Integer(0), + reader.Integer(9), reader.Integer(9)}})) + } + return g +} + +// draw puts one more form XObject in the first page's resources. +func (g *tagged) draw(name reader.Name, ref reader.Ref) { g.drawn[name] = ref } + +// writePages puts the three pages down. +func (g *tagged) writePages() { + for i, ref := range g.page { + page := reader.Dict{"Type": reader.Name("Page"), "Parent": g.pages, + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), + reader.Integer(99), reader.Integer(99)}, + "Annots": reader.Array{g.annot[i]}, + "StructParents": reader.Integer(int64(i)), + "Contents": marked(g.w, 0, 1), + } + if i == 0 { + page["Resources"] = reader.Dict{"XObject": g.drawn} + } + g.w.Put(ref, page) + } + g.w.Put(g.pages, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{g.page[0], g.page[1], g.page[2]}, + "Count": reader.Integer(3)}) +} + +// finish writes the file with the given structure tree over the pages. +func (g *tagged) finish(t *testing.T, root reader.Dict) []byte { + t.Helper() + g.writePages() + catalog := reader.Dict{"Type": reader.Name("Catalog"), "Pages": g.pages, + "MarkInfo": reader.Dict{"Marked": reader.Bool(true)}} + if root != nil { + catalog["StructTreeRoot"] = g.w.Add(root) + } + out, err := g.w.Finish(reader.Dict{"Root": g.w.Add(catalog)}) + if err != nil { + t.Fatal(err) + } + return out +} + +// elem writes one structure element. +func (g *tagged) elem(kind reader.Name, extra reader.Dict, kids ...reader.Object) reader.Ref { + d := reader.Dict{"Type": reader.Name("StructElem"), "S": kind} + for k, v := range extra { + d[k] = v + } + switch len(kids) { + case 0: + case 1: + d["K"] = kids[0] + default: + d["K"] = reader.Array(kids) + } + return g.w.Add(d) +} + +// wholeTree is the structure tree the tests measure against: every shape a +// real one has, over three pages. +func wholeTree(g *tagged) reader.Dict { + sect := g.elem("Sect", reader.Dict{"Pg": g.page[0]}, + reader.Integer(0), + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(1)}, + reader.Dict{"Type": reader.Name("OBJR"), "Obj": g.annot[0], "Pg": g.page[0]}, + ) + // A mark inside the form XObject the first page draws. + inStream := g.elem("H1", reader.Dict{"Pg": g.page[0], "ID": reader.String("aaa")}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": g.stream, "Pg": g.page[0]}) + // An empty table cell: the shape of the document, not a claim about it. + empty := g.elem("TD", reader.Dict{"Pg": g.page[0]}) + second := g.elem("P", reader.Dict{"Pg": g.page[1], "ID": reader.String("bbb")}, + reader.Integer(0), + reader.Dict{"Type": reader.Name("OBJR"), "Obj": g.annot[1]}, + ) + emptyGone := g.elem("TD", reader.Dict{"Pg": g.page[1]}) + // An element with no page of its own, whose child has one. + third := g.elem("Div", nil, + g.elem("P", reader.Dict{"Pg": g.page[2]}, reader.Integer(1))) + return reader.Dict{ + "Type": reader.Name("StructTreeRoot"), + "RoleMap": reader.Dict{"Sect": reader.Name("Div")}, + "ClassMap": reader.Dict{"warm": reader.Dict{"O": reader.Name("Layout")}}, + "IDTree": reader.Dict{"Names": reader.Array{}}, + "K": g.elem("Document", nil, + sect, inStream, empty, second, emptyGone, third), + } +} + +// treeOf reads back the structure tree of a rebuilt document. +func treeOf(t *testing.T, d *reader.Document, catalog reader.Dict) reader.Dict { + t.Helper() + root, ok := d.GetDict(catalog, "StructTreeRoot") + if !ok { + t.Fatal("the rebuilt document has no structure tree") + } + return root +} + +// topOf lists the elements at the top of a rebuilt tree. +func topOf(t *testing.T, d *reader.Document, root reader.Dict) []reader.Object { + t.Helper() + kids := kidsOf(t, d, root) + if len(kids) != 1 { + t.Fatalf("the tree has %d elements at the top, wanted one", len(kids)) + } + doc, ok := reader.ToDict(kids[0]) + if !ok { + t.Fatalf("the top of the tree is %v", kids[0]) + } + return kidsOf(t, d, doc) +} + +// namesOf lists the kinds of a handful of elements, for a failure message. +func namesOf(list []reader.Object) []string { + var out []string + for _, k := range list { + e, _ := reader.ToDict(k) + s, _ := reader.ToName(e.Get("S")) + out = append(out, string(s)) + } + return out +} + +// kidsOf lists one element's children, resolved. +func kidsOf(t *testing.T, d *reader.Document, elem reader.Dict) []reader.Object { + t.Helper() + entry, _ := d.Resolve(elem.Get("K")) + if arr, ok := reader.ToArray(entry); ok { + out := make([]reader.Object, 0, len(arr)) + for _, e := range arr { + got, _ := d.Resolve(e) + out = append(out, got) + } + return out + } + return []reader.Object{entry} +} + +// numsOf reads a rebuilt parent tree into a map. +func numsOf(t *testing.T, d *reader.Document, root reader.Dict) map[int64]reader.Object { + t.Helper() + out := map[int64]reader.Object{} + pt, ok := d.GetDict(root, "ParentTree") + if !ok { + return out + } + arr, ok := reader.ToArray(pt.Get("Nums")) + if !ok { + t.Fatal("the parent tree has no /Nums") + } + last := int64(-1) + for i := 0; i+1 < len(arr); i += 2 { + key, ok := reader.ToInt(arr[i]) + if !ok { + t.Fatalf("the parent tree is keyed by %v", arr[i]) + } + if key <= last { + t.Errorf("the parent tree's keys are out of order: %d after %d", key, last) + } + last = key + got, _ := d.Resolve(arr[i+1]) + out[key] = got + } + return out +} + +// marksOn lists the mark numbers in one page's content, so that what the tree +// claims can be held against what the page actually draws. +func marksOn(t *testing.T, d *reader.Document, page int) map[int64]bool { + t.Helper() + ops, err := d.PageOperations(page) + if err != nil { + t.Fatal(err) + } + out := map[int64]bool{} + for _, op := range ops { + if op.Operator != "BDC" || len(op.Operands) < 2 { + continue + } + props, ok := reader.ToDict(op.Operands[1]) + if !ok { + continue + } + if n, ok := reader.ToInt(props.Get("MCID")); ok { + out[n] = true + } + } + return out +} + +func TestARebuiltDocumentKeepsItsStructure(t *testing.T) { + g := newTagged() + src := g.finish(t, wholeTree(g)) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + root := treeOf(t, d, catalog) + + for _, key := range []reader.Name{"RoleMap", "ClassMap", "ParentTree", "IDTree"} { + if root.Get(key).Kind() == reader.KindNull { + t.Errorf("the rebuilt tree lost /%s", key) + } + } + kids := topOf(t, d, root) + if len(kids) != 6 { + t.Fatalf("the document element has %d children (%v), wanted six", len(kids), namesOf(kids)) + } + // The first element's own page must be a page of this document, and its + // marks must be marks the page really draws. + sect, _ := reader.ToDict(kids[0]) + first, _ := d.PageRef(1) + if pg, ok := sect.Get("Pg").(reader.Ref); !ok || pg != first { + t.Errorf("the first element is on %v, wanted page one at %v", sect.Get("Pg"), first) + } + drawn := marksOn(t, d, 1) + for _, kid := range kidsOf(t, d, sect) { + if n, ok := reader.ToInt(kid); ok && !drawn[n] { + t.Errorf("the tree names mark %d, which page one does not draw", n) + } + } + // The page says under what number its marks are filed, and the number + // tree, looked up under it, says which element owns each mark. + nums := numsOf(t, d, root) + page, err := d.Page(1) + if err != nil { + t.Fatal(err) + } + key, ok := reader.ToInt(page.Get("StructParents")) + if !ok { + t.Fatal("page one does not say where its structure is filed") + } + arr, ok := reader.ToArray(nums[key]) + if !ok { + t.Fatalf("the number tree holds %v under page one's key %d", nums[key], key) + } + if len(arr) < 2 { + t.Fatalf("page one's entry has %d places, wanted at least two", len(arr)) + } + for _, mark := range []int64{0, 1} { + if _, ok := arr[mark].(reader.Ref); !ok { + t.Errorf("mark %d of page one is owned by %v", mark, arr[mark]) + } + } + // The annotation is filed under a number of its own, and that entry names + // one element rather than an array. + annots, _ := reader.ToArray(page.Get("Annots")) + if len(annots) != 1 { + t.Fatalf("page one has %d annotations", len(annots)) + } + annot, _ := d.GetDict(page, "Annots") + _ = annot + first0, _ := reader.ToDict(mustGet(t, d, annots[0])) + akey, ok := reader.ToInt(first0.Get("StructParent")) + if !ok { + t.Fatal("the annotation does not say where it is filed") + } + if akey == key { + t.Errorf("the annotation and the page are filed under the same number %d", akey) + } + if _, ok := nums[akey].(reader.Dict); !ok { + if _, isRef := nums[akey].(reader.Ref); !isRef { + t.Errorf("the annotation's entry is %v, wanted an element", nums[akey]) + } + } + if next, ok := reader.ToInt(root.Get("ParentTreeNextKey")); !ok || next <= akey { + t.Errorf("the next free number is %v, and %d is taken", root.Get("ParentTreeNextKey"), akey) + } + // The identifiers of the elements that survived, in order. + ids, ok := d.GetDict(root, "IDTree") + if !ok { + t.Fatal("the rebuilt tree has no /IDTree") + } + names, ok := reader.ToArray(ids.Get("Names")) + if !ok || len(names) != 4 { + t.Fatalf("the identifier tree holds %v", ids.Get("Names")) + } + if s, _ := reader.ToString(names[0]); string(s) != "aaa" { + t.Errorf("the first identifier is %q", s) + } + if s, _ := reader.ToString(names[2]); string(s) != "bbb" { + t.Errorf("the second identifier is %q", s) + } +} + +func TestStructureFollowsThePagesThatSurvive(t *testing.T) { + g := newTagged() + src := g.finish(t, wholeTree(g)) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Select("1,3")) }) + root := treeOf(t, d, catalog) + // The element on the dropped page goes, and so does the empty cell that + // sat on it: four of the six are left. + kids := topOf(t, d, root) + if len(kids) != 4 { + t.Fatalf("%d children left (%v), wanted four", len(kids), namesOf(kids)) + } + // Every page named by what is left is a page of this document, and the + // numbers have been handed out afresh. + nums := numsOf(t, d, root) + for i := 1; i <= d.PageCount(); i++ { + page, err := d.Page(i) + if err != nil { + t.Fatal(err) + } + key, ok := reader.ToInt(page.Get("StructParents")) + if !ok { + t.Fatalf("page %d says nothing about where its structure is filed", i) + } + arr, ok := reader.ToArray(nums[key]) + if !ok { + t.Fatalf("page %d's key %d holds %v", i, key, nums[key]) + } + drawn := marksOn(t, d, i) + for mark, owner := range arr { + if owner.Kind() == reader.KindNull { + continue + } + if !drawn[int64(mark)] { + t.Errorf("page %d is filed as owning mark %d, which it does not draw", i, mark) + } + } + } + // The identifier of the element that went is not carried: it would name + // nothing. + ids, _ := d.GetDict(root, "IDTree") + names, _ := reader.ToArray(ids.Get("Names")) + if len(names) != 2 { + t.Fatalf("the identifier tree holds %v, wanted only the surviving one", names) + } +} + +func TestADuplicatedPageCarriesNoStructure(t *testing.T) { + // An element names one page, so only the first copy can be the one the + // structure describes. The second must not be left carrying a number that + // would send a reader to the first copy's elements. + g := newTagged() + src := g.finish(t, wholeTree(g)) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Select("1,1,2,3")) }) + treeOf(t, d, catalog) + first, err := d.Page(1) + if err != nil { + t.Fatal(err) + } + if first.Get("StructParents").Kind() == reader.KindNull { + t.Error("the first copy of the page carries no structure") + } + second, err := d.Page(2) + if err != nil { + t.Fatal(err) + } + if got := second.Get("StructParents"); got.Kind() != reader.KindNull { + t.Errorf("the second copy of the page is filed under %v", got) + } +} + +func TestStructureIsNotCarriedAcrossFiles(t *testing.T) { + g := newTagged() + src := g.finish(t, wholeTree(g)) + one, err := Open(src) + if err != nil { + t.Fatal(err) + } + two, err := Open(src) + if err != nil { + t.Fatal(err) + } + one.Append(two) + out, err := one.Bytes() + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + catalog, err := d.Catalog() + if err != nil { + t.Fatal(err) + } + if got := catalog.Get("StructTreeRoot"); got.Kind() != reader.KindNull { + t.Errorf("two files were given one structure tree: %v", got) + } + // And no page may be left saying it is filed in a tree that is not there. + for i := 1; i <= d.PageCount(); i++ { + page, err := d.Page(i) + if err != nil { + t.Fatal(err) + } + if got := page.Get("StructParents"); got.Kind() != reader.KindNull { + t.Errorf("page %d still says it is filed under %v", i, got) + } + } +} + +func TestStructureGoesWithTheAnnotationsWhenTheyGo(t *testing.T) { + g := newTagged() + src := g.finish(t, wholeTree(g)) + for _, c := range []struct { + name string + do func(*Doc) + }{ + {"flattened", func(doc *Doc) { doc.Flatten() }}, + {"without annotations", func(doc *Doc) { doc.RemoveAnnotations() }}, + } { + t.Run(c.name, func(t *testing.T) { + d, catalog := rebuilt(t, src, c.do) + root := treeOf(t, d, catalog) + for _, kid := range topOf(t, d, root) { + elem, _ := reader.ToDict(kid) + for _, own := range kidsOf(t, d, elem) { + if o, ok := reader.ToDict(own); ok { + if kind, _ := reader.ToName(o.Get("Type")); kind == "OBJR" { + t.Errorf("the tree still points at an annotation: %v", o) + } + } + } + } + }) + } +} + +func TestAStructureTreeWithNothingInItIsNotWritten(t *testing.T) { + for _, c := range []struct { + name string + root func(*tagged) reader.Dict + }{ + {"no tree at all", func(*tagged) reader.Dict { return nil }}, + {"a root with no children", func(*tagged) reader.Dict { + return reader.Dict{"Type": reader.Name("StructTreeRoot"), + "RoleMap": reader.Dict{"Sect": reader.Name("Div")}} + }}, + {"a root whose only child is a mark", func(g *tagged) reader.Dict { + return reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(0)}} + }}, + {"a root whose only child is an annotation", func(g *tagged) reader.Dict { + return reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Dict{"Type": reader.Name("OBJR"), "Obj": g.annot[0]}} + }}, + {"a root whose children are not elements", func(g *tagged) reader.Dict { + return reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Array{reader.Name("nonsense"), reader.Integer(3)}} + }}, + } { + t.Run(c.name, func(t *testing.T) { + g := newTagged() + src := g.finish(t, c.root(g)) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + if got := catalog.Get("StructTreeRoot"); got.Kind() != reader.KindNull { + t.Errorf("a tree with nothing in it was written as %v", got) + } + for i := 1; i <= d.PageCount(); i++ { + page, _ := d.Page(i) + if got := page.Get("StructParents"); got.Kind() != reader.KindNull { + t.Errorf("page %d says it is filed under %v", i, got) + } + } + }) + } +} + +func TestAnElementThatNamesNothingIsDropped(t *testing.T) { + g := newTagged() + // Each of these describes nothing that is still there, and each must go + // rather than be written naming something that is not. + loop := g.w.Reserve() + g.w.Put(loop, reader.Dict{"Type": reader.Name("StructElem"), "S": reader.Name("Loop"), + "Pg": g.page[0], "K": loop}) + deep := g.page[0] + chain := reader.Object(reader.Integer(0)) + for i := 0; i < maxStructDepth+4; i++ { + chain = g.elem("Deep", reader.Dict{"Pg": deep}, chain) + } + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Array{ + // A page written into the element rather than referred to. + g.elem("A", reader.Dict{"Pg": reader.Dict{"Type": reader.Name("Page")}}, + reader.Integer(0)), + // A page that is not a page of this document. + g.elem("B", reader.Dict{"Pg": g.notAPage}, reader.Integer(0)), + // An empty element on a page written into it, which cannot be + // placed and so is not the shape of anything. + g.elem("C", reader.Dict{"Pg": g.notAPage}), + // A mark with no page anywhere above it. + g.elem("D", nil, reader.Integer(0)), + // Marks no page could have. + g.elem("E", reader.Dict{"Pg": g.page[0]}, + reader.Integer(-1), reader.Integer(maxMarksPerPage)), + // A child that is neither a mark nor a dictionary. + g.elem("F", reader.Dict{"Pg": g.page[0]}, reader.Name("nonsense")), + // A marked-content reference with no mark number. + g.elem("G", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR")}), + // A reference to an annotation this document does not have. + g.elem("H", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("OBJR"), "Obj": g.notAPage}), + // An element that is its own child. + loop, + // A chain deeper than anything a document has. + chain, + // One that does survive, so that there is a tree to look at. + g.elem("Keeper", reader.Dict{"Pg": g.page[0]}, reader.Integer(1)), + }} + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + tree := treeOf(t, d, catalog) + kids := kidsOf(t, d, tree) + if len(kids) != 1 || namesOf(kids)[0] != "Keeper" { + t.Fatalf("%d elements survived (%v), wanted only the keeper", len(kids), namesOf(kids)) + } +} + +func TestAMarkInsideAStreamThePageDraws(t *testing.T) { + g := newTagged() + notAStream := g.w.Add(reader.Dict{"Type": reader.Name("Whatever")}) + keyless := g.w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form")}, + Raw: []byte("/P <> BDC BT ET EMC\n")}) + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Array{ + // Inside the form XObject the first page draws: carried, and filed + // under the number that stream carries. + g.elem("Kept", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": g.stream}), + // Inside a stream no page draws. + g.elem("Loose", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(5), + "Stm": g.loose}), + // A stream written into the reference rather than referred to. + g.elem("Direct", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": reader.Dict{}}), + // Something that is not a stream at all. + g.elem("NotAStream", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": notAStream}), + // A stream that says nothing about where its marks are filed. + g.elem("Keyless", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": keyless}), + // A mark inside a stream, with no page anywhere above it. + g.elem("Unplaced", nil, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": g.stream}), + }} + // The first page has to draw the keyless stream too, or it would be + // refused for that rather than for having no key. + g.draw("Fm1", keyless) + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + tree := treeOf(t, d, catalog) + kids := kidsOf(t, d, tree) + if len(kids) != 1 { + t.Fatalf("%d elements survived (%v), wanted only the one whose stream the page draws", + len(kids), namesOf(kids)) + } + kept, _ := reader.ToDict(kids[0]) + mcr, ok := reader.ToDict(kidsOf(t, d, kept)[0]) + if !ok { + t.Fatal("the mark inside the stream is gone") + } + stm, ok := mcr.Get("Stm").(reader.Ref) + if !ok { + t.Fatalf("the reference names %v rather than a stream of this file", mcr.Get("Stm")) + } + // The copy of the stream keeps the number it was filed under, so the + // numbers handed to the pages must start above it. + stream, ok := reader.ToStream(mustGet(t, d, stm)) + if !ok { + t.Fatal("the stream the mark is in was not carried") + } + key, ok := reader.ToInt(stream.Dict.Get("StructParents")) + if !ok { + t.Fatal("the copied stream says nothing about where its marks are filed") + } + nums := numsOf(t, d, tree) + arr, ok := reader.ToArray(nums[key]) + if !ok || len(arr) != 5 { + t.Fatalf("the stream's key %d holds %v", key, nums[key]) + } + if _, ok := arr[4].(reader.Ref); !ok { + t.Errorf("mark four of the stream is owned by %v", arr[4]) + } + for i := 1; i <= d.PageCount(); i++ { + page, _ := d.Page(i) + if got, ok := reader.ToInt(page.Get("StructParents")); ok && got <= key { + t.Errorf("page %d was handed number %d, which the stream already holds", i, got) + } + } +} + +func TestAnElementWhoseOwnPageGoesButWhoseContentStays(t *testing.T) { + // Its own page is gone and some of its content is not. Saying nothing + // about a page would leave it inheriting whatever is above it, which is a + // page it is not on — the mistake worth avoiding, since a reader trusts it. + g := newTagged() + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Array{ + g.elem("Displaced", reader.Dict{"Pg": g.page[1]}, + g.elem("P", reader.Dict{"Pg": g.page[0]}, reader.Integer(0))), + // One whose only content is an annotation on a page that stays. + g.elem("Standing", reader.Dict{"Pg": g.page[1]}, + reader.Dict{"Type": reader.Name("OBJR"), "Obj": g.annot[0]}), + }} + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Select("1")) }) + tree := treeOf(t, d, catalog) + kids := kidsOf(t, d, tree) + if len(kids) != 2 { + t.Fatalf("%d elements survived (%v), wanted both", len(kids), namesOf(kids)) + } + only, _ := d.PageRef(1) + displaced, _ := reader.ToDict(kids[0]) + if pg, ok := displaced.Get("Pg").(reader.Ref); !ok || pg != only { + t.Errorf("the displaced element says it is on %v, wanted the one page left at %v", + displaced.Get("Pg"), only) + } + standing, _ := reader.ToDict(kids[1]) + if got := standing.Get("Pg"); got.Kind() != reader.KindNull { + t.Errorf("the element standing for an annotation claims page %v of its own", got) + } +} + +func TestMarksNamedTheLongWay(t *testing.T) { + g := newTagged() + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": reader.Array{ + // A reference that says which page the mark is on itself. + g.elem("Kept", nil, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(1), + "Pg": g.page[0]}), + // One naming a mark no page could have. + g.elem("Gone", nil, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(-1), + "Pg": g.page[0]}), + }} + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + tree := treeOf(t, d, catalog) + kids := kidsOf(t, d, tree) + if len(kids) != 1 || namesOf(kids)[0] != "Kept" { + t.Fatalf("%d elements survived (%v), wanted only the one naming a real mark", + len(kids), namesOf(kids)) + } + kept, _ := reader.ToDict(kids[0]) + mcr, ok := reader.ToDict(kidsOf(t, d, kept)[0]) + if !ok { + t.Fatal("the reference is gone") + } + first, _ := d.PageRef(1) + if pg, ok := mcr.Get("Pg").(reader.Ref); !ok || pg != first { + t.Errorf("the reference says the mark is on %v, wanted page one at %v", mcr.Get("Pg"), first) + } +} + +func TestTwoDrawnStreamsAndOneIdentifierTwice(t *testing.T) { + g := newTagged() + other := g.w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "StructParents": reader.Integer(2), + }, Raw: []byte("/P <> BDC BT ET EMC\n")}) + g.draw("Fm1", other) + // Something the page draws that is not a stream at all. + g.draw("Fm2", g.notAPage) + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "IDTree": reader.Dict{"Names": reader.Array{}}, + "K": reader.Array{ + g.elem("A", reader.Dict{"Pg": g.page[0], "ID": reader.String("aaa")}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(4), + "Stm": g.stream}), + g.elem("B", reader.Dict{"Pg": g.page[0], "ID": reader.String("aaa")}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(1), + "Stm": other}), + // A reference into something the page draws that is not a stream. + g.elem("C", reader.Dict{"Pg": g.page[0]}, + reader.Dict{"Type": reader.Name("MCR"), "MCID": reader.Integer(1), + "Stm": g.notAPage}), + }} + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + tree := treeOf(t, d, catalog) + kids := kidsOf(t, d, tree) + if len(kids) != 2 { + t.Fatalf("%d elements survived (%v), wanted the two whose streams are streams", + len(kids), namesOf(kids)) + } + // Two streams, each keeping its own number, and both below the numbers the + // pages were handed. + nums := numsOf(t, d, tree) + for _, key := range []int64{2, 7} { + if _, ok := reader.ToArray(nums[key]); !ok { + t.Errorf("the number tree holds %v under the stream's own key %d", nums[key], key) + } + } + for i := 1; i <= d.PageCount(); i++ { + page, _ := d.Page(i) + if got, ok := reader.ToInt(page.Get("StructParents")); ok && got <= 7 { + t.Errorf("page %d was handed number %d, which a stream already holds", i, got) + } + } + // One identifier naming two elements: a name tree has one entry per key. + ids, _ := d.GetDict(tree, "IDTree") + names, _ := reader.ToArray(ids.Get("Names")) + if len(names) != 2 { + t.Errorf("the identifier tree holds %v, wanted one entry", names) + } +} + +func TestResourcesNestedDeeperThanAnythingReal(t *testing.T) { + // The walk that settles which numbers are already taken has to stop + // somewhere, and a file can nest a dictionary without end. + g := newTagged() + nest := reader.Object(reader.Dict{"StructParents": reader.Integer(4)}) + for i := 0; i < maxStructDepth+4; i++ { + nest = reader.Dict{"Deeper": nest} + } + g.drawn["Nest"] = nest + root := reader.Dict{"Type": reader.Name("StructTreeRoot"), + "K": g.elem("Keeper", reader.Dict{"Pg": g.page[0]}, reader.Integer(0))} + src := g.finish(t, root) + d, catalog := rebuilt(t, src, func(doc *Doc) { mustDo(t, doc.Rotate("all", 90)) }) + tree := treeOf(t, d, catalog) + if kids := kidsOf(t, d, tree); len(kids) != 1 { + t.Fatalf("%d elements survived (%v)", len(kids), namesOf(kids)) + } +} diff --git a/write.go b/write.go index 11c4602..db772bb 100644 --- a/write.go +++ b/write.go @@ -18,6 +18,16 @@ type pageKey struct { num int } +// A builtPage is one page of the file being written: where it was borrowed +// from, the number it was given here, and its dictionary, which is still open +// for changes until it is put down. +type builtPage struct { + src *reader.Document + num int + ref reader.Ref + dict reader.Dict +} + // Bytes writes the document out as a PDF file. Every borrowed page is copied // out of the file it came from with the attributes it inherited written onto // it, so pages from different documents keep their own geometry and resources. @@ -54,6 +64,18 @@ func (d *Doc) Bytes() ([]byte, error) { for i, p := range d.pages { dicts[i] = d.buildPage(w, p, pagesRef, where, kept) } + + // The catalogue is rebuilt before the pages are put down, not after: what + // it carries across may have something to write onto a page — the number + // under which a page's structure is filed — and a page that has already + // been written cannot be added to. + built := make([]builtPage, len(d.pages)) + for i, p := range d.pages { + built[i] = builtPage{src: p.src, num: p.number, ref: refs[i], dict: dicts[i]} + } + catalog := reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef} + d.keepCatalogue(w, catalog, kept, built) + kids := make(reader.Array, 0, len(d.pages)) for i := range d.pages { w.Put(refs[i], dicts[i]) @@ -64,9 +86,6 @@ func (d *Doc) Bytes() ([]byte, error) { "Kids": kids, "Count": reader.Integer(len(kids)), }) - - catalog := reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef} - d.keepCatalogue(w, catalog, kept) kept.write(w) if outlines := d.writeOutlines(w, where, refs); outlines != nil { catalog["Outlines"] = outlines @@ -115,6 +134,13 @@ var rebuiltPageKeys = map[reader.Name]bool{ "Rotate": true, // this document decides which way up a page goes "Annots": true, // rebuilt so that links can be remapped "AA": true, // a page's own actions run without anyone asking + + // A page's number in the structure tree's parent tree is handed out + // afresh, because the pages are not the ones the source had. Keeping the + // number a page had in a file it is no longer part of is worse than + // dropping it: a reader would look the number up and be told, with every + // confidence, about somebody else's page. + "StructParents": true, } // sanitisedPageKeys are the entries of a page that only a sanitised file