diff --git a/README.md b/README.md index c06b7d3..4ca9f45 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,278 @@ # latex — go-pdfkit -Reconstruct LaTeX source from a PDF. +[![License](https://img.shields.io/badge/license-BSD--3--Clause-blue)](LICENSE) +[![Go](https://img.shields.io/badge/go-1.26.4%2B-00ADD8)](https://go.dev/dl/) +[![coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](#the-gate) -Under construction. +**Reconstruct LaTeX source from a PDF.** Pure Go, no cgo, no dependencies +outside the fleet. + +```go +d, _ := reader.Open(pdfBytes) +doc, _ := latex.Reconstruct(d, latex.Options{}) +os.WriteFile("out.tex", []byte(doc.String()), 0o644) +doc.WriteFiles(".") // the pictures the \includegraphics commands name +``` + +A PDF that came from TeX does not contain a document. It contains the marks TeX +made putting one on paper: glyphs at absolute positions, from fonts whose names +are the only surviving record of what the author asked for, and a few thin +rectangles. Everything else — that this was a paragraph, that this word was +`\emph`, that these eleven glyphs and one rule were `\frac{a+b}{c}` — was +consumed by the typesetting and has to be worked out again from the geometry. + +This package does that work. It is a **reconstruction and not a recovery**: two +different sources typeset to the same page, and where the geometry cannot tell +them apart this makes a choice and says which one. + +## What it reconstructs + +**Paragraphs and lines.** Baselines into lines, lines into paragraphs. A +paragraph is found from three things at once — the first line of one is +indented, the last line of one is short, and a document that uses vertical +space instead of an indent leaves a wider gap — with the short-line test only +trusted after checking, over the whole document, that its lines reach the right +margin at all. + +**Two columns.** Found from the gutter: a strip down the middle of the page that +no glyph enters. This has to be found *before* the lines are built, because two +columns set to the same grid put their lines on the same baselines, and a page +read by baseline alone comes back with every left-hand line joined to the +right-hand line beside it. + +**Font changes.** `\textbf`, `\emph`, `\texttt`, `\textsf`, `\textsc`, read from +the names of the fonts the text was set in — CMBX10, CMTI10, LMRoman10-Bold, +NimbusRomNo9L-Medi. Nothing looks inside the font program: a font's own idea of +its weight is often wrong, and a subsetted face carries no useful flags at all. + +**Section headings and their levels.** From the number the author gave them +where there is one — "3.2.1" is a `\subsubsection` and nothing else — and from +the ranking of heading sizes across the document where there is not. An +unnumbered heading comes back starred. The title of a paper becomes `\title` +and `\maketitle`. + +**Mathematics.** The part that matters, and the part that is difficult. + +- **Super- and subscripts**, recursively: a smaller glyph raised or dropped. +- **Fractions**: a horizontal rule with material above and below it. Both + producers are read — pdfTeX fills a rectangle, xdvipdfmx (what XeTeX and + tectonic write through) strokes a segment — because a reader that knows only + one of them silently loses every fraction in half the world's PDFs. +- **Radicals**: the hook glyph plus the bar drawn over the radicand, which is + what says how far the radicand reaches. +- **Big operators with their limits**, recognised by the limits sitting *over* + the operator rather than after it. +- **`\left(` … `\right)`** from the extension family's grown delimiters, when + they pair up in the equation; when they do not, the plain character. +- **Greek and about 250 symbols**, and the operator names — `\sin`, `\log`, + `\max` — that TeX sets upright. +- **Numbered displays** come back as an `equation` environment with the number + taken off, since writing it back would give the equation two numbers. + +**Where an equation starts and stops** is itself a reconstruction: TeX sets the +`$` and the text around it in the same roman font, so the boundary leaves no +trace. The three math families are the seeds; around each one the equation is +grown over the characters the roman font also drew — the digits, the `+`, the +parentheses — recognised by being the sort of character an equation contains +*and* by being closer than a word space. Both halves matter. There is also a +rule for one widely used font package, Fourier, which sets its math *letters* in +the text italic face, so that "(L, P)" is drawn with roman parentheses and +italic letters and nothing in any font name says it is an equation: a letter set +in italic, alone, with roman on both sides of it, is a variable, because +emphasis applies to words and a word is more than one letter. + +**Pictures**, as `\includegraphics` with the file written out beside the source. +A JPEG or a JPEG 2000 is written as it stands; plain samples in DeviceGray, +DeviceRGB or a stencil mask become a PNG; anything else becomes a `\framebox` of +the right size rather than a command pointing at a file that is not there. + +**Output that compiles.** The last thing done to an equation is to read it back +the way TeX will and repair what TeX would refuse: a script with nothing to be +the script of, two scripts of one kind on one letter, a brace left open, a +`\left` whose `\right` was on the next line. This is not a nicety — an aborted +compile loses the whole document rather than the one equation. + +## What it does not + +- **Bibliographies, citations, cross-references, labels.** A `\cite` that + typeset to "[14]" comes back as the characters `[14]`; there is nothing on the + page that says otherwise. Same for `\ref` and `\label`. +- **Tables.** The rules are read (a fraction needs them) but nothing is made of + a grid of them. A table comes back as the lines of text it is made of. +- **Footnotes**, which come back as text at the foot of the page. +- **Telling a class's italic from the author's `\emph`.** A theorem environment + sets its body in italic, and the page records only the italic. Every run of it + comes back as `\emph{...}`, which typesets the same and reads nothing like the + source: `\emph` is emitted about six times as often as the authors wrote it, + and `\textbf` about three. The same applies to any class that sets something + bold or italic on its own account. +- **A figure drawn rather than placed.** `\includegraphics` comes back for a + picture the page places as an image; a plot that the author included as a + vector PDF is not an image at all but a few thousand drawing operations + inlined into the page, and this recovers about a third of the + `\includegraphics` commands in the corpus for that reason. +- **Colour.** +- **Accents.** A PDF draws `\hat{L}` as an L and a circumflex placed over it, + two glyphs, and there is no way to write that back without knowing which glyph + the accent belongs to. The accent is dropped and the letter kept, rather than + a character being written into the source that no engine will set. +- **The abstract environment**, which comes back as a centred paragraph. +- **The preamble.** It is reconstructed from what the body needs — amsmath, + amssymb, graphicx, and the page geometry measured off the first page — rather + than from what the author wrote, which no PDF records. +- **Whitespace fidelity.** +- **A page whose producer scales text with the text matrix rather than with the + font size.** `github.com/go-pdfkit/extract` reports the font size, and a + producer that sets `/F1 1 Tf` and scales by twelve reports a size of one for + everything on the page. TeX-produced PDFs — which is what this is for — set + the real size; drawing programs often do not, and on such a page the size + tests that find headings and scripts have nothing to work with. + + +## Measured + +On arXiv source packages held locally: **399 papers attempted, 275 typeset by +tectonic** (a real TeX engine) into the PDFs this then read back — **7 262 +pages**. The other 124 would not compile at all: they want packages, classes or +figures that tectonic could not supply. Everything below is the distribution +over those 275, not one example of one. + +### Does it compile? + +**274 of 275 (99.6%)** of the reconstructions are accepted by tectonic. + +That number is the whole point of the repair pass, and it was not free. Over the +same set of papers, as the faults were found and fixed, it went + +| | reconstructions that compile | +|---|---| +| before any repair | **27%** | +| after the first repair pass | **55%** | +| after three more faults were fixed | **97–99%** | + +The faults were: a script with nothing to be the script of; two scripts of one +kind on one letter; a brace left open; a `\left` whose `\right` was on the next +line or inside a different group; a prime, which TeX reads as a superscript, so +a following subscript lands on a letter that already has one; a bare `\sqrt` +with no radicand; and — the one that no amount of reading the code would have +found — that `\rightarrow` begins with the six characters of `\right`, so the +first version of the pass that strips an unmatched `\left` turned every limit in +every document into the undefined command `\thetaarrow`. + +### How close is it to what the author wrote? + +The reconstruction is compared with the paper's own `.tex` files, with comments +stripped, commands and braces removed, and mathematics compared separately. + +| measure | q1 | **median** | q3 | +|---|---|---|---| +| words, F1 of the bag | 0.56 | **0.67** | 0.75 | +| word bigrams, F1 (order-sensitive) | 0.37 | **0.49** | 0.58 | +| mathematics, F1 of the token multiset | 0.32 | **0.61** | 0.75 | + +The word figure is a floor rather than a score: the denominator is the author's +whole source, which contains a preamble, macro definitions, commented-out +paragraphs and a bibliography, none of which ever reach the page and none of +which this could recover. + + +### Which of it comes back, construct by construct + +Counting the commands in the 276 reconstructions against the commands in the +`.tex` files they came from. The denominator is generous — it counts every +`.tex` in the package, including files the main document never inputs, and +macro definitions — so these are lower bounds on the right-hand side and the +ratios understate. They are still the clearest statement of what works. + +| construct | in the reconstructions | in the sources | +|---|---|---| +| `\section{` | 1 331 | 2 707 | +| `\subsection{` | 1 432 | 3 572 | +| `\begin{equation}` | 4 673 | 9 516 | +| `\frac{` | 6 607 | 24 066 | +| `\sqrt{` | 994 | 3 796 | +| `\left` | 1 832 | 17 238 | +| `\includegraphics` | 1 082 | 3 094 | +| `\texttt{` | 2 983 | 2 377 | +| `\textbf{` | 20 270 | 6 893 | +| `\emph{` | 40 253 | 6 424 | +| `\begin{tabular}` | **0** | 746 | + +Three of those rows are worth reading carefully. + +**`\emph` and `\textbf` are over-emitted, by six times and three.** A theorem +environment sets its whole body in italic, and this has no way to know that the +italic came from `\begin{theorem}` rather than from `\emph` — so it writes one +`\emph` per run of italic text. The page typesets the same; the source does not +look like the author's. The same happens to `\textbf` wherever a class sets +something bold on its own account. This costs on the comparison with the source +and nothing at all on the comparison with the page. + +**`\left` comes back one time in nine.** That is mostly correct rather than +mostly missing: a `\left(` around something that fits on one line is drawn from +the ordinary roman font at the ordinary size, and is then indistinguishable from +a plain `(`, which typesets identically. Only a delimiter that actually grew +leaves evidence, and that is what this recovers. + +**`tabular` is zero, by choice.** Tables were the last item of the brief and +were not reached; a table comes back as the lines of text it is made of. +### Round trip: typeset the reconstruction and compare the pixels + +The reconstruction is set again — once by the fleet's own `go-tex/engine`, once +by tectonic — and each rendering is compared with the original page by +`go-pdfkit/render`. + +The comparison is **where the ink lands**, not the mean pixel difference: a dark +pixel in one page counts as matched when there is a dark pixel within two pixels +of it in the other, scored as an F1. Mean absolute difference has a blind spot +on a page that is mostly white — drawing the right thing one pixel off scores +worse than drawing nothing — and this corpus shows it plainly, below. + +The engine's own fidelity has to be separated from the reconstruction's, so the +same engine also sets the paper's **true source**, and the two are compared: + +| page one of 275 papers, ink F1 against the original | q1 | **median** | q3 | +|---|---|---|---| +| go-tex sets **the reconstruction** | 0.32 | **0.400** | 0.47 | +| go-tex sets **the author's own source** | 0.19 | **0.315** | 0.43 | +| tectonic sets **the reconstruction** | 0.26 | **0.320** | 0.40 | + +**The reconstruction lands closer to the original page than the true source +does, in 274 of 275 papers**, when both are set by the fleet's engine. That is +not a claim that the reconstruction is better than the source. It is a statement +about what the engine can read: the reconstruction is plain `article` LaTeX with +amsmath and graphicx, which go-tex sets in full, while a real arXiv paper pulls +in classes and packages it drops. A reconstruction faithful enough to stand in +for the source under an engine that cannot read the source is the useful thing +being measured here. + +### The blind spot, demonstrated + +The same 275 comparisons, scored by **mean absolute pixel difference** instead: + +| | q1 | **median** | q3 | +|---|---|---|---| +| go-tex sets the reconstruction | 0.071 | **0.090** | 0.104 | +| go-tex sets the author's own source | 0.054 | **0.075** | 0.093 | + +By that measure the reconstruction is *worse* in **274 of 275** — the exact +opposite verdict. The reason is that go-tex sets less of the true source than of +the reconstruction, so its page is emptier, and on a page that is 95% white an +emptier page is nearer the original by mean difference however much of the +document it has lost. Both numbers are reported because only one of them is +answering the question. + +### What it costs + +Median 18 seconds per paper end to end, which is dominated by the two TeX +compiles; the reconstruction itself is a fraction of a second per page. +## The gate + +`go vet` clean, `gofmt` clean, `CGO_ENABLED=0`, **exact 100% statement +coverage**, and a build for linux amd64/arm64/riscv64/loong64/ppc64le/s390x, +js/wasm, darwin/arm64 and windows/amd64. Nothing outside the standard library +and `github.com/go-pdfkit/{reader,extract,pdffont}`. + +## Licence + +BSD-3-Clause. diff --git a/atom.go b/atom.go new file mode 100644 index 0000000..e71ec19 --- /dev/null +++ b/atom.go @@ -0,0 +1,228 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import ( + "sort" + + "github.com/go-pdfkit/extract" + "github.com/go-pdfkit/reader" +) + +// An atom is one piece of text on the page together with what its font says +// about it. It is what github.com/go-pdfkit/extract gives back, with the +// font's name looked up in the page's resources and read (see font.go), since +// the resource name a run carries — F10, TT0 — says nothing on its own. +type atom struct { + text string + // x and y are where the piece starts and the baseline it sits on, in + // points up from the bottom left of the visible page. + x, y float64 + // width is how far the pen moved drawing it and size how tall it is. + width, size float64 + // space is how wide a space would be in this font at this size, which is + // what decides whether a gap between two pieces is a word break. + space float64 + sh shape +} + +// right is where the piece ends. +func (a atom) right() float64 { return a.x + a.width } + +// midX is its horizontal centre. +func (a atom) midX() float64 { return a.x + a.width/2 } + +// page is everything one page of a document says, in the form the rest of this +// package works on. +type page struct { + atoms []atom + rules []rule + images []extract.Image + // width and height are the visible page's own size in points. + width, height float64 +} + +// readPage reads the i'th page, counting from one. +func readPage(d *reader.Document, i int) (page, error) { + runs, err := extract.Runs(d, i) + if err != nil { + return page{}, err + } + // The page dictionary cannot fail to be read once extract has read the + // page itself: both go through the same page tree, and it has just been + // walked. + dict, _ := d.Page(i) + box := visibleBox(d, dict) + shapes := fontShapes(d, dict) + p := page{ + width: box[2] - box[0], + height: box[3] - box[1], + rules: rules(d, i, point{box[0], box[1]}), + } + p.images, _ = extract.Images(d, i) + for _, r := range runs { + // Text drawn in the invisible mode is kept. It is how a scanner + // puts what it read underneath the picture it read it from, and on + // such a page it is the only text there is. + if r.Text == "" { + continue + } + p.atoms = append(p.atoms, atom{ + text: r.Text, + x: r.X, + y: r.Y, + width: r.Width, + size: r.Size, + space: r.Space, + sh: shapes[r.Font], + }) + } + sort.SliceStable(p.atoms, func(i, j int) bool { + if p.atoms[i].y != p.atoms[j].y { + return p.atoms[i].y > p.atoms[j].y + } + return p.atoms[i].x < p.atoms[j].x + }) + return p, nil +} + +// fontShapes reads every font the page names, so that a run's resource name can +// be turned into what the font says about the text. A form XObject the page +// draws has resources of its own, whose names may collide with the page's; the +// page's win, which is the wrong answer only for a document that reuses one +// name for two different faces. +func fontShapes(d *reader.Document, pageDict reader.Dict) map[string]shape { + out := map[string]shape{} + collectFonts(d, pageDict, out, 0) + return out +} + +// maxResourceDepth is how deeply this follows a form's own resources before +// taking the page to be drawing itself. +const maxResourceDepth = 8 + +// collectFonts adds the fonts of one resource-carrying dictionary, and of the +// forms it names, without overwriting a name already found. +func collectFonts(d *reader.Document, dict reader.Dict, out map[string]shape, depth int) { + if depth > maxResourceDepth { + return + } + res, ok := d.GetDict(dict, "Resources") + if !ok { + return + } + if fonts, ok := d.GetDict(res, "Font"); ok { + for name, ref := range fonts { + if _, seen := out[string(name)]; seen { + continue + } + f, ok := d.GetDict(reader.Dict{"f": ref}, "f") + if !ok { + continue + } + out[string(name)] = classify(fontName(d, f)) + } + } + if xobj, ok := d.GetDict(res, "XObject"); ok { + for _, ref := range xobj { + form, ok := d.GetDict(reader.Dict{"f": ref}, "f") + if !ok { + continue + } + collectFonts(d, form, out, depth+1) + } + } +} + +// fontName is the name a font dictionary gives itself. A Type0 font keeps the +// interesting part in its descendant, whose /BaseFont says the same thing, so +// the outer one is enough; a font with no name at all comes back empty and is +// classified as plain roman, which is what an unnamed font usually is. +func fontName(d *reader.Document, f reader.Dict) string { + o, err := d.Resolve(f.Get("BaseFont")) + if err != nil { + return "" + } + if n, ok := o.(reader.Name); ok { + return string(n) + } + return "" +} + +// visibleBox is the area of the page that is shown, which is what extract +// reports its coordinates against. +func visibleBox(d *reader.Document, pageDict reader.Dict) [4]float64 { + for _, key := range []reader.Name{"CropBox", "MediaBox"} { + if b, ok := rectangle(d, pageDict.Get(key)); ok { + return b + } + } + return [4]float64{0, 0, 612, 792} +} + +// rectangle reads a PDF rectangle, put the right way round. +func rectangle(d *reader.Document, o reader.Object) ([4]float64, bool) { + var out [4]float64 + r, err := d.Resolve(o) + if err != nil { + return out, false + } + arr, ok := reader.ToArray(r) + if !ok || len(arr) < 4 { + return out, false + } + for i := 0; i < 4; i++ { + v, err := d.Resolve(arr[i]) + if err != nil { + return out, false + } + f, ok := reader.ToFloat(v) + if !ok { + return out, false + } + out[i] = f + } + if out[0] > out[2] { + out[0], out[2] = out[2], out[0] + } + if out[1] > out[3] { + out[1], out[3] = out[3], out[1] + } + return out, true +} + +// The bounds a word space is believed within, as a fraction of the font's size. +// Every text face ever cut sets a space somewhere between a fifth and a half of +// its size; a figure outside that did not come from the font. +const ( + minSpace = 0.22 + maxSpace = 0.55 + // fallbackSpace is what a space is taken to be when the font's own + // figure is not believable. A third of the size is what the roman faces + // TeX ships use. + fallbackSpace = 0.33 +) + +// wordSpace is how wide a word space is here, which is what says whether a gap +// between two pieces of text is a word break or only kerning. +// +// It is not simply what the font says, because what the font says is sometimes +// not a space at all. A composite font is addressed by two-byte codes and has +// no character at 32; asking it how wide a space is returns whatever its +// fallback width happens to be, and one real producer in this corpus answers +// with the font size itself. Taking that at face value makes every gap on the +// page look like kerning, and a whole document comes back as one long word. So +// an answer outside what a space can be is replaced by what a space usually is. +// +// A math font has no word spaces at all, and says so by returning nothing: +// there are no word breaks inside an equation. +func (a atom) wordSpace() float64 { + if a.sh.isMath() { + return 0 + } + if a.space < minSpace*a.size || a.space > maxSpace*a.size { + return fallbackSpace * a.size + } + return a.space +} diff --git a/block.go b/block.go new file mode 100644 index 0000000..d47b57a --- /dev/null +++ b/block.go @@ -0,0 +1,670 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import ( + "fmt" + "sort" + "strings" + + "github.com/go-pdfkit/extract" +) + +// This file decides what each line of the page was, and writes it out. +// +// A PDF has no paragraphs. It has lines, and the paragraph has to be inferred +// from three things TeX leaves behind: the first line of one is indented, the +// last line of one is short, and a document that uses vertical space instead of +// an indent leaves a wider gap. The first two are only usable together with a +// check that the document is justified at all — in ragged-right text every line +// is short and the "last line" test fires on all of them, which is why it is +// measured over the whole document before it is trusted on any one line. +// +// A heading is a line set larger or bolder than the body and short enough not to +// be a line of it. Where the author numbered their sections the number is still +// there at the front of the line, and it says the level exactly — "3.2.1" is a +// \subsubsection and nothing else — which is far better evidence than the size. +// Where they did not, the sizes are ranked across the document and the ranking +// gives the level. + +// A frame is one page, ready to be written out. +type frame struct { + lines []line + rules []rule + images []extract.Image + twoColumn bool +} + +// An emitter writes the body of the reconstructed document. +type emitter struct { + opt Options + out strings.Builder + files []File + // body is the size the document's running text is set at, and levels + // the distinct sizes its headings use, largest first. + body float64 + levels []float64 + // justified says whether the document's lines reach the right margin, + // which is what makes a short line evidence of a paragraph ending. + justified bool + // leading is how far apart the document set its lines. + leading float64 + // para is the paragraph being built and prev the line it last took. + para []string + prev line + hasPrev bool + // paraRight is how far right the paragraph being built has reached, + // which is what a short line is short against. + paraRight float64 + // centred says the paragraph being built is set in the middle of its + // column, so that a block of centred lines comes out as one \begin{center} + // rather than as one per line. + centred bool + images int + title string + // opening says the emitter is still on the first page, which is the + // only place a title can be. + opening bool +} + +// indentShare is how far a line must start beyond its column's margin to be the +// first of a paragraph, as a fraction of the body size. LaTeX's \parindent for a +// ten-point article is fifteen points, so half the body size is a wide margin +// for error either way. +const indentShare = 0.5 + +// paragraphGap is how much more than a line's own leading a vertical gap must +// be before it ends a paragraph on its own. +const paragraphGap = 1.7 + +// shortLine is how far short of the right margin a line must stop, in body +// sizes, to be the last of a paragraph. +const shortLine = 2.5 + +// centreShare is how near a column's centre a line's centre must be, as a +// fraction of the column's width, to be centred. +const centreShare = 0.06 + +// write puts one page into the document. +func (e *emitter) write(f frame) { + for _, it := range interleave(f) { + if it.image != nil { + e.flush() + e.figure(*it.image) + continue + } + e.line(*it.line, f.rules) + } +} + +// An item is something to be written, from a page read top to bottom. +type item struct { + line *line + image *extract.Image + y float64 +} + +// interleave puts a page's pictures among its lines, so that a figure comes out +// where it sat rather than all of them at the end. +func interleave(f frame) []item { + items := make([]item, 0, len(f.lines)+len(f.images)) + for i := range f.lines { + items = append(items, item{line: &f.lines[i], y: f.lines[i].y}) + } + for i := range f.images { + items = append(items, item{image: &f.images[i], y: f.images[i].Y + f.images[i].DrawnHeight}) + } + // The lines are already in reading order, which on a two-column page is + // not the order they sit in; sorting by height would undo that, so the + // pictures are put in by position and the lines left where they are. + sort.SliceStable(items, func(a, b int) bool { + return items[a].image != nil && items[b].image == nil && items[a].y > items[b].y + }) + return items +} + +// line writes one line, as whatever it turns out to be. +func (e *emitter) line(l line, rs []rule) { + segs := segments(l) + if len(segs) == 0 { + return + } + switch { + case e.heading(l, segs): + e.flush() + e.section(l, segs) + case display(l, segs): + e.flush() + e.display(l, segs, rs) + default: + e.text(l, segs, rs) + } + e.prev, e.hasPrev = l, true +} + +// text adds a line to the paragraph being built, starting a new one where the +// evidence says the last ended. +func (e *emitter) text(l line, segs []segment, rs []rule) { + if c := centred(l); c != e.centred && len(e.para) > 0 { + e.flush() + e.centred = c + } else if len(e.para) == 0 { + e.centred = c + } + if e.breaks(l) { + e.flush() + } + e.paraRight = max(e.paraRight, l.x1) + e.para = append(e.para, lineText(l, segs, rs, false)) +} + +// breaks reports whether a line starts a new paragraph. +func (e *emitter) breaks(l line) bool { + if !e.hasPrev || len(e.para) == 0 { + return false + } + if e.prev.y-l.y > paragraphGap*e.body { + return true + } + // Neither of the horizontal tests means anything in a centred block, + // where every line starts and stops somewhere different by design. + if e.centred { + return false + } + switch { + case l.x0 > e.prev.x0+indentShare*e.body && l.x0 > l.left+indentShare*e.body: + return true + case e.justified && e.prev.x1 < e.paraRight-shortLine*e.body: + return true + } + return false +} + +// flush ends the paragraph being built. +func (e *emitter) flush() { + if len(e.para) == 0 { + return + } + s := mend(e.para) + if e.centred { + s = "\\begin{center}\n" + s + "\n\\end{center}" + } + e.paragraph(s) + e.para = e.para[:0] + e.paraRight = 0 +} + +// paragraph writes a finished paragraph. +func (e *emitter) paragraph(s string) { + if strings.TrimSpace(s) == "" { + return + } + e.out.WriteString(s) + e.out.WriteString("\n\n") +} + +// mend joins a paragraph's lines. A line that ends in a hyphen and is followed +// by a lowercase letter was broken by TeX rather than by the author, so the +// hyphen goes and the word is put back together; a hyphen followed by anything +// else was written. +func mend(parts []string) string { + var b strings.Builder + for i, p := range parts { + if i > 0 { + s := b.String() + switch { + case !brokenWord(s): + b.WriteByte(' ') + case startsLower(p): + // TeX put the hyphen in, so it comes out again. + b.Reset() + b.WriteString(s[:len(s)-1]) + } + } + b.WriteString(p) + } + return b.String() +} + +// startsLower reports whether text begins with a lowercase letter. +func startsLower(s string) bool { + for _, r := range s { + return r >= 'a' && r <= 'z' + } + return false +} + +// heading reports whether a line is a section heading. +func (e *emitter) heading(l line, segs []segment) bool { + if allMath(segs) || l.x1-l.x0 > 0.9*(l.right-l.left) { + return false + } + if l.size > e.body*1.08 { + return true + } + return l.size > e.body*0.9 && allBold(l) +} + +// section writes a heading at the level the document says it is. +func (e *emitter) section(l line, segs []segment) { + text := strings.TrimSpace(lineText(l, segs, nil, true)) + number, rest := splitNumber(text) + level := len(number) + if level == 0 { + level = e.rank(l.size) + } + if e.opening && e.title == "" && number == nil && centred(l) && l.size >= e.levels[0] { + e.title = rest + return + } + star := "*" + if number != nil { + star = "" + } + e.out.WriteString(sectionName(level) + star + "{" + rest + "}\n\n") +} + +// sectionNames are the levels a heading may be written at. A document nested +// deeper than this is written at the deepest one rather than being given a level +// LaTeX has no command for. +var sectionNames = []string{`\section`, `\subsection`, `\subsubsection`, `\paragraph`} + +// sectionName is the command for a level, counting from one. +func sectionName(level int) string { + if level > len(sectionNames) { + level = len(sectionNames) + } + return sectionNames[level-1] +} + +// rank is the level a heading of a given size sits at, from where that size +// falls among all the heading sizes in the document. +func (e *emitter) rank(size float64) int { + for i, s := range e.levels { + if size > s*0.99 { + return i + 1 + } + } + return len(e.levels) +} + +// splitNumber takes a section number off the front of a heading and returns its +// parts and what is left. "3.2 Results" is a \subsection because the number has +// two parts, which is better evidence of the level than any measurement. +func splitNumber(s string) ([]string, string) { + i := 0 + for i < len(s) && (isDigit(s[i]) || s[i] == '.' || (i == 0 && isASCIILetter(s[i]) && len(s) > 1 && s[1] == '.')) { + i++ + } + head := strings.TrimSuffix(s[:i], ".") + rest := strings.TrimSpace(s[i:]) + if head == "" || rest == "" || !isDigit(head[0]) && !isASCIILetter(head[0]) { + return nil, s + } + parts := strings.Split(head, ".") + for _, p := range parts { + if p == "" { + return nil, s + } + } + return parts, rest +} + +// isDigit reports whether a byte is a decimal digit. +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +// display reports whether a line is a displayed equation: all mathematics, and +// set apart from the margin the way TeX sets a display. +func display(l line, segs []segment) bool { + if !allMath(trimNumber(segs)) { + return false + } + width := l.right - l.left + return centred(l) || l.x0 > l.left+0.06*width || len(trimNumber(segs)) < len(segs) +} + +// display writes a displayed equation, in the numbered environment when the +// document numbered it. +func (e *emitter) display(l line, segs []segment, rs []rule) { + kept := trimNumber(segs) + body := displayBody(kept, rs) + if body == "" { + return + } + if len(kept) < len(segs) { + e.out.WriteString("\\begin{equation}\n" + body + "\n\\end{equation}\n\n") + return + } + e.out.WriteString("\\[\n" + body + "\n\\]\n\n") +} + +// trimNumber drops an equation number from the end of a line. TeX sets it at the +// right margin in the roman font, in parentheses, and it is not part of the +// equation — writing it back would give the equation two numbers. +func trimNumber(segs []segment) []segment { + if len(segs) < 2 { + return segs + } + last := segs[len(segs)-1] + if last.math || len(last.atoms) != 1 { + return segs + } + a := last.atoms[0] + l := a.text + if len(l) < 3 || l[0] != '(' || l[len(l)-1] != ')' { + return segs + } + return segs[:len(segs)-1] +} + +// allMath reports whether every piece of a line is mathematics. +func allMath(segs []segment) bool { + for _, s := range segs { + if !s.math { + return false + } + } + return len(segs) > 0 +} + +// allBold reports whether every piece of a line is set bold. +func allBold(l line) bool { + for _, a := range l.atoms { + if !a.sh.bold { + return false + } + } + return len(l.atoms) > 0 +} + +// centred reports whether a line sits in the middle of its column with space at +// both ends. +func centred(l line) bool { + width := l.right - l.left + if width <= 0 { + return false + } + return abs((l.x0+l.x1)/2-(l.left+l.right)/2) < centreShare*width && + l.x0 > l.left+centreShare*width +} + +// figure writes a picture, and keeps the file it needs. +func (e *emitter) figure(im extract.Image) { + s, f := figure(im, e.images) + if s == "" { + return + } + e.images++ + if f != nil { + e.files = append(e.files, *f) + } + e.paragraph("\\begin{center}\n" + s + "\n\\end{center}") +} + +// lineText writes one line: its equations between dollars and its text with the +// markup its fonts call for. +func lineText(l line, segs []segment, rs []rule, bare bool) string { + var b strings.Builder + for i, s := range segs { + text := s.source(rs, bare) + if text == "" { + continue + } + if i > 0 && gapBetween(segs[i-1], s) { + b.WriteByte(' ') + } + b.WriteString(text) + } + return b.String() +} + +// source is one piece of a line written out. An equation with nothing left in +// it - every glyph of it was an accent this could not place - is written as +// nothing rather than as an empty pair of dollars, which is not an empty +// equation but the start of a displayed one. +func (s segment) source(rs []rule, bare bool) string { + if !s.math { + return runText(s.atoms, bare) + } + if m := mathSource(s.atoms, mathRules(s.atoms, rs)); m != "" { + return "$" + m + "$" + } + return "" +} + +// displayBody is a displayed equation written without the dollars, since the +// environment around it opens the mathematics itself. A line that is mostly an +// equation but carries a word or two comes back with those words in \text, +// which is what an author writes for them. +func displayBody(segs []segment, rs []rule) string { + var parts []string + for _, s := range segs { + if s.math { + if m := mathSource(s.atoms, mathRules(s.atoms, rs)); m != "" { + parts = append(parts, m) + } + continue + } + if t := strings.TrimSpace(runText(s.atoms, false)); t != "" { + parts = append(parts, `\text{`+t+`}`) + } + } + return strings.Join(parts, " ") +} + +// gapBetween reports whether two neighbouring pieces of a line have a word space +// between them. +func gapBetween(left, right segment) bool { + a := left.atoms[len(left.atoms)-1] + b := right.atoms[0] + space := max(a.wordSpace(), b.wordSpace()) + if space == 0 { + return false + } + return b.x-a.right() > wordGap*space +} + +// runText writes a stretch of ordinary text, putting the word spaces back and +// wrapping each run of one style in the command that produced it. The space +// between two words of different styles goes between the two commands rather +// than inside either: \\textbf{bold} words, never \\textbf{bold } words, which +// sets a trailing space in bold and is not what the author wrote. +// +// bare suppresses the markup, for a heading: \\section already sets its argument +// bold, and wrapping it in \\textbf as well would double it. +func runText(atoms []atom, bare bool) string { + var b, run strings.Builder + cur := atoms[0].sh.textual() + flush := func() { + if bare { + b.WriteString(run.String()) + run.Reset() + return + } + b.WriteString(markup(cur, run.String())) + run.Reset() + } + for i, a := range atoms { + if i > 0 { + prev := atoms[i-1] + gap := false + if space := max(prev.wordSpace(), a.wordSpace()); space > 0 { + gap = a.x-prev.right() > wordGap*space + } + if a.sh.textual() != cur { + flush() + cur = a.sh.textual() + if gap { + b.WriteByte(' ') + gap = false + } + } + if gap { + run.WriteByte(' ') + } + } + run.WriteString(escapeText(a.text)) + } + flush() + return b.String() +} + +// measure works out, over the whole document, the size its running text is set +// at, the sizes its headings use, and whether its lines reach the right margin. +func (e *emitter) measure(frames []frame) { + weight := map[int]float64{} + reach, total := 0, 0 + for _, f := range frames { + for _, l := range f.lines { + for _, a := range l.atoms { + weight[int(a.size*20+0.5)] += a.width + } + } + } + e.body = heaviest(weight) + for _, f := range frames { + for _, l := range f.lines { + if l.size > e.body*1.08 || (l.size > e.body*0.9 && allBold(l)) { + e.levels = append(e.levels, l.size) + continue + } + total++ + if l.x1 > l.right-0.5*e.body { + reach++ + } + } + } + e.justified = total > 0 && reach*10 > total*6 + e.levels = distinct(e.levels) + e.leading = e.leadingOf(frames) +} + +// heaviest is the size the most ink was set in, which is the body size: a +// document has more running text than anything else, by a wide margin. +func heaviest(weight map[int]float64) float64 { + best, found := 0, -1.0 + for k, v := range weight { + if v > found || (v == found && k < best) { + best, found = k, v + } + } + return float64(best) / 20 +} + +// distinct is the heading sizes, largest first, with sizes within a twentieth of +// each other counted as one level. +func distinct(sizes []float64) []float64 { + sort.Float64s(sizes) + var out []float64 + for i := len(sizes) - 1; i >= 0; i-- { + if len(out) == 0 || sizes[i] < out[len(out)-1]*0.95 { + out = append(out, sizes[i]) + } + } + if len(out) == 0 { + out = []float64{1e9} + } + return out +} + +// preamble is the document's opening, which says only what the reconstruction +// actually needs: the classes of symbol it may have written, the graphics it may +// have referred to, and the shape of the page it came off. +func preamble(opt Options, f frame, width, height, size, leading float64, title string) string { + var b strings.Builder + class := opt.Class + if class == "" { + class = "article" + } + options := "" + if f.twoColumn { + options = "[twocolumn]" + } + fmt.Fprintf(&b, "\\documentclass%s{%s}\n", options, class) + b.WriteString("\\usepackage{amsmath}\n\\usepackage{amssymb}\n\\usepackage{graphicx}\n") + l, r, t, bot := textBlock(f, width, height) + fmt.Fprintf(&b, "\\usepackage[paperwidth=%.1fpt,paperheight=%.1fpt,"+ + "left=%.1fpt,right=%.1fpt,top=%.1fpt,bottom=%.1fpt]{geometry}\n", + width, height, l, r, t, bot) + if size > 0 && leading > 0 { + // The size the document was actually set at, rather than the ten + // point the class would otherwise choose. A paper whose body is nine + // point set on eleven, put back as ten on twelve, drifts a point and + // a half a line: by the twentieth line it is a whole line out, and + // every comparison with the original page is then comparing text + // against the gap between two other pieces of text. + fmt.Fprintf(&b, "\\AtBeginDocument{\\fontsize{%.2fpt}{%.2fpt}\\selectfont}\n", size, leading) + } + if title != "" { + fmt.Fprintf(&b, "\\title{%s}\n\\author{}\n\\date{}\n", title) + } + return b.String() +} + +// textBlock is the four margins the page was set with, taken from where its +// text actually sits: how far in it starts on the left, how far the longest +// line stops from the right, and the same top and bottom. Setting all four is +// worth more than setting one, because a paper is rarely symmetrical and a +// reconstruction with the wrong measure rebreaks every line in it. +// +// A page with no text on it is given an inch all round, which is the default. +func textBlock(f frame, width, height float64) (left, right, top, bottom float64) { + if len(f.lines) == 0 { + return 72, 72, 72, 72 + } + x0, x1 := width, 0.0 + y0, y1 := height, 0.0 + for _, l := range f.lines { + x0, x1 = min(x0, l.x0), max(x1, l.x1) + y0, y1 = min(y0, l.y), max(y1, l.y+l.size) + } + return x0, max(width-x1, 0), max(height-y1, 0), max(y0, 0) +} + +// brokenWord reports whether a line ended in the middle of a word. TeX breaks a +// word across two lines with a single hyphen; a line that ends in a dash the +// author wrote ends in two or three of them, and one that ends in a hyphenated +// compound - Levi-Civita broken after the Levi - ends in one as well, which is +// why the hyphen itself is only removed when what follows is lowercase. +func brokenWord(s string) bool { + return strings.HasSuffix(s, "-") && !strings.HasSuffix(s, "--") +} + +// The bounds a believable leading falls within, as a multiple of the body size. +// TeX sets a ten-point document on twelve-point leading; a paper set tighter +// than the size itself, or looser than double, has a gap between paragraphs or +// a figure in the way rather than a leading. +const ( + minLeading = 0.9 + maxLeading = 2.0 +) + +// leadingOf is how far apart the document sets its lines, which together with +// the body size is what decides where every line of the reconstruction lands. +// +// It is worth setting. A paper whose body is nine point set on eleven, put back +// as LaTeX's ten on twelve, drifts a point and a half a line: by the twentieth +// line it is a whole line out, and every measurement that compares the +// reconstruction with the original page compares text against the gap between +// two other pieces of text. Zero means the document did not say, and the class +// decides. +func (e *emitter) leadingOf(frames []frame) float64 { + var gaps []float64 + for _, f := range frames { + for i := 1; i < len(f.lines); i++ { + a, b := f.lines[i-1], f.lines[i] + if a.size < e.body*0.98 || b.size < e.body*0.98 || a.left != b.left { + continue + } + if d := a.y - b.y; d > minLeading*e.body && d < maxLeading*e.body { + gaps = append(gaps, d) + } + } + } + if len(gaps) == 0 { + return 0 + } + sort.Float64s(gaps) + return gaps[len(gaps)/2] +} diff --git a/build_test.go b/build_test.go new file mode 100644 index 0000000..6166c78 --- /dev/null +++ b/build_test.go @@ -0,0 +1,105 @@ +package latex + +import ( + "fmt" + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// A face is a font a test page draws in: the name that says what it is, and a +// map from the codes the content stream uses to the characters they stand for, +// which is how a real math font tells a reader that code 11 is alpha. +type face struct { + base string + uni map[byte]rune +} + +// pageWith builds a one-page document from a content stream and the fonts it +// names. The page is 200 by 200 unless the test says otherwise. +func pageWith(t *testing.T, content string, faces map[string]face) *reader.Document { + t.Helper() + return pagesWith(t, []string{content}, faces, [4]int{0, 0, 200, 200}) +} + +// pagesWith builds a document of several pages, all with the same fonts and box. +func pagesWith(t *testing.T, contents []string, faces map[string]face, box [4]int) *reader.Document { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + res := reader.Dict{} + if len(faces) > 0 { + fonts := reader.Dict{} + for name, f := range faces { + fonts[reader.Name(name)] = w.Add(fontDict(w, f)) + } + res["Font"] = fonts + } + kids := reader.Array{} + for _, c := range contents { + kids = append(kids, w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(box[0]), reader.Integer(box[1]), + reader.Integer(box[2]), reader.Integer(box[3])}, + "Contents": w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte(c)}), + "Resources": res, + })) + } + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": kids, "Count": reader.Integer(len(kids))}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d +} + +// fontDict is a simple font of even width, so that a test can work out where +// every character lands: half an em each. +func fontDict(w *reader.Writer, f face) reader.Dict { + widths := make(reader.Array, 0, 224) + for i := 32; i < 256; i++ { + widths = append(widths, reader.Integer(500)) + } + d := reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("Type1"), + "BaseFont": reader.Name(f.base), "FirstChar": reader.Integer(32), + "LastChar": reader.Integer(255), "Widths": widths, + "Encoding": reader.Name("WinAnsiEncoding"), + } + if f.uni != nil { + d["ToUnicode"] = w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte(toUnicode(f.uni))}) + } + return d +} + +// toUnicode writes the CMap that says what each code stands for. +func toUnicode(m map[byte]rune) string { + var b strings.Builder + b.WriteString("/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n") + b.WriteString("1 begincodespacerange\n<00> \nendcodespacerange\n") + fmt.Fprintf(&b, "%d beginbfchar\n", len(m)) + for code, r := range m { + fmt.Fprintf(&b, "<%02X> <%04X>\n", code, r) + } + b.WriteString("endbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n") + return b.String() +} + +// show is one piece of text drawn at a place, in a font, at a size. +func show(font string, size float64, x, y float64, text string) string { + // A PDF string literal escapes its backslashes and its parentheses. + lit := strings.NewReplacer(`\`, `\\`, `(`, `\(`, `)`, `\)`).Replace(text) + return fmt.Sprintf("BT /%s %g Tf 1 0 0 1 %g %g Tm (%s) Tj ET\n", font, size, x, y, lit) +} + +// bar is a filled rectangle, which is how pdfTeX draws a rule. +func bar(x, y, w, h float64) string { + return fmt.Sprintf("%g %g %g %g re f\n", x, y, w, h) +} diff --git a/column.go b/column.go new file mode 100644 index 0000000..6abefb3 --- /dev/null +++ b/column.go @@ -0,0 +1,119 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +// This file puts a page's lines into reading order, which on a two-column paper +// is not the order they appear in from the top of the page down. +// +// The test for two columns is the gutter: a band down the middle of the page +// that lines stop at rather than cross. A one-column page has lines crossing the +// middle constantly; a two-column page has almost none, and the few there are — +// the title, a wide figure, a section heading set across both — are the ones +// that have to stay where they are rather than being sorted into a column. +// +// So the page is cut at every full-width line. Between two of them the lines are +// read left column first and right column second, which is what a reader does. + +// gutterBand is how wide the band down the middle of the page is, as a fraction +// of the page's width. A two-column layout leaves about a twentieth of the page +// empty there; this is narrower, so that a line reaching a little into the +// gutter still counts as staying in its column. +const gutterBand = 0.02 + +// minColumnLines is how many lines each column must have before the page is +// read as two. A page with three lines on the left and three on the right is +// more likely to be a table or a title block than a two-column layout. +const minColumnLines = 5 + +// A side is where a line sits relative to the gutter. +type side int + +const ( + full side = iota + leftSide + rightSide +) + +// order returns the page's lines in reading order, and reports whether the page +// was read as two columns. Each line comes back knowing the margins of the +// column it is in, which is what an indent is measured against. +func order(lines []line, width float64) ([]line, bool) { + sides := make([]side, len(lines)) + mid, band := width/2, gutterBand*width + var nleft, nright, nfull int + for i, l := range lines { + switch { + case l.x0 < mid-band && l.x1 > mid+band: + sides[i], nfull = full, nfull+1 + case l.x1 <= mid+band: + sides[i], nleft = leftSide, nleft+1 + default: + sides[i], nright = rightSide, nright+1 + } + } + if nleft < minColumnLines || nright < minColumnLines || nfull*4 > nleft+nright { + return margins(lines, nil), false + } + var out []line + var outSides []side + var run []int + take := func(i int) { + out = append(out, lines[i]) + outSides = append(outSides, sides[i]) + } + flush := func() { + for _, want := range []side{leftSide, rightSide} { + for _, i := range run { + if sides[i] == want { + take(i) + } + } + } + run = run[:0] + } + for i := range lines { + if sides[i] == full { + flush() + take(i) + continue + } + run = append(run, i) + } + flush() + return margins(out, outSides), true +} + +// margins gives each line the left and right edge of the column it is in, so +// that an indent, a centred line and an equation number can each be recognised +// by where they sit within it. With no sides given every line shares the page's. +func margins(lines []line, sides []side) []line { + var bounds [3][2]float64 + for i := range bounds { + bounds[i] = [2]float64{1e9, -1e9} + } + at := func(i int) side { + if sides == nil { + return full + } + return sides[i] + } + for i, l := range lines { + s := at(i) + bounds[s][0] = min(bounds[s][0], l.x0) + bounds[s][1] = max(bounds[s][1], l.x1) + } + // A line set across the whole page is measured against the page rather + // than against either column, so its bounds take in both. + if sides != nil { + for _, s := range []side{leftSide, rightSide} { + bounds[full][0] = min(bounds[full][0], bounds[s][0]) + bounds[full][1] = max(bounds[full][1], bounds[s][1]) + } + } + for i := range lines { + s := at(i) + lines[i].left, lines[i].right = bounds[s][0], bounds[s][1] + } + return lines +} diff --git a/figure.go b/figure.go new file mode 100644 index 0000000..ffb5897 --- /dev/null +++ b/figure.go @@ -0,0 +1,145 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import ( + "bytes" + "fmt" + "image" + "image/color" + "image/png" + + "github.com/go-pdfkit/extract" + "github.com/go-pdfkit/reader" +) + +// This file turns the pictures a page places back into \includegraphics, and +// writes the pictures out beside the source so that the command has something +// to point at. +// +// A picture that arrives already encoded — a JPEG, a JPEG 2000 — is written +// straight out, because re-encoding it would lose something for nothing and +// graphicx reads both. A picture that arrives as plain samples has to be given +// a container, and PNG is the one the standard library can write and graphicx +// can read. Reading the samples means reading the colour space, and this reads +// the two that account for nearly all of them plus a stencil mask; a picture in +// any other — separation inks, an indexed palette, a Lab space — is not written +// out, and the figure becomes a frame of the right size with a note in it +// rather than a command pointing at a file that is not there. + +// A File is a picture pulled out of the document, to be written beside the +// reconstructed source. +type File struct { + // Name is what the \includegraphics command refers to. + Name string + Data []byte +} + +// minFigure is how large a picture must be drawn, in points, to be worth a +// figure. Below that it is a rule, a bullet or a logo fragment, and a document +// full of \includegraphics of three-point images is worse than one without. +const minFigure = 12 + +// figure writes a picture as a graphic, and the file it needs. +func figure(im extract.Image, n int) (string, *File) { + if im.DrawnWidth < minFigure || im.DrawnHeight < minFigure { + return "", nil + } + box := fmt.Sprintf("[width=%.1fpt,height=%.1fpt]", im.DrawnWidth, im.DrawnHeight) + name, data, ok := picture(im, n) + if !ok { + return fmt.Sprintf(`\framebox[%.1fpt]{\rule{0pt}{%.1fpt}unreadable image}`, + im.DrawnWidth, im.DrawnHeight), nil + } + return `\includegraphics` + box + `{` + name + `}`, &File{Name: name, Data: data} +} + +// picture is the file to write for an image, and false for one this cannot put +// into a container graphicx reads. +func picture(im extract.Image, n int) (string, []byte, bool) { + switch im.Filter { + case "DCTDecode": + return fmt.Sprintf("image%03d.jpg", n), im.Data, true + case "JPXDecode": + return fmt.Sprintf("image%03d.jp2", n), im.Data, true + } + pix, ok := samples(im) + if !ok { + return "", nil, false + } + var buf bytes.Buffer + // png.Encode writes an image this function built itself into a buffer + // that cannot fail, so the only error it could return is one that cannot + // happen; the encoding of a Gray or an NRGBA is always representable. + _ = png.Encode(&buf, pix) + return fmt.Sprintf("image%03d.png", n), buf.Bytes(), true +} + +// samples turns unfiltered image data into a picture, for the colour spaces +// that can be read without a full renderer behind them. +func samples(im extract.Image) (image.Image, bool) { + bits, _ := reader.ToInt(im.Dict.Get("BitsPerComponent")) + mask, _ := reader.ToBool(im.Dict.Get("ImageMask")) + w, h := im.Width, im.Height + if w <= 0 || h <= 0 { + return nil, false + } + if mask { + return stencil(im.Data, w, h) + } + space, _ := im.Dict.Get("ColorSpace").(reader.Name) + switch { + case bits == 8 && (space == "DeviceGray" || space == "CalGray" || space == "G"): + return gray(im.Data, w, h) + case bits == 8 && (space == "DeviceRGB" || space == "CalRGB" || space == "RGB"): + return rgb(im.Data, w, h) + } + return nil, false +} + +// stencil reads a one-bit mask, in which a set bit is a hole and a clear one is +// paint — which is the way round PDF states it when there is no /Decode array. +func stencil(data []byte, w, h int) (image.Image, bool) { + stride := (w + 7) / 8 + if len(data) < stride*h { + return nil, false + } + out := image.NewGray(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + bit := data[y*stride+x/8] >> (7 - uint(x)%8) & 1 + out.SetGray(x, y, color.Gray{Y: bit * 255}) + } + } + return out, true +} + +// gray reads eight-bit greyscale samples. +func gray(data []byte, w, h int) (image.Image, bool) { + if len(data) < w*h { + return nil, false + } + out := image.NewGray(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + copy(out.Pix[y*out.Stride:], data[y*w:(y+1)*w]) + } + return out, true +} + +// rgb reads eight-bit colour samples. +func rgb(data []byte, w, h int) (image.Image, bool) { + if len(data) < w*h*3 { + return nil, false + } + out := image.NewNRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + s := (y*w + x) * 3 + d := out.PixOffset(x, y) + out.Pix[d], out.Pix[d+1], out.Pix[d+2] = data[s], data[s+1], data[s+2] + out.Pix[d+3] = 255 + } + } + return out, true +} diff --git a/figure_test.go b/figure_test.go new file mode 100644 index 0000000..d6b3f8c --- /dev/null +++ b/figure_test.go @@ -0,0 +1,148 @@ +package latex + +import ( + "bytes" + "image/png" + "strings" + "testing" + + "github.com/go-pdfkit/extract" + "github.com/go-pdfkit/reader" +) + +func TestAPictureBecomesAGraphic(t *testing.T) { + im := extract.Image{ + Width: 2, Height: 2, DrawnWidth: 100, DrawnHeight: 50, + Filter: "DCTDecode", Data: []byte("jpeg bytes"), + } + s, f := figure(im, 3) + if !strings.Contains(s, `\includegraphics[width=100.0pt,height=50.0pt]{image003.jpg}`) { + t.Errorf("got %q", s) + } + if f == nil || f.Name != "image003.jpg" || string(f.Data) != "jpeg bytes" { + t.Errorf("got %+v", f) + } + im.Filter = "JPXDecode" + if _, f := figure(im, 0); f == nil || f.Name != "image000.jp2" { + t.Errorf("got %+v", f) + } +} + +func TestASmallPictureIsNotAFigure(t *testing.T) { + // A three-point image is a bullet or a rule, and a document full of + // \includegraphics of them is worse than one without. + if s, f := figure(extract.Image{DrawnWidth: 3, DrawnHeight: 3}, 0); s != "" || f != nil { + t.Errorf("got %q %+v", s, f) + } + if s, _ := figure(extract.Image{DrawnWidth: 100, DrawnHeight: 3}, 0); s != "" { + t.Errorf("got %q", s) + } +} + +func TestPlainSamplesBecomeAPNG(t *testing.T) { + grey := extract.Image{ + Width: 2, Height: 2, DrawnWidth: 40, DrawnHeight: 40, + Data: []byte{0, 64, 128, 255}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceGray")}, + } + s, f := figure(grey, 1) + if f == nil || f.Name != "image001.png" || !strings.Contains(s, "image001.png") { + t.Fatalf("got %q %+v", s, f) + } + pix, err := png.Decode(bytes.NewReader(f.Data)) + if err != nil { + t.Fatal(err) + } + if r, _, _, _ := pix.At(1, 0).RGBA(); r>>8 != 64 { + t.Errorf("the second sample came out %d", r>>8) + } + colour := extract.Image{ + Width: 1, Height: 2, DrawnWidth: 40, DrawnHeight: 40, + Data: []byte{255, 0, 0, 0, 0, 255}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceRGB")}, + } + _, f = figure(colour, 2) + if f == nil { + t.Fatal("no file") + } + pix, _ = png.Decode(bytes.NewReader(f.Data)) + if r, _, b, _ := pix.At(0, 1).RGBA(); r != 0 || b>>8 != 255 { + t.Errorf("the second sample came out %v", pix.At(0, 1)) + } +} + +func TestAStencilMaskBecomesAPNG(t *testing.T) { + mask := extract.Image{ + Width: 8, Height: 1, DrawnWidth: 40, DrawnHeight: 40, + Data: []byte{0b10000000}, + Dict: reader.Dict{"ImageMask": reader.Bool(true)}, + } + _, f := figure(mask, 0) + if f == nil { + t.Fatal("no file") + } + pix, err := png.Decode(bytes.NewReader(f.Data)) + if err != nil { + t.Fatal(err) + } + if r, _, _, _ := pix.At(0, 0).RGBA(); r>>8 != 255 { + t.Errorf("the set bit came out %d", r>>8) + } + if r, _, _, _ := pix.At(1, 0).RGBA(); r>>8 != 0 { + t.Errorf("the clear bit came out %d", r>>8) + } +} + +func TestAPictureThatCannotBeWrittenBecomesAFrame(t *testing.T) { + for _, im := range []extract.Image{ + // A colour space this does not read. + {Width: 1, Height: 1, DrawnWidth: 40, DrawnHeight: 40, Data: []byte{0}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceCMYK")}}, + // Four bits a sample. + {Width: 1, Height: 1, DrawnWidth: 40, DrawnHeight: 40, Data: []byte{0}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(4), "ColorSpace": reader.Name("DeviceGray")}}, + // A picture with no samples in it at all. + {Width: 4, Height: 4, DrawnWidth: 40, DrawnHeight: 40, Data: nil, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceGray")}}, + {Width: 4, Height: 4, DrawnWidth: 40, DrawnHeight: 40, Data: nil, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceRGB")}}, + {Width: 4, Height: 4, DrawnWidth: 40, DrawnHeight: 40, Data: nil, + Dict: reader.Dict{"ImageMask": reader.Bool(true)}}, + // A picture with no size. + {Width: 0, Height: 0, DrawnWidth: 40, DrawnHeight: 40, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8)}}, + } { + s, f := figure(im, 0) + if f != nil || !strings.Contains(s, `\framebox`) { + t.Errorf("%v gave %q %+v", im.Dict, s, f) + } + } +} + +func TestAPictureTooLargeToEncode(t *testing.T) { + // png.Encode refuses a picture whose bounds it cannot write. + if _, _, ok := picture(extract.Image{Width: 1, Height: 1, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceGray")}, + Data: []byte{0}}, 0); !ok { + t.Error("a one-pixel greyscale picture could not be written") + } + if _, ok := samples(extract.Image{Width: 1, Height: 1, Data: []byte{0}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("CalGray")}}); !ok { + t.Error("CalGray was not read") + } + if _, ok := samples(extract.Image{Width: 1, Height: 1, Data: []byte{0, 0, 0}, + Dict: reader.Dict{"BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("CalRGB")}}); !ok { + t.Error("CalRGB was not read") + } +} + +func TestWritingThePicturesOut(t *testing.T) { + dir := t.TempDir() + doc := &Document{Files: []File{{Name: "a.png", Data: []byte("x")}}} + if err := doc.WriteFiles(dir); err != nil { + t.Fatal(err) + } + if err := doc.WriteFiles(dir + "/nowhere"); err == nil { + t.Error("writing into a directory that does not exist succeeded") + } +} diff --git a/font.go b/font.go new file mode 100644 index 0000000..1bea838 --- /dev/null +++ b/font.go @@ -0,0 +1,130 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "strings" + +// This file works out what a font was, from the only thing the page reliably +// says about it: its name. A PDF made by TeX does not record that a word was +// bold, or that a letter was a mathematical variable. It records that the word +// was drawn in CMBX10 and the letter in CMMI10 — and those names, which TeX has +// used unchanged since 1980, say exactly that. Every later family kept the +// convention or spelled it out in words (LMRoman10-Bold, NimbusRomNo9L-Medi), +// so the same reading works for them too. +// +// Nothing here looks inside the font program. A font's own idea of its weight +// is often wrong, and a subsetted face carries no useful flags at all; the name +// the document gives it is what the author's \textbf actually turned into. + +// A shape is what a font says about the text drawn in it. +type shape struct { + bold bool + italic bool + mono bool + sans bool + smallCaps bool + // math is the part a math font plays, and mathNone for a text font. A + // text font still appears inside mathematics — digits and parentheses + // come from the roman face — which is why this cannot be the only test + // for whether something is an equation. + math mathRole +} + +// A mathRole is the part a math font plays in an equation. +type mathRole int + +const ( + // mathNone is a font that is not one of the math families. + mathNone mathRole = iota + // mathLetter is the math italic family (CMMI): variables, and the + // lowercase Greek letters, which TeX treats as variables too. + mathLetter + // mathSymbol is the symbol family (CMSY, and the AMS extensions): + // relations, operators, arrows, and the calligraphic capitals. + mathSymbol + // mathExt is the extension family (CMEX): the big operators, and the + // delimiters grown to fit what they enclose. + mathExt +) + +// isMath reports whether the font is one of the math families. +func (s shape) isMath() bool { return s.math != mathNone } + +// textual reports whether two pieces of text should be wrapped in the same +// markup. Only the text properties count: two math fonts differ in role all the +// time inside one equation and that is not a change of markup. +func (s shape) textual() shape { + s.math = mathNone + return s +} + +// classify reads a font's name. What comes in is the /BaseFont of the font +// dictionary, which for an embedded subset carries a six-letter tag and a plus +// sign in front (FKPJSN+CMSY10) and for a Type0 font a suffix naming the +// encoding (LMRoman10-Bold-Identity-H). Both are cut away first. +func classify(base string) shape { + n := strings.ToUpper(baseName(base)) + var s shape + switch { + case mathFamily(n, "ITAL", "LETTER", "MI"): + s.math = mathLetter + case mathFamily(n, "SYM", "SY"): + s.math = mathSymbol + case mathFamily(n, "EXT", "EX"): + s.math = mathExt + case hasAny(n, "CMMI", "CMMIB", "MATHITALIC", "MTMI", "RMMI", "TXMI", "PXMI", "EURM", "EUMI"): + s.math = mathLetter + case hasAny(n, "CMSY", "CMBSY", "MATHSYMBOL", "MSAM", "MSBM", "MTSY", "RMSY", "TXSY", "PXSY", + "EUFM", "EUSM", "RSFS", "WASY", "BBOLD", "DSROM", "MNSYMBOL"): + s.math = mathSymbol + case hasAny(n, "CMEX", "MATHEXTENSION", "MTEX", "RMEX", "TXEX", "PXEX", "ESINT", "STMARY", "EUEX"): + s.math = mathExt + } + if s.isMath() { + // A math font is drawn slanted or upright as its family decides; + // saying it is italic would put \emph round every variable. + return s + } + s.mono = hasAny(n, "CMTT", "CMITT", "CMSLTT", "MONO", "TYPEWRITER", "COURIER", "NIMBUSMON") + s.sans = !s.mono && hasAny(n, "CMSS", "SANS", "HELVETIC", "ARIAL", "NIMBUSSAN") + s.smallCaps = hasAny(n, "CMCSC", "CAPS", "SMALLCAPS", "-SC", "SC10") + s.bold = hasAny(n, "CMBX", "CMB10", "CMBXTI", "CMBXSL", "CMSSBX", "CMFIB", "BOLD", "-MEDI", "SEMIBOLD", "BLACK", "HEAVY") + s.italic = hasAny(n, "CMTI", "CMSL", "CMITT", "CMSLTT", "CMBXTI", "CMBXSL", "CMFI", "ITALIC", "OBLIQUE", "SLANT") + return s +} + +// baseName strips a subset tag and a Type0 encoding suffix from a /BaseFont. +func baseName(base string) string { + if i := strings.IndexByte(base, '+'); i >= 0 && i == 6 { + base = base[i+1:] + } + for _, suffix := range []string{"-Identity-H", "-Identity-V", "-UniGB-UCS2-H", "-UniJIS-UCS2-H"} { + base = strings.TrimSuffix(base, suffix) + } + return base +} + +// hasAny reports whether the name contains any of the marks. +func hasAny(name string, marks ...string) bool { + for _, m := range marks { + if strings.Contains(name, m) { + return true + } + } + return false +} + +// mathFamily reports whether a font's name says it is a math font of a +// particular kind: the word MATH together with a word naming the part it plays. +// +// This is what catches the families that are not descended from Computer +// Modern's naming. Fourier calls its three fonts Fourier-Math-Letters-Italic, +// Fourier-Math-Symbols and Fourier-Math-Extension, and a reader that knows only +// CMMI and CMSY sees a paper set in it as prose with a great many one-letter +// italic words in it. The word MATH on its own is not enough: a font called +// STIXTwoMath is a general-purpose face that one producer in this corpus sets +// entire documents in, running text included. +func mathFamily(name string, roles ...string) bool { + return strings.Contains(name, "MATH") && hasAny(name, roles...) +} diff --git a/font_test.go b/font_test.go new file mode 100644 index 0000000..978b0d0 --- /dev/null +++ b/font_test.go @@ -0,0 +1,74 @@ +package latex + +import "testing" + +func TestReadingWhatAFontSays(t *testing.T) { + cases := []struct { + base string + want shape + }{ + {"FKPJSN+CMR10", shape{}}, + {"CMBX12", shape{bold: true}}, + {"CMTI10", shape{italic: true}}, + {"CMBXTI10", shape{bold: true, italic: true}}, + {"CMTT10", shape{mono: true}}, + {"CMSS10", shape{sans: true}}, + {"CMCSC10", shape{smallCaps: true}}, + {"CMSL10", shape{italic: true}}, + {"LMRoman10-Bold-Identity-H", shape{bold: true}}, + {"LMRoman10-Italic-Identity-H", shape{italic: true}}, + {"LMMono10-Regular", shape{mono: true}}, + {"NimbusRomNo9L-Medi", shape{bold: true}}, + {"Helvetica-Oblique", shape{sans: true, italic: true}}, + {"Courier-Bold", shape{mono: true, bold: true}}, + {"CMMI10", shape{math: mathLetter}}, + {"CMSY7", shape{math: mathSymbol}}, + {"CMEX10", shape{math: mathExt}}, + {"MSBM10", shape{math: mathSymbol}}, + {"Fourier-Math-Letters-Italic", shape{math: mathLetter}}, + {"Fourier-Math-Symbols", shape{math: mathSymbol}}, + {"Fourier-Math-Extension", shape{math: mathExt}}, + {"LMMathItalic10-Regular", shape{math: mathLetter}}, + {"EUEX10", shape{math: mathExt}}, + {"EURM10", shape{math: mathLetter}}, + {"RSFS10", shape{math: mathSymbol}}, + // A general-purpose face with Math in its name is not a math family: + // one producer sets whole documents in it, running text included. + {"STIXTwoMath-Regular", shape{}}, + {"", shape{}}, + // A subset tag is only a tag when it is six letters and a plus. + {"AB+CMBX10", shape{bold: true}}, + } + for _, c := range cases { + if got := classify(c.base); got != c.want { + t.Errorf("%s reads as %+v, want %+v", c.base, got, c.want) + } + } +} + +func TestAMathFontIsNeitherBoldNorItalic(t *testing.T) { + // CMMI is a slanted face, but a variable is not emphasis; saying so would + // put \emph round every letter of every equation. + s := classify("CMMI10") + if s.italic || s.bold || !s.isMath() { + t.Errorf("CMMI10 reads as %+v", s) + } + if s.textual().math != mathNone { + t.Error("textual() kept the math role") + } +} + +func TestStrippingAFontsDecoration(t *testing.T) { + for _, c := range [][2]string{ + {"ABCDEF+CMR10", "CMR10"}, + {"CMR10", "CMR10"}, + {"ABC+CMR10", "ABC+CMR10"}, + {"X-Identity-V", "X"}, + {"Y-UniGB-UCS2-H", "Y"}, + {"Z-UniJIS-UCS2-H", "Z"}, + } { + if got := baseName(c[0]); got != c[1] { + t.Errorf("baseName(%q) = %q, want %q", c[0], got, c[1]) + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b64958d --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/go-pdfkit/latex + +go 1.26.4 + +require ( + github.com/go-pdfkit/extract v0.1.0 + github.com/go-pdfkit/reader v0.4.1 +) + +require ( + github.com/go-opentype/opentype v0.9.0 // indirect + github.com/go-pdfkit/pdffont v0.2.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e33f2d7 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/go-opentype/fonts v0.8.0 h1:77i3VPIH90GbstzNb21mk+an4WvEOe2idC6W+J0n0fw= +github.com/go-opentype/fonts v0.8.0/go.mod h1:C6yQL2apHItfEZ5hztpsHF0S5mlX/hklLlq/Z5fRG/g= +github.com/go-opentype/opentype v0.9.0 h1:GFgcJ3nwTDp4NJr5O+Paw7lhZx5Jv/R+noZwvhYDlkM= +github.com/go-opentype/opentype v0.9.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= +github.com/go-pdfkit/extract v0.1.0 h1:7IJDJMH43l2wJZD+DkW5StmeAS1NDt2FnT2LBceXTog= +github.com/go-pdfkit/extract v0.1.0/go.mod h1:3ejdi87IrdR60wmjzLswdqzBzHCgFlrLtxkMxRN1KSU= +github.com/go-pdfkit/pdffont v0.2.0 h1:yAp/oR5Z2kkqs4r0GWMalZMC7rc7XSCZXgwIypbpMWM= +github.com/go-pdfkit/pdffont v0.2.0/go.mod h1:y4vo5DgT95e57C3XxIWfA/xss+x6RwZwyj6KWdJc86s= +github.com/go-pdfkit/reader v0.4.1 h1:pRxFqRjsn7H/VsGfWb9nYWyFuDgTU2Pjmoq/f5mgVq4= +github.com/go-pdfkit/reader v0.4.1/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8= diff --git a/latex.go b/latex.go new file mode 100644 index 0000000..8995f43 --- /dev/null +++ b/latex.go @@ -0,0 +1,154 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +// Package latex reconstructs LaTeX source from a PDF. +// +// A PDF that came from TeX does not contain a document. It contains the marks +// TeX made putting one on paper: glyphs at absolute positions, from fonts whose +// names are the only surviving record of what the author asked for, and a few +// thin rectangles. Everything else — that this was a paragraph, that this word +// was \emph, that these eleven glyphs and one rule were \frac{a+b}{c} — was +// consumed by the typesetting and has to be worked out again from the geometry. +// +// This package does that work and writes out source that, put back through a +// TeX engine, sets something close to the page it was read from. It is a +// reconstruction and not a recovery: two different sources typeset to the same +// page, and where the geometry cannot tell them apart this makes a choice and +// says in its own documentation which one. +// +// # What it reconstructs +// +// - Paragraphs and lines, including two-column layouts, indentation, centred +// lines, and the hyphens TeX put in that the author did not. +// - Bold, italic, typewriter, sans and small-capital text, from the names of +// the fonts it was set in. +// - Section headings and their levels, from the numbers the author gave them +// where there are numbers and from the ranking of their sizes where there +// are not. +// - Inline and displayed mathematics: superscripts and subscripts, fractions, +// radicals, big operators with their limits, grown delimiters as \left and +// \right, the Greek alphabet, and about two hundred and fifty symbols. +// Numbered displays come back as an equation environment. +// - Pictures, as \includegraphics with the file written out beside the source. +// +// # What it does not +// +// Bibliographies, citations, cross-references, labels, footnotes, tables, +// colour, and whitespace fidelity. A \cite that typeset to "[14]" comes back as +// the characters [14]; there is nothing on the page that says otherwise. The +// preamble is reconstructed from what the body needs rather than from what the +// author wrote, which no PDF records. +package latex + +import ( + "os" + "path/filepath" + "strings" + + "github.com/go-pdfkit/reader" +) + +// Options say how a document is to be reconstructed. The zero value asks for +// every page of an article. +type Options struct { + // First and Last are the range of pages to read, counting from one. Zero + // means the first page and the last respectively. A range that names a + // page the document has not is an error rather than an empty document: + // a caller asking for page forty of a three-page paper has made a + // mistake and is better told about it. + First, Last int + // Class is the document class to write. Empty means article. + Class string +} + +// A Document is the reconstructed source, and the files it refers to. +type Document struct { + // Preamble is everything before \begin{document}. + Preamble string + // Body is everything between \begin{document} and \end{document}. + Body string + // Files are the pictures the body's \includegraphics commands name, to + // be written in the same directory as the source. + Files []File + // Pages is how many pages were read. + Pages int +} + +// String is the whole source, ready to be typeset. +func (d *Document) String() string { + return d.Preamble + "\n\\begin{document}\n\n" + d.Body + "\n\\end{document}\n" +} + +// WriteFiles writes the pictures the body refers to into a directory, which +// must already exist. +func (d *Document) WriteFiles(dir string) error { + for _, f := range d.Files { + if err := os.WriteFile(filepath.Join(dir, f.Name), f.Data, 0o644); err != nil { + return err + } + } + return nil +} + +// Reconstruct reads a document and writes LaTeX source for it. +func Reconstruct(d *reader.Document, opt Options) (*Document, error) { + first, last := pageRange(opt, d.PageCount()) + var frames []frame + width, height := 612.0, 792.0 + for i := first; i <= last; i++ { + p, err := readPage(d, i) + if err != nil { + return nil, err + } + lines, two := order(groupLines(p.atoms, p.rules, p.width), p.width) + frames = append(frames, frame{ + lines: lines, rules: p.rules, images: p.images, twoColumn: two, + }) + width, height = p.width, p.height + } + e := &emitter{opt: opt} + e.measure(frames) + for i := range frames { + frames[i].lines = stripRunningHeads(frames[i].lines, e.body) + } + for i, f := range frames { + e.opening = i == 0 + e.write(f) + } + e.flush() + out := &Document{ + Body: strings.TrimRight(e.out.String(), "\n") + "\n", + Files: e.files, + Pages: len(frames), + } + if len(frames) > 0 { + out.Preamble = preamble(opt, frames[0], width, height, e.body, e.leading, e.title) + } + if e.title != "" { + out.Body = "\\maketitle\n\n" + out.Body + } + return out, nil +} + +// Source is the reconstructed source of a whole document, as one string. +func Source(d *reader.Document) (string, error) { + doc, err := Reconstruct(d, Options{}) + if err != nil { + return "", err + } + return doc.String(), nil +} + +// pageRange puts the requested range inside the document. +func pageRange(opt Options, count int) (int, int) { + first, last := opt.First, opt.Last + if first < 1 { + first = 1 + } + // Only an unset Last means "to the end". A caller that names a page the + // document has not is told so rather than handed an empty document. + if last < 1 { + last = count + } + return first, last +} diff --git a/latex_test.go b/latex_test.go new file mode 100644 index 0000000..30d690d --- /dev/null +++ b/latex_test.go @@ -0,0 +1,342 @@ +package latex + +import ( + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// The fonts a test page draws in. The names are the ones TeX has used since +// 1980, which is what the classifier reads; the maps are what a real math font +// carries so that a reader can tell code 11 from alpha. +var ( + roman = face{base: "ABCDEF+CMR10"} + bold = face{base: "ABCDEF+CMBX10"} + italic = face{base: "ABCDEF+CMTI10"} + mono = face{base: "ABCDEF+CMTT10"} + sansf = face{base: "ABCDEF+CMSS10"} + caps = face{base: "ABCDEF+CMCSC10"} + mathit = face{base: "ABCDEF+CMMI10", uni: map[byte]rune{'a': 'α', 'b': 'β'}} + symbol = face{base: "ABCDEF+CMSY10", uni: map[byte]rune{'r': '√', 'i': '∈', 'm': '−'}} + extend = face{base: "ABCDEF+CMEX10", uni: map[byte]rune{'s': '∑', '(': '(', ')': ')'}} +) + +func allFaces() map[string]face { + return map[string]face{ + "R": roman, "B": bold, "I": italic, "T": mono, "S": sansf, "C": caps, + "M": mathit, "Y": symbol, "X": extend, + } +} + +// body is the reconstructed body of a one-page document. +func body(t *testing.T, content string) string { + t.Helper() + d := pageWith(t, content, allFaces()) + doc, err := Reconstruct(d, Options{}) + if err != nil { + t.Fatal(err) + } + return strings.TrimSpace(doc.Body) +} + +func TestProseAndItsMarkup(t *testing.T) { + got := body(t, show("R", 10, 20, 150, "This is")+ + show("B", 10, 60, 150, "bold")+ + show("R", 10, 85, 150, "and")+ + show("I", 10, 105, 150, "slanted")+ + show("R", 10, 145, 150, "and")+ + show("T", 10, 165, 150, "fixed")) + want := `This is \textbf{bold} and \emph{slanted} and \texttt{fixed}` + if got != want { + t.Errorf("got %s\nwant %s", got, want) + } +} + +func TestSansAndSmallCaps(t *testing.T) { + got := body(t, show("R", 10, 20, 150, "a")+ + show("S", 10, 30, 150, "bb")+ + show("R", 10, 45, 150, "c")+ + show("C", 10, 55, 150, "dd")) + if !strings.Contains(got, `\textsf{bb}`) || !strings.Contains(got, `\textsc{dd}`) { + t.Errorf("got %s", got) + } +} + +func TestWordsAndKerning(t *testing.T) { + // Two pieces a space apart are two words; two pieces touching are one. + got := body(t, show("R", 10, 20, 150, "on")+show("R", 10, 30, 150, "e")+ + show("R", 10, 45, 150, "two")) + if got != "one two" { + t.Errorf("got %q", got) + } +} + +func TestSuperscriptsAndSubscripts(t *testing.T) { + got := body(t, show("M", 10, 20, 100, "x")+ + show("R", 7, 25, 104, "2")+ + show("R", 10, 30, 100, "+")+ + show("M", 10, 37, 100, "y")+ + show("R", 7, 42, 97, "1")) + if got != `$x^{2}+y_{1}$` { + t.Errorf("got %q", got) + } +} + +func TestGreekAndSymbols(t *testing.T) { + got := body(t, show("M", 10, 20, 100, "a")+ + show("Y", 10, 26, 100, "i")+ + show("M", 10, 34, 100, "b")) + if got != `$\alpha\in\beta$` { + t.Errorf("got %q", got) + } +} + +func TestAFractionIsARuleWithMaterialEitherSide(t *testing.T) { + got := body(t, show("M", 7, 20, 104, "a")+ + show("M", 7, 20, 96, "b")+ + bar(19.5, 100, 4.5, 0.4)) + if got != `$\frac{\alpha}{\beta}$` { + t.Errorf("got %q", got) + } +} + +func TestARadicalReachesAsFarAsItsBar(t *testing.T) { + got := body(t, show("M", 10, 28, 100, "ab")+ + show("Y", 10, 20, 104, "r")+ + bar(25, 107, 14, 0.4)) + if got != `$\sqrt{\alpha\beta}$` { + t.Errorf("got %q", got) + } +} + +func TestABigOperatorTakesLimits(t *testing.T) { + got := body(t, show("X", 10, 20, 105, "s")+ + show("M", 7, 21, 111, "b")+ + show("M", 7, 21, 99, "a")+ + show("M", 10, 34, 100, "ab")) + if got != `$\sum_{\alpha}^{\beta}\alpha\beta$` { + t.Errorf("got %q", got) + } +} + +func TestGrownDelimitersBecomeLeftAndRight(t *testing.T) { + got := body(t, show("X", 10, 20, 102, "(")+ + show("M", 10, 26, 100, "ab")+ + show("X", 10, 37, 102, ")")) + if got != `$\left(\alpha\beta\right)$` { + t.Errorf("got %q", got) + } + // An opening delimiter with no closing one is written plainly: \left + // with no \right does not compile. + got = body(t, show("X", 10, 20, 102, "(")+show("M", 10, 26, 100, "ab")) + if strings.Contains(got, `\left`) { + t.Errorf("an unbalanced delimiter was written as \\left: %q", got) + } +} + +func TestAnEquationStopsAtTheProseAroundIt(t *testing.T) { + // "and" is a word, not a variable, so the equation does not reach it; + // the "+" is arithmetic set in the roman font, so it does. + got := body(t, show("R", 10, 20, 150, "and")+ + show("M", 10, 40, 150, "x")+ + show("R", 10, 46, 150, "+")+ + show("M", 10, 53, 150, "a")+ + show("R", 10, 59, 150, ".")+ + show("R", 10, 66, 150, "text")) + if !strings.Contains(got, `and $x+\alpha$. text`) { + t.Errorf("got %q", got) + } +} + +func TestAnOperatorNameIsSetUpright(t *testing.T) { + got := body(t, show("M", 10, 20, 100, "a")+ + show("R", 10, 27, 100, "sin")+ + show("M", 10, 43, 100, "b")) + if !strings.Contains(got, `\sin`) { + t.Errorf("got %q", got) + } + got = body(t, show("M", 10, 20, 100, "a")+ + show("R", 10, 27, 100, "det")+ + show("M", 10, 43, 100, "b")) + if !strings.Contains(got, `\det`) { + t.Errorf("got %q", got) + } +} + +func TestSectionsFromTheirNumbers(t *testing.T) { + // Enough body text for the body size to be the body's, then headings. + content := show("B", 14, 20, 180, "1 One") + + show("R", 10, 20, 160, "Some running text on a line of its own here.") + + show("B", 12, 20, 140, "1.1 Two") + + show("R", 10, 20, 120, "More running text on a second line of prose.") + + show("B", 11, 20, 100, "1.1.1 Three") + + show("R", 10, 20, 80, "Yet more running text to weigh the body size.") + got := body(t, content) + for _, want := range []string{`\section{One}`, `\subsection{Two}`, `\subsubsection{Three}`} { + if !strings.Contains(got, want) { + t.Errorf("%s missing from\n%s", want, got) + } + } +} + +func TestAnUnnumberedHeadingIsStarred(t *testing.T) { + content := show("B", 14, 20, 180, "Introduction") + + show("R", 10, 20, 160, "Some running text on a line of its own here.") + + show("R", 10, 20, 145, "More running text to make the body size clear.") + if got := body(t, content); !strings.Contains(got, `\section*{Introduction}`) { + t.Errorf("got %s", got) + } +} + +func TestADisplayedEquationAndItsNumber(t *testing.T) { + content := show("R", 10, 10, 180, "Body text that sets the margins of the column.") + + show("M", 10, 80, 150, "ab") + + show("R", 10, 160, 150, "(1)") + + show("R", 10, 10, 120, "More body text that sets the margins here too.") + got := body(t, content) + if !strings.Contains(got, "\\begin{equation}\n\\alpha\\beta\n\\end{equation}") { + t.Errorf("got %s", got) + } + // Without a number it is an unnumbered display. + content = show("R", 10, 10, 180, "Body text that sets the margins of the column.") + + show("M", 10, 80, 150, "ab") + + show("R", 10, 10, 120, "More body text that sets the margins here too.") + if got := body(t, content); !strings.Contains(got, "\\[\n\\alpha\\beta\n\\]") { + t.Errorf("got %s", got) + } +} + +func TestParagraphsFromIndentation(t *testing.T) { + content := show("R", 10, 20, 180, "The first paragraph starts here and runs on") + + show("R", 10, 10, 168, "to a second line that reaches the margin too.") + + show("R", 10, 20, 156, "The second paragraph is indented like this one") + + show("R", 10, 10, 144, "and also runs to a second line at the margin.") + got := body(t, content) + if n := strings.Count(got, "\n\n"); n != 1 { + t.Errorf("%d paragraph breaks in\n%s", n, got) + } +} + +func TestAHyphenAtTheEndOfALineIsPutBack(t *testing.T) { + content := show("R", 10, 10, 180, "a word that has been hyphen-") + + show("R", 10, 10, 168, "ated across two lines of text here") + if got := body(t, content); !strings.Contains(got, "hyphenated") { + t.Errorf("got %s", got) + } + // A hyphen followed by a capital was written by the author. + content = show("R", 10, 10, 180, "the well-known Levi-") + + show("R", 10, 10, 168, "Civita symbol appears in this line of text") + if got := body(t, content); !strings.Contains(got, "Levi-Civita") { + t.Errorf("got %s", got) + } +} + +func TestACentredBlockComesBackAsOne(t *testing.T) { + content := show("R", 10, 20, 300, "Body text that reaches right across the column here.") + + show("R", 10, 135, 280, "middle") + + show("R", 10, 137, 268, "again") + + show("R", 10, 20, 250, "Body text that reaches right across again here too.") + got := strings.TrimSpace(wide(t, content).Body) + if strings.Count(got, `\begin{center}`) != 1 { + t.Errorf("got %s", got) + } +} + +func TestReadingTwoColumns(t *testing.T) { + var b strings.Builder + for i := 0; i < 6; i++ { + y := float64(300 - i*12) + b.WriteString(show("R", 10, 20, y, "left column line here")) + b.WriteString(show("R", 10, 320, y, "right column line here")) + } + doc := wide(t, b.String()) + if !strings.Contains(doc.Preamble, "twocolumn") { + t.Errorf("the page was not read as two columns:\n%s", doc.Preamble) + } + first := strings.Index(doc.Body, "right column line here") + if strings.LastIndex(doc.Body, "left column line here") > first { + t.Errorf("the columns were interleaved:\n%s", doc.Body) + } +} +func TestAPageNumberIsNotText(t *testing.T) { + content := show("R", 10, 10, 180, "Body text on the first line of the page here.") + + show("R", 10, 10, 168, "Body text on the second line of the page too.") + + show("R", 10, 95, 20, "7") + if got := body(t, content); strings.Contains(got, "7") { + t.Errorf("the page number survived: %s", got) + } +} + +func TestTheWholeSource(t *testing.T) { + d := pageWith(t, show("R", 10, 20, 150, "hello"), allFaces()) + s, err := Source(d) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`\documentclass{article}`, `\begin{document}`, "hello", `\end{document}`} { + if !strings.Contains(s, want) { + t.Errorf("%s missing from\n%s", want, s) + } + } +} + +func TestChoosingPagesAndClass(t *testing.T) { + d := pagesWith(t, []string{ + show("R", 10, 20, 150, "first"), + show("R", 10, 20, 150, "second"), + show("R", 10, 20, 150, "third"), + }, allFaces(), [4]int{0, 0, 200, 200}) + doc, err := Reconstruct(d, Options{First: 2, Last: 2, Class: "report"}) + if err != nil { + t.Fatal(err) + } + if doc.Pages != 1 || !strings.Contains(doc.Body, "second") || strings.Contains(doc.Body, "first") { + t.Errorf("got %d pages:\n%s", doc.Pages, doc.Body) + } + if !strings.Contains(doc.Preamble, `\documentclass{report}`) { + t.Errorf("got %s", doc.Preamble) + } + // An unset range is the whole document. + doc, err = Reconstruct(d, Options{}) + if err != nil || doc.Pages != 3 { + t.Errorf("got %d pages, %v", doc.Pages, err) + } +} + +func TestADocumentThatCannotBeRead(t *testing.T) { + // A page whose contents are missing still opens, and reads as nothing. + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + page := w.Add(reader.Dict{"Type": reader.Name("Page"), "Parent": pagesRef}) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + doc, err := Reconstruct(d, Options{}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(doc.Body) != "" { + t.Errorf("an empty page says %q", doc.Body) + } +} + +// wide is the reconstructed body of a one-page document on a page big enough +// for a layout to be laid out on. +func wide(t *testing.T, content string) *Document { + t.Helper() + d := pagesWith(t, []string{content}, allFaces(), [4]int{0, 0, 600, 400}) + doc, err := Reconstruct(d, Options{}) + if err != nil { + t.Fatal(err) + } + return doc +} diff --git a/line.go b/line.go new file mode 100644 index 0000000..c46d487 --- /dev/null +++ b/line.go @@ -0,0 +1,391 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "sort" + +// This file puts the page's text back into lines, which is harder than it +// sounds for exactly one reason: mathematics does not sit on the baseline. +// +// A page of prose is easy. TeX gives every piece of a line the same baseline, so +// grouping by that one number is enough. But the moment an equation appears, the +// same line carries a superscript four points up, a subscript three points down, +// the two halves of a fraction on either side of a rule, the limits of a big +// operator further out still, a radical sign raised so its bar clears the +// radicand, and a \left( grown from the extension family and lifted so that its +// middle lands on the maths axis. Every one of those has to end up on the line +// it belongs to, and none of them may drag in the line above. +// +// The obvious way — walk down the page attaching each baseline to the one before +// — does not work, and fails in a way that is worth writing down because the +// output looks plausible. A superscript sits ABOVE the line it belongs to, so +// walking downwards meets it first and has to decide what it is before the line +// exists; a radical sign raised seven points sits nearer the line above it than +// the one it is part of. The \sqrt{x} on one line ends up in the sentence on the +// line before, and the page still reads as prose. +// +// So this goes the other way round. The biggest pieces are placed first, because +// a page's structure is carried by its body text and its headings, and they are +// never displaced. Everything smaller is then offered to the lines that already +// exist and goes to the nearest one that will have it — which for a superscript +// caught between two lines is the one four points away rather than the one eight +// points away, whichever came first down the page. + +// bandTolerance is how far apart two baselines may be and still be the same +// one. TeX writes a baseline as an exact number, but a rounded one, and two +// pieces set by different routes can differ in the last place. +const bandTolerance = 0.35 + +// The reaches a displaced piece is allowed, each a fraction of the size of the +// line it is offered to. +const ( + // sameBaseline is the slack for two pieces meant to be level. + sameBaseline = 0.25 + // scriptSpan is how far a smaller piece may sit from the line it hangs + // from. A superscript rises about four tenths of the size and the upper + // limit of a big operator three quarters of it. + scriptSpan = 0.85 + // mathReach is how far a piece drawn in a symbol or extension font may be + // lifted. A radical sign and a grown delimiter are raised by design, at + // the same nominal size as their surroundings, so the size test cannot + // catch them — but they are only ever raised, never dropped, and that is + // what keeps this from reaching down into the next line. + mathReach = 0.85 + // sideReach is how far to either side of a line a piece may sit and + // still belong to it, which stops a script in one column being claimed + // by a line in the other. + sideReach = 2.0 + // scriptSize is how much smaller a piece must be than the line it hangs + // from. TeX sets a first-level script at seven tenths of the size. + scriptSize = 0.92 +) + +// A line is everything on one baseline, plus the parts of it that were shifted +// off that baseline. +type line struct { + atoms []atom + // y is the baseline the line's main material sits on and size how tall + // that material is. + y, size float64 + // x0 and x1 are the line's left and right ends. + x0, x1 float64 + // left and right are the margins of the column the line is in, which is + // what an indent and a centred line are measured against. + left, right float64 +} + +// groupLines puts the page's atoms into lines. The atoms must already be sorted +// from the top of the page down, which readPage does. +func groupLines(atoms []atom, rules []rule, width float64) []line { + bs := bands(atoms) + if x, ok := gutter(atoms, width); ok { + bs = split(bs, x) + } + order := make([]int, len(bs)) + for i := range order { + order[i] = i + } + sort.SliceStable(order, func(a, b int) bool { + x, y := bs[order[a]], bs[order[b]] + if x.size != y.size { + return x.size > y.size + } + return x.width > y.width + }) + var lines []line + for _, i := range order { + if k := nearest(lines, bs[i], rules); k >= 0 { + lines[k] = attach(lines[k], bs[i]) + continue + } + lines = append(lines, seed(bs[i])) + } + sort.SliceStable(lines, func(a, b int) bool { return lines[a].y > lines[b].y }) + for i := range lines { + sort.SliceStable(lines[i].atoms, func(a, b int) bool { + return lines[i].atoms[a].x < lines[i].atoms[b].x + }) + } + return lines +} + +// A band is the atoms that share one baseline. +type band struct { + atoms []atom + y float64 + size float64 + width float64 + x0, x1 float64 + allMath bool +} + +// bands groups atoms that share a baseline. +func bands(atoms []atom) []band { + var out []band + for _, a := range atoms { + if n := len(out); n > 0 && abs(out[n-1].y-a.y) <= bandTolerance { + out[n-1].atoms = append(out[n-1].atoms, a) + continue + } + out = append(out, band{atoms: []atom{a}, y: a.y, x0: a.x, x1: a.right()}) + } + for i := range out { + out[i] = measured(out[i].atoms) + } + return out +} + +// nearest is the line a band belongs to, or -1 when it starts one of its own. +// Where more than one line would take it, the closest wins: a superscript +// caught between two lines belongs to the one it is nearer. +func nearest(lines []line, b band, rules []rule) int { + best, found := -1, 0.0 + for i, l := range lines { + if !accepts(l, b, rules) { + continue + } + if d := abs(b.y - l.y); best < 0 || d < found { + best, found = i, d + } + } + return best +} + +// accepts reports whether a line will take a band. Bands are offered largest +// first, so a band is never larger than the line it is offered to by more than +// rounding. +func accepts(l line, b band, rules []rule) bool { + if b.size > l.size*1.02 { + return false + } + if b.x0 > l.x1+sideReach*l.size || b.x1 < l.x0-sideReach*l.size { + return false + } + dy := b.y - l.y + switch { + case abs(dy) < sameBaseline*l.size: + return true + case b.size < scriptSize*l.size && abs(dy) < scriptSpan*l.size: + return true + case b.size < scriptSize*l.size && abs(dy) < limitSpan*l.size && overOperator(l, b): + return true + case b.allMath && dy > 0 && dy < mathReach*l.size: + return true + case sharesBar(l, b, rules): + return true + } + return false +} + +// seed starts a line from a band. +func seed(b band) line { + return line{atoms: b.atoms, y: b.y, size: b.size, x0: b.x0, x1: b.x1} +} + +// attach adds a band to a line, which keeps the baseline and size it was +// seeded with: those belong to the line's own material, not to what hangs off +// it. +func attach(l line, b band) line { + l.atoms = append(l.atoms, b.atoms...) + l.x0, l.x1 = min(l.x0, b.x0), max(l.x1, b.x1) + return l +} + +// headFoot is how far from the rest of the page a short line must sit, in body +// sizes, to be a running head or a page number rather than part of the text. +const headFoot = 2.2 + +// headFootWidth is how much of the column such a line may fill. +const headFootWidth = 0.4 + +// stripRunningHeads drops the page number and the running head. Neither was in +// the source: LaTeX put them there, and writing them back would put them on the +// page twice — once as text in the middle of a paragraph and once again by the +// page style. +func stripRunningHeads(lines []line, body float64) []line { + for len(lines) > 1 && isolated(lines[0], lines[1], body) { + lines = lines[1:] + } + for len(lines) > 1 && isolated(lines[len(lines)-1], lines[len(lines)-2], body) { + lines = lines[:len(lines)-1] + } + return lines +} + +// isolated reports whether an edge line is short, set well away from its +// neighbour, and no larger than the body text. The size test is what keeps a +// section heading at the top of a page — which is also short, and also set off +// from what follows it — from being thrown away as a running head. +func isolated(edge, next line, body float64) bool { + return edge.size <= body*1.05 && + edge.x1-edge.x0 < headFootWidth*(edge.right-edge.left) && + abs(edge.y-next.y) > headFoot*body +} + +// fracReach is how far apart the two halves of a displayed fraction may sit, +// as a multiple of the size they are set at. A display sets both halves at full +// size, one above the bar and one below, and TeX leaves room for the tallest +// thing in each. +const fracReach = 2.4 + +// sharesBar reports whether a rule between a line and a band is a fraction bar +// with one of them above it and the other below. +// +// This is the case the size and reach tests above cannot cover. A superscript +// is recognised by being smaller; the two halves of a DISPLAYED fraction are +// both set at full size, three points above and three points below a bar, and +// nothing about either of them says they are not two ordinary lines of text one +// after the other. The bar says it. Without this, \frac{d}{dt} at the head of a +// displayed equation comes out as three separate lines - a d, the rest of the +// equation, and a dt - which compiles, and is wrong. +func sharesBar(l line, b band, rules []rule) bool { + if abs(b.y-l.y) > fracReach*l.size { + return false + } + for _, r := range rules { + // The bar sits between the two baselines, give or take: a fraction + // is set about the maths axis, which is a quarter of the size above + // the baseline of the equation it is in, so a bar with the whole + // fraction below it still lies a little above the line. + if !r.wide() || r.midY() < min(l.y, b.y)-0.2*l.size || r.midY() > max(l.y, b.y)+0.8*l.size { + continue + } + // The band has to be one half of the fraction: entirely within the + // bar's reach, and not much narrower than it. A bar is drawn a + // little wider than the wider of the two halves it separates, so a + // rule many times the width of what sits under it is a table's line + // and not a fraction's - which is what keeps a row of a table from + // being folded into the row above. + if b.x0 < r.x0-barSlack || b.x1 > r.x1+barSlack { + continue + } + if r.x1-r.x0 > 2*(b.x1-b.x0)+4 { + continue + } + // And the line has to be the equation the fraction belongs to: its + // other half, or the material either side of it. + if overlaps(r, l.x0-l.size, l.x1+l.size) { + return true + } + } + return false +} + +// overlaps reports whether a stretch of the page falls within a rule's reach. +func overlaps(r rule, x0, x1 float64) bool { + return x1 > r.x0-barSlack && x0 < r.x1+barSlack +} + +// limitSpan is how far the limits of a big operator may sit from the baseline +// of the equation they are part of, as a multiple of its size. A displayed +// \sum sets its limits further out than a superscript by half again, which is +// most of the way to the next line - so this reach is only offered to a band +// that is actually sitting over such an operator. +const limitSpan = 1.3 + +// overOperator reports whether a band is centred on a big operator the line +// already holds, which is what a limit is and a superscript is not: TeX puts a +// superscript AFTER what it belongs to and a limit ABOVE it. +func overOperator(l line, b band) bool { + mid := (b.x0 + b.x1) / 2 + for _, a := range l.atoms { + if a.sh.math != mathExt { + continue + } + if mid > a.x-l.size && mid < a.right()+l.size { + return true + } + } + return false +} + +// The gutter of a two-column page: how wide the empty strip down the middle +// must be, in points, and how far from the centre it may sit. +const ( + gutterWidth = 8.0 + gutterZone = 0.15 +) + +// gutter is where a page's columns are divided, and false for a page with one +// column. +// +// This has to be found before the lines are built rather than after. Two columns +// set to the same grid put their lines on the SAME baselines, so a page read by +// baseline alone comes back with every line of the left column joined to the +// line of the right column beside it - and the join happens before anything has +// had a chance to notice there are two columns. What gives it away is a strip +// down the middle of the page that no glyph enters, which is what a gutter is. +func gutter(atoms []atom, width float64) (float64, bool) { + if width <= 0 || len(atoms) == 0 { + return 0, false + } + lo, hi := int(width*(0.5-gutterZone)), int(width*(0.5+gutterZone)) + covered := make([]bool, hi-lo+1) + var left, right bool + for _, a := range atoms { + if a.x < float64(lo) { + left = true + } + if a.right() > float64(hi) { + right = true + } + for i := max(int(a.x)-lo, 0); i <= min(int(a.right())-lo, len(covered)-1); i++ { + covered[i] = true + } + } + if !left || !right { + return 0, false + } + best, run, end := 0, 0, 0 + for i, c := range covered { + if c { + run = 0 + continue + } + run++ + if run > best { + best, end = run, i + } + } + if float64(best) < gutterWidth { + return 0, false + } + return float64(lo+end) - float64(best)/2, true +} + +// split cuts every band at the gutter, so that the left column's line and the +// right column's are two lines rather than one. +func split(bs []band, x float64) []band { + var out []band + for _, b := range bs { + var lhs, rhs []atom + for _, a := range b.atoms { + if a.midX() < x { + lhs = append(lhs, a) + continue + } + rhs = append(rhs, a) + } + if len(lhs) == 0 || len(rhs) == 0 { + out = append(out, b) + continue + } + out = append(out, measured(lhs), measured(rhs)) + } + return out +} + +// measured is a band built from atoms that already share a baseline. +func measured(atoms []atom) band { + b := band{atoms: atoms, y: atoms[0].y, x0: atoms[0].x, x1: atoms[0].right(), allMath: true} + for _, a := range atoms { + b.size = max(b.size, a.size) + b.width += a.width + b.x0, b.x1 = min(b.x0, a.x), max(b.x1, a.right()) + if !a.sh.isMath() { + b.allMath = false + } + } + return b +} diff --git a/math.go b/math.go new file mode 100644 index 0000000..ee71eb0 --- /dev/null +++ b/math.go @@ -0,0 +1,427 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import ( + "sort" + "strings" + "unicode" +) + +// This file rebuilds an equation from the marks TeX left behind, which is the +// part of this package that is actually difficult. +// +// TeX does not write mathematics into a PDF. It writes glyphs, each at an +// absolute place, from the three families it sets maths in: the math italic +// family for variables and the lowercase Greek, the symbol family for +// relations and operators, and the extension family for the big operators and +// for delimiters grown to fit. Everything the source said about structure is +// gone, and what is left is geometry: +// +// - a superscript is a smaller glyph raised above the baseline, a subscript a +// smaller glyph dropped below it; +// - a fraction is a horizontal rule with material above it and material below +// it, both set smaller than what surrounds them; +// - a radical is a hook from the symbol family followed by a rule drawn over +// everything it covers, which is what says where the radicand ends; +// - a big operator's limits are the same shift, but centred on the operator +// rather than following it; +// - \left( and \right) are the same characters as ( and ), drawn from the +// extension family at whatever size fits, and raised so that their middle +// lands on the maths axis. +// +// So the reconstruction is a recursive decomposition of a set of placed glyphs. +// Take the widest rule that has material above and below it: that is the +// outermost fraction, and the material either side of it is what comes before +// and after. Recurse into each part with that rule removed. When no rule is +// left, walk what remains from left to right, and after each glyph take the +// smaller shifted ones that follow as its scripts. Recurse into those too. +// +// The decomposition is honest about what it cannot see. Nothing in the geometry +// distinguishes \alpha from a variable named by the same character, or \cdot +// from \bullet drawn small, or an author's \mathrm{Re} from the \Re symbol. +// Where a choice has to be made it is made once, in mathnames.go, and written +// down there. + +// A mathScope is the rules an equation may still use. Rules are consumed as the +// decomposition goes down, so that a fraction inside a numerator is found by the +// recursion rather than by the same rule matching twice. +type mathScope struct { + rules []rule + // pairs says whether grown delimiters may be written as \left and + // \right. They may only when the equation has as many of one as of the + // other; an equation cut in half by a line break has not, and \left with + // no \right does not compile. + pairs bool +} + +// scriptDrop is how far a piece must leave the baseline, as a fraction of its +// own size, before it counts as a script rather than as rounding. +const scriptDrop = 0.12 + +// mathSource writes an equation. items are its glyphs in order across the page, +// rs the rules that fall within it. +func mathSource(items []atom, rs []rule) string { + m := mathScope{rules: rs, pairs: balanced(items)} + return repair(strings.TrimSpace(m.render(items))) +} + +// balanced reports whether the grown delimiters in an equation pair up. +func balanced(items []atom) bool { + open, closed := 0, 0 + for _, a := range items { + if a.sh.math != mathExt { + continue + } + for _, r := range a.text { + if _, ok := openDelimiter[r]; ok { + open++ + continue + } + if _, ok := closeDelimiter[r]; ok { + closed++ + } + } + } + return open > 0 && open == closed +} + +// render writes a group of glyphs, taking the outermost fraction first. +func (m mathScope) render(items []atom) string { + if len(items) == 0 { + return "" + } + if s, ok := m.fraction(items); ok { + return s + } + return m.sequence(items) +} + +// fraction finds the widest rule with material both above and below it, and +// writes what it finds as \frac, with whatever lies to either side of the rule +// written around it. +func (m mathScope) fraction(items []atom) (string, bool) { + best, found := -1, 0.0 + for i, r := range m.rules { + if !m.bar(r, items) { + continue + } + if w := r.x1 - r.x0; w > found { + best, found = i, w + } + } + if best < 0 { + return "", false + } + r := m.rules[best] + inner := m.without(best) + var before, above, below, after []atom + for _, a := range items { + switch { + case !r.spans(a.midX(), barSlack): + if a.midX() < r.x0 { + before = append(before, a) + } else { + after = append(after, a) + } + case a.y > r.midY(): + above = append(above, a) + default: + below = append(below, a) + } + } + return inner.render(before) + `\frac{` + inner.render(above) + `}{` + + inner.render(below) + `}` + inner.render(after), true +} + +// barSlack is how far outside a rule a glyph's centre may sit and still count +// as under it. A fraction bar is drawn a shade wider than its material, but a +// glyph beside the fraction starts a good deal further away than this. +const barSlack = 1.0 + +// bar reports whether a rule is a fraction bar for these glyphs: wide, with +// material both above and below it, and not the bar of a radical — which is +// drawn hard against the radical sign's right edge and has nothing under it +// that is not also under the sign. +func (m mathScope) bar(r rule, items []atom) bool { + if !r.wide() { + return false + } + var up, down bool + for _, a := range items { + if radicalOf(a) && abs(a.right()-r.x0) < radicalReach { + return false + } + if !r.spans(a.midX(), barSlack) { + continue + } + if a.y > r.midY() { + up = true + } else { + down = true + } + } + return up && down +} + +// without is the scope with one rule taken out. +func (m mathScope) without(i int) mathScope { + rest := make([]rule, 0, len(m.rules)-1) + rest = append(rest, m.rules[:i]...) + rest = append(rest, m.rules[i+1:]...) + return mathScope{rules: rest, pairs: m.pairs} +} + +// radicalReach is how far a radical's bar may start from the sign's right edge. +const radicalReach = 2.5 + +// radicalSign is the character a radical is drawn with, U+221A. +const radicalSign = 0x221A + +// radicalOf reports whether a glyph is a radical sign. +func radicalOf(a atom) bool { return strings.ContainsRune(a.text, radicalSign) } + +// sequence writes a group with no fraction left in it: left to right, each +// glyph followed by the smaller shifted ones that hang from it. +func (m mathScope) sequence(items []atom) string { + base, size := level(items) + var out []string + for i := 0; i < len(items); { + a := items[i] + if radicalOf(a) { + if s, n, ok := m.radical(items, i); ok { + out = append(out, s) + i = n + continue + } + } + out = append(out, m.glyph(a)) + i++ + j := i + for j < len(items) && shifted(items[j], base, size) { + j++ + } + if j > i { + out = append(out, m.scripts(items[i:j], base)) + i = j + } + } + return join(out) +} + +// radical writes \sqrt. The bar drawn over the radicand says how far it +// reaches; without one there is nothing to say, and the sign is written as a +// bare \surd instead. +func (m mathScope) radical(items []atom, i int) (string, int, bool) { + a := items[i] + for k, r := range m.rules { + if !r.wide() || abs(r.x0-a.right()) >= radicalReach || r.midY() <= a.y { + continue + } + end := i + 1 + for end < len(items) && r.spans(items[end].midX(), barSlack) { + end++ + } + if end == i+1 { + continue + } + return `\sqrt{` + m.without(k).render(items[i+1:end]) + `}`, end, true + } + return "", 0, false +} + +// level is the baseline a group sits on and the size it is set at. +// +// The size is the largest there is. The baseline is the one the most ink sits +// on at that size — not the one under the widest single glyph, because in +// \\sum_{i=1}^n f(x_i) the widest glyph at full size is the \\sum itself, and a +// big operator is drawn RAISED so that it straddles the maths axis. Taking its +// baseline as the group’s puts the upper limit below it and both limits come +// back as subscripts. Glyphs the extension family raises by construction, and +// the radical sign, are left out of the vote for the same reason. +func level(items []atom) (float64, float64) { + size := 0.0 + for _, a := range items { + size = max(size, a.size) + } + y, ok := vote(items, size, true) + if !ok { + y, _ = vote(items, size, false) + } + return y, size +} + +// vote is the baseline carrying the most width at a group’s own size, and false +// when nothing qualifies. grounded leaves out the glyphs that are raised +// whatever the equation says. +func vote(items []atom, size float64, grounded bool) (float64, bool) { + weight := map[float64]float64{} + for _, a := range items { + if a.size <= scriptSize*size { + continue + } + if grounded && (a.sh.math == mathExt || radicalOf(a)) { + continue + } + weight[a.y] += a.width + } + best, found := 0.0, -1.0 + for y, w := range weight { + if w > found || (w == found && y > best) { + best, found = y, w + } + } + return best, found >= 0 +} + +// shifted reports whether a glyph is a script of what precedes it: smaller than +// the level it is being read at, and off its baseline. +func shifted(a atom, base, size float64) bool { + return a.size < scriptSize*size && abs(a.y-base) > scriptDrop*a.size +} + +// scripts writes the superscripts and subscripts that hang from one glyph. +// Both are written even when only one is there, and each is recursed into, +// since a script may hold a fraction of its own. +func (m mathScope) scripts(items []atom, base float64) string { + var up, down []atom + for _, a := range items { + if a.y > base { + up = append(up, a) + continue + } + down = append(down, a) + } + out := "" + if len(down) > 0 { + out += `_{` + m.render(down) + `}` + } + if len(up) > 0 { + out += `^{` + m.render(up) + `}` + } + return out +} + +// glyph writes one piece of text as mathematics. +func (m mathScope) glyph(a atom) string { + if s, ok := m.operatorName(a); ok { + return s + } + // The characters of one run need the same gaps between them as the runs + // of one equation. A run holding an epsilon and an e is drawn as two + // glyphs but arrives as one piece of text, and writing it out without + // a gap gives \\varepsilone, which is not a command any engine has. + var parts []string + for _, r := range a.text { + parts = append(parts, m.rune(a, r)) + } + return join(parts) +} + +// operatorName catches a run of upright letters, which inside an equation is +// either one of the names TeX has a command for — \sin, \log, \max — or a word +// the author set with \mathrm. +func (m mathScope) operatorName(a atom) (string, bool) { + if a.sh.isMath() || len([]rune(a.text)) < 2 { + return "", false + } + for _, r := range a.text { + if !unicode.IsLetter(r) { + return "", false + } + } + if functionName[a.text] { + return `\` + a.text, true + } + return `\mathrm{` + a.text + `}`, true +} + +// rune writes one character as mathematics. +func (m mathScope) rune(a atom, r rune) string { + if a.sh.math == mathExt && m.pairs { + if s, ok := openDelimiter[r]; ok { + return `\left` + s + } + if s, ok := closeDelimiter[r]; ok { + return `\right` + s + } + } + if s, ok := mathCommand[r]; ok { + return s + } + if decoration(r) { + return "" + } + if unicode.IsLetter(r) || mathPlain(r) { + return string(r) + } + return escapeText(string(r)) +} + +// join puts an equation's pieces together, with a space wherever leaving one +// out would run a command's name into what follows it. +func join(parts []string) string { + var b strings.Builder + for _, p := range parts { + if p == "" { + continue + } + if needsGap(b.String(), p) { + b.WriteByte(' ') + } + b.WriteString(p) + } + return b.String() +} + +// needsGap reports whether a control word at the end of what is written would +// swallow the start of what comes next. +func needsGap(done, next string) bool { + if done == "" || next == "" { + return false + } + last := rune(done[len(done)-1]) + if !unicode.IsLetter(last) || !unicode.IsLetter(rune(next[0])) { + return false + } + return endsInControlWord(done) +} + +// endsInControlWord reports whether the text ends inside a \command rather than +// in ordinary letters. +func endsInControlWord(s string) bool { + i := len(s) + for i > 0 && isASCIILetter(s[i-1]) { + i-- + } + return i > 0 && s[i-1] == '\\' +} + +// isASCIILetter reports whether a byte is one of the letters a control word may +// be spelled with. +func isASCIILetter(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + +// mathRules is the rules that fall inside an equation's box, which is what the +// decomposition is allowed to use. +func mathRules(items []atom, rs []rule) []rule { + x0, x1 := items[0].x, items[0].right() + ylo, yhi := items[0].y, items[0].y + size := 0.0 + for _, a := range items { + x0, x1 = min(x0, a.x), max(x1, a.right()) + ylo, yhi = min(ylo, a.y), max(yhi, a.y) + size = max(size, a.size) + } + var out []rule + for _, r := range rs { + if r.x0 >= x0-barSlack && r.x1 <= x1+barSlack && + r.midY() > ylo-0.5*size && r.midY() < yhi+1.2*size { + out = append(out, r) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].x0 < out[j].x0 }) + return out +} diff --git a/math_test.go b/math_test.go new file mode 100644 index 0000000..ff5ac4c --- /dev/null +++ b/math_test.go @@ -0,0 +1,147 @@ +package latex + +import "testing" + +var ( + mi = shape{math: mathLetter} + sy = shape{math: mathSymbol} + ex = shape{math: mathExt} +) + +func TestMaterialEitherSideOfAFraction(t *testing.T) { + items := []atom{ + at("k", 10, 100, 5, 10, mi), + at("a", 20, 104, 4, 7, mi), + at("b", 20, 96, 4, 7, mi), + at("m", 30, 100, 5, 10, mi), + } + got := mathSource(items, []rule{{19, 99.8, 24, 100.2}}) + if got != `k\frac{a}{b}m` { + t.Errorf("got %q", got) + } +} + +func TestWhatIsNotAFractionBar(t *testing.T) { + items := []atom{at("a", 20, 104, 4, 7, mi), at("b", 20, 96, 4, 7, mi)} + // An upright rule divides columns, not a fraction. + if got := mathSource(items, []rule{{21, 90, 21.4, 110}}); got != "ab" { + t.Errorf("got %q", got) + } + // A rule with everything on one side of it is not a bar. + if got := mathSource(items, []rule{{19, 90, 24, 90.4}}); got != "ab" { + t.Errorf("got %q", got) + } + // A rule the glyphs sit well to one side of has nothing under it. + side := []atom{at("a", 200, 104, 4, 7, mi), at("b", 200, 96, 4, 7, mi)} + if got := mathSource(side, []rule{{19, 99.8, 24, 100.2}}); got != "ab" { + t.Errorf("got %q", got) + } +} + +func TestARadicalWithNothingUnderIt(t *testing.T) { + // A radical sign with no bar over it is a sign and nothing more. + items := []atom{at("√", 20, 104, 8, 10, sy), at("x", 30, 100, 5, 10, mi)} + if got := mathSource(items, nil); got != `\surd x` { + t.Errorf("got %q", got) + } + // A bar that starts nowhere near the sign is not its bar. + if got := mathSource(items, []rule{{90, 108, 120, 108.4}}); got != `\surd x` { + t.Errorf("got %q", got) + } + // A bar below the sign is not its bar either. + if got := mathSource(items, []rule{{28, 96, 40, 96.4}}); got != `\surd x` { + t.Errorf("got %q", got) + } + // A bar with nothing under it covers no radicand. + if got := mathSource(items, []rule{{28, 108, 29, 108.4}}); got != `\surd x` { + t.Errorf("got %q", got) + } + // An upright rule beside the sign is not a bar. + if got := mathSource(items, []rule{{28, 108, 28.4, 130}}); got != `\surd x` { + t.Errorf("got %q", got) + } +} + +func TestAGroupOfNothingButExtensionGlyphs(t *testing.T) { + // Every glyph is raised by construction, so the baseline has to be + // taken from them after all rather than from anything standing on it. + items := []atom{at("∑", 20, 105, 10, 10, ex), at("∏", 32, 105, 10, 10, ex)} + if got := mathSource(items, nil); got != `\sum\prod` { + t.Errorf("got %q", got) + } +} + +func TestWordsInsideAnEquation(t *testing.T) { + // A run of upright letters TeX has a command for. + items := []atom{at("log", 20, 100, 15, 10, shape{}), at("x", 36, 100, 5, 10, mi)} + if got := mathSource(items, nil); got != `\log x` { + t.Errorf("got %q", got) + } + // One it has not was set with \mathrm. + items[0] = at("diag", 20, 100, 20, 10, shape{}) + if got := mathSource(items, nil); got != `\mathrm{diag}x` { + t.Errorf("got %q", got) + } + // A run with something in it that is not a letter is not a name. + items[0] = at("x2", 20, 100, 10, 10, shape{}) + if got := mathSource(items, nil); got != `x2x` { + t.Errorf("got %q", got) + } +} + +func TestCharactersAnEquationCannotName(t *testing.T) { + // A free-standing accent is left out: there is no way to say which + // glyph it belonged to. + if got := mathSource([]atom{at("x", 20, 100, 5, 10, mi), at("ˆ", 25, 100, 3, 10, sy)}, nil); got != "x" { + t.Errorf("got %q", got) + } + // A character with no meaning in mathematics is escaped so that it + // compiles rather than being written raw. + if got := mathSource([]atom{at("%", 20, 100, 5, 10, sy)}, nil); got != `\%` { + t.Errorf("got %q", got) + } +} + +func TestWhichRulesAnEquationMayUse(t *testing.T) { + items := []atom{at("a", 20, 104, 4, 7, mi), at("b", 20, 96, 4, 7, mi)} + in := []rule{ + {19, 99.8, 24, 100.2}, + {19.5, 99.9, 23, 100.1}, + // Far off to the right: another equation's. + {300, 99.8, 340, 100.2}, + } + got := mathRules(items, in) + if len(got) != 2 || got[0].x0 != 19 { + t.Errorf("got %+v", got) + } +} + +func TestAnEquationWithNothingLeftInIt(t *testing.T) { + // Every glyph was an accent that could not be placed. Writing this as + // $$ would not be an empty equation but the start of a displayed one. + if got := mathSource([]atom{at("ˆ", 20, 100, 3, 10, sy)}, nil); got != "" { + t.Errorf("got %q", got) + } + seg := segment{atoms: []atom{at("ˆ", 20, 100, 3, 10, sy)}, math: true} + if got := lineText(line{}, []segment{seg, seg}, nil, false); got != "" { + t.Errorf("got %q", got) + } + if got := displayBody([]segment{seg}, nil); got != "" { + t.Errorf("got %q", got) + } + e := &emitter{} + e.display(line{}, []segment{seg}, nil) + if e.out.String() != "" { + t.Errorf("got %q", e.out.String()) + } +} + +func TestAWordInsideADisplayedEquation(t *testing.T) { + segs := []segment{ + {atoms: []atom{at("x", 20, 100, 5, 10, mi)}, math: true}, + {atoms: []atom{at("for", 30, 100, 15, 10, shape{})}}, + } + if got := displayBody(segs, nil); got != `x \text{for}` { + t.Errorf("got %q", got) + } +} diff --git a/mathnames.go b/mathnames.go new file mode 100644 index 0000000..360cd8e --- /dev/null +++ b/mathnames.go @@ -0,0 +1,166 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "unicode" + +// This file is the dictionary between what a glyph turned into and what the +// author wrote. A PDF says the character is U+03B1; the source said \alpha. +// There is no way to derive one from the other, so it is written out. +// +// The characters come from the font's /ToUnicode map, or failing that from the +// glyph names in its built-in encoding — either way that work is already done +// by github.com/go-pdfkit/pdffont, and what arrives here is a rune. Some of +// these are unambiguous (U+2211 is \sum and nothing else); some are a choice +// this makes and should say so. U+2212 is emitted as a plain minus because that +// is what TeX's - means in maths, and U+00D7 as \times rather than \x. Where a +// symbol has an AMS spelling and a plain one, the plain one wins, so that the +// output needs amsmath and amssymb but not more. + +// mathCommand is what a rune is written as inside mathematics. A rune that is +// not here is emitted as itself, which is right for the letters, the digits and +// the handful of characters TeX takes literally. +var mathCommand = map[rune]string{ + // Lowercase Greek, which TeX sets from the math italic family. + 'α': `\alpha`, 'β': `\beta`, 'γ': `\gamma`, 'δ': `\delta`, + 'ε': `\epsilon`, 'ζ': `\zeta`, 'η': `\eta`, 'θ': `\theta`, + 'ι': `\iota`, 'κ': `\kappa`, 'λ': `\lambda`, 'μ': `\mu`, + 'ν': `\nu`, 'ξ': `\xi`, 'π': `\pi`, 'ρ': `\rho`, + 'σ': `\sigma`, 'τ': `\tau`, 'υ': `\upsilon`, 'φ': `\phi`, + 'χ': `\chi`, 'ψ': `\psi`, 'ω': `\omega`, + 'ϑ': `\vartheta`, 'ϕ': `\varphi`, 'ϖ': `\varpi`, 'ϱ': `\varrho`, + 'ς': `\varsigma`, 'ϵ': `\varepsilon`, 'ϝ': `\digamma`, + // Uppercase Greek. Only the ones TeX has a command for: the rest are + // Latin capitals in the roman font and come through as themselves. + 'Γ': `\Gamma`, 'Δ': `\Delta`, 'Θ': `\Theta`, 'Λ': `\Lambda`, + 'Ξ': `\Xi`, 'Π': `\Pi`, 'Σ': `\Sigma`, 'Υ': `\Upsilon`, + 'Φ': `\Phi`, 'Ψ': `\Psi`, 'Ω': `\Omega`, + // Relations. + '≤': `\leq`, '≥': `\geq`, '≠': `\neq`, '≈': `\approx`, + '≡': `\equiv`, '∼': `\sim`, '≃': `\simeq`, '≅': `\cong`, + '∝': `\propto`, '≪': `\ll`, '≫': `\gg`, '≺': `\prec`, '≻': `\succ`, + '⪯': `\preceq`, '⪰': `\succeq`, '≍': `\asymp`, '≐': `\doteq`, + '⊥': `\perp`, '∥': `\parallel`, '∣': `\mid`, '⊢': `\vdash`, '⊣': `\dashv`, + '⊨': `\models`, '≜': `\triangleq`, '≔': `:=`, + // Set relations. + '∈': `\in`, '∉': `\notin`, '∋': `\ni`, '⊂': `\subset`, '⊃': `\supset`, + '⊆': `\subseteq`, '⊇': `\supseteq`, '∅': `\emptyset`, + // Binary operators. + '±': `\pm`, '∓': `\mp`, '×': `\times`, '÷': `\div`, '∗': `\ast`, + '⋆': `\star`, '∘': `\circ`, '∙': `\bullet`, '·': `\cdot`, '⋅': `\cdot`, + '∪': `\cup`, '∩': `\cap`, '⊎': `\uplus`, '⊓': `\sqcap`, '⊔': `\sqcup`, + '∨': `\vee`, '∧': `\wedge`, '∖': `\setminus`, '≀': `\wr`, + '⊕': `\oplus`, '⊖': `\ominus`, '⊗': `\otimes`, '⊘': `\oslash`, + '⊙': `\odot`, '†': `\dagger`, '‡': `\ddagger`, '⨿': `\amalg`, + '△': `\bigtriangleup`, '▽': `\bigtriangledown`, '◁': `\triangleleft`, + '▷': `\triangleright`, + // Arrows. + '←': `\leftarrow`, '→': `\rightarrow`, '↔': `\leftrightarrow`, + '⇐': `\Leftarrow`, '⇒': `\Rightarrow`, '⇔': `\Leftrightarrow`, + '↑': `\uparrow`, '↓': `\downarrow`, '↕': `\updownarrow`, + '⇑': `\Uparrow`, '⇓': `\Downarrow`, '⇕': `\Updownarrow`, + '↦': `\mapsto`, '⟶': `\longrightarrow`, '⟵': `\longleftarrow`, + '⟹': `\Longrightarrow`, '⟸': `\Longleftarrow`, '⟷': `\longleftrightarrow`, + '↩': `\hookleftarrow`, '↪': `\hookrightarrow`, '⇀': `\rightharpoonup`, + '⇁': `\rightharpoondown`, '↼': `\leftharpoonup`, '↽': `\leftharpoondown`, + '↗': `\nearrow`, '↘': `\searrow`, '↙': `\swarrow`, '↖': `\nwarrow`, + // Miscellaneous. + '∞': `\infty`, '∂': `\partial`, '∇': `\nabla`, '∀': `\forall`, + '∃': `\exists`, '¬': `\neg`, '∠': `\angle`, '□': `\Box`, + 'ℓ': `\ell`, 'ℏ': `\hbar`, 'ℜ': `\Re`, 'ℑ': `\Im`, '℘': `\wp`, + 'ℵ': `\aleph`, '′': `'`, '″': `''`, '‴': `'''`, + '…': `\ldots`, '⋯': `\cdots`, '⋮': `\vdots`, '⋱': `\ddots`, + '♭': `\flat`, '♮': `\natural`, '♯': `\sharp`, '♣': `\clubsuit`, + '♦': `\diamondsuit`, '♥': `\heartsuit`, '♠': `\spadesuit`, + '✓': `\checkmark`, '∴': `\therefore`, '∵': `\because`, + // A degree sign is a superscript, and a superscript needs something to + // be the superscript of. + '\u00B0': `{}^\circ`, '\u00AF': `-`, '\u2016': `\|`, + // The big operators, which take limits. + '∑': `\sum`, '∏': `\prod`, '∐': `\coprod`, '∫': `\int`, + '∮': `\oint`, '⋃': `\bigcup`, '⋂': `\bigcap`, '⨁': `\bigoplus`, + '⨂': `\bigotimes`, '⨀': `\bigodot`, '⋁': `\bigvee`, '⋀': `\bigwedge`, + '⨆': `\bigsqcup`, '∬': `\iint`, '∭': `\iiint`, + // Delimiters. + '⟨': `\langle`, '⟩': `\rangle`, '⌈': `\lceil`, '⌉': `\rceil`, + '\u230A': `\lfloor`, '\u230B': `\rfloor`, + // A radical sign this could not find a bar for has no radicand either, + // and \sqrt with nothing after it does not compile: \surd is the sign + // on its own, and \sqrt{...} is written out by the radical rule. + '\u221A': `\surd`, + // A minus sign is TeX's plain hyphen in maths, and a dash inside an + // equation is always one. + '\u2212': `-`, '\u2013': `-`, '\u2014': `-`, + // Characters Unicode has more than one code for. A font's map answers + // with whichever its maker chose, and an equation set in one of them + // would otherwise come back with a raw character in it. + '\u2126': `\Omega`, '\u00B5': `\mu`, '\u2206': `\Delta`, + '\u2019': `'`, +} + +// bigOperator reports whether a rune is one of the operators whose scripts +// become limits. LaTeX places them itself, so the only thing that matters here +// is knowing that a script sitting under such an operator rather than after it +// is still a subscript. +var bigOperator = map[rune]bool{ + '∑': true, '∏': true, '∐': true, '∫': true, '∮': true, + '⋃': true, '⋂': true, '⨁': true, '⨂': true, '⨀': true, + '⋁': true, '⋀': true, '⨆': true, '∬': true, '∭': true, +} + +// openDelimiter is what a rune is written as after \left. +var openDelimiter = map[rune]string{ + '(': `(`, '[': `[`, '{': `\{`, '⟨': `\langle`, + '⌈': `\lceil`, '⌊': `\lfloor`, '|': `|`, '‖': `\|`, + '↑': `\uparrow`, '⇑': `\Uparrow`, '/': `/`, +} + +// closeDelimiter is what a rune is written as after \right. +var closeDelimiter = map[rune]string{ + ')': `)`, ']': `]`, '}': `\}`, '⟩': `\rangle`, + '⌉': `\rceil`, '⌋': `\rfloor`, '|': `|`, '‖': `\|`, + '↓': `\downarrow`, '⇓': `\Downarrow`, '\\': `\backslash`, +} + +// functionName is the set of operator names TeX has a command for. A run of +// roman letters inside an equation is one of these, or it is a word the author +// wrote with \mathrm; both are set upright, and the only way to tell them apart +// is this list. +var functionName = map[string]bool{ + "arccos": true, "arcsin": true, "arctan": true, "arg": true, + "cos": true, "cosh": true, "cot": true, "coth": true, "csc": true, + "deg": true, "det": true, "dim": true, "exp": true, "gcd": true, + "hom": true, "inf": true, "ker": true, "lg": true, "lim": true, + "liminf": true, "limsup": true, "ln": true, "log": true, "max": true, + "min": true, "mod": true, "Pr": true, "sec": true, "sin": true, + "sinh": true, "sup": true, "tan": true, "tanh": true, +} + +// mathPlain reports whether a rune drawn in a text font inside an equation may +// be written as itself. Digits, the arithmetic that lives in the roman font, +// and the punctuation TeX takes literally. +func mathPlain(r rune) bool { + if r >= '0' && r <= '9' { + return true + } + switch r { + case '+', '=', '<', '>', '(', ')', '[', ']', '/', '!', ':', ';', + ',', '.', '|', '\'', '-', '*', '?', '@': + return true + } + return false +} + +// decoration reports whether a character is a free-standing accent rather than +// something to set. +// +// A PDF draws \hat{L} as an L and a circumflex placed over it, two glyphs; the +// font's map calls the second one U+02C6, a modifier letter. There is no way to +// write that back as \hat without knowing which glyph it belongs to and how far +// the accent's own box was shifted, which this does not attempt. What it does +// do is leave the accent out rather than write a character into the source that +// no engine will set - the reconstruction loses the hat and keeps the L. +func decoration(r rune) bool { + return unicode.In(r, unicode.Sk, unicode.Lm, unicode.Mn) +} diff --git a/matrix.go b/matrix.go new file mode 100644 index 0000000..4eb47a2 --- /dev/null +++ b/matrix.go @@ -0,0 +1,49 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +// A matrix is the six numbers a PDF transform is written as. Only three things +// are ever asked of one here — compose two, move a point, and say how much a +// length is stretched — so this carries its own rather than depending on a +// drawing library for three lines of arithmetic. +type matrix struct{ a, b, c, d, e, f float64 } + +// mul is the transform that applies m first and then n, which is the order a +// PDF composes them in. +func (m matrix) mul(n matrix) matrix { + return matrix{ + a: m.a*n.a + m.b*n.c, + b: m.a*n.b + m.b*n.d, + c: m.c*n.a + m.d*n.c, + d: m.c*n.b + m.d*n.d, + e: m.e*n.a + m.f*n.c + n.e, + f: m.e*n.b + m.f*n.d + n.f, + } +} + +// apply moves a point. +func (m matrix) apply(x, y float64) (float64, float64) { + return m.a*x + m.c*y + m.e, m.b*x + m.d*y + m.f +} + +// scale is how much the transform stretches a length, taken as the geometric +// mean of its two axes so that a rotation counts as no stretch at all. +func (m matrix) scale() float64 { + x := m.a*m.a + m.b*m.b + y := m.c*m.c + m.d*m.d + return sqrt(sqrt(x * y)) +} + +// sqrt is Newton's method, which is here rather than math.Sqrt only to keep the +// arithmetic in one place with the rest of the geometry. +func sqrt(v float64) float64 { + if v <= 0 { + return 0 + } + x := v + for i := 0; i < 24; i++ { + x = (x + v/x) / 2 + } + return x +} diff --git a/matrix_test.go b/matrix_test.go new file mode 100644 index 0000000..f02d92b --- /dev/null +++ b/matrix_test.go @@ -0,0 +1,39 @@ +package latex + +import "testing" + +func TestComposingTransforms(t *testing.T) { + move := matrix{1, 0, 0, 1, 5, 7} + twice := matrix{2, 0, 0, 2, 0, 0} + // Moving then doubling doubles the move. + x, y := move.mul(twice).apply(1, 1) + if x != 12 || y != 16 { + t.Errorf("got (%v,%v), want (12,16)", x, y) + } + if s := twice.scale(); s != 2 { + t.Errorf("a doubling stretches by %v", s) + } + // A rotation stretches nothing. + if s := (matrix{0, 1, -1, 0, 0, 0}).scale(); s < 0.999 || s > 1.001 { + t.Errorf("a quarter turn stretches by %v", s) + } + // A transform that collapses the page stretches by nothing at all. + if s := (matrix{0, 0, 0, 0, 0, 0}).scale(); s != 0 { + t.Errorf("a collapse stretches by %v", s) + } +} + +func TestSquareRoot(t *testing.T) { + if v := sqrt(9); v < 2.999 || v > 3.001 { + t.Errorf("sqrt(9) = %v", v) + } + if v := sqrt(-1); v != 0 { + t.Errorf("sqrt(-1) = %v", v) + } +} + +func TestSizeWithoutSign(t *testing.T) { + if abs(-3) != 3 || abs(3) != 3 { + t.Error("abs is wrong") + } +} diff --git a/page_test.go b/page_test.go new file mode 100644 index 0000000..aac1c83 --- /dev/null +++ b/page_test.go @@ -0,0 +1,157 @@ +package latex + +import ( + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// built is a one-page document assembled by hand, for the tests that need more +// of a page than a content stream and a font. +type built struct { + w *reader.Writer + res reader.Dict + content string + box [4]int + contents reader.Object +} + +func newBuilt() *built { + return &built{w: reader.NewWriter("1.7"), res: reader.Dict{}, box: [4]int{0, 0, 200, 200}} +} + +func (b *built) open(t *testing.T) *reader.Document { + t.Helper() + pagesRef := b.w.Reserve() + contents := b.contents + if contents == nil { + contents = b.w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte(b.content)}) + } + page := b.w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(b.box[0]), reader.Integer(b.box[1]), + reader.Integer(b.box[2]), reader.Integer(b.box[3])}, + "Contents": contents, "Resources": b.res, + }) + b.w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + root := b.w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := b.w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestAPictureOnThePageBecomesAFigure(t *testing.T) { + b := newBuilt() + img := b.w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(2), + "BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceGray"), + }, Raw: []byte{0, 64, 128, 255}}) + tiny := b.w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(1), "Height": reader.Integer(1), + "BitsPerComponent": reader.Integer(8), "ColorSpace": reader.Name("DeviceGray"), + }, Raw: []byte{0}}) + b.res["XObject"] = reader.Dict{"Im0": img, "Im1": tiny} + b.res["Font"] = reader.Dict{"R": b.w.Add(fontDict(b.w, roman))} + b.content = show("R", 10, 20, 180, "Text above the picture on this page.") + + "q 100 0 0 60 20 80 cm /Im0 Do Q\n" + + "q 4 0 0 4 20 40 cm /Im1 Do Q\n" + + show("R", 10, 20, 20, "Text below the picture on this page.") + doc, err := Reconstruct(b.open(t), Options{}) + if err != nil { + t.Fatal(err) + } + if len(doc.Files) != 1 || !strings.Contains(doc.Body, `\includegraphics`) { + t.Fatalf("got %d files:\n%s", len(doc.Files), doc.Body) + } + if !strings.Contains(doc.Body, "width=100.0pt,height=60.0pt") { + t.Errorf("got %s", doc.Body) + } + // The four-point picture is a rule or a bullet, not a figure. + if strings.Count(doc.Body, `\includegraphics`) != 1 { + t.Errorf("got %s", doc.Body) + } +} + +func TestATitleAndItsMaketitle(t *testing.T) { + b := newBuilt() + b.box = [4]int{0, 0, 400, 300} + b.res["Font"] = reader.Dict{ + "R": b.w.Add(fontDict(b.w, roman)), + "H": b.w.Add(fontDict(b.w, face{base: "CMBX17"})), + } + b.content = show("H", 17, 113, 280, "A Title") + + show("R", 10, 20, 240, "Body text that runs right across the page here.") + + show("R", 10, 20, 228, "More body text that runs across the page as well.") + + show("R", 10, 20, 216, "Still more body text to weigh the body size down.") + doc, err := Reconstruct(b.open(t), Options{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(doc.Preamble, `\title{A Title}`) { + t.Errorf("preamble is\n%s", doc.Preamble) + } + if !strings.HasPrefix(doc.Body, `\maketitle`) || strings.Contains(doc.Body, "A Title") { + t.Errorf("body is\n%s", doc.Body) + } +} + +func TestTheSameFontNameInAPageAndInAForm(t *testing.T) { + // The page's own font wins, which is the wrong answer only for a + // document that gives one name to two different faces. + b := newBuilt() + form := b.w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"Font": reader.Dict{"R": b.w.Add(fontDict(b.w, bold))}}, + }, Raw: []byte("")}) + b.res["Font"] = reader.Dict{"R": b.w.Add(fontDict(b.w, roman))} + b.res["XObject"] = reader.Dict{"F": form} + b.content = show("R", 10, 20, 150, "plain") + if got, err := Reconstruct(b.open(t), Options{}); err != nil { + t.Fatal(err) + } else if strings.Contains(got.Body, `\textbf`) { + t.Errorf("the form's font won: %s", got.Body) + } +} + +func TestARunWhoseCharactersCannotBeWorkedOut(t *testing.T) { + // A symbolic font with no map and glyphs named after numbers says + // nothing about what its codes stand for. extract reports the run as + // unreadable with no text in it, and there is nothing to write. + b := newBuilt() + b.res["Font"] = reader.Dict{"N": b.w.Add(reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("Type1"), + "BaseFont": reader.Name("Odd"), "FirstChar": reader.Integer(65), + "LastChar": reader.Integer(66), "Widths": reader.Array{reader.Integer(500), reader.Integer(500)}, + "Encoding": reader.Dict{"Differences": reader.Array{reader.Integer(65), + reader.Name("g12"), reader.Name("g13")}}, + })} + b.content = show("N", 10, 20, 150, "AB") + doc, err := Reconstruct(b.open(t), Options{}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(doc.Body) != "" { + t.Errorf("got %q", doc.Body) + } +} + +func TestAPageWhoseContentCannotBeRead(t *testing.T) { + b := newBuilt() + b.contents = b.w.Add(&reader.Stream{ + Dict: reader.Dict{"Filter": reader.Name("FlateDecode")}, + Raw: []byte("not compressed at all"), + }) + if _, err := Source(b.open(t)); err == nil { + t.Error("a page whose content will not decode read without error") + } +} diff --git a/region.go b/region.go new file mode 100644 index 0000000..710e6e7 --- /dev/null +++ b/region.go @@ -0,0 +1,190 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "unicode" + +// This file decides where an equation starts and stops, which is a question the +// page does not answer. +// +// TeX sets the $ and the text around it in the same roman font, so the boundary +// leaves no trace. What it does leave is the three math families: a glyph from +// the math italic, symbol or extension family is inside an equation and nothing +// else is. Those are the seeds. Around each one the equation is grown outwards +// over the glyphs that the roman font also drew but that belong to the maths — +// the digits, the + and the =, the parentheses — recognised by two things +// together: they are the sort of character an equation contains, and they are +// close enough to their neighbour that TeX did not put a word space there. +// +// Both halves matter. Without the character test, "and" gets eaten whenever it +// happens to sit near an equation; without the spacing test, the "2" of +// "section 2" joins the equation on the same line. + +// wordGap is how wide a gap must be, as a fraction of the font's own space, to +// be a word break rather than kerning. TeX kerns by a point or two and spaces by +// three or four, so a third of a space separates them cleanly. +const wordGap = 0.35 + +// mathGap is how far an equation may reach over a gap to take in a character +// its own font did not draw, as a fraction of the size it is set at. TeX’s +// widest ordinary inter-atom space is five eighteenths of an em on each side of +// a relation, so a shade under half an em covers both. +const mathGap = 0.45 + +// A segment is a stretch of one line that is either all mathematics or all +// text. +type segment struct { + atoms []atom + math bool +} + +// segments splits a line into its equations and the text between them. +func segments(l line) []segment { + if len(l.atoms) == 0 { + return nil + } + inMath := grow(l) + var out []segment + for i := 0; i < len(l.atoms); { + j := i + for j < len(l.atoms) && inMath[j] == inMath[i] { + j++ + } + out = append(out, segment{atoms: l.atoms[i:j], math: inMath[i]}) + i = j + } + return out +} + +// grow marks which of a line's glyphs are inside an equation: the ones drawn in +// a math family, the single italic letters standing among roman, and the ones +// the growth reaches from them. +func grow(l line) []bool { + in := make([]bool, len(l.atoms)) + any := false + for i, a := range l.atoms { + if a.sh.isMath() || italicVariable(l.atoms, i) { + in[i], any = true, true + } + } + if !any { + return in + } + for i := range in { + if !in[i] { + continue + } + for j := i - 1; j >= 0 && !in[j] && absorbs(l.atoms[j], l.atoms[j+1], l.atoms[j]); j-- { + in[j] = true + } + for j := i + 1; j < len(l.atoms) && !in[j] && absorbs(l.atoms[j-1], l.atoms[j], l.atoms[j]); j++ { + in[j] = true + } + } + return dropTrailing(l.atoms, in) +} + +// dropTrailing takes the punctuation off the end of each equation. TeX leaves no +// space between $x$ and the full stop after it, so the growth reaches over the +// gap every time; a comma inside an equation is never the last thing in it, +// which is what tells the two apart. +func dropTrailing(atoms []atom, in []bool) []bool { + for i := len(in) - 1; i >= 0; i-- { + if !in[i] || (i+1 < len(in) && in[i+1]) { + continue + } + for j := i; j >= 0 && in[j] && trailing(atoms[j]); j-- { + in[j] = false + } + } + return in +} + +// absorbs reports whether an equation reaches over the gap between two +// neighbouring glyphs to take in outer, the one that is not yet part of it. +func absorbs(left, right, outer atom) bool { + if !mathLike(outer) { + return false + } + gap := right.x - left.right() + // TeX puts a thin, medium or thick space around the operators inside an + // equation - up to five eighteenths of an em - which is wider than the + // kerning a word space has to be told from, so the reach here is measured + // against the font size rather than against the space. What keeps a word + // of prose out is mathLike, not the distance. + return gap < mathGap*max(left.size, right.size) || + gap < wordGap*max(left.wordSpace(), right.wordSpace()) +} + +// mathLike reports whether a piece of text drawn in a text font is the sort an +// equation contains. A single letter is a variable set upright; two or more are +// an operator name; digits and the arithmetic characters are themselves. A word +// of ordinary prose is none of those. +func mathLike(a atom) bool { + letters := 0 + for _, r := range a.text { + switch { + case unicode.IsLetter(r): + letters++ + case mathPlain(r) || r == ' ': + default: + return false + } + } + if letters == 0 { + return true + } + return letters == len([]rune(a.text)) && + (letters == 1 || functionName[a.text]) +} + +// trailing reports whether a glyph is the punctuation that follows an equation +// rather than part of it. TeX leaves no space between $x$ and the full stop +// after it, so without this test every equation at the end of a sentence +// swallows the sentence's punctuation. +func trailing(a atom) bool { + if a.sh.isMath() { + return false + } + for _, r := range a.text { + if r != '.' && r != ',' && r != ';' && r != ':' { + return false + } + } + return len(a.text) > 0 +} + +// italicVariable reports whether a single italic letter standing among roman +// text is a mathematical variable. +// +// It is here because one widely used font package makes the font names useless +// for this. Fourier sets its Greek and its operators in Fourier-Math-Letters and +// Fourier-Math-Symbols, but it sets the ordinary math LETTERS in Utopia-Italic — +// the same face the text italic uses. A paper set in it has "(L, P)" drawn with +// the parentheses roman and the L and the P italic, and nothing in any font name +// says that this is an equation rather than two emphasised letters. +// +// What does say so is the arrangement. A letter set in italic, alone, with roman +// on both sides of it, is a variable: emphasis applies to words, and a word is +// more than one letter. The neighbours have to be checked rather than only the +// letter, because a producer that emits one glyph per showing operator turns a +// whole italic sentence into single italic letters, and every one of them would +// otherwise qualify. +func italicVariable(atoms []atom, i int) bool { + a := atoms[i] + if a.sh.isMath() || !a.sh.italic { + return false + } + rs := []rune(a.text) + if len(rs) != 1 || !unicode.IsLetter(rs[0]) { + return false + } + if i > 0 && atoms[i-1].sh.italic { + return false + } + if i+1 < len(atoms) && atoms[i+1].sh.italic { + return false + } + return true +} diff --git a/repair.go b/repair.go new file mode 100644 index 0000000..f25dfce --- /dev/null +++ b/repair.go @@ -0,0 +1,182 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "strings" + +// This file makes sure that what comes out compiles. +// +// A reconstruction is worth nothing if TeX will not read it back, and the +// reconstruction of an equation is put together from geometry rather than from +// grammar, so it can end up saying things that no source ever would: a +// superscript with nothing to be the superscript of, because the glyph it +// belonged to was set in a font this could not read; two subscripts on one +// letter, because a script group was split by a rule; a brace left open, +// because the material a \frac needed ran off the end of the line; a \left with +// its \right on the next line. +// +// Every one of those aborts the compile, and aborting the compile loses the +// whole document rather than the one equation. So the last thing done to an +// equation is to read it back the way TeX will and repair what TeX would refuse. +// The repairs are the smallest ones that keep the meaning: an empty group in +// front of an orphaned script, an empty group between two scripts that would +// otherwise land on the same letter, the missing braces, and - for a delimiter +// that lost its partner - the plain character in place of the grown one. +// +// This was not written from first principles. It was written from a run over +// the corpus: of the first ninety-nine papers reconstructed, twenty-seven +// compiled. The four faults above are what the other seventy-two were. + +// repair returns an equation that TeX will read, which is not always the one it +// was given. +func repair(s string) string { + rs := []rune(s) + // A \left whose \right is missing, or is inside a group of its own, + // does not compile, and there is nothing to pair it with: the equation + // was cut in half by a line break or by a fraction bar. The delimiter + // itself is still right, only its growing is lost. + // + // This is judged on whole control words. \rightarrow starts with the + // six characters of \right, and a pass that works on the text rather + // than on the tokens turns every limit in the document into the + // undefined command \thetaarrow. + strip := !wellPaired(rs) + var b strings.Builder + // levels is the stack of groups, each remembering whether anything in it + // can carry a script and which scripts it has already been given. + levels := []group{{}} + depth := func() *group { return &levels[len(levels)-1] } + for i := 0; i < len(rs); i++ { + switch r := rs[i]; { + case r == '{': + levels = append(levels, group{}) + b.WriteRune(r) + case r == '}': + if len(levels) == 1 { + // A closing brace with nothing open would end whatever + // group the surrounding text is in. + continue + } + levels = levels[:len(levels)-1] + depth().base = true + b.WriteRune(r) + case r == '^' || r == '_' || r == '\'': + // A prime is TeX's own shorthand for a superscript, and lands + // on the same letter as one. + kind := r + if r == '\'' { + kind = '^' + } + g := depth() + if !g.base || g.taken(kind) { + b.WriteString("{}") + g.reset() + } + g.mark(kind) + b.WriteRune(r) + case r == '\\': + word, next := controlWord(rs, i) + if !strip || (word != "left" && word != "right") { + b.WriteString(string(rs[i:next])) + depth().start() + } + i = next - 1 + default: + b.WriteRune(r) + if r != ' ' { + depth().start() + } + } + } + // Whatever is still open is closed, in the order it was opened. + b.WriteString(strings.Repeat("}", len(levels)-1)) + return b.String() +} + +// controlWord reads the command starting at the backslash at i: its name, and +// where it ends. A control symbol - a backslash and one character that is not a +// letter - has that character as its name. +func controlWord(rs []rune, i int) (string, int) { + j := i + 1 + for j < len(rs) && isASCIILetter(byte(rs[j])) { + j++ + } + if j == i+1 && j < len(rs) { + j++ + } + return string(rs[i+1 : j]), j +} + +// A group is what is known about one level of braces: whether it holds anything +// a script could attach to, and which scripts have already attached. +type group struct { + base bool + sub bool + sup bool +} + +// taken reports whether this level already has a script of that kind. +func (g *group) taken(r rune) bool { + if r == '_' { + return g.sub + } + return g.sup +} + +// mark records that a script of that kind has been given. +func (g *group) mark(r rune) { + if r == '_' { + g.sub = true + return + } + g.sup = true +} + +// start records something a script could attach to, which is also the end of +// whatever the last thing's scripts were. +func (g *group) start() { + g.base, g.sub, g.sup = true, false, false +} + +// reset forgets the scripts given so far, for a level that has just been handed +// an empty group to hang the next one from. +func (g *group) reset() { + g.sub, g.sup = false, false +} + +// wellPaired reports whether every \left has its \right after it and inside the +// same group. +// +// Counting them is not enough. A fraction whose numerator holds the \left and +// whose denominator holds the \right has one of each and does not compile: TeX +// reads each group on its own, and a \right it meets with no \left open is an +// error however many there are elsewhere in the equation. That happens whenever +// a grown delimiter straddles a fraction bar, which the decomposition splits. +func wellPaired(rs []rune) bool { + open := []int{0} + for i := 0; i < len(rs); i++ { + switch rs[i] { + case '{': + open = append(open, 0) + case '}': + if len(open) == 1 || open[len(open)-1] != 0 { + return false + } + open = open[:len(open)-1] + case '\\': + word, next := controlWord(rs, i) + i = next - 1 + switch word { + case "left": + open[len(open)-1]++ + case "right": + if open[len(open)-1] == 0 { + return false + } + open[len(open)-1]-- + } + } + } + return len(open) == 1 && open[0] == 0 +} diff --git a/repair_test.go b/repair_test.go new file mode 100644 index 0000000..64d2c1b --- /dev/null +++ b/repair_test.go @@ -0,0 +1,89 @@ +package latex + +import "testing" + +func TestRepairingAnEquationTeXWouldRefuse(t *testing.T) { + cases := [][2]string{ + // A script with nothing to be the script of. + {`^{2}`, `{}^{2}`}, + // A plus is an atom and may carry a script; that is legal TeX. + {`+^{i}x`, `+^{i}x`}, + // A script at the head of a group has nothing before it. + {`{^{i}}`, `{{}^{i}}`}, + // Two scripts of the same kind on one letter. + {`x_{1}_{2}`, `x_{1}{}_{2}`}, + {`x^{1}^{2}`, `x^{1}{}^{2}`}, + // One of each is what a big operator has, and is left alone. + {`x_{1}^{2}`, `x_{1}^{2}`}, + // A new letter carries its own scripts. + {`x_{1}y_{2}`, `x_{1}y_{2}`}, + // A group that closes is something a script can attach to. + {`{ab}^{2}`, `{ab}^{2}`}, + // Braces left open are closed, and braces closed too often dropped. + {`\frac{a}{b`, `\frac{a}{b}`}, + {`a}b`, `ab`}, + // A command is something a script can attach to. + {`\alpha^{2}`, `\alpha^{2}`}, + // A \left with no \right loses the growing and keeps the delimiter. + {`\left(x`, `(x`}, + {`\left(x\right)`, `\left(x\right)`}, + // A control symbol is one character long. + {`\{x^{2}`, `\{x^{2}`}, + // A space is not something a script can attach to. + {` ^{2}`, ` {}^{2}`}, + // A backslash at the very end is left as it is. + {`x\`, `x\`}, + {"", ""}, + } + for _, c := range cases { + if got := repair(c[0]); got != c[1] { + t.Errorf("repair(%q) = %q, want %q", c[0], got, c[1]) + } + } +} + +func TestARightArrowIsNotAnUnmatchedRight(t *testing.T) { + // \rightarrow starts with the six characters of \right. Stripping an + // unmatched \left by working on the text rather than on the tokens turns + // every limit in the document into \thetaarrow, which is not a command. + if got := repair(`\left(\theta\rightarrow0`); got != `(\theta\rightarrow0` { + t.Errorf("got %q", got) + } + if got := repair(`\theta\rightarrow0`); got != `\theta\rightarrow0` { + t.Errorf("got %q", got) + } +} + +func TestAPrimeIsASuperscript(t *testing.T) { + // TeX reads x' as x^{\prime}, so a subscript after one lands on a letter + // that already has a superscript. + if got := repair(`q_{a}'_{b}`); got != `q_{a}'{}_{b}` { + t.Errorf("got %q", got) + } + if got := repair(`q'`); got != `q'` { + t.Errorf("got %q", got) + } +} + +func TestDelimitersThatDoNotNest(t *testing.T) { + // One of each, but the \right is in a group of its own: TeX reads each + // group on its own and refuses it. + if got := repair(`\frac{\left(a}{b\right)}`); got != `\frac{(a}{b)}` { + t.Errorf("got %q", got) + } + // A \right before its \left. + if got := repair(`a\right)b\left(`); got != `a)b(` { + t.Errorf("got %q", got) + } + // A brace that closes with nothing open is dropped before the pairing is + // judged, so the pairing sees what TeX will see. + if got := repair(`\left(a\right)`); got != `\left(a\right)` { + t.Errorf("got %q", got) + } + if got := repair(`{\left(a\right)}`); got != `{\left(a\right)}` { + t.Errorf("got %q", got) + } + if got := repair(`}\left(a\right)`); got != `(a)` { + t.Errorf("got %q", got) + } +} diff --git a/rule.go b/rule.go new file mode 100644 index 0000000..baf325b --- /dev/null +++ b/rule.go @@ -0,0 +1,245 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "github.com/go-pdfkit/reader" + +// This file finds the straight lines on a page, because two of the things this +// package has to reconstruct are not text at all. +// +// A fraction is a rule with its numerator above and its denominator below: TeX +// draws \frac{a+b}{c} as five glyphs and one thin horizontal bar, and without +// the bar there is nothing to say the a+b and the c belong together rather than +// being two lines of an equation. A radical is the same — a hook glyph followed +// by a bar over the radicand, and the bar is what says how far the radicand +// reaches. Ruled tables are made of the same material. +// +// So this walks the content stream a second time, ignoring the text and keeping +// only the marks. Two producers draw a rule two different ways and both have to +// be read: pdfTeX emits a filled rectangle (re … f), while xdvipdfmx — what +// XeTeX and tectonic write through — emits a stroked segment (m … l … S) whose +// thickness is the line width. A reader that knows only one of them silently +// loses every fraction in half the world's PDFs. + +// A rule is a straight mark on the page, given as the box it covers in the same +// coordinates the text comes back in: up from the bottom left of what is +// visible, in points. +type rule struct { + x0, y0, x1, y1 float64 +} + +// wide reports whether the mark is longer across than it is tall, which is what +// separates a fraction bar or a table's horizontal line from a column divider. +func (r rule) wide() bool { return r.x1-r.x0 >= r.y1-r.y0 } + +// midY is the height of the mark's centre. +func (r rule) midY() float64 { return (r.y0 + r.y1) / 2 } + +// midX is the horizontal centre of the mark. +func (r rule) midX() float64 { return (r.x0 + r.x1) / 2 } + +// spans reports whether x falls within the mark's horizontal reach, with a +// little slack at each end: a fraction bar is drawn a shade wider than the +// material it covers, but not always wider than a glyph's own side bearings. +func (r rule) spans(x, slack float64) bool { + return x >= r.x0-slack && x <= r.x1+slack +} + +// defaultLineWidth is the thickness of a stroke drawn with a width of zero, +// which PDF defines as the thinnest line the device can manage. A fraction bar +// is never drawn that way, but a hairline table rule is, and giving it no +// thickness at all would make it a mark with no height. +const defaultLineWidth = 0.4 + +// pathState is the part of the graphics state that says where a mark lands. +type pathState struct { + ctm matrix + width float64 +} + +// A pathScan walks a content stream keeping the marks and dropping the text. +type pathScan struct { + g pathState + stack []pathState + // rects are the rectangles the current path has collected, in page + // coordinates, waiting to learn whether the path will be filled. + rects []rule + // segs are the straight straight pieces of the current path, waiting to learn + // whether it will be stroked. + segs []stroke + // curved records that the path contains something this cannot reduce to + // a straight mark, so the whole path is dropped rather than half kept. + curved bool + // cur is where the current subpath is, for a lineto to draw from. + cur point + out []rule +} + +// A point is a place on the page. +type point struct{ x, y float64 } + +// A stroke is a straight piece of a path, in page coordinates. +type stroke struct{ a, b point } + +// rules is every straight mark on the i'th page, counting from one. Marks drawn +// inside a form XObject are not reached: a form has its own content stream and +// its own transform, and following them is the renderer's work rather than +// this package's. A fraction TeX put in a form — which it does not do — would +// be missed. +func rules(d *reader.Document, i int, origin point) []rule { + ops, err := d.PageOperations(i) + if err != nil { + return nil + } + s := &pathScan{g: pathState{ + ctm: matrix{1, 0, 0, 1, -origin.x, -origin.y}, + width: 1, + }} + for _, op := range ops { + s.step(op) + } + return s.out +} + +// step reads one operation. +func (s *pathScan) step(op reader.Operation) { + n := numbers(op.Operands) + switch op.Operator { + case "q": + s.stack = append(s.stack, s.g) + case "Q": + if len(s.stack) > 0 { + s.g = s.stack[len(s.stack)-1] + s.stack = s.stack[:len(s.stack)-1] + } + case "cm": + if len(n) >= 6 { + s.g.ctm = matrix{n[0], n[1], n[2], n[3], n[4], n[5]}.mul(s.g.ctm) + } + case "w": + if len(n) >= 1 { + s.g.width = n[0] + } + case "gs": + // An external graphics state can carry a line width, but reading it + // means resolving a resource; a rule whose width comes from one is + // rare enough to leave at the default rather than guess. + case "m": + if len(n) >= 2 { + s.cur = s.place(n[0], n[1]) + } + case "l": + if len(n) >= 2 { + p := s.place(n[0], n[1]) + s.segs = append(s.segs, stroke{s.cur, p}) + s.cur = p + } + case "c", "v", "y": + s.curved = true + case "re": + if len(n) >= 4 { + s.rect(n[0], n[1], n[2], n[3]) + } + case "f", "F", "f*", "B", "B*", "b", "b*": + s.paint(true, op.Operator != "f" && op.Operator != "F" && op.Operator != "f*") + case "S", "s": + s.paint(false, true) + case "n": + s.reset() + } +} + +// place moves a point into page coordinates. +func (s *pathScan) place(x, y float64) point { + px, py := s.g.ctm.apply(x, y) + return point{px, py} +} + +// rect records a rectangle, put the right way round. +func (s *pathScan) rect(x, y, w, h float64) { + a := s.place(x, y) + b := s.place(x+w, y+h) + if a.x > b.x { + a.x, b.x = b.x, a.x + } + if a.y > b.y { + a.y, b.y = b.y, a.y + } + s.rects = append(s.rects, rule{a.x, a.y, b.x, b.y}) +} + +// paint ends a path, keeping what it drew. A filled rectangle is a mark as it +// stands; a stroked segment becomes one by giving it the line's thickness. +func (s *pathScan) paint(fill, stroke bool) { + if s.curved { + s.reset() + return + } + // A rectangle is one mark whether it is filled, stroked, or both: an + // operator that does both must not put it in twice. + if fill || stroke { + s.out = append(s.out, s.rects...) + } + if stroke { + half := s.strokeHalfWidth() + for _, g := range s.segs { + if r, ok := g.mark(half); ok { + s.out = append(s.out, r) + } + } + } + s.reset() +} + +// strokeHalfWidth is half the thickness a stroke gets on the page. +func (s *pathScan) strokeHalfWidth() float64 { + w := s.g.width + if w <= 0 { + w = defaultLineWidth + } + return w * s.g.ctm.scale() / 2 +} + +// mark turns a straight segment into the box it covers, and reports false for +// one that runs at an angle: a rule is horizontal or vertical, and a diagonal +// line is part of a drawing. +func (g stroke) mark(half float64) (rule, bool) { + dx, dy := abs(g.b.x-g.a.x), abs(g.b.y-g.a.y) + switch { + case dy <= half && dx > 0: + y := (g.a.y + g.b.y) / 2 + return rule{min(g.a.x, g.b.x), y - half, max(g.a.x, g.b.x), y + half}, true + case dx <= half && dy > 0: + x := (g.a.x + g.b.x) / 2 + return rule{x - half, min(g.a.y, g.b.y), x + half, max(g.a.y, g.b.y)}, true + } + return rule{}, false +} + +// reset clears the current path. +func (s *pathScan) reset() { + s.rects, s.segs, s.curved = nil, nil, false +} + +// numbers reads the numeric operands of an operation, stopping at the first +// that is not a number since an operator's numbers come first. +func numbers(ops []reader.Object) []float64 { + out := make([]float64, 0, len(ops)) + for _, o := range ops { + v, ok := reader.ToFloat(o) + if !ok { + break + } + out = append(out, v) + } + return out +} + +// abs is the size of a number without its sign. +func abs(v float64) float64 { + if v < 0 { + return -v + } + return v +} diff --git a/rule_test.go b/rule_test.go new file mode 100644 index 0000000..ee755ed --- /dev/null +++ b/rule_test.go @@ -0,0 +1,130 @@ +package latex + +import ( + "testing" + + "github.com/go-pdfkit/reader" +) + +// marks is what a content stream draws, read back as straight marks. +func marks(t *testing.T, content string) []rule { + t.Helper() + d := pageWith(t, content, nil) + return rules(d, 1, point{0, 0}) +} + +func TestAFilledRectangleIsAMark(t *testing.T) { + got := marks(t, "10 20 30 2 re f") + if len(got) != 1 || got[0] != (rule{10, 20, 40, 22}) { + t.Fatalf("got %+v", got) + } + if !got[0].wide() || got[0].midY() != 21 || got[0].midX() != 25 { + t.Errorf("got %+v", got[0]) + } + if !got[0].spans(9.5, 1) || got[0].spans(5, 1) { + t.Error("spans is wrong") + } + // A rectangle written the other way round is the same mark. + if got := marks(t, "40 22 -30 -2 re f"); len(got) != 1 || got[0] != (rule{10, 20, 40, 22}) { + t.Errorf("got %+v", got) + } + // A tall rectangle is a column divider rather than a rule across. + if got := marks(t, "10 20 2 30 re f"); len(got) != 1 || got[0].wide() { + t.Errorf("got %+v", got) + } +} + +func TestAStrokedLineIsAMark(t *testing.T) { + // xdvipdfmx, which is what XeTeX writes through, draws a rule as a + // stroked segment whose thickness is the line width. + got := marks(t, "1 w 10 20 m 40 20 l S") + if len(got) != 1 || got[0] != (rule{10, 19.5, 40, 20.5}) { + t.Fatalf("got %+v", got) + } + // A vertical one. + if got := marks(t, "2 w 10 20 m 10 50 l S"); len(got) != 1 || got[0].wide() { + t.Errorf("got %+v", got) + } + // A width of nought is the thinnest the device can draw, not nothing. + got = marks(t, "0 w 10 20 m 40 20 l S") + if len(got) != 1 || abs(got[0].y1-got[0].y0-defaultLineWidth) > 1e-9 { + t.Errorf("got %+v", got) + } + // A line at an angle is part of a drawing, not a rule. + if got := marks(t, "1 w 10 20 m 40 60 l S"); len(got) != 0 { + t.Errorf("got %+v", got) + } + // A stroked rectangle is kept as it stands. + if got := marks(t, "10 20 30 2 re S"); len(got) != 1 { + t.Errorf("got %+v", got) + } + // A path that is only closed and stroked draws its segments too. + if got := marks(t, "10 20 m 40 20 l s"); len(got) != 1 { + t.Errorf("got %+v", got) + } +} + +func TestTheTransformMovesAMark(t *testing.T) { + got := marks(t, "q 2 0 0 2 5 5 cm 10 20 30 2 re f Q 0 0 1 1 re f") + if len(got) != 2 || got[0] != (rule{25, 45, 85, 49}) { + t.Fatalf("got %+v", got) + } + // The transform was put back by Q, so the second mark is where it says. + if got[1] != (rule{0, 0, 1, 1}) { + t.Errorf("got %+v", got[1]) + } + // A Q with nothing pushed is ignored rather than fatal. + if got := marks(t, "Q 0 0 1 1 re f"); len(got) != 1 { + t.Errorf("got %+v", got) + } + // The page's own origin is taken off. + d := pageWith(t, "10 20 30 2 re f", nil) + if got := rules(d, 1, point{5, 5}); got[0] != (rule{5, 15, 35, 17}) { + t.Errorf("got %+v", got) + } +} + +func TestAPathThatIsNotStraightIsDropped(t *testing.T) { + for _, c := range []string{ + "10 20 m 15 25 20 30 25 20 c f", + "10 20 m 15 25 25 20 v f", + "10 20 m 15 25 25 20 y f", + } { + if got := marks(t, c); len(got) != 0 { + t.Errorf("%s gave %+v", c, got) + } + } + // A path used only for clipping paints nothing. + if got := marks(t, "10 20 30 2 re W n"); len(got) != 0 { + t.Errorf("got %+v", got) + } +} + +func TestEveryWayOfPainting(t *testing.T) { + for _, op := range []string{"f", "F", "f*", "B", "B*", "b", "b*"} { + if got := marks(t, "10 20 30 2 re "+op); len(got) != 1 { + t.Errorf("%s painted %d marks", op, len(got)) + } + } + // An operator with too few operands does nothing rather than panicking. + for _, c := range []string{"1 2 3 re f", "1 cm 0 0 1 1 re f", "w 0 0 1 1 re f", + "1 m 0 0 1 1 re f", "1 l 0 0 1 1 re f", "/G gs 0 0 1 1 re f"} { + marks(t, c) + } +} + +func TestReadingTheNumbersOfAnOperation(t *testing.T) { + // The numbers of an operation come first; reading stops at the first + // operand that is not one. + got := numbers([]reader.Object{reader.Integer(1), reader.Real(2.5), reader.Name("x"), reader.Integer(3)}) + if len(got) != 2 || got[0] != 1 || got[1] != 2.5 { + t.Errorf("got %v", got) + } +} + +func TestAPageWithNoContentHasNoMarks(t *testing.T) { + d := pageWith(t, "10 20 30 2 re f", nil) + if got := rules(d, 2, point{0, 0}); got != nil { + t.Errorf("page two of a one-page document has marks: %+v", got) + } +} diff --git a/text.go b/text.go new file mode 100644 index 0000000..147d199 --- /dev/null +++ b/text.go @@ -0,0 +1,90 @@ +// Copyright (c) the go-pdfkit/latex authors. +// SPDX-License-Identifier: BSD-3-Clause + +package latex + +import "strings" + +// This file turns text back into source: the characters a document drew, put +// back into the form that would draw them again, and the font changes along a +// line put back into the commands that caused them. +// +// The escaping is the ordinary LaTeX list. The interesting part is the other +// direction — a run of characters set in CMBX10 was written \textbf{…}, and to +// say so the runs of one style have to be found and wrapped. That is done at +// the line level rather than the word level, because a style that changes for +// one word and back is one \textbf and not three. + +// specials are the characters LaTeX reads as instructions, and what has to be +// written to get the character itself. +var specials = map[rune]string{ + '#': `\#`, '$': `\$`, '%': `\%`, '&': `\&`, '_': `\_`, + '{': `\{`, '}': `\}`, + '~': `\textasciitilde{}`, '^': `\textasciicircum{}`, + '\\': `\textbackslash{}`, + // The characters a text font draws for a ligature or a dash, put back + // into what an author types for them. + 'ff': `ff`, 'fi': `fi`, 'fl': `fl`, 'ffi': `ffi`, 'ffl': `ffl`, + '–': `--`, '—': `---`, '‘': "`", '’': `'`, + '“': "``", '”': `''`, '−': `-`, + '…': `\ldots{}`, '¡': `!`, '¿': `?`, + '°': `\textdegree{}`, '©': `\textcopyright{}`, '§': `\S{}`, + '¶': `\P{}`, '†': `\dag{}`, '‡': `\ddag{}`, '•': `\textbullet{}`, + '£': `\pounds{}`, '€': `\texteuro{}`, '×': `\texttimes{}`, +} + +// escapeText writes text so that LaTeX draws it rather than obeying it. +// +// A character that only mathematics has a name for is written as +// mathematics. This happens more often than it should: a symbol the +// reconstruction did not manage to bring inside an equation still has to +// come out as something that compiles, and \ensuremath{\partial} typesets +// where a bare U+2202 does not - not in a document whose preamble this +// package wrote, which does not load an input encoding for it. +func escapeText(s string) string { + var b strings.Builder + for _, r := range s { + if rep, ok := specials[r]; ok { + b.WriteString(rep) + continue + } + if r > 0x7F { + if cmd, ok := mathCommand[r]; ok { + b.WriteString(`\ensuremath{` + cmd + `}`) + continue + } + if decoration(r) { + continue + } + } + b.WriteRune(r) + } + return b.String() +} + +// wrappers are the commands that put a style back, in the order they are +// applied when a piece of text has more than one. A face that is both bold and +// italic becomes \textbf{\emph{…}}. +type wrapper struct { + on func(shape) bool + name string +} + +var wrappers = []wrapper{ + {func(s shape) bool { return s.mono }, `\texttt`}, + {func(s shape) bool { return s.sans }, `\textsf`}, + {func(s shape) bool { return s.smallCaps }, `\textsc`}, + {func(s shape) bool { return s.bold }, `\textbf`}, + {func(s shape) bool { return s.italic }, `\emph`}, +} + +// markup wraps text in whatever commands its font calls for. Plain roman gets +// nothing, which is most of a document. +func markup(s shape, text string) string { + for i := len(wrappers) - 1; i >= 0; i-- { + if wrappers[i].on(s) { + text = wrappers[i].name + `{` + text + `}` + } + } + return text +} diff --git a/text_test.go b/text_test.go new file mode 100644 index 0000000..9e4d135 --- /dev/null +++ b/text_test.go @@ -0,0 +1,53 @@ +package latex + +import "testing" + +func TestWritingTextBackAsSource(t *testing.T) { + cases := [][2]string{ + {"plain", "plain"}, + {"100%", `100\%`}, + {"a_b", `a\_b`}, + {"{x}", `\{x\}`}, + {"#$&", `\#\$\&`}, + {"~^", `\textasciitilde{}\textasciicircum{}`}, + {`\`, `\textbackslash{}`}, + {"fine", "fine"}, + {"ffflffiffl", "ffflffiffl"}, + {"– —", `-- ---`}, + {"‘a’", "`a'"}, + {"“a”", "``a''"}, + // A symbol only mathematics has a name for is written as mathematics + // so that it compiles at all. + {"∂", `\ensuremath{\partial}`}, + // A free-standing accent is dropped rather than written out. + {"Lˆ", "L"}, + // A character with no special meaning is left alone. + {"café", "café"}, + } + for _, c := range cases { + if got := escapeText(c[0]); got != c[1] { + t.Errorf("escapeText(%q) = %q, want %q", c[0], got, c[1]) + } + } +} + +func TestPuttingTheMarkupBack(t *testing.T) { + cases := []struct { + s shape + want string + }{ + {shape{}, "x"}, + {shape{bold: true}, `\textbf{x}`}, + {shape{italic: true}, `\emph{x}`}, + {shape{mono: true}, `\texttt{x}`}, + {shape{sans: true}, `\textsf{x}`}, + {shape{smallCaps: true}, `\textsc{x}`}, + {shape{bold: true, italic: true}, `\textbf{\emph{x}}`}, + {shape{sans: true, bold: true}, `\textsf{\textbf{x}}`}, + } + for _, c := range cases { + if got := markup(c.s, "x"); got != c.want { + t.Errorf("markup(%+v) = %q, want %q", c.s, got, c.want) + } + } +} diff --git a/unit_test.go b/unit_test.go new file mode 100644 index 0000000..026542c --- /dev/null +++ b/unit_test.go @@ -0,0 +1,586 @@ +package latex + +import ( + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// at is one piece of text for the tests that work below the level of a page. +func at(text string, x, y, w, size float64, s shape) atom { + return atom{text: text, x: x, y: y, width: w, size: size, space: size / 3, sh: s} +} + +func TestHowWideAWordSpaceIs(t *testing.T) { + // What the font says, when what it says is believable. + a := atom{size: 10, space: 3} + if a.wordSpace() != 3 { + t.Errorf("got %v", a.wordSpace()) + } + // A composite font has no character at 32 and answers with whatever its + // fallback width is; one real producer answers with the font size, and + // taking that at face value turns a page into one long word. + a.space = 10 + if abs(a.wordSpace()-3.3) > 1e-9 { + t.Errorf("got %v", a.wordSpace()) + } + a.space = 0.5 + if abs(a.wordSpace()-3.3) > 1e-9 { + t.Errorf("got %v", a.wordSpace()) + } + // A math font has no word spaces at all. + a = atom{size: 10, space: 3, sh: shape{math: mathLetter}} + if a.wordSpace() != 0 { + t.Errorf("got %v", a.wordSpace()) + } +} + +func TestReadingAPageThatIsNotThere(t *testing.T) { + d := pageWith(t, "", nil) + if _, err := readPage(d, 4); err == nil { + t.Error("page four of a one-page document read without error") + } + if _, err := Reconstruct(d, Options{First: 4, Last: 4}); err == nil { + t.Error("reconstructing a page that is not there succeeded") + } + if _, err := Source(d); err != nil { + t.Error(err) + } +} + +func TestAPageBoxThatCannotBeRead(t *testing.T) { + d := pageWith(t, "", nil) + // Not an array, too short, not numbers: each falls back to the default. + for _, o := range []reader.Object{ + reader.Name("x"), + reader.Array{reader.Integer(0)}, + reader.Array{reader.Integer(0), reader.Integer(0), reader.Name("x"), reader.Integer(1)}, + } { + if _, ok := rectangle(d, o); ok { + t.Errorf("%v read as a rectangle", o) + } + } + // A box written back to front is put the right way round. + got, ok := rectangle(d, reader.Array{reader.Integer(9), reader.Integer(9), + reader.Integer(1), reader.Integer(1)}) + if !ok || got != [4]float64{1, 1, 9, 9} { + t.Errorf("got %v %v", got, ok) + } + // A page with no box at all is taken to be letter paper. + if got := visibleBox(d, reader.Dict{}); got != [4]float64{0, 0, 612, 792} { + t.Errorf("got %v", got) + } +} + +func TestAFontWithNoName(t *testing.T) { + d := pageWith(t, "", nil) + if got := fontName(d, reader.Dict{}); got != "" { + t.Errorf("got %q", got) + } + if got := fontName(d, reader.Dict{"BaseFont": reader.Integer(3)}); got != "" { + t.Errorf("got %q", got) + } +} + +func TestFontsInsideAForm(t *testing.T) { + // A form XObject carries resources of its own, and the fonts it names + // have to be read too or the text it draws comes back unclassified. + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + inner := w.Add(reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"Font": reader.Dict{ + "G": w.Add(fontDict(w, face{base: "CMBX10"})), + // A font entry that is not a dictionary is skipped. + "H": reader.Integer(7), + }}, + }) + page := w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), reader.Integer(200), reader.Integer(200)}, + "Contents": w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte("")}), + "Resources": reader.Dict{ + "Font": reader.Dict{"F": w.Add(fontDict(w, face{base: "CMR10"}))}, + "XObject": reader.Dict{"X": inner, "Y": reader.Integer(9)}, + }, + }) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + dict, err := d.Page(1) + if err != nil { + t.Fatal(err) + } + got := fontShapes(d, dict) + if !got["G"].bold || got["F"].bold { + t.Errorf("got %+v", got) + } + // A dictionary with no resources contributes nothing, and neither does + // one nested deeper than a page ever nests. + empty := map[string]shape{} + collectFonts(d, reader.Dict{}, empty, 0) + collectFonts(d, dict, empty, maxResourceDepth+1) + if len(empty) != 0 { + t.Errorf("got %+v", empty) + } +} + +func TestWhichBandsALineWillTake(t *testing.T) { + body := line{y: 100, size: 10, x0: 20, x1: 60} + // A band far to the side belongs to another column, not to this line. + far := measured([]atom{at("x", 300, 104, 4, 7, shape{})}) + if accepts(body, far, nil) { + t.Error("a band in the next column was taken") + } + // A band larger than the line is never offered to it. + big := measured([]atom{at("x", 25, 104, 4, 20, shape{})}) + if accepts(body, big, nil) { + t.Error("a larger band was taken") + } + // A band the same size, level with the line, and nowhere near it + // vertically, is the next line. + next := measured([]atom{at("x", 25, 80, 4, 10, shape{})}) + if accepts(body, next, nil) { + t.Error("the next line was taken") + } + // A symbol raised above the line belongs to it; the same symbol dropped + // below it belongs to the line beneath. + up := measured([]atom{at("√", 25, 106, 4, 10, shape{math: mathSymbol})}) + down := measured([]atom{at("√", 25, 94, 4, 10, shape{math: mathSymbol})}) + if !accepts(body, up, nil) || accepts(body, down, nil) { + t.Error("a raised symbol was read wrongly") + } +} + +func TestLimitsOfABigOperator(t *testing.T) { + op := at("∑", 20, 100, 10, 10, shape{math: mathExt}) + l := line{atoms: []atom{op, at("f", 34, 100, 5, 10, shape{math: mathLetter})}, + y: 100, size: 10, x0: 20, x1: 39} + // A limit sits over the operator and may be further off than a script. + limit := measured([]atom{at("n", 21, 111, 4, 7, shape{math: mathLetter})}) + if !accepts(l, limit, nil) { + t.Error("the limit was not taken") + } + // A script the same distance away but not over an operator is not. + plain := line{atoms: []atom{at("f", 34, 100, 5, 10, shape{})}, y: 100, size: 10, x0: 34, x1: 39} + if accepts(plain, measured([]atom{at("n", 34, 111, 4, 7, shape{})}), nil) { + t.Error("a band eleven points away was taken as a script") + } + if overOperator(plain, limit) { + t.Error("a line with no big operator has something over one") + } +} + +func TestWhenARuleBindsTwoBands(t *testing.T) { + l := line{atoms: []atom{at("d", 20, 100, 5, 10, shape{math: mathLetter})}, + y: 100, size: 10, x0: 20, x1: 25} + below := measured([]atom{at("dt", 20, 92, 9, 10, shape{math: mathLetter})}) + frac := rule{19, 97.5, 29, 98} + if !accepts(l, below, []rule{frac}) { + t.Error("the denominator was not taken") + } + // A rule far too wide for what sits under it is a table's line. + if accepts(l, below, []rule{{19, 97.5, 300, 98}}) { + t.Error("a table rule was taken as a fraction bar") + } + // A band that reaches outside the rule is not one of its halves. + wideBand := measured([]atom{at("dt", 20, 92, 40, 10, shape{math: mathLetter})}) + if accepts(l, wideBand, []rule{frac}) { + t.Error("a band wider than the bar was taken") + } + // A band too far below to be part of the same fraction. + away := measured([]atom{at("dt", 20, 60, 9, 10, shape{math: mathLetter})}) + if accepts(l, away, []rule{frac}) { + t.Error("a band thirty points away was taken") + } + // An upright rule is not a fraction bar. + if accepts(l, below, []rule{{22, 90, 22.5, 110}}) { + t.Error("a column divider was taken as a fraction bar") + } + // A rule outside the two baselines is not between them. + if accepts(l, below, []rule{{19, 130, 29, 130.5}}) { + t.Error("a rule above both was taken as a bar between them") + } +} + +func TestPageNumbersAndRunningHeads(t *testing.T) { + mk := func(y, x1 float64) line { + return line{y: y, size: 10, x0: 20, x1: x1, left: 20, right: 200} + } + lines := []line{mk(190, 30), mk(160, 200), mk(148, 200), mk(20, 30)} + got := stripRunningHeads(lines, 10) + if len(got) != 2 || got[0].y != 160 { + t.Errorf("got %+v", got) + } + // A heading at the top of a page is short and set off from what follows + // it too, and must not be thrown away with the running heads. + head := line{y: 190, size: 14, x0: 20, x1: 60, left: 20, right: 200} + if got := stripRunningHeads([]line{head, mk(160, 200)}, 10); len(got) != 2 { + t.Errorf("the heading was stripped: %+v", got) + } + // A single line is left alone. + if got := stripRunningHeads(lines[:1], 10); len(got) != 1 { + t.Errorf("got %+v", got) + } +} + +func TestFindingTheGutter(t *testing.T) { + if _, ok := gutter(nil, 100); ok { + t.Error("a page with nothing on it has a gutter") + } + if _, ok := gutter([]atom{at("x", 0, 0, 5, 10, shape{})}, 0); ok { + t.Error("a page with no width has a gutter") + } + // Text on one side only is one column. + if _, ok := gutter([]atom{at("x", 0, 0, 5, 10, shape{})}, 100); ok { + t.Error("text on the left alone made a gutter") + } + if _, ok := gutter([]atom{at("x", 90, 0, 5, 10, shape{})}, 100); ok { + t.Error("text on the right alone made a gutter") + } + // Text on both sides with nothing between them is two columns. + both := []atom{at("x", 0, 0, 40, 10, shape{}), at("y", 60, 0, 40, 10, shape{})} + x, ok := gutter(both, 100) + if !ok || x < 40 || x > 60 { + t.Errorf("got %v %v", x, ok) + } + // Text right across the middle is one column. + if _, ok := gutter([]atom{at("x", 0, 0, 100, 10, shape{})}, 100); ok { + t.Error("a full-width line made a gutter") + } +} + +func TestSplittingABandAtTheGutter(t *testing.T) { + one := measured([]atom{at("x", 0, 0, 5, 10, shape{})}) + if got := split([]band{one}, 50); len(got) != 1 { + t.Errorf("a band on one side was split: %+v", got) + } +} + +func TestWritingAnEquationsPieces(t *testing.T) { + // A control word runs into a letter, so a space goes between them; it + // does not run into a brace or a digit. + if got := join([]string{`\alpha`, "x"}); got != `\alpha x` { + t.Errorf("got %q", got) + } + if got := join([]string{`\alpha`, "1"}); got != `\alpha1` { + t.Errorf("got %q", got) + } + if got := join([]string{"x", "y"}); got != "xy" { + t.Errorf("got %q", got) + } + if got := join([]string{"", "x", ""}); got != "x" { + t.Errorf("got %q", got) + } + if needsGap("", "x") || needsGap("x", "") { + t.Error("an empty piece needs a gap") + } + if endsInControlWord("abc") || !endsInControlWord(`\alpha`) { + t.Error("endsInControlWord is wrong") + } +} + +func TestWhichCharactersAnEquationWritesPlainly(t *testing.T) { + for _, r := range "0123456789+=<>()[]/!:;,.|'-*?@" { + if !mathPlain(r) { + t.Errorf("%q is not plain", r) + } + } + for _, r := range "$%#&" { + if mathPlain(r) { + t.Errorf("%q is plain", r) + } + } +} + +func TestWhatCountsAsMathematics(t *testing.T) { + if !mathLike(at("2", 0, 0, 5, 10, shape{})) { + t.Error("a digit is not math-like") + } + if !mathLike(at("x", 0, 0, 5, 10, shape{})) { + t.Error("a single letter is not math-like") + } + if !mathLike(at("sin", 0, 0, 5, 10, shape{})) { + t.Error("an operator name is not math-like") + } + if mathLike(at("and", 0, 0, 5, 10, shape{})) { + t.Error("a word is math-like") + } + if mathLike(at("and?x", 0, 0, 5, 10, shape{})) { + t.Error("a word with punctuation in it is math-like") + } + if !mathLike(at("+ =", 0, 0, 5, 10, shape{})) { + t.Error("arithmetic with a space in it is not math-like") + } +} + +func TestASingleItalicLetterAmongRoman(t *testing.T) { + rom, it := shape{}, shape{italic: true} + atoms := []atom{ + at("(", 0, 0, 5, 10, rom), + at("L", 5, 0, 5, 10, it), + at(",", 10, 0, 5, 10, rom), + at("wide", 15, 0, 20, 10, it), + at("x", 35, 0, 5, 10, it), + } + if !italicVariable(atoms, 1) { + t.Error("a lone italic letter is not a variable") + } + if italicVariable(atoms, 0) { + t.Error("a roman character is a variable") + } + // A word is emphasis, not a variable. + if italicVariable(atoms, 3) { + t.Error("an italic word is a variable") + } + // A letter with italic beside it is part of italic text. + if italicVariable(atoms, 4) { + t.Error("a letter next to italic text is a variable") + } + // A digit is not a variable, and neither is anything set in a math font + // (which is a seed already). + more := []atom{at("7", 0, 0, 5, 10, it), at("x", 0, 0, 5, 10, shape{math: mathLetter})} + if italicVariable(more, 0) || italicVariable(more, 1) { + t.Error("got a variable") + } +} + +func TestSplittingASectionNumberOff(t *testing.T) { + for _, c := range []struct { + in string + parts int + rest string + }{ + {"1 One", 1, "One"}, + {"1.2 Two", 2, "Two"}, + {"1.2.3 Three", 3, "Three"}, + {"A.1 Appendix", 2, "Appendix"}, + {"Introduction", 0, "Introduction"}, + {"1.", 0, "1."}, + {"1..2 Two", 0, "1..2 Two"}, + {"", 0, ""}, + } { + parts, rest := splitNumber(c.in) + if len(parts) != c.parts || rest != c.rest { + t.Errorf("%q gave %v %q", c.in, parts, rest) + } + } +} + +func TestWhichCommandAHeadingLevelUses(t *testing.T) { + if sectionName(1) != `\section` || sectionName(4) != `\paragraph` || sectionName(9) != `\paragraph` { + t.Error("sectionName is wrong") + } + e := &emitter{levels: []float64{14, 12}} + if e.rank(14) != 1 || e.rank(12) != 2 || e.rank(9) != 2 { + t.Errorf("rank gave %d %d %d", e.rank(14), e.rank(12), e.rank(9)) + } +} + +func TestTakingAnEquationNumberOff(t *testing.T) { + mathSeg := segment{atoms: []atom{at("x", 0, 0, 5, 10, shape{math: mathLetter})}, math: true} + num := segment{atoms: []atom{at("(1)", 90, 0, 15, 10, shape{})}} + if got := trimNumber([]segment{mathSeg, num}); len(got) != 1 { + t.Error("the number was kept") + } + if got := trimNumber([]segment{mathSeg}); len(got) != 1 { + t.Error("a lone equation lost something") + } + if got := trimNumber([]segment{mathSeg, mathSeg}); len(got) != 2 { + t.Error("an equation was taken for a number") + } + two := segment{atoms: []atom{at("(", 0, 0, 5, 10, shape{}), at("1)", 5, 0, 5, 10, shape{})}} + if got := trimNumber([]segment{mathSeg, two}); len(got) != 2 { + t.Error("a two-piece tail was taken for a number") + } + word := segment{atoms: []atom{at("(x", 90, 0, 15, 10, shape{})}} + if got := trimNumber([]segment{mathSeg, word}); len(got) != 2 { + t.Error("a bracket with no closing one was taken for a number") + } + if got := trimNumber([]segment{mathSeg, {atoms: []atom{at("()", 0, 0, 5, 10, shape{})}}}); len(got) != 2 { + t.Error("an empty pair of brackets was taken for a number") + } +} + +func TestTheSmallJudgementsOfALine(t *testing.T) { + if allBold(line{}) { + t.Error("a line with nothing on it is bold") + } + if allMath(nil) { + t.Error("a line with nothing on it is mathematics") + } + if centred(line{left: 10, right: 10}) { + t.Error("a column with no width has a centre") + } + if startsLower("") || startsLower("A") || !startsLower("a") { + t.Error("startsLower is wrong") + } + if brokenWord("a--") || !brokenWord("a-") { + t.Error("brokenWord is wrong") + } + // Two pieces with no idea how wide a space is have no space between + // them: both are mathematics. + m := shape{math: mathLetter} + if gapBetween(segment{atoms: []atom{at("x", 0, 0, 5, 10, m)}}, + segment{atoms: []atom{at("y", 50, 0, 5, 10, m)}}) { + t.Error("two math pieces were given a word space") + } +} + +func TestAnEmptyParagraphIsNotWritten(t *testing.T) { + e := &emitter{} + e.paragraph(" ") + e.flush() + if e.out.String() != "" { + t.Errorf("got %q", e.out.String()) + } +} + +func TestALineWithNothingOnIt(t *testing.T) { + e := &emitter{body: 10} + e.line(line{}, nil) + if e.out.String() != "" || len(e.para) != 0 { + t.Error("an empty line wrote something") + } + if segments(line{}) != nil { + t.Error("an empty line has segments") + } +} + +// cyclic builds a document with an indirect reference that points at itself +// through another, which is the one thing that makes resolving one fail. +func cyclic(t *testing.T) (*reader.Document, reader.Ref) { + t.Helper() + w := reader.NewWriter("1.7") + a, b := w.Reserve(), w.Reserve() + w.Put(a, b) + w.Put(b, a) + pagesRef := w.Reserve() + page := w.Add(reader.Dict{"Type": reader.Name("Page"), "Parent": pagesRef, "MediaBox": a}) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d, a +} + +func TestAReferenceThatCannotBeFollowed(t *testing.T) { + d, ref := cyclic(t) + if _, ok := rectangle(d, ref); ok { + t.Error("a reference chain that never ends read as a rectangle") + } + if _, ok := rectangle(d, reader.Array{ref, reader.Integer(0), reader.Integer(1), reader.Integer(1)}); ok { + t.Error("a rectangle with an endless element read") + } + if got := fontName(d, reader.Dict{"BaseFont": ref}); got != "" { + t.Errorf("got %q", got) + } + // The page falls back to letter paper, and reconstructs as nothing. + doc, err := Reconstruct(d, Options{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(doc.Preamble, "paperwidth=612.0pt") { + t.Errorf("got %s", doc.Preamble) + } +} + +func TestSourceOfADocumentWithNoSuchPage(t *testing.T) { + d := pageWith(t, "", nil) + if _, err := Reconstruct(d, Options{First: 9, Last: 9}); err == nil { + t.Error("no error") + } +} + +func TestAnEquationReachingLeftwards(t *testing.T) { + // The growth goes both ways: the 2 of 2x is roman, and belongs to the + // equation the x seeds. + l := line{atoms: []atom{ + at("2", 20, 100, 5, 10, shape{}), + at("x", 25, 100, 5, 10, mi), + }, y: 100, size: 10} + segs := segments(l) + if len(segs) != 1 || !segs[0].math { + t.Errorf("got %+v", segs) + } +} + +func TestACharacterThatIsNeitherLetterNorArithmetic(t *testing.T) { + if mathLike(at("#", 0, 0, 5, 10, shape{})) { + t.Error("a hash is math-like") + } +} + +func TestAnItalicLetterWithItalicAfterIt(t *testing.T) { + atoms := []atom{ + at("x", 0, 0, 5, 10, shape{italic: true}), + at("word", 5, 0, 20, 10, shape{italic: true}), + } + if italicVariable(atoms, 0) { + t.Error("a letter with italic after it is a variable") + } +} + +func TestAFullWidthLineOnATwoColumnPage(t *testing.T) { + mk := func(x0, x1, y float64) line { return line{x0: x0, x1: x1, y: y, size: 10} } + var lines []line + lines = append(lines, mk(10, 190, 300)) // a heading across both columns + for i := 0; i < 6; i++ { + y := float64(280 - i*12) + lines = append(lines, mk(10, 90, y), mk(110, 190, y)) + } + got, two := order(lines, 200) + if !two { + t.Fatal("the page was not read as two columns") + } + if got[0].x0 != 10 || got[0].x1 != 190 { + t.Errorf("the full-width line did not come first: %+v", got[0]) + } + // The full-width line is measured against the page, the others against + // their own column. + if got[0].right != 190 || got[1].right != 90 { + t.Errorf("margins are %v and %v", got[0].right, got[1].right) + } + // Every left line comes before every right one. + for i := 1; i < 7; i++ { + if got[i].x0 != 10 { + t.Errorf("line %d is at %v", i, got[i].x0) + } + } +} + +func TestParagraphsFromASpaceAndFromAShortLine(t *testing.T) { + e := &emitter{body: 10, justified: true, hasPrev: true, para: []string{"x"}} + e.prev = line{y: 200, x0: 20, x1: 200, left: 20, right: 200} + e.paraRight = 200 + // A gap much wider than the leading ends a paragraph. + if !e.breaks(line{y: 178, x0: 20, left: 20, right: 200}) { + t.Error("a wide gap did not end the paragraph") + } + // So does a line that stopped well short of the right margin. + e.prev.x1 = 150 + if !e.breaks(line{y: 188, x0: 20, left: 20, right: 200}) { + t.Error("a short line did not end the paragraph") + } + // A centred block is neither indented nor short in any useful sense. + e.centred = true + if e.breaks(line{y: 188, x0: 60, left: 20, right: 200}) { + t.Error("a centred line ended the paragraph") + } +}