From b3aba37b233c6484dfcce9f0ca62bad105f65737 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 28 Aug 2026 11:21:00 +0200 Subject: [PATCH] program: read a symbolic TrueType font through its own character map A simple TrueType font flagged symbolic, carrying an embedded FontFile2, with neither /Encoding nor /ToUnicode, yielded no text at all. /Users/Shared/pdfforms/us-dol/1999-schedule-i.pdf is one: pdftotext reads 1105 words off page 1 and this package read 10, although the page's codes are plain ASCII and the content stream literally says (Department of Labor) Tj. The guard is right and stays. pdffont does fall back to StandardEncoding when a font has no /Encoding, so names are assigned; Font.Text then refuses them for a symbolic font, because a mathematical symbolic font puts a capital gamma where the standard encoding puts an inverted exclamation mark and reading the name would give the wrong letter with nothing to say so. The answer has to come from the program, which is what the fallback installed here is for. It only knew one way to ask: the program's own code-to-name encoding, which only a PostScript program carries. Font.GlyphIndexByCode is documented to report nothing for every TrueType font, so the last resort returned nothing at the first step for all six embedded fonts of that file. So ask the other way, through the font's character map. The way in is the one poppler uses for a font the document gave no encoding (GfxFont.cc, Gfx8BitFont::getCodeToGIDMap, case 2): the Microsoft Symbol subtable if there is one, else the Macintosh Roman one, addressed by the raw code and then by 0xF000 + code, which is where such subtables are conventionally written. The way out is the Unicode subtable, inverted -- the font's own statement of which character that glyph is for. Without a Unicode subtable there is no way out and it says nothing: a glyph on its own says nothing about which character it is, and guessing would produce exactly the wrong letter the guard exists to refuse. Nor is a private-use or control character reported, which reads no better than silence and would wrongly look like a page that had been read. Order is unchanged: /ToUnicode, then a name the document chose in /Differences, then the program. The program stays the last resort. Measured. The four US Department of Labor forms, page 1, against pdftotext on the same page: schedule I 10 -> 1103 words (poppler 1105), 5500 schedule A 10 -> 244 (248), schedule D 10 -> 312 (314), schedule G 10 -> 227 (229). Across the 1633 real forms of the corpus, first three pages, four files gain text and none loses any; across 118843 arXiv documents not one byte of text changes, because 98% of the fonts in that shape there are PostScript programs the name route already answered for. --- cmapbuild_test.go | 109 +++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- program.go | 135 +++++++++++++++++++++++++++++++++++++---- program_test.go | 149 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 386 insertions(+), 13 deletions(-) create mode 100644 cmapbuild_test.go diff --git a/cmapbuild_test.go b/cmapbuild_test.go new file mode 100644 index 0000000..c73959e --- /dev/null +++ b/cmapbuild_test.go @@ -0,0 +1,109 @@ +package extract + +import ( + "encoding/binary" + "sort" + + "github.com/go-opentype/fonts" +) + +// A symbolic TrueType font embedded in a PDF is addressed through its own +// character map, and which subtables it carries decides what can be read back +// out of it. Real fonts carry whichever their maker chose, so these build one +// carrying exactly the subtables a test is about, around the glyphs of a real +// font so the rest of the program stays true. + +// cmapSpec is one subtable of a synthetic cmap table: the platform and +// encoding it claims to be written for, and the codes it maps. +type cmapSpec struct { + platform uint16 + encoding uint16 + codes map[rune]uint16 +} + +// fontWithCmaps re-emits a real TrueType font with its character map replaced +// by the given subtables, leaving its glyphs, metrics and every other table +// alone. +func fontWithCmaps(specs []cmapSpec) []byte { + src := fonts.MostLegible() + type table struct { + tag string + data []byte + } + var tables []table + n := int(binary.BigEndian.Uint16(src[4:])) + for i := range n { + rec := src[12+i*16:] + tag := string(rec[:4]) + if tag == "cmap" { + continue + } + off := int(binary.BigEndian.Uint32(rec[8:])) + length := int(binary.BigEndian.Uint32(rec[12:])) + tables = append(tables, table{tag, src[off : off+length]}) + } + tables = append(tables, table{"cmap", cmapTableOf(specs)}) + sort.Slice(tables, func(i, j int) bool { return tables[i].tag < tables[j].tag }) + + out := make([]byte, 12+16*len(tables)) + binary.BigEndian.PutUint32(out, 0x00010000) + binary.BigEndian.PutUint16(out[4:], uint16(len(tables))) + for i, t := range tables { + for len(out)%4 != 0 { + out = append(out, 0) + } + rec := out[12+i*16:] + copy(rec[:4], t.tag) + binary.BigEndian.PutUint32(rec[8:], uint32(len(out))) + binary.BigEndian.PutUint32(rec[12:], uint32(len(t.data))) + out = append(out, t.data...) + } + return out +} + +// cmapTableOf builds a cmap table whose records carry the platform and +// encoding each spec asks for. +func cmapTableOf(specs []cmapSpec) []byte { + head := make([]byte, 4+8*len(specs)) + binary.BigEndian.PutUint16(head[2:], uint16(len(specs))) + body := []byte{} + for i, s := range specs { + rec := head[4+i*8:] + binary.BigEndian.PutUint16(rec, s.platform) + binary.BigEndian.PutUint16(rec[2:], s.encoding) + binary.BigEndian.PutUint32(rec[4:], uint32(len(head)+len(body))) + body = append(body, cmap4Of(s.codes)...) + } + return append(head, body...) +} + +// cmap4Of builds a format-4 subtable, one segment per code, with the sentinel +// segment the format requires. +func cmap4Of(codes map[rune]uint16) []byte { + runes := make([]int, 0, len(codes)) + for r := range codes { + runes = append(runes, int(r)) + } + sort.Ints(runes) + runes = append(runes, 0xFFFF) + + be := binary.BigEndian + seg := len(runes) + out := make([]byte, 14+8*seg+2) + be.PutUint16(out, 4) + be.PutUint16(out[2:], uint16(len(out))) + be.PutUint16(out[6:], uint16(seg*2)) + for i, r := range runes { + // A code the caller did not ask for -- the sentinel among them -- + // takes the delta the sentinel segment is required to carry, which + // lands 0xFFFF on .notdef. + delta := uint16(1) + if g, ok := codes[rune(r)]; ok { + delta = g - uint16(r) + } + be.PutUint16(out[14+2*i:], uint16(r)) // endCode + be.PutUint16(out[16+2*seg+2*i:], uint16(r)) // startCode + be.PutUint16(out[16+4*seg+2*i:], delta) // idDelta + } + return out +} diff --git a/go.mod b/go.mod index 2544fd8..659805f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/go-pdfkit/extract go 1.26.4 require ( - github.com/go-opentype/opentype v0.9.0 + github.com/go-opentype/opentype v0.12.0 github.com/go-pdfkit/pdffont v0.3.0 github.com/go-pdfkit/reader v0.6.0 ) diff --git a/go.sum b/go.sum index ec9a233..2f6a954 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/go-opentype/fonts v0.8.0 h1:77i3VPIH90GbstzNb21mk+an4WvEOe2idC6W+J0n0fw= github.com/go-opentype/fonts v0.8.0/go.mod h1:C6yQL2apHItfEZ5hztpsHF0S5mlX/hklLlq/Z5fRG/g= -github.com/go-opentype/opentype v0.9.0 h1:GFgcJ3nwTDp4NJr5O+Paw7lhZx5Jv/R+noZwvhYDlkM= -github.com/go-opentype/opentype v0.9.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= +github.com/go-opentype/opentype v0.12.0 h1:wBlcDi+3ZaNZXEt5z+Ixr11/cYYwi5W+jX6yTl/qr1I= +github.com/go-opentype/opentype v0.12.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= github.com/go-pdfkit/pdffont v0.3.0 h1:G5DKcAmsZJ0e17QhSrcUaL7PKEXKjUx4P/iGMl7bAbI= github.com/go-pdfkit/pdffont v0.3.0/go.mod h1:bfmNLna1l1CljNX/Utg55YzFylovfmI7sJnvgA3bzKI= github.com/go-pdfkit/reader v0.6.0 h1:KAabNOYUcTlZlNBTbG9bEhWP1NiZhjuhzUdavdTdfes= diff --git a/program.go b/program.go index e387101..9ad68d3 100644 --- a/program.go +++ b/program.go @@ -13,6 +13,13 @@ import ( // and an assumed encoding is a bad guess at it — a mathematical font puts a // capital gamma where the standard encoding puts an inverted exclamation mark. // The program knows, and this is how it is asked. +// +// There are two ways to ask, and a program answers at most one of them. A +// PostScript program names its glyphs and carries an encoding from code to +// name, so the name is the answer. A TrueType program usually does neither — +// a subset of one names nothing worth reading and has no code-to-name +// encoding at all — but it does carry character maps, and walking through one +// to the glyph and back out of another says which character the glyph is for. func attachProgram(f *pdffont.Font) { if f.Kind() == pdffont.Composite { return @@ -25,23 +32,131 @@ func attachProgram(f *pdffont.Font) { if err != nil { return } + maps := chooseCharacterMaps(program) // Only a simple font reaches here, and its codes are bytes. f.SetFallback(func(code int) (string, bool) { - gid, ok := program.GlyphIndexByCode(byte(code)) - if !ok { - return "", false + if r, ok := runeByName(program, code); ok { + return string(r), true } - // A glyph the program does not name comes back as no name at all, - // which names no character either. - name, _ := program.GlyphName(gid) - r, ok := pdffont.RuneOfGlyphName(name) - if !ok { - return "", false + if r, ok := runeByCharacterMap(program, maps, code); ok { + return string(r), true } - return string(r), true + return "", false }) } +// runeByName asks the program's own encoding what it calls a code, and reads +// the character out of that name. Only a PostScript program — a Type 1 one, or +// the CFF outlines of an OpenType font — carries such an encoding. +func runeByName(program *opentype.Font, code int) (rune, bool) { + gid, ok := program.GlyphIndexByCode(byte(code)) + if !ok { + return 0, false + } + // A glyph the program does not name comes back as no name at all, + // which names no character either. + name, _ := program.GlyphName(gid) + return pdffont.RuneOfGlyphName(name) +} + +// characterMaps says which of a program's cmap subtables are worth addressing, +// by index, or -1 for one the program does not carry. +type characterMaps struct { + // symbol is the Microsoft Symbol subtable, platform 3 encoding 0: a font's + // own codes, conventionally written at 0xF000 + code. + symbol int + // mac is the Macintosh Roman subtable, platform 1 encoding 0, indexed by + // single bytes. + mac int + // unicode is a Unicode subtable — Microsoft Unicode, platform 3 encoding 1 + // or 10, or anything on platform 0 — indexed by codepoint. This is the one + // that is inverted, because it is the only one whose codes are characters. + unicode int +} + +// chooseCharacterMaps picks out the subtables a font is addressed through and +// the one that says which character a glyph is for. +// +// A font can carry several subtables of a kind; the first of each is taken, +// which is what a font that repeats one means by repeating it. +func chooseCharacterMaps(program *opentype.Font) characterMaps { + m := characterMaps{symbol: -1, mac: -1, unicode: -1} + for i := range program.NumCharacterMaps() { + platform, encoding, _, _ := program.CharacterMap(i) + switch { + case platform == 3 && encoding == 0: + if m.symbol < 0 { + m.symbol = i + } + case platform == 1 && encoding == 0: + if m.mac < 0 { + m.mac = i + } + case platform == 0, platform == 3 && (encoding == 1 || encoding == 10): + if m.unicode < 0 { + m.unicode = i + } + } + } + return m +} + +// runeByCharacterMap walks a code through the font's own character map to a +// glyph, and back out of its Unicode character map to the character that glyph +// stands for. +// +// The way in is the one poppler uses for a font the document gave no encoding: +// the Microsoft Symbol subtable if there is one, else the Macintosh Roman one, +// addressed by the raw code and then, failing that, by 0xF000 + code, which is +// where such subtables are conventionally written. +// +// The way out is the Unicode subtable, inverted. Without one there is no way +// out: a glyph on its own says nothing about which character it is, and a +// guess would be exactly the wrong letter the caller's guard exists to refuse. +func runeByCharacterMap(program *opentype.Font, m characterMaps, code int) (rune, bool) { + if m.unicode < 0 { + return 0, false + } + in := m.symbol + if in < 0 { + in = m.mac + } + if in < 0 { + return 0, false + } + gid, ok := program.GlyphIndexInMap(in, rune(code)) + if !ok { + gid, ok = program.GlyphIndexInMap(in, rune(0xF000|code)) + } + if !ok { + return 0, false + } + r, ok := program.RuneOfGlyphInMap(m.unicode, gid) + if !ok || !readableRune(r) { + return 0, false + } + return r, true +} + +// readableRune reports whether a character recovered this way is worth +// reporting as text. +// +// A private-use codepoint is not: it means whatever the font decided it means +// and nothing outside the font can read it, so a page full of them searches no +// better than a page of nothing and looks, wrongly, like it was read. Control +// characters are refused for the same reason — a page does not say them. +func readableRune(r rune) bool { + switch { + case r < 0x20, r >= 0x7F && r <= 0x9F: + return false + case r >= 0xE000 && r <= 0xF8FF: + return false + case r >= 0xF0000: + return false + } + return true +} + // readProgram decodes an embedded font program. Which key it arrived under // says what it is: FontFile2 is TrueType, FontFile a PostScript Type 1 // program, and FontFile3 either a bare CFF one or a whole OpenType font — the diff --git a/program_test.go b/program_test.go index ac603cc..153ffb0 100644 --- a/program_test.go +++ b/program_test.go @@ -170,3 +170,152 @@ func TestWhatTheProgramCannotSay(t *testing.T) { t.Error("the codes nothing could name did not say so") } } + +// cmapFont is a font whose descriptor says symbolic and whose dictionary says +// nothing else: no Encoding, no ToUnicode. Its own character map is then the +// only thing left that knows what its codes are. +func cmapFont(w *reader.Writer, specs []cmapSpec) reader.Dict { + ttf := fontWithCmaps(specs) + file := w.Add(&reader.Stream{Dict: reader.Dict{"Length1": reader.Integer(len(ttf))}, Raw: ttf}) + return reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("TrueType"), + "BaseFont": reader.Name("Test"), + "FontDescriptor": w.Add(reader.Dict{ + "Type": reader.Name("FontDescriptor"), "FontName": reader.Name("Test"), + "Flags": reader.Integer(4), "FontFile2": file}), + } +} + +// saysThrough reads one page drawn in a font carrying exactly these subtables. +func saysThrough(t *testing.T, content string, specs []cmapSpec) string { + t.Helper() + d := pageWith(t, content, func(w *reader.Writer, res reader.Dict) { + res["Font"].(reader.Dict)["F2"] = w.Add(cmapFont(w, specs)) + }) + text, err := Text(d, 1) + if err != nil { + t.Fatal(err) + } + return text +} + +func TestASymbolicTrueTypeFontReadThroughItsCharacterMap(t *testing.T) { + // The way in is the Microsoft Symbol subtable, whose codes conventionally + // live at 0xF000 + code; the way out is the Unicode subtable, inverted. + // Code 'A' reaches glyph 5 at 0xF041, and the font says glyph 5 is 'Z'. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }) + if text != "Z" { + t.Errorf("the page says %q, want %q", text, "Z") + } +} + +func TestACodeAddressedWithoutTheSymbolOffset(t *testing.T) { + // A symbol subtable written at the raw code rather than at 0xF000 + code + // is tried first, which is the order poppler uses. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (B) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{'B': 7}}, + {3, 1, map[rune]uint16{'W': 7}}, + }) + if text != "W" { + t.Errorf("the page says %q, want %q", text, "W") + } +} + +func TestAFontAddressedThroughItsMacintoshRomanMap(t *testing.T) { + // With no Microsoft Symbol subtable the Macintosh Roman one is the way in. + // Platform 0 serves as the way out just as Microsoft Unicode does. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 6}}, + {0, 3, map[rune]uint16{'Q': 6}}, + }) + if text != "Q" { + t.Errorf("the page says %q, want %q", text, "Q") + } +} + +func TestTheFirstSubtableOfAKindIsTheOneTaken(t *testing.T) { + // A font that repeats a kind of subtable means the first of them. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + {3, 0, map[rune]uint16{0xF041: 8}}, + {1, 0, map[rune]uint16{'A': 8}}, + {1, 0, map[rune]uint16{'A': 9}}, + {3, 1, map[rune]uint16{'Z': 5}}, + {3, 10, map[rune]uint16{'Y': 5}}, + }) + if text != "Z" { + t.Errorf("the page says %q, want %q", text, "Z") + } +} + +func TestWhatTheCharacterMapRouteRefusesToSay(t *testing.T) { + // Every way this can honestly come up empty. A guess would be exactly the + // wrong letter the symbolic guard exists to refuse, so it says nothing. + cases := []struct { + why string + specs []cmapSpec + }{ + {"no way out: nothing says which character a glyph is", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + }}, + {"no way in: the font is not addressed by its own codes", []cmapSpec{ + {3, 1, map[rune]uint16{'A': 5}}, + }}, + {"no subtable of any kind this route knows", []cmapSpec{ + {7, 7, map[rune]uint16{'A': 5}}, + }}, + {"the code reaches no glyph, at either address", []cmapSpec{ + {3, 0, map[rune]uint16{0xF042: 5}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }}, + {"no code in the Unicode map reaches that glyph", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 9}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }}, + {"the glyph is a private-use character, which nothing can read", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0xE000: 5}}, + }}, + {"the glyph is a control character, which a page does not say", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0x0B: 5}}, + }}, + {"the glyph is a C1 control character", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0x85: 5}}, + }}, + } + for _, c := range cases { + if text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", c.specs); text != "" { + t.Errorf("%s: the page says %q, want nothing", c.why, text) + } + } +} + +func TestWhichCharactersAreWorthReporting(t *testing.T) { + // A format-4 subtable cannot reach beyond the basic plane, so the high + // private-use planes are checked here rather than through a whole page. + cases := map[rune]bool{ + 'A': true, + ' ': true, + 0x00: false, // NUL + 0x1F: false, // a C0 control + 0x7F: false, // delete + 0x85: false, // a C1 control + 0xA0: true, // no-break space, the first character past them + 0xE000: false, // private use + 0xF8FF: false, // private use, last + 0xF900: true, // a compatibility ideograph, just past it + 0x1F600: true, // an astral character a font may really mean + 0xF0000: false, // supplementary private use area A + 0x10FFFF: false, // supplementary private use area B + } + for r, want := range cases { + if got := readableRune(r); got != want { + t.Errorf("readableRune(%#x) = %v want %v", r, got, want) + } + } +}