diff --git a/cmapbuild_test.go b/cmapbuild_test.go new file mode 100644 index 0000000..c73959e --- /dev/null +++ b/cmapbuild_test.go @@ -0,0 +1,109 @@ +package extract + +import ( + "encoding/binary" + "sort" + + "github.com/go-opentype/fonts" +) + +// A symbolic TrueType font embedded in a PDF is addressed through its own +// character map, and which subtables it carries decides what can be read back +// out of it. Real fonts carry whichever their maker chose, so these build one +// carrying exactly the subtables a test is about, around the glyphs of a real +// font so the rest of the program stays true. + +// cmapSpec is one subtable of a synthetic cmap table: the platform and +// encoding it claims to be written for, and the codes it maps. +type cmapSpec struct { + platform uint16 + encoding uint16 + codes map[rune]uint16 +} + +// fontWithCmaps re-emits a real TrueType font with its character map replaced +// by the given subtables, leaving its glyphs, metrics and every other table +// alone. +func fontWithCmaps(specs []cmapSpec) []byte { + src := fonts.MostLegible() + type table struct { + tag string + data []byte + } + var tables []table + n := int(binary.BigEndian.Uint16(src[4:])) + for i := range n { + rec := src[12+i*16:] + tag := string(rec[:4]) + if tag == "cmap" { + continue + } + off := int(binary.BigEndian.Uint32(rec[8:])) + length := int(binary.BigEndian.Uint32(rec[12:])) + tables = append(tables, table{tag, src[off : off+length]}) + } + tables = append(tables, table{"cmap", cmapTableOf(specs)}) + sort.Slice(tables, func(i, j int) bool { return tables[i].tag < tables[j].tag }) + + out := make([]byte, 12+16*len(tables)) + binary.BigEndian.PutUint32(out, 0x00010000) + binary.BigEndian.PutUint16(out[4:], uint16(len(tables))) + for i, t := range tables { + for len(out)%4 != 0 { + out = append(out, 0) + } + rec := out[12+i*16:] + copy(rec[:4], t.tag) + binary.BigEndian.PutUint32(rec[8:], uint32(len(out))) + binary.BigEndian.PutUint32(rec[12:], uint32(len(t.data))) + out = append(out, t.data...) + } + return out +} + +// cmapTableOf builds a cmap table whose records carry the platform and +// encoding each spec asks for. +func cmapTableOf(specs []cmapSpec) []byte { + head := make([]byte, 4+8*len(specs)) + binary.BigEndian.PutUint16(head[2:], uint16(len(specs))) + body := []byte{} + for i, s := range specs { + rec := head[4+i*8:] + binary.BigEndian.PutUint16(rec, s.platform) + binary.BigEndian.PutUint16(rec[2:], s.encoding) + binary.BigEndian.PutUint32(rec[4:], uint32(len(head)+len(body))) + body = append(body, cmap4Of(s.codes)...) + } + return append(head, body...) +} + +// cmap4Of builds a format-4 subtable, one segment per code, with the sentinel +// segment the format requires. +func cmap4Of(codes map[rune]uint16) []byte { + runes := make([]int, 0, len(codes)) + for r := range codes { + runes = append(runes, int(r)) + } + sort.Ints(runes) + runes = append(runes, 0xFFFF) + + be := binary.BigEndian + seg := len(runes) + out := make([]byte, 14+8*seg+2) + be.PutUint16(out, 4) + be.PutUint16(out[2:], uint16(len(out))) + be.PutUint16(out[6:], uint16(seg*2)) + for i, r := range runes { + // A code the caller did not ask for -- the sentinel among them -- + // takes the delta the sentinel segment is required to carry, which + // lands 0xFFFF on .notdef. + delta := uint16(1) + if g, ok := codes[rune(r)]; ok { + delta = g - uint16(r) + } + be.PutUint16(out[14+2*i:], uint16(r)) // endCode + be.PutUint16(out[16+2*seg+2*i:], uint16(r)) // startCode + be.PutUint16(out[16+4*seg+2*i:], delta) // idDelta + } + return out +} diff --git a/go.mod b/go.mod index 2544fd8..659805f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/go-pdfkit/extract go 1.26.4 require ( - github.com/go-opentype/opentype v0.9.0 + github.com/go-opentype/opentype v0.12.0 github.com/go-pdfkit/pdffont v0.3.0 github.com/go-pdfkit/reader v0.6.0 ) diff --git a/go.sum b/go.sum index ec9a233..2f6a954 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/go-opentype/fonts v0.8.0 h1:77i3VPIH90GbstzNb21mk+an4WvEOe2idC6W+J0n0fw= github.com/go-opentype/fonts v0.8.0/go.mod h1:C6yQL2apHItfEZ5hztpsHF0S5mlX/hklLlq/Z5fRG/g= -github.com/go-opentype/opentype v0.9.0 h1:GFgcJ3nwTDp4NJr5O+Paw7lhZx5Jv/R+noZwvhYDlkM= -github.com/go-opentype/opentype v0.9.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= +github.com/go-opentype/opentype v0.12.0 h1:wBlcDi+3ZaNZXEt5z+Ixr11/cYYwi5W+jX6yTl/qr1I= +github.com/go-opentype/opentype v0.12.0/go.mod h1:AOixevJf7XQaH7+WG+OMIOZEbYPXfMqklVk26Y6YTUU= github.com/go-pdfkit/pdffont v0.3.0 h1:G5DKcAmsZJ0e17QhSrcUaL7PKEXKjUx4P/iGMl7bAbI= github.com/go-pdfkit/pdffont v0.3.0/go.mod h1:bfmNLna1l1CljNX/Utg55YzFylovfmI7sJnvgA3bzKI= github.com/go-pdfkit/reader v0.6.0 h1:KAabNOYUcTlZlNBTbG9bEhWP1NiZhjuhzUdavdTdfes= diff --git a/program.go b/program.go index e387101..9ad68d3 100644 --- a/program.go +++ b/program.go @@ -13,6 +13,13 @@ import ( // and an assumed encoding is a bad guess at it — a mathematical font puts a // capital gamma where the standard encoding puts an inverted exclamation mark. // The program knows, and this is how it is asked. +// +// There are two ways to ask, and a program answers at most one of them. A +// PostScript program names its glyphs and carries an encoding from code to +// name, so the name is the answer. A TrueType program usually does neither — +// a subset of one names nothing worth reading and has no code-to-name +// encoding at all — but it does carry character maps, and walking through one +// to the glyph and back out of another says which character the glyph is for. func attachProgram(f *pdffont.Font) { if f.Kind() == pdffont.Composite { return @@ -25,23 +32,131 @@ func attachProgram(f *pdffont.Font) { if err != nil { return } + maps := chooseCharacterMaps(program) // Only a simple font reaches here, and its codes are bytes. f.SetFallback(func(code int) (string, bool) { - gid, ok := program.GlyphIndexByCode(byte(code)) - if !ok { - return "", false + if r, ok := runeByName(program, code); ok { + return string(r), true } - // A glyph the program does not name comes back as no name at all, - // which names no character either. - name, _ := program.GlyphName(gid) - r, ok := pdffont.RuneOfGlyphName(name) - if !ok { - return "", false + if r, ok := runeByCharacterMap(program, maps, code); ok { + return string(r), true } - return string(r), true + return "", false }) } +// runeByName asks the program's own encoding what it calls a code, and reads +// the character out of that name. Only a PostScript program — a Type 1 one, or +// the CFF outlines of an OpenType font — carries such an encoding. +func runeByName(program *opentype.Font, code int) (rune, bool) { + gid, ok := program.GlyphIndexByCode(byte(code)) + if !ok { + return 0, false + } + // A glyph the program does not name comes back as no name at all, + // which names no character either. + name, _ := program.GlyphName(gid) + return pdffont.RuneOfGlyphName(name) +} + +// characterMaps says which of a program's cmap subtables are worth addressing, +// by index, or -1 for one the program does not carry. +type characterMaps struct { + // symbol is the Microsoft Symbol subtable, platform 3 encoding 0: a font's + // own codes, conventionally written at 0xF000 + code. + symbol int + // mac is the Macintosh Roman subtable, platform 1 encoding 0, indexed by + // single bytes. + mac int + // unicode is a Unicode subtable — Microsoft Unicode, platform 3 encoding 1 + // or 10, or anything on platform 0 — indexed by codepoint. This is the one + // that is inverted, because it is the only one whose codes are characters. + unicode int +} + +// chooseCharacterMaps picks out the subtables a font is addressed through and +// the one that says which character a glyph is for. +// +// A font can carry several subtables of a kind; the first of each is taken, +// which is what a font that repeats one means by repeating it. +func chooseCharacterMaps(program *opentype.Font) characterMaps { + m := characterMaps{symbol: -1, mac: -1, unicode: -1} + for i := range program.NumCharacterMaps() { + platform, encoding, _, _ := program.CharacterMap(i) + switch { + case platform == 3 && encoding == 0: + if m.symbol < 0 { + m.symbol = i + } + case platform == 1 && encoding == 0: + if m.mac < 0 { + m.mac = i + } + case platform == 0, platform == 3 && (encoding == 1 || encoding == 10): + if m.unicode < 0 { + m.unicode = i + } + } + } + return m +} + +// runeByCharacterMap walks a code through the font's own character map to a +// glyph, and back out of its Unicode character map to the character that glyph +// stands for. +// +// The way in is the one poppler uses for a font the document gave no encoding: +// the Microsoft Symbol subtable if there is one, else the Macintosh Roman one, +// addressed by the raw code and then, failing that, by 0xF000 + code, which is +// where such subtables are conventionally written. +// +// The way out is the Unicode subtable, inverted. Without one there is no way +// out: a glyph on its own says nothing about which character it is, and a +// guess would be exactly the wrong letter the caller's guard exists to refuse. +func runeByCharacterMap(program *opentype.Font, m characterMaps, code int) (rune, bool) { + if m.unicode < 0 { + return 0, false + } + in := m.symbol + if in < 0 { + in = m.mac + } + if in < 0 { + return 0, false + } + gid, ok := program.GlyphIndexInMap(in, rune(code)) + if !ok { + gid, ok = program.GlyphIndexInMap(in, rune(0xF000|code)) + } + if !ok { + return 0, false + } + r, ok := program.RuneOfGlyphInMap(m.unicode, gid) + if !ok || !readableRune(r) { + return 0, false + } + return r, true +} + +// readableRune reports whether a character recovered this way is worth +// reporting as text. +// +// A private-use codepoint is not: it means whatever the font decided it means +// and nothing outside the font can read it, so a page full of them searches no +// better than a page of nothing and looks, wrongly, like it was read. Control +// characters are refused for the same reason — a page does not say them. +func readableRune(r rune) bool { + switch { + case r < 0x20, r >= 0x7F && r <= 0x9F: + return false + case r >= 0xE000 && r <= 0xF8FF: + return false + case r >= 0xF0000: + return false + } + return true +} + // readProgram decodes an embedded font program. Which key it arrived under // says what it is: FontFile2 is TrueType, FontFile a PostScript Type 1 // program, and FontFile3 either a bare CFF one or a whole OpenType font — the diff --git a/program_test.go b/program_test.go index ac603cc..153ffb0 100644 --- a/program_test.go +++ b/program_test.go @@ -170,3 +170,152 @@ func TestWhatTheProgramCannotSay(t *testing.T) { t.Error("the codes nothing could name did not say so") } } + +// cmapFont is a font whose descriptor says symbolic and whose dictionary says +// nothing else: no Encoding, no ToUnicode. Its own character map is then the +// only thing left that knows what its codes are. +func cmapFont(w *reader.Writer, specs []cmapSpec) reader.Dict { + ttf := fontWithCmaps(specs) + file := w.Add(&reader.Stream{Dict: reader.Dict{"Length1": reader.Integer(len(ttf))}, Raw: ttf}) + return reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("TrueType"), + "BaseFont": reader.Name("Test"), + "FontDescriptor": w.Add(reader.Dict{ + "Type": reader.Name("FontDescriptor"), "FontName": reader.Name("Test"), + "Flags": reader.Integer(4), "FontFile2": file}), + } +} + +// saysThrough reads one page drawn in a font carrying exactly these subtables. +func saysThrough(t *testing.T, content string, specs []cmapSpec) string { + t.Helper() + d := pageWith(t, content, func(w *reader.Writer, res reader.Dict) { + res["Font"].(reader.Dict)["F2"] = w.Add(cmapFont(w, specs)) + }) + text, err := Text(d, 1) + if err != nil { + t.Fatal(err) + } + return text +} + +func TestASymbolicTrueTypeFontReadThroughItsCharacterMap(t *testing.T) { + // The way in is the Microsoft Symbol subtable, whose codes conventionally + // live at 0xF000 + code; the way out is the Unicode subtable, inverted. + // Code 'A' reaches glyph 5 at 0xF041, and the font says glyph 5 is 'Z'. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }) + if text != "Z" { + t.Errorf("the page says %q, want %q", text, "Z") + } +} + +func TestACodeAddressedWithoutTheSymbolOffset(t *testing.T) { + // A symbol subtable written at the raw code rather than at 0xF000 + code + // is tried first, which is the order poppler uses. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (B) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{'B': 7}}, + {3, 1, map[rune]uint16{'W': 7}}, + }) + if text != "W" { + t.Errorf("the page says %q, want %q", text, "W") + } +} + +func TestAFontAddressedThroughItsMacintoshRomanMap(t *testing.T) { + // With no Microsoft Symbol subtable the Macintosh Roman one is the way in. + // Platform 0 serves as the way out just as Microsoft Unicode does. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 6}}, + {0, 3, map[rune]uint16{'Q': 6}}, + }) + if text != "Q" { + t.Errorf("the page says %q, want %q", text, "Q") + } +} + +func TestTheFirstSubtableOfAKindIsTheOneTaken(t *testing.T) { + // A font that repeats a kind of subtable means the first of them. + text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + {3, 0, map[rune]uint16{0xF041: 8}}, + {1, 0, map[rune]uint16{'A': 8}}, + {1, 0, map[rune]uint16{'A': 9}}, + {3, 1, map[rune]uint16{'Z': 5}}, + {3, 10, map[rune]uint16{'Y': 5}}, + }) + if text != "Z" { + t.Errorf("the page says %q, want %q", text, "Z") + } +} + +func TestWhatTheCharacterMapRouteRefusesToSay(t *testing.T) { + // Every way this can honestly come up empty. A guess would be exactly the + // wrong letter the symbolic guard exists to refuse, so it says nothing. + cases := []struct { + why string + specs []cmapSpec + }{ + {"no way out: nothing says which character a glyph is", []cmapSpec{ + {3, 0, map[rune]uint16{0xF041: 5}}, + }}, + {"no way in: the font is not addressed by its own codes", []cmapSpec{ + {3, 1, map[rune]uint16{'A': 5}}, + }}, + {"no subtable of any kind this route knows", []cmapSpec{ + {7, 7, map[rune]uint16{'A': 5}}, + }}, + {"the code reaches no glyph, at either address", []cmapSpec{ + {3, 0, map[rune]uint16{0xF042: 5}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }}, + {"no code in the Unicode map reaches that glyph", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 9}}, + {3, 1, map[rune]uint16{'Z': 5}}, + }}, + {"the glyph is a private-use character, which nothing can read", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0xE000: 5}}, + }}, + {"the glyph is a control character, which a page does not say", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0x0B: 5}}, + }}, + {"the glyph is a C1 control character", []cmapSpec{ + {1, 0, map[rune]uint16{'A': 5}}, + {3, 1, map[rune]uint16{0x85: 5}}, + }}, + } + for _, c := range cases { + if text := saysThrough(t, "BT /F2 10 Tf 20 100 Td (A) Tj ET", c.specs); text != "" { + t.Errorf("%s: the page says %q, want nothing", c.why, text) + } + } +} + +func TestWhichCharactersAreWorthReporting(t *testing.T) { + // A format-4 subtable cannot reach beyond the basic plane, so the high + // private-use planes are checked here rather than through a whole page. + cases := map[rune]bool{ + 'A': true, + ' ': true, + 0x00: false, // NUL + 0x1F: false, // a C0 control + 0x7F: false, // delete + 0x85: false, // a C1 control + 0xA0: true, // no-break space, the first character past them + 0xE000: false, // private use + 0xF8FF: false, // private use, last + 0xF900: true, // a compatibility ideograph, just past it + 0x1F600: true, // an astral character a font may really mean + 0xF0000: false, // supplementary private use area A + 0x10FFFF: false, // supplementary private use area B + } + for r, want := range cases { + if got := readableRune(r); got != want { + t.Errorf("readableRune(%#x) = %v want %v", r, got, want) + } + } +}