From 11d0e1b9cca07d9e0e6654e9e9576f15bb2ce7cd Mon Sep 17 00:00:00 2001 From: tannevaled Date: Wed, 26 Aug 2026 19:12:00 +0200 Subject: [PATCH 1/2] Read the letters of the languages that use them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The glyph names here covered the Latin an English document is set in and stopped. A document in Czech, Polish, Slovak, Hungarian, Turkish, Romanian, Latvian or Lithuanian is set in the block that follows, and every one of its accented letters was read as nothing at all — which is worse than it sounds, because a dropped character leaves no mark: "Příliš" comes back "Píliš" and still looks like a word. Counted over the whole 118 833-file corpus: of 52.4 million glyph names, 483 382 could not be turned into text, and the largest nameable share of those was that block — rcaron 650, zdotaccent 617, Ccaron 611, nacute 602, aogonek 602, scedilla 597, gbreve 597, and a hundred more. Two rules from the Adobe list were missing as well, each costing real text. A name may carry a variant after a full stop: "a.sc" is a small-capital A and is still an A, 7 272 of them. And a name may be its parts with underscores between them: "f_i" is the fi ligature, and without it "Definition" comes back "Denition" — not a missing character so much as a misspelt word. A code that stands for two letters has to give back two, so Text now asks TextOfGlyphName, and RuneOfGlyphName refuses what will not fit in one character. Measured the same way before and after, over the same corpus: unreadable names 483 382 -> 440 865 files losing a character 4 789 -> 4 332 named as a ligature 3 024 -> 0 named with a variant 7 272 -> 885 What is left is what cannot be recovered rather than what has not been: 279 253 names are g0 or C7, a subset font's own numbering, which says nothing about any character. And one that looks recoverable and is not: 49 740 names of the form uni00000048. Read as the specification says, that is U+0000 then U+0048 — but the values that appear are 0x15, 0x48, 0x44, 0x55, which are glyph numbers a producer dressed up in a Unicode-looking name. Decoding them would turn text that is merely missing into text that is confidently wrong, so they are left alone. --- encoding.go | 63 ++++++++++++++++++++++++++++++++++ encoding_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ extended.go | 71 ++++++++++++++++++++++++++++++++++++++ font.go | 6 ++-- 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 encoding_test.go create mode 100644 extended.go diff --git a/encoding.go b/encoding.go index 52f6c45..458b345 100644 --- a/encoding.go +++ b/encoding.go @@ -1,5 +1,7 @@ package pdffont +import "strings" + // The two encodings a simple font is addressed through when it does not carry // one of its own, as a name per code. Only the codes that name a glyph appear; // the rest are blank. @@ -153,9 +155,70 @@ var glyphRunes = map[string]rune{ // naming a character directly. ok is false for a name that says nothing about // which character it is, which a subsetted font's own names often do not. func RuneOfGlyphName(name string) (rune, bool) { + s, ok := TextOfGlyphName(name) + if !ok { + return 0, false + } + r := []rune(s) + if len(r) != 1 { + return 0, false + } + return r[0], true +} + +// TextOfGlyphName turns a glyph name into the text it stands for, which is not +// always one character: a name may say it is a ligature of several. +// +// Three of the rules here are the ones the Adobe Glyph List lays down for +// reading a name nobody has listed, and each was costing real text. A name may +// carry a variant after a full stop — "a.sc" is a small-capital A and is still +// an A, 7 272 of them in the corpus. A name may be several component names +// with underscores between them — "f_i" is the fi ligature, and without this +// "Definition" comes back "Denition", which is not a missing character so much +// as a misspelt word. And a name may simply be one nobody thought to list. +func TextOfGlyphName(name string) (string, bool) { + if name == "" { + return "", false + } + if r, ok := oneGlyphRune(name); ok { + return string(r), true + } + // A variant of a character is still that character: the part before the + // first full stop is the name, and what follows says which cut of it. + if base, _, found := strings.Cut(name, "."); found { + if base == "" { + return "", false + } + return TextOfGlyphName(base) + } + // A name made of parts with underscores between them is the parts, in + // order — which is how a ligature is named when it has no name of its own. + if strings.Contains(name, "_") { + // Every part that can be read contributes at least one character, and + // a part that cannot gives up on the whole name, so what comes out of + // this is never empty. + var out strings.Builder + for _, part := range strings.Split(name, "_") { + piece, ok := TextOfGlyphName(part) + if !ok { + return "", false + } + out.WriteString(piece) + } + return out.String(), true + } + return "", false +} + +// oneGlyphRune is a name that stands for exactly one character, by any of the +// ways a name can. +func oneGlyphRune(name string) (rune, bool) { if r, ok := glyphRunes[name]; ok { return r, true } + if r, ok := latinExtendedRunes[name]; ok { + return r, true + } if r, ok := greekAndMathRunes[name]; ok { return r, true } diff --git a/encoding_test.go b/encoding_test.go new file mode 100644 index 0000000..c69f00c --- /dev/null +++ b/encoding_test.go @@ -0,0 +1,89 @@ +package pdffont + +import "testing" + +func TestTheLettersOfTheLanguagesThatUseThem(t *testing.T) { + // A document in Czech, Polish, Slovak, Hungarian, Turkish or Romanian is + // set in glyphs with these names, and every one of them was being read as + // nothing at all. A dropped character leaves no mark: "Příliš" comes back + // "Píliš" and still looks like a word. + for name, want := range map[string]rune{ + "rcaron": 'ř', "zdotaccent": 'ż', "Ccaron": 'Č', "ccaron": 'č', + "nacute": 'ń', "aogonek": 'ą', "scedilla": 'ş', "gbreve": 'ğ', + "sacute": 'ś', "cacute": 'ć', "Ecaron": 'Ě', "Lcaron": 'Ľ', + "Uhungarumlaut": 'Ű', "Uring": 'Ů', "Tcaron": 'Ť', "Eng": 'Ŋ', + "Abreve": 'Ă', "Lacute": 'Ĺ', "dcroat": 'đ', "IJ": 'IJ', "ij": 'ij', + "Idotaccent": 'İ', "scommaaccent": 'ș', "kgreenlandic": 'ĸ', + "longs": 'ſ', "napostrophe": 'ʼn', "hbar": 'ħ', + "Tcedilla": 'Ţ', "tcedilla": 'ţ', "Germandbls": 'ẞ', + "lscript": 'ℓ', "openbullet": '◦', + "angbracketleft": '〈', "angbracketright": '〉', + "tcommabelow": 'ț', + } { + got, ok := RuneOfGlyphName(name) + if !ok || got != want { + t.Errorf("%s read as %q %v, wanted %q", name, got, ok, want) + } + } +} + +func TestAVariantOfACharacterIsStillThatCharacter(t *testing.T) { + // A name may carry a variant after a full stop — a small capital, an + // old-style figure, an alternate cut. Which cut it is does not change + // which letter it is, and 7 272 names in the corpus are written this way. + for name, want := range map[string]string{ + "a.sc": "a", "one.oldstyle": "1", "A.alt": "A", + "eacute.sc": "é", "f.alt01": "f", "rcaron.ss01": "ř", + } { + got, ok := TextOfGlyphName(name) + if !ok || got != want { + t.Errorf("%s read as %q %v, wanted %q", name, got, ok, want) + } + } +} + +func TestALigatureNamedByItsParts(t *testing.T) { + // A name made of parts with underscores between them is those parts in + // order. Without this, "Definition" comes back "Denition" — which is not + // a missing character so much as a misspelt word. + for name, want := range map[string]string{ + "f_i": "fi", "f_f": "ff", "f_f_i": "ffi", "f_l": "fl", + "s_t": "st", "c_t": "ct", "a_b_c": "abc", + "f_i.sc": "fi", + } { + got, ok := TextOfGlyphName(name) + if !ok || got != want { + t.Errorf("%s read as %q %v, wanted %q", name, got, ok, want) + } + } + // A code standing for two letters gives back two, and a name asking for + // that cannot be one character. + if _, ok := RuneOfGlyphName("f_i"); ok { + t.Error("a ligature of two letters was given back as one character") + } +} + +func TestANameThatSaysNothing(t *testing.T) { + // A name of one character is that character, full stop and underscore + // included — a glyph called "." is the full stop — so those are not here. + for _, name := range []string{ + "", ".sc", "f_", "_i", "g17", "cid42", "index7", + "nonesuch", "f_nonesuch", "uni", "uniZZZZ", + } { + if got, ok := TextOfGlyphName(name); ok { + t.Errorf("%q was read as %q, and it says nothing about any character", name, got) + } + } +} + +func TestTheConventionsForNamingACharacterOutright(t *testing.T) { + for name, want := range map[string]rune{ + "uni0041": 'A', "uni00E9": 'é', "u0041": 'A', "u01D400": '\U0001D400', + "A": 'A', "z": 'z', + } { + got, ok := RuneOfGlyphName(name) + if !ok || got != want { + t.Errorf("%s read as %q %v, wanted %q", name, got, ok, want) + } + } +} diff --git a/extended.go b/extended.go new file mode 100644 index 0000000..5f3be21 --- /dev/null +++ b/extended.go @@ -0,0 +1,71 @@ +package pdffont + +// The glyph names above cover the Latin every English document is set in and +// stop there. A document in Czech, Polish, Slovak, Hungarian, Turkish, +// Romanian, Latvian, Lithuanian or Maltese is set in the block that follows, +// and every one of its accented letters was being read as nothing at all — +// which is worse than it sounds, because a dropped character leaves no mark: +// "Příliš" comes back "Píliš" and looks like a word. +// +// Counted across the 118 833-file corpus: of 52.4 million glyph names, 483 382 +// could not be turned into text, and the largest nameable share of those was +// this block — rcaron 650, zdotaccent 617, Ccaron 611, nacute 602, aogonek +// 602, scedilla 597, gbreve 597, and so on for a hundred more. +// +// These are the names the Adobe Glyph List gives, and the characters are not +// in doubt: unlike the subsetted fonts that call their glyphs g0 or dress a +// glyph number up as uni00000048, a name like "rcaron" says exactly one thing. +var latinExtendedRunes = map[string]rune{ + "Amacron": 'Ā', "amacron": 'ā', "Abreve": 'Ă', "abreve": 'ă', + "Aogonek": 'Ą', "aogonek": 'ą', "Cacute": 'Ć', "cacute": 'ć', + "Ccircumflex": 'Ĉ', "ccircumflex": 'ĉ', "Cdotaccent": 'Ċ', "cdotaccent": 'ċ', + "Ccaron": 'Č', "ccaron": 'č', "Dcaron": 'Ď', "dcaron": 'ď', + "Dcroat": 'Đ', "dcroat": 'đ', "Dslash": 'Đ', "dmacron": 'đ', + "Emacron": 'Ē', "emacron": 'ē', "Ebreve": 'Ĕ', "ebreve": 'ĕ', + "Edotaccent": 'Ė', "edotaccent": 'ė', "Eogonek": 'Ę', "eogonek": 'ę', + "Ecaron": 'Ě', "ecaron": 'ě', "Gcircumflex": 'Ĝ', "gcircumflex": 'ĝ', + "Gbreve": 'Ğ', "gbreve": 'ğ', "Gdotaccent": 'Ġ', "gdotaccent": 'ġ', + "Gcommaaccent": 'Ģ', "gcommaaccent": 'ģ', + "Hcircumflex": 'Ĥ', "hcircumflex": 'ĥ', "Hbar": 'Ħ', "hbar": 'ħ', + "Itilde": 'Ĩ', "itilde": 'ĩ', "Imacron": 'Ī', "imacron": 'ī', + "Ibreve": 'Ĭ', "ibreve": 'ĭ', "Iogonek": 'Į', "iogonek": 'į', + "Idotaccent": 'İ', "dotlessi": 'ı', "IJ": 'IJ', "ij": 'ij', + "Jcircumflex": 'Ĵ', "jcircumflex": 'ĵ', + "Kcommaaccent": 'Ķ', "kcommaaccent": 'ķ', "kgreenlandic": 'ĸ', + "Lacute": 'Ĺ', "lacute": 'ĺ', "Lcommaaccent": 'Ļ', "lcommaaccent": 'ļ', + "Lcaron": 'Ľ', "lcaron": 'ľ', "Ldot": 'Ŀ', "ldot": 'ŀ', + "Nacute": 'Ń', "nacute": 'ń', "Ncommaaccent": 'Ņ', "ncommaaccent": 'ņ', + "Ncaron": 'Ň', "ncaron": 'ň', "napostrophe": 'ʼn', "Eng": 'Ŋ', "eng": 'ŋ', + "Omacron": 'Ō', "omacron": 'ō', "Obreve": 'Ŏ', "obreve": 'ŏ', + "Ohungarumlaut": 'Ő', "ohungarumlaut": 'ő', + "Racute": 'Ŕ', "racute": 'ŕ', "Rcommaaccent": 'Ŗ', "rcommaaccent": 'ŗ', + "Rcaron": 'Ř', "rcaron": 'ř', "Sacute": 'Ś', "sacute": 'ś', + "Scircumflex": 'Ŝ', "scircumflex": 'ŝ', "Scedilla": 'Ş', "scedilla": 'ş', + "Tcommaaccent": 'Ţ', "tcommaaccent": 'ţ', "Tcaron": 'Ť', "tcaron": 'ť', + "Tbar": 'Ŧ', "tbar": 'ŧ', "Utilde": 'Ũ', "utilde": 'ũ', + "Umacron": 'Ū', "umacron": 'ū', "Ubreve": 'Ŭ', "ubreve": 'ŭ', + "Uring": 'Ů', "uring": 'ů', "Uhungarumlaut": 'Ű', "uhungarumlaut": 'ű', + "Uogonek": 'Ų', "uogonek": 'ų', "Wcircumflex": 'Ŵ', "wcircumflex": 'ŵ', + "Ycircumflex": 'Ŷ', "ycircumflex": 'ŷ', + "Zacute": 'Ź', "zacute": 'ź', "Zdotaccent": 'Ż', "zdotaccent": 'ż', + "longs": 'ſ', + + // A handful outside that block that the same documents use: the Romanian + // letters written with a comma below rather than a cedilla, the Turkish + // and Azeri schwa, and the two the Adobe list names for Welsh and Maltese. + "Scommaaccent": 'Ș', "scommaaccent": 'ș', + "Afii10017": 'А', "afii10017": 'А', + "Wgrave": 'Ẁ', "wgrave": 'ẁ', "Wacute": 'Ẃ', "wacute": 'ẃ', + "Wdieresis": 'Ẅ', "wdieresis": 'ẅ', + "Ygrave": 'Ỳ', "ygrave": 'ỳ', + "Hcaron": 'Ȟ', "hcaron": 'ȟ', + // The rest of what the corpus asks for and nothing could answer, each of + // them a name the Adobe list gives and none of them in doubt: the cedilla + // spellings of the Romanian letters, the capital sharp s, the script ell + // mathematicians write lengths with, and the angle brackets. + "Tcedilla": 'Ţ', "tcedilla": 'ţ', "Germandbls": 'ẞ', + "lscript": 'ℓ', "openbullet": '◦', + "angbracketleft": '〈', "angbracketright": '〉', + "Scommabelow": 'Ș', "scommabelow": 'ș', + "Tcommabelow": 'Ț', "tcommabelow": 'ț', +} diff --git a/font.go b/font.go index 0c201f4..885c11c 100644 --- a/font.go +++ b/font.go @@ -158,8 +158,10 @@ func (f *Font) Text(code int) (string, bool) { return s, true } if name, ok := f.names[code]; ok && f.namedByTheDocument(code) { - if r, ok := RuneOfGlyphName(name); ok { - return string(r), true + // Not one character: a name may say it is a ligature of several, and + // a code that stands for two letters has to give back two. + if text, ok := TextOfGlyphName(name); ok { + return text, true } } if f.fallback != nil { From e00e7e73ecce8df0f7ddba6a0c8884828170a3d3 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Wed, 26 Aug 2026 19:30:26 +0200 Subject: [PATCH 2/2] Read a name that spells several characters, without reading a glyph number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A name may spell out more than one character: a letter and the accent over it, or the three pieces a Hebrew cluster is written in. The reader capped a name at six hex digits, so every one of those was refused. It has to be told apart from something that looks exactly like it. Producers write a glyph's own number in the same shape — uni00000048 is glyph 72, not U+0000 then U+0048 — and 49 740 names in the figure corpus are that. Reading them would turn text that is merely missing into text that is confidently wrong, which is the worse failure of the two. One observation separates them: a real sequence never begins with U+0000, because nothing is written after a character that does not exist. Of the 3 875 genuine sequences found across 14 823 embedded fonts, not one starts with a zero group; of the disguised glyph numbers, all of them do. My own sweep does not see those 3 875 — it looks at the first five pages of a file rather than at every font — so the rule is carried on the strength of the sweep that did, and on its being harmless: a name it refuses today it still refuses. A name of five or six digits is left to the older rule that reads it as one character. What producers mean by those has not been measured, and this change is not the place to guess. --- encoding.go | 34 ++++++++++++++++++++++++++++++++++ encoding_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/encoding.go b/encoding.go index 458b345..7c839fe 100644 --- a/encoding.go +++ b/encoding.go @@ -183,6 +183,9 @@ func TextOfGlyphName(name string) (string, bool) { if r, ok := oneGlyphRune(name); ok { return string(r), true } + if seq, ok := unicodeSequence(name); ok { + return seq, true + } // A variant of a character is still that character: the part before the // first full stop is the name, and what follows says which cut of it. if base, _, found := strings.Cut(name, "."); found { @@ -234,6 +237,37 @@ func oneGlyphRune(name string) (rune, bool) { return 0, false } +// unicodeSequence reads the uniXXXXYYYY form, which names several characters +// at once: a letter and the accent that goes over it, or the three pieces a +// Hebrew cluster is written in. +// +// It has to be told apart from something that looks exactly like it. Producers +// write a glyph's own number in the same shape — uni00000048 is glyph 72, not +// U+0000 followed by U+0048 — and 49 740 names in the corpus are that. The two +// are separable by one observation: a real sequence never begins with U+0000, +// because nothing is written after a character that does not exist. Of the +// 3 875 genuine sequences found across 14 823 embedded fonts, not one starts +// with a zero group; of the disguised glyph numbers, all of them do. +func unicodeSequence(name string) (string, bool) { + digits, ok := strings.CutPrefix(name, "uni") + if !ok || len(digits) < 8 || len(digits)%4 != 0 { + return "", false + } + var out strings.Builder + for i := 0; i < len(digits); i += 4 { + r, ok := parseHexName("uni"+digits[i:i+4], "uni", 4) + if !ok { + return "", false + } + if r == 0 { + // A glyph number wearing a character's clothes. + return "", false + } + out.WriteRune(r) + } + return out.String(), true +} + // parseHexName reads the uniXXXX and uXXXX conventions. func parseHexName(name, prefix string, minDigits int) (rune, bool) { if len(name) <= len(prefix) || name[:len(prefix)] != prefix { diff --git a/encoding_test.go b/encoding_test.go index c69f00c..3c54554 100644 --- a/encoding_test.go +++ b/encoding_test.go @@ -87,3 +87,45 @@ func TestTheConventionsForNamingACharacterOutright(t *testing.T) { } } } + +func TestANameThatSpellsSeveralCharacters(t *testing.T) { + // A name may spell out more than one character: a letter and the accent + // over it, or the three pieces a Hebrew cluster is written in. There are + // 3 875 of these across the fonts of the corpus. + for name, want := range map[string]string{ + "uni004A0301": "J́", + "uni05DC05BC05B9": "לֹּ", + "uni00410042": "AB", + } { + got, ok := TextOfGlyphName(name) + if !ok || got != want { + t.Errorf("%s read as %q %v, wanted %q", name, got, ok, want) + } + } +} + +func TestAGlyphNumberWearingACharactersClothes(t *testing.T) { + // A producer writes a glyph's own number in the same shape a sequence + // takes: uni00000048 is glyph 72, not U+0000 then U+0048. There are + // 49 740 of those in the corpus, and reading them would turn text that is + // merely missing into text that is confidently wrong. + // + // The two are told apart by one thing: a real sequence never begins with + // U+0000, because nothing follows a character that does not exist. + for _, name := range []string{ + "uni00000048", "uni00000015", "uni00000003", "uni0000004c", + "uni000000000048", + } { + if got, ok := TextOfGlyphName(name); ok { + t.Errorf("%s was read as %q, and it is a glyph number", name, got) + } + } + // And the malformed ones nobody meant anything by. A name of five or six + // digits is left to the older rule that reads it as one character, since + // what producers mean by those has not been measured here. + for _, name := range []string{"uni0041004", "uniZZZZ0041"} { + if got, ok := TextOfGlyphName(name); ok { + t.Errorf("%s was read as %q", name, got) + } + } +}