From 850a992e67677ba91b0c599ec67a67eb2a0ddb42 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 27 Aug 2026 15:18:34 +0200 Subject: [PATCH] Decode a fax, so a scanned form is not a blank page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scanned form is a fax. 67 of the 1 633 real forms in the corpus — the eleven issuing bodies, not the vendor test suites — carry a CCITT-encoded image, 263 images between them, and 6 of their pages have nothing else on them at all. Those pages drew entirely blank, and a blank page is the failure a reader notices before any other. WHY IT BELONGS IN THE READER The other filters this package stops at carry an image with its own idea of how many components it has and how deep they are. A fax does not: it produces bilevel samples, one bit a pixel, rows padded to a byte, and the stream dictionary says what they mean. That is a byte stream, so it is a filter — and decoding it here means every caller gets it rather than each writing its own. /CCITTFaxDecode and /CCF therefore leave ImageFilter's list, which is a change to a documented contract and is written up as one. Group 3 one-dimensional, Group 3 mixed and Group 4 are all decoded, with /K, /Columns, /Rows, /BlackIs1, /EncodedByteAlign and /EndOfBlock read. THE REFERENCE READ BEFORE WRITING IT ITU-T T.4 and T.6, by way of the tables and the changing-element algorithm in golang.org/x/image/ccitt — read, not imported, because this package has no dependencies outside the standard library and gains none. The 218 code table entries were extracted mechanically from that package's gen.go rather than retyped, because a table of 218 variable-length codes transcribed by hand is a table with a mistake in it. MEASURED AGAINST THAT REFERENCE, IMAGE BY IMAGE Every CCITT image in the corpus, decoded by both and compared byte for byte, with both sides working the height out for themselves where /Rows is absent: 1 633 real forms 263 images 260 byte-identical 0 differ 3 the reference refuses arXiv sample 47 images 47 byte-identical 0 differ The three the reference gives up on ("invalid code", "invalid offset") are pages 2, 3 and 4 of fr-cerfa/cerfa_10455.pdf, where it stops and we decode all 3 504 rows — exactly the /Height the image dictionary names. Being more forgiving than the reference is deliberate: a damaged scan comes back as far as it got, because refusing it turns a form into a blank page. TWO THINGS THE MEASUREMENT FOUND THAT READING THE SPECIFICATION DID NOT Zero bits decode as perfectly good two-dimensional modes — pass mode is 0001, vertical-left-3 is 0000010 — so a decoder that does not stop at the padding goes on inventing rows that look like the last real one. fr-cerfa/cerfa_10701 gave 1 636 rows for a 208-row image that way, every surplus row with plausible ink on it. Both /EndOfBlock's marker and a plain "the rest is fill" check are now honoured. The fuzz target found the second: on a truncated fax the early return skipped the padding that makes the answer as long as /Rows promised, so two bytes of input gave one row where the caller had asked for four, with no error to say so. 6.7 million fuzz executions since, with none failing. WHAT IT CHANGES WHEN A PAGE IS DRAWN Through render, on the first page of all 1 633 real forms: 69 pages change. One goes from blank to drawn — fr-cerfa/cerfa_11818, 0 inked pixels to 28 989. Fourteen gain ink. **Fifty-one lose ink**, and that is the interesting number: those pages were not blank before, they were noise. 273 of the corpus's image masks carry an encoded filter — 236 CCITT, 9 JBIG2 — and render hands a mask to its stencil path before asking whether the bytes are still compressed, so a fax was being drawn as a stencil of its own compressed bytes. Decoding it replaces that noise with the picture, which has less ink in it because most of a form is white. The remaining 9 are a defect in render, filed separately. 100% statement coverage, go vet and -race clean, nine cross-compile targets, nothing outside the standard library. Co-Authored-By: Claude Opus 5 --- ccitt.go | 579 +++++++++++++++++++++++++++++++++++++++++++++++++ ccitt_test.go | 516 +++++++++++++++++++++++++++++++++++++++++++ filter.go | 11 +- filter_test.go | 11 +- 4 files changed, 1113 insertions(+), 4 deletions(-) create mode 100644 ccitt.go create mode 100644 ccitt_test.go diff --git a/ccitt.go b/ccitt.go new file mode 100644 index 0000000..386d032 --- /dev/null +++ b/ccitt.go @@ -0,0 +1,579 @@ +package reader + +import "fmt" + +// CCITT Group 3 and Group 4 fax decoding, as used by /CCITTFaxDecode. +// +// WHY THIS IS HERE +// +// A scanned form is a fax. 67 of the 1 633 real forms in the corpus — the +// eleven issuing bodies, not the vendor test suites — carry a CCITT-encoded +// image, 263 images between them, and 6 of their pages have nothing else on +// them at all. Those six pages drew entirely blank, and a blank page is the +// failure a reader notices before any other. +// +// WHY IT BELONGS IN THE READER RATHER THAN IN A RENDERER +// +// The other filters this package stops at — DCT, JPX, JBIG2 — carry an image +// with its own idea of how many components it has and how deep they are. This +// one does not: it produces bilevel samples, one bit a pixel, rows padded to a +// byte, and the stream dictionary says what they mean. That is a byte stream, +// so it is a filter, and every caller gets it rather than each writing its own. +// +// THE REFERENCE READ BEFORE WRITING THIS +// +// ITU-T Recommendations T.4 (Group 3) and T.6 (Group 4), by way of the tables +// and the changing-element algorithm in golang.org/x/image/ccitt — read, not +// imported, because this package has no dependencies outside the standard +// library and gains none. The code tables below were extracted mechanically +// from that package's gen.go rather than retyped, because a table of 218 +// variable-length codes transcribed by hand is a table with a mistake in it. +// +// The algorithm is the one T.6 Figure 1 describes. A row is decoded into one +// byte a pixel — 0xFF white, 0x00 black — and the row before it is the +// reference against which the two-dimensional modes are resolved: +// +// b1 b2 +// v v +// prev: BBBBBwwwwwBBBwwwww +// curr: BBBwwwwwBBBBBBwwww +// ^ ^ ^ +// a0 a1 a2 +// +// a0 is where the pen is; a1 the next colour change to its right on this row; +// b1 the first change on the row above that is to the right of a0 and of the +// opposite colour to it; b2 the next change after b1. Pass mode says a1 is at +// or beyond b2; horizontal mode reads two runs and ignores the row above; +// vertical mode puts a1 at b1 plus an offset between -3 and +3. + +// The two-dimensional modes of T.4 Table 1. +const ( + modePass = iota + modeH + modeV0 + modeVR1 + modeVR2 + modeVR3 + modeVL1 + modeVL2 + modeVL3 + modeExt +) + +// A ccittCode is one row of a code table: the value a code names, and the bits +// that name it, most significant first. +type ccittCode struct { + value int + bits string +} + +// ccittModeCodes is Table 1 of ITU-T T.4: the two-dimensional modes. +var ccittModeCodes = []ccittCode{ + {modePass, "0001"}, {modeH, "001"}, {modeV0, "1"}, {modeVR1, "011"}, + {modeVR2, "000011"}, {modeVR3, "0000011"}, {modeVL1, "010"}, {modeVL2, "000010"}, + {modeVL3, "0000010"}, {modeExt, "0000001"}, +} + +// ccittWhiteCodes is Tables 2 and 3 of ITU-T T.4 for a white run: the +// terminating codes 0 to 63, then the make-up codes in steps of 64. +var ccittWhiteCodes = []ccittCode{ + {0, "00110101"}, {1, "000111"}, {2, "0111"}, {3, "1000"}, + {4, "1011"}, {5, "1100"}, {6, "1110"}, {7, "1111"}, + {8, "10011"}, {9, "10100"}, {10, "00111"}, {11, "01000"}, + {12, "001000"}, {13, "000011"}, {14, "110100"}, {15, "110101"}, + {16, "101010"}, {17, "101011"}, {18, "0100111"}, {19, "0001100"}, + {20, "0001000"}, {21, "0010111"}, {22, "0000011"}, {23, "0000100"}, + {24, "0101000"}, {25, "0101011"}, {26, "0010011"}, {27, "0100100"}, + {28, "0011000"}, {29, "00000010"}, {30, "00000011"}, {31, "00011010"}, + {32, "00011011"}, {33, "00010010"}, {34, "00010011"}, {35, "00010100"}, + {36, "00010101"}, {37, "00010110"}, {38, "00010111"}, {39, "00101000"}, + {40, "00101001"}, {41, "00101010"}, {42, "00101011"}, {43, "00101100"}, + {44, "00101101"}, {45, "00000100"}, {46, "00000101"}, {47, "00001010"}, + {48, "00001011"}, {49, "01010010"}, {50, "01010011"}, {51, "01010100"}, + {52, "01010101"}, {53, "00100100"}, {54, "00100101"}, {55, "01011000"}, + {56, "01011001"}, {57, "01011010"}, {58, "01011011"}, {59, "01001010"}, + {60, "01001011"}, {61, "00110010"}, {62, "00110011"}, {63, "00110100"}, + {64, "11011"}, {128, "10010"}, {192, "010111"}, {256, "0110111"}, + {320, "00110110"}, {384, "00110111"}, {448, "01100100"}, {512, "01100101"}, + {576, "01101000"}, {640, "01100111"}, {704, "011001100"}, {768, "011001101"}, + {832, "011010010"}, {896, "011010011"}, {960, "011010100"}, {1024, "011010101"}, + {1088, "011010110"}, {1152, "011010111"}, {1216, "011011000"}, {1280, "011011001"}, + {1344, "011011010"}, {1408, "011011011"}, {1472, "010011000"}, {1536, "010011001"}, + {1600, "010011010"}, {1664, "011000"}, {1728, "010011011"}, {1792, "00000001000"}, + {1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"}, + {2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"}, + {2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"}, +} + +// ccittBlackCodes is Tables 2 and 3 of ITU-T T.4 for a black run. +var ccittBlackCodes = []ccittCode{ + {0, "0000110111"}, {1, "010"}, {2, "11"}, {3, "10"}, + {4, "011"}, {5, "0011"}, {6, "0010"}, {7, "00011"}, + {8, "000101"}, {9, "000100"}, {10, "0000100"}, {11, "0000101"}, + {12, "0000111"}, {13, "00000100"}, {14, "00000111"}, {15, "000011000"}, + {16, "0000010111"}, {17, "0000011000"}, {18, "0000001000"}, {19, "00001100111"}, + {20, "00001101000"}, {21, "00001101100"}, {22, "00000110111"}, {23, "00000101000"}, + {24, "00000010111"}, {25, "00000011000"}, {26, "000011001010"}, {27, "000011001011"}, + {28, "000011001100"}, {29, "000011001101"}, {30, "000001101000"}, {31, "000001101001"}, + {32, "000001101010"}, {33, "000001101011"}, {34, "000011010010"}, {35, "000011010011"}, + {36, "000011010100"}, {37, "000011010101"}, {38, "000011010110"}, {39, "000011010111"}, + {40, "000001101100"}, {41, "000001101101"}, {42, "000011011010"}, {43, "000011011011"}, + {44, "000001010100"}, {45, "000001010101"}, {46, "000001010110"}, {47, "000001010111"}, + {48, "000001100100"}, {49, "000001100101"}, {50, "000001010010"}, {51, "000001010011"}, + {52, "000000100100"}, {53, "000000110111"}, {54, "000000111000"}, {55, "000000100111"}, + {56, "000000101000"}, {57, "000001011000"}, {58, "000001011001"}, {59, "000000101011"}, + {60, "000000101100"}, {61, "000001011010"}, {62, "000001100110"}, {63, "000001100111"}, + {64, "0000001111"}, {128, "000011001000"}, {192, "000011001001"}, {256, "000001011011"}, + {320, "000000110011"}, {384, "000000110100"}, {448, "000000110101"}, {512, "0000001101100"}, + {576, "0000001101101"}, {640, "0000001001010"}, {704, "0000001001011"}, {768, "0000001001100"}, + {832, "0000001001101"}, {896, "0000001110010"}, {960, "0000001110011"}, {1024, "0000001110100"}, + {1088, "0000001110101"}, {1152, "0000001110110"}, {1216, "0000001110111"}, {1280, "0000001010010"}, + {1344, "0000001010011"}, {1408, "0000001010100"}, {1472, "0000001010101"}, {1536, "0000001011010"}, + {1600, "0000001011011"}, {1664, "0000001100100"}, {1728, "0000001100101"}, {1792, "00000001000"}, + {1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"}, + {2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"}, + {2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"}, +} + +// A ccittTable decodes one of the code tables. Codes are grouped by their +// length, which is the whole of the decoding: read a bit, lengthen the code by +// one, and ask whether a code of that length says anything. A prefix-free code +// cannot answer twice, so the first answer is the only one. +type ccittTable struct { + byLength []map[uint32]int +} + +// newCCITTTable groups a table's codes by length. The bits are read from the +// strings the tables are written in, so what the code says and what the +// specification says are the same text. +func newCCITTTable(codes []ccittCode) *ccittTable { + longest := 0 + for _, c := range codes { + if len(c.bits) > longest { + longest = len(c.bits) + } + } + t := &ccittTable{byLength: make([]map[uint32]int, longest+1)} + for _, c := range codes { + var v uint32 + for _, b := range c.bits { + v <<= 1 + if b == '1' { + v |= 1 + } + } + n := len(c.bits) + if t.byLength[n] == nil { + t.byLength[n] = map[uint32]int{} + } + t.byLength[n][v] = c.value + } + return t +} + +var ( + ccittModes = newCCITTTable(ccittModeCodes) + ccittWhite = newCCITTTable(ccittWhiteCodes) + ccittBlack = newCCITTTable(ccittBlackCodes) +) + +// A ccittBits reads one bit at a time, most significant first, which is the +// order a fax is written in. +type ccittBits struct { + data []byte + pos int // in bits +} + +func (b *ccittBits) next() (uint32, bool) { + if b.pos >= 8*len(b.data) { + return 0, false + } + bit := (b.data[b.pos/8] >> (7 - uint(b.pos%8))) & 1 + b.pos++ + return uint32(bit), true +} + +// align moves to the next byte boundary, which /EncodedByteAlign asks for at +// the start of every row. +func (b *ccittBits) align() { b.pos = (b.pos + 7) &^ 7 } + +func (b *ccittBits) exhausted() bool { return b.pos >= 8*len(b.data) } + +// restIsFill says whether nothing but zero bits remain. A fax is padded to a +// byte, and often to rather more than a byte, and zeros decode as perfectly +// good two-dimensional modes: pass mode is 0001 and vertical-left-3 is 0000010, +// so a decoder that does not stop here goes on inventing rows that look like +// the last real one. One French form gave 1 636 rows for a 208-row image that +// way, and every one of the surplus rows had plausible ink on it. +func (b *ccittBits) restIsFill() bool { + for i := b.pos; i < 8*len(b.data); i++ { + if b.data[i/8]&(1<<(7-uint(i%8))) != 0 { + return false + } + } + return true +} + +// read decodes one code. It returns false at the end of the data or on a code +// the table does not name — the caller decides which of those is fatal, since a +// truncated fax is common and worth showing as far as it goes. +func (t *ccittTable) read(b *ccittBits) (int, bool) { + var code uint32 + for n := 1; n < len(t.byLength); n++ { + bit, ok := b.next() + if !ok { + return 0, false + } + code = code<<1 | bit + if v, ok := t.byLength[n][code]; ok { + return v, true + } + } + return 0, false +} + +// skipEOL consumes an end-of-line code, 000000000001, if one is next, and any +// fill bits before it. A Group 3 fax may have one at the end of every row, may +// have none at all, and may have several together at the end; all three are +// found in the corpus, so none of them may be an error. +func (b *ccittBits) skipEOL() bool { + start := b.pos + zeros := 0 + for { + bit, ok := b.next() + if !ok { + b.pos = start + return false + } + if bit == 0 { + zeros++ + continue + } + if zeros >= 11 { + return true + } + b.pos = start + return false + } +} + +// ccittParams are the decode parameters of /CCITTFaxDecode, with the defaults +// the specification gives. +type ccittParams struct { + k int // <0 Group 4, 0 Group 3 one-dimensional, >0 Group 3 mixed + columns int // default 1728 + rows int // 0 means as many as the data holds + blackIs1 bool // false: a 0 bit is black + encodedByteAlign bool + endOfBlock bool // default true: the data ends with a block marker +} + +func ccittParamsOf(parm Dict, r Resolver) ccittParams { + return ccittParams{ + k: intParm(parm, "K", 0, r), + columns: intParm(parm, "Columns", 1728, r), + rows: intParm(parm, "Rows", 0, r), + blackIs1: boolParm(parm, "BlackIs1", false, r), + encodedByteAlign: boolParm(parm, "EncodedByteAlign", false, r), + endOfBlock: boolParm(parm, "EndOfBlock", true, r), + } +} + +// endOfBlockEOLs is how many end-of-line codes in a row mean the data has +// ended: two for Group 4, which has none between its rows, and six for Group 3, +// where every row may legitimately be followed by one. +func endOfBlockEOLs(k int) int { + if k < 0 { + return 2 + } + return 6 +} + +// maxCCITTPixels bounds what one image may decode to, the way maxDecodedSize +// bounds a Flate stream. A fax names its own width and height in the decode +// parameters, so a file can ask for an image of any size it likes; 2^28 pixels +// is more than four times A4 at 600 dots to the inch. +var maxCCITTPixels = 1 << 28 + +// ccittDecode turns a fax into bilevel samples: one bit a pixel, each row +// padded to a byte boundary, which is what every other image filter in this +// package produces and what a stream dictionary's /Width, /Height and +// /BitsPerComponent describe. +func ccittDecode(data []byte, p ccittParams) ([]byte, error) { + if p.columns <= 0 { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d columns", p.columns) + } + if p.rows < 0 { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d rows", p.rows) + } + if p.rows > 0 && p.columns > maxCCITTPixels/p.rows { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d by %d pixels, "+ + "past the limit of %d", p.columns, p.rows, maxCCITTPixels) + } + b := &ccittBits{data: data} + stride := (p.columns + 7) / 8 + curr := make([]byte, p.columns) + prev := []byte(nil) + out := []byte(nil) + for row := 0; p.rows == 0 || row < p.rows; row++ { + if p.rows == 0 && len(out)/stride*p.columns >= maxCCITTPixels { + return nil, fmt.Errorf("reader: /CCITTFaxDecode ran past %d pixels "+ + "without naming how many rows it has", maxCCITTPixels) + } + if p.encodedByteAlign { + b.align() + } + // A Group 3 row may be introduced by an end-of-line code. Consuming it + // here rather than after the row means a file that has them and a file + // that does not are read the same way — and counting them is how the + // end of the data announces itself, since the block marker is nothing + // but end-of-line codes in a row. + eols := 0 + for b.skipEOL() { + eols++ + } + if p.endOfBlock && eols >= endOfBlockEOLs(p.k) { + break + } + if b.exhausted() || b.restIsFill() { + break + } + twoDimensional := p.k < 0 + if p.k > 0 { + // In mixed mode a bit after the end-of-line says which the row is: + // 1 for one-dimensional, 0 for two. + // The data was checked for bits a moment ago, so this cannot + // fail; the value is taken rather than the error handled, because + // a branch that cannot be reached cannot be tested. + bit, _ := b.next() + twoDimensional = bit == 0 + } + if err := ccittRow(b, curr, prev, twoDimensional); err != nil { + if len(out) == 0 { + return nil, err + } + // A truncated fax is worth showing as far as it got: the rows + // already decoded are real, and refusing them turns a damaged + // scan into a blank page. Leaving the loop rather than returning + // here matters — the padding below is what makes the answer as + // long as /Rows promised, and a fuzz case of two bytes found that + // returning early handed the caller one row where it had asked + // for four, with no error to say so. + break + } + out = append(out, ccittPack(curr, stride, p.blackIs1)...) + if prev == nil { + prev = make([]byte, p.columns) + } + copy(prev, curr) + } + // A file that says how many rows it has gets that many, padded with white + // if the data ran out, so the samples match the /Height the dictionary + // promises rather than being short by a row. + if p.rows > 0 { + white := make([]byte, stride) + if !p.blackIs1 { + for i := range white { + white[i] = 0xFF + } + } + for len(out) < p.rows*stride { + out = append(out, white...) + } + } + return out, nil +} + +// ccittPack turns a row of one byte a pixel into one bit a pixel. With +// /BlackIs1 false — the default — a 0 bit is black, so white becomes a 1. +func ccittPack(row []byte, stride int, blackIs1 bool) []byte { + out := make([]byte, stride) + for x, v := range row { + white := v != 0 + if white != blackIs1 { + out[x/8] |= 1 << (7 - uint(x%8)) + } + } + return out +} + +// ccittRow decodes one row into curr, one byte a pixel: 0xFF white, 0x00 black. +func ccittRow(b *ccittBits, curr, prev []byte, twoDimensional bool) error { + for i := range curr { + curr[i] = 0 + } + at, white, first := 0, true, true + for at < len(curr) { + if !twoDimensional { + n, err := ccittRun(b, white) + if err != nil { + return err + } + if at+n > len(curr) { + return fmt.Errorf("reader: a fax row of %d pixels named %d", + len(curr), at+n) + } + fill(curr[at:at+n], white) + at += n + white = !white + first = false + continue + } + mode, ok := ccittModes.read(b) + if !ok { + return fmt.Errorf("reader: a fax row named no mode this reader knows") + } + switch mode { + case modePass: + // ccittFindB starts where the pen is and only walks forward, + // stopping at the end of the row, so b2 is always a pixel of this + // row at or after the pen. There is nothing to check. + b2 := ccittFindB(prev, curr, at, white, first, true) + fill(curr[at:b2], white) + at = b2 + case modeH: + // Two runs, of the current colour and then the other, neither of + // them looking at the row above. + for i := 0; i < 2; i++ { + n, err := ccittRun(b, white) + if err != nil { + return err + } + if at+n > len(curr) { + return fmt.Errorf("reader: a fax row of %d pixels named %d", + len(curr), at+n) + } + fill(curr[at:at+n], white) + at += n + white = !white + } + // Horizontal mode reads a pair, so the two flips above have put + // the colour back where it started; there is nothing to undo. + case modeV0, modeVR1, modeVR2, modeVR3, modeVL1, modeVL2, modeVL3: + a1 := ccittFindB(prev, curr, at, white, first, false) + ccittOffset(mode) + if a1 < at || a1 > len(curr) { + return fmt.Errorf("reader: a fax vertical mode named pixel %d of %d", + a1, len(curr)) + } + fill(curr[at:a1], white) + at = a1 + white = !white + default: + return fmt.Errorf("reader: a fax named an extension mode, which " + + "this reader does not decode") + } + first = false + } + return nil +} + +// ccittOffset is how far a vertical mode moves a1 from b1. +func ccittOffset(mode int) int { + switch mode { + case modeVR1: + return 1 + case modeVR2: + return 2 + case modeVR3: + return 3 + case modeVL1: + return -1 + case modeVL2: + return -2 + case modeVL3: + return -3 + } + return 0 +} + +// ccittRun reads one run length, which is a sequence of make-up codes ending in +// a terminating code below 64. +func ccittRun(b *ccittBits, white bool) (int, error) { + table := ccittBlack + if white { + table = ccittWhite + } + total := 0 + for { + n, ok := table.read(b) + if !ok { + return 0, fmt.Errorf("reader: a fax run named no length this reader knows") + } + total += n + if total > maxCCITTPixels { + return 0, fmt.Errorf("reader: a fax named a run of %d pixels", total) + } + if n <= 63 { + return total, nil + } + } +} + +// ccittFindB finds b1, or b2 when second is set: the changing elements on the +// row above, as T.6 Figure 1 defines them. +func ccittFindB(prev, curr []byte, at int, white, first, second bool) int { + // The row above the first row is implicitly all white, so it has no + // changing elements and both b values are at the end of the row. + if prev == nil { + return len(curr) + } + i := at + if first { + // a0 is implicitly one pixel before the row, on white. b1 is the first + // black pixel above; b2 the first white pixel after that. + for i < len(prev) && prev[i] != 0 { + i++ + } + if second { + for i < len(prev) && prev[i] == 0 { + i++ + } + } + return i + } + // Walk past the run above that is of the opposite colour to the pen, then + // past the run of the pen's own colour: what follows is b1. + opposite := byte(0xFF) + if white { + opposite = 0 + } + for i < len(prev) && prev[i] == opposite { + i++ + } + same := ^opposite + for i < len(prev) && prev[i] == same { + i++ + } + if second { + for i < len(prev) && prev[i] == opposite { + i++ + } + } + return i +} + +// boolParm reads a boolean decode parameter, falling back to its default. +func boolParm(parm Dict, key Name, def bool, r Resolver) bool { + if parm == nil { + return def + } + v, err := Resolve(parm.Get(key), r) + if err != nil { + return def + } + if b, ok := v.(Bool); ok { + return bool(b) + } + return def +} + +// fill paints a run of pixels, one byte each. +func fill(row []byte, white bool) { + v := byte(0) + if white { + v = 0xFF + } + for i := range row { + row[i] = v + } +} diff --git a/ccitt_test.go b/ccitt_test.go new file mode 100644 index 0000000..a95a588 --- /dev/null +++ b/ccitt_test.go @@ -0,0 +1,516 @@ +package reader + +import ( + "errors" + "strings" + "testing" +) + +// The tests here write a fax the way the specification does, as a string of +// '0' and '1', and read the result back as a string of 'W' and 'B'. Neither +// end of that is the decoder's own arithmetic, so a test that passes says the +// decoder agrees with the specification rather than with itself. + +// faxBits assembles a fax from names: "W4" and "B2" are runs, "V0" and "VR2" +// and "P" and "H" are modes, "EOL" is an end-of-line code, and a run of digits +// is written out as it stands. +func faxBits(t *testing.T, parts ...string) []byte { + t.Helper() + var sb strings.Builder + for _, p := range parts { + switch { + case p == "EOL": + sb.WriteString("000000000001") + case p == "P": + sb.WriteString(codeBits(t, ccittModeCodes, modePass)) + case p == "H": + sb.WriteString(codeBits(t, ccittModeCodes, modeH)) + case p == "EXT": + sb.WriteString(codeBits(t, ccittModeCodes, modeExt)) + case strings.HasPrefix(p, "V"): + sb.WriteString(codeBits(t, ccittModeCodes, modeNamed(t, p))) + case p[0] == 'W' || p[0] == 'B': + table := ccittWhiteCodes + if p[0] == 'B' { + table = ccittBlackCodes + } + sb.WriteString(codeBits(t, table, atoi(t, p[1:]))) + default: + for _, c := range p { + if c != '0' && c != '1' { + t.Fatalf("faxBits: %q is not a name or a run of bits", p) + } + } + sb.WriteString(p) + } + } + // Pad the last byte with zeros, which is what a fax does and what the + // decoder must not read as another row. + bits := sb.String() + out := make([]byte, (len(bits)+7)/8) + for i, c := range bits { + if c == '1' { + out[i/8] |= 1 << (7 - uint(i%8)) + } + } + return out +} + +func codeBits(t *testing.T, table []ccittCode, value int) string { + t.Helper() + for _, c := range table { + if c.value == value { + return c.bits + } + } + t.Fatalf("codeBits: no code for %d", value) + return "" +} + +func modeNamed(t *testing.T, s string) int { + t.Helper() + switch s { + case "V0": + return modeV0 + case "VR1": + return modeVR1 + case "VR2": + return modeVR2 + case "VR3": + return modeVR3 + case "VL1": + return modeVL1 + case "VL2": + return modeVL2 + case "VL3": + return modeVL3 + } + t.Fatalf("modeNamed: %q", s) + return 0 +} + +func atoi(t *testing.T, s string) int { + t.Helper() + n := 0 + for _, c := range s { + if c < '0' || c > '9' { + t.Fatalf("atoi: %q", s) + } + n = n*10 + int(c-'0') + } + return n +} + +// faxRows reads decoded samples back as 'W' and 'B' per pixel, one string a +// row. With /BlackIs1 false — the default — a 0 bit is black, which is the +// convention this asserts. +func faxRows(t *testing.T, data []byte, columns int, blackIs1 bool) []string { + t.Helper() + stride := (columns + 7) / 8 + var out []string + for i := 0; i+stride <= len(data); i += stride { + var sb strings.Builder + for x := 0; x < columns; x++ { + set := data[i+x/8]&(1<<(7-uint(x%8))) != 0 + if set == blackIs1 { + sb.WriteByte('B') + } else { + sb.WriteByte('W') + } + } + out = append(out, sb.String()) + } + return out +} + +func decodeFax(t *testing.T, data []byte, p ccittParams) []string { + t.Helper() + out, err := ccittDecode(data, p) + if err != nil { + t.Fatalf("ccittDecode: %v", err) + } + return faxRows(t, out, p.columns, p.blackIs1) +} + +func g4(columns int) ccittParams { + return ccittParams{k: -1, columns: columns, endOfBlock: true} +} + +func TestAGroup4RowInHorizontalMode(t *testing.T) { + // Horizontal mode reads two runs and pays no attention to the row above, + // so it is the one mode that can start a picture from nothing. + data := faxBits(t, "H", "W3", "B5") + got := decodeFax(t, data, g4(8)) + want := []string{"WWWBBBBB"} + if len(got) != 1 || got[0] != want[0] { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestAGroup4RowCopiesTheOneAboveIt(t *testing.T) { + // V0 says the colour changes exactly where it changed on the row above, + // which is how a fax of a form spends most of its bits. + data := faxBits(t, "H", "W3", "B5", "V0", "V0") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q, want two rows of WWWBBBBB", got) + } +} + +func TestTheVerticalModesMoveTheChangeSideways(t *testing.T) { + // Each vertical mode puts the change at b1 plus its own offset. Against a + // first row that changes at pixel 3, VR1 changes at 4 and VL1 at 2. + for _, tc := range []struct { + mode string + want string + }{ + {"V0", "WWWBBBBB"}, + {"VR1", "WWWWBBBB"}, + {"VR2", "WWWWWBBB"}, + {"VR3", "WWWWWWBB"}, + {"VL1", "WWBBBBBB"}, + {"VL2", "WBBBBBBB"}, + {"VL3", "BBBBBBBB"}, + } { + t.Run(tc.mode, func(t *testing.T) { + data := faxBits(t, "H", "W3", "B5", tc.mode, "V0") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 { + t.Fatalf("got %d rows, want 2: %q", len(got), got) + } + if got[1] != tc.want { + t.Errorf("%s gave %q, want %q", tc.mode, got[1], tc.want) + } + }) + } +} + +func TestGroup4PassMode(t *testing.T) { + // Pass mode says the white run under way reaches at least as far as b2: + // the black run on the row above is passed over rather than followed. + // Against a first row of three white and five black, b2 is the end of the + // row, so one pass mode fills the whole second row white and finishes it — + // which is why nothing follows it here. + data := faxBits(t, "H", "W3", "B5", "P") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[1] != "WWWWWWWW" { + t.Errorf("pass mode gave %q, want the black above passed over", got[1]) + } +} + +func TestAGroup3OneDimensionalRow(t *testing.T) { + // Group 3 with K = 0 is nothing but runs, white first, and a row may be + // followed by an end-of-line code or by nothing at all. + for _, tc := range []struct { + name string + parts []string + }{ + {"with end-of-line codes", []string{"W2", "B6", "EOL", "W4", "B4"}}, + {"without", []string{"W2", "B6", "W4", "B4"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := decodeFax(t, faxBits(t, tc.parts...), ccittParams{ + k: 0, columns: 8, endOfBlock: true}) + if len(got) != 2 || got[0] != "WWBBBBBB" || got[1] != "WWWWBBBB" { + t.Errorf("got %q", got) + } + }) + } +} + +func TestAGroup3MixedRowSaysWhichKindItIs(t *testing.T) { + // With K > 0 a bit after the end-of-line says whether the row that follows + // is one-dimensional (1) or two-dimensional (0). + data := faxBits(t, "EOL", "1", "W3", "B5", "EOL", "0", "V0", "V0") + got := decodeFax(t, data, ccittParams{k: 1, columns: 8, endOfBlock: true}) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q, want two rows of WWWBBBBB", got) + } +} + +func TestBlackIs1TurnsTheSamplesOver(t *testing.T) { + data := faxBits(t, "H", "W3", "B5") + p := g4(8) + p.blackIs1 = true + got := decodeFax(t, data, p) + if len(got) != 1 || got[0] != "WWWBBBBB" { + t.Errorf("got %q: the picture must not change, only the bits that say it", got) + } + // And the bits themselves are the other way round. + out, err := ccittDecode(data, p) + if err != nil { + t.Fatal(err) + } + if out[0] != 0b00011111 { + t.Errorf("first byte is %08b, want 00011111", out[0]) + } +} + +func TestEncodedByteAlignStartsEveryRowOnAByte(t *testing.T) { + // "H W3 B5" is 3 + 4 + 4 = 11 bits, so the second row would begin + // mid-byte. With the flag set the decoder skips to the boundary, and the + // bits in between are ignored rather than read as a mode. + first := "001" + codeBits(t, ccittWhiteCodes, 3) + codeBits(t, ccittBlackCodes, 5) + pad := strings.Repeat("0", 8-len(first)%8) + data := faxBits(t, first+pad, codeBits(t, ccittModeCodes, modeV0), + codeBits(t, ccittModeCodes, modeV0)) + p := g4(8) + p.encodedByteAlign = true + got := decodeFax(t, data, p) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q", got) + } +} + +func TestRowsGivenIsHonouredExactly(t *testing.T) { + // A file that says how many rows it has gets that many: short data is + // padded with white, so the samples match the /Height the dictionary + // promises rather than being a row short. + data := faxBits(t, "H", "W3", "B5") + p := g4(8) + p.rows = 3 + got := decodeFax(t, data, p) + if len(got) != 3 { + t.Fatalf("got %d rows, want 3: %q", len(got), got) + } + if got[1] != "WWWWWWWW" || got[2] != "WWWWWWWW" { + t.Errorf("the padding rows are %q and %q, want white", got[1], got[2]) + } +} + +func TestAMakeUpCodeAddsToTheRunBeforeIt(t *testing.T) { + // A run longer than 63 is a make-up code and then a terminating one. 64 + // plus 0 is the shortest way to say sixty-four. + data := faxBits(t, "H", "W64", "W0", "B1") + got := decodeFax(t, data, g4(65)) + if len(got) != 1 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[0] != strings.Repeat("W", 64)+"B" { + t.Errorf("got %q", got[0]) + } +} + +func TestATruncatedFaxComesBackAsFarAsItGot(t *testing.T) { + // A damaged scan is worth showing: the rows already decoded are real, and + // refusing them turns a form into a blank page. + data := faxBits(t, "H", "W3", "B5", "V0", "V0", "H", "W3") + got := decodeFax(t, data, g4(8)) + if len(got) < 2 { + t.Fatalf("got %d rows, want at least the two that decoded: %q", len(got), got) + } + if got[0] != "WWWBBBBB" { + t.Errorf("the first row came back as %q", got[0]) + } +} + +func TestAFaxThatSaysNothingSensibleIsRefused(t *testing.T) { + for _, tc := range []struct { + name string + data []byte + p ccittParams + }{ + {"no columns", nil, ccittParams{k: -1, columns: 0}}, + {"negative rows", nil, ccittParams{k: -1, columns: 8, rows: -1}}, + {"more pixels than anyone can want", + nil, ccittParams{k: -1, columns: 1 << 20, rows: 1 << 20}}, + {"an extension mode", faxBits(t, "EXT", "000"), g4(8)}, + {"a run longer than the row", faxBits(t, "H", "W63", "B63"), g4(8)}, + {"a mode this reader does not know", faxBits(t, "00000001"), g4(8)}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := ccittDecode(tc.data, tc.p); err == nil { + t.Error("no error") + } + }) + } +} + +func TestAFaxWithNoRowsAtAllDecodesToNothing(t *testing.T) { + // All-zero data is fill, not a picture. Zeros decode as perfectly good + // modes — pass mode is 0001 — so a decoder that does not stop here invents + // rows for as long as it is asked to. + out, err := ccittDecode(make([]byte, 64), g4(8)) + if err != nil { + t.Fatal(err) + } + if len(out) != 0 { + t.Errorf("%d bytes out of nothing but fill", len(out)) + } +} + +func TestTheBlockMarkerEndsTheData(t *testing.T) { + // Two end-of-line codes in a row are Group 4's way of saying it has + // finished; six are Group 3's. What follows them is not a row. + g4Data := faxBits(t, "H", "W3", "B5", "EOL", "EOL", "H", "W1", "B7") + if got := decodeFax(t, g4Data, g4(8)); len(got) != 1 { + t.Errorf("Group 4 gave %d rows past its block marker: %q", len(got), got) + } + g3 := []string{"W3", "B5"} + for i := 0; i < 6; i++ { + g3 = append(g3, "EOL") + } + g3 = append(g3, "W1", "B7") + got := decodeFax(t, faxBits(t, g3...), ccittParams{k: 0, columns: 8, endOfBlock: true}) + if len(got) != 1 { + t.Errorf("Group 3 gave %d rows past its block marker: %q", len(got), got) + } + if endOfBlockEOLs(-1) != 2 || endOfBlockEOLs(0) != 6 || endOfBlockEOLs(4) != 6 { + t.Error("endOfBlockEOLs disagrees with the two cases above") + } +} + +func TestEndOfBlockCanBeTurnedOff(t *testing.T) { + // A file may say its data has no block marker, in which case the codes + // that would be one are read as whatever they are. + p := g4(8) + p.endOfBlock = false + p.rows = 1 + got := decodeFax(t, faxBits(t, "H", "W3", "B5"), p) + if len(got) != 1 || got[0] != "WWWBBBBB" { + t.Errorf("got %q", got) + } +} + +func TestTheDecodeParametersComeFromTheDictionary(t *testing.T) { + got := ccittParamsOf(Dict{ + "K": Integer(-1), + "Columns": Integer(120), + "Rows": Integer(30), + "BlackIs1": Bool(true), + "EncodedByteAlign": Bool(true), + "EndOfBlock": Bool(false), + }, nil) + want := ccittParams{k: -1, columns: 120, rows: 30, blackIs1: true, + encodedByteAlign: true, endOfBlock: false} + if got != want { + t.Errorf("got %+v, want %+v", got, want) + } + // No dictionary at all means the defaults the specification gives. + def := ccittParamsOf(nil, nil) + if def != (ccittParams{k: 0, columns: 1728, rows: 0, blackIs1: false, + encodedByteAlign: false, endOfBlock: true}) { + t.Errorf("defaults are %+v", def) + } + // A parameter of the wrong kind is not a parameter. + if boolParm(Dict{"BlackIs1": Integer(1)}, "BlackIs1", false, nil) { + t.Error("an integer was read as a boolean") + } + if !boolParm(Dict{"BlackIs1": Ref{Num: 9}}, "BlackIs1", true, failingResolver) { + t.Error("a reference that will not resolve did not fall back to the default") + } +} + +// failingResolver refuses every reference, which is what a damaged file's +// cross-reference table amounts to. +func failingResolver(Ref) (Object, error) { + return nil, errors.New("reader: no such object") +} + +func TestAFaxReachesTheCallerThroughTheFilterChain(t *testing.T) { + // The point of decoding here rather than in a renderer: /CCITTFaxDecode is + // no longer handed back encoded, so every caller gets samples. + data := faxBits(t, "H", "W3", "B5") + out, img, err := Decode(Dict{ + "Filter": Name("CCITTFaxDecode"), + "DecodeParms": Dict{"K": Integer(-1), "Columns": Integer(8)}, + }, data, nil) + if err != nil { + t.Fatal(err) + } + if img != "" { + t.Errorf("the chain stopped at /%s", img) + } + if len(out) != 1 || out[0] != 0b11100000 { + t.Errorf("got %08b", out) + } + // The abbreviated name an inline image uses reaches the same code. + out2, _, err := Decode(Dict{ + "Filter": Name("CCF"), + "DecodeParms": Dict{"K": Integer(-1), "Columns": Integer(8)}, + }, data, nil) + if err != nil || len(out2) != 1 || out2[0] != out[0] { + t.Errorf("/CCF gave %08b, %v", out2, err) + } +} + +func FuzzCCITTDecode(f *testing.F) { + f.Add(faxBits(f2t(f), "H", "W3", "B5")) + f.Add([]byte{0, 0, 0, 0}) + f.Add([]byte{0xff, 0xff}) + f.Fuzz(func(t *testing.T, data []byte) { + for _, k := range []int{-1, 0, 1} { + for _, cols := range []int{1, 8, 1728} { + // No panic, and never more than the bound allows. + out, err := ccittDecode(data, ccittParams{k: k, columns: cols, + rows: 4, endOfBlock: true}) + if err == nil && len(out) != 4*((cols+7)/8) { + t.Fatalf("K=%d columns=%d gave %d bytes for 4 rows", + k, cols, len(out)) + } + } + } + }) +} + +// f2t lets the fuzz seed use the helpers, which take a *testing.T. +func f2t(f *testing.F) *testing.T { return &testing.T{} } + +func TestTheChangingElementsAreFoundPartWayThroughARow(t *testing.T) { + // b1 and b2 are found by walking the row above from where the pen is. The + // interesting case is a mode that is not the first of its row, because + // then the walk starts inside the row rather than before it — and the pen + // may be either colour by that point. + // + // The first row here changes four times, so the second row's four vertical + // modes each start their walk from a different place and from alternating + // colours; the third row's pass mode is preceded by a vertical one, so it + // too begins part way along. + data := faxBits(t, + "H", "W4", "B4", "H", "W4", "B4", // WWWWBBBBWWWWBBBB + "V0", "V0", "V0", "V0", // copied exactly + "V0", "P", "V0", // white to 4, then pass over the black above + ) + got := decodeFax(t, data, g4(16)) + if len(got) != 3 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[0] != "WWWWBBBBWWWWBBBB" { + t.Errorf("first row %q", got[0]) + } + if got[1] != "WWWWBBBBWWWWBBBB" { + t.Errorf("second row %q, want the first copied", got[1]) + } + if got[2] != "WWWWBBBBBBBBBBBB" { + t.Errorf("third row %q, want the black above passed over", got[2]) + } +} + +func TestAFaxIsBoundedEvenWhenItSaysNothingAboutItsHeight(t *testing.T) { + // A fax that does not say how many rows it has is decoded until the data + // runs out, so the bound is the only thing standing between a file and the + // heap. The bound is lowered here rather than a hundred megabytes of fax + // written, which is what maxDecodedSize does for Flate. + was := maxCCITTPixels + maxCCITTPixels = 16 + defer func() { maxCCITTPixels = was }() + + // Four rows of eight pixels is thirty-two, which is past sixteen. + data := faxBits(t, "H", "W3", "B5", "V0", "V0", "V0", "V0", "V0", "V0") + if _, err := ccittDecode(data, g4(8)); err == nil { + t.Error("no error from a fax past the bound") + } + // A run may be built up from make-up codes without ever terminating, and + // is bounded the same way. + maxCCITTPixels = 100 + long := []string{"H"} + for i := 0; i < 8; i++ { + long = append(long, "W64") + } + if _, err := ccittDecode(faxBits(t, long...), g4(1728)); err == nil { + t.Error("no error from a run past the bound") + } +} diff --git a/filter.go b/filter.go index a21da63..25e0bad 100644 --- a/filter.go +++ b/filter.go @@ -16,9 +16,16 @@ var maxDecodedSize int64 = 1 << 30 // ImageFilter reports whether a filter yields an encoded image rather than a // byte stream. [Decode] stops at one of these and hands the caller the still // encoded bytes, because decoding them is an image decoder's job. +// +// /CCITTFaxDecode was on this list and is not any more. It does not carry an +// image with its own idea of how many components it has and how deep they are, +// the way DCT and JPX do: it produces bilevel samples, one bit a pixel, and the +// stream dictionary says what they mean. That is a byte stream, so it is a +// filter — and decoding it here means every caller gets it rather than each +// writing its own. See ccitt.go. func ImageFilter(n Name) bool { switch n { - case "DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode": + case "DCTDecode", "DCT", "JPXDecode", "JBIG2Decode": return true } return false @@ -130,6 +137,8 @@ func applyFilter(f Name, data []byte, parm Dict, r Resolver) ([]byte, error) { return ascii85Decode(data) case "RunLengthDecode", "RL": return runLengthDecode(data) + case "CCITTFaxDecode", "CCF": + return ccittDecode(data, ccittParamsOf(parm, r)) } return nil, fmt.Errorf("reader: unsupported filter /%s", f) } diff --git a/filter_test.go b/filter_test.go index 8196b1a..7b9f698 100644 --- a/filter_test.go +++ b/filter_test.go @@ -41,13 +41,18 @@ func rawDeflateBytes(t *testing.T, data []byte) []byte { } func TestImageFilter(t *testing.T) { - for _, n := range []Name{"DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode"} { + for _, n := range []Name{"DCTDecode", "DCT", "JPXDecode", "JBIG2Decode"} { if !ImageFilter(n) { t.Errorf("ImageFilter(%s) = false", n) } } - if ImageFilter("FlateDecode") { - t.Error("ImageFilter(FlateDecode) = true") + // A fax is not one of them any more: it decodes to bilevel samples, which + // is a byte stream, so the filter chain finishes it rather than handing it + // back encoded. See ccitt.go. + for _, n := range []Name{"FlateDecode", "CCITTFaxDecode", "CCF"} { + if ImageFilter(n) { + t.Errorf("ImageFilter(%s) = true", n) + } } }