diff --git a/ccitt.go b/ccitt.go new file mode 100644 index 0000000..386d032 --- /dev/null +++ b/ccitt.go @@ -0,0 +1,579 @@ +package reader + +import "fmt" + +// CCITT Group 3 and Group 4 fax decoding, as used by /CCITTFaxDecode. +// +// WHY THIS IS HERE +// +// A scanned form is a fax. 67 of the 1 633 real forms in the corpus — the +// eleven issuing bodies, not the vendor test suites — carry a CCITT-encoded +// image, 263 images between them, and 6 of their pages have nothing else on +// them at all. Those six pages drew entirely blank, and a blank page is the +// failure a reader notices before any other. +// +// WHY IT BELONGS IN THE READER RATHER THAN IN A RENDERER +// +// The other filters this package stops at — DCT, JPX, JBIG2 — carry an image +// with its own idea of how many components it has and how deep they are. This +// one does not: it produces bilevel samples, one bit a pixel, rows padded to a +// byte, and the stream dictionary says what they mean. That is a byte stream, +// so it is a filter, and every caller gets it rather than each writing its own. +// +// THE REFERENCE READ BEFORE WRITING THIS +// +// ITU-T Recommendations T.4 (Group 3) and T.6 (Group 4), by way of the tables +// and the changing-element algorithm in golang.org/x/image/ccitt — read, not +// imported, because this package has no dependencies outside the standard +// library and gains none. The code tables below were extracted mechanically +// from that package's gen.go rather than retyped, because a table of 218 +// variable-length codes transcribed by hand is a table with a mistake in it. +// +// The algorithm is the one T.6 Figure 1 describes. A row is decoded into one +// byte a pixel — 0xFF white, 0x00 black — and the row before it is the +// reference against which the two-dimensional modes are resolved: +// +// b1 b2 +// v v +// prev: BBBBBwwwwwBBBwwwww +// curr: BBBwwwwwBBBBBBwwww +// ^ ^ ^ +// a0 a1 a2 +// +// a0 is where the pen is; a1 the next colour change to its right on this row; +// b1 the first change on the row above that is to the right of a0 and of the +// opposite colour to it; b2 the next change after b1. Pass mode says a1 is at +// or beyond b2; horizontal mode reads two runs and ignores the row above; +// vertical mode puts a1 at b1 plus an offset between -3 and +3. + +// The two-dimensional modes of T.4 Table 1. +const ( + modePass = iota + modeH + modeV0 + modeVR1 + modeVR2 + modeVR3 + modeVL1 + modeVL2 + modeVL3 + modeExt +) + +// A ccittCode is one row of a code table: the value a code names, and the bits +// that name it, most significant first. +type ccittCode struct { + value int + bits string +} + +// ccittModeCodes is Table 1 of ITU-T T.4: the two-dimensional modes. +var ccittModeCodes = []ccittCode{ + {modePass, "0001"}, {modeH, "001"}, {modeV0, "1"}, {modeVR1, "011"}, + {modeVR2, "000011"}, {modeVR3, "0000011"}, {modeVL1, "010"}, {modeVL2, "000010"}, + {modeVL3, "0000010"}, {modeExt, "0000001"}, +} + +// ccittWhiteCodes is Tables 2 and 3 of ITU-T T.4 for a white run: the +// terminating codes 0 to 63, then the make-up codes in steps of 64. +var ccittWhiteCodes = []ccittCode{ + {0, "00110101"}, {1, "000111"}, {2, "0111"}, {3, "1000"}, + {4, "1011"}, {5, "1100"}, {6, "1110"}, {7, "1111"}, + {8, "10011"}, {9, "10100"}, {10, "00111"}, {11, "01000"}, + {12, "001000"}, {13, "000011"}, {14, "110100"}, {15, "110101"}, + {16, "101010"}, {17, "101011"}, {18, "0100111"}, {19, "0001100"}, + {20, "0001000"}, {21, "0010111"}, {22, "0000011"}, {23, "0000100"}, + {24, "0101000"}, {25, "0101011"}, {26, "0010011"}, {27, "0100100"}, + {28, "0011000"}, {29, "00000010"}, {30, "00000011"}, {31, "00011010"}, + {32, "00011011"}, {33, "00010010"}, {34, "00010011"}, {35, "00010100"}, + {36, "00010101"}, {37, "00010110"}, {38, "00010111"}, {39, "00101000"}, + {40, "00101001"}, {41, "00101010"}, {42, "00101011"}, {43, "00101100"}, + {44, "00101101"}, {45, "00000100"}, {46, "00000101"}, {47, "00001010"}, + {48, "00001011"}, {49, "01010010"}, {50, "01010011"}, {51, "01010100"}, + {52, "01010101"}, {53, "00100100"}, {54, "00100101"}, {55, "01011000"}, + {56, "01011001"}, {57, "01011010"}, {58, "01011011"}, {59, "01001010"}, + {60, "01001011"}, {61, "00110010"}, {62, "00110011"}, {63, "00110100"}, + {64, "11011"}, {128, "10010"}, {192, "010111"}, {256, "0110111"}, + {320, "00110110"}, {384, "00110111"}, {448, "01100100"}, {512, "01100101"}, + {576, "01101000"}, {640, "01100111"}, {704, "011001100"}, {768, "011001101"}, + {832, "011010010"}, {896, "011010011"}, {960, "011010100"}, {1024, "011010101"}, + {1088, "011010110"}, {1152, "011010111"}, {1216, "011011000"}, {1280, "011011001"}, + {1344, "011011010"}, {1408, "011011011"}, {1472, "010011000"}, {1536, "010011001"}, + {1600, "010011010"}, {1664, "011000"}, {1728, "010011011"}, {1792, "00000001000"}, + {1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"}, + {2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"}, + {2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"}, +} + +// ccittBlackCodes is Tables 2 and 3 of ITU-T T.4 for a black run. +var ccittBlackCodes = []ccittCode{ + {0, "0000110111"}, {1, "010"}, {2, "11"}, {3, "10"}, + {4, "011"}, {5, "0011"}, {6, "0010"}, {7, "00011"}, + {8, "000101"}, {9, "000100"}, {10, "0000100"}, {11, "0000101"}, + {12, "0000111"}, {13, "00000100"}, {14, "00000111"}, {15, "000011000"}, + {16, "0000010111"}, {17, "0000011000"}, {18, "0000001000"}, {19, "00001100111"}, + {20, "00001101000"}, {21, "00001101100"}, {22, "00000110111"}, {23, "00000101000"}, + {24, "00000010111"}, {25, "00000011000"}, {26, "000011001010"}, {27, "000011001011"}, + {28, "000011001100"}, {29, "000011001101"}, {30, "000001101000"}, {31, "000001101001"}, + {32, "000001101010"}, {33, "000001101011"}, {34, "000011010010"}, {35, "000011010011"}, + {36, "000011010100"}, {37, "000011010101"}, {38, "000011010110"}, {39, "000011010111"}, + {40, "000001101100"}, {41, "000001101101"}, {42, "000011011010"}, {43, "000011011011"}, + {44, "000001010100"}, {45, "000001010101"}, {46, "000001010110"}, {47, "000001010111"}, + {48, "000001100100"}, {49, "000001100101"}, {50, "000001010010"}, {51, "000001010011"}, + {52, "000000100100"}, {53, "000000110111"}, {54, "000000111000"}, {55, "000000100111"}, + {56, "000000101000"}, {57, "000001011000"}, {58, "000001011001"}, {59, "000000101011"}, + {60, "000000101100"}, {61, "000001011010"}, {62, "000001100110"}, {63, "000001100111"}, + {64, "0000001111"}, {128, "000011001000"}, {192, "000011001001"}, {256, "000001011011"}, + {320, "000000110011"}, {384, "000000110100"}, {448, "000000110101"}, {512, "0000001101100"}, + {576, "0000001101101"}, {640, "0000001001010"}, {704, "0000001001011"}, {768, "0000001001100"}, + {832, "0000001001101"}, {896, "0000001110010"}, {960, "0000001110011"}, {1024, "0000001110100"}, + {1088, "0000001110101"}, {1152, "0000001110110"}, {1216, "0000001110111"}, {1280, "0000001010010"}, + {1344, "0000001010011"}, {1408, "0000001010100"}, {1472, "0000001010101"}, {1536, "0000001011010"}, + {1600, "0000001011011"}, {1664, "0000001100100"}, {1728, "0000001100101"}, {1792, "00000001000"}, + {1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"}, + {2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"}, + {2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"}, +} + +// A ccittTable decodes one of the code tables. Codes are grouped by their +// length, which is the whole of the decoding: read a bit, lengthen the code by +// one, and ask whether a code of that length says anything. A prefix-free code +// cannot answer twice, so the first answer is the only one. +type ccittTable struct { + byLength []map[uint32]int +} + +// newCCITTTable groups a table's codes by length. The bits are read from the +// strings the tables are written in, so what the code says and what the +// specification says are the same text. +func newCCITTTable(codes []ccittCode) *ccittTable { + longest := 0 + for _, c := range codes { + if len(c.bits) > longest { + longest = len(c.bits) + } + } + t := &ccittTable{byLength: make([]map[uint32]int, longest+1)} + for _, c := range codes { + var v uint32 + for _, b := range c.bits { + v <<= 1 + if b == '1' { + v |= 1 + } + } + n := len(c.bits) + if t.byLength[n] == nil { + t.byLength[n] = map[uint32]int{} + } + t.byLength[n][v] = c.value + } + return t +} + +var ( + ccittModes = newCCITTTable(ccittModeCodes) + ccittWhite = newCCITTTable(ccittWhiteCodes) + ccittBlack = newCCITTTable(ccittBlackCodes) +) + +// A ccittBits reads one bit at a time, most significant first, which is the +// order a fax is written in. +type ccittBits struct { + data []byte + pos int // in bits +} + +func (b *ccittBits) next() (uint32, bool) { + if b.pos >= 8*len(b.data) { + return 0, false + } + bit := (b.data[b.pos/8] >> (7 - uint(b.pos%8))) & 1 + b.pos++ + return uint32(bit), true +} + +// align moves to the next byte boundary, which /EncodedByteAlign asks for at +// the start of every row. +func (b *ccittBits) align() { b.pos = (b.pos + 7) &^ 7 } + +func (b *ccittBits) exhausted() bool { return b.pos >= 8*len(b.data) } + +// restIsFill says whether nothing but zero bits remain. A fax is padded to a +// byte, and often to rather more than a byte, and zeros decode as perfectly +// good two-dimensional modes: pass mode is 0001 and vertical-left-3 is 0000010, +// so a decoder that does not stop here goes on inventing rows that look like +// the last real one. One French form gave 1 636 rows for a 208-row image that +// way, and every one of the surplus rows had plausible ink on it. +func (b *ccittBits) restIsFill() bool { + for i := b.pos; i < 8*len(b.data); i++ { + if b.data[i/8]&(1<<(7-uint(i%8))) != 0 { + return false + } + } + return true +} + +// read decodes one code. It returns false at the end of the data or on a code +// the table does not name — the caller decides which of those is fatal, since a +// truncated fax is common and worth showing as far as it goes. +func (t *ccittTable) read(b *ccittBits) (int, bool) { + var code uint32 + for n := 1; n < len(t.byLength); n++ { + bit, ok := b.next() + if !ok { + return 0, false + } + code = code<<1 | bit + if v, ok := t.byLength[n][code]; ok { + return v, true + } + } + return 0, false +} + +// skipEOL consumes an end-of-line code, 000000000001, if one is next, and any +// fill bits before it. A Group 3 fax may have one at the end of every row, may +// have none at all, and may have several together at the end; all three are +// found in the corpus, so none of them may be an error. +func (b *ccittBits) skipEOL() bool { + start := b.pos + zeros := 0 + for { + bit, ok := b.next() + if !ok { + b.pos = start + return false + } + if bit == 0 { + zeros++ + continue + } + if zeros >= 11 { + return true + } + b.pos = start + return false + } +} + +// ccittParams are the decode parameters of /CCITTFaxDecode, with the defaults +// the specification gives. +type ccittParams struct { + k int // <0 Group 4, 0 Group 3 one-dimensional, >0 Group 3 mixed + columns int // default 1728 + rows int // 0 means as many as the data holds + blackIs1 bool // false: a 0 bit is black + encodedByteAlign bool + endOfBlock bool // default true: the data ends with a block marker +} + +func ccittParamsOf(parm Dict, r Resolver) ccittParams { + return ccittParams{ + k: intParm(parm, "K", 0, r), + columns: intParm(parm, "Columns", 1728, r), + rows: intParm(parm, "Rows", 0, r), + blackIs1: boolParm(parm, "BlackIs1", false, r), + encodedByteAlign: boolParm(parm, "EncodedByteAlign", false, r), + endOfBlock: boolParm(parm, "EndOfBlock", true, r), + } +} + +// endOfBlockEOLs is how many end-of-line codes in a row mean the data has +// ended: two for Group 4, which has none between its rows, and six for Group 3, +// where every row may legitimately be followed by one. +func endOfBlockEOLs(k int) int { + if k < 0 { + return 2 + } + return 6 +} + +// maxCCITTPixels bounds what one image may decode to, the way maxDecodedSize +// bounds a Flate stream. A fax names its own width and height in the decode +// parameters, so a file can ask for an image of any size it likes; 2^28 pixels +// is more than four times A4 at 600 dots to the inch. +var maxCCITTPixels = 1 << 28 + +// ccittDecode turns a fax into bilevel samples: one bit a pixel, each row +// padded to a byte boundary, which is what every other image filter in this +// package produces and what a stream dictionary's /Width, /Height and +// /BitsPerComponent describe. +func ccittDecode(data []byte, p ccittParams) ([]byte, error) { + if p.columns <= 0 { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d columns", p.columns) + } + if p.rows < 0 { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d rows", p.rows) + } + if p.rows > 0 && p.columns > maxCCITTPixels/p.rows { + return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d by %d pixels, "+ + "past the limit of %d", p.columns, p.rows, maxCCITTPixels) + } + b := &ccittBits{data: data} + stride := (p.columns + 7) / 8 + curr := make([]byte, p.columns) + prev := []byte(nil) + out := []byte(nil) + for row := 0; p.rows == 0 || row < p.rows; row++ { + if p.rows == 0 && len(out)/stride*p.columns >= maxCCITTPixels { + return nil, fmt.Errorf("reader: /CCITTFaxDecode ran past %d pixels "+ + "without naming how many rows it has", maxCCITTPixels) + } + if p.encodedByteAlign { + b.align() + } + // A Group 3 row may be introduced by an end-of-line code. Consuming it + // here rather than after the row means a file that has them and a file + // that does not are read the same way — and counting them is how the + // end of the data announces itself, since the block marker is nothing + // but end-of-line codes in a row. + eols := 0 + for b.skipEOL() { + eols++ + } + if p.endOfBlock && eols >= endOfBlockEOLs(p.k) { + break + } + if b.exhausted() || b.restIsFill() { + break + } + twoDimensional := p.k < 0 + if p.k > 0 { + // In mixed mode a bit after the end-of-line says which the row is: + // 1 for one-dimensional, 0 for two. + // The data was checked for bits a moment ago, so this cannot + // fail; the value is taken rather than the error handled, because + // a branch that cannot be reached cannot be tested. + bit, _ := b.next() + twoDimensional = bit == 0 + } + if err := ccittRow(b, curr, prev, twoDimensional); err != nil { + if len(out) == 0 { + return nil, err + } + // A truncated fax is worth showing as far as it got: the rows + // already decoded are real, and refusing them turns a damaged + // scan into a blank page. Leaving the loop rather than returning + // here matters — the padding below is what makes the answer as + // long as /Rows promised, and a fuzz case of two bytes found that + // returning early handed the caller one row where it had asked + // for four, with no error to say so. + break + } + out = append(out, ccittPack(curr, stride, p.blackIs1)...) + if prev == nil { + prev = make([]byte, p.columns) + } + copy(prev, curr) + } + // A file that says how many rows it has gets that many, padded with white + // if the data ran out, so the samples match the /Height the dictionary + // promises rather than being short by a row. + if p.rows > 0 { + white := make([]byte, stride) + if !p.blackIs1 { + for i := range white { + white[i] = 0xFF + } + } + for len(out) < p.rows*stride { + out = append(out, white...) + } + } + return out, nil +} + +// ccittPack turns a row of one byte a pixel into one bit a pixel. With +// /BlackIs1 false — the default — a 0 bit is black, so white becomes a 1. +func ccittPack(row []byte, stride int, blackIs1 bool) []byte { + out := make([]byte, stride) + for x, v := range row { + white := v != 0 + if white != blackIs1 { + out[x/8] |= 1 << (7 - uint(x%8)) + } + } + return out +} + +// ccittRow decodes one row into curr, one byte a pixel: 0xFF white, 0x00 black. +func ccittRow(b *ccittBits, curr, prev []byte, twoDimensional bool) error { + for i := range curr { + curr[i] = 0 + } + at, white, first := 0, true, true + for at < len(curr) { + if !twoDimensional { + n, err := ccittRun(b, white) + if err != nil { + return err + } + if at+n > len(curr) { + return fmt.Errorf("reader: a fax row of %d pixels named %d", + len(curr), at+n) + } + fill(curr[at:at+n], white) + at += n + white = !white + first = false + continue + } + mode, ok := ccittModes.read(b) + if !ok { + return fmt.Errorf("reader: a fax row named no mode this reader knows") + } + switch mode { + case modePass: + // ccittFindB starts where the pen is and only walks forward, + // stopping at the end of the row, so b2 is always a pixel of this + // row at or after the pen. There is nothing to check. + b2 := ccittFindB(prev, curr, at, white, first, true) + fill(curr[at:b2], white) + at = b2 + case modeH: + // Two runs, of the current colour and then the other, neither of + // them looking at the row above. + for i := 0; i < 2; i++ { + n, err := ccittRun(b, white) + if err != nil { + return err + } + if at+n > len(curr) { + return fmt.Errorf("reader: a fax row of %d pixels named %d", + len(curr), at+n) + } + fill(curr[at:at+n], white) + at += n + white = !white + } + // Horizontal mode reads a pair, so the two flips above have put + // the colour back where it started; there is nothing to undo. + case modeV0, modeVR1, modeVR2, modeVR3, modeVL1, modeVL2, modeVL3: + a1 := ccittFindB(prev, curr, at, white, first, false) + ccittOffset(mode) + if a1 < at || a1 > len(curr) { + return fmt.Errorf("reader: a fax vertical mode named pixel %d of %d", + a1, len(curr)) + } + fill(curr[at:a1], white) + at = a1 + white = !white + default: + return fmt.Errorf("reader: a fax named an extension mode, which " + + "this reader does not decode") + } + first = false + } + return nil +} + +// ccittOffset is how far a vertical mode moves a1 from b1. +func ccittOffset(mode int) int { + switch mode { + case modeVR1: + return 1 + case modeVR2: + return 2 + case modeVR3: + return 3 + case modeVL1: + return -1 + case modeVL2: + return -2 + case modeVL3: + return -3 + } + return 0 +} + +// ccittRun reads one run length, which is a sequence of make-up codes ending in +// a terminating code below 64. +func ccittRun(b *ccittBits, white bool) (int, error) { + table := ccittBlack + if white { + table = ccittWhite + } + total := 0 + for { + n, ok := table.read(b) + if !ok { + return 0, fmt.Errorf("reader: a fax run named no length this reader knows") + } + total += n + if total > maxCCITTPixels { + return 0, fmt.Errorf("reader: a fax named a run of %d pixels", total) + } + if n <= 63 { + return total, nil + } + } +} + +// ccittFindB finds b1, or b2 when second is set: the changing elements on the +// row above, as T.6 Figure 1 defines them. +func ccittFindB(prev, curr []byte, at int, white, first, second bool) int { + // The row above the first row is implicitly all white, so it has no + // changing elements and both b values are at the end of the row. + if prev == nil { + return len(curr) + } + i := at + if first { + // a0 is implicitly one pixel before the row, on white. b1 is the first + // black pixel above; b2 the first white pixel after that. + for i < len(prev) && prev[i] != 0 { + i++ + } + if second { + for i < len(prev) && prev[i] == 0 { + i++ + } + } + return i + } + // Walk past the run above that is of the opposite colour to the pen, then + // past the run of the pen's own colour: what follows is b1. + opposite := byte(0xFF) + if white { + opposite = 0 + } + for i < len(prev) && prev[i] == opposite { + i++ + } + same := ^opposite + for i < len(prev) && prev[i] == same { + i++ + } + if second { + for i < len(prev) && prev[i] == opposite { + i++ + } + } + return i +} + +// boolParm reads a boolean decode parameter, falling back to its default. +func boolParm(parm Dict, key Name, def bool, r Resolver) bool { + if parm == nil { + return def + } + v, err := Resolve(parm.Get(key), r) + if err != nil { + return def + } + if b, ok := v.(Bool); ok { + return bool(b) + } + return def +} + +// fill paints a run of pixels, one byte each. +func fill(row []byte, white bool) { + v := byte(0) + if white { + v = 0xFF + } + for i := range row { + row[i] = v + } +} diff --git a/ccitt_test.go b/ccitt_test.go new file mode 100644 index 0000000..a95a588 --- /dev/null +++ b/ccitt_test.go @@ -0,0 +1,516 @@ +package reader + +import ( + "errors" + "strings" + "testing" +) + +// The tests here write a fax the way the specification does, as a string of +// '0' and '1', and read the result back as a string of 'W' and 'B'. Neither +// end of that is the decoder's own arithmetic, so a test that passes says the +// decoder agrees with the specification rather than with itself. + +// faxBits assembles a fax from names: "W4" and "B2" are runs, "V0" and "VR2" +// and "P" and "H" are modes, "EOL" is an end-of-line code, and a run of digits +// is written out as it stands. +func faxBits(t *testing.T, parts ...string) []byte { + t.Helper() + var sb strings.Builder + for _, p := range parts { + switch { + case p == "EOL": + sb.WriteString("000000000001") + case p == "P": + sb.WriteString(codeBits(t, ccittModeCodes, modePass)) + case p == "H": + sb.WriteString(codeBits(t, ccittModeCodes, modeH)) + case p == "EXT": + sb.WriteString(codeBits(t, ccittModeCodes, modeExt)) + case strings.HasPrefix(p, "V"): + sb.WriteString(codeBits(t, ccittModeCodes, modeNamed(t, p))) + case p[0] == 'W' || p[0] == 'B': + table := ccittWhiteCodes + if p[0] == 'B' { + table = ccittBlackCodes + } + sb.WriteString(codeBits(t, table, atoi(t, p[1:]))) + default: + for _, c := range p { + if c != '0' && c != '1' { + t.Fatalf("faxBits: %q is not a name or a run of bits", p) + } + } + sb.WriteString(p) + } + } + // Pad the last byte with zeros, which is what a fax does and what the + // decoder must not read as another row. + bits := sb.String() + out := make([]byte, (len(bits)+7)/8) + for i, c := range bits { + if c == '1' { + out[i/8] |= 1 << (7 - uint(i%8)) + } + } + return out +} + +func codeBits(t *testing.T, table []ccittCode, value int) string { + t.Helper() + for _, c := range table { + if c.value == value { + return c.bits + } + } + t.Fatalf("codeBits: no code for %d", value) + return "" +} + +func modeNamed(t *testing.T, s string) int { + t.Helper() + switch s { + case "V0": + return modeV0 + case "VR1": + return modeVR1 + case "VR2": + return modeVR2 + case "VR3": + return modeVR3 + case "VL1": + return modeVL1 + case "VL2": + return modeVL2 + case "VL3": + return modeVL3 + } + t.Fatalf("modeNamed: %q", s) + return 0 +} + +func atoi(t *testing.T, s string) int { + t.Helper() + n := 0 + for _, c := range s { + if c < '0' || c > '9' { + t.Fatalf("atoi: %q", s) + } + n = n*10 + int(c-'0') + } + return n +} + +// faxRows reads decoded samples back as 'W' and 'B' per pixel, one string a +// row. With /BlackIs1 false — the default — a 0 bit is black, which is the +// convention this asserts. +func faxRows(t *testing.T, data []byte, columns int, blackIs1 bool) []string { + t.Helper() + stride := (columns + 7) / 8 + var out []string + for i := 0; i+stride <= len(data); i += stride { + var sb strings.Builder + for x := 0; x < columns; x++ { + set := data[i+x/8]&(1<<(7-uint(x%8))) != 0 + if set == blackIs1 { + sb.WriteByte('B') + } else { + sb.WriteByte('W') + } + } + out = append(out, sb.String()) + } + return out +} + +func decodeFax(t *testing.T, data []byte, p ccittParams) []string { + t.Helper() + out, err := ccittDecode(data, p) + if err != nil { + t.Fatalf("ccittDecode: %v", err) + } + return faxRows(t, out, p.columns, p.blackIs1) +} + +func g4(columns int) ccittParams { + return ccittParams{k: -1, columns: columns, endOfBlock: true} +} + +func TestAGroup4RowInHorizontalMode(t *testing.T) { + // Horizontal mode reads two runs and pays no attention to the row above, + // so it is the one mode that can start a picture from nothing. + data := faxBits(t, "H", "W3", "B5") + got := decodeFax(t, data, g4(8)) + want := []string{"WWWBBBBB"} + if len(got) != 1 || got[0] != want[0] { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestAGroup4RowCopiesTheOneAboveIt(t *testing.T) { + // V0 says the colour changes exactly where it changed on the row above, + // which is how a fax of a form spends most of its bits. + data := faxBits(t, "H", "W3", "B5", "V0", "V0") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q, want two rows of WWWBBBBB", got) + } +} + +func TestTheVerticalModesMoveTheChangeSideways(t *testing.T) { + // Each vertical mode puts the change at b1 plus its own offset. Against a + // first row that changes at pixel 3, VR1 changes at 4 and VL1 at 2. + for _, tc := range []struct { + mode string + want string + }{ + {"V0", "WWWBBBBB"}, + {"VR1", "WWWWBBBB"}, + {"VR2", "WWWWWBBB"}, + {"VR3", "WWWWWWBB"}, + {"VL1", "WWBBBBBB"}, + {"VL2", "WBBBBBBB"}, + {"VL3", "BBBBBBBB"}, + } { + t.Run(tc.mode, func(t *testing.T) { + data := faxBits(t, "H", "W3", "B5", tc.mode, "V0") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 { + t.Fatalf("got %d rows, want 2: %q", len(got), got) + } + if got[1] != tc.want { + t.Errorf("%s gave %q, want %q", tc.mode, got[1], tc.want) + } + }) + } +} + +func TestGroup4PassMode(t *testing.T) { + // Pass mode says the white run under way reaches at least as far as b2: + // the black run on the row above is passed over rather than followed. + // Against a first row of three white and five black, b2 is the end of the + // row, so one pass mode fills the whole second row white and finishes it — + // which is why nothing follows it here. + data := faxBits(t, "H", "W3", "B5", "P") + got := decodeFax(t, data, g4(8)) + if len(got) != 2 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[1] != "WWWWWWWW" { + t.Errorf("pass mode gave %q, want the black above passed over", got[1]) + } +} + +func TestAGroup3OneDimensionalRow(t *testing.T) { + // Group 3 with K = 0 is nothing but runs, white first, and a row may be + // followed by an end-of-line code or by nothing at all. + for _, tc := range []struct { + name string + parts []string + }{ + {"with end-of-line codes", []string{"W2", "B6", "EOL", "W4", "B4"}}, + {"without", []string{"W2", "B6", "W4", "B4"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := decodeFax(t, faxBits(t, tc.parts...), ccittParams{ + k: 0, columns: 8, endOfBlock: true}) + if len(got) != 2 || got[0] != "WWBBBBBB" || got[1] != "WWWWBBBB" { + t.Errorf("got %q", got) + } + }) + } +} + +func TestAGroup3MixedRowSaysWhichKindItIs(t *testing.T) { + // With K > 0 a bit after the end-of-line says whether the row that follows + // is one-dimensional (1) or two-dimensional (0). + data := faxBits(t, "EOL", "1", "W3", "B5", "EOL", "0", "V0", "V0") + got := decodeFax(t, data, ccittParams{k: 1, columns: 8, endOfBlock: true}) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q, want two rows of WWWBBBBB", got) + } +} + +func TestBlackIs1TurnsTheSamplesOver(t *testing.T) { + data := faxBits(t, "H", "W3", "B5") + p := g4(8) + p.blackIs1 = true + got := decodeFax(t, data, p) + if len(got) != 1 || got[0] != "WWWBBBBB" { + t.Errorf("got %q: the picture must not change, only the bits that say it", got) + } + // And the bits themselves are the other way round. + out, err := ccittDecode(data, p) + if err != nil { + t.Fatal(err) + } + if out[0] != 0b00011111 { + t.Errorf("first byte is %08b, want 00011111", out[0]) + } +} + +func TestEncodedByteAlignStartsEveryRowOnAByte(t *testing.T) { + // "H W3 B5" is 3 + 4 + 4 = 11 bits, so the second row would begin + // mid-byte. With the flag set the decoder skips to the boundary, and the + // bits in between are ignored rather than read as a mode. + first := "001" + codeBits(t, ccittWhiteCodes, 3) + codeBits(t, ccittBlackCodes, 5) + pad := strings.Repeat("0", 8-len(first)%8) + data := faxBits(t, first+pad, codeBits(t, ccittModeCodes, modeV0), + codeBits(t, ccittModeCodes, modeV0)) + p := g4(8) + p.encodedByteAlign = true + got := decodeFax(t, data, p) + if len(got) != 2 || got[0] != "WWWBBBBB" || got[1] != "WWWBBBBB" { + t.Errorf("got %q", got) + } +} + +func TestRowsGivenIsHonouredExactly(t *testing.T) { + // A file that says how many rows it has gets that many: short data is + // padded with white, so the samples match the /Height the dictionary + // promises rather than being a row short. + data := faxBits(t, "H", "W3", "B5") + p := g4(8) + p.rows = 3 + got := decodeFax(t, data, p) + if len(got) != 3 { + t.Fatalf("got %d rows, want 3: %q", len(got), got) + } + if got[1] != "WWWWWWWW" || got[2] != "WWWWWWWW" { + t.Errorf("the padding rows are %q and %q, want white", got[1], got[2]) + } +} + +func TestAMakeUpCodeAddsToTheRunBeforeIt(t *testing.T) { + // A run longer than 63 is a make-up code and then a terminating one. 64 + // plus 0 is the shortest way to say sixty-four. + data := faxBits(t, "H", "W64", "W0", "B1") + got := decodeFax(t, data, g4(65)) + if len(got) != 1 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[0] != strings.Repeat("W", 64)+"B" { + t.Errorf("got %q", got[0]) + } +} + +func TestATruncatedFaxComesBackAsFarAsItGot(t *testing.T) { + // A damaged scan is worth showing: the rows already decoded are real, and + // refusing them turns a form into a blank page. + data := faxBits(t, "H", "W3", "B5", "V0", "V0", "H", "W3") + got := decodeFax(t, data, g4(8)) + if len(got) < 2 { + t.Fatalf("got %d rows, want at least the two that decoded: %q", len(got), got) + } + if got[0] != "WWWBBBBB" { + t.Errorf("the first row came back as %q", got[0]) + } +} + +func TestAFaxThatSaysNothingSensibleIsRefused(t *testing.T) { + for _, tc := range []struct { + name string + data []byte + p ccittParams + }{ + {"no columns", nil, ccittParams{k: -1, columns: 0}}, + {"negative rows", nil, ccittParams{k: -1, columns: 8, rows: -1}}, + {"more pixels than anyone can want", + nil, ccittParams{k: -1, columns: 1 << 20, rows: 1 << 20}}, + {"an extension mode", faxBits(t, "EXT", "000"), g4(8)}, + {"a run longer than the row", faxBits(t, "H", "W63", "B63"), g4(8)}, + {"a mode this reader does not know", faxBits(t, "00000001"), g4(8)}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := ccittDecode(tc.data, tc.p); err == nil { + t.Error("no error") + } + }) + } +} + +func TestAFaxWithNoRowsAtAllDecodesToNothing(t *testing.T) { + // All-zero data is fill, not a picture. Zeros decode as perfectly good + // modes — pass mode is 0001 — so a decoder that does not stop here invents + // rows for as long as it is asked to. + out, err := ccittDecode(make([]byte, 64), g4(8)) + if err != nil { + t.Fatal(err) + } + if len(out) != 0 { + t.Errorf("%d bytes out of nothing but fill", len(out)) + } +} + +func TestTheBlockMarkerEndsTheData(t *testing.T) { + // Two end-of-line codes in a row are Group 4's way of saying it has + // finished; six are Group 3's. What follows them is not a row. + g4Data := faxBits(t, "H", "W3", "B5", "EOL", "EOL", "H", "W1", "B7") + if got := decodeFax(t, g4Data, g4(8)); len(got) != 1 { + t.Errorf("Group 4 gave %d rows past its block marker: %q", len(got), got) + } + g3 := []string{"W3", "B5"} + for i := 0; i < 6; i++ { + g3 = append(g3, "EOL") + } + g3 = append(g3, "W1", "B7") + got := decodeFax(t, faxBits(t, g3...), ccittParams{k: 0, columns: 8, endOfBlock: true}) + if len(got) != 1 { + t.Errorf("Group 3 gave %d rows past its block marker: %q", len(got), got) + } + if endOfBlockEOLs(-1) != 2 || endOfBlockEOLs(0) != 6 || endOfBlockEOLs(4) != 6 { + t.Error("endOfBlockEOLs disagrees with the two cases above") + } +} + +func TestEndOfBlockCanBeTurnedOff(t *testing.T) { + // A file may say its data has no block marker, in which case the codes + // that would be one are read as whatever they are. + p := g4(8) + p.endOfBlock = false + p.rows = 1 + got := decodeFax(t, faxBits(t, "H", "W3", "B5"), p) + if len(got) != 1 || got[0] != "WWWBBBBB" { + t.Errorf("got %q", got) + } +} + +func TestTheDecodeParametersComeFromTheDictionary(t *testing.T) { + got := ccittParamsOf(Dict{ + "K": Integer(-1), + "Columns": Integer(120), + "Rows": Integer(30), + "BlackIs1": Bool(true), + "EncodedByteAlign": Bool(true), + "EndOfBlock": Bool(false), + }, nil) + want := ccittParams{k: -1, columns: 120, rows: 30, blackIs1: true, + encodedByteAlign: true, endOfBlock: false} + if got != want { + t.Errorf("got %+v, want %+v", got, want) + } + // No dictionary at all means the defaults the specification gives. + def := ccittParamsOf(nil, nil) + if def != (ccittParams{k: 0, columns: 1728, rows: 0, blackIs1: false, + encodedByteAlign: false, endOfBlock: true}) { + t.Errorf("defaults are %+v", def) + } + // A parameter of the wrong kind is not a parameter. + if boolParm(Dict{"BlackIs1": Integer(1)}, "BlackIs1", false, nil) { + t.Error("an integer was read as a boolean") + } + if !boolParm(Dict{"BlackIs1": Ref{Num: 9}}, "BlackIs1", true, failingResolver) { + t.Error("a reference that will not resolve did not fall back to the default") + } +} + +// failingResolver refuses every reference, which is what a damaged file's +// cross-reference table amounts to. +func failingResolver(Ref) (Object, error) { + return nil, errors.New("reader: no such object") +} + +func TestAFaxReachesTheCallerThroughTheFilterChain(t *testing.T) { + // The point of decoding here rather than in a renderer: /CCITTFaxDecode is + // no longer handed back encoded, so every caller gets samples. + data := faxBits(t, "H", "W3", "B5") + out, img, err := Decode(Dict{ + "Filter": Name("CCITTFaxDecode"), + "DecodeParms": Dict{"K": Integer(-1), "Columns": Integer(8)}, + }, data, nil) + if err != nil { + t.Fatal(err) + } + if img != "" { + t.Errorf("the chain stopped at /%s", img) + } + if len(out) != 1 || out[0] != 0b11100000 { + t.Errorf("got %08b", out) + } + // The abbreviated name an inline image uses reaches the same code. + out2, _, err := Decode(Dict{ + "Filter": Name("CCF"), + "DecodeParms": Dict{"K": Integer(-1), "Columns": Integer(8)}, + }, data, nil) + if err != nil || len(out2) != 1 || out2[0] != out[0] { + t.Errorf("/CCF gave %08b, %v", out2, err) + } +} + +func FuzzCCITTDecode(f *testing.F) { + f.Add(faxBits(f2t(f), "H", "W3", "B5")) + f.Add([]byte{0, 0, 0, 0}) + f.Add([]byte{0xff, 0xff}) + f.Fuzz(func(t *testing.T, data []byte) { + for _, k := range []int{-1, 0, 1} { + for _, cols := range []int{1, 8, 1728} { + // No panic, and never more than the bound allows. + out, err := ccittDecode(data, ccittParams{k: k, columns: cols, + rows: 4, endOfBlock: true}) + if err == nil && len(out) != 4*((cols+7)/8) { + t.Fatalf("K=%d columns=%d gave %d bytes for 4 rows", + k, cols, len(out)) + } + } + } + }) +} + +// f2t lets the fuzz seed use the helpers, which take a *testing.T. +func f2t(f *testing.F) *testing.T { return &testing.T{} } + +func TestTheChangingElementsAreFoundPartWayThroughARow(t *testing.T) { + // b1 and b2 are found by walking the row above from where the pen is. The + // interesting case is a mode that is not the first of its row, because + // then the walk starts inside the row rather than before it — and the pen + // may be either colour by that point. + // + // The first row here changes four times, so the second row's four vertical + // modes each start their walk from a different place and from alternating + // colours; the third row's pass mode is preceded by a vertical one, so it + // too begins part way along. + data := faxBits(t, + "H", "W4", "B4", "H", "W4", "B4", // WWWWBBBBWWWWBBBB + "V0", "V0", "V0", "V0", // copied exactly + "V0", "P", "V0", // white to 4, then pass over the black above + ) + got := decodeFax(t, data, g4(16)) + if len(got) != 3 { + t.Fatalf("got %d rows: %q", len(got), got) + } + if got[0] != "WWWWBBBBWWWWBBBB" { + t.Errorf("first row %q", got[0]) + } + if got[1] != "WWWWBBBBWWWWBBBB" { + t.Errorf("second row %q, want the first copied", got[1]) + } + if got[2] != "WWWWBBBBBBBBBBBB" { + t.Errorf("third row %q, want the black above passed over", got[2]) + } +} + +func TestAFaxIsBoundedEvenWhenItSaysNothingAboutItsHeight(t *testing.T) { + // A fax that does not say how many rows it has is decoded until the data + // runs out, so the bound is the only thing standing between a file and the + // heap. The bound is lowered here rather than a hundred megabytes of fax + // written, which is what maxDecodedSize does for Flate. + was := maxCCITTPixels + maxCCITTPixels = 16 + defer func() { maxCCITTPixels = was }() + + // Four rows of eight pixels is thirty-two, which is past sixteen. + data := faxBits(t, "H", "W3", "B5", "V0", "V0", "V0", "V0", "V0", "V0") + if _, err := ccittDecode(data, g4(8)); err == nil { + t.Error("no error from a fax past the bound") + } + // A run may be built up from make-up codes without ever terminating, and + // is bounded the same way. + maxCCITTPixels = 100 + long := []string{"H"} + for i := 0; i < 8; i++ { + long = append(long, "W64") + } + if _, err := ccittDecode(faxBits(t, long...), g4(1728)); err == nil { + t.Error("no error from a run past the bound") + } +} diff --git a/filter.go b/filter.go index a21da63..25e0bad 100644 --- a/filter.go +++ b/filter.go @@ -16,9 +16,16 @@ var maxDecodedSize int64 = 1 << 30 // ImageFilter reports whether a filter yields an encoded image rather than a // byte stream. [Decode] stops at one of these and hands the caller the still // encoded bytes, because decoding them is an image decoder's job. +// +// /CCITTFaxDecode was on this list and is not any more. It does not carry an +// image with its own idea of how many components it has and how deep they are, +// the way DCT and JPX do: it produces bilevel samples, one bit a pixel, and the +// stream dictionary says what they mean. That is a byte stream, so it is a +// filter — and decoding it here means every caller gets it rather than each +// writing its own. See ccitt.go. func ImageFilter(n Name) bool { switch n { - case "DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode": + case "DCTDecode", "DCT", "JPXDecode", "JBIG2Decode": return true } return false @@ -130,6 +137,8 @@ func applyFilter(f Name, data []byte, parm Dict, r Resolver) ([]byte, error) { return ascii85Decode(data) case "RunLengthDecode", "RL": return runLengthDecode(data) + case "CCITTFaxDecode", "CCF": + return ccittDecode(data, ccittParamsOf(parm, r)) } return nil, fmt.Errorf("reader: unsupported filter /%s", f) } diff --git a/filter_test.go b/filter_test.go index 8196b1a..7b9f698 100644 --- a/filter_test.go +++ b/filter_test.go @@ -41,13 +41,18 @@ func rawDeflateBytes(t *testing.T, data []byte) []byte { } func TestImageFilter(t *testing.T) { - for _, n := range []Name{"DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode"} { + for _, n := range []Name{"DCTDecode", "DCT", "JPXDecode", "JBIG2Decode"} { if !ImageFilter(n) { t.Errorf("ImageFilter(%s) = false", n) } } - if ImageFilter("FlateDecode") { - t.Error("ImageFilter(FlateDecode) = true") + // A fax is not one of them any more: it decodes to bilevel samples, which + // is a byte stream, so the filter chain finishes it rather than handing it + // back encoded. See ccitt.go. + for _, n := range []Name{"FlateDecode", "CCITTFaxDecode", "CCF"} { + if ImageFilter(n) { + t.Errorf("ImageFilter(%s) = true", n) + } } }