|
| 1 | +// What gzip costs on top of the bytes it carries, measured rather than |
| 2 | +// written down. |
| 3 | +// |
| 4 | +// This file exists because a number that was written down turned out to be a |
| 5 | +// fact about one Go release. The size of a TAR.GZ is arithmetic - it has to |
| 6 | +// be, because the bytes pass through deflate and building the archive to |
| 7 | +// measure it would make a preview cost what the run costs. The arithmetic |
| 8 | +// needs to know what the gzip stream adds, and until 2026-09-01 that was three |
| 9 | +// constants in targz.go. |
| 10 | +// |
| 11 | +// Go 1.27.0 changed one of them. The block that closes a level zero stream |
| 12 | +// went from a five byte empty STORED block to a two byte one, so every archive |
| 13 | +// came out three bytes short of its plan and the format refused to write |
| 14 | +// anything at all - measured on every size tried, from 64 kB to 10 MB. The |
| 15 | +// engine was right to refuse. The arithmetic was describing Go 1.26. |
| 16 | +// |
| 17 | +// Bumping the constant would have worked until the next release. Measuring |
| 18 | +// asks the library that is actually linked, so it holds for the one after that |
| 19 | +// too. The model is the same under both releases and only the constants move: |
| 20 | +// |
| 21 | +// overhead(n) = base + perBlock * ceil(n / storeBlock) |
| 22 | +// |
| 23 | +// Measured 2026-09-01, level zero: perBlock is 5 under both, base is 23 under |
| 24 | +// go1.26.7 and 20 under go1.27.0. |
| 25 | +// |
| 26 | +// One honest limit, because it would otherwise look like this file is proven |
| 27 | +// and it is only half proven. Replacing the measurement with today's two |
| 28 | +// constants written down would pass every test in this repository, today, on |
| 29 | +// this compiler - the mutation runner was pointed at exactly that and it |
| 30 | +// cannot go red. What the measurement buys is the NEXT release, and no test |
| 31 | +// that runs today can demonstrate that. The mutations here cover the |
| 32 | +// arithmetic being wrong; they cannot cover it being right for the wrong |
| 33 | +// reason. That is why this comment is long: it is the only thing standing |
| 34 | +// between a later reader and a tidy simplification back to the bug. |
| 35 | +package targz |
| 36 | + |
| 37 | +import ( |
| 38 | + "bytes" |
| 39 | + "compress/gzip" |
| 40 | + "fmt" |
| 41 | + "io" |
| 42 | + "sync" |
| 43 | +) |
| 44 | + |
| 45 | +// framing is what a level zero gzip stream costs beyond its content. |
| 46 | +type framing struct { |
| 47 | + // base is the header, the trailer, and the block that closes the stream. |
| 48 | + base int64 |
| 49 | + // perBlock is what each stored block of content costs. |
| 50 | + perBlock int64 |
| 51 | +} |
| 52 | + |
| 53 | +// gzipFraming measures the framing once and hands back the same answer after. |
| 54 | +// |
| 55 | +// Lazy rather than at init because the answer is only needed when a size is |
| 56 | +// being worked out, and a package that measures something on every program |
| 57 | +// start makes every command pay for the one that needs it. |
| 58 | +var measuredFraming = sync.OnceValues(measureFraming) |
| 59 | + |
| 60 | +// measureFraming works the two constants out from three compressions, and then |
| 61 | +// checks the model against a fourth. |
| 62 | +// |
| 63 | +// Two points settle the line and the third says whether it is a line at all. |
| 64 | +// Without that check a change to the BLOCK SIZE - rather than to the cost of a |
| 65 | +// block - would be read as a change to the constants, and the arithmetic would |
| 66 | +// be quietly wrong instead of loudly refused. That is the failure this whole |
| 67 | +// file exists to stop happening a second time, so it is worth one more |
| 68 | +// compression of a buffer that is already in memory. |
| 69 | +func measureFraming() (framing, error) { |
| 70 | + one, err := storedOverhead(storeBlock) |
| 71 | + if err != nil { |
| 72 | + return framing{}, err |
| 73 | + } |
| 74 | + two, err := storedOverhead(2 * storeBlock) |
| 75 | + if err != nil { |
| 76 | + return framing{}, err |
| 77 | + } |
| 78 | + |
| 79 | + f := framing{perBlock: two - one} |
| 80 | + f.base = one - f.perBlock |
| 81 | + |
| 82 | + // Two independent checks the two points above cannot make on their own: a |
| 83 | + // third multiple of the block, and the empty stream, which is base alone. |
| 84 | + three, err := storedOverhead(3 * storeBlock) |
| 85 | + if err != nil { |
| 86 | + return framing{}, err |
| 87 | + } |
| 88 | + empty, err := storedOverhead(0) |
| 89 | + if err != nil { |
| 90 | + return framing{}, err |
| 91 | + } |
| 92 | + if want := f.base + 3*f.perBlock; three != want { |
| 93 | + return framing{}, fmt.Errorf( |
| 94 | + "targz: this build of Go frames a gzip stream in a shape this tool does not understand. "+ |
| 95 | + "Three blocks of content cost %d B where the two measured before them predict %d B, "+ |
| 96 | + "so the size of an archive cannot be worked out without building it", |
| 97 | + three, want) |
| 98 | + } |
| 99 | + if empty != f.base { |
| 100 | + return framing{}, fmt.Errorf( |
| 101 | + "targz: this build of Go frames an empty gzip stream at %d B where the measurement says %d B, "+ |
| 102 | + "so the size of an archive cannot be worked out without building it", |
| 103 | + empty, f.base) |
| 104 | + } |
| 105 | + return f, nil |
| 106 | +} |
| 107 | + |
| 108 | +// storedOverhead is what gzip adds to n bytes at compression level zero. |
| 109 | +// |
| 110 | +// The content is zeros, and that is safe precisely because the level is zero: |
| 111 | +// stored blocks carry their input unchanged, so the framing does not depend on |
| 112 | +// what is in them. At any other level it would. |
| 113 | +func storedOverhead(n int64) (int64, error) { |
| 114 | + var out bytes.Buffer |
| 115 | + w, err := gzip.NewWriterLevel(&out, gzip.NoCompression) |
| 116 | + if err != nil { |
| 117 | + return 0, err |
| 118 | + } |
| 119 | + if n > 0 { |
| 120 | + if _, err := io.CopyN(w, zeros{}, n); err != nil { |
| 121 | + return 0, err |
| 122 | + } |
| 123 | + } |
| 124 | + if err := w.Close(); err != nil { |
| 125 | + return 0, err |
| 126 | + } |
| 127 | + return int64(out.Len()) - n, nil |
| 128 | +} |
| 129 | + |
| 130 | +// zeros is an endless run of zero bytes, so the measurement allocates one |
| 131 | +// small buffer rather than the megabyte it reads. |
| 132 | +type zeros struct{} |
| 133 | + |
| 134 | +func (zeros) Read(p []byte) (int, error) { |
| 135 | + for i := range p { |
| 136 | + p[i] = 0 |
| 137 | + } |
| 138 | + return len(p), nil |
| 139 | +} |
0 commit comments