From 87278bc0332ca1dac8a80d272ffc79b3705cab5b Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 21 Aug 2026 20:58:38 +0000 Subject: [PATCH] Leave the unwritten spans of a formatted image as holes An image is formatted at the capacity the filesystem it will hold is given, and two of its structures are sized from that capacity and written out empty: the journal, which e2fsprogs convention caps at a gigabyte for a filesystem this large, and the inodes of the table past the ones the image holds. Writing them cost the host a gigabyte of disk for every image unpacked, before any of them held anything. An integration run unpacking twenty images spent twenty gigabytes on the emptiness alone and took the disk out from under the machine. Both spans read as zero, and so does a hole, so a reader of the image sees the same bytes either way and the host keeps the blocks. The bitmaps of the groups past the ones holding content are already placed this way. The last block of the journal is written rather than sought over. A journal that does not fit in the image failed at the I/O layer when the whole span was written, and the journal inode's extent is recorded on the strength of that, so the block that would be the first to fail is still written. An empty image at that capacity held 1,107,685,376 bytes of the host's disk and now holds a thirtieth of that, which is what the same image formatted without a journal has always held. --- .../ContainerizationEXT4/EXT4+Formatter.swift | 13 +-- .../ContainerizationEXT4/EXT4+Journal.swift | 23 +++-- .../TestEXT4Sparse.swift | 85 +++++++++++++++++++ 3 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 Tests/ContainerizationEXT4Tests/TestEXT4Sparse.swift diff --git a/Sources/ContainerizationEXT4/EXT4+Formatter.swift b/Sources/ContainerizationEXT4/EXT4+Formatter.swift index 708b9e1b0..4f461441e 100644 --- a/Sources/ContainerizationEXT4/EXT4+Formatter.swift +++ b/Sources/ContainerizationEXT4/EXT4+Formatter.swift @@ -1023,11 +1023,14 @@ extension EXT4 { } let tableSize: UInt64 = UInt64(EXT4.InodeSize) * blockGroups * inodesPerGroup let rest = tableSize - UInt64(self.inodes.count) * EXT4.InodeSize - let zeroBlock = Array.init(repeating: 0, count: Int(self.blockSize)) - for _ in 0..<(rest / self.blockSize) { - try self.handle.write(contentsOf: zeroBlock) - } - try self.handle.write(contentsOf: Array.init(repeating: 0, count: Int(rest % self.blockSize))) + // The inodes past the ones written are free, and the table reads as + // zero for the whole span they cover. A hole reads as zero as well, + // so the span is skipped rather than written: the bytes a reader + // sees are the same, and the blocks a filesystem holding the image + // gives up to them are not. The bitmaps of the groups past the ones + // holding content are placed by the same seek, and what follows here + // writes at a higher offset, so the file still reaches its length. + try self.handle.seek(toOffset: self.pos + rest) return inodeTableOffset } diff --git a/Sources/ContainerizationEXT4/EXT4+Journal.swift b/Sources/ContainerizationEXT4/EXT4+Journal.swift index 5e3511f6a..aa235972a 100644 --- a/Sources/ContainerizationEXT4/EXT4+Journal.swift +++ b/Sources/ContainerizationEXT4/EXT4+Journal.swift @@ -126,17 +126,22 @@ extension EXT4.Formatter { private func zeroJournalBlocks(count: UInt32) throws { guard count > 0 else { return } - let chunkSize = 1.mib() // Safe: both operands are UInt32, so their product peaks at ~17 TiB, which fits // in Int64 (the width of Int on all 64-bit Apple platforms). - let totalBytes = Int(count) * Int(self.blockSize) - let zeroBuf = [UInt8](repeating: 0, count: min(Int(chunkSize), totalBytes)) - var remaining = totalBytes - while remaining > 0 { - let toWrite = min(zeroBuf.count, remaining) - try self.handle.write(contentsOf: zeroBuf[0.. UInt64 { + var st = stat() + guard stat(path.string, &st) == 0 else { + throw EXT4.Formatter.Error.notFound(path) + } + return UInt64(st.st_blocks) * 512 + } + + private func format(capacity: UInt64, journal: EXT4.JournalConfig?) throws -> (path: FilePath, allocated: UInt64, length: UInt64) { + let path = FilePath( + FileManager.default.uniqueTemporaryDirectory() + .appendingPathComponent("ext4.img.delme.sparse", isDirectory: false)) + let formatter = try EXT4.Formatter(path, minDiskSize: capacity, journal: journal) + try formatter.create(path: FilePath("/test"), mode: EXT4.Inode.Mode(.S_IFDIR, 0o755)) + try formatter.close() + let handle = try FileHandle(forReadingFrom: path.url) + let length = try handle.seekToEnd() + try handle.close() + return (path, try allocatedBytes(of: path), length) + } + + /// An image formatted at the capacity a container's filesystem is given + /// reports that capacity and occupies a small fraction of it. The bound is + /// far above what the structures of an empty filesystem come to and far + /// below the gigabyte a written-out journal alone would add, so it holds + /// whatever the layout does and fails if a span goes back to being written. + @Test func emptyImageAtContainerCapacityOccupiesLittle() throws { + let capacity: UInt64 = 512.gib() + let result = try format(capacity: capacity, journal: .init(defaultMode: .ordered)) + defer { try? FileManager.default.removeItem(at: result.path.url) } + + #expect(result.length >= capacity) + #expect( + result.allocated < 256.mib(), + "an empty image of \(capacity) bytes occupies \(result.allocated) bytes" + ) + } + + /// The journal is the largest of those spans, so an image given one and an + /// image given none occupy nearly the same. + @Test func theJournalCostsLittleUntilItHoldsSomething() throws { + let capacity: UInt64 = 512.gib() + let journaled = try format(capacity: capacity, journal: .init(defaultMode: .ordered)) + defer { try? FileManager.default.removeItem(at: journaled.path.url) } + let plain = try format(capacity: capacity, journal: nil) + defer { try? FileManager.default.removeItem(at: plain.path.url) } + + #expect( + journaled.allocated < plain.allocated + 64.mib(), + "journaled image occupies \(journaled.allocated) against \(plain.allocated) without one" + ) + } +}