diff --git a/AGENTS.md b/AGENTS.md index 8ee8bc5..95cd3fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ activate it first and drop the override): | corpus audit | `.venv/bin/python dev/audit_corpus.py ` (every listing over every file, escapes and undecoded kinds tallied) | | OMAP vs Windows | `.venv/bin/python dev/validate_omap_against_windows.py --fetch` (needs Windows DLLs and network; nothing is redistributed) | -Expected state on a clean tree: **636 tests pass**, `ruff check` and `ty check` +Expected state on a clean tree: **645 tests pass**, `ruff check` and `ty check` both clean, and the fuzzer reports no escaped exception. Run tests, lint *and* a quick fuzz pass before considering work complete. diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d6f02..0fc0f27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ resolve *differently* would be breaking, and would say so here. ### Added +- `Diagnostics.c13_truncations` records where and why a C13 line-info walk + stopped early, accompanied by an explanatory warning in `Diagnostics.warnings`. + `C13Truncation` is exported in `purepdb.__all__`. - `diagnose()` explains a PDB whose modules carry no symbol stream at all. That is what `link.exe /PDBSTRIPPED` writes, and what every public symbol file on Microsoft's symbol server is: publics, section headers and FPO data, with @@ -166,6 +169,27 @@ resolve *differently* would be breaking, and would say so here. size cannot be less than what was just read out of the block, so the bytes consumed are now the floor; a damaged size landing inside the entries used to report lines made of line-record bytes. +- `PDB.named_streams()` returns `{}` when the PDB Info stream is absent or + invalid, matching its documented contract and preventing `MsfError` from + escaping `string_table()`, `lines()`, and `diagnose()`. +- `PublicsStream.parse()` raises `PdbError` rather than `ValueError` on + truncated input, ensuring `PdbError` remains the only exception surface a + caller must handle. Plain `PdbError` rather than `MsfError`: the publics + stream is not an MSF container. +- `DbiStream.parse()` checks its substream sizes. A negative one raises + `MsfError`: it makes the next substream's offset go backwards, so every + later slice would alias bytes that were never substream data. A size past + the end of the stream is read as far as the stream goes and recorded in + `DbiStream.substream_overrun`, which `diagnose()` reports as + `Diagnostics.dbi_overrun` with a warning naming the substream -- a file + missing the last eight bytes of its debug header still yields every + function, and now says what it is missing. (Raising for the overrun too was + tried on this branch and reverted: it turned such a file into one that + would not open, with no diagnostic to say why.) +- C13 trailing bytes are reported as a truncation only when they hold + something. Fewer than eight bytes after the last subsection cannot be a + header; zero ones are a producer's padding, and the lines before them are + all present. - A module stream signed C7 (`0`) or C11 (`1`) has its signature stripped like a C13 one. Only C13 was recognised, so the other two left the signature word on the front of the records, where it was read as a record header and every diff --git a/purepdb/__init__.py b/purepdb/__init__.py index 5b81bd1..95c3efd 100644 --- a/purepdb/__init__.py +++ b/purepdb/__init__.py @@ -9,6 +9,7 @@ empty list. """ +from .c13 import C13Truncation from .codeview import ( CompileInfo, Constant, @@ -48,6 +49,7 @@ __all__ = [ "PDB", + "C13Truncation", "CompileInfo", "Constant", "DataSymbol", diff --git a/purepdb/c13.py b/purepdb/c13.py index e5acbcf..4d41bbf 100644 --- a/purepdb/c13.py +++ b/purepdb/c13.py @@ -80,7 +80,16 @@ class LineEntry: is_statement: bool -def iter_subsections(data: bytes): +@dataclass +class C13Truncation: + """Where a C13 subsection or line walk stopped early, and why.""" + + offset: int + reason: str + + +def iter_subsections(data: bytes, *, + truncation: list[C13Truncation] | None = None): """Yield each C13 subsection. Stops at the first header that does not fit. A subsection whose kind carries `DEBUG_S_IGNORE` is stepped over rather @@ -94,11 +103,26 @@ def iter_subsections(data: bytes): kind, length = struct.unpack_from(" len(data) - pos: + if truncation is not None: + truncation.append(C13Truncation( + pos - 8, + f"subsection {kind:#x} length {length} runs " + f"{length - (len(data) - pos)} bytes past the end of the " + f"{len(data)}-byte C13 stream", + )) return if not kind & DEBUG_S_IGNORE: yield Subsection(kind=kind, payload=data[pos : pos + length]) pos += length pos += -pos % 4 + # Fewer than eight bytes after the last subsection cannot be a header. + # Zero bytes there are padding -- a producer rounding the region up -- + # and only bytes with something in them are evidence of a cut. + if pos < len(data) and truncation is not None and any(data[pos:]): + truncation.append(C13Truncation( + pos, + f"{len(data) - pos} trailing byte(s) are too few for a subsection header", + )) def parse_file_checksums(payload: bytes) -> dict[int, int]: @@ -118,9 +142,15 @@ def parse_file_checksums(payload: bytes) -> dict[int, int]: return out -def parse_lines(payload: bytes) -> list[LineEntry]: +def parse_lines(payload: bytes, *, + truncation: list[C13Truncation] | None = None) -> list[LineEntry]: """Decode one DEBUG_S_LINES subsection.""" if len(payload) < _LINE_FRAGMENT_HEADER.size: + if truncation is not None and len(payload) > 0: + truncation.append(C13Truncation( + 0, + f"DEBUG_S_LINES payload is {len(payload)} byte(s), too short for header", + )) return [] base_offset, segment, flags, _code_size = _LINE_FRAGMENT_HEADER.unpack_from(payload, 0) have_columns = bool(flags & LF_HAVE_COLUMNS) @@ -134,7 +164,13 @@ def parse_lines(payload: bytes) -> list[LineEntry]: if have_columns: needed += num_lines * 4 if needed > len(payload) - entries_at: - break + if truncation is not None: + truncation.append(C13Truncation( + pos, + f"line block at byte {pos:#x} needs {needed} bytes for {num_lines} " + f"lines, exceeding remaining {len(payload) - entries_at} bytes", + )) + return out # The entries are a packed array, which `iter_unpack` walks without # a per-entry offset computation; the column entries, if any, follow @@ -156,4 +192,10 @@ def parse_lines(payload: bytes) -> list[LineEntry]: # header and report the lines it made of them; the bytes consumed are # the floor. pos += max(block_size, _LINE_BLOCK_HEADER.size + needed) + if pos < len(payload) and truncation is not None and any(payload[pos:]): + truncation.append(C13Truncation( + pos, + f"{len(payload) - pos} trailing byte(s) in DEBUG_S_LINES are " + f"too few for a block header", + )) return out diff --git a/purepdb/dbi.py b/purepdb/dbi.py index 5b311dc..1dcae1e 100644 --- a/purepdb/dbi.py +++ b/purepdb/dbi.py @@ -123,6 +123,7 @@ class SectionContribution: module_index: int # index into DbiStream.modules def contains(self, segment: int, offset: int) -> bool: + """Whether this contribution covers the given segment:offset.""" return self.segment == segment and self.offset <= offset < self.offset + self.size @@ -142,6 +143,7 @@ def __init__(self, contributions: list[SectionContribution]): self._keys = [(c.segment, c.offset) for c in self._sorted] def __len__(self) -> int: + """Total number of contributions, including empty ones.""" return len(self.contributions) def find(self, segment: int, offset: int) -> SectionContribution | None: @@ -179,6 +181,11 @@ class DbiStream: Non-None means `modules` is short: the modules past that point were never read, and every symbol in their streams is absent.""" + substream_overrun: str | None = None + """The first substream whose declared size ran past the end of the DBI + stream, or None. That substream was read as far as the stream goes and + every one after it as empty, so modules, contributions, the section map + or the debug-header slots may be missing -- which `diagnose()` says.""" @property def is_stripped(self) -> bool: @@ -258,19 +265,50 @@ def parse(cls, data: bytes) -> DbiStream: ) off = _HEADER.size + + # The two ways a substream size can be wrong are not the same kind of + # wrong. A negative size makes the *next* substream's offset go + # backwards, so every later slice aliases the header or an earlier + # substream -- bytes that were never substream data -- and anything + # read from them is fiction. That raises. A size past the end of the + # stream is a stream cut short: what is there is real, the parsers + # below already stop at the end of what they are handed, and a file + # missing the last eight bytes of its debug header used to yield every + # function. So it is read as far as it goes and recorded, once, where + # `diagnose()` can turn it into a sentence -- an empty result with an + # explanation rather than a refusal, which is the contract. + def _check_substream(name: str, size: int) -> int: + if size < 0: + raise MsfError(f"DBI {name} substream size is negative ({size})") + if off + size > len(data): + if self.substream_overrun is None: + self.substream_overrun = ( + f"the {name} substream starts at byte {off} and claims " + f"{size} bytes of a {len(data)}-byte stream") + return max(0, len(data) - off) + return size + + modinfo_size = _check_substream("ModuleInfo", modinfo_size) self.modules, self.module_list_stopped_at = _parse_module_list( data[off : off + modinfo_size] ) off += modinfo_size + + seccontrib_size = _check_substream("SectionContribution", seccontrib_size) self.section_contributions = _parse_section_contributions( data[off : off + seccontrib_size] ) off += seccontrib_size + + secmap_size = _check_substream("SectionMap", secmap_size) self.section_map = parse_section_map(data[off : off + secmap_size]) off += secmap_size - off += srcinfo_size - off += tsmap_size - off += ec_size + + off += _check_substream("SourceInfo", srcinfo_size) + off += _check_substream("TypeServerMap", tsmap_size) + off += _check_substream("EC", ec_size) + + dbg_hdr_size = _check_substream("OptionalDebugHeader", dbg_hdr_size) self.dbg_header = _parse_dbg_header(data[off : off + dbg_hdr_size]) return self diff --git a/purepdb/gsi.py b/purepdb/gsi.py index 5643397..02270bb 100644 --- a/purepdb/gsi.py +++ b/purepdb/gsi.py @@ -32,6 +32,8 @@ import struct from dataclasses import dataclass +from .msf import PdbError + _PUBLICS_HEADER = struct.Struct( " PublicsStream: if len(data) < _PUBLICS_HEADER.size: - raise ValueError("publics stream too small for its header") + raise PdbError("publics stream too small for its header") (sym_hash, addr_map_size, num_thunks, size_of_thunk, isect, _pad, off_thunk, num_sections) = _PUBLICS_HEADER.unpack_from(data, 0) start = _PUBLICS_HEADER.size + sym_hash end = start + addr_map_size if end > len(data): - raise ValueError( + raise PdbError( f"publics address map runs past end of stream " f"(need {end} bytes, have {len(data)})" ) diff --git a/purepdb/pdb.py b/purepdb/pdb.py index 3892a7b..ee09eef 100644 --- a/purepdb/pdb.py +++ b/purepdb/pdb.py @@ -340,6 +340,13 @@ class Diagnostics: files teaches a reader to skip the list, which costs the entries that do matter. The count and the note beside it in `purepdb diagnose` are the explanation, in the report rather than in the alarm channel.""" + c13_truncations: list[tuple[str, c13.C13Truncation]] = field(default_factory=list) + """Where a C13 line-info walk stopped short of consuming its buffer, and why.""" + dbi_overrun: str | None = None + """A DBI substream whose declared size ran past the stream, if any. It was + read as far as the stream goes and the substreams after it as empty, so + modules, section contributions, the section map or the debug-header slots + may be missing.""" unrecognised_signatures: dict[int, int] = field(default_factory=dict) """Module streams whose first word is not a CodeView signature (C7, C11 or C13), counted by that word. Their records are not read: the layout after @@ -527,6 +534,14 @@ def warnings(self) -> list[str]: "is in the pre-optimisation address space and does not match " "the shipped image" ) + if self.dbi_overrun is not None: + out.append( + f"the DBI stream is shorter than its header claims " + f"({self.dbi_overrun}); that substream was read as far as the " + f"stream goes and the ones after it as empty, so modules, " + f"section contributions, the section map or the debug-header " + f"slots may be missing" + ) if self.module_list_stopped_at is not None: out.append( f"the module list stopped at byte " @@ -587,6 +602,13 @@ def warnings(self) -> list[str]: f"/names stream is not, so file-name offsets cannot be resolved " f"and lines() yields nothing" ) + if self.c13_truncations: + where, first = self.c13_truncations[0] + out.append( + f"{len(self.c13_truncations)} C13 line-info section(s) stopped early; " + f"lines after that point are missing. First: {where} at " + f"byte {first.offset:#x} ({first.reason})" + ) if self.pdb_info_error is not None: out.append( f"the PDB Info stream cannot be read ({self.pdb_info_error}), " @@ -814,7 +836,7 @@ def publics_stream(self) -> PublicsStream | None: return None try: return PublicsStream.parse(self.msf.read_stream(idx)) - except (ValueError, struct.error): + except (PdbError, struct.error): return None def public_symbols(self) -> list[codeview.PublicSymbol]: @@ -1304,7 +1326,12 @@ def named_streams(self) -> dict[str, int]: `/names` and `/LinkInfo` are what real linkers put here. """ - return parse_named_stream_map(self.msf.read_stream(STREAM_PDB_INFO)) + if not self.msf.is_valid_stream(STREAM_PDB_INFO): + return {} + try: + return parse_named_stream_map(self.msf.read_stream(STREAM_PDB_INFO)) + except MsfError: + return {} def string_table(self) -> StringTable | None: """The `/names` global string table, or None when the PDB has none.""" @@ -1400,6 +1427,7 @@ def diagnose(self) -> Diagnostics: malformed = 0 malformed_inline = 0 line_bytes = 0 + c13_truncations: list[tuple[str, c13.C13Truncation]] = [] signatures: dict[int, int] = {} proc_records = 0 placed_sites = 0 @@ -1414,7 +1442,24 @@ def diagnose(self) -> Diagnostics: # exactly what `module_procs()` returns since `extract_procs` drops # only the records `parse_proc` raises on. for mod in self.dbi.modules: - line_bytes += len(self.module_c13_bytes(mod)) + c13_bytes = self.module_c13_bytes(mod) + line_bytes += len(c13_bytes) + if c13_bytes: + # The truncation reports come from re-decoding every + # DEBUG_S_LINES payload here and discarding the entries: + # 18ms of the 180ms diagnose() costs on the 3 MB sqlite + # fixture. Deliberate: `parse_lines` is the only + # implementation of the block walk, + # and a header-only twin kept beside it would drift from it. + # `lines()` cannot be reused for this: it is a generator a + # caller may never consume, and it resolves file names, + # which this loop does not want. + mod_c13_report: list[c13.C13Truncation] = [] + for sub in c13.iter_subsections(c13_bytes, truncation=mod_c13_report): + if sub.kind == c13.DEBUG_S_LINES: + c13.parse_lines(sub.payload, truncation=mod_c13_report) + for t in mod_c13_report: + c13_truncations.append((f"module {mod.index} ({mod.module_name})", t)) signature, body = self._module_symbols(mod) if signature is not None: signatures[signature] = signatures.get(signature, 0) + 1 @@ -1527,9 +1572,11 @@ def diagnose(self) -> Diagnostics: has_string_table=self.string_table() is not None, pdb_info_error=pdb_info_error, module_list_stopped_at=self.dbi.module_list_stopped_at, + dbi_overrun=self.dbi.substream_overrun, private_symbols_stripped=self.dbi.is_stripped, linker_version=self.dbi.toolchain_version, thread_local_records=thread_locals, + c13_truncations=c13_truncations, unrecognised_signatures=signatures, ) diff --git a/purepdb/reader.py b/purepdb/reader.py index b9edd4b..0379c2e 100644 --- a/purepdb/reader.py +++ b/purepdb/reader.py @@ -87,10 +87,5 @@ def cstring(self) -> str: self.pos = end + 1 return s.decode("utf-8", errors="replace") - def align(self, boundary: int) -> None: - rem = self.pos % boundary - if rem: - self.pos += boundary - rem - def seek(self, pos: int) -> None: self.pos = pos diff --git a/tests/_synth.py b/tests/_synth.py index 7ba1f69..52784f7 100644 --- a/tests/_synth.py +++ b/tests/_synth.py @@ -9,6 +9,7 @@ from __future__ import annotations import struct +from collections.abc import Sequence from purepdb.dbi import SEC_CONTRIB_V2, SEC_CONTRIB_VER60 from purepdb.msf import BIG_MSF_MAGIC @@ -18,13 +19,16 @@ def _ceil_div(a, b): return (a + b - 1) // b -def build_msf(streams: list[bytes], block_size: int = 512) -> bytes: +def build_msf(streams: Sequence[bytes | None], block_size: int = 512) -> bytes: """Serialise a list of streams into a valid MSF 7.00 container.""" bs = block_size next_block = 3 # 0=superblock, 1=FPM1, 2=FPM2 placed = [] # (size, [block indices], data) for data in streams: + if data is None: + placed.append((0xFFFFFFFF, [], b"")) + continue n = _ceil_div(len(data), bs) if data else 0 blocks = list(range(next_block, next_block + n)) next_block += n diff --git a/tests/test_cli.py b/tests/test_cli.py index 2d2ada6..71a5135 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -446,6 +446,29 @@ def test_modules_accounts_for_a_contribution_with_no_module(tmp_path, capsys): assert err[:2] == ["", "1 modules"] +def test_diagnose_reports_a_c13_section_that_stops_early(tmp_path, capsys): + from purepdb import c13 + from tests._synth import names_stream, pdb_info_stream + + symbols = gproc32("main", 1, 0x10) + region = struct.pack(" 0 assert d.has_string_table + assert d.c13_truncations == [] assert not any("/names" in w for w in d.warnings) + assert not any("C13 line-info" in w for w in d.warnings) def test_line_info_without_the_names_stream_is_warned_about(): @@ -361,3 +363,60 @@ def test_a_pdb_with_no_line_info_is_not_warned_about(): d = pdb.diagnose() assert d.line_bytes == 0 assert not any("/names" in w for w in d.warnings) + assert d.c13_truncations == [] + + +def test_truncated_c13_subsection_header_is_warned_about(): + # Only 4 bytes: not enough for the 8-byte (kind, length) subsection header. + pdb = _pdb(c13_region=b"\x01\x00\x00\x00") + d = pdb.diagnose() + assert len(d.c13_truncations) == 1 + assert any("C13 line-info section(s) stopped early" in w for w in d.warnings) + + +def test_c13_subsection_length_past_end_is_warned_about(): + # Header claims 500 bytes payload, but only 8 bytes remain. + region = struct.pack(" list: pdb.original_sections, pdb.functions(), pdb.public_symbols(), + pdb.publics_stream(), pdb.module_procs(), pdb.data_symbols(), pdb.thread_locals(),