Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ activate it first and drop the override):
| corpus audit | `.venv/bin/python dev/audit_corpus.py <dir>` (every listing over every file, escapes and undecoded kinds tallied) |
| OMAP vs Windows | `.venv/bin/python dev/validate_omap_against_windows.py <dll-dir> --fetch` (needs Windows DLLs and network; nothing is redistributed) |

Expected state on a clean tree: **636 tests pass**, `ruff check` and `ty check`
Expected state on a clean tree: **645 tests pass**, `ruff check` and `ty check`
both clean, and the fuzzer reports no escaped exception. Run tests, lint *and*
a quick fuzz pass before considering work complete.

Expand Down
24 changes: 24 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ resolve *differently* would be breaking, and would say so here.

### Added

- `Diagnostics.c13_truncations` records where and why a C13 line-info walk
stopped early, accompanied by an explanatory warning in `Diagnostics.warnings`.
`C13Truncation` is exported in `purepdb.__all__`.
- `diagnose()` explains a PDB whose modules carry no symbol stream at all. That
is what `link.exe /PDBSTRIPPED` writes, and what every public symbol file on
Microsoft's symbol server is: publics, section headers and FPO data, with
Expand Down Expand Up @@ -166,6 +169,27 @@ resolve *differently* would be breaking, and would say so here.
size cannot be less than what was just read out of the block, so the bytes
consumed are now the floor; a damaged size landing inside the entries used
to report lines made of line-record bytes.
- `PDB.named_streams()` returns `{}` when the PDB Info stream is absent or
invalid, matching its documented contract and preventing `MsfError` from
escaping `string_table()`, `lines()`, and `diagnose()`.
- `PublicsStream.parse()` raises `PdbError` rather than `ValueError` on
truncated input, ensuring `PdbError` remains the only exception surface a
caller must handle. Plain `PdbError` rather than `MsfError`: the publics
stream is not an MSF container.
- `DbiStream.parse()` checks its substream sizes. A negative one raises
`MsfError`: it makes the next substream's offset go backwards, so every
later slice would alias bytes that were never substream data. A size past
the end of the stream is read as far as the stream goes and recorded in
`DbiStream.substream_overrun`, which `diagnose()` reports as
`Diagnostics.dbi_overrun` with a warning naming the substream -- a file
missing the last eight bytes of its debug header still yields every
function, and now says what it is missing. (Raising for the overrun too was
tried on this branch and reverted: it turned such a file into one that
would not open, with no diagnostic to say why.)
- C13 trailing bytes are reported as a truncation only when they hold
something. Fewer than eight bytes after the last subsection cannot be a
header; zero ones are a producer's padding, and the lines before them are
all present.
- A module stream signed C7 (`0`) or C11 (`1`) has its signature stripped like
a C13 one. Only C13 was recognised, so the other two left the signature word
on the front of the records, where it was read as a record header and every
Expand Down
2 changes: 2 additions & 0 deletions purepdb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
empty list.
"""

from .c13 import C13Truncation
from .codeview import (
CompileInfo,
Constant,
Expand Down Expand Up @@ -48,6 +49,7 @@

__all__ = [
"PDB",
"C13Truncation",
"CompileInfo",
"Constant",
"DataSymbol",
Expand Down
48 changes: 45 additions & 3 deletions purepdb/c13.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,16 @@ class LineEntry:
is_statement: bool


def iter_subsections(data: bytes):
@dataclass
class C13Truncation:
"""Where a C13 subsection or line walk stopped early, and why."""

offset: int
reason: str


def iter_subsections(data: bytes, *,
truncation: list[C13Truncation] | None = None):
"""Yield each C13 subsection. Stops at the first header that does not fit.

A subsection whose kind carries `DEBUG_S_IGNORE` is stepped over rather
Expand All @@ -94,11 +103,26 @@ def iter_subsections(data: bytes):
kind, length = struct.unpack_from("<II", data, pos)
pos += 8
if length > len(data) - pos:
if truncation is not None:
truncation.append(C13Truncation(
pos - 8,
f"subsection {kind:#x} length {length} runs "
f"{length - (len(data) - pos)} bytes past the end of the "
f"{len(data)}-byte C13 stream",
))
return
if not kind & DEBUG_S_IGNORE:
yield Subsection(kind=kind, payload=data[pos : pos + length])
pos += length
pos += -pos % 4
# Fewer than eight bytes after the last subsection cannot be a header.
# Zero bytes there are padding -- a producer rounding the region up --
# and only bytes with something in them are evidence of a cut.
if pos < len(data) and truncation is not None and any(data[pos:]):
truncation.append(C13Truncation(
pos,
f"{len(data) - pos} trailing byte(s) are too few for a subsection header",
))


def parse_file_checksums(payload: bytes) -> dict[int, int]:
Expand All @@ -118,9 +142,15 @@ def parse_file_checksums(payload: bytes) -> dict[int, int]:
return out


def parse_lines(payload: bytes) -> list[LineEntry]:
def parse_lines(payload: bytes, *,
truncation: list[C13Truncation] | None = None) -> list[LineEntry]:
"""Decode one DEBUG_S_LINES subsection."""
if len(payload) < _LINE_FRAGMENT_HEADER.size:
if truncation is not None and len(payload) > 0:
truncation.append(C13Truncation(
0,
f"DEBUG_S_LINES payload is {len(payload)} byte(s), too short for header",
))
return []
base_offset, segment, flags, _code_size = _LINE_FRAGMENT_HEADER.unpack_from(payload, 0)
have_columns = bool(flags & LF_HAVE_COLUMNS)
Expand All @@ -134,7 +164,13 @@ def parse_lines(payload: bytes) -> list[LineEntry]:
if have_columns:
needed += num_lines * 4
if needed > len(payload) - entries_at:
break
if truncation is not None:
truncation.append(C13Truncation(
pos,
f"line block at byte {pos:#x} needs {needed} bytes for {num_lines} "
f"lines, exceeding remaining {len(payload) - entries_at} bytes",
))
return out

# The entries are a packed array, which `iter_unpack` walks without
# a per-entry offset computation; the column entries, if any, follow
Expand All @@ -156,4 +192,10 @@ def parse_lines(payload: bytes) -> list[LineEntry]:
# header and report the lines it made of them; the bytes consumed are
# the floor.
pos += max(block_size, _LINE_BLOCK_HEADER.size + needed)
if pos < len(payload) and truncation is not None and any(payload[pos:]):
truncation.append(C13Truncation(
pos,
f"{len(payload) - pos} trailing byte(s) in DEBUG_S_LINES are "
f"too few for a block header",
))
return out
44 changes: 41 additions & 3 deletions purepdb/dbi.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ class SectionContribution:
module_index: int # index into DbiStream.modules

def contains(self, segment: int, offset: int) -> bool:
"""Whether this contribution covers the given segment:offset."""
return self.segment == segment and self.offset <= offset < self.offset + self.size


Expand All @@ -142,6 +143,7 @@ def __init__(self, contributions: list[SectionContribution]):
self._keys = [(c.segment, c.offset) for c in self._sorted]

def __len__(self) -> int:
"""Total number of contributions, including empty ones."""
return len(self.contributions)

def find(self, segment: int, offset: int) -> SectionContribution | None:
Expand Down Expand Up @@ -179,6 +181,11 @@ class DbiStream:

Non-None means `modules` is short: the modules past that point were never
read, and every symbol in their streams is absent."""
substream_overrun: str | None = None
"""The first substream whose declared size ran past the end of the DBI
stream, or None. That substream was read as far as the stream goes and
every one after it as empty, so modules, contributions, the section map
or the debug-header slots may be missing -- which `diagnose()` says."""

@property
def is_stripped(self) -> bool:
Expand Down Expand Up @@ -258,19 +265,50 @@ def parse(cls, data: bytes) -> DbiStream:
)

off = _HEADER.size

# The two ways a substream size can be wrong are not the same kind of
# wrong. A negative size makes the *next* substream's offset go
# backwards, so every later slice aliases the header or an earlier
# substream -- bytes that were never substream data -- and anything
# read from them is fiction. That raises. A size past the end of the
# stream is a stream cut short: what is there is real, the parsers
# below already stop at the end of what they are handed, and a file
# missing the last eight bytes of its debug header used to yield every
# function. So it is read as far as it goes and recorded, once, where
# `diagnose()` can turn it into a sentence -- an empty result with an
# explanation rather than a refusal, which is the contract.
def _check_substream(name: str, size: int) -> int:
if size < 0:
raise MsfError(f"DBI {name} substream size is negative ({size})")
if off + size > len(data):
if self.substream_overrun is None:
self.substream_overrun = (
f"the {name} substream starts at byte {off} and claims "
f"{size} bytes of a {len(data)}-byte stream")
return max(0, len(data) - off)
return size

modinfo_size = _check_substream("ModuleInfo", modinfo_size)
self.modules, self.module_list_stopped_at = _parse_module_list(
data[off : off + modinfo_size]
)
off += modinfo_size

seccontrib_size = _check_substream("SectionContribution", seccontrib_size)
self.section_contributions = _parse_section_contributions(
data[off : off + seccontrib_size]
)
off += seccontrib_size

secmap_size = _check_substream("SectionMap", secmap_size)
self.section_map = parse_section_map(data[off : off + secmap_size])
off += secmap_size
off += srcinfo_size
off += tsmap_size
off += ec_size

off += _check_substream("SourceInfo", srcinfo_size)
off += _check_substream("TypeServerMap", tsmap_size)
off += _check_substream("EC", ec_size)

dbg_hdr_size = _check_substream("OptionalDebugHeader", dbg_hdr_size)
self.dbg_header = _parse_dbg_header(data[off : off + dbg_hdr_size])
return self

Expand Down
6 changes: 4 additions & 2 deletions purepdb/gsi.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import struct
from dataclasses import dataclass

from .msf import PdbError

_PUBLICS_HEADER = struct.Struct(
"<I" # SymHash -- byte size of the hash-table portion
"I" # AddrMap -- byte size of the address map
Expand Down Expand Up @@ -70,14 +72,14 @@ class PublicsStream:
@classmethod
def parse(cls, data: bytes) -> PublicsStream:
if len(data) < _PUBLICS_HEADER.size:
raise ValueError("publics stream too small for its header")
raise PdbError("publics stream too small for its header")
(sym_hash, addr_map_size, num_thunks, size_of_thunk,
isect, _pad, off_thunk, num_sections) = _PUBLICS_HEADER.unpack_from(data, 0)

start = _PUBLICS_HEADER.size + sym_hash
end = start + addr_map_size
if end > len(data):
raise ValueError(
raise PdbError(
f"publics address map runs past end of stream "
f"(need {end} bytes, have {len(data)})"
)
Expand Down
53 changes: 50 additions & 3 deletions purepdb/pdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,13 @@ class Diagnostics:
files teaches a reader to skip the list, which costs the entries that do
matter. The count and the note beside it in `purepdb diagnose` are the
explanation, in the report rather than in the alarm channel."""
c13_truncations: list[tuple[str, c13.C13Truncation]] = field(default_factory=list)
"""Where a C13 line-info walk stopped short of consuming its buffer, and why."""
dbi_overrun: str | None = None
"""A DBI substream whose declared size ran past the stream, if any. It was
read as far as the stream goes and the substreams after it as empty, so
modules, section contributions, the section map or the debug-header slots
may be missing."""
unrecognised_signatures: dict[int, int] = field(default_factory=dict)
"""Module streams whose first word is not a CodeView signature (C7, C11 or
C13), counted by that word. Their records are not read: the layout after
Expand Down Expand Up @@ -527,6 +534,14 @@ def warnings(self) -> list[str]:
"is in the pre-optimisation address space and does not match "
"the shipped image"
)
if self.dbi_overrun is not None:
out.append(
f"the DBI stream is shorter than its header claims "
f"({self.dbi_overrun}); that substream was read as far as the "
f"stream goes and the ones after it as empty, so modules, "
f"section contributions, the section map or the debug-header "
f"slots may be missing"
)
if self.module_list_stopped_at is not None:
out.append(
f"the module list stopped at byte "
Expand Down Expand Up @@ -587,6 +602,13 @@ def warnings(self) -> list[str]:
f"/names stream is not, so file-name offsets cannot be resolved "
f"and lines() yields nothing"
)
if self.c13_truncations:
where, first = self.c13_truncations[0]
out.append(
f"{len(self.c13_truncations)} C13 line-info section(s) stopped early; "
f"lines after that point are missing. First: {where} at "
f"byte {first.offset:#x} ({first.reason})"
)
if self.pdb_info_error is not None:
out.append(
f"the PDB Info stream cannot be read ({self.pdb_info_error}), "
Expand Down Expand Up @@ -814,7 +836,7 @@ def publics_stream(self) -> PublicsStream | None:
return None
try:
return PublicsStream.parse(self.msf.read_stream(idx))
except (ValueError, struct.error):
except (PdbError, struct.error):
return None

def public_symbols(self) -> list[codeview.PublicSymbol]:
Expand Down Expand Up @@ -1304,7 +1326,12 @@ def named_streams(self) -> dict[str, int]:

`/names` and `/LinkInfo` are what real linkers put here.
"""
return parse_named_stream_map(self.msf.read_stream(STREAM_PDB_INFO))
if not self.msf.is_valid_stream(STREAM_PDB_INFO):
return {}
try:
return parse_named_stream_map(self.msf.read_stream(STREAM_PDB_INFO))
except MsfError:
return {}

def string_table(self) -> StringTable | None:
"""The `/names` global string table, or None when the PDB has none."""
Expand Down Expand Up @@ -1400,6 +1427,7 @@ def diagnose(self) -> Diagnostics:
malformed = 0
malformed_inline = 0
line_bytes = 0
c13_truncations: list[tuple[str, c13.C13Truncation]] = []
signatures: dict[int, int] = {}
proc_records = 0
placed_sites = 0
Expand All @@ -1414,7 +1442,24 @@ def diagnose(self) -> Diagnostics:
# exactly what `module_procs()` returns since `extract_procs` drops
# only the records `parse_proc` raises on.
for mod in self.dbi.modules:
line_bytes += len(self.module_c13_bytes(mod))
c13_bytes = self.module_c13_bytes(mod)
line_bytes += len(c13_bytes)
if c13_bytes:
# The truncation reports come from re-decoding every
# DEBUG_S_LINES payload here and discarding the entries:
# 18ms of the 180ms diagnose() costs on the 3 MB sqlite
# fixture. Deliberate: `parse_lines` is the only
# implementation of the block walk,
# and a header-only twin kept beside it would drift from it.
# `lines()` cannot be reused for this: it is a generator a
# caller may never consume, and it resolves file names,
# which this loop does not want.
mod_c13_report: list[c13.C13Truncation] = []
for sub in c13.iter_subsections(c13_bytes, truncation=mod_c13_report):
if sub.kind == c13.DEBUG_S_LINES:
c13.parse_lines(sub.payload, truncation=mod_c13_report)
for t in mod_c13_report:
c13_truncations.append((f"module {mod.index} ({mod.module_name})", t))
signature, body = self._module_symbols(mod)
if signature is not None:
signatures[signature] = signatures.get(signature, 0) + 1
Expand Down Expand Up @@ -1527,9 +1572,11 @@ def diagnose(self) -> Diagnostics:
has_string_table=self.string_table() is not None,
pdb_info_error=pdb_info_error,
module_list_stopped_at=self.dbi.module_list_stopped_at,
dbi_overrun=self.dbi.substream_overrun,
private_symbols_stripped=self.dbi.is_stripped,
linker_version=self.dbi.toolchain_version,
thread_local_records=thread_locals,
c13_truncations=c13_truncations,
unrecognised_signatures=signatures,
)

Expand Down
5 changes: 0 additions & 5 deletions purepdb/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,5 @@ def cstring(self) -> str:
self.pos = end + 1
return s.decode("utf-8", errors="replace")

def align(self, boundary: int) -> None:
rem = self.pos % boundary
if rem:
self.pos += boundary - rem

def seek(self, pos: int) -> None:
self.pos = pos
Loading