From 959c5acca2de3db6521b55dd81a7bd4544ec4d4b Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 13:04:55 +0200 Subject: [PATCH 01/14] feat(reader): add an internal content-stream interpreter Lands the ISO 32000-2 section 7.8.2 operand-stack interpreter that PR 5 (text extraction) and PR 11 (image extraction) will share: Form XObject recursion with depth/cycle/budget guards, inline images per section 8.9.7, graphics and text state tracking, and a content-mode lexer relaxation for BX/EX compatibility sections. Malformed or unsupported content is reported through the diagnostics channel and interpretation continues, matching this reader's existing notify-and-continue policy rather than aborting the page. Public surface is minimal by design: PdfReaderOptions.MaxFormXObjectDepth (tighten-only, default 32) and nine new PdfReaderDiagnosticCode values in the 3xx block. The interpreter itself, its visitor interface, and the graphics/text state types stay internal until a real caller lands. --- CHANGELOG.md | 13 + docs/reader-guide.md | 32 +- .../Content/ContentInterpreter.cs | 1301 +++++++++++++++++ .../Content/ContentOperators.cs | 139 ++ src/VellumPdf.Reader/Content/GraphicsState.cs | 60 + .../Content/IContentVisitor.cs | 76 + .../Content/InlineImageAbbreviations.cs | 77 + src/VellumPdf.Reader/Content/Matrix.cs | 38 + src/VellumPdf.Reader/Content/TextState.cs | 45 + .../PdfDocumentReader.Content.cs | 19 + src/VellumPdf.Reader/PdfLexer.cs | 40 + src/VellumPdf.Reader/PdfObjectParser.cs | 8 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 111 ++ src/VellumPdf.Reader/PdfReaderOptions.cs | 22 + src/VellumPdf.Reader/PublicAPI.Unshipped.txt | 11 + src/VellumPdf.Reader/README.md | 5 +- src/VellumPdf.Reader/ReaderLimits.cs | 30 +- .../ContentInterpreterTests.cs | 834 +++++++++++ .../PdfLexerContentModeTests.cs | 165 +++ .../PdfReaderDiagnosticCodeTests.cs | 18 + .../PdfReaderOptionsTests.cs | 31 + 21 files changed, 3056 insertions(+), 19 deletions(-) create mode 100644 src/VellumPdf.Reader/Content/ContentInterpreter.cs create mode 100644 src/VellumPdf.Reader/Content/ContentOperators.cs create mode 100644 src/VellumPdf.Reader/Content/GraphicsState.cs create mode 100644 src/VellumPdf.Reader/Content/IContentVisitor.cs create mode 100644 src/VellumPdf.Reader/Content/InlineImageAbbreviations.cs create mode 100644 src/VellumPdf.Reader/Content/Matrix.cs create mode 100644 src/VellumPdf.Reader/Content/TextState.cs create mode 100644 src/VellumPdf.Reader/PdfDocumentReader.Content.cs create mode 100644 tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs create mode 100644 tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2062dec0..232aea38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). leaf and node limits each end the walk the instant they are reported, so at most one of that pair, plus the depth code's first occurrence, is ever retained in the same walk. Computed lazily, on first access, and cached for the reader's lifetime. (#98) +- **A new option, `PdfReaderOptions.MaxFormXObjectDepth`, and nine new `PdfReaderDiagnosticCode` + values in a `3xx` block reserved for content streams.** The reader now has an internal + content-stream interpreter (ISO 32000-2 §7.8.2), shared machinery the two extraction milestones + still ahead of it (text, then images) will build on; a caller cannot invoke it directly in this + release, so the nine codes below cannot yet be reported to one. `MaxFormXObjectDepth` (default 32, + tighten-only like the three resource limits it joins) is the one part of this a caller sets + today: a ceiling on Form XObject recursion depth (§8.10) the interpreter enforces once something + does call it, reporting `FormXObjectDepthExceeded` and continuing rather than recursing + unboundedly. The rest of the interpreter follows the same policy throughout: a malformed or + unsupported construct is reported, not thrown, and interpretation continues past it. That is + what the other eight codes (`ContentStreamLexError`, `UnknownOperator`, `OperandStackMalformed`, + `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, `InlineImageMalformed`, + `ContentStreamTooLarge`) each describe. (#98) ### Changed diff --git a/docs/reader-guide.md b/docs/reader-guide.md index f00764c6..6955add1 100644 --- a/docs/reader-guide.md +++ b/docs/reader-guide.md @@ -63,6 +63,7 @@ var options = new PdfReaderOptions MaxDecodedStreamBytes = 64 * 1024 * 1024, ReconstructionBudgetMultiplier = 4, MaxDiagnostics = 200, + MaxFormXObjectDepth = 16, }; using var reader = PdfReader.Open(File.OpenRead("input.pdf"), options); @@ -80,19 +81,24 @@ real `startxref` chain left for `/Prev` to extend, and a recovered trailer's `/I enough to carry into a new revision. Reconstruction also refuses outright the instant it finds any sign the document is encrypted, rather than guessing at a key. -**`MaxDecodedStreamBytes`**, **`ReconstructionBudgetMultiplier`**, and **`MaxDiagnostics`** are all -**tighten-only**. None is a spec requirement — ISO 32000-2 Annex C.1 notes that "a particular PDF -processor running on a particular device and in a particular operating environment will always have -practical limits", and Annex C.3 adds that available memory is "often much less in mobile devices -than desktop computers." The defaults (512 MiB decoded-stream ceiling, an ×8 multiplier on -reconstruction's `max(1 MiB, N × file length)` work budget, a 1000-entry diagnostics cap) are this -library's own choice for a desktop host, not something Annex C mandates. A caller on a more -constrained device, or hardening against a decompression bomb, a file engineered to burn CPU across -many decoy candidates, or a document that would otherwise report the same recoverable condition on -a huge number of objects, can lower any of the three. Raising any of them above its default throws -`ArgumentOutOfRangeException` at `Open` time: nothing above the shipped defaults has been exercised -as a safe ceiling, so these options can only make the reader stricter than it already is, never -looser. +**`MaxDecodedStreamBytes`**, **`ReconstructionBudgetMultiplier`**, **`MaxDiagnostics`**, and +**`MaxFormXObjectDepth`** are all **tighten-only**. None is a spec requirement: ISO 32000-2 Annex +C.1 notes that "a particular PDF processor running on a particular device and in a particular +operating environment will always have practical limits", and Annex C.3 adds that available memory +is "often much less in mobile devices than desktop computers." The defaults (512 MiB decoded-stream +ceiling, an ×8 multiplier on reconstruction's `max(1 MiB, N × file length)` work budget, a +1000-entry diagnostics cap, 32 levels of Form XObject recursion) are this library's own choice for +a desktop host, not something Annex C mandates. A caller on a more constrained device, or hardening +against a decompression bomb, a file engineered to burn CPU across many decoy candidates, a +document that would otherwise report the same recoverable condition on a huge number of objects, or +a page that nests Form XObjects (ISO 32000-2 §8.10) deeper than a caller wants to follow, can lower +any of the four. Raising any of them above its default throws `ArgumentOutOfRangeException` at +`Open` time: nothing above the shipped defaults has been exercised as a safe ceiling, so these +options can only make the reader stricter than it already is, never looser. +`MaxFormXObjectDepth` governs an internal content-stream interpreter this package does not expose +directly yet. Text and image extraction, still ahead on the roadmap below, will be its first +callers. It has no visible effect until then, but validates at `Open` time regardless, alongside +the other three. --- diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs new file mode 100644 index 00000000..96d7f5ef --- /dev/null +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -0,0 +1,1301 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using System.Buffers.Text; +using VellumPdf.Core; +using VellumPdf.Document; + +namespace VellumPdf.Reader.Content; + +/// +/// An operand-stack content-stream interpreter (ISO 32000-2 §7.8.2): walks a page's /Contents +/// (or a Form XObject's own content, recursively), tracking the graphics and text state, and reports +/// every recognised operator, inline image, and Form XObject boundary to an . +/// Shared machinery for the reader's later text and image extraction (#98): this type positions +/// nothing and resolves no font. It keeps state current and delimits structure. +/// +/// +/// Not thread-safe, and not reentrant across concurrent calls on the same +/// instance, matching every other stateful type in this package. A malformed document degrades +/// (a diagnostic and best-effort recovery) rather than throwing; the sole exception is +/// , which is allowed to propagate: "cannot continue" +/// belongs to the exception channel the rest of this reader already uses it for. +/// +internal sealed class ContentInterpreter +{ + // ISO 32000-2 §7.8.2 gives an operator's own operands no declared bound; this reader's own + // ceiling against a hostile or corrupted stream that never emits an operator at all. + private const int MaxOperandsPerOperator = 32; + + // §9.4.3's own TJ array holds a mix of strings and numeric adjustments; this reader's own + // ceiling on how many elements one such array may carry. + private const int MaxTjElements = 8192; + + // §8.4.4's q/Q pair; this reader's own ceiling on how deep a legitimate document nests them. + private const int MaxGraphicsStateDepth = 64; + + // §14.6.2's BMC/BDC/EMC nesting; this reader's own ceiling, mirroring MaxGraphicsStateDepth. + private const int MaxMarkedContentDepth = 64; + + // This interpreter's own budget on total successful Form XObject recursions across one page + // (§8.10), independent of PdfReaderOptions.MaxFormXObjectDepth, which bounds nesting DEPTH + // rather than the total COUNT of forms a page may draw. A wide, shallow graph (one page + // invoking the same shallow form thousands of times) is not caught by a depth cap at all. + private const int MaxFormInvocationsPerPage = 4096; + + // This reader's own ceiling on the total decoded bytes one page's /Contents may contribute, + // summed across every stream in the array (ISO 32000-2 §7.7.3.3 Table 31). + private const long MaxContentBytes = 64L * 1024 * 1024; + + private static readonly PdfName XObjectSubtypeForm = new("Form"); + private static readonly PdfName ImageMaskKey = new("ImageMask"); + private static readonly PdfName WidthKey = new("Width"); + private static readonly PdfName HeightKey = new("Height"); + private static readonly PdfName BitsPerComponentKey = new("BitsPerComponent"); + private static readonly PdfName MatrixKey = new("Matrix"); + private static readonly PdfName BBoxKey = new("BBox"); + private static readonly PdfName FontKey = new("Font"); + + private readonly PdfDocumentReader _reader; + private readonly ReaderLimits _limits; + + // ── Per-Run mutable state: reset at the top of Run, threaded through the recursive descent + // into Form XObjects via StreamContext rather than saved/restored on instance fields. ────────── + private GraphicsState _gs = new(); + private readonly Stack _gsStack = new(); + private readonly TextState _textState = new(); + private readonly List _operands = []; + private bool _operandOverflow; + private int _bxDepth; + private int _markedContentDepth; + private readonly HashSet _openForms = []; + private int _formDepth; + private int _formInvocations; + private readonly HashSet _reportedUnknownOperators = []; + private ReadOnlyMemory _currentBuffer; + + /// The current graphics state, the top of the q/Q stack, readable from + /// inside an callback. Mutated in place; a callback that needs a + /// value after the interpreter moves on must copy it. + internal GraphicsState GraphicsState => _gs; + + /// The current text-positioning state, readable the same way as + /// . + internal TextState TextState => _textState; + + /// Creates an interpreter that resolves resources and streams through + /// , under that reader's own . + internal ContentInterpreter(PdfDocumentReader reader) + { + _reader = reader; + _limits = reader.Limits; + } + + /// + /// Interprets 's content (ISO 32000-2 §7.8.2), reporting every event to + /// and every recoverable condition through this reader's diagnostics + /// channel, scoped per call via . + /// Never throws for a malformed document; is the + /// one exception allowed to propagate (see this type's own remarks). + /// + internal void Run(PdfReadPage page, IContentVisitor visitor) + { + ArgumentNullException.ThrowIfNull(page); + ArgumentNullException.ThrowIfNull(visitor); + + _gs = new GraphicsState(); + _gsStack.Clear(); + _textState.BeginText(); + _operands.Clear(); + _operandOverflow = false; + _bxDepth = 0; + _markedContentDepth = 0; + _openForms.Clear(); + _formDepth = 0; + _formInvocations = 0; + _reportedUnknownOperators.Clear(); + + var diagnostics = _reader.CreateContentDiagnosticScope(); + var pageIndex = page.Index; + + var buffer = BuildPageContentBuffer(page, diagnostics, pageIndex, out var soleObjectNumber); + if (buffer.IsEmpty) + return; + + var ctx = new StreamContext(page.Resources, soleObjectNumber); + InterpretStream(buffer, ctx, visitor, pageIndex, diagnostics); + } + + // ── /Contents resolution and concatenation (ISO 32000-2 §7.7.3.3 Table 31) ───────────────────── + + private ReadOnlyMemory BuildPageContentBuffer( + PdfReadPage page, DiagnosticSink diagnostics, int pageIndex, out int? soleObjectNumber) + { + soleObjectNumber = null; + var raw = page.Dictionary.Get(PdfName.Contents); + if (raw is null or PdfNull) + return ReadOnlyMemory.Empty; // Optional; a page with no content draws nothing. + + var chunks = new List(); + var contributingStreams = 0; + int? soleObjectNumberLocal = null; + + void AddElement(PdfObject element) + { + if (element is not PdfIndirectReference elementRef) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + "A /Contents element is not an indirect reference to a stream (ISO 32000-2 " + + "§7.7.3.3 Table 31); it was skipped.", + pageIndex: pageIndex); + return; + } + + var stream = _reader.ResolveStream(elementRef); + if (stream is null) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"/Contents object {elementRef.ObjectNumber} does not resolve to a stream; it " + + "was skipped.", + elementRef.ObjectNumber, pageIndex: pageIndex); + return; + } + + byte[]? decoded; + try + { + decoded = _reader.GetDecodedStreamData(stream); + } + catch (InvalidDataException) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"Content stream object {stream.ObjectNumber} failed to decode; it was skipped.", + stream.ObjectNumber, pageIndex: pageIndex); + return; + } + + if (decoded is null) + { + // An image filter (DCTDecode, JPXDecode, ...) in the chain is never valid on a + // content stream, which must decode to PDF operator syntax. + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"Content stream object {stream.ObjectNumber} carries an image filter and " + + "cannot be decoded as content; it was skipped.", + stream.ObjectNumber, pageIndex: pageIndex); + return; + } + + chunks.Add(decoded); + contributingStreams++; + soleObjectNumberLocal = contributingStreams == 1 ? stream.ObjectNumber : null; + } + + if (raw is PdfArray directArray) + { + foreach (var element in Enumerate(directArray)) + AddElement(element); + } + else + { + // A single reference: could name a stream directly, or (nonconformant but tolerated) + // an array object. + var resolved = _reader.ResolveValue(raw); + if (resolved is PdfArray arr) + { + foreach (var element in Enumerate(arr)) + AddElement(element); + } + else + { + AddElement(raw); + } + } + + soleObjectNumber = soleObjectNumberLocal; + return Concatenate(chunks, diagnostics, pageIndex); + } + + private static IEnumerable Enumerate(PdfArray array) + { + for (var i = 0; i < array.Count; i++) + yield return array[i]; + } + + // Joins each stream's decoded bytes with a single '\n' between them, per ISO 32000-2 §7.7.3.3 + // Table 31's own text: "the division between streams may occur only at the boundaries between + // lexical tokens". So a token split across two streams (e.g. one ending "BT" and the next + // starting immediately with "ET") is not glued into one token by naive concatenation. Enforces + // MaxContentBytes across the total, truncating at the last whitespace boundary within budget + // rather than mid-token, so what IS interpreted is a clean prefix rather than one broken by an + // artificial cut. + private static ReadOnlyMemory Concatenate( + List chunks, DiagnosticSink diagnostics, int pageIndex) + { + if (chunks.Count == 0) + return ReadOnlyMemory.Empty; + + long total = 0; + foreach (var chunk in chunks) + total += chunk.Length + 1; // +1 for the separator this method inserts after each chunk + + if (total <= MaxContentBytes) + { + var buffer = new byte[total]; + var pos = 0; + foreach (var chunk in chunks) + { + chunk.CopyTo(buffer, pos); + pos += chunk.Length; + buffer[pos++] = (byte)'\n'; + } + return buffer; + } + + // Over budget: copy whole chunks while they fit, then take as much of the chunk that would + // overflow as fits, backing off to the nearest preceding whitespace byte so the cut falls on + // a token boundary rather than through the middle of one. + var capped = new byte[MaxContentBytes]; + var written = 0; + foreach (var chunk in chunks) + { + var remaining = (int)Math.Min(MaxContentBytes - written, int.MaxValue); + if (remaining <= 0) + break; + + if (chunk.Length + 1 <= remaining) + { + chunk.CopyTo(capped, written); + written += chunk.Length; + capped[written++] = (byte)'\n'; + continue; + } + + var take = Math.Min(chunk.Length, remaining); + while (take > 0 && !PdfLexer.IsWhitespaceByte(chunk[take - 1])) + take--; + Array.Copy(chunk, capped, take); + written += take; + break; + } + + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamTooLarge, + $"The page's /Contents exceeded the {MaxContentBytes / (1024 * 1024)} MiB decoded-size " + + "cap; interpretation stopped there.", + pageIndex: pageIndex); + + return new ReadOnlyMemory(capped, 0, written); + } + + // ── Main interpretation loop ───────────────────────────────────────────────────────────────── + + /// Resources and diagnostic-attribution identity for one content stream being + /// interpreted: the page's own for the top-level call, a Form XObject's own (falling back to + /// its invoker's per §8.10.2) for a recursive one. + private readonly record struct StreamContext(PdfDictionary? Resources, int? DiagObjectNumber); + + private void InterpretStream( + ReadOnlyMemory data, StreamContext ctx, IContentVisitor visitor, int pageIndex, + DiagnosticSink diagnostics) + { + var outerBuffer = _currentBuffer; + _currentBuffer = data; + try + { + var lexer = new PdfLexer(data, contentStreamMode: true); + var parser = new PdfObjectParser(lexer); + + try + { + while (true) + { + lexer.SkipWhitespaceAndComments(); + if (lexer.AtEnd) + break; + + var offset = lexer.Position; + var token = lexer.NextToken(); + if (token.Kind == TokenKind.EndOfInput) + break; + + switch (token.Kind) + { + case TokenKind.Integer or TokenKind.Real: + HandleNumber(token, ctx, diagnostics, pageIndex); + break; + + case TokenKind.LiteralString: + PushOperand( + PdfObjectParser.DecodeLiteralString(token.Raw), ctx, diagnostics, + pageIndex); + break; + + case TokenKind.HexString: + PushOperand( + PdfObjectParser.DecodeHexString(token.Raw), ctx, diagnostics, + pageIndex); + break; + + case TokenKind.Name: + PushOperand(PdfObjectParser.ParseName(token), ctx, diagnostics, pageIndex); + break; + + case TokenKind.ArrayBegin: + lexer.Seek(offset); + PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); + break; + + case TokenKind.DictBegin: + lexer.Seek(offset); + PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); + break; + + case TokenKind.Keyword: + { + var raw = token.Raw.Span; + if (raw.SequenceEqual("true"u8)) + PushOperand(PdfBoolean.True, ctx, diagnostics, pageIndex); + else if (raw.SequenceEqual("false"u8)) + PushOperand(PdfBoolean.False, ctx, diagnostics, pageIndex); + else if (raw.SequenceEqual("null"u8)) + PushOperand(PdfNull.Instance, ctx, diagnostics, pageIndex); + else if (raw.SequenceEqual("BI"u8)) + { + if (!HandleInlineImage(lexer, parser, ctx, visitor, diagnostics, pageIndex, offset)) + goto endOfStream; + } + else + { + var name = System.Text.Encoding.Latin1.GetString(raw); + HandleOperator(name, offset, ctx, visitor, diagnostics, pageIndex); + } + break; + } + + default: + throw new InvalidDataException( + $"Unexpected token {token.Kind} at content-stream offset {offset}."); + } + } + endOfStream:; + } + catch (InvalidDataException) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + "The content stream's syntax could not be interpreted past this point; " + + "interpretation of it stopped here.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + } + finally + { + _currentBuffer = outerBuffer; + } + } + + // ── Operand collection ─────────────────────────────────────────────────────────────────────── + + private void HandleNumber(Token token, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (TryParseOperandNumber(token.Raw.Span, token.Kind == TokenKind.Real, out var value)) + { + PushOperand(value!, ctx, diagnostics, pageIndex); + return; + } + + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + "A numeric operand did not parse, or was not finite; it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + + // System.Buffers.Text.Utf8Parser backs this, but against a normalised copy of the token's + // bytes, not the raw span, because PDF's own numeric grammar + // (ISO 32000-2 §7.3.3) allows a bare leading or trailing decimal point ("-.5", "6.") that the + // BCL's own double formats do not universally accept the same way across runtimes; padding a + // missing digit on either side of '.' sidesteps that without reimplementing number parsing. + private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, out PdfObject? result) + { + result = null; + if (raw.IsEmpty) + return false; + + var negative = false; + var span = raw; + if (span[0] is (byte)'+' or (byte)'-') + { + negative = span[0] == (byte)'-'; + span = span[1..]; + } + + if (!isReal) + { + if (span.IsEmpty || !Utf8Parser.TryParse(span, out long value, out var consumed) || consumed != span.Length) + return false; + result = new PdfInteger(negative ? -value : value); + return true; + } + + Span padded = stackalloc byte[span.Length + 2]; + var len = 0; + if (span.IsEmpty || span[0] == (byte)'.') + padded[len++] = (byte)'0'; + span.CopyTo(padded[len..]); + len += span.Length; + if (len == 0 || padded[len - 1] == (byte)'.') + padded[len++] = (byte)'0'; + + if (!Utf8Parser.TryParse(padded[..len], out double d, out var consumedReal) || consumedReal != len) + return false; + if (negative) + d = -d; + if (!double.IsFinite(d)) + return false; + + result = new PdfReal(d); + return true; + } + + private void PushOperand(PdfObject value, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_operandOverflow) + return; + + if (_operands.Count >= MaxOperandsPerOperator) + { + _operandOverflow = true; + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"More than {MaxOperandsPerOperator} operands accumulated before an operator; the " + + "next operator was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + + _operands.Add(value); + } + + private void ClearOperands() + { + _operands.Clear(); + _operandOverflow = false; + } + + // ── Operator dispatch ──────────────────────────────────────────────────────────────────────── + + private void HandleOperator( + string name, int offset, StreamContext ctx, IContentVisitor visitor, DiagnosticSink diagnostics, + int pageIndex) + { + if (!ContentOperators.IsKnown(name)) + { + // ISO 32000-2 §7.8.2: "an error shall occur" outside a compatibility section; this + // reader instead notifies and continues. Deliberately does NOT clear the operand stack + // (this reader's own leniency, distinct from Table 33's "ignored ... along with + // operands" for a genuine future operator inside BX/EX): the most common way an + // unrecognised keyword appears in an otherwise-conforming stream is a stray "R" left + // over from indirect-reference syntax that §7.8.2 forbids in content streams at all + // ("Indirect objects and object references shall not be permitted"), and the operands + // that precede it usually belong to whatever REAL operator follows, not to "R" itself. + if (_bxDepth == 0 && _reportedUnknownOperators.Add(name)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.UnknownOperator, + $"'{name}' is not one of the operators ISO 32000-2 Annex A Table A.1 defines; " + + "it was ignored.", + pageIndex: pageIndex); + } + return; + } + + if (name is "BX") + { + _bxDepth++; + EmitAndClear(name, offset, visitor); + return; + } + if (name is "EX") + { + if (_bxDepth > 0) + _bxDepth--; + EmitAndClear(name, offset, visitor); + return; + } + + var expected = ContentOperators.ExpectedOperandCount(name); + if (_operandOverflow) + { + ClearOperands(); + return; // Already reported when the overflow itself happened. + } + if (expected != ContentOperators.Variable && _operands.Count != expected) + { + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"'{name}' expects {expected} operand(s) but {_operands.Count} were on the stack; " + + "it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + return; + } + + // ISO 32000-2 §7.8.2: "Dictionaries shall be permitted as operands only by certain specific + // operators". BDC and DP (§14.6.2) are the only two in Annex A Table A.1 that take one. + if (name is not ("BDC" or "DP")) + { + foreach (var operand in _operands) + { + if (operand is not PdfDictionary) + continue; + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"'{name}' does not accept a dictionary operand (ISO 32000-2 §7.8.2); it was " + + "dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + return; + } + } + + switch (name) + { + case "TJ": + if (_operands[0] is not PdfArray tjArray || tjArray.Count > MaxTjElements) + { + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"TJ's array operand is missing or exceeds {MaxTjElements} elements; it was " + + "dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + return; + } + break; + + case "q": + PushGraphicsState(ctx, diagnostics, pageIndex); + break; + + case "Q": + PopGraphicsState(ctx, diagnostics, pageIndex); + break; + + case "cm": + _gs.Ctm = new Matrix( + NumberOperand(0), NumberOperand(1), NumberOperand(2), NumberOperand(3), + NumberOperand(4), NumberOperand(5)).Concat(_gs.Ctm); + break; + + case "BT": + _textState.BeginText(); + break; + + case "Tc": + _gs.CharSpacing = NumberOperand(0); + break; + + case "Tw": + _gs.WordSpacing = NumberOperand(0); + break; + + case "Tz": + _gs.HorizontalScaling = NumberOperand(0); + break; + + case "TL": + _gs.Leading = NumberOperand(0); + break; + + case "Tf": + ValidateFontResource(ctx, diagnostics, pageIndex); + _gs.Font = _operands[0]; + _gs.FontSize = NumberOperand(1); + break; + + case "Tr": + _gs.RenderMode = (int)NumberOperand(0); + break; + + case "Ts": + _gs.Rise = NumberOperand(0); + break; + + case "Td": + _textState.MoveTextPosition(NumberOperand(0), NumberOperand(1)); + break; + + case "TD": + { + var ty = NumberOperand(1); + _gs.Leading = -ty; + _textState.MoveTextPosition(NumberOperand(0), ty); + break; + } + + case "Tm": + _textState.SetTextMatrix(new Matrix( + NumberOperand(0), NumberOperand(1), NumberOperand(2), NumberOperand(3), + NumberOperand(4), NumberOperand(5))); + break; + + case "T*": + _textState.MoveTextPosition(0, -_gs.Leading); + break; + + case "BDC" or "BMC": + PushMarkedContent(ctx, diagnostics, pageIndex); + break; + + case "EMC": + PopMarkedContent(ctx, diagnostics, pageIndex); + break; + + case "cs" or "CS": + ValidateColorSpaceResource(name, ctx, diagnostics, pageIndex); + break; + + case "sh": + ValidateNamedResource(name, ShadingKey, ctx, diagnostics, pageIndex); + break; + + case "gs": + HandleExtGState(ctx, diagnostics, pageIndex); + break; + + case "Do": + HandleDo(offset, ctx, visitor, diagnostics, pageIndex); + return; // HandleDo emits "Do" itself before recursing. + + default: + break; // Recognised but state-inert: path, colour, clipping, rendering operators. + } + + EmitAndClear(name, offset, visitor); + } + + private void EmitAndClear(string name, int offset, IContentVisitor visitor) + { + visitor.OnOperator(name, _operands, offset); + ClearOperands(); + } + + private double NumberOperand(int index) => _operands[index] switch + { + PdfInteger i => i.Value, + PdfReal r => r.Value, + _ => 0, + }; + + // ── q/Q, BMC/BDC/EMC ───────────────────────────────────────────────────────────────────────── + + private void PushGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_gsStack.Count >= MaxGraphicsStateDepth) + { + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"The graphics-state stack exceeded {MaxGraphicsStateDepth} nested 'q' saves; " + + "further saves were ignored.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + _gsStack.Push(_gs); + _gs = _gs.Clone(); + } + + private void PopGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_gsStack.Count == 0) + { + // An unbalanced 'q' still open at end of stream is fine (nothing downstream needs the + // state restored past the last operator this interpreter saw); an unbalanced 'Q' is the + // opposite problem (a restore with nothing to restore), so this one is reported. + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + "'Q' with no matching 'q' on the graphics-state stack; it was ignored.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + _gs = _gsStack.Pop(); + } + + private void PushMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_markedContentDepth >= MaxMarkedContentDepth) + { + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"Marked-content nesting exceeded {MaxMarkedContentDepth} levels; further " + + "'BMC'/'BDC' operators were ignored.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + _markedContentDepth++; + } + + private void PopMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_markedContentDepth == 0) + { + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + "'EMC' with no matching 'BMC'/'BDC'; it was ignored.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + _markedContentDepth--; + } + + // ── Resource lookups (ISO 32000-2 §7.8.3) ─────────────────────────────────────────────────── + + private static readonly PdfName ShadingKey = PdfName.Shading; + + private static readonly HashSet _standaloneColorSpaceNames = + new(StringComparer.Ordinal) { "DeviceGray", "DeviceRGB", "DeviceCMYK", "Pattern" }; + + private void ValidateFontResource(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) => + ValidateNamedResource("Tf", PdfName.Font, ctx, diagnostics, pageIndex); + + private void ValidateColorSpaceResource( + string op, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_operands[0] is not PdfName csName || _standaloneColorSpaceNames.Contains(csName.Value)) + return; // §8.6.3: the four device/pattern spaces are never resource-dictionary entries. + ValidateNamedResource(op, PdfName.ColorSpace, ctx, diagnostics, pageIndex); + } + + private void ValidateNamedResource( + string op, PdfName category, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_operands.Count == 0 || _operands[0] is not PdfName name) + return; + + if (ctx.Resources is not null && TryGetResource(ctx.Resources, category, name, out _)) + return; + + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'{op}' names '/{name.Value}', absent from the applicable /Resources /{category.Value} " + + "dictionary.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + + private bool TryGetResource(PdfDictionary resources, PdfName category, PdfName name, out PdfObject value) + { + value = PdfNull.Instance; + var categoryRaw = resources.Get(category); + if (categoryRaw is null) + return false; + if (_reader.ResolveValue(categoryRaw) is not PdfDictionary categoryDict) + return false; + var raw = categoryDict.Get(name); + if (raw is null or PdfNull) + return false; + value = raw; + return true; + } + + // ── gs (ISO 32000-2 §8.4.5 Table 57) ──────────────────────────────────────────────────────── + + private void HandleExtGState(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (_operands[0] is not PdfName gsName) + return; + + if (ctx.Resources is null || !TryGetResource(ctx.Resources, PdfName.ExtGState, gsName, out var raw)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'gs' names '/{gsName.Value}', absent from the applicable /Resources /ExtGState " + + "dictionary.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + + if (_reader.ResolveValue(raw) is not PdfDictionary extGState) + return; + + // Table 57: /Font is "an array of the form [font size] where font shall be an indirect + // reference to a font dictionary". Every other ExtGState entry is out of scope for this + // interpreter: it neither positions text nor renders colour or transparency. + if (extGState.Get(FontKey) is PdfArray fontArray && fontArray.Count == 2) + { + _gs.Font = fontArray[0]; + _gs.FontSize = ReadNumber(fontArray[1]); + } + } + + private static double ReadNumber(PdfObject obj) => obj switch + { + PdfInteger i => i.Value, + PdfReal r => r.Value, + _ => 0, + }; + + // ── Do / Form XObjects (ISO 32000-2 §8.10) ────────────────────────────────────────────────── + + private void HandleDo( + int offset, StreamContext ctx, IContentVisitor visitor, DiagnosticSink diagnostics, int pageIndex) + { + var xobjectNameOperand = _operands.Count == 1 ? _operands[0] : null; + EmitAndClear("Do", offset, visitor); + + if (xobjectNameOperand is not PdfName xobjectName) + return; + + if (ctx.Resources is null || !TryGetResource(ctx.Resources, PdfName.XObject, xobjectName, out var entryRaw)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'Do' names '/{xobjectName.Value}', absent from the applicable /Resources /XObject " + + "dictionary.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return; + } + + if (entryRaw is not PdfIndirectReference xobjectRef) + return; + + var stream = _reader.ResolveStream(xobjectRef); + if (stream is null) + return; + + if (stream.Dictionary.Get(PdfName.Subtype) is not PdfName subtype || !subtype.Equals(XObjectSubtypeForm)) + return; // An Image XObject, or anything else: no recursion; the caller already got Do. + + var objectNumber = stream.ObjectNumber; + + if (_formInvocations >= MaxFormInvocationsPerPage) + { + diagnostics.Report( + PdfReaderDiagnosticCode.FormXObjectBudgetExceeded, + $"The page invoked more than {MaxFormInvocationsPerPage} Form XObjects; further " + + "'Do' recursions were skipped for the rest of the page.", + pageIndex: pageIndex); + return; + } + + if (_formDepth >= _limits.MaxFormXObjectDepth) + { + diagnostics.Report( + PdfReaderDiagnosticCode.FormXObjectDepthExceeded, + $"Form XObject recursion exceeded {_limits.MaxFormXObjectDepth} levels; this 'Do' " + + "was not followed.", + objectNumber, pageIndex: pageIndex); + return; + } + + if (!_openForms.Add(objectNumber)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.FormXObjectCycle, + $"Form XObject {objectNumber} invokes itself, directly or through a chain of nested " + + "'Do' operators; the recursive invocation was skipped.", + objectNumber, pageIndex: pageIndex); + return; + } + + _formInvocations++; + _formDepth++; + try + { + var formDict = stream.Dictionary; + var matrix = ReadFormMatrix(formDict); + var bbox = ReadFormBBox(formDict); + // §8.10.2: a form's /Resources is optional but strongly recommended; when absent, the + // invoking content stream's own resources apply. + var formResources = ResolveDictionaryEntry(formDict, PdfName.Resources) ?? ctx.Resources; + + visitor.OnFormBegin(formDict, matrix, bbox, objectNumber, offset); + try + { + byte[]? decoded; + try + { + decoded = _reader.GetDecodedStreamData(stream); + } + catch (InvalidDataException) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"Form XObject {objectNumber}'s content stream failed to decode.", + objectNumber, pageIndex: pageIndex); + decoded = null; + } + + if (decoded is not null) + { + var formCtx = new StreamContext(formResources, objectNumber); + InterpretStream(decoded, formCtx, visitor, pageIndex, diagnostics); + } + } + finally + { + visitor.OnFormEnd(objectNumber); + } + } + finally + { + _formDepth--; + _openForms.Remove(objectNumber); + } + } + + private PdfDictionary? ResolveDictionaryEntry(PdfDictionary dict, PdfName key) => + dict.Get(key) is { } raw ? _reader.ResolveValue(raw) as PdfDictionary : null; + + private Matrix ReadFormMatrix(PdfDictionary formDict) + { + if (formDict.Get(MatrixKey) is PdfArray arr && arr.Count == 6 && TryReadNumbers(arr, out var v)) + return new Matrix(v[0], v[1], v[2], v[3], v[4], v[5]); + return Matrix.Identity; // §8.10.2 Table 93's own default. + } + + private PdfRectangle? ReadFormBBox(PdfDictionary formDict) + { + if (formDict.Get(BBoxKey) is PdfArray arr && arr.Count == 4 && TryReadNumbers(arr, out var v)) + { + return new PdfRectangle( + Math.Min(v[0], v[2]), Math.Min(v[1], v[3]), Math.Max(v[0], v[2]), Math.Max(v[1], v[3])); + } + return null; + } + + private bool TryReadNumbers(PdfArray array, out double[] values) + { + values = new double[array.Count]; + for (var i = 0; i < array.Count; i++) + { + var resolved = _reader.ResolveValue(array[i]); + if (resolved is not (PdfInteger or PdfReal)) + return false; + values[i] = ReadNumber(resolved); + } + return true; + } + + // ── Inline images (ISO 32000-2 §8.9.7) ────────────────────────────────────────────────────── + + /// Parses one BI…ID…EI inline image starting with BI + /// already consumed by the caller. Returns when the image's key/value + /// dictionary or data could not be delimited at all: the caller stops interpreting this + /// stream, since nothing past this point can be resynchronised reliably. + private bool HandleInlineImage( + PdfLexer lexer, PdfObjectParser parser, StreamContext ctx, IContentVisitor visitor, + DiagnosticSink diagnostics, int pageIndex, int biOffset) + { + var dict = new PdfDictionary(); + + while (true) + { + lexer.SkipWhitespaceAndComments(); + if (lexer.AtEnd) + { + ReportInlineImageMalformed( + "the 'ID' operator was never reached before the end of the content stream", ctx, + diagnostics, pageIndex); + return false; + } + + var keyTok = lexer.NextToken(); + if (keyTok.Kind == TokenKind.Keyword && keyTok.Raw.Span.SequenceEqual("ID"u8)) + break; + + if (keyTok.Kind != TokenKind.Name) + { + ReportInlineImageMalformed( + "a key name or 'ID' was expected in the inline image dictionary", ctx, diagnostics, + pageIndex); + return false; + } + + var key = InlineImageAbbreviations.ExpandKey(PdfObjectParser.ParseName(keyTok)); + var isColorSpaceKey = key.Equals(PdfName.ColorSpace); + var isFilterKey = key.Equals(PdfName.Filter); + + lexer.SkipWhitespaceAndComments(); + var valueStart = lexer.Position; + var valueTok = lexer.NextToken(); + + PdfObject value; + if (valueTok.Kind == TokenKind.Name && (isColorSpaceKey || isFilterKey)) + { + value = InlineImageAbbreviations.ExpandColorSpaceOrFilterName( + PdfObjectParser.ParseName(valueTok), isColorSpaceKey); + } + else if (valueTok.Kind == TokenKind.ArrayBegin && isFilterKey) + { + lexer.Seek(valueStart); + var arr = (PdfArray)parser.ParseObject(); + var items = new List(arr.Count); + for (var i = 0; i < arr.Count; i++) + { + items.Add(arr[i] is PdfName elName + ? InlineImageAbbreviations.ExpandColorSpaceOrFilterName(elName, isColorSpace: false) + : arr[i]); + } + value = new PdfArray(items); + } + else + { + lexer.Seek(valueStart); + value = parser.ParseObject(); + } + + dict.Set(key, value); + } + + // §8.9.7: "Unless the image uses ASCIIHexDecode or ASCII85Decode ..., the ID operator shall + // be followed by a single white-space character, and the next character shall be + // interpreted as the first byte of image data." Consuming at most one whitespace byte here + // is correct for every filter, including AHx/A85, since a producer is free to include that + // one separating byte regardless (the spec exempts them from being REQUIRED to, not from + // being ALLOWED to). + if (lexer.TryPeek() is var b && b >= 0 && PdfLexer.IsWhitespaceByte((byte)b)) + lexer.Seek(lexer.Position + 1); + + var dataStart = lexer.Position; + var filterNames = CollectFilterNames(dict); + var hasDisallowedFilter = filterNames.Any(f => + f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); + + var length = TryLengthFromDictionary(dict, dataStart, out var lengthPastEnd); + if (lengthPastEnd) + { + ReportInlineImageMalformed( + "/L names a length past the end of the content stream", ctx, diagnostics, pageIndex); + } + + if (length is null && filterNames.Count == 0) + length = TryComputeUnfilteredLength(dict, ctx, dataStart, diagnostics, pageIndex); + + if (length is null) + { + var scanEnd = ScanForEi(dataStart); + if (scanEnd is null) + { + ReportInlineImageMalformed( + "no 'EI' operator delimiting the image data could be found", ctx, diagnostics, + pageIndex); + return false; + } + length = scanEnd.Value - dataStart; + } + + var dataEnd = dataStart + length.Value; + if (dataEnd < dataStart || dataEnd > _currentBuffer.Length) + { + ReportInlineImageMalformed( + "the computed image data length runs past the end of the content stream", ctx, + diagnostics, pageIndex); + return false; + } + + var data = _currentBuffer.Slice(dataStart, length.Value); + + var resyncPos = SkipToEi(dataEnd); + if (resyncPos is null) + { + ReportInlineImageMalformed( + "no 'EI' operator was found at the computed end of the image data", ctx, diagnostics, + pageIndex); + return false; + } + + lexer.Seek(resyncPos.Value); + + if (hasDisallowedFilter) + { + ReportInlineImageMalformed( + "the image uses a filter (JBIG2Decode, JPXDecode, or Crypt) never valid on an " + + "inline image (ISO 32000-2 §8.9.7)", ctx, diagnostics, pageIndex); + return true; + } + + visitor.OnInlineImage(dict, data, biOffset); + return true; + } + + private void ReportInlineImageMalformed( + string reason, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) => + diagnostics.Report( + PdfReaderDiagnosticCode.InlineImageMalformed, $"Inline image malformed: {reason}.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + + private List CollectFilterNames(PdfDictionary dict) + { + var names = new List(); + if (dict.Get(PdfName.Filter) is { } filterVal) + { + switch (filterVal) + { + case PdfName n: names.Add(n); break; + case PdfArray arr: + for (var i = 0; i < arr.Count; i++) + if (arr[i] is PdfName elName) + names.Add(elName); + break; + } + } + return names; + } + + // Tier (a): /L, PDF 2.0's own Table 91 entry (§8.9.7). See InlineImageAbbreviations' own remarks + // for why this cites Table 91 rather than Table 93, which ISO 32000-2 reserves for a Form + // XObject dictionary's unrelated entries (§8.10.2). + private int? TryLengthFromDictionary(PdfDictionary dict, int dataStart, out bool pastEnd) + { + pastEnd = false; + if (dict.Get(PdfName.Length) is not PdfInteger lengthObj || lengthObj.Value < 0) + return null; + + var length = lengthObj.Value; + if (length > int.MaxValue || dataStart + length > _currentBuffer.Length) + { + pastEnd = true; + return null; + } + + return (int)length; + } + + // Tier (b): unfiltered data, Height x rowBytes, where rowBytes = ceil(Width x BitsPerComponent + // x components / 8) (§8.9.7). + private int? TryComputeUnfilteredLength( + PdfDictionary dict, StreamContext ctx, int dataStart, DiagnosticSink diagnostics, int pageIndex) + { + var isMask = dict.Get(ImageMaskKey) is PdfBoolean { Value: true }; + var width = ReadIntEntry(dict, WidthKey); + var height = ReadIntEntry(dict, HeightKey); + var bpc = isMask ? 1 : ReadIntEntry(dict, BitsPerComponentKey); + + if (width is null || height is null || bpc is null || width < 0 || height < 0 || bpc <= 0) + { + ReportInlineImageMalformed( + "an unfiltered image is missing /W, /H, or /BPC needed to compute its data length", + ctx, diagnostics, pageIndex); + return null; + } + + var components = isMask ? 1 : ResolveComponentCount(dict, ctx, diagnostics, pageIndex); + if (components <= 0) + return null; // Unknown colour space: fall back to the EI scan (tier c). + + var rowBytes = ((long)width.Value * bpc.Value * components + 7) / 8; + var total = rowBytes * height.Value; + if (total < 0 || dataStart + total > _currentBuffer.Length) + return null; // Fall back to the EI scan rather than trusting a runaway computed size. + + return (int)total; + } + + private static int? ReadIntEntry(PdfDictionary dict, PdfName key) => dict.Get(key) switch + { + PdfInteger i => (int)i.Value, + PdfReal r => (int)r.Value, + _ => null, + }; + + private int ResolveComponentCount( + PdfDictionary dict, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (dict.Get(PdfName.ColorSpace) is not PdfName csName) + return -1; + + if (csName.Equals(PdfName.DeviceRGB)) return 3; + if (csName.Value == "DeviceCMYK") return 4; + if (csName.Value == "DeviceGray") return 1; + if (csName.Value == "Indexed") return 1; + + // A named resource colour space (§8.9.7: "the value of the ColorSpace entry may also be the + // name of a colour space in the ColorSpace subdictionary of the current resource + // dictionary"). Resolving its component count in general needs full colour-space semantics + // (ICCBased /N, DeviceN's component array, ...) this interpreter does not implement; + // reporting ResourceMissing when the name is absent, and otherwise falling back to the EI + // scan (tier c) either way, keeps this interpreter's own scope bounded to delimiting the + // image rather than fully understanding its colour space. + if (ctx.Resources is null || !TryGetResource(ctx.Resources, PdfName.ColorSpace, csName, out _)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"An inline image's /CS names '/{csName.Value}', absent from the applicable " + + "/Resources /ColorSpace dictionary.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + return -1; + } + + // Tier (c): scan for whitespace-EI-whitespace/EOF, accepted only when what follows lexes as + // operators or EOF in content mode (the false-EI-inside-DCT-data problem). + private int? ScanForEi(int dataStart) + { + var span = _currentBuffer.Span; + for (var i = dataStart; i + 1 < span.Length; i++) + { + if (span[i] != (byte)'E' || span[i + 1] != (byte)'I') + continue; + + var precededByWhitespace = i == dataStart || PdfLexer.IsWhitespaceByte(span[i - 1]); + if (!precededByWhitespace) + continue; + + var after = i + 2; + var followedOk = after >= span.Length + || PdfLexer.IsWhitespaceByte(span[after]) || PdfLexer.IsDelimiterByte(span[after]); + if (!followedOk) + continue; + + if (!LooksLikeResyncPoint(after)) + continue; + + var dataEnd = i > dataStart && PdfLexer.IsWhitespaceByte(span[i - 1]) ? i - 1 : i; + return dataEnd; + } + return null; + } + + // Confirms an 'EI' candidate at exactly a known offset (used once tier a/b already computed a + // length) by requiring the bytes there literally spell "EI" preceded and followed the way §8.9.7 + // describes; unlike ScanForEi this does not search, it verifies one position. + private int? SkipToEi(int dataEnd) + { + var pos = dataEnd; + var span = _currentBuffer.Span; + while (pos < span.Length && PdfLexer.IsWhitespaceByte(span[pos])) + pos++; + if (pos + 1 >= span.Length || span[pos] != (byte)'E' || span[pos + 1] != (byte)'I') + return null; + return pos + 2; + } + + // Bounded lookahead: lexes up to a handful of tokens in content mode from a candidate resync + // point and accepts it if none of them throws before either running out of tokens to try or + // reaching end of input. This is what rejects a coincidental "EI" byte pair sitting inside + // DCT-compressed binary data, which is followed by more binary noise the lexer chokes on almost + // immediately (an unterminated literal string or hex string is the most common trip). + private bool LooksLikeResyncPoint(int pos) + { + var probe = new PdfLexer(_currentBuffer, contentStreamMode: true); + probe.Seek(pos); + try + { + for (var i = 0; i < 8; i++) + { + if (probe.AtEnd) + return true; + if (probe.NextToken().Kind == TokenKind.EndOfInput) + return true; + } + return true; + } + catch (InvalidDataException) + { + return false; + } + } +} diff --git a/src/VellumPdf.Reader/Content/ContentOperators.cs b/src/VellumPdf.Reader/Content/ContentOperators.cs new file mode 100644 index 00000000..fb92c6d8 --- /dev/null +++ b/src/VellumPdf.Reader/Content/ContentOperators.cs @@ -0,0 +1,139 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +namespace VellumPdf.Reader.Content; + +/// +/// The 73 content-stream operators ISO 32000-2 Annex A Table A.1 lists, alphabetically, with each +/// one's operand count for 's stack-discipline check. A count of +/// marks the four colour operators (SC, sc, SCN, +/// scn) whose own tables (§8.6.8) give them 1 to 4 numeric operands plus, for the N +/// suffix, an optional trailing pattern name: no single fixed count describes them, so this +/// interpreter accepts any operand count for them rather than reporting +/// on a legitimately variable call. +/// +internal static class ContentOperators +{ + /// Sentinel operand count for an operator this interpreter does not arity-check. + internal const int Variable = -1; + + private static readonly Dictionary _arity = new(StringComparer.Ordinal) + { + // Table 59: path-painting operators. + ["b"] = 0, + ["B"] = 0, + ["b*"] = 0, + ["B*"] = 0, + ["f"] = 0, + ["F"] = 0, // deprecated alias of f (ISO 32000-2 §8.5.3.1) + ["f*"] = 0, + ["n"] = 0, + ["s"] = 0, + ["S"] = 0, + + // Table 33: compatibility operators. + ["BX"] = 0, + ["EX"] = 0, + + // Table 351/352: marked-content operators. + ["BDC"] = 2, + ["BMC"] = 1, + ["DP"] = 2, + ["EMC"] = 0, + ["MP"] = 1, + + // Table 105/107: text object / text-showing operators. + ["BT"] = 0, + ["ET"] = 0, + ["Tj"] = 1, + ["TJ"] = 1, + ["'"] = 1, + ["\""] = 3, + + // Table 58: path construction operators. + ["c"] = 6, + ["h"] = 0, + ["l"] = 2, + ["m"] = 2, + ["re"] = 4, + ["v"] = 4, + ["y"] = 4, + + // Table 56: graphics state operators. + ["cm"] = 6, + ["d"] = 2, + ["gs"] = 1, + ["i"] = 1, + ["j"] = 1, + ["J"] = 1, + ["M"] = 1, + ["ri"] = 1, + ["w"] = 1, + + // Table 73: colour operators. + ["CS"] = 1, + ["cs"] = 1, + ["G"] = 1, + ["g"] = 1, + ["K"] = 4, + ["k"] = 4, + ["RG"] = 3, + ["rg"] = 3, + ["SC"] = Variable, + ["sc"] = Variable, + ["SCN"] = Variable, + ["scn"] = Variable, + + // Table 60: clipping path operators. + ["W"] = 0, + ["W*"] = 0, + + // Table 76: shading operator. + ["sh"] = 1, + + // Table 86: XObject operator. + ["Do"] = 1, + + // Table 90: inline image operators. Never reach the generic arity check (ContentInterpreter + // intercepts BI before generic operand collection begins), listed here only so the known-set + // membership check (IsKnown) recognises them as legitimate operators. + ["BI"] = 0, + ["ID"] = 0, + ["EI"] = 0, + + // Table 103: text state operators. + ["Tc"] = 1, + ["Tf"] = 2, + ["Tr"] = 1, + ["Ts"] = 1, + ["Tw"] = 1, + ["Tz"] = 1, + ["TL"] = 1, + + // Table 106: text-positioning operators. + ["Td"] = 2, + ["TD"] = 2, + ["Tm"] = 6, + ["T*"] = 0, + + // Table 111: Type 3 font operators. + ["d0"] = 2, + ["d1"] = 6, + + // Table 56: graphics state save/restore. + ["q"] = 0, + ["Q"] = 0, + }; + + /// True when is one of the 73 operators Annex A Table + /// A.1 lists. + internal static bool IsKnown(string operatorName) => _arity.ContainsKey(operatorName); + + /// + /// The operand count expects, or for the + /// four colour operators this interpreter does not arity-check. Throws + /// for a name would report false for; + /// every call site checks that first. + /// + internal static int ExpectedOperandCount(string operatorName) => _arity[operatorName]; +} diff --git a/src/VellumPdf.Reader/Content/GraphicsState.cs b/src/VellumPdf.Reader/Content/GraphicsState.cs new file mode 100644 index 00000000..49783d5b --- /dev/null +++ b/src/VellumPdf.Reader/Content/GraphicsState.cs @@ -0,0 +1,60 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using VellumPdf.Core; + +namespace VellumPdf.Reader.Content; + +/// +/// The subset of a content stream's graphics state (ISO 32000-2 §8.4) this interpreter tracks: +/// the current transformation matrix, and the text state parameters §9.3.1 makes part of the +/// graphics state (so they save and restore with q/Q along with everything else +/// here). Colour, line width, dash pattern, clipping path, and every other §8.4 parameter are +/// recognised at the operator level (Annex A Table A.1) but not tracked: this interpreter's job is +/// to keep and the text parameters current for a later caller (text extraction, +/// then image extraction, per #98) to read during a visitor callback, not to reproduce a full +/// graphics pipeline. +/// +internal sealed class GraphicsState +{ + /// The current transformation matrix, concatenated by cm (§8.3.4). + internal Matrix Ctm { get; set; } = Matrix.Identity; + + /// Character spacing, Tc (§9.3.2). Added to the horizontal displacement of every + /// glyph shown, including the byte that follows a stretch of encoded space. + internal double CharSpacing { get; set; } + + /// Word spacing, Tw (§9.3.3). Added only after a single-byte code 32. + internal double WordSpacing { get; set; } + + /// Horizontal scaling, Tz (§9.3.4), as a percentage; 100 is unscaled. + internal double HorizontalScaling { get; set; } = 100; + + /// Leading, TL (§9.3.5): the line-to-line advance T* and ' use, and + /// what TD sets from its own ty operand. + internal double Leading { get; set; } + + /// + /// The font operand from the last Tf or gs-with-/Font (§9.3.6, §8.4.5 Table + /// 57): a naming a /Resources /Font entry for Tf, or the + /// an ExtGState's own /Font array names as its font directly (Table + /// 57 requires that to be an indirect reference to a font dictionary) for gs. Not resolved + /// by this interpreter either way: font resolution and glyph positioning are for a later caller + /// to do, not this interpreter's own job. + /// + internal PdfObject? Font { get; set; } + + /// Font size in unscaled text-space units, from the same Tf or gs call + /// that set . + internal double FontSize { get; set; } + + /// Text rendering mode, Tr (§9.3.7): 0–7 per Table 104, not validated here. + internal int RenderMode { get; set; } + + /// Text rise, Ts (§9.3.8): vertical displacement, in unscaled text-space units. + internal double Rise { get; set; } + + /// Deep-copies this state for a q push; Q discards the top and restores + /// the state it copied from. + internal GraphicsState Clone() => (GraphicsState)MemberwiseClone(); +} diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs new file mode 100644 index 00000000..032bee81 --- /dev/null +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -0,0 +1,76 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using VellumPdf.Core; +using VellumPdf.Document; + +namespace VellumPdf.Reader.Content; + +/// +/// Receives the events produces while walking a page's content +/// stream (ISO 32000-2 §7.8.2). A caller reads and +/// from inside these callbacks to see the state as of the +/// event; both are mutated in place, so a callback that needs a value after the interpreter has +/// moved on must copy it out rather than hold the reference. +/// +internal interface IContentVisitor +{ + /// + /// Called for every recognised operator (Annex A Table A.1) the interpreter accepts, i.e. after + /// its own operand-count and stack-discipline checks pass; a malformed or unrecognised operator + /// never reaches this callback (see + /// and ). + /// + /// The operator keyword, e.g. "Tj" or "re". + /// + /// This operator's operands, in the order they appeared. Owned by the interpreter and reused for + /// the next operator once this call returns, so a callback that needs an operand's value after + /// returning must copy it out, not hold the list. + /// + /// The byte offset, within the buffer currently being interpreted (the + /// page's own concatenated content, or the current Form XObject's own content), of the operator + /// keyword's first byte. + void OnOperator(string operatorName, IReadOnlyList operands, int offset); + + /// + /// Called once an inline image (ISO 32000-2 §8.9.7) has been fully delimited and its dictionary + /// decoded, with Table 91/92 abbreviations already expanded to their full names. Not called at all + /// for an image this interpreter could not delimit or decode + /// (see ). + /// + /// The inline image's key/value pairs, with every Table 91/92 + /// abbreviation already expanded (e.g. /W to /Width, /CS /G to + /// /ColorSpace /DeviceGray). + /// The image's own (still filtered, undecoded) sample data: the bytes between + /// ID and EI, excluding the delimiting white space. + /// The byte offset of the BI operator that began this image. + void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset); + + /// + /// Called immediately before the interpreter recurses into a Form XObject's own content, once a + /// Do operator (already reported through ) has been resolved to a + /// /Subtype /Form stream and every recursion guard (depth, cycle, per-page budget) has + /// passed. Matched by exactly one call once the form's own content + /// finishes interpreting, even if that content raises further nested + /// calls of its own in between. + /// + /// The form XObject's own stream dictionary. + /// The form's /Matrix (ISO 32000-2 §8.10.2 Table 93), or + /// when absent or malformed. The interpreter does not compose this + /// with the CTM itself; that is left to the caller. + /// The form's /BBox (Table 93, Required), or + /// when absent or malformed. + /// The form stream's own indirect object number. + /// The byte offset of the Do operator that invoked this form, in the + /// buffer the interpreter was walking at the time, i.e. the INVOKING stream's own offset space, + /// not the form's. + void OnFormBegin( + PdfDictionary formDictionary, Matrix formMatrix, PdfRectangle? boundingBox, int objectNumber, + int offset); + + /// Called once a Form XObject's own content has finished interpreting, matching the + /// most recent unmatched call. + /// The same form stream object number + /// reported. + void OnFormEnd(int objectNumber); +} diff --git a/src/VellumPdf.Reader/Content/InlineImageAbbreviations.cs b/src/VellumPdf.Reader/Content/InlineImageAbbreviations.cs new file mode 100644 index 00000000..8efe8d06 --- /dev/null +++ b/src/VellumPdf.Reader/Content/InlineImageAbbreviations.cs @@ -0,0 +1,77 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using VellumPdf.Core; + +namespace VellumPdf.Reader.Content; + +/// +/// The abbreviations ISO 32000-2 §8.9.7 permits inside an inline image's BI…ID +/// key-value pairs: Table 91's key names, and Table 92's colour-space and filter-name values. +/// +/// +/// /L (the inline-image length entry) belongs to Table 91, alongside every other +/// inline-image key abbreviation; ISO 32000-2 reserves Table 93 for a Form XObject dictionary's own +/// entries (§8.10.2, an unrelated table two clauses later), so this implementation and its +/// citations cite Table 91 for it. +/// +internal static class InlineImageAbbreviations +{ + // Table 91: Entries in an inline image object. + private static readonly Dictionary _keys = new(StringComparer.Ordinal) + { + ["BPC"] = new PdfName("BitsPerComponent"), + ["CS"] = PdfName.ColorSpace, + ["D"] = new PdfName("Decode"), + ["DP"] = new PdfName("DecodeParms"), + ["F"] = PdfName.Filter, + ["H"] = new PdfName("Height"), + ["IM"] = new PdfName("ImageMask"), + ["I"] = new PdfName("Interpolate"), // ambiguous with Table 92's /Indexed "I"; see below + ["L"] = PdfName.Length, + ["W"] = new PdfName("Width"), + }; + + // Table 92: Additional abbreviations in an inline image object (colour spaces and filter names). + // "I" is deliberately excluded from this table: the same one-letter abbreviation "I" stands for + // /Interpolate as a Table 91 KEY abbreviation and for /Indexed as a Table 92 colour-space VALUE + // abbreviation, so which it means depends on where it appears (a dictionary key vs. a /CS + // value), not on the bytes alone. ExpandKey and ExpandColorSpaceOrFilterName each resolve it + // from their own position in the caller's parse, not from this shared table. + private static readonly Dictionary _colorSpacesAndFilters = new(StringComparer.Ordinal) + { + ["G"] = new PdfName("DeviceGray"), + ["RGB"] = PdfName.DeviceRGB, + ["CMYK"] = new PdfName("DeviceCMYK"), + ["AHx"] = new PdfName("ASCIIHexDecode"), + ["A85"] = new PdfName("ASCII85Decode"), + ["LZW"] = new PdfName("LZWDecode"), + ["Fl"] = PdfName.FlateDecode, + ["RL"] = new PdfName("RunLengthDecode"), + ["CCF"] = PdfName.CCITTFaxDecode, + ["DCT"] = PdfName.DCTDecode, + }; + + private static readonly PdfName _indexed = new("Indexed"); + + /// Expands a Table 91 key abbreviation to its full name, or returns + /// unchanged when it is not one of Table 91's abbreviations (including + /// when it is already a full name). + internal static PdfName ExpandKey(PdfName key) => + _keys.TryGetValue(key.Value, out var full) ? full : key; + + /// + /// Expands a Table 92 colour-space-or-filter-name abbreviation. Colour space and filter names + /// share one lookup because Table 92 lists them together and neither can collide with the + /// other's abbreviations (distinct strings), except for the "I" ambiguity documented on + /// : is what tells this + /// method the caller's "I" is a colour-space value (→ /Indexed) rather than the Table 91 key + /// abbreviation (→ /Interpolate) handles instead. + /// + internal static PdfName ExpandColorSpaceOrFilterName(PdfName name, bool isColorSpace) + { + if (isColorSpace && name.Value == "I") // Indexed (Table 92); ExpandKey handles Interpolate + return _indexed; + return _colorSpacesAndFilters.TryGetValue(name.Value, out var full) ? full : name; + } +} diff --git a/src/VellumPdf.Reader/Content/Matrix.cs b/src/VellumPdf.Reader/Content/Matrix.cs new file mode 100644 index 00000000..887fea27 --- /dev/null +++ b/src/VellumPdf.Reader/Content/Matrix.cs @@ -0,0 +1,38 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +namespace VellumPdf.Reader.Content; + +/// +/// A PDF transformation matrix (ISO 32000-2 §8.3.4), written in content streams as six operands +/// a b c d e f representing +/// [ a b 0 ; c d 0 ; e f 1 ]. Applied to a row vector on the left: a point +/// (x, y) maps to (a·x + c·y + e, b·x + d·y + f). +/// +internal readonly record struct Matrix(double A, double B, double C, double D, double E, double F) +{ + /// The identity matrix [1 0 0 1 0 0]: §8.3.4's own default for /Matrix + /// and the CTM at the start of every content stream. + internal static readonly Matrix Identity = new(1, 0, 0, 1, 0, 0); + + /// + /// Composes this matrix with so that applying the result to a point is + /// the same as applying this matrix first, then , i.e. + /// this.Concat(other) == this × other in row-vector convention. This is the operation + /// cm uses to fold its operand matrix into the CTM (§8.3.4: "the new matrix shall be the + /// result of premultiplying the specified matrix with the current matrix", + /// CTM_new = M × CTM_old, i.e. m.Concat(ctm)), and the one Td/TD use + /// to fold a translation into the text line matrix (§9.4.2). + /// + internal Matrix Concat(Matrix other) => new( + A: A * other.A + B * other.C, + B: A * other.B + B * other.D, + C: C * other.A + D * other.C, + D: C * other.B + D * other.D, + E: E * other.A + F * other.C + other.E, + F: E * other.B + F * other.D + other.F); + + /// Builds the translation matrix [1 0 0 1 tx ty] that Td/TD + /// concatenate onto the text line matrix (§9.4.2). + internal static Matrix Translation(double tx, double ty) => new(1, 0, 0, 1, tx, ty); +} diff --git a/src/VellumPdf.Reader/Content/TextState.cs b/src/VellumPdf.Reader/Content/TextState.cs new file mode 100644 index 00000000..9b447168 --- /dev/null +++ b/src/VellumPdf.Reader/Content/TextState.cs @@ -0,0 +1,45 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +namespace VellumPdf.Reader.Content; + +/// +/// The text-positioning matrices ISO 32000-2 §9.4.2 tracks per text object: the text matrix and +/// text line matrix. Deliberately separate from : §9.4.2 says these two +/// matrices are "not part of the graphics state" and are not saved or restored by q/Q. +/// Only BT resets them (to identity), and only Td, TD, Tm, and +/// T* update them. Not stacked; the interpreter owns exactly one live instance. +/// +internal sealed class TextState +{ + /// The text matrix, Tm: maps text space to the CTM's user space. + internal Matrix TextMatrix { get; set; } = Matrix.Identity; + + /// The text line matrix: the text matrix at the start of the current line, what + /// Td/TD/T* advance from. + internal Matrix TextLineMatrix { get; set; } = Matrix.Identity; + + /// BT (§9.4.1): resets both matrices to identity at the start of a text object. + internal void BeginText() + { + TextMatrix = Matrix.Identity; + TextLineMatrix = Matrix.Identity; + } + + /// + /// Td/TD (§9.4.2): advances the text line matrix by [1 0 0 1 tx ty] + /// premultiplied against the current text line matrix, then makes the text matrix track it. + /// + internal void MoveTextPosition(double tx, double ty) + { + TextLineMatrix = Matrix.Translation(tx, ty).Concat(TextLineMatrix); + TextMatrix = TextLineMatrix; + } + + /// Tm (§9.4.2): replaces both matrices outright rather than concatenating. + internal void SetTextMatrix(Matrix m) + { + TextMatrix = m; + TextLineMatrix = m; + } +} diff --git a/src/VellumPdf.Reader/PdfDocumentReader.Content.cs b/src/VellumPdf.Reader/PdfDocumentReader.Content.cs new file mode 100644 index 00000000..64e2b4ee --- /dev/null +++ b/src/VellumPdf.Reader/PdfDocumentReader.Content.cs @@ -0,0 +1,19 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +namespace VellumPdf.Reader; + +public sealed partial class PdfDocumentReader +{ + /// + /// Creates a fresh scope (see ) + /// forwarding into this reader's own diagnostics. ContentInterpreter.Run is the first real + /// caller of (see that method's own remarks), creating + /// one scope per page interpreted so a caller who runs the interpreter over the same page more + /// than once (text extraction, then image extraction, per #98) gets a fresh, page-scoped + /// dedupe set each time rather than sharing one that silently goes quiet on a second pass's own + /// first occurrence of a condition the first pass already reported into it. Reports made through + /// the returned scope still land in , under this reader's own cap. + /// + internal DiagnosticSink CreateContentDiagnosticScope() => _diagnostics.CreateScope(); +} diff --git a/src/VellumPdf.Reader/PdfLexer.cs b/src/VellumPdf.Reader/PdfLexer.cs index 8be3a032..3e8cbf33 100644 --- a/src/VellumPdf.Reader/PdfLexer.cs +++ b/src/VellumPdf.Reader/PdfLexer.cs @@ -77,6 +77,18 @@ internal sealed class PdfLexer { private readonly ReadOnlyMemory _data; + // Off by default: every existing consumer (the object parser, the 11 Conformance rules that + // build their own PdfLexer, and every non-content test) relies on the file-structure lexer's + // stricter rule that a lone '{', '}', or unmatched '>' is malformed PDF (they never appear + // outside a content stream at all: ISO 32000-2 §7.2.2 lists them as delimiters with no syntax + // of their own anywhere else). A content stream is a different grammar: it may carry them as + // one-byte PostScript-heritage tokens inside a BX/EX compatibility section (§7.8.2), which + // ContentInterpreter needs to lex as harmless unknown-operator keywords instead of aborting the + // whole page over a construct §7.8.2 explicitly says to tolerate. Opt-in, not a runtime + // detection of "is this a content stream", keeps that decision with the caller that already + // knows which grammar applies instead of guessing from the bytes. + private readonly bool _contentStreamMode; + /// Current byte offset within . public int Position { get; private set; } @@ -93,6 +105,19 @@ public PdfLexer(ReadOnlyMemory data, int offset = 0) Position = offset; } + /// + /// Creates a lexer over in content-stream mode (ISO 32000-2 §7.8.2): a + /// lone {, }, or unmatched > lexes as a one-byte + /// token instead of throwing . Every other token kind, and every + /// other lexer behaviour, is byte-identical to the default constructor's. This is strictly an + /// opt-in relaxation, never a stricter mode. + /// + internal PdfLexer(ReadOnlyMemory data, bool contentStreamMode) + { + _data = data; + _contentStreamMode = contentStreamMode; + } + // ── ISO 32000-2 §7.2.2 — whitespace bytes ───────────────────────────── private static bool IsWhitespace(byte b) => b is 0 or 9 or 10 or 12 or 13 or 32; @@ -224,6 +249,14 @@ public Token NextToken() Position += 2; return new Token(TokenKind.DictEnd, _data.Slice(start, 2)); } + if (_contentStreamMode) + { + // §7.8.2's compatibility section (BX/EX) tolerates PostScript-heritage tokens this + // grammar has no other use for; ContentInterpreter treats the resulting one-byte + // Keyword as just another unrecognised operator, same as '{' and '}' below. + Position++; + return new Token(TokenKind.Keyword, _data.Slice(start, 1)); + } throw new InvalidDataException( $"Unexpected '>' at offset {Position} (not part of '>>'); malformed PDF."); } @@ -244,6 +277,13 @@ public Token NextToken() if (IsRegular(b)) return ReadKeyword(start); + if (_contentStreamMode && (b == (byte)'{' || b == (byte)'}')) + { + // See the '>' branch above and this lexer's content-stream-mode constructor doc. + Position++; + return new Token(TokenKind.Keyword, _data.Slice(start, 1)); + } + throw new InvalidDataException( $"Unexpected byte 0x{b:X2} at offset {Position}."); } diff --git a/src/VellumPdf.Reader/PdfObjectParser.cs b/src/VellumPdf.Reader/PdfObjectParser.cs index 5962e40c..1117f8ac 100644 --- a/src/VellumPdf.Reader/PdfObjectParser.cs +++ b/src/VellumPdf.Reader/PdfObjectParser.cs @@ -292,7 +292,13 @@ private static PdfReal ParseReal(Token token) return new PdfReal(d); } - private static PdfName ParseName(Token token) + /// + /// Decodes a token's raw bytes (including the leading /) into + /// a , applying #XX escapes (ISO 32000-2 §7.3.5). Internal, not + /// private, so ContentInterpreter's own name-operand decoding shares this rather than + /// duplicating it. + /// + internal static PdfName ParseName(Token token) { // token.Raw includes the leading '/' var raw = token.Raw.Span[1..]; // skip '/' diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index c6388852..67b73ced 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -349,6 +349,108 @@ public enum PdfReaderDiagnosticCode /// PageTreeNodeLimitExceeded = 207, + // ── 3xx: content streams ──────────────────────────────────────────────────────────────────── + + /// + /// The content-stream interpreter (ISO 32000-2 §7.8.2) hit an + /// from the lexer or object parser partway through a page's content, or a member of the page's + /// /Contents array (§7.7.3.3 Table 31) did not resolve to a usable stream at all: a + /// non-stream element, a reference that fails to resolve, or a stream whose filter chain could + /// not be decoded (including one carrying an image filter, which this interpreter never + /// attempts to decode as content). Interpretation of that stream stops at the point of failure; + /// operators already reported to the caller's visitor before the failure are kept, and, for a + /// multi-stream /Contents array specifically, interpretation resumes with the next + /// stream in the array rather than abandoning the whole page. + /// + ContentStreamLexError = 300, + + /// + /// A content-stream keyword token is not one of the 73 operators ISO 32000-2 Annex A Table A.1 + /// lists, and it did not appear inside a BX/EX compatibility section (§7.8.2). ISO + /// 32000-2 says "an error shall occur" for this case outside such a section; this reader instead + /// reports it and continues, the same notify-and-continue choice every other diagnostic in this + /// channel makes. Reported at most once per (page, operator name) rather than once per + /// occurrence, since a producer that emits a future operator this reader does not know about + /// typically emits it many times on the same page. Silent inside a compatibility section, per + /// Table 33's own text: "Unrecognised operators ... shall be ignored without error." + /// + UnknownOperator = 301, + + /// + /// A content stream's operand-stack discipline broke down in one of several ways this + /// interpreter groups under one code rather than one each, since every case has the same + /// remedy: drop the offending operator (or, for an unbalanced Q/EMC, drop the + /// pop) and keep interpreting. Covers: more than 32 operands accumulated before an operator + /// (§7.8.2 gives an operator's operands no declared bound of its own; this reader's own + /// ceiling), a TJ array (§9.4.3) with more than 8192 elements, a number token that does + /// not parse or is not finite, a dictionary operand on an operator other than BDC/DP + /// (§7.8.2: "Dictionaries shall be permitted as operands only by certain specific operators"), + /// a known operator invoked with the wrong operand count for its own arity (Annex A Table A.1), + /// an unbalanced Q with no matching q on the graphics-state stack, more than 64 + /// nested q saves, or an unbalanced EMC/deeply nested BMC/BDC + /// (§14.6.2) past the same 64-deep cap. An unbalanced q still open at the end of a + /// content stream is not reported: nothing downstream of this interpreter needs the graphics + /// state restored past the last operator it saw. + /// + OperandStackMalformed = 302, + + /// + /// A Form XObject Do (ISO 32000-2 §8.10) recursed past + /// levels deep. Descent into that subtree + /// stops; the Do operator itself is still reported to the caller's visitor, only the + /// recursive walk into the form's own content is skipped. + /// + FormXObjectDepthExceeded = 303, + + /// + /// A Form XObject's own content, directly or through a chain of nested Do invocations, + /// draws itself again: the same indirect object number already open on the interpreter's own + /// recursion stack (ISO 32000-2 §8.10 describes no cycle of this kind as legal; a form's content + /// is ordinary content that may invoke any XObject, so nothing in the format itself prevents a + /// producer from writing one). Reported once per cycle found; the recursive invocation that + /// would close the cycle is skipped rather than recursing forever. + /// + FormXObjectCycle = 304, + + /// + /// A single page invoked Form XObjects (successful Do recursions, counted across the + /// whole page, not per subtree) more than 4096 times. Descent into any further form stops for + /// the rest of the page; operators already reported before the budget was reached are kept, and + /// interpretation of the page's own (non-form) content continues past the point where the + /// budget was hit. + /// + FormXObjectBudgetExceeded = 305, + + /// + /// A Do, gs, Tf, cs/CS, or sh operator, or an inline + /// image's /CS (/ColorSpace) entry, named a resource absent from the applicable + /// /Resources subdictionary (ISO 32000-2 §7.8.3): the page's own, or, inside a Form + /// XObject, that form's own /Resources falling back to its parent's when absent + /// (§8.10.2). The operator is still reported to the caller's visitor; only the interpreter's own + /// attempt to resolve the name failed. + /// + ResourceMissing = 306, + + /// + /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded: a filter this + /// interpreter never applies to inline image data (JBIG2Decode, JPXDecode, or + /// Crypt: §8.9.7 itself excludes all three from inline-image use), a missing ID + /// or EI operator, a missing /W, /H, or /BPC where the image's shape + /// requires one to compute the data length, or an /L (§8.9.7, Table 91; PDF 2.0) past the + /// end of the stream. The image is skipped (its data is still delimited well enough for + /// interpretation of the rest of the content stream to continue), and no inline-image callback + /// is raised for it. + /// + InlineImageMalformed = 307, + + /// + /// A page's /Contents (ISO 32000-2 §7.7.3.3 Table 31), concatenated across every stream + /// in the array with a newline inserted between streams so a token is never glued across a + /// stream boundary, exceeded 64 MiB of decoded bytes. Interpretation proceeds up to the cap and + /// stops there; operators reported before the cap was reached are kept. + /// + ContentStreamTooLarge = 308, + // ── 9xx: reserved ─────────────────────────────────────────────────────────────────────────── /// @@ -408,6 +510,15 @@ internal static class PdfReaderDiagnosticSeverities PdfReaderDiagnosticCode.PageAttributeInvalid => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.PageTreeNodeMalformed => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.ContentStreamLexError => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.UnknownOperator => PdfReaderDiagnosticSeverity.Info, + PdfReaderDiagnosticCode.OperandStackMalformed => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.FormXObjectDepthExceeded => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.FormXObjectCycle => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.FormXObjectBudgetExceeded => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.ResourceMissing => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.InlineImageMalformed => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.ContentStreamTooLarge => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.DiagnosticsSuppressed => PdfReaderDiagnosticSeverity.Warning, _ => throw new UnreachableException($"No severity is mapped for {code}."), }; diff --git a/src/VellumPdf.Reader/PdfReaderOptions.cs b/src/VellumPdf.Reader/PdfReaderOptions.cs index dcf7f0b9..61ff36ba 100644 --- a/src/VellumPdf.Reader/PdfReaderOptions.cs +++ b/src/VellumPdf.Reader/PdfReaderOptions.cs @@ -124,4 +124,26 @@ public sealed class PdfReaderOptions /// of 1000 or below the floor of 1. /// public int MaxDiagnostics { get; init; } = ReaderLimits.DefaultMaxDiagnostics; + + /// + /// The nesting-depth ceiling the internal content-stream interpreter enforces on a recursive + /// Form XObject Do invocation (ISO 32000-2 §8.10): a form that itself draws a form, that + /// draws a form, and so on. Defaults to 32. ISO 32000-2 places no limit on this: a form's own + /// content stream is ordinary PDF syntax that may invoke any XObject, including another form, + /// so the ceiling is this processor's own choice against adversarial input (Annex C.1, + /// informative, on practical processing limits), the same rationale as + /// above. Exceeding it reports + /// PdfReaderDiagnosticCode.FormXObjectDepthExceeded and stops descending into that + /// subtree rather than recursing further (an uncatchable + /// is the alternative a hostile self-referential or deeply-chained form graph would otherwise + /// risk). Tighten-only, matching , + /// , and above: nothing + /// about this cap is a spec requirement, so a caller may lower it but not raise it past the + /// shipped default. + /// + /// + /// Thrown by when set above the default + /// of 32 or below the floor of 1. + /// + public int MaxFormXObjectDepth { get; init; } = ReaderLimits.DefaultMaxFormXObjectDepth; } diff --git a/src/VellumPdf.Reader/PublicAPI.Unshipped.txt b/src/VellumPdf.Reader/PublicAPI.Unshipped.txt index 808eddd0..225d8c53 100644 --- a/src/VellumPdf.Reader/PublicAPI.Unshipped.txt +++ b/src/VellumPdf.Reader/PublicAPI.Unshipped.txt @@ -37,15 +37,22 @@ VellumPdf.Reader.PdfReaderDiagnostic.PageIndex.get -> int? VellumPdf.Reader.PdfReaderDiagnostic.Severity.get -> VellumPdf.Reader.PdfReaderDiagnosticSeverity override VellumPdf.Reader.PdfReaderDiagnostic.ToString() -> string! VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.ContentStreamLexError = 300 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.ContentStreamTooLarge = 308 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.DecodeParmsMalformed = 108 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.DecodedStreamLimitExceeded = 111 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.DiagnosticsSuppressed = 900 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.FilterArrayElementNotName = 106 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.FilterNull = 105 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.FilterValueMalformed = 107 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.FormXObjectBudgetExceeded = 305 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.FormXObjectCycle = 304 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.FormXObjectDepthExceeded = 303 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.InlineImageMalformed = 307 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.ObjectGenerationMismatch = 104 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.ObjectHeaderMismatch = 103 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.ObjectStreamContainerUnreadable = 102 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.OperandStackMalformed = 302 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.OrphanedObjectStreamMembersDropped = 101 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.PageAttributeInvalid = 205 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.PageTreeCycle = 201 -> VellumPdf.Reader.PdfReaderDiagnosticCode @@ -55,7 +62,9 @@ VellumPdf.Reader.PdfReaderDiagnosticCode.PageTreeLeafLimitExceeded = 203 -> Vell VellumPdf.Reader.PdfReaderDiagnosticCode.PageTreeMissing = 200 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded = 207 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.PageTreeNodeMalformed = 206 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.ResourceMissing = 306 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.UnknownFilter = 110 -> VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.UnknownOperator = 301 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.UnsupportedPredictor = 109 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.XrefReconstructed = 100 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticSeverity @@ -69,6 +78,8 @@ VellumPdf.Reader.PdfReaderOptions.MaxDecodedStreamBytes.get -> long VellumPdf.Reader.PdfReaderOptions.MaxDecodedStreamBytes.init -> void VellumPdf.Reader.PdfReaderOptions.MaxDiagnostics.get -> int VellumPdf.Reader.PdfReaderOptions.MaxDiagnostics.init -> void +VellumPdf.Reader.PdfReaderOptions.MaxFormXObjectDepth.get -> int +VellumPdf.Reader.PdfReaderOptions.MaxFormXObjectDepth.init -> void VellumPdf.Reader.PdfReaderOptions.Password.get -> string? VellumPdf.Reader.PdfReaderOptions.Password.init -> void VellumPdf.Reader.PdfReaderOptions.PdfReaderOptions() -> void diff --git a/src/VellumPdf.Reader/README.md b/src/VellumPdf.Reader/README.md index fd5544d0..ab4f715e 100644 --- a/src/VellumPdf.Reader/README.md +++ b/src/VellumPdf.Reader/README.md @@ -15,8 +15,9 @@ The PDF reader of **[VellumPdf](https://github.com/Tim81/VellumPDF)**, a depende is encrypted rather than guessing at a key. `PdfDocumentReader.WasReconstructed` reports whether a given document took this path. - Tighten-only resource limits (`PdfReaderOptions.MaxDecodedStreamBytes`, - `ReconstructionBudgetMultiplier`) for a caller hardening against a decompression bomb or a file - engineered to burn CPU — both can only lower the shipped default, never raise it. + `ReconstructionBudgetMultiplier`, `MaxDiagnostics`, `MaxFormXObjectDepth`) for a caller hardening + against a decompression bomb, a file engineered to burn CPU, or a deeply nested Form XObject + graph: each can only lower the shipped default, never raise it. - Exposes the document catalog and digital signatures. Stream decoding is internal for now: the public surface reads structure and signatures, not page content — see the roadmap below. - Writes a decrypted copy of an encrypted document (`PdfDocumentReader.SaveDecrypted`), refusing a diff --git a/src/VellumPdf.Reader/ReaderLimits.cs b/src/VellumPdf.Reader/ReaderLimits.cs index de0bdda4..a628d3cc 100644 --- a/src/VellumPdf.Reader/ReaderLimits.cs +++ b/src/VellumPdf.Reader/ReaderLimits.cs @@ -30,11 +30,16 @@ namespace VellumPdf.Reader; /// The cap enforces on — /// see . /// +/// +/// The nesting-depth ceiling ContentInterpreter enforces on recursive Form XObject Do +/// invocations (ISO 32000-2 §8.10); see . +/// internal readonly record struct ReaderLimits( long MaxDecodedBytes, long MaxAggregateReconstructionDecodeBytes, int ReconstructionBudgetMultiplier, - int MaxDiagnostics) + int MaxDiagnostics, + int MaxFormXObjectDepth) { /// The processor's own choice of default per-decode ceiling: 512 MiB. internal const long DefaultMaxDecodedBytes = 512L * 1024 * 1024; @@ -63,9 +68,20 @@ internal readonly record struct ReaderLimits( /// internal const int MinMaxDiagnostics = 1; + /// The processor's own choice of default Form XObject recursion depth ceiling: 32. + internal const int DefaultMaxFormXObjectDepth = 32; + + /// + /// The floor a caller may tighten down to: 1 + /// (a Form XObject may still be invoked, but may not itself invoke another one). + /// + internal const int MinMaxFormXObjectDepth = 1; + /// The library's built-in ceilings — what every read used before this option existed. internal static ReaderLimits Defaults { get; } = - new(DefaultMaxDecodedBytes, DefaultMaxDecodedBytes, DefaultReconstructionBudgetMultiplier, DefaultMaxDiagnostics); + new( + DefaultMaxDecodedBytes, DefaultMaxDecodedBytes, DefaultReconstructionBudgetMultiplier, + DefaultMaxDiagnostics, DefaultMaxFormXObjectDepth); /// /// Validates 's three resource knobs and resolves them into the @@ -115,6 +131,14 @@ internal static ReaderLimits Resolve(PdfReaderOptions options) $"{nameof(PdfReaderOptions.MaxDiagnostics)} must be between " + $"{MinMaxDiagnostics} and {DefaultMaxDiagnostics}."); - return new ReaderLimits(maxDecodedBytes, maxDecodedBytes, multiplier, maxDiagnostics); + var maxFormXObjectDepth = options.MaxFormXObjectDepth; + if (maxFormXObjectDepth < MinMaxFormXObjectDepth || maxFormXObjectDepth > DefaultMaxFormXObjectDepth) + throw new ArgumentOutOfRangeException( + nameof(PdfReaderOptions.MaxFormXObjectDepth), maxFormXObjectDepth, + $"{nameof(PdfReaderOptions.MaxFormXObjectDepth)} must be between " + + $"{MinMaxFormXObjectDepth} and {DefaultMaxFormXObjectDepth}."); + + return new ReaderLimits( + maxDecodedBytes, maxDecodedBytes, multiplier, maxDiagnostics, maxFormXObjectDepth); } } diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs new file mode 100644 index 00000000..351981bb --- /dev/null +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -0,0 +1,834 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using System.IO.Compression; +using System.Text; +using CsCheck; +using VellumPdf.Core; +using VellumPdf.Document; +using VellumPdf.Reader.Content; + +namespace VellumPdf.Reader.Tests; + +/// +/// Exercises the internal content-stream interpreter (#98, part 3): +/// walking a page's /Contents per ISO 32000-2 §7.8.2. Fixtures are hand-built byte strings +/// (the PageTreeTests style) so tests control exact operator sequences, resource shapes, and +/// deliberately malformed constructs a document writer cannot produce. +/// +public sealed class ContentInterpreterTests +{ + // ── Fixture builders ───────────────────────────────────────────────────────────────────────── + + private sealed record Obj(int Num, string Dict, byte[]? Stream = null); + + private static byte[] BuildPdf(int rootObjectNumber, params Obj[] objects) + { + var ms = new MemoryStream(); + void W(string s) => ms.Write(Encoding.ASCII.GetBytes(s)); + + W("%PDF-1.7\n"); + + var maxNum = objects.Max(o => o.Num); + var offsets = new int?[maxNum + 1]; + foreach (var obj in objects.OrderBy(o => o.Num)) + { + offsets[obj.Num] = (int)ms.Position; + if (obj.Stream is null) + { + W($"{obj.Num} 0 obj\n{obj.Dict}\nendobj\n"); + } + else + { + var trimmed = obj.Dict.TrimEnd(); + var withLength = trimmed[..^2].TrimEnd() + $" /Length {obj.Stream.Length} >>"; + W($"{obj.Num} 0 obj\n{withLength}\nstream\n"); + ms.Write(obj.Stream); + W("\nendstream\nendobj\n"); + } + } + + var xrefOffset = (int)ms.Position; + W($"xref\n0 {maxNum + 1}\n"); + W("0000000000 65535 f \n"); + for (var i = 1; i <= maxNum; i++) + { + W(offsets[i] is { } offset + ? $"{offset:D10} 00000 n \n" + : "0000000000 65535 f \n"); + } + W($"trailer\n<< /Size {maxNum + 1} /Root {rootObjectNumber} 0 R >>\n"); + W($"startxref\n{xrefOffset}\n%%EOF\n"); + + return ms.ToArray(); + } + + private static byte[] Flate(byte[] raw) + { + var ms = new MemoryStream(); + using (var z = new ZLibStream(ms, CompressionLevel.Fastest, leaveOpen: true)) + z.Write(raw); + return ms.ToArray(); + } + + /// Builds a one-page document whose page's own content is , + /// with as its (already-formatted) /Resources value and + /// any (fonts, XObjects, ExtGStates, ...) alongside it. + private static byte[] BuildPageDoc( + string content, string resourcesDict = "<< >>", params Obj[] extraObjects) + { + var objs = new List + { + new(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + $"/Resources {resourcesDict} /Contents 4 0 R >>"), + new(4, "<< >>", Encoding.ASCII.GetBytes(content)), + }; + objs.AddRange(extraObjects); + return BuildPdf(1, [.. objs]); + } + + /// Same as , but takes the page's content as raw bytes. + /// Needed whenever a fixture embeds bytes outside the ASCII range, which + /// Encoding.ASCII.GetBytes would otherwise silently replace with '?'. + private static byte[] BuildPageDocRaw( + byte[] content, string resourcesDict = "<< >>", params Obj[] extraObjects) + { + var objs = new List + { + new(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + $"/Resources {resourcesDict} /Contents 4 0 R >>"), + new(4, "<< >>", content), + }; + objs.AddRange(extraObjects); + return BuildPdf(1, [.. objs]); + } + + private sealed class RecordingVisitor : IContentVisitor + { + public List<(string Op, List Operands, int Offset)> Operators { get; } = []; + public List<(PdfDictionary Dict, byte[] Data, int Offset)> InlineImages { get; } = []; + public List<(PdfDictionary Dict, Matrix Matrix, PdfRectangle? BBox, int ObjectNumber, int Offset)> FormBegins { get; } = []; + public List FormEnds { get; } = []; + + public void OnOperator(string operatorName, IReadOnlyList operands, int offset) => + Operators.Add((operatorName, [.. operands], offset)); + + public void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset) => + InlineImages.Add((dictionary, data.ToArray(), offset)); + + public void OnFormBegin( + PdfDictionary formDictionary, Matrix formMatrix, PdfRectangle? boundingBox, int objectNumber, + int offset) => + FormBegins.Add((formDictionary, formMatrix, boundingBox, objectNumber, offset)); + + public void OnFormEnd(int objectNumber) => FormEnds.Add(objectNumber); + } + + private static (PdfDocumentReader Reader, ContentInterpreter Interpreter, RecordingVisitor Visitor) Run( + byte[] pdfBytes, PdfReaderOptions? options = null) + { + var reader = PdfReader.Open(pdfBytes, options ?? new PdfReaderOptions()); + var page = reader.GetPage(0); + var interpreter = new ContentInterpreter(reader); + var visitor = new RecordingVisitor(); + interpreter.Run(page, visitor); + return (reader, interpreter, visitor); + } + + // ── Operand types (known-answer) ──────────────────────────────────────────────────────────── + + [Fact] + public void OperandTypes_areParsedWithExactValuesAndShapes() + { + const string content = + "5 w\n" + + "-.5 6. -.5 6. re\n" + + "(foo \\) bar (nested) baz) Tj\n" + + "<48656C6C6F> Tj\n" + + "/Name#20Test ri\n" + + "[3 2] 0 d\n" + + "[(A) -120 (B) [1 2] 5] TJ\n" + + "/Span << /MCID 1 /Foo (bar) >> BDC\n" + + "EMC\n" + + "true false null \"\n"; + + var (_, _, visitor) = Run(BuildPageDoc(content)); + + var w = visitor.Operators.Single(o => o.Op == "w"); + Assert.Equal(5, ((PdfInteger)w.Operands[0]).Value); + + var re = visitor.Operators.Single(o => o.Op == "re"); + Assert.Equal(-0.5, ((PdfReal)re.Operands[0]).Value); + Assert.Equal(6.0, ((PdfReal)re.Operands[1]).Value); + Assert.Equal(-0.5, ((PdfReal)re.Operands[2]).Value); + Assert.Equal(6.0, ((PdfReal)re.Operands[3]).Value); + + var tjCalls = visitor.Operators.Where(o => o.Op == "Tj").ToList(); + Assert.Equal(2, tjCalls.Count); + var literal = (PdfLiteralString)tjCalls[0].Operands[0]; + Assert.Equal("foo ) bar (nested) baz", Encoding.ASCII.GetString(literal.Bytes.Span)); + var hex = (PdfHexString)tjCalls[1].Operands[0]; + Assert.Equal("Hello", Encoding.ASCII.GetString(hex.Bytes.Span)); + + var ri = visitor.Operators.Single(o => o.Op == "ri"); + Assert.Equal("Name Test", ((PdfName)ri.Operands[0]).Value); + + var d = visitor.Operators.Single(o => o.Op == "d"); + var dashArray = (PdfArray)d.Operands[0]; + Assert.Equal(2, dashArray.Count); + Assert.Equal(3, ((PdfInteger)dashArray[0]).Value); + Assert.Equal(0, ((PdfInteger)d.Operands[1]).Value); + + var tj = visitor.Operators.Single(o => o.Op == "TJ"); + var tjArray = (PdfArray)tj.Operands[0]; + Assert.Equal(5, tjArray.Count); + Assert.Equal("A", Encoding.ASCII.GetString(((PdfLiteralString)tjArray[0]).Bytes.Span)); + Assert.Equal(-120, ((PdfInteger)tjArray[1]).Value); + Assert.Equal("B", Encoding.ASCII.GetString(((PdfLiteralString)tjArray[2]).Bytes.Span)); + var nested = (PdfArray)tjArray[3]; + Assert.Equal(2, nested.Count); + Assert.Equal(1, ((PdfInteger)nested[0]).Value); + Assert.Equal(2, ((PdfInteger)nested[1]).Value); + Assert.Equal(5, ((PdfInteger)tjArray[4]).Value); + + var bdc = visitor.Operators.Single(o => o.Op == "BDC"); + Assert.Equal("Span", ((PdfName)bdc.Operands[0]).Value); + var props = (PdfDictionary)bdc.Operands[1]; + Assert.Equal(1, ((PdfInteger)props.Get(new PdfName("MCID"))!).Value); + + Assert.Contains(visitor.Operators, o => o.Op == "EMC" && o.Operands.Count == 0); + + var quote = visitor.Operators.Single(o => o.Op == "\""); + Assert.Equal(3, quote.Operands.Count); + Assert.Same(PdfBoolean.True, quote.Operands[0]); + Assert.Same(PdfBoolean.False, quote.Operands[1]); + Assert.Same(PdfNull.Instance, quote.Operands[2]); + } + + // ── BX/EX compatibility sections ──────────────────────────────────────────────────────────── + + [Fact] + public void UnknownOperator_outsideBX_isReportedOncePerName_andInsideBX_isSilent() + { + const string content = "Zork\nZork\nBX\nZork\n{ pop }\n> \nEX\n"; + + var (reader, _, _) = Run(BuildPageDoc(content)); + + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); + Assert.Single(reports); + Assert.Equal(PdfReaderDiagnosticSeverity.Info, reports[0].Severity); + } + + [Fact] + public void CurlyBracesAndLoneGreaterThan_insideBX_doNotAbortThePage() + { + const string content = "BX\n{ pop } \n> \nEX\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + // ── Operand-stack and graphics-state caps ─────────────────────────────────────────────────── + + [Fact] + public void OperandStackCap_32IsOk_33IsMalformed() + { + // 32 numeric operands, none consumed by a real operator (so this pins the CAP itself, not + // any one operator's own arity): the 32nd push must not itself overflow. + var okContent = string.Join(' ', Enumerable.Repeat("1", 32)); + var overContent = string.Join(' ', Enumerable.Repeat("1", 33)); + + var (okReader, _, _) = Run(BuildPageDoc(okContent)); + Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + + var (overReader, _, _) = Run(BuildPageDoc(overContent)); + Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + } + + [Fact] + public void TjArrayCap_8192IsOk_8193IsMalformed() + { + var okArray = "[" + string.Concat(Enumerable.Repeat("0 ", 8192)) + "] TJ\n"; + var overArray = "[" + string.Concat(Enumerable.Repeat("0 ", 8193)) + "] TJ\n"; + + var (okReader, _, okVisitor) = Run(BuildPageDoc(okArray)); + Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.Single(okVisitor.Operators, o => o.Op == "TJ"); + + var (overReader, _, overVisitor) = Run(BuildPageDoc(overArray)); + Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(overVisitor.Operators, o => o.Op == "TJ"); + } + + [Fact] + public void UnbalancedQ_isIgnoredWithADiagnostic() + { + var (reader, _, visitor) = Run(BuildPageDoc("Q\n1 w\n")); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // interpretation continued + } + + // ── q/Q/cm matrix state, text state ────────────────────────────────────────────────────────── + + [Fact] + public void GraphicsStateStack_savesAndRestoresTheCtm() + { + var interpreter = RunAndKeepInterpreter( + BuildPageDoc("q\n2 0 0 2 10 20 cm\nQ\n"), out _); + + Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + } + + [Fact] + public void Cm_concatenatesOntoTheCurrentCtm() + { + var interpreter = RunAndKeepInterpreter( + BuildPageDoc("2 0 0 2 10 20 cm\n"), out _); + + Assert.Equal(new Matrix(2, 0, 0, 2, 10, 20), interpreter.GraphicsState.Ctm); + } + + [Fact] + public void TextStateOperators_setTheExpectedFields() + { + const string content = + "1 Tc 2 Tw 150 Tz 12 TL /F1 24 Tf 2 Tr 3 Ts\n" + + "10 20 Td\n" + + "5 -6 TD\n" + + "1 0 0 1 100 200 Tm\n" + + "T*\n"; + var interpreter = RunAndKeepInterpreter( + BuildPageDoc(content, "<< /Font << /F1 5 0 R >> >>", + new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), + out _); + + Assert.Equal(1, interpreter.GraphicsState.CharSpacing); + Assert.Equal(2, interpreter.GraphicsState.WordSpacing); + Assert.Equal(150, interpreter.GraphicsState.HorizontalScaling); + Assert.Equal(6, interpreter.GraphicsState.Leading); // TD's ty=-6 sets TL=-(-6)=6 + Assert.Equal("F1", ((PdfName)interpreter.GraphicsState.Font!).Value); + Assert.Equal(24, interpreter.GraphicsState.FontSize); + Assert.Equal(2, interpreter.GraphicsState.RenderMode); + Assert.Equal(3, interpreter.GraphicsState.Rise); + + // After Tm replaces both matrices with [1 0 0 1 100 200], T* moves by (0, -TL=-6): + // Tlm_new = [1 0 0 1 0 -6] x [1 0 0 1 100 200] = [1 0 0 1 100 194]. + Assert.Equal(new Matrix(1, 0, 0, 1, 100, 194), interpreter.TextState.TextMatrix); + } + + [Fact] + public void BT_resetsTheTextMatrices() + { + const string content = "1 0 0 1 100 200 Tm\nBT\n"; + var interpreter = RunAndKeepInterpreter(BuildPageDoc(content), out _); + + Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); + Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + } + + private static ContentInterpreter RunAndKeepInterpreter(byte[] pdfBytes, out PdfDocumentReader reader) + { + reader = PdfReader.Open(pdfBytes); + var interpreter = new ContentInterpreter(reader); + interpreter.Run(reader.GetPage(0), new RecordingVisitor()); + return interpreter; + } + + // ── gs with /Font ──────────────────────────────────────────────────────────────────────────── + + [Fact] + public void Gs_withFont_surfacesTheFontSelectionToTheState() + { + var interpreter = RunAndKeepInterpreter( + BuildPageDoc( + "/G1 gs\n", + "<< /ExtGState << /G1 6 0 R >> >>", + new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"), + new Obj(6, "<< /Type /ExtGState /Font [5 0 R 18] >>")), + out _); + + var fontRef = Assert.IsType(interpreter.GraphicsState.Font); + Assert.Equal(5, fontRef.ObjectNumber); + Assert.Equal(18, interpreter.GraphicsState.FontSize); + } + + [Fact] + public void Gs_namingAMissingExtGState_reportsResourceMissing() + { + var (reader, _, _) = Run(BuildPageDoc("/Absent gs\n", "<< /ExtGState << >> >>")); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + } + + // ── Form XObjects ──────────────────────────────────────────────────────────────────────────── + + private static Obj[] BuildFormChain(int count, string leafExtra = "") + { + // Form N invokes Form N+1; Form `count` is the one final recursion is expected to skip. + var objs = new List(); + for (var i = 1; i <= count; i++) + { + var body = i < count ? $"/F{i + 1} Do" : leafExtra; + var resources = i < count + ? $"<< /XObject << /F{i + 1} {11 + i} 0 R >> >>" + : "<< >>"; + objs.Add(new Obj( + 10 + i, + $"<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources {resources} >>", + Encoding.ASCII.GetBytes(body))); + } + return [.. objs]; + } + + [Fact] + public void NestedForms_toDepth32_ok_33_reportsDepthExceeded() + { + var forms = BuildFormChain(33); + var doc = BuildPageDoc("/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", forms); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.FormXObjectDepthExceeded); + Assert.Equal(33, visitor.Operators.Count(o => o.Op == "Do")); + Assert.Equal(32, visitor.FormBegins.Count); + Assert.Equal(32, visitor.FormEnds.Count); + } + + [Fact] + public void NestedForms_atDepth32_reportsNoDepthExceeded() + { + var forms = BuildFormChain(32); + var doc = BuildPageDoc("/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", forms); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.FormXObjectDepthExceeded); + Assert.Equal(32, visitor.FormBegins.Count); + } + + [Fact] + public void SelfReferencingForm_reportsCycleOnce() + { + var doc = BuildPageDoc( + "/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] " + + "/Resources << /XObject << /F1 11 0 R >> >> >>", + "/F1 Do"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + var cycles = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.FormXObjectCycle).ToList(); + Assert.Single(cycles); + Assert.Single(visitor.FormBegins); + Assert.Single(visitor.FormEnds); + } + + [Fact] + public void FormDrawn4097Times_reportsBudgetExceeded_andThePageStillCompletes() + { + var pageContent = string.Concat(Enumerable.Repeat("/F1 Do\n", 4097)) + "1 w\n"; + var doc = BuildPageDoc( + pageContent, "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", [])); + + var (reader, _, visitor) = Run(doc); + + var budget = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.FormXObjectBudgetExceeded).ToList(); + Assert.Single(budget); + Assert.Equal(4097, visitor.Operators.Count(o => o.Op == "Do")); + Assert.True(visitor.FormBegins.Count <= 4096); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // the page itself still finished + } + + [Fact] + public void Form_withoutOwnResources_fallsBackToTheParentsResources() + { + var doc = BuildPageDoc( + "/F1 Do\n", "<< /Shading << /Sh1 20 0 R >> /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "/Sh1 sh"u8.ToArray()), + new Obj(20, "<< /ShadingType 2 /ColorSpace /DeviceGray /Coords [0 0 1 1] >>")); + + var (reader, _, _) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + } + + [Fact] + public void Form_matrixAndBBox_areHandedToTheVisitor() + { + var doc = BuildPageDoc( + "/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, + "<< /Type /XObject /Subtype /Form /BBox [1 2 3 4] /Matrix [2 0 0 2 5 6] >>", + [])); + + var (_, _, visitor) = Run(doc); + + var begin = Assert.Single(visitor.FormBegins); + Assert.Equal(new Matrix(2, 0, 0, 2, 5, 6), begin.Matrix); + Assert.NotNull(begin.BBox); + Assert.Equal(1, begin.BBox!.LlX); + Assert.Equal(2, begin.BBox.LlY); + Assert.Equal(3, begin.BBox.UrX); + Assert.Equal(4, begin.BBox.UrY); + Assert.Equal(11, begin.ObjectNumber); + } + + // ── Inline images ──────────────────────────────────────────────────────────────────────────── + + [Fact] + public void UnfilteredGray8Bit_computedLength_skipsEmbeddedEiBytes() + { + byte[] pixelData = [0x45, 0x49, 0x10, 0x20]; // literally contains 'E','I' + var content = BuildInlineImageContent( + "/W 2 /H 2 /BPC 8 /CS /G", pixelData, trailingOperator: "Q"); + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(pixelData)); + Assert.Equal(2, ((PdfInteger)img.Dict.Get(new PdfName("Width"))!).Value); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void L_isHonoured() + { + byte[] data = "ABCDEFGH"u8.ToArray(); + var content = + "BI /W 2 /H 2 /BPC 8 /CS /RGB /F /AHx /L " + data.Length + " ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + } + + [Fact] + public void L_pastTheEnd_reportsMalformed_andRecoversViaTheEiScan() + { + byte[] data = "ABCDEFGH"u8.ToArray(); + var content = + "BI /F /AHx /L 999999 ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void FilteredAHx_usesTheEiScan() + { + byte[] data = "0123456789ABCDEF"u8.ToArray(); + var content = "BI /F /AHx ID " + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + } + + [Fact] + public void DctInlineImage_withAFalseEiInsideItsData_isSkippedByTheScan() + { + // A whitespace-delimited "EI" followed by an unterminated literal string, which never + // closes anywhere in the rest of the content stream. The resync probe's lex attempt from + // that point throws, so ScanForEi rejects it and keeps looking for the real one. + var falseCandidate = " EI (unterminated "u8.ToArray(); + byte[] jpegNoise = [0x01, 0x02, 0xFF, 0xD8, 0xFF]; + var data = jpegNoise.Concat(falseCandidate).Concat(new byte[] { 3, 4, 5 }).ToArray(); + + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void AbbreviationExpansion_isPinned() + { + var content = "BI /W 2 /H 2 /BPC 8 /CS /G /F [/AHx /Fl] ID XYZDATA1234 EI\nQ\n"; + + var (_, _, visitor) = Run(BuildPageDoc(content)); + + var img = Assert.Single(visitor.InlineImages); + Assert.Equal(2, ((PdfInteger)img.Dict.Get(new PdfName("Width"))!).Value); + Assert.Equal( + "DeviceGray", ((PdfName)img.Dict.Get(PdfName.ColorSpace)!).Value); + var filters = (PdfArray)img.Dict.Get(PdfName.Filter)!; + Assert.Equal("ASCIIHexDecode", ((PdfName)filters[0]).Value); + Assert.Equal("FlateDecode", ((PdfName)filters[1]).Value); + } + + [Fact] + public void NamedColorSpace_resolvesThroughResourcesColorSpace() + { + var content = "BI /W 1 /H 1 /BPC 8 /CS /MyCS ID X EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc( + content, "<< /ColorSpace << /MyCS [/Indexed /DeviceRGB 0 (\\000\\000\\000)] >> >>")); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Single(visitor.InlineImages); + } + + [Fact] + public void NamedColorSpace_whenAbsentFromResources_reportsResourceMissing() + { + var content = "BI /W 1 /H 1 /BPC 8 /CS /MyCS ID X EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Single(visitor.InlineImages); // still delimited via the EI scan fallback + } + + [Fact] + public void JpxDecodeInlineImage_reportsMalformed_andTheStreamContinuesAfterIt() + { + var content = "BI /W 1 /H 1 /BPC 8 /F /JPXDecode ID XYZ EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Empty(visitor.InlineImages); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void MissingId_reportsMalformed() + { + var (reader, _, visitor) = Run(BuildPageDoc("BI /W 1 /H 1")); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Empty(visitor.InlineImages); + } + + private static string BuildInlineImageContent( + string keys, byte[] data, string trailingOperator) + { + var sb = new StringBuilder(); + sb.Append("BI ").Append(keys).Append(" ID "); + var prefix = sb.ToString(); + var suffix = " EI\n" + trailingOperator + "\n"; + return prefix + Encoding.Latin1.GetString(data) + suffix; + } + + // ── /Contents array concatenation ──────────────────────────────────────────────────────────── + + [Fact] + public void TokenSplitAcrossStreams_isNotGlued() + { + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [4 0 R 5 0 R] >>"), + new Obj(4, "<< >>", "q\nBT"u8.ToArray()), // ends mid-token, no trailing whitespace + new Obj(5, "<< >>", "ET\nQ"u8.ToArray())); // begins with no leading whitespace + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.UnknownOperator); + Assert.Equal(["q", "BT", "ET", "Q"], visitor.Operators.Select(o => o.Op)); + } + + [Fact] + public void NonStreamContentsElement_isSkippedWithLexError() + { + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [5 0 R 6 0 R] >>"), + new Obj(5, "42"), // not a stream at all + new Obj(6, "<< >>", "1 w"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // object 6's content still ran + } + + [Fact] + public void ContentsStreamWithAnImageFilter_reportsLexError() + { + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< /Filter /DCTDecode >>", [0xFF, 0xD8, 0xFF, 0xD9])); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Empty(visitor.Operators); + } + + // ── 64 MiB content cap ─────────────────────────────────────────────────────────────────────── + + [Fact] + public void ContentExceeding64MiB_reportsTooLarge_andKeepsTheOperatorsBeforeTheCap() + { + var unit = "0 0 1 1 re\n"u8.ToArray(); + var repeats = (67 * 1024 * 1024 / unit.Length) + 200_000; + var raw = new byte[unit.Length * repeats]; + for (var i = 0; i < repeats; i++) + unit.CopyTo(raw, i * unit.Length); + + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< /Filter /FlateDecode >>", Flate(raw))); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge); + Assert.NotEmpty(visitor.Operators); + Assert.True(visitor.Operators.Count < repeats); + Assert.All(visitor.Operators, o => + { + Assert.Equal("re", o.Op); + Assert.Equal(4, o.Operands.Count); + }); + } + + // ── Fuzzing ────────────────────────────────────────────────────────────────────────────────── + + private static readonly byte[] FuzzSeed = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << /Font << /F1 6 0 R >> /XObject << /X1 7 0 R >> " + + "/ExtGState << /G1 8 0 R >> >> /Contents 4 0 R >>"), + new Obj(4, "<< >>", Encoding.ASCII.GetBytes( + "q\n2 0 0 2 10 20 cm\nBT\n/F1 12 Tf\n(Hi) Tj\n[(A) -10 (B)] TJ\nET\n" + + "/G1 gs\n/X1 Do\nBI /W 2 /H 2 /BPC 8 /CS /G ID \x01\x02\x03\x04 EI\nQ\n")), + new Obj(6, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"), + new Obj( + 7, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Matrix [1 0 0 1 0 0] >>", + "1 w\n"u8.ToArray()), + new Obj(8, "<< /Type /ExtGState /Font [6 0 R 10] >>")); + + private readonly record struct MutationOp(int Kind, int Position, byte Value, int Length); + + private static readonly Gen MutationOpGen = + Gen.Select( + Gen.Int[0, 5], Gen.Int[0, int.MaxValue], Gen.Byte, Gen.Int[1, 32], + (kind, position, value, length) => new MutationOp(kind, position, value, length)); + + private static readonly Gen FuzzInputGen = + MutationOpGen.Array[1, 8].Select(ops => Mutate(FuzzSeed, ops)); + + private static byte[] Mutate(byte[] seed, MutationOp[] ops) + { + var buffer = new List(seed); + foreach (var op in ops) + { + if (buffer.Count == 0) { buffer.Add(op.Value); continue; } + var position = op.Position % buffer.Count; + switch (op.Kind) + { + case 0: buffer[position] ^= (byte)(1 << (op.Value % 8)); break; + case 1: buffer[position] = op.Value; break; + case 2: buffer.RemoveAt(position); break; + case 3: buffer.Insert(position, op.Value); break; + case 4: + var length = Math.Min(op.Length, buffer.Count - position); + if (length > 0 && buffer.Count + length <= 1 << 20) + buffer.InsertRange(position, buffer.GetRange(position, length)); + break; + case 5: + var cut = position + 1; + if (cut < buffer.Count) + buffer.RemoveRange(cut, buffer.Count - cut); + break; + } + if (buffer.Count > 1 << 20) + buffer.RemoveRange(1 << 20, buffer.Count - (1 << 20)); + } + return buffer.Count == 0 ? [0] : [.. buffer]; + } + + [Fact] + public void Fuzz_run_neverThrowsOutsideTheDeclaredVocabulary_andAlwaysTerminates() + => FuzzInputGen.Sample(AssertInterpreterIsRobust, iter: FuzzBudget.Iterations); + + private static void AssertInterpreterIsRobust(byte[] bytes) + { + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + try + { + using var reader = PdfReader.Open(bytes, new PdfReaderOptions + { + MaxDecodedStreamBytes = ReaderLimits.MinMaxDecodedBytes, + }); + if (reader.PageCount == 0) + return; + var interpreter = new ContentInterpreter(reader); + interpreter.Run(reader.GetPage(0), new RecordingVisitor()); + } + catch (Exception ex) when (ex is InvalidDataException or UnsupportedPdfFeatureException or PdfPasswordException) + { + // Acceptable outcome; see ParserFuzzTests' own class doc for the same policy this + // interpreter follows: a robustness oracle, not a conformance one. + } + Assert.True( + stopwatch.Elapsed <= TimeSpan.FromSeconds(4), + $"content interpretation took {stopwatch.Elapsed} on a {bytes.Length}-byte input."); + } + + private static class FuzzBudget + { + private const long DefaultIterations = 3_000; + + internal static long Iterations + { + get + { + var raw = Environment.GetEnvironmentVariable("VELLUMPDF_FUZZ_ITER"); + return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : DefaultIterations; + } + } + } +} diff --git a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs new file mode 100644 index 00000000..a79cec5e --- /dev/null +++ b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs @@ -0,0 +1,165 @@ +// Copyright © Timothy van der Ham (@Tim81) +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; + +namespace VellumPdf.Reader.Tests; + +/// +/// Pins 's content-stream mode (#98, part 3): the default constructor's +/// behaviour must stay byte-identical (it still throws on a lone {, }, or unmatched +/// >), and the new internal PdfLexer(ReadOnlyMemory<byte>, bool) constructor +/// must accept those same three bytes as one-byte tokens (ISO +/// 32000-2 §7.8.2) while lexing every other token kind identically to the default constructor. +/// +public sealed class PdfLexerContentModeTests +{ + // ── Default constructor: unchanged behaviour ──────────────────────────────────────────────── + + [Theory] + [InlineData("{")] + [InlineData("}")] + [InlineData(">")] + public void DefaultConstructor_stillThrowsOnPostScriptHeritageBytes(string input) + { + var lexer = new PdfLexer(Encoding.ASCII.GetBytes(input)); + + Assert.Throws(() => lexer.NextToken()); + } + + [Fact] + public void DefaultConstructor_stillThrowsOnUnmatchedGreaterThan_evenMidStream() + { + var lexer = new PdfLexer(Encoding.ASCII.GetBytes("1 0 obj > endobj")); + lexer.NextToken(); // 1 + lexer.NextToken(); // 0 + lexer.NextToken(); // obj + + Assert.Throws(() => lexer.NextToken()); + } + + // ── Content-stream mode: the three bytes become one-byte Keyword tokens ──────────────────────── + + [Theory] + [InlineData("{")] + [InlineData("}")] + [InlineData(">")] + public void ContentStreamMode_lexesPostScriptHeritageBytesAsOneByteKeywordTokens(string input) + { + var bytes = Encoding.ASCII.GetBytes(input); + var lexer = new PdfLexer(bytes, contentStreamMode: true); + + var token = lexer.NextToken(); + + Assert.Equal(TokenKind.Keyword, token.Kind); + Assert.Equal(1, token.Raw.Length); + Assert.Equal(input[0], (char)token.Raw.Span[0]); + Assert.Equal(1, lexer.Position); + } + + [Fact] + public void ContentStreamMode_lexesADictionaryEndImmediatelyAfterALoneGreaterThan() + { + // "> >>": a lone '>' keyword token, then a real dictionary-end token right after it. + var lexer = new PdfLexer(Encoding.ASCII.GetBytes("> >>"), contentStreamMode: true); + + var first = lexer.NextToken(); + var second = lexer.NextToken(); + + Assert.Equal(TokenKind.Keyword, first.Kind); + Assert.Equal(TokenKind.DictEnd, second.Kind); + } + + [Fact] + public void ContentStreamMode_stillLexesADoubleGreaterThanAsDictEnd_notTwoLoneKeywords() + { + var lexer = new PdfLexer(Encoding.ASCII.GetBytes(">>"), contentStreamMode: true); + + var token = lexer.NextToken(); + + Assert.Equal(TokenKind.DictEnd, token.Kind); + Assert.Equal(2, lexer.Position); + } + + [Fact] + public void ContentStreamMode_insideACompatibilitySection_lexesAWholeBxToExSequenceWithoutThrowing() + { + // A PostScript-heritage compatibility fragment ISO 32000-2 §7.8.2 permits inside BX/EX. + var bytes = Encoding.ASCII.GetBytes("BX { pop } EX"); + var lexer = new PdfLexer(bytes, contentStreamMode: true); + + var kinds = new List(); + Token tok; + while ((tok = lexer.NextToken()).Kind != TokenKind.EndOfInput) + kinds.Add(tok.Kind); + + Assert.Equal( + [ + TokenKind.Keyword, // BX + TokenKind.Keyword, // { + TokenKind.Keyword, // pop + TokenKind.Keyword, // } + TokenKind.Keyword, // EX + ], + kinds); + } + + // ── Every other token kind lexes identically in both modes ───────────────────────────────────── + + // MemberData parameters must be public-accessible types (CS0051), so TokenKind (internal, via + // this test assembly's InternalsVisibleTo friendship) is named by string here and parsed back + // inside the theory body instead of being the parameter type itself. + public static IEnumerable AllTokenKindFixtures() + { + yield return ["123", nameof(TokenKind.Integer)]; + yield return ["-.5", nameof(TokenKind.Real)]; + yield return ["6.", nameof(TokenKind.Real)]; + yield return ["/Name#20With#23Escape", nameof(TokenKind.Name)]; + yield return ["(literal (nested) string)", nameof(TokenKind.LiteralString)]; + yield return ["<48656C6C6F>", nameof(TokenKind.HexString)]; + yield return ["[", nameof(TokenKind.ArrayBegin)]; + yield return ["]", nameof(TokenKind.ArrayEnd)]; + yield return ["<<", nameof(TokenKind.DictBegin)]; + yield return ["true", nameof(TokenKind.Keyword)]; + yield return ["Tj", nameof(TokenKind.Keyword)]; + } + + [Theory] + [MemberData(nameof(AllTokenKindFixtures))] + public void EveryOtherTokenKind_lexesIdenticallyInBothModes(string input, string expectedKindName) + { + var expectedKind = Enum.Parse(expectedKindName); + var bytes = Encoding.ASCII.GetBytes(input); + var defaultLexer = new PdfLexer(bytes); + var contentLexer = new PdfLexer(bytes, contentStreamMode: true); + + var defaultToken = defaultLexer.NextToken(); + var contentToken = contentLexer.NextToken(); + + Assert.Equal(expectedKind, defaultToken.Kind); + Assert.Equal(defaultToken.Kind, contentToken.Kind); + Assert.True(defaultToken.Raw.Span.SequenceEqual(contentToken.Raw.Span)); + Assert.Equal(defaultLexer.Position, contentLexer.Position); + } + + [Fact] + public void EveryTokenKind_overOneFixtureCoveringAllOfThem_lexesIdenticallyInBothModes() + { + const string fixture = + "q 1 0 0 1 10 20 cm /F1 12 Tf (Hello) Tj <48656C6C6F> Tj [1 2 3] true false null Q"; + var bytes = Encoding.ASCII.GetBytes(fixture); + var defaultLexer = new PdfLexer(bytes); + var contentLexer = new PdfLexer(bytes, contentStreamMode: true); + + while (true) + { + var a = defaultLexer.NextToken(); + var b = contentLexer.NextToken(); + Assert.Equal(a.Kind, b.Kind); + Assert.True(a.Raw.Span.SequenceEqual(b.Raw.Span)); + if (a.Kind == TokenKind.EndOfInput) + break; + } + Assert.Equal(defaultLexer.Position, contentLexer.Position); + } +} diff --git a/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs b/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs index a5c84c6e..ca62264d 100644 --- a/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs @@ -36,6 +36,15 @@ public sealed class PdfReaderDiagnosticCodeTests [PdfReaderDiagnosticCode.PageAttributeInvalid] = 2, [PdfReaderDiagnosticCode.PageTreeNodeMalformed] = 2, [PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded] = 2, + [PdfReaderDiagnosticCode.ContentStreamLexError] = 3, + [PdfReaderDiagnosticCode.UnknownOperator] = 3, + [PdfReaderDiagnosticCode.OperandStackMalformed] = 3, + [PdfReaderDiagnosticCode.FormXObjectDepthExceeded] = 3, + [PdfReaderDiagnosticCode.FormXObjectCycle] = 3, + [PdfReaderDiagnosticCode.FormXObjectBudgetExceeded] = 3, + [PdfReaderDiagnosticCode.ResourceMissing] = 3, + [PdfReaderDiagnosticCode.InlineImageMalformed] = 3, + [PdfReaderDiagnosticCode.ContentStreamTooLarge] = 3, [PdfReaderDiagnosticCode.DiagnosticsSuppressed] = 9, }; @@ -134,6 +143,15 @@ private static PdfReaderDiagnostic MakeDiagnostic(PdfReaderDiagnosticCode code) [PdfReaderDiagnosticCode.PageAttributeInvalid] = (205, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.PageTreeNodeMalformed] = (206, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded] = (207, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.ContentStreamLexError] = (300, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.UnknownOperator] = (301, PdfReaderDiagnosticSeverity.Info), + [PdfReaderDiagnosticCode.OperandStackMalformed] = (302, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.FormXObjectDepthExceeded] = (303, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.FormXObjectCycle] = (304, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.FormXObjectBudgetExceeded] = (305, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.ResourceMissing] = (306, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.InlineImageMalformed] = (307, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.ContentStreamTooLarge] = (308, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.DiagnosticsSuppressed] = (900, PdfReaderDiagnosticSeverity.Warning), }; diff --git a/tests/VellumPdf.Reader.Tests/PdfReaderOptionsTests.cs b/tests/VellumPdf.Reader.Tests/PdfReaderOptionsTests.cs index 80c3aba3..f22b203e 100644 --- a/tests/VellumPdf.Reader.Tests/PdfReaderOptionsTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfReaderOptionsTests.cs @@ -103,6 +103,37 @@ public void ToString_doesNotContainThePassword() Assert.DoesNotContain("correct horse battery staple", options.ToString()); } + [Theory] + [InlineData(0)] + [InlineData(33)] + public void MaxFormXObjectDepth_outsideRange_throwsArgumentOutOfRangeException(int value) + { + var bytes = Load("plaintext-baseline.pdf"); + + var ex = Assert.Throws( + () => PdfReader.Open(bytes, new PdfReaderOptions { MaxFormXObjectDepth = value })); + + Assert.Equal(nameof(PdfReaderOptions.MaxFormXObjectDepth), ex.ParamName); + } + + [Theory] + [InlineData(1)] + [InlineData(32)] + public void MaxFormXObjectDepth_atTheFloorOrCeiling_isAccepted(int value) + { + var bytes = Load("plaintext-baseline.pdf"); + + using var reader = PdfReader.Open(bytes, new PdfReaderOptions { MaxFormXObjectDepth = value }); + + Assert.NotNull(reader.Catalog); + } + + [Fact] + public void DefaultOptions_carryTheDefaultMaxFormXObjectDepth() + { + Assert.Equal(32, new PdfReaderOptions().MaxFormXObjectDepth); + } + private static byte[] Load(string name) { using var s = Assembly.GetExecutingAssembly().GetManifestResourceStream(name) From 99777c2fb0c73b43927bccbec3ee0245a3715139 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 13:46:21 +0200 Subject: [PATCH 02/14] Require operators after a candidate EI in the inline-image scan LooksLikeResyncPoint accepted any EI whose following bytes lexed without throwing, but a run of binary bytes outside the whitespace and delimiter sets lexes as one Keyword token, so a false EI inside DCT data followed by more image data passed the probe more often than not. The probe now also requires every keyword it sees to be an Annex A Table A.1 operator (or true/false/null); a test pins the binary-noise case. Also: drop the per-name UnknownOperator set, which the sink's (code, object, page) dedupe made redundant, and say so in the test; the CreateScope remark names its first caller; ReaderLimits.Resolve doc counts the fourth knob. --- .../Content/ContentInterpreter.cs | 32 ++++++++++++----- src/VellumPdf.Reader/DiagnosticSink.cs | 8 ++--- src/VellumPdf.Reader/ReaderLimits.cs | 14 ++++---- .../ContentInterpreterTests.cs | 34 +++++++++++++++++-- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 96d7f5ef..4b4c04f2 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -71,7 +71,6 @@ internal sealed class ContentInterpreter private readonly HashSet _openForms = []; private int _formDepth; private int _formInvocations; - private readonly HashSet _reportedUnknownOperators = []; private ReadOnlyMemory _currentBuffer; /// The current graphics state, the top of the q/Q stack, readable from @@ -113,7 +112,6 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _openForms.Clear(); _formDepth = 0; _formInvocations = 0; - _reportedUnknownOperators.Clear(); var diagnostics = _reader.CreateContentDiagnosticScope(); var pageIndex = page.Index; @@ -497,12 +495,14 @@ private void HandleOperator( // ISO 32000-2 §7.8.2: "an error shall occur" outside a compatibility section; this // reader instead notifies and continues. Deliberately does NOT clear the operand stack // (this reader's own leniency, distinct from Table 33's "ignored ... along with - // operands" for a genuine future operator inside BX/EX): the most common way an + // operands" for a later PDF version's operator inside BX/EX): the most common way an // unrecognised keyword appears in an otherwise-conforming stream is a stray "R" left // over from indirect-reference syntax that §7.8.2 forbids in content streams at all // ("Indirect objects and object references shall not be permitted"), and the operands // that precede it usually belong to whatever REAL operator follows, not to "R" itself. - if (_bxDepth == 0 && _reportedUnknownOperators.Add(name)) + // The sink's dedupe key is (code, object, page), so only the first unknown name on a + // page is recorded; a second distinct name on the same page is dropped by the sink. + if (_bxDepth == 0) { diagnostics.Report( PdfReaderDiagnosticCode.UnknownOperator, @@ -1274,10 +1274,15 @@ private int ResolveComponentCount( } // Bounded lookahead: lexes up to a handful of tokens in content mode from a candidate resync - // point and accepts it if none of them throws before either running out of tokens to try or - // reaching end of input. This is what rejects a coincidental "EI" byte pair sitting inside - // DCT-compressed binary data, which is followed by more binary noise the lexer chokes on almost - // immediately (an unterminated literal string or hex string is the most common trip). + // point and accepts it only if none of them throws and every keyword among them is an operator + // Annex A Table A.1 defines (or true/false/null). The lexer alone is a weak filter for a + // coincidental "EI" byte pair inside DCT-compressed data: any run of bytes outside §7.2.2's + // whitespace and delimiter sets lexes as one Keyword token, so binary noise after a false EI + // very often lexes cleanly. Requiring the keywords to be operators is what rejects it, since + // a byte run like 0x8F 0x12 0xC4 is never an operator name. The one construct this rejects + // wrongly is an unknown operator inside a BX/EX section right after an inline image, which + // then falls through to a later EI candidate; the false-positive cost of accepting binary + // noise (the rest of the stream lost to ContentStreamLexError) is the worse of the two. private bool LooksLikeResyncPoint(int pos) { var probe = new PdfLexer(_currentBuffer, contentStreamMode: true); @@ -1288,8 +1293,17 @@ private bool LooksLikeResyncPoint(int pos) { if (probe.AtEnd) return true; - if (probe.NextToken().Kind == TokenKind.EndOfInput) + var token = probe.NextToken(); + if (token.Kind == TokenKind.EndOfInput) return true; + if (token.Kind != TokenKind.Keyword) + continue; + + var raw = token.Raw.Span; + if (raw.SequenceEqual("true"u8) || raw.SequenceEqual("false"u8) || raw.SequenceEqual("null"u8)) + continue; + if (!ContentOperators.IsKnown(System.Text.Encoding.Latin1.GetString(raw))) + return false; } return true; } diff --git a/src/VellumPdf.Reader/DiagnosticSink.cs b/src/VellumPdf.Reader/DiagnosticSink.cs index 2300e27e..7b5d8da9 100644 --- a/src/VellumPdf.Reader/DiagnosticSink.cs +++ b/src/VellumPdf.Reader/DiagnosticSink.cs @@ -91,10 +91,10 @@ private DiagnosticSink(DiagnosticSink parent) : this(parent._cap) /// it at all ( returns before is ever called). /// /// - /// Unused in this PR (#385 lands only the document-level sink; a per-operation result — the - /// first candidate is #98's text extraction — is what will actually call this). Present now, - /// and exercised directly by DiagnosticSinkTests, so the forwarding contract is pinned - /// before anything depends on it rather than designed against its first real caller. + /// Landed with #385 ahead of any caller, exercised directly by DiagnosticSinkTests so the + /// forwarding contract was pinned before anything depended on it. The first caller is + /// ContentInterpreter.Run (#98), one scope per page interpreted, through + /// . /// /// Each scope holds its own cap-bounded dedupe set (see ), so N /// live scopes retain up to O(N × cap) between them — scopes are meant to be short-lived, diff --git a/src/VellumPdf.Reader/ReaderLimits.cs b/src/VellumPdf.Reader/ReaderLimits.cs index a628d3cc..24fa4c86 100644 --- a/src/VellumPdf.Reader/ReaderLimits.cs +++ b/src/VellumPdf.Reader/ReaderLimits.cs @@ -84,7 +84,7 @@ internal readonly record struct ReaderLimits( DefaultMaxDiagnostics, DefaultMaxFormXObjectDepth); /// - /// Validates 's three resource knobs and resolves them into the + /// Validates 's four resource knobs and resolves them into the /// limits threaded through one read. /// /// @@ -93,19 +93,21 @@ internal readonly record struct ReaderLimits( /// practical limits", and Annex C.3 (informative) adds that available memory is "often much less /// in mobile devices than desktop computers" — the ceiling is this processor's own choice, not a /// spec requirement, so , - /// , and - /// are each a safe upper bound a caller may only lower, never raise. A value under the + /// , , and + /// are each a safe upper bound a caller may only lower, never raise. A value under the /// corresponding floor is rejected too: below or /// , an otherwise ordinary document routinely /// fails to decode or reconstruct at all; below every report - /// would turn into a suppression count, disabling the channel entirely. Each of these is a + /// would turn into a suppression count, disabling the channel entirely; below + /// no Form XObject could be entered at all. Each of these is a /// configuration mistake worth surfacing immediately, here, rather than as a confusing /// exception from a different layer downstream. /// /// /// , - /// , or - /// is outside its allowed range. + /// , + /// , or + /// is outside its allowed range. /// internal static ReaderLimits Resolve(PdfReaderOptions options) { diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 351981bb..18777cf3 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -214,15 +214,18 @@ public void OperandTypes_areParsedWithExactValuesAndShapes() // ── BX/EX compatibility sections ──────────────────────────────────────────────────────────── [Fact] - public void UnknownOperator_outsideBX_isReportedOncePerName_andInsideBX_isSilent() + public void UnknownOperator_outsideBX_isReportedOncePerPage_andInsideBX_isSilent() { - const string content = "Zork\nZork\nBX\nZork\n{ pop }\n> \nEX\n"; + // Two distinct unknown names outside BX/EX: the sink dedupes on (code, object, page), so + // the page records the first one only. + const string content = "Zork\nZork\nBlat\nBX\nZork\n{ pop }\n> \nEX\n"; var (reader, _, _) = Run(BuildPageDoc(content)); var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); Assert.Single(reports); Assert.Equal(PdfReaderDiagnosticSeverity.Info, reports[0].Severity); + Assert.Contains("'Zork'", reports[0].Message); } [Fact] @@ -571,6 +574,33 @@ public void DctInlineImage_withAFalseEiInsideItsData_isSkippedByTheScan() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void DctInlineImage_withAFalseEiFollowedByBinaryNoise_isSkippedByTheScan() + { + // The harder false candidate: " EI " followed by bytes outside ISO 32000-2 §7.2.2's + // whitespace and delimiter sets, which the lexer accepts as one Keyword token without + // throwing. Only the probe's operator check (the keyword is not in Annex A Table A.1) + // rejects this one; the real EI follows and is accepted. + var falseCandidate = " EI "u8.ToArray(); + byte[] noiseAfter = [0x8F, 0x12, 0xC4, 0x7A, 0x20, 0xFE, 0x01]; + byte[] jpegNoise = [0xFF, 0xD8, 0xFF, 0xE0, 0x00]; + var data = jpegNoise.Concat(falseCandidate).Concat(noiseAfter).ToArray(); + + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Equal("Q", Assert.Single(visitor.Operators).Op); + } + [Fact] public void AbbreviationExpansion_isPinned() { From 3d9882065992e9154117027db97f897449746c9f Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 16:02:33 +0200 Subject: [PATCH 03/14] Fix content-interpreter review findings from PR #402 round 1 Do on a Form XObject now brackets the form's content in an implicit q/Q, marked-content, and BX/EX save and restore (ISO 32000-2 section 8.10.1 steps a and e), using stack-depth floors rather than a real push so a form's own state cannot leak into its invoker and the implicit save costs nothing against MaxGraphicsStateDepth. An unknown operator inside BX/EX now drops its operands per Table 33 ("ignored without error", operands included), rather than only suppressing the report. The bounded resync probe for a false inline-image EI (LooksLikeResyncPoint) now lexes at most 128 bytes and 8 tokens per candidate instead of the rest of the buffer, fixing the quadratic scan (100 KB content: 18s before, well under a second after) while accepting an unknown-but-printable operator or a BX/EX section it previously rejected. A failed tier-a/tier-b inline-image length now falls back to the EI scan instead of losing the rest of the content stream, matching the existing /L-past-the-end recovery path; an ID followed by CR LF consumes both bytes as one separator per section 7.2.3, with a one-byte-earlier retry for a binary payload whose first byte is itself LF. Table 92's colour-space abbreviations now expand inside a /CS array (the one composite inline colour space section 8.9.7 allows), and an array led by /Indexed correctly counts one component. The 64 MiB content budget is now a per-Run running total across the page's own /Contents and every Form XObject it draws, not just the page's own content; ContentStreamTooLarge and FormXObjectBudgetExceeded now use ReportRetained so they survive MaxDiagnostics exhaustion the same way PageTreeWalker's own walk-stop codes do. Also: the Array.Copy overload in Concatenate copied a truncated second stream over the first instead of appending it; MaxOperandsPerOperator is 64 rather than 32, since Table 73's scn can legally need more than 32 for a DeviceN space; a new ContentLimitExceeded code separates this reader's own processing ceilings from producer-side malformation, which stays under OperandStackMalformed; UnknownOperator is Warning severity, not Info; /Matrix, /BBox, and ExtGState's /Font now resolve through an indirect reference before their shape check; a negative /L and an invalid /W, /H, or /BPC are now reported instead of silently falling through; and the fuzz test's try/catch around Open/GetPage no longer also swallows a Run-time InvalidDataException, which surfaced a pre-existing gap where resolving /Contents, a resource, or a Form XObject could still throw past Run's own no-throw promise; Run now catches that at the outermost level and reports it as a diagnostic. --- CHANGELOG.md | 21 +- docs/reader-guide.md | 2 +- .../Content/ContentInterpreter.cs | 531 ++++++++++++---- .../Content/IContentVisitor.cs | 5 +- src/VellumPdf.Reader/PdfLexer.cs | 19 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 83 ++- src/VellumPdf.Reader/PublicAPI.Unshipped.txt | 1 + .../ContentInterpreterTests.cs | 572 +++++++++++++++++- .../PdfLexerContentModeTests.cs | 34 ++ .../PdfReaderDiagnosticCodeTests.cs | 4 +- 10 files changed, 1111 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 232aea38..1d95858f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,19 +62,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). leaf and node limits each end the walk the instant they are reported, so at most one of that pair, plus the depth code's first occurrence, is ever retained in the same walk. Computed lazily, on first access, and cached for the reader's lifetime. (#98) -- **A new option, `PdfReaderOptions.MaxFormXObjectDepth`, and nine new `PdfReaderDiagnosticCode` +- **A new option, `PdfReaderOptions.MaxFormXObjectDepth`, and ten new `PdfReaderDiagnosticCode` values in a `3xx` block reserved for content streams.** The reader now has an internal content-stream interpreter (ISO 32000-2 §7.8.2), shared machinery the two extraction milestones still ahead of it (text, then images) will build on; a caller cannot invoke it directly in this - release, so the nine codes below cannot yet be reported to one. `MaxFormXObjectDepth` (default 32, + release, so the ten codes below cannot yet be reported to one. `MaxFormXObjectDepth` (default 32, tighten-only like the three resource limits it joins) is the one part of this a caller sets today: a ceiling on Form XObject recursion depth (§8.10) the interpreter enforces once something does call it, reporting `FormXObjectDepthExceeded` and continuing rather than recursing - unboundedly. The rest of the interpreter follows the same policy throughout: a malformed or + unboundedly. `Do` on a Form XObject brackets the form's own content in an implicit save and + restore of the graphics state, marked-content nesting, and BX/EX compatibility depth, mirroring + §8.10.1's own steps a) and e), so nothing the form does to any of the three leaks into the page + that invoked it. The rest of the interpreter follows the same policy throughout: a malformed or unsupported construct is reported, not thrown, and interpretation continues past it. That is - what the other eight codes (`ContentStreamLexError`, `UnknownOperator`, `OperandStackMalformed`, - `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, `InlineImageMalformed`, - `ContentStreamTooLarge`) each describe. (#98) + what the other nine codes describe: `ContentStreamLexError`, `UnknownOperator` (`Warning` + severity: an operator this reader skipped is a best-effort reading, not proof the output is + correct), `OperandStackMalformed` for a producer-side malformation, `ContentLimitExceeded` for + this reader's own processing ceilings instead (an operand-count, `TJ`-array, `q`-depth, or + marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, + `InlineImageMalformed`, and `ContentStreamTooLarge` (the same 64 MiB decoded-content budget now + covers every Form XObject a page draws, not only its own `/Contents`, and every invocation of a + form counts again, since the interpretation cost this bounds scales with how many times a form + is drawn). (#98) ### Changed diff --git a/docs/reader-guide.md b/docs/reader-guide.md index 6955add1..58a57778 100644 --- a/docs/reader-guide.md +++ b/docs/reader-guide.md @@ -357,7 +357,7 @@ throws `PdfPasswordException` if the document needs a real one. best-effort: a wrong guess at the object graph during recovery produces a wrong, but internally consistent, decrypted copy. -**`MaxDecodedStreamBytes`, `ReconstructionBudgetMultiplier`, and `MaxDiagnostics` only go down.** +**`MaxDecodedStreamBytes`, `ReconstructionBudgetMultiplier`, `MaxDiagnostics`, and `MaxFormXObjectDepth` only go down.** Raising any of them above its shipped default throws at `Open` time rather than silently clamping — there is no way to ask the reader to trust a file more than its own defaults do. diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 4b4c04f2..a0b33f80 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -20,12 +20,29 @@ namespace VellumPdf.Reader.Content; /// (a diagnostic and best-effort recovery) rather than throwing; the sole exception is /// , which is allowed to propagate: "cannot continue" /// belongs to the exception channel the rest of this reader already uses it for. +/// +/// Retains at most TWO diagnostics past per +/// (see ), one of each code: +/// fires at most once per run, because +/// the first truncation it reports, whether against the page's own /Contents or a Form +/// XObject's, drives the run's own decoded-bytes budget to zero, and every later stream this run +/// touches then takes the budget-already-spent skip silently instead of truncating (and reporting) +/// again; fires at most once +/// because its own report carries no object number, so the sink's own (code, object, page) dedupe +/// already collapses every recursion past the 4096-invocation ceiling to one entry on its own. +/// /// internal sealed class ContentInterpreter { // ISO 32000-2 §7.8.2 gives an operator's own operands no declared bound; this reader's own - // ceiling against a hostile or corrupted stream that never emits an operator at all. - private const int MaxOperandsPerOperator = 32; + // ceiling against a hostile or corrupted stream that never emits an operator at all. 64, not + // 32: Annex C.1 Table C.1 (informative) records 32 as the DeviceN colourant-count limit an + // earlier PDF version recommended, but §8.6.6.5 itself allows "an arbitrary number" of + // colourants, and Table 73's scn operator takes one numeric component per colourant plus an + // optional trailing pattern name, so a legal scn call against a DeviceN space with 32 or more + // colourants needs 33 or more operands; 32 as this reader's own ceiling would reject a legal + // call, not just a hostile one. + private const int MaxOperandsPerOperator = 64; // §9.4.3's own TJ array holds a mix of strings and numeric adjustments; this reader's own // ceiling on how many elements one such array may carry. @@ -34,7 +51,7 @@ internal sealed class ContentInterpreter // §8.4.4's q/Q pair; this reader's own ceiling on how deep a legitimate document nests them. private const int MaxGraphicsStateDepth = 64; - // §14.6.2's BMC/BDC/EMC nesting; this reader's own ceiling, mirroring MaxGraphicsStateDepth. + // §14.6.1's BMC/BDC/EMC nesting; this reader's own ceiling, mirroring MaxGraphicsStateDepth. private const int MaxMarkedContentDepth = 64; // This interpreter's own budget on total successful Form XObject recursions across one page @@ -43,10 +60,20 @@ internal sealed class ContentInterpreter // invoking the same shallow form thousands of times) is not caught by a depth cap at all. private const int MaxFormInvocationsPerPage = 4096; - // This reader's own ceiling on the total decoded bytes one page's /Contents may contribute, - // summed across every stream in the array (ISO 32000-2 §7.7.3.3 Table 31). + // This reader's own ceiling on the total decoded content bytes one Run interprets: the page's + // own /Contents (ISO 32000-2 §7.7.3.3 Table 31) and every Form XObject invocation on that page + // (§8.10), combined. Tracked as a running per-Run budget (_contentBytesRemaining) rather than + // checked once against /Contents alone, since a small file drawing one large form many times + // can interpret far more total content than its own /Contents ever declares. private const long MaxContentBytes = 64L * 1024 * 1024; + // The bounded resync probe (see LooksLikeResyncPoint) lexes at most this many bytes, and at + // most ProbeTokens tokens, from each 'EI' candidate: bounding the work per candidate is what + // keeps ScanForEi linear in the content length rather than quadratic (a probe that lexed to + // the end of the buffer from every candidate cost O(N) per candidate, O(N^2) overall). + private const int ProbeWindowBytes = 128; + private const int ProbeTokens = 8; + private static readonly PdfName XObjectSubtypeForm = new("Form"); private static readonly PdfName ImageMaskKey = new("ImageMask"); private static readonly PdfName WidthKey = new("Width"); @@ -71,8 +98,18 @@ internal sealed class ContentInterpreter private readonly HashSet _openForms = []; private int _formDepth; private int _formInvocations; + private long _contentBytesRemaining; private ReadOnlyMemory _currentBuffer; + // The graphics-state stack depth, marked-content depth, and BX/EX depth a 'Q', 'EMC', or 'EX' + // may not pop or decrement below. All three are 0 for the page's own top-level content and set + // to the invoker's own depth for the duration of one Form XObject's content (see HandleDo): + // ISO 32000-2 §8.10.1 brackets a form's content in an implicit q/Q pair the form's own content + // must not be able to see past, in either direction. + private int _gsFloor; + private int _markedContentFloor; + private int _bxFloor; + /// The current graphics state, the top of the q/Q stack, readable from /// inside an callback. Mutated in place; a callback that needs a /// value after the interpreter moves on must copy it. @@ -112,16 +149,37 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _openForms.Clear(); _formDepth = 0; _formInvocations = 0; + _contentBytesRemaining = MaxContentBytes; + _gsFloor = 0; + _markedContentFloor = 0; + _bxFloor = 0; var diagnostics = _reader.CreateContentDiagnosticScope(); var pageIndex = page.Index; - var buffer = BuildPageContentBuffer(page, diagnostics, pageIndex, out var soleObjectNumber); - if (buffer.IsEmpty) - return; + try + { + var buffer = BuildPageContentBuffer(page, diagnostics, pageIndex, out var soleObjectNumber); + if (buffer.IsEmpty) + return; - var ctx = new StreamContext(page.Resources, soleObjectNumber); - InterpretStream(buffer, ctx, visitor, pageIndex, diagnostics); + var ctx = new StreamContext(page.Resources, soleObjectNumber); + InterpretStream(buffer, ctx, visitor, pageIndex, diagnostics); + } + catch (InvalidDataException ex) + { + // The outermost guard for a malformed indirect-reference chain reached through + // resource, XObject, or Form XObject resolution (a corrupt cross-reference offset, + // say): PdfDocumentReader.Resolve and friends can throw here even though this + // interpreter's own lexer and parser never do (InterpretStream's own catch already + // handles those). Consistent with this type's own class doc promise that + // InvalidDataException never escapes Run, and with the notify-and-continue policy + // every other diagnostic in this channel follows. + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"The page's content could not be fully resolved: {ex.Message}", + pageIndex: pageIndex); + } } // ── /Contents resolution and concatenation (ISO 32000-2 §7.7.3.3 Table 31) ───────────────────── @@ -214,7 +272,14 @@ void AddElement(PdfObject element) } soleObjectNumber = soleObjectNumberLocal; - return Concatenate(chunks, diagnostics, pageIndex); + var buffer = Concatenate(chunks, diagnostics, pageIndex, _contentBytesRemaining, out var truncated); + // A truncation here already spent the whole per-Run budget (see Concatenate's own remarks + // on why it always reports at most once): forcing the remainder to exactly zero, rather + // than the few leftover bytes the whitespace-boundary back-off may have left unused, is + // what keeps a later Form XObject from triggering a SECOND ContentStreamTooLarge report + // against a different object number once this run is already over budget. + _contentBytesRemaining = truncated ? 0 : _contentBytesRemaining - buffer.Length; + return buffer; } private static IEnumerable Enumerate(PdfArray array) @@ -227,12 +292,16 @@ private static IEnumerable Enumerate(PdfArray array) // Table 31's own text: "the division between streams may occur only at the boundaries between // lexical tokens". So a token split across two streams (e.g. one ending "BT" and the next // starting immediately with "ET") is not glued into one token by naive concatenation. Enforces - // MaxContentBytes across the total, truncating at the last whitespace boundary within budget - // rather than mid-token, so what IS interpreted is a clean prefix rather than one broken by an - // artificial cut. + // budget (this Run's own remaining share of MaxContentBytes) across the total, truncating at + // the last whitespace boundary within budget rather than mid-token, so what IS interpreted is a + // clean prefix rather than one broken by an artificial cut. Reports and sets + // truncated = true at most once: the caller (BuildPageContentBuffer, or HandleDo for a Form + // XObject's own decoded content) is responsible for driving the run's remaining budget to zero + // once this returns true, so a later stream never re-triggers the report. private static ReadOnlyMemory Concatenate( - List chunks, DiagnosticSink diagnostics, int pageIndex) + List chunks, DiagnosticSink diagnostics, int pageIndex, long budget, out bool truncated) { + truncated = false; if (chunks.Count == 0) return ReadOnlyMemory.Empty; @@ -240,7 +309,7 @@ private static ReadOnlyMemory Concatenate( foreach (var chunk in chunks) total += chunk.Length + 1; // +1 for the separator this method inserts after each chunk - if (total <= MaxContentBytes) + if (total <= budget) { var buffer = new byte[total]; var pos = 0; @@ -256,11 +325,12 @@ private static ReadOnlyMemory Concatenate( // Over budget: copy whole chunks while they fit, then take as much of the chunk that would // overflow as fits, backing off to the nearest preceding whitespace byte so the cut falls on // a token boundary rather than through the middle of one. - var capped = new byte[MaxContentBytes]; + var cappedLength = (int)Math.Min(budget, int.MaxValue); + var capped = new byte[cappedLength]; var written = 0; foreach (var chunk in chunks) { - var remaining = (int)Math.Min(MaxContentBytes - written, int.MaxValue); + var remaining = cappedLength - written; if (remaining <= 0) break; @@ -272,23 +342,32 @@ private static ReadOnlyMemory Concatenate( continue; } - var take = Math.Min(chunk.Length, remaining); - while (take > 0 && !PdfLexer.IsWhitespaceByte(chunk[take - 1])) - take--; - Array.Copy(chunk, capped, take); + var take = TruncateAtWhitespaceBoundary(chunk, Math.Min(chunk.Length, remaining)); + Array.Copy(chunk, 0, capped, written, take); written += take; break; } - diagnostics.Report( + truncated = true; + diagnostics.ReportRetained( PdfReaderDiagnosticCode.ContentStreamTooLarge, $"The page's /Contents exceeded the {MaxContentBytes / (1024 * 1024)} MiB decoded-size " - + "cap; interpretation stopped there.", + + "cap shared with every Form XObject it draws; interpretation stopped there.", pageIndex: pageIndex); return new ReadOnlyMemory(capped, 0, written); } + // Backs a byte-budget cut off to the nearest preceding whitespace byte, so neither Concatenate + // (across a /Contents array) nor HandleDo (a single Form XObject's own decoded content) ever + // cuts a token in half when the per-Run content budget runs out mid-chunk. + private static int TruncateAtWhitespaceBoundary(ReadOnlySpan chunk, int take) + { + while (take > 0 && !PdfLexer.IsWhitespaceByte(chunk[take - 1])) + take--; + return take; + } + // ── Main interpretation loop ───────────────────────────────────────────────────────────────── /// Resources and diagnostic-attribution identity for one content stream being @@ -431,6 +510,13 @@ private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, o span = span[1..]; } + // §7.3.3 allows "an optional sign" (singular). A second sign character immediately after + // the first ("--5", "-+5") is not this grammar's syntax; rejecting it here, rather than + // letting the digit scan below fail on it less directly, keeps the failure attributable to + // the actual malformation instead of a coincidentally-empty digit run. + if (!span.IsEmpty && span[0] is (byte)'+' or (byte)'-') + return false; + if (!isReal) { if (span.IsEmpty || !Utf8Parser.TryParse(span, out long value, out var consumed) || consumed != span.Length) @@ -468,7 +554,7 @@ private void PushOperand(PdfObject value, StreamContext ctx, DiagnosticSink diag { _operandOverflow = true; diagnostics.Report( - PdfReaderDiagnosticCode.OperandStackMalformed, + PdfReaderDiagnosticCode.ContentLimitExceeded, $"More than {MaxOperandsPerOperator} operands accumulated before an operator; the " + "next operator was dropped.", ctx.DiagObjectNumber, pageIndex: pageIndex); @@ -492,17 +578,24 @@ private void HandleOperator( { if (!ContentOperators.IsKnown(name)) { - // ISO 32000-2 §7.8.2: "an error shall occur" outside a compatibility section; this - // reader instead notifies and continues. Deliberately does NOT clear the operand stack - // (this reader's own leniency, distinct from Table 33's "ignored ... along with - // operands" for a later PDF version's operator inside BX/EX): the most common way an - // unrecognised keyword appears in an otherwise-conforming stream is a stray "R" left - // over from indirect-reference syntax that §7.8.2 forbids in content streams at all - // ("Indirect objects and object references shall not be permitted"), and the operands - // that precede it usually belong to whatever REAL operator follows, not to "R" itself. - // The sink's dedupe key is (code, object, page), so only the first unknown name on a - // page is recorded; a second distinct name on the same page is dropped by the sink. - if (_bxDepth == 0) + // Two different rules apply here depending on _bxDepth. Inside a BX/EX compatibility + // section, Table 33 is explicit: "Unrecognised operators (along with their operands) + // shall be ignored without error until the balancing EX operator is encountered", so + // the operand stack IS cleared, and nothing is reported. Outside one, §7.8.2 says "an + // error shall occur"; this reader instead notifies and continues, and deliberately does + // NOT clear the operand stack, a leniency of this reader's own rather than anything + // Table 33 asks for: the most common way an unrecognised keyword appears in an + // otherwise-conforming stream is a stray "R" left over from indirect-reference syntax + // that §7.8.2 forbids in content streams at all ("Indirect objects and object + // references shall not be permitted"), and the operands that precede it usually belong + // to whatever REAL operator follows, not to "R" itself. The sink's dedupe key is + // (code, object, page), so only the first unknown name on a page is recorded; a second + // distinct name on the same page is dropped by the sink. + if (_bxDepth > 0) + { + ClearOperands(); + } + else { diagnostics.Report( PdfReaderDiagnosticCode.UnknownOperator, @@ -521,7 +614,7 @@ private void HandleOperator( } if (name is "EX") { - if (_bxDepth > 0) + if (_bxDepth > _bxFloor) _bxDepth--; EmitAndClear(name, offset, visitor); return; @@ -565,12 +658,20 @@ private void HandleOperator( switch (name) { case "TJ": - if (_operands[0] is not PdfArray tjArray || tjArray.Count > MaxTjElements) + if (_operands[0] is not PdfArray tjArray) { diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, - $"TJ's array operand is missing or exceeds {MaxTjElements} elements; it was " - + "dropped.", + "TJ's operand is not an array; it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + return; + } + if (tjArray.Count > MaxTjElements) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentLimitExceeded, + $"TJ's array operand exceeds {MaxTjElements} elements; it was dropped.", ctx.DiagObjectNumber, pageIndex: pageIndex); ClearOperands(); return; @@ -698,7 +799,7 @@ private void PushGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, in if (_gsStack.Count >= MaxGraphicsStateDepth) { diagnostics.Report( - PdfReaderDiagnosticCode.OperandStackMalformed, + PdfReaderDiagnosticCode.ContentLimitExceeded, $"The graphics-state stack exceeded {MaxGraphicsStateDepth} nested 'q' saves; " + "further saves were ignored.", ctx.DiagObjectNumber, pageIndex: pageIndex); @@ -710,11 +811,14 @@ private void PushGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, in private void PopGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) { - if (_gsStack.Count == 0) + if (_gsStack.Count <= _gsFloor) { // An unbalanced 'q' still open at end of stream is fine (nothing downstream needs the // state restored past the last operator this interpreter saw); an unbalanced 'Q' is the - // opposite problem (a restore with nothing to restore), so this one is reported. + // opposite problem (a restore with nothing to restore), so this one is reported. Inside + // a Form XObject's own content, _gsFloor is that form's own entry depth (see HandleDo), + // so a form's own 'Q' can pop no further than where the form started, and a 'Q' the + // page itself already owns is never available for the form's content to pop. diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, "'Q' with no matching 'q' on the graphics-state stack; it was ignored.", @@ -729,7 +833,7 @@ private void PushMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, in if (_markedContentDepth >= MaxMarkedContentDepth) { diagnostics.Report( - PdfReaderDiagnosticCode.OperandStackMalformed, + PdfReaderDiagnosticCode.ContentLimitExceeded, $"Marked-content nesting exceeded {MaxMarkedContentDepth} levels; further " + "'BMC'/'BDC' operators were ignored.", ctx.DiagObjectNumber, pageIndex: pageIndex); @@ -740,7 +844,7 @@ private void PushMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, in private void PopMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) { - if (_markedContentDepth == 0) + if (_markedContentDepth <= _markedContentFloor) { diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, @@ -821,9 +925,13 @@ private void HandleExtGState(StreamContext ctx, DiagnosticSink diagnostics, int return; // Table 57: /Font is "an array of the form [font size] where font shall be an indirect - // reference to a font dictionary". Every other ExtGState entry is out of scope for this - // interpreter: it neither positions text nor renders colour or transparency. - if (extGState.Get(FontKey) is PdfArray fontArray && fontArray.Count == 2) + // reference to a font dictionary". §7.3.10 lets any dictionary entry be given as an + // indirect reference, not only the ones a table says must be one, so /Font's own value is + // resolved before the shape check the same way a form XObject's /Matrix and /BBox are. + // Every other ExtGState entry is out of scope for this interpreter: it neither positions + // text nor renders colour or transparency. + if (extGState.Get(FontKey) is { } fontRaw && _reader.ResolveValue(fontRaw) is PdfArray fontArray + && fontArray.Count == 2) { _gs.Font = fontArray[0]; _gs.FontSize = ReadNumber(fontArray[1]); @@ -872,7 +980,7 @@ private void HandleDo( if (_formInvocations >= MaxFormInvocationsPerPage) { - diagnostics.Report( + diagnostics.ReportRetained( PdfReaderDiagnosticCode.FormXObjectBudgetExceeded, $"The page invoked more than {MaxFormInvocationsPerPage} Form XObjects; further " + "'Do' recursions were skipped for the rest of the page.", @@ -928,10 +1036,86 @@ private void HandleDo( decoded = null; } + if (decoded is not null) + { + // Every invocation of a form counts its bytes again against this Run's own + // shared budget: the cost being bounded is interpretation WORK, and a form + // drawn many times is interpreted that many times, not decoded-and-cached once. + if (_contentBytesRemaining <= 0) + { + decoded = null; // Budget already spent; skip this invocation's content. + } + else if (decoded.Length > _contentBytesRemaining) + { + var take = TruncateAtWhitespaceBoundary( + decoded, (int)Math.Min(decoded.Length, _contentBytesRemaining)); + decoded = decoded.AsSpan(0, take).ToArray(); + diagnostics.ReportRetained( + PdfReaderDiagnosticCode.ContentStreamTooLarge, + $"Form XObject {objectNumber}'s content pushed this Run's combined " + + $"page-and-forms budget past {MaxContentBytes / (1024 * 1024)} MiB; " + + "interpretation of it stopped there.", + objectNumber, pageIndex: pageIndex); + // See BuildPageContentBuffer's own remark on why this is forced to exactly + // zero rather than left at whatever the whitespace back-off did not use. + _contentBytesRemaining = 0; + } + else + { + _contentBytesRemaining -= decoded.Length; + } + } + if (decoded is not null) { var formCtx = new StreamContext(formResources, objectNumber); - InterpretStream(decoded, formCtx, visitor, pageIndex, diagnostics); + + // ISO 32000-2 §8.10.1: Do on a form XObject "a) Saves the current graphics + // state, as if by invoking the q operator" before interpreting the form's own + // content, and "e) Restores [it], as if by invoking the Q operator" once done. + // Implemented with floors rather than an actual _gsStack.Push/Pop so the + // implicit save does not itself consume the invoker's own + // MaxGraphicsStateDepth budget, and the form's own content cannot 'Q' past its + // own entry point back into the invoker's own saves. Marked-content nesting + // (§14.6.1) and BX/EX depth get the same bracketing: nothing a form does to + // either may leak into the invoker once Do returns. Text state (_textState, + // the Tm/Tlm pair) is deliberately NOT saved here: Do is not itself a + // text-showing operator and is not permitted inside a text object (§8.2 + // Figure 9 / Table 51's own allowed-operator sets), so there is no open text + // object for a form's own content to disturb in the first place. + var savedGs = _gs; + var savedGsStackCount = _gsStack.Count; + var savedMarkedContentDepth = _markedContentDepth; + var savedBxDepth = _bxDepth; + var savedGsFloor = _gsFloor; + var savedMarkedContentFloor = _markedContentFloor; + var savedBxFloor = _bxFloor; + + _gsFloor = savedGsStackCount; + _markedContentFloor = savedMarkedContentDepth; + _bxFloor = savedBxDepth; + _gs = _gs.Clone(); + + // §7.8.2: operands never carry across an operator, and Do is the operator + // here, so neither an operand left over from before this Do nor one trailing + // the form's own content should reach the operator that follows on either side. + ClearOperands(); + try + { + InterpretStream(decoded, formCtx, visitor, pageIndex, diagnostics); + } + finally + { + ClearOperands(); + while (_gsStack.Count > savedGsStackCount) + _gsStack.Pop(); + _gs = savedGs; + _markedContentDepth = savedMarkedContentDepth; + _bxDepth = savedBxDepth; + _gsFloor = savedGsFloor; + _markedContentFloor = savedMarkedContentFloor; + _bxFloor = savedBxFloor; + } } } finally @@ -951,14 +1135,19 @@ private void HandleDo( private Matrix ReadFormMatrix(PdfDictionary formDict) { - if (formDict.Get(MatrixKey) is PdfArray arr && arr.Count == 6 && TryReadNumbers(arr, out var v)) + // §7.3.10 permits any dictionary entry to be an indirect reference; Table 93 gives /Matrix + // no direct-only restriction, so the entry is resolved before the shape check below. + if (formDict.Get(MatrixKey) is { } raw && _reader.ResolveValue(raw) is PdfArray arr + && arr.Count == 6 && TryReadNumbers(arr, out var v)) return new Matrix(v[0], v[1], v[2], v[3], v[4], v[5]); return Matrix.Identity; // §8.10.2 Table 93's own default. } private PdfRectangle? ReadFormBBox(PdfDictionary formDict) { - if (formDict.Get(BBoxKey) is PdfArray arr && arr.Count == 4 && TryReadNumbers(arr, out var v)) + // Same reasoning as ReadFormMatrix above: /BBox may be indirect too. + if (formDict.Get(BBoxKey) is { } raw && _reader.ResolveValue(raw) is PdfArray arr + && arr.Count == 4 && TryReadNumbers(arr, out var v)) { return new PdfRectangle( Math.Min(v[0], v[2]), Math.Min(v[1], v[3]), Math.Max(v[0], v[2]), Math.Max(v[1], v[3])); @@ -1028,15 +1217,22 @@ private bool HandleInlineImage( value = InlineImageAbbreviations.ExpandColorSpaceOrFilterName( PdfObjectParser.ParseName(valueTok), isColorSpaceKey); } - else if (valueTok.Kind == TokenKind.ArrayBegin && isFilterKey) + else if (valueTok.Kind == TokenKind.ArrayBegin && (isFilterKey || isColorSpaceKey)) { lexer.Seek(valueStart); var arr = (PdfArray)parser.ParseObject(); var items = new List(arr.Count); for (var i = 0; i < arr.Count; i++) { - items.Add(arr[i] is PdfName elName - ? InlineImageAbbreviations.ExpandColorSpaceOrFilterName(elName, isColorSpace: false) + // §8.9.7 permits exactly one composite inline colour space, "a limited form of + // Indexed colour space" whose base is a device space, written as an array: + // [/I baseSpace hival lookup]. Only elements 0 and 1 are colour-space NAMES + // eligible for Table 92 expansion there (hival is a number, lookup a string or + // stream); a /Filter array has no such shape restriction, so every element of + // one is eligible. + var eligible = isFilterKey || i < 2; + items.Add(arr[i] is PdfName elName && eligible + ? InlineImageAbbreviations.ExpandColorSpaceOrFilterName(elName, isColorSpace: isColorSpaceKey) : arr[i]); } value = new PdfArray(items); @@ -1055,16 +1251,32 @@ private bool HandleInlineImage( // interpreted as the first byte of image data." Consuming at most one whitespace byte here // is correct for every filter, including AHx/A85, since a producer is free to include that // one separating byte regardless (the spec exempts them from being REQUIRED to, not from - // being ALLOWED to). - if (lexer.TryPeek() is var b && b >= 0 && PdfLexer.IsWhitespaceByte((byte)b)) - lexer.Seek(lexer.Position + 1); + // being ALLOWED to). §7.2.3: "The combination of a CARRIAGE RETURN followed immediately by + // a LINE FEED shall be treated as one EOL marker", so a CR immediately followed by an LF + // is consumed as that ONE separator, not as the separator plus a data byte. + var consumedCrLf = false; + if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 + && PdfLexer.IsWhitespaceByte((byte)separatorByte)) + { + if (separatorByte == (byte)'\r' && lexer.Position + 1 < _currentBuffer.Length + && _currentBuffer.Span[lexer.Position + 1] == (byte)'\n') + { + lexer.Seek(lexer.Position + 2); + consumedCrLf = true; + } + else + { + lexer.Seek(lexer.Position + 1); + } + } var dataStart = lexer.Position; var filterNames = CollectFilterNames(dict); var hasDisallowedFilter = filterNames.Any(f => f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); - var length = TryLengthFromDictionary(dict, dataStart, out var lengthPastEnd); + var length = TryLengthFromDictionary(dict, dataStart, ctx, diagnostics, pageIndex, out var lengthPastEnd); + var usedTierA = length is not null; if (lengthPastEnd) { ReportInlineImageMalformed( @@ -1074,6 +1286,7 @@ private bool HandleInlineImage( if (length is null && filterNames.Count == 0) length = TryComputeUnfilteredLength(dict, ctx, dataStart, diagnostics, pageIndex); + var lengthFromScan = false; if (length is null) { var scanEnd = ScanForEi(dataStart); @@ -1085,6 +1298,7 @@ private bool HandleInlineImage( return false; } length = scanEnd.Value - dataStart; + lengthFromScan = true; } var dataEnd = dataStart + length.Value; @@ -1097,8 +1311,53 @@ private bool HandleInlineImage( } var data = _currentBuffer.Slice(dataStart, length.Value); - var resyncPos = SkipToEi(dataEnd); + + // A tier-a (/L) or tier-b (computed) length that does not land on 'EI' is symmetric with + // the /L-past-the-end case above: both recover through the same EI scan (tier c) rather + // than losing the rest of the content stream outright. Tier c itself is excluded here + // (lengthFromScan) since it already IS that fallback. + if (resyncPos is null && !lengthFromScan) + { + var tierName = usedTierA ? "/L" : "the unfiltered image's computed length"; + ReportInlineImageMalformed( + $"the image data length from {tierName} did not land on an 'EI' operator", ctx, + diagnostics, pageIndex); + + if (consumedCrLf) + { + // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for + // binary image data that reading is ambiguous: a producer may have meant only the + // CR as the ID separator, with the LF as the image's own first byte. Retry once + // with the data window shifted one byte earlier, before falling back to the scan, + // since a payload that happens to begin with LF right after a CR separator is + // exactly the case the CR-LF-as-one-marker choice above would otherwise misjudge. + var retryStart = dataStart - 1; + var retryEnd = retryStart + length.Value; + if (retryStart >= 0 && retryEnd <= _currentBuffer.Length) + { + var retryResync = SkipToEi(retryEnd); + if (retryResync is not null) + { + dataStart = retryStart; + data = _currentBuffer.Slice(dataStart, length.Value); + resyncPos = retryResync; + } + } + } + + if (resyncPos is null) + { + var scanEnd = ScanForEi(dataStart); + if (scanEnd is not null) + { + length = scanEnd.Value - dataStart; + data = _currentBuffer.Slice(dataStart, length.Value); + resyncPos = SkipToEi(dataStart + length.Value); + } + } + } + if (resyncPos is null) { ReportInlineImageMalformed( @@ -1148,11 +1407,19 @@ private List CollectFilterNames(PdfDictionary dict) // Tier (a): /L, PDF 2.0's own Table 91 entry (§8.9.7). See InlineImageAbbreviations' own remarks // for why this cites Table 91 rather than Table 93, which ISO 32000-2 reserves for a Form // XObject dictionary's unrelated entries (§8.10.2). - private int? TryLengthFromDictionary(PdfDictionary dict, int dataStart, out bool pastEnd) + private int? TryLengthFromDictionary( + PdfDictionary dict, int dataStart, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex, + out bool pastEnd) { pastEnd = false; - if (dict.Get(PdfName.Length) is not PdfInteger lengthObj || lengthObj.Value < 0) + if (dict.Get(PdfName.Length) is not PdfInteger lengthObj) + return null; + + if (lengthObj.Value < 0) + { + ReportInlineImageMalformed("'/L' is negative; it was ignored", ctx, diagnostics, pageIndex); return null; + } var length = lengthObj.Value; if (length > int.MaxValue || dataStart + length > _currentBuffer.Length) @@ -1174,11 +1441,14 @@ private List CollectFilterNames(PdfDictionary dict) var height = ReadIntEntry(dict, HeightKey); var bpc = isMask ? 1 : ReadIntEntry(dict, BitsPerComponentKey); - if (width is null || height is null || bpc is null || width < 0 || height < 0 || bpc <= 0) + // Table 87 types Width, Height, and BitsPerComponent as positive integers. ReadIntEntry + // already turns a non-integer or an out-of-int-range value into "missing" (null); zero or + // negative is the one shape left for this method itself to reject. + if (width is null || height is null || bpc is null || width <= 0 || height <= 0 || bpc <= 0) { ReportInlineImageMalformed( - "an unfiltered image is missing /W, /H, or /BPC needed to compute its data length", - ctx, diagnostics, pageIndex); + "an unfiltered image is missing, or carries an invalid, /W, /H or /BPC needed to " + + "compute its data length", ctx, diagnostics, pageIndex); return null; } @@ -1194,23 +1464,42 @@ private List CollectFilterNames(PdfDictionary dict) return (int)total; } + // Table 87 types Width, Height, and BitsPerComponent as integers; a PdfReal there, or an + // integer outside int's range, is invalid rather than merely absent, but both are treated as + // "missing" here so TryComputeUnfilteredLength's own null check catches either uniformly. private static int? ReadIntEntry(PdfDictionary dict, PdfName key) => dict.Get(key) switch { - PdfInteger i => (int)i.Value, - PdfReal r => (int)r.Value, + PdfInteger i when i.Value >= int.MinValue && i.Value <= int.MaxValue => (int)i.Value, _ => null, }; private int ResolveComponentCount( PdfDictionary dict, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) { - if (dict.Get(PdfName.ColorSpace) is not PdfName csName) + var csValue = dict.Get(PdfName.ColorSpace); + + // §8.9.7's one composite inline colour space: [/Indexed base hival lookup]. Indexed + // samples are always single-component index values (§8.6.6.3) regardless of the base + // space's own component count, so this is the one array shape this method needs to + // recognise without resolving the base space at all. + if (csValue is PdfArray csArray && csArray.Count > 0 && csArray[0] is PdfName firstName + && firstName.Value == "Indexed") + return 1; + + if (csValue is not PdfName csName) return -1; if (csName.Equals(PdfName.DeviceRGB)) return 3; if (csName.Value == "DeviceCMYK") return 4; if (csName.Value == "DeviceGray") return 1; - if (csName.Value == "Indexed") return 1; + + // A bare "/Indexed" name, rather than the array form above, is not itself a legal colour + // space (§8.6.6.3 requires the array shape); it is also not a /Resources /ColorSpace + // entry name a producer would define, so looking it up there and reporting + // ResourceMissing when it is (as expected) absent would be wrong twice over. Falling back + // to the EI scan silently is the simplest correct outcome. + if (csName.Value == "Indexed") + return -1; // A named resource colour space (§8.9.7: "the value of the ColorSpace entry may also be the // name of a colour space in the ColorSpace subdictionary of the current resource @@ -1230,8 +1519,8 @@ private int ResolveComponentCount( return -1; } - // Tier (c): scan for whitespace-EI-whitespace/EOF, accepted only when what follows lexes as - // operators or EOF in content mode (the false-EI-inside-DCT-data problem). + // Tier (c): scan for whitespace-EI-whitespace/EOF, accepted only when the bounded probe just + // past the candidate (LooksLikeResyncPoint) does not reject it. private int? ScanForEi(int dataStart) { var span = _currentBuffer.Span; @@ -1273,43 +1562,83 @@ private int ResolveComponentCount( return pos + 2; } - // Bounded lookahead: lexes up to a handful of tokens in content mode from a candidate resync - // point and accepts it only if none of them throws and every keyword among them is an operator - // Annex A Table A.1 defines (or true/false/null). The lexer alone is a weak filter for a - // coincidental "EI" byte pair inside DCT-compressed data: any run of bytes outside §7.2.2's - // whitespace and delimiter sets lexes as one Keyword token, so binary noise after a false EI - // very often lexes cleanly. Requiring the keywords to be operators is what rejects it, since - // a byte run like 0x8F 0x12 0xC4 is never an operator name. The one construct this rejects - // wrongly is an unknown operator inside a BX/EX section right after an inline image, which - // then falls through to a later EI candidate; the false-positive cost of accepting binary - // noise (the rest of the stream lost to ContentStreamLexError) is the worse of the two. + // Bounded lookahead: lexes at most ProbeTokens tokens from a candidate resync point, over at + // most ProbeWindowBytes bytes, and accepts it unless one of those tokens is a keyword this + // probe cannot justify as legitimate content-stream syntax. Bounding both the token count and + // the byte window is what keeps ScanForEi linear in the content length: an earlier version of + // this probe lexed all the way to the end of the buffer from every candidate, which cost O(N) + // work per candidate and made a content stream built from many false 'EI' candidates (a + // filtered image followed by literal " EI (" text repeated many times, say) quadratic overall. + // + // A non-keyword token (a number, name, string, array or dictionary delimiter, whatever its + // bytes) is neutral: it neither accepts nor rejects the candidate on its own. A keyword is + // accepted when it is a Table A.1 operator (or true/false/null, or the one-byte content-mode + // keywords '{', '}', '>' this lexer's own content-stream mode produces), and otherwise accepted + // only when every one of its bytes is printable ASCII (0x21 to 0x7E): an unknown-but-printable + // keyword is the kind of thing §7.8.2 already tolerates outside a compatibility section (a + // future operator this reader does not know yet, a stray "R"), while a byte run containing + // anything else is binary noise a coincidental "EI" byte pair inside DCT- or JPX-compressed + // data would otherwise be mistaken for legitimate syntax. A 'BI' keyword ends the probe with + // acceptance immediately, without lexing further: the bytes after ITS OWN following 'ID' are + // raw image data and must not be judged as tokens at all. private bool LooksLikeResyncPoint(int pos) { - var probe = new PdfLexer(_currentBuffer, contentStreamMode: true); - probe.Seek(pos); - try + // Whether this window is itself an artificial cap, i.e. more buffer exists beyond it that + // this probe deliberately does not look at. Only THAT case makes "ran off the window" + // inconclusive rather than an outright rejection: a short buffer where the window reaches + // the buffer's own true end behaves exactly like the unbounded lexer this probe replaces + // (a token that never closes anywhere is malformed, full stop), so it must still reject + // there. Matters for a short fixture, or a content stream that stops mid-token, where the + // window and "the rest of the buffer" happen to coincide. + var remaining = _currentBuffer.Length - pos; + var windowClipped = remaining > ProbeWindowBytes; + var windowLength = Math.Min(ProbeWindowBytes, remaining); + var window = _currentBuffer.Slice(pos, windowLength); + var probe = new PdfLexer(window, contentStreamMode: true); + + for (var i = 0; i < ProbeTokens; i++) { - for (var i = 0; i < 8; i++) + if (probe.AtEnd) + return true; + + Token token; + try { - if (probe.AtEnd) - return true; - var token = probe.NextToken(); - if (token.Kind == TokenKind.EndOfInput) - return true; - if (token.Kind != TokenKind.Keyword) - continue; + token = probe.NextToken(); + } + catch (InvalidDataException) + { + // Ran off the end of the window mid-token (an unterminated literal or hex string + // straddling the boundary): inconclusive, since PdfLexer's own string readers + // advance Position as they go, so Position sitting at or past the window's own + // length here means the failure was purely the window's own limit, not malformed + // bytes the probe saw, PROVIDED the window was clipped in the first place (see + // above). Anything else, including running off a window that already reached the + // buffer's own true end, rejects instead. + return windowClipped && probe.Position >= window.Length; + } - var raw = token.Raw.Span; - if (raw.SequenceEqual("true"u8) || raw.SequenceEqual("false"u8) || raw.SequenceEqual("null"u8)) - continue; - if (!ContentOperators.IsKnown(System.Text.Encoding.Latin1.GetString(raw))) + if (token.Kind == TokenKind.EndOfInput) + return true; + if (token.Kind != TokenKind.Keyword) + continue; + + var raw = token.Raw.Span; + if (raw.SequenceEqual("BI"u8)) + return true; + if (raw.SequenceEqual("true"u8) || raw.SequenceEqual("false"u8) || raw.SequenceEqual("null"u8)) + continue; + if (raw.Length == 1 && (raw[0] == (byte)'{' || raw[0] == (byte)'}' || raw[0] == (byte)'>')) + continue; + if (ContentOperators.IsKnown(System.Text.Encoding.Latin1.GetString(raw))) + continue; + + foreach (var b in raw) + { + if (b is < (byte)'!' or > (byte)'~') return false; } - return true; - } - catch (InvalidDataException) - { - return false; } + return true; } } diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs index 032bee81..3055eb7f 100644 --- a/src/VellumPdf.Reader/Content/IContentVisitor.cs +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -59,7 +59,10 @@ internal interface IContentVisitor /// when absent or malformed. The interpreter does not compose this /// with the CTM itself; that is left to the caller. /// The form's /BBox (Table 93, Required), or - /// when absent or malformed. + /// when the entry is absent (Table 93 marks it Required, so a + /// here is itself a malformation the visitor may report) or does not + /// resolve to a four-number array. This interpreter raises no new diagnostic of its own for + /// either case. /// The form stream's own indirect object number. /// The byte offset of the Do operator that invoked this form, in the /// buffer the interpreter was walking at the time, i.e. the INVOKING stream's own offset space, diff --git a/src/VellumPdf.Reader/PdfLexer.cs b/src/VellumPdf.Reader/PdfLexer.cs index 3e8cbf33..28b9576c 100644 --- a/src/VellumPdf.Reader/PdfLexer.cs +++ b/src/VellumPdf.Reader/PdfLexer.cs @@ -79,14 +79,17 @@ internal sealed class PdfLexer // Off by default: every existing consumer (the object parser, the 11 Conformance rules that // build their own PdfLexer, and every non-content test) relies on the file-structure lexer's - // stricter rule that a lone '{', '}', or unmatched '>' is malformed PDF (they never appear - // outside a content stream at all: ISO 32000-2 §7.2.2 lists them as delimiters with no syntax - // of their own anywhere else). A content stream is a different grammar: it may carry them as - // one-byte PostScript-heritage tokens inside a BX/EX compatibility section (§7.8.2), which - // ContentInterpreter needs to lex as harmless unknown-operator keywords instead of aborting the - // whole page over a construct §7.8.2 explicitly says to tolerate. Opt-in, not a runtime - // detection of "is this a content stream", keeps that decision with the caller that already - // knows which grammar applies instead of guessing from the bytes. + // stricter rule that a lone '{', '}', or unmatched '>' is malformed PDF. §7.2.3 Table 2 lists + // '{' and '}' as delimiters with no syntax of their own in the file-structure grammar; the one + // place they mean anything is inside a Type 4 PostScript calculator function body (§7.10.5), + // which this lexer is never pointed at, so the file-structure grammar itself has no production + // for a lone one and it stays malformed in default mode. A content stream is a different + // grammar: those same bytes are junk with no meaning of their own there either, but §7.8.2's + // BX/EX rule is the reason to tolerate junk inside a compatibility section rather than abort + // the whole page over it, so ContentInterpreter needs to lex them as harmless one-byte + // unknown-operator keywords instead. Opt-in, not a runtime detection of "is this a content + // stream", keeps that decision with the caller that already knows which grammar applies + // instead of guessing from the bytes. private readonly bool _contentStreamMode; /// Current byte offset within . diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 67b73ced..541e09a0 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -369,10 +369,11 @@ public enum PdfReaderDiagnosticCode /// lists, and it did not appear inside a BX/EX compatibility section (§7.8.2). ISO /// 32000-2 says "an error shall occur" for this case outside such a section; this reader instead /// reports it and continues, the same notify-and-continue choice every other diagnostic in this - /// channel makes. Reported at most once per (page, operator name) rather than once per - /// occurrence, since a producer that emits a future operator this reader does not know about - /// typically emits it many times on the same page. Silent inside a compatibility section, per - /// Table 33's own text: "Unrecognised operators ... shall be ignored without error." + /// channel makes. Reported at most once per page, because the sink's dedupe key is + /// (code, object, page) and this report carries no object number: a second distinct + /// unrecognised name on the same page is deduped away rather than reported. Silent inside a + /// compatibility section, per Table 33's own text: "Unrecognised operators ... shall be ignored + /// without error." /// UnknownOperator = 301, @@ -380,17 +381,18 @@ public enum PdfReaderDiagnosticCode /// A content stream's operand-stack discipline broke down in one of several ways this /// interpreter groups under one code rather than one each, since every case has the same /// remedy: drop the offending operator (or, for an unbalanced Q/EMC, drop the - /// pop) and keep interpreting. Covers: more than 32 operands accumulated before an operator - /// (§7.8.2 gives an operator's operands no declared bound of its own; this reader's own - /// ceiling), a TJ array (§9.4.3) with more than 8192 elements, a number token that does - /// not parse or is not finite, a dictionary operand on an operator other than BDC/DP - /// (§7.8.2: "Dictionaries shall be permitted as operands only by certain specific operators"), - /// a known operator invoked with the wrong operand count for its own arity (Annex A Table A.1), - /// an unbalanced Q with no matching q on the graphics-state stack, more than 64 - /// nested q saves, or an unbalanced EMC/deeply nested BMC/BDC - /// (§14.6.2) past the same 64-deep cap. An unbalanced q still open at the end of a - /// content stream is not reported: nothing downstream of this interpreter needs the graphics - /// state restored past the last operator it saw. + /// pop) and keep interpreting. Every case here is a PRODUCER-side malformation, the document + /// itself is wrong, not merely bigger than this reader is willing to process; see + /// for the four cases that are this reader's own processing + /// ceiling instead. Covers: a number token that does not parse, is not finite, or carries a + /// second sign character (--5, -+5: §7.3.3 allows only "an optional sign"), a + /// dictionary operand on an operator other than BDC/DP (§7.8.2: "Dictionaries + /// shall be permitted as operands only by certain specific operators"), a known operator + /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an unbalanced + /// Q with no matching q on the graphics-state stack, or an unbalanced EMC + /// with no matching BMC/BDC (§14.6.1). An unbalanced q still open at the + /// end of a content stream is not reported: nothing downstream of this interpreter needs the + /// graphics state restored past the last operator it saw. /// OperandStackMalformed = 302, @@ -417,7 +419,9 @@ public enum PdfReaderDiagnosticCode /// whole page, not per subtree) more than 4096 times. Descent into any further form stops for /// the rest of the page; operators already reported before the budget was reached are kept, and /// interpretation of the page's own (non-form) content continues past the point where the - /// budget was hit. + /// budget was hit. Reported through DiagnosticSink.ReportRetained: a condition that ends + /// the page's own form recursion for good is worth surfacing even once + /// is spent on earlier, unrelated conditions. /// FormXObjectBudgetExceeded = 305, @@ -432,25 +436,49 @@ public enum PdfReaderDiagnosticCode ResourceMissing = 306, /// - /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded: a filter this - /// interpreter never applies to inline image data (JBIG2Decode, JPXDecode, or - /// Crypt: §8.9.7 itself excludes all three from inline-image use), a missing ID - /// or EI operator, a missing /W, /H, or /BPC where the image's shape - /// requires one to compute the data length, or an /L (§8.9.7, Table 91; PDF 2.0) past the - /// end of the stream. The image is skipped (its data is still delimited well enough for + /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded, or one of its + /// dictionary entries was itself invalid: a filter this interpreter never applies to inline + /// image data (JBIG2Decode, JPXDecode, or Crypt: §8.9.7 itself excludes + /// all three from inline-image use), a missing ID or EI operator, a missing, + /// non-integer, or non-positive /W, /H, or /BPC where the image's shape + /// requires one to compute the data length (Table 87 types all three as positive integers), a + /// negative /L (§8.9.7, Table 91; PDF 2.0), an /L or computed length past the end + /// of the stream, or a computed length (from /L or from the image's own shape) that does + /// not land on the following EI operator, in which case this reader retries the EI scan + /// before giving up. The image is skipped (its data is still delimited well enough for /// interpretation of the rest of the content stream to continue), and no inline-image callback /// is raised for it. /// InlineImageMalformed = 307, /// - /// A page's /Contents (ISO 32000-2 §7.7.3.3 Table 31), concatenated across every stream - /// in the array with a newline inserted between streams so a token is never glued across a - /// stream boundary, exceeded 64 MiB of decoded bytes. Interpretation proceeds up to the cap and - /// stops there; operators reported before the cap was reached are kept. + /// This run's combined decoded-content budget, 64 MiB shared across the page's own + /// /Contents (ISO 32000-2 §7.7.3.3 Table 31) and every Form XObject it draws (§8.10), was + /// exceeded. /Contents is concatenated across every stream in its array with a newline + /// inserted between streams so a token is never glued across a stream boundary; a Form XObject + /// is counted again on every invocation, not once per distinct form object, since the + /// interpretation work a repeatedly-drawn form costs scales with invocations, not with how many + /// distinct form objects a page names. Interpretation proceeds up to the point the budget ran + /// out and stops there; operators reported before that point are kept. Reported through + /// DiagnosticSink.ReportRetained, and at most once per run: the truncation this reports + /// also drives the run's own remaining budget to exactly zero, so no later stream in the same + /// run can trigger a second report. /// ContentStreamTooLarge = 308, + /// + /// A content stream hit one of this reader's own processing ceilings rather than being + /// malformed by its producer: more than 64 operands accumulated before an operator (§7.8.2 + /// gives an operator's own operand count no declared bound of its own), a TJ array + /// (§9.4.3) with more than 8192 elements, more than 64 nested q saves, or marked-content + /// nesting (§14.6.1) past the same 64-deep cap. Split out from + /// (#402) so a caller can tell "this file hit a limit of this reader" apart from "this file is + /// malformed", a distinction the two codes sharing one value made impossible to draw. The + /// offending operator, or push, is dropped and interpretation continues, the same recovery + /// uses. + /// + ContentLimitExceeded = 309, + // ── 9xx: reserved ─────────────────────────────────────────────────────────────────────────── /// @@ -511,7 +539,7 @@ internal static class PdfReaderDiagnosticSeverities PdfReaderDiagnosticCode.PageTreeNodeMalformed => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.ContentStreamLexError => PdfReaderDiagnosticSeverity.Warning, - PdfReaderDiagnosticCode.UnknownOperator => PdfReaderDiagnosticSeverity.Info, + PdfReaderDiagnosticCode.UnknownOperator => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.OperandStackMalformed => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.FormXObjectDepthExceeded => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.FormXObjectCycle => PdfReaderDiagnosticSeverity.Warning, @@ -519,6 +547,7 @@ internal static class PdfReaderDiagnosticSeverities PdfReaderDiagnosticCode.ResourceMissing => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.InlineImageMalformed => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.ContentStreamTooLarge => PdfReaderDiagnosticSeverity.Warning, + PdfReaderDiagnosticCode.ContentLimitExceeded => PdfReaderDiagnosticSeverity.Warning, PdfReaderDiagnosticCode.DiagnosticsSuppressed => PdfReaderDiagnosticSeverity.Warning, _ => throw new UnreachableException($"No severity is mapped for {code}."), }; diff --git a/src/VellumPdf.Reader/PublicAPI.Unshipped.txt b/src/VellumPdf.Reader/PublicAPI.Unshipped.txt index 225d8c53..f54877a3 100644 --- a/src/VellumPdf.Reader/PublicAPI.Unshipped.txt +++ b/src/VellumPdf.Reader/PublicAPI.Unshipped.txt @@ -37,6 +37,7 @@ VellumPdf.Reader.PdfReaderDiagnostic.PageIndex.get -> int? VellumPdf.Reader.PdfReaderDiagnostic.Severity.get -> VellumPdf.Reader.PdfReaderDiagnosticSeverity override VellumPdf.Reader.PdfReaderDiagnostic.ToString() -> string! VellumPdf.Reader.PdfReaderDiagnosticCode +VellumPdf.Reader.PdfReaderDiagnosticCode.ContentLimitExceeded = 309 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.ContentStreamLexError = 300 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.ContentStreamTooLarge = 308 -> VellumPdf.Reader.PdfReaderDiagnosticCode VellumPdf.Reader.PdfReaderDiagnosticCode.DecodeParmsMalformed = 108 -> VellumPdf.Reader.PdfReaderDiagnosticCode diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 18777cf3..0b550b5a 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -224,7 +224,7 @@ public void UnknownOperator_outsideBX_isReportedOncePerPage_andInsideBX_isSilent var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); Assert.Single(reports); - Assert.Equal(PdfReaderDiagnosticSeverity.Info, reports[0].Severity); + Assert.Equal(PdfReaderDiagnosticSeverity.Warning, reports[0].Severity); Assert.Contains("'Zork'", reports[0].Message); } @@ -239,38 +239,75 @@ public void CurlyBracesAndLoneGreaterThan_insideBX_doNotAbortThePage() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void UnknownOperatorInsideBX_dropsItsOperands_perTable33() + { + // Table 33: "Unrecognised operators (along with their operands) shall be ignored without + // error until the balancing EX operator is encountered" (#402). Before the fix, the + // unknown-operator branch never cleared the operand stack regardless of _bxDepth, so the + // 1 2 3 preceding SomeFutureOp survived to be misread as w's own operand. + const string content = "BX\n1 2 3 SomeFutureOp\n5 w\nEX\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Equal(["BX", "w", "EX"], visitor.Operators.Select(o => o.Op)); + var w = visitor.Operators.Single(o => o.Op == "w"); + Assert.Equal(5, ((PdfInteger)w.Operands[0]).Value); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.UnknownOperator); + } + // ── Operand-stack and graphics-state caps ─────────────────────────────────────────────────── [Fact] - public void OperandStackCap_32IsOk_33IsMalformed() + public void OperandStackCap_64IsOk_65IsALimitNotAMalformation() { - // 32 numeric operands, none consumed by a real operator (so this pins the CAP itself, not - // any one operator's own arity): the 32nd push must not itself overflow. - var okContent = string.Join(' ', Enumerable.Repeat("1", 32)); - var overContent = string.Join(' ', Enumerable.Repeat("1", 33)); + // 64 numeric operands, none consumed by an operator (so this pins the CAP itself, not any + // one operator's own arity): the 64th push must not itself overflow. 64, not 32: + // Table 73's scn operator can legally take 33+ operands for a DeviceN space with many + // colourants (see the cap's own comment in ContentInterpreter), so 32 rejected a legal call. + var okContent = string.Join(' ', Enumerable.Repeat("1", 64)); + var overContent = string.Join(' ', Enumerable.Repeat("1", 65)); var (okReader, _, _) = Run(BuildPageDoc(okContent)); - Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); var (overReader, _, _) = Run(BuildPageDoc(overContent)); - Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + // This reader's own ceiling, not a producer-side malformation: ContentLimitExceeded (#402), + // not OperandStackMalformed. + Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); } [Fact] - public void TjArrayCap_8192IsOk_8193IsMalformed() + public void TjArrayCap_8192IsOk_8193IsALimitNotAMalformation() { var okArray = "[" + string.Concat(Enumerable.Repeat("0 ", 8192)) + "] TJ\n"; var overArray = "[" + string.Concat(Enumerable.Repeat("0 ", 8193)) + "] TJ\n"; var (okReader, _, okVisitor) = Run(BuildPageDoc(okArray)); - Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(okReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); Assert.Single(okVisitor.Operators, o => o.Op == "TJ"); var (overReader, _, overVisitor) = Run(BuildPageDoc(overArray)); - Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.Contains(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(overReader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); Assert.DoesNotContain(overVisitor.Operators, o => o.Op == "TJ"); } + [Fact] + public void TjOperand_notAnArray_reportsOperandStackMalformed_notALimit() + { + // A wrong TYPE (a producer-side malformation) must stay OperandStackMalformed even though + // the ELEMENT-COUNT cap right beside it in the same switch case moved to ContentLimitExceeded. + var (reader, _, visitor) = Run(BuildPageDoc("5 TJ\n1 w\n")); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(visitor.Operators, o => o.Op == "TJ"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + [Fact] public void UnbalancedQ_isIgnoredWithADiagnostic() { @@ -488,6 +525,154 @@ public void Form_matrixAndBBox_areHandedToTheVisitor() Assert.Equal(11, begin.ObjectNumber); } + [Fact] + public void Form_matrixAndBBox_resolveThroughAnIndirectReference() + { + // §7.3.10 lets any dictionary entry be given as an indirect reference; Table 93 gives + // /Matrix and /BBox no direct-only restriction, so both must resolve before the shape + // check ContentInterpreter runs against them (#402). + var doc = BuildPageDoc( + "/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox 12 0 R /Matrix 13 0 R >>", []), + new Obj(12, "[1 2 3 4]"), + new Obj(13, "[2 0 0 2 5 6]")); + + var (_, _, visitor) = Run(doc); + + var begin = Assert.Single(visitor.FormBegins); + Assert.Equal(new Matrix(2, 0, 0, 2, 5, 6), begin.Matrix); + Assert.NotNull(begin.BBox); + Assert.Equal(1, begin.BBox!.LlX); + Assert.Equal(4, begin.BBox.UrY); + } + + // ── Do brackets a form's content in an implicit q/Q (ISO 32000-2 §8.10.1) ─────────────────── + + [Fact] + public void Do_onAFormThatChangesTheCtm_doesNotLeakTheChangeIntoTheInvoker() + { + var doc = BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", + "3 0 0 3 0 0 cm"u8.ToArray())); + + var interpreter = RunAndKeepInterpreter(doc, out var reader); + + Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + } + + [Fact] + public void Do_onAFormWithUnbalancedQ_doesNotPopThePagesOwnSave() + { + var doc = BuildPageDoc( + "q\n/F1 Do\nQ\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "Q Q Q"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + // The form's three stray 'Q's each report against the form's own object number, 11, and + // the page's own 'q'/'Q' pairing (opened before Do, closed after it) stays untouched: the + // page's own Q must not itself be reported as unbalanced. + var malformed = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed).ToList(); + Assert.NotEmpty(malformed); + Assert.All(malformed, d => Assert.Equal(11, d.ObjectNumber)); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void Do_onAQOnlyForm_drawn70Times_producesNoGraphicsStateDepthDiagnostic() + { + var pageContent = string.Concat(Enumerable.Repeat("/F1 Do\n", 70)) + "1 w\n"; + var doc = BuildPageDoc( + pageContent, "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "q\n"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void Do_onAFormWithAStrayEmcAndUnbalancedBX_leavesThePageStateBalanced() + { + var doc = BuildPageDoc( + "BDC\n/F1 Do\nEMC\nZorkAfterDo\n", + "<< /XObject << /F1 12 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "EMC"u8.ToArray()), + new Obj( + 12, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] " + + "/Resources << /XObject << /F1 11 0 R >> >> >>", + "/Span << >> BDC\n/F1 Do\n"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + // The innermost form's stray EMC is reported against ITS OWN object number (11), and the + // page's own BDC (opened before Do, closed by the page's own EMC after it) still balances: + // no diagnostic against the page (null object number) for an unmatched EMC. + var emcReports = reader.Diagnostics + .Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed + && d.Message.Contains("EMC", StringComparison.Ordinal)) + .ToList(); + Assert.Contains(emcReports, d => d.ObjectNumber == 11); + Assert.DoesNotContain(emcReports, d => d.ObjectNumber is null); + + // The middle form's unbalanced BX leaves the PAGE outside a compatibility section once Do + // returns (the floor resets _bxDepth back to 0, not to "still inside BX"), so the page's + // own unknown operator right after Do is reported rather than silently swallowed. + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.UnknownOperator + && d.Message.Contains("ZorkAfterDo", StringComparison.Ordinal)); + } + + // ── The 64 MiB content budget covers Form XObject invocations too (#402) ─────────────────── + + [Fact] + public void FormDrawnRepeatedly_countsTowardTheSameContentBudgetAsThePage() + { + // A form whose own decoded content is ~20 MiB, drawn 4 times: well past the combined + // 64 MiB page-and-forms budget on the 4th invocation, without any single form or the + // page's own /Contents alone being anywhere near the cap. + var unit = "0 0 1 1 re\n"u8.ToArray(); + var repeatsPerForm = (20 * 1024 * 1024) / unit.Length; + var formBody = new byte[unit.Length * repeatsPerForm]; + for (var i = 0; i < repeatsPerForm; i++) + unit.CopyTo(formBody, i * unit.Length); + + var pageContent = string.Concat(Enumerable.Repeat("/F1 Do\n", 4)) + "1 w\n"; + var doc = BuildPageDoc( + pageContent, "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", formBody)); + + var (reader, _, visitor) = Run(doc); + + var tooLarge = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge).ToList(); + Assert.Single(tooLarge); + + // Expected operator count computed from the byte budget, not read off the output: the + // whole 64 MiB budget divided by one form's own byte length gives how many WHOLE forms + // fit, each contributing repeatsPerForm 're' operators (the page's own /Contents is a few + // bytes and negligible against a 64 MiB budget). + var wholeFormsThatFit = (int)(ContentInterpreterBudget.MaxContentBytes / formBody.Length); + var reOps = visitor.Operators.Where(o => o.Op == "re").ToList(); + Assert.True(reOps.Count >= wholeFormsThatFit * repeatsPerForm); + Assert.True(reOps.Count < 4 * repeatsPerForm); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // the page's own content after the last Do still ran + } + + // Mirrors ContentInterpreter's own private MaxContentBytes so the test above can compute an + // expected operator count from the budget rather than reading it off the interpreter's output. + private static class ContentInterpreterBudget + { + internal const long MaxContentBytes = 64L * 1024 * 1024; + } + // ── Inline images ──────────────────────────────────────────────────────────────────────────── [Fact] @@ -537,6 +722,39 @@ public void L_pastTheEnd_reportsMalformed_andRecoversViaTheEiScan() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void NegativeL_reportsMalformed_andRecoversViaTheEiScan() + { + byte[] data = "ABCDEFGH"u8.ToArray(); + var content = "BI /F /AHx /L -1 ID " + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Theory] + [InlineData("4294967298")] // outside int's range + [InlineData("2.9")] // a PdfReal, not a PdfInteger (Table 87 types /W as integer) + public void InvalidW_takesTheEiScanPath_withTheMissingOrInvalidReport(string invalidWidth) + { + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /W {invalidWidth} /H 2 /BPC 8 /CS /G ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed + && d.Message.Contains("missing, or carries an invalid", StringComparison.Ordinal)); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + } + [Fact] public void FilteredAHx_usesTheEiScan() { @@ -601,6 +819,218 @@ public void DctInlineImage_withAFalseEiFollowedByBinaryNoise_isSkippedByTheScan( Assert.Equal("Q", Assert.Single(visitor.Operators).Op); } + // ── The bounded resync probe (#402): linear-time scan, less strict, still rejects noise ──── + + [Fact] + public void ManyFalseEiCandidates_doesNotThrow_andReportsADiagnostic() + { + // Reproduces the pre-#402 quadratic blowup: N repeats of " EI (" (a false candidate + // followed by the start of a literal string) after a DCT-filtered image's own data. The + // unbounded probe this used to run from EVERY candidate re-lexed all the way to the end of + // the buffer looking for the string's own closing ')', which never comes; O(N) work per + // candidate made the whole scan O(N^2) (measured pre-fix: 100 KB content, 18 s; 400 KB, + // 305 s, from a 1.2 KB Flate-compressed source). The bounded probe caps the per-candidate + // cost, so this reads (with `dotnet test`'s own default timeout as the actual regression + // guard, per this repo's no-wall-clock-assertion rule) rather than hanging. + const int n = 20_000; + var falseCandidate = " EI ("u8.ToArray(); + var noise = new byte[falseCandidate.Length * n]; + for (var i = 0; i < n; i++) + falseCandidate.CopyTo(noise, i * falseCandidate.Length); + + var content = "BI /F /DCT ID "u8.ToArray() + .Concat([0xFF, 0xD8, 0xFF]) + .Concat(noise) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, _) = Run(doc); + + Assert.Contains( + reader.Diagnostics, + d => d.Code is PdfReaderDiagnosticCode.ContentStreamLexError + or PdfReaderDiagnosticCode.InlineImageMalformed); + } + + [Fact] + public void FalseEiCandidate_followedByAnUnknownButPrintableOperator_isAccepted() + { + // 'PS' is an unrecognised-but-printable operator name (§7.8.2 tolerates it outside + // BX/EX); the probe accepts it as a plausible operator rather than rejecting the whole + // candidate on account of it, so the false EI right after the image data still resolves + // correctly, and 'PS' itself reaches the ordinary unknown-operator path once the + // interpreter's own main loop gets there. + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "q\nBI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI PS Q 1 w\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.UnknownOperator + && d.Message.Contains("'PS'", StringComparison.Ordinal)); + Assert.Equal(["q", "Q", "w"], visitor.Operators.Select(o => o.Op)); + } + + [Fact] + public void FalseEiCandidate_followedByABxExSection_isAccepted() + { + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "q\nBI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI BX { } EX Q 1 w\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void TwoConsecutiveInlineImages_theSecondWithAShortDictionary_bothDelimitCorrectly() + { + byte[] data1 = [0x10, 0x20, 0x30, 0x40]; // no 'E'/'I' bytes: unambiguous EI scan + byte[] data2 = [0x99]; // /H is absent, so this one falls to the EI scan too + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data1).Concat(" EI\n"u8.ToArray()) + .Concat("BI /IM true /W 8 ID "u8.ToArray()) + .Concat(data2).Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.Equal(2, visitor.InlineImages.Count); + Assert.True(visitor.InlineImages[0].Data.AsSpan().SequenceEqual(data1)); + Assert.True(visitor.InlineImages[1].Data.AsSpan().SequenceEqual(data2)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindow_isAccepted() + { + // The probe's own bounded lexer runs off its 128-byte window mid-string here (the literal + // is 200 bytes, longer than the window), the inconclusive case this reader accepts rather + // than rejects: PdfLexer's string readers advance Position as they read, so reaching the + // window's own end here means the window ran out, not that the bytes were malformed. + var longLiteral = new string('X', 200); + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(Encoding.ASCII.GetBytes($" EI ({longLiteral}) Tj\nQ\n")) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + var tj = Assert.Single(visitor.Operators, o => o.Op == "Tj"); + var str = (PdfLiteralString)tj.Operands[0]; + Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); + } + + // ── A failed tier-a/tier-b end falls back to the scan instead of losing the stream (#402) ── + + [Fact] + public void L_oneShort_reportsMalformed_andRecoversViaTheEiScan() + { + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /F /AHx /L {data.Length - 1} ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void IdFollowedByCrLf_unfilteredImage_treatsBothBytesAsOneSeparator() + { + var content = "BI /W 2 /H 2 /BPC 8 /CS /G ID\r\nABCD EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("ABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void IdFollowedByCrLf_withExplicitL_treatsBothBytesAsOneSeparator() + { + var content = "BI /F /AHx /L 4 ID\r\nABCD EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("ABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void W_zero_reportsMalformed_andRecoversViaTheEiScan() + { + var content = "BI /W 0 /H 2 /BPC 8 /CS /G ID ABCD EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("ABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + // ── Table 92 abbreviations inside a /CS array (#402) ──────────────────────────────────────── + + [Fact] + public void CsArray_tableNinetyTwoAbbreviationsExpand_andIndexedComponentCountDrivesTierB() + { + // §8.9.7's one composite inline colour space: [/I baseSpace hival lookup]. /I and /RGB + // (the array's first two elements) are Table 92 abbreviations; 1 and the hex string are + // left alone. An array whose first element is /Indexed counts one component regardless of + // the base space (§8.6.6.3: an Indexed sample is always a single index value), so tier b + // computes the data length without resolving /DeviceRGB's own component count at all. A + // literal " EI x" is embedded inside the tier-b-computed 5-byte data window: if this + // regressed to the EI scan (tier c) instead, that decoy would be mistaken for the + // terminator, and the image would come out 1 byte long instead of 5. + var content = "BI /W 5 /H 1 /BPC 8 /CS [/I /RGB 1 <000000FFFFFF>] ID EI x EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + var img = Assert.Single(visitor.InlineImages); + var csArray = (PdfArray)img.Dict.Get(PdfName.ColorSpace)!; + Assert.Equal(4, csArray.Count); + Assert.Equal("Indexed", ((PdfName)csArray[0]).Value); + Assert.Equal("DeviceRGB", ((PdfName)csArray[1]).Value); + Assert.Equal(1, ((PdfInteger)csArray[2]).Value); + Assert.True(img.Data.AsSpan().SequenceEqual(" EI x"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + [Fact] public void AbbreviationExpansion_isPinned() { @@ -761,6 +1191,96 @@ public void ContentExceeding64MiB_reportsTooLarge_andKeepsTheOperatorsBeforeTheC }); } + [Fact] + public void ContentExceeding64MiB_acrossTwoStreams_appendsTheSecondStreamAfterTheFirst_notOverIt() + { + // A regression pin for a wrong Array.Copy overload (#402): copying the truncated tail + // chunk to index 0 of the capped buffer, rather than to `written`, silently overwrote the + // FIRST stream's own already-copied bytes once a second stream needed truncating. The + // single-stream cap test above cannot catch this: it never has anything already written + // when the truncation branch runs (`written == 0` there), so the wrong overload and the + // right one produce identical output in that one-stream case. + var unitA = "0 0 1 1 re\n"u8.ToArray(); + var repeatsA = 40 * 1024 * 1024 / unitA.Length; + var rawA = new byte[unitA.Length * repeatsA]; + for (var i = 0; i < repeatsA; i++) + unitA.CopyTo(rawA, i * unitA.Length); + + var unitB = "1 w\n"u8.ToArray(); + var repeatsB = 30 * 1024 * 1024 / unitB.Length; + var rawB = new byte[unitB.Length * repeatsB]; + for (var i = 0; i < repeatsB; i++) + unitB.CopyTo(rawB, i * unitB.Length); + + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [4 0 R 5 0 R] >>"), + new Obj(4, "<< /Filter /FlateDecode >>", Flate(rawA)), + new Obj(5, "<< /Filter /FlateDecode >>", Flate(rawB))); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + + var reOps = visitor.Operators.TakeWhile(o => o.Op == "re").ToList(); + Assert.Equal(repeatsA, reOps.Count); + Assert.All(reOps, o => Assert.Equal(4, o.Operands.Count)); + + var rest = visitor.Operators.Skip(reOps.Count).ToList(); + Assert.NotEmpty(rest); + Assert.True(rest.Count < repeatsB); + Assert.All(rest, o => + { + Assert.Equal("w", o.Op); + Assert.Single(o.Operands); + }); + } + + // ── ContentStreamTooLarge and FormXObjectBudgetExceeded use ReportRetained (#402) ─────────── + + [Fact] + public void ContentStreamTooLarge_isRetainedEvenOnceMaxDiagnosticsIsAlreadySpent() + { + // Two pages: page 0 spends the whole cap (MaxDiagnostics = 1) on an ordinary UnknownOperator + // report; page 1's /Contents then exceeds the 64 MiB budget. Without ReportRetained, that + // second report would be silently dropped in favour of the DiagnosticsSuppressed sentinel, + // since the sink's ordinary cap (shared across the whole reader, not per page) is already + // full by the time page 1 is interpreted (#398 set this rule for PageTreeWalker; this is + // the content-interpreter counterpart). + var unit = "0 0 1 1 re\n"u8.ToArray(); + var repeats = (67 * 1024 * 1024 / unit.Length) + 200_000; + var raw = new byte[unit.Length * repeats]; + for (var i = 0; i < repeats; i++) + unit.CopyTo(raw, i * unit.Length); + + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< >>", "Zork\n"u8.ToArray()), + new Obj(6, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 7 0 R >>"), + new Obj(7, "<< /Filter /FlateDecode >>", Flate(raw))); + + var reader = PdfReader.Open(doc, new PdfReaderOptions { MaxDiagnostics = 1 }); + var interpreter = new ContentInterpreter(reader); + + interpreter.Run(reader.GetPage(0), new RecordingVisitor()); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.UnknownOperator); + + interpreter.Run(reader.GetPage(1), new RecordingVisitor()); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge); + } + // ── Fuzzing ────────────────────────────────────────────────────────────────────────────────── private static readonly byte[] FuzzSeed = BuildPdf( @@ -826,23 +1346,43 @@ public void Fuzz_run_neverThrowsOutsideTheDeclaredVocabulary_andAlwaysTerminates private static void AssertInterpreterIsRobust(byte[] bytes) { + // Two separate try blocks, not one covering Open/GetPage/Run together: ContentInterpreter's + // own class doc promises InvalidDataException never escapes Run (UnsupportedPdfFeatureException + // is the one exception allowed to). One try block covering all three would make a Run-time + // InvalidDataException indistinguishable from an open-time one, silently accepting a + // regression in that promise as just another "acceptable outcome". var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + PdfDocumentReader? reader = null; + PdfReadPage? page = null; try { - using var reader = PdfReader.Open(bytes, new PdfReaderOptions + reader = PdfReader.Open(bytes, new PdfReaderOptions { MaxDecodedStreamBytes = ReaderLimits.MinMaxDecodedBytes, }); - if (reader.PageCount == 0) - return; - var interpreter = new ContentInterpreter(reader); - interpreter.Run(reader.GetPage(0), new RecordingVisitor()); + if (reader.PageCount != 0) + page = reader.GetPage(0); } catch (Exception ex) when (ex is InvalidDataException or UnsupportedPdfFeatureException or PdfPasswordException) { // Acceptable outcome; see ParserFuzzTests' own class doc for the same policy this // interpreter follows: a robustness oracle, not a conformance one. } + + if (page is not null) + { + try + { + var interpreter = new ContentInterpreter(reader!); + interpreter.Run(page, new RecordingVisitor()); + } + catch (UnsupportedPdfFeatureException) + { + // The one exception Run's own class doc allows to propagate. + } + } + + reader?.Dispose(); Assert.True( stopwatch.Elapsed <= TimeSpan.FromSeconds(4), $"content interpretation took {stopwatch.Elapsed} on a {bytes.Length}-byte input."); diff --git a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs index a79cec5e..7e769858 100644 --- a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs @@ -81,6 +81,40 @@ public void ContentStreamMode_stillLexesADoubleGreaterThanAsDictEnd_notTwoLoneKe Assert.Equal(2, lexer.Position); } + [Fact] + public void ContentStreamMode_lexesTwoLoneGreaterThansSplitByWhitespace_asTwoOneByteKeywords() + { + // ">>" is one DictEnd token (above); "> >", with whitespace between the two bytes, is two + // separate one-byte Keyword tokens instead. + var lexer = new PdfLexer(Encoding.ASCII.GetBytes("> >"), contentStreamMode: true); + + var first = lexer.NextToken(); + var second = lexer.NextToken(); + var third = lexer.NextToken(); + + Assert.Equal(TokenKind.Keyword, first.Kind); + Assert.Equal(1, first.Raw.Length); + Assert.Equal(TokenKind.Keyword, second.Kind); + Assert.Equal(1, second.Raw.Length); + Assert.Equal(TokenKind.EndOfInput, third.Kind); + } + + [Fact] + public void ContentStreamMode_seekingPastAOneByteKeyword_lexesTheFollowingTokenNormally() + { + var lexer = new PdfLexer(Encoding.ASCII.GetBytes("{Tj"), contentStreamMode: true); + + var brace = lexer.NextToken(); + Assert.Equal(TokenKind.Keyword, brace.Kind); + Assert.Equal(1, lexer.Position); + + lexer.Seek(1); + var next = lexer.NextToken(); + + Assert.Equal(TokenKind.Keyword, next.Kind); + Assert.Equal("Tj", Encoding.ASCII.GetString(next.Raw.Span)); + } + [Fact] public void ContentStreamMode_insideACompatibilitySection_lexesAWholeBxToExSequenceWithoutThrowing() { diff --git a/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs b/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs index ca62264d..bba71d2c 100644 --- a/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfReaderDiagnosticCodeTests.cs @@ -45,6 +45,7 @@ public sealed class PdfReaderDiagnosticCodeTests [PdfReaderDiagnosticCode.ResourceMissing] = 3, [PdfReaderDiagnosticCode.InlineImageMalformed] = 3, [PdfReaderDiagnosticCode.ContentStreamTooLarge] = 3, + [PdfReaderDiagnosticCode.ContentLimitExceeded] = 3, [PdfReaderDiagnosticCode.DiagnosticsSuppressed] = 9, }; @@ -144,7 +145,7 @@ private static PdfReaderDiagnostic MakeDiagnostic(PdfReaderDiagnosticCode code) [PdfReaderDiagnosticCode.PageTreeNodeMalformed] = (206, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.PageTreeNodeLimitExceeded] = (207, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.ContentStreamLexError] = (300, PdfReaderDiagnosticSeverity.Warning), - [PdfReaderDiagnosticCode.UnknownOperator] = (301, PdfReaderDiagnosticSeverity.Info), + [PdfReaderDiagnosticCode.UnknownOperator] = (301, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.OperandStackMalformed] = (302, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.FormXObjectDepthExceeded] = (303, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.FormXObjectCycle] = (304, PdfReaderDiagnosticSeverity.Warning), @@ -152,6 +153,7 @@ private static PdfReaderDiagnostic MakeDiagnostic(PdfReaderDiagnosticCode code) [PdfReaderDiagnosticCode.ResourceMissing] = (306, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.InlineImageMalformed] = (307, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.ContentStreamTooLarge] = (308, PdfReaderDiagnosticSeverity.Warning), + [PdfReaderDiagnosticCode.ContentLimitExceeded] = (309, PdfReaderDiagnosticSeverity.Warning), [PdfReaderDiagnosticCode.DiagnosticsSuppressed] = (900, PdfReaderDiagnosticSeverity.Warning), }; From 298693eaf701b5103ea973d21862df3f0d157f66 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 16:22:02 +0200 Subject: [PATCH 04/14] Tidy round-1 prose in the CHANGELOG and the 302 code doc The CHANGELOG bullet said the content budget "now covers" forms, but the interpreter is new in this release, so there is no earlier state for "now" to contrast with; it describes the budget as one per page instead. The OperandStackMalformed doc had a comma splice. --- CHANGELOG.md | 8 ++++---- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d95858f..a9f482bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,10 +80,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). correct), `OperandStackMalformed` for a producer-side malformation, `ContentLimitExceeded` for this reader's own processing ceilings instead (an operand-count, `TJ`-array, `q`-depth, or marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, - `InlineImageMalformed`, and `ContentStreamTooLarge` (the same 64 MiB decoded-content budget now - covers every Form XObject a page draws, not only its own `/Contents`, and every invocation of a - form counts again, since the interpretation cost this bounds scales with how many times a form - is drawn). (#98) + `InlineImageMalformed`, and `ContentStreamTooLarge` (one 64 MiB decoded-content budget per page, + shared between its own `/Contents` and every Form XObject it draws; each invocation of a form + counts again, since the interpretation cost this bounds scales with how many times a form is + drawn). (#98) ### Changed diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 541e09a0..79d8b336 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -381,8 +381,8 @@ public enum PdfReaderDiagnosticCode /// A content stream's operand-stack discipline broke down in one of several ways this /// interpreter groups under one code rather than one each, since every case has the same /// remedy: drop the offending operator (or, for an unbalanced Q/EMC, drop the - /// pop) and keep interpreting. Every case here is a PRODUCER-side malformation, the document - /// itself is wrong, not merely bigger than this reader is willing to process; see + /// pop) and keep interpreting. Every case here is a producer-side malformation (the document + /// itself is wrong, not merely bigger than this reader is willing to process); see /// for the four cases that are this reader's own processing /// ceiling instead. Covers: a number token that does not parse, is not finite, or carries a /// second sign character (--5, -+5: §7.3.3 allows only "an optional sign"), a From b892675685e5f9f14dede9fc23d1ae8180110ea6 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 19:37:25 +0200 Subject: [PATCH 05/14] Fix content-interpreter review findings from PR #402 round 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Charge the per-Run content-decode budget as each /Contents element or Form XObject invocation decodes, not only after every element already did: a /Contents array naming the same oversized stream many times no longer holds every decode in memory before the cap gets a chance to stop it, and a Form XObject invocation skips its own decode entirely once the budget is already spent instead of decoding and discarding. Give an over-cap 'q'/'BMC' push a credit a later 'Q'/'EMC' spends before reporting an unbalanced pop, so a producer that legitimately nests past this reader's own ceiling and balances every nest is not also accused of an unbalanced pop. Clear the operand stack on every unrecognised operator, not only inside a BX/EX section, per §7.8.2. Save and restore the invoker's text matrices around a Form XObject invocation, since the form's own content can open an independent text object regardless of the invoker's state; report when 'Do' itself occurs inside a text object. Concatenate a form's own /Matrix into the CTM before interpreting its content (§8.10.1 b), so a visitor can read the composed value. Check the Form XObject cycle guard before the depth cap so the more informative code wins, and report a /XObject entry that resolves but is not a usable Form or Image stream. Replace the inline-image resync probe's single-window inconclusive accept, which could mask a malformed stream once a token ran past the first window, with a second bounded retry at a much larger window before rejecting. Skip whitespace after ID for every filter named anywhere in a /Filter array when ASCIIHexDecode or ASCII85Decode is present, not only the one byte every other filter gets. Delay the CR-as-EOL malformed report until after the one-byte-earlier retry it exists for has also failed. Report a non-integer /L and reject a /BPC outside {1, 2, 4, 8, 16} the same way an invalid /W or /H already is. Fix two fabricated ISO 32000-2 quotations (Matrix's own §8.3.4 cm citation, TextState's §9.4.1 citation) and several wrong table/clause references, and correct doc comments that no longer matched what the interpreter does. --- CHANGELOG.md | 8 +- .../Content/ContentInterpreter.cs | 511 +++++++++++---- .../Content/ContentOperators.cs | 45 +- .../Content/IContentVisitor.cs | 19 +- src/VellumPdf.Reader/Content/Matrix.cs | 5 +- src/VellumPdf.Reader/Content/TextState.cs | 11 +- src/VellumPdf.Reader/DiagnosticSink.cs | 10 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 39 +- .../ContentInterpreterTests.cs | 586 +++++++++++++++++- .../PdfLexerContentModeTests.cs | 22 + 10 files changed, 1101 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f482bc..43035454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,7 +83,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). `InlineImageMalformed`, and `ContentStreamTooLarge` (one 64 MiB decoded-content budget per page, shared between its own `/Contents` and every Form XObject it draws; each invocation of a form counts again, since the interpretation cost this bounds scales with how many times a form is - drawn). (#98) + drawn), charged against the budget as each `/Contents` element or form invocation decodes rather + than only once all of them already have, so a `/Contents` array naming the same oversized stream + many times cannot hold every decode in memory before the cap gets a chance to stop it. `Do` on a + Form XObject also concatenates the form's own `/Matrix` into the graphics state's CTM (§8.10.1 b) + before interpreting its content, so a caller reading the CTM from inside the form's own content + sees the composed value, not the invoker's own CTM with the form's matrix left for it to apply + separately. (#98) ### Changed diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index a0b33f80..36fb709b 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -36,12 +36,12 @@ internal sealed class ContentInterpreter { // ISO 32000-2 §7.8.2 gives an operator's own operands no declared bound; this reader's own // ceiling against a hostile or corrupted stream that never emits an operator at all. 64, not - // 32: Annex C.1 Table C.1 (informative) records 32 as the DeviceN colourant-count limit an - // earlier PDF version recommended, but §8.6.6.5 itself allows "an arbitrary number" of - // colourants, and Table 73's scn operator takes one numeric component per colourant plus an - // optional trailing pattern name, so a legal scn call against a DeviceN space with 32 or more - // colourants needs 33 or more operands; 32 as this reader's own ceiling would reject a legal - // call, not just a hostile one. + // 32: Annex C.2 Table C.1 (informative; Annex C.1 is only the annex's own general preamble) + // records 32 as the DeviceN colourant-count limit an earlier PDF version recommended, but + // §8.6.6.5 itself allows "an arbitrary number" of colourants, and Table 73's scn operator + // takes one numeric component per colourant plus an optional trailing pattern name, so a + // legal scn call against a DeviceN space with 32 or more colourants needs 33 or more operands; + // 32 as this reader's own ceiling would reject a legal call, not just a hostile one. private const int MaxOperandsPerOperator = 64; // §9.4.3's own TJ array holds a mix of strings and numeric adjustments; this reader's own @@ -74,7 +74,17 @@ internal sealed class ContentInterpreter private const int ProbeWindowBytes = 128; private const int ProbeTokens = 8; + // The second, larger window LooksLikeResyncPoint re-probes with when a token runs off the + // clipped ProbeWindowBytes window: a legitimate token following a false 'EI' candidate (a long + // literal string, say) can run well past 128 bytes without being image data at all, so a + // candidate is rejected only once a token also runs off THIS window (#402 round 2; the + // previous single-window probe treated running off a clipped window as inconclusive and + // accepted the candidate either way, which let a sufficiently long unterminated token past the + // window mask a malformed stream). + private const int ExtendedProbeWindowBytes = 4096; + private static readonly PdfName XObjectSubtypeForm = new("Form"); + private static readonly PdfName XObjectSubtypeImage = new("Image"); private static readonly PdfName ImageMaskKey = new("ImageMask"); private static readonly PdfName WidthKey = new("Width"); private static readonly PdfName HeightKey = new("Height"); @@ -95,12 +105,24 @@ internal sealed class ContentInterpreter private bool _operandOverflow; private int _bxDepth; private int _markedContentDepth; + private bool _inTextObject; private readonly HashSet _openForms = []; private int _formDepth; private int _formInvocations; private long _contentBytesRemaining; private ReadOnlyMemory _currentBuffer; + // Pushes PushGraphicsState/PushMarkedContent dropped for being over MaxGraphicsStateDepth or + // MaxMarkedContentDepth: a matching 'Q'/'EMC' consumes one of these before it may report an + // unbalanced pop (see PopGraphicsState/PopMarkedContent), so a producer that legitimately + // nests past this reader's own ceiling and then balances every one of those nests does not + // ALSO get accused of an unbalanced pop purely because this reader declined to push the state + // it was asked to (#402 round 2). Saved and restored across a Form XObject invocation + // (HandleDo) the same way _gsFloor/_markedContentFloor are, so a form's own credit and the + // invoker's own credit can never be consumed across that boundary. + private int _ignoredGsPushes; + private int _ignoredMcPushes; + // The graphics-state stack depth, marked-content depth, and BX/EX depth a 'Q', 'EMC', or 'EX' // may not pop or decrement below. All three are 0 for the page's own top-level content and set // to the invoker's own depth for the duration of one Form XObject's content (see HandleDo): @@ -119,6 +141,16 @@ internal sealed class ContentInterpreter /// . internal TextState TextState => _textState; + /// + /// Test-only visibility into how many content streams (the page's own /Contents + /// elements, and Form XObject invocations) this decoded, so a test + /// can pin how early the per-Run content budget stops further decoding without asserting on + /// wall-clock time or process memory (#402 round 2: peak heap scales with how many oversized + /// streams get decoded before the budget is charged, not with the operator count those streams + /// produce). + /// + internal int ContentStreamsDecoded { get; private set; } + /// Creates an interpreter that resolves resources and streams through /// , under that reader's own . internal ContentInterpreter(PdfDocumentReader reader) @@ -146,6 +178,7 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _operandOverflow = false; _bxDepth = 0; _markedContentDepth = 0; + _inTextObject = false; _openForms.Clear(); _formDepth = 0; _formInvocations = 0; @@ -153,6 +186,9 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _gsFloor = 0; _markedContentFloor = 0; _bxFloor = 0; + _ignoredGsPushes = 0; + _ignoredMcPushes = 0; + ContentStreamsDecoded = 0; var diagnostics = _reader.CreateContentDiagnosticScope(); var pageIndex = page.Index; @@ -196,8 +232,26 @@ private ReadOnlyMemory BuildPageContentBuffer( var contributingStreams = 0; int? soleObjectNumberLocal = null; + // Tracks decoded bytes charged so far in THIS loop, separate from _contentBytesRemaining + // itself (which Concatenate below still consumes its own share of, against the full + // per-Run budget, once the loop finishes): stopping further decodes here is what keeps a + // /Contents array naming the same oversized stream many times from decoding, and holding + // in memory, every one of them before Concatenate ever gets a chance to truncate. Before + // this fix, peak heap scaled with element count x MaxDecodedStreamBytes, since every + // element decoded in full regardless of how far over budget earlier ones already were: one + // 20 MiB-decoding stream referenced 128 times from a 103 KB file measured at 4662 MiB peak + // managed heap while interpreting exactly the same operators as a four-element array would + // (#402 round 2). A single element is still bounded only by MaxDecodedStreamBytes, not by + // this budget: GetDecodedStreamData itself throws before this method ever sees a decode + // larger than that limit. + var decodedSoFar = 0L; + var budgetSpent = false; + void AddElement(PdfObject element) { + if (budgetSpent) + return; + if (element is not PdfIndirectReference elementRef) { diagnostics.Report( @@ -245,9 +299,22 @@ void AddElement(PdfObject element) return; } + ContentStreamsDecoded++; chunks.Add(decoded); contributingStreams++; soleObjectNumberLocal = contributingStreams == 1 ? stream.ObjectNumber : null; + + decodedSoFar += decoded.Length + 1L; // the separator byte Concatenate also charges below + if (decodedSoFar > _contentBytesRemaining) + { + budgetSpent = true; + diagnostics.ReportRetained( + PdfReaderDiagnosticCode.ContentStreamTooLarge, + $"The page's /Contents exceeded the {MaxContentBytes / (1024 * 1024)} MiB " + + "decoded-size cap shared with every Form XObject it draws; interpretation " + + "stopped there.", + pageIndex: pageIndex); + } } if (raw is PdfArray directArray) @@ -578,24 +645,23 @@ private void HandleOperator( { if (!ContentOperators.IsKnown(name)) { - // Two different rules apply here depending on _bxDepth. Inside a BX/EX compatibility - // section, Table 33 is explicit: "Unrecognised operators (along with their operands) - // shall be ignored without error until the balancing EX operator is encountered", so - // the operand stack IS cleared, and nothing is reported. Outside one, §7.8.2 says "an - // error shall occur"; this reader instead notifies and continues, and deliberately does - // NOT clear the operand stack, a leniency of this reader's own rather than anything - // Table 33 asks for: the most common way an unrecognised keyword appears in an - // otherwise-conforming stream is a stray "R" left over from indirect-reference syntax - // that §7.8.2 forbids in content streams at all ("Indirect objects and object - // references shall not be permitted"), and the operands that precede it usually belong - // to whatever REAL operator follows, not to "R" itself. The sink's dedupe key is - // (code, object, page), so only the first unknown name on a page is recorded; a second - // distinct name on the same page is dropped by the sink. - if (_bxDepth > 0) - { - ClearOperands(); - } - else + // Reported only outside a BX/EX compatibility section. Inside one, Table 33 is + // explicit: "Unrecognised operators (along with their operands) shall be ignored + // without error until the balancing EX operator is encountered." Outside one, §7.8.2 + // says "an error shall occur"; this reader instead notifies and continues, the same + // notify-and-continue choice every other diagnostic in this channel makes. The sink's + // dedupe key is (code, object, page), so only the first unknown name on a page is + // recorded; a second distinct name on the same page is dropped by the sink. + // + // Either way, the operand stack is cleared: §7.8.2 "operands shall not be left over + // when an operator finishes execution" applies to an unrecognised keyword's own + // (no-op) execution just as much as to a recognised one (#402 round 2; an earlier + // version kept the operands outside BX/EX on the theory that a stray "R" left over + // from indirect-reference syntax §7.8.2 forbids in content streams usually belonged to + // whatever REAL operator followed rather than to "R" itself, but that leniency broke a + // differently-shaped input just as easily: '10 20 Zork' ahead of '1 w' silently fed + // the leftover 20 into 'w' as its own operand instead of 1). + if (_bxDepth <= 0) { diagnostics.Report( PdfReaderDiagnosticCode.UnknownOperator, @@ -603,6 +669,7 @@ private void HandleOperator( + "it was ignored.", pageIndex: pageIndex); } + ClearOperands(); return; } @@ -694,6 +761,11 @@ private void HandleOperator( case "BT": _textState.BeginText(); + _inTextObject = true; + break; + + case "ET": + _inTextObject = false; break; case "Tc": @@ -798,6 +870,15 @@ private void PushGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, in { if (_gsStack.Count >= MaxGraphicsStateDepth) { + // The push itself is dropped, but the 'q' it belongs to is still credited toward a + // LATER 'Q': nothing in §7.8.2 or §8.4.4 bounds how deep a legitimate document nests + // 'q'/'Q', so a producer that nests past this reader's own ceiling and then balances + // every one of those nests must not ALSO be accused of an unbalanced 'Q' purely + // because this reader declined to push the state it was asked to (#402 round 2; without + // this credit, 65 balanced 'q'...'Q' pairs reported ContentLimitExceeded AND + // OperandStackMalformed together, and desynchronised GraphicsState.Ctm from the actual + // nesting besides). + _ignoredGsPushes++; diagnostics.Report( PdfReaderDiagnosticCode.ContentLimitExceeded, $"The graphics-state stack exceeded {MaxGraphicsStateDepth} nested 'q' saves; " @@ -813,6 +894,16 @@ private void PopGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, int { if (_gsStack.Count <= _gsFloor) { + // A pop with nothing left on the stack to restore first spends a credit from an earlier + // over-cap push (see PushGraphicsState) before it may report an unbalanced 'Q': only + // once that credit is exhausted is this the "restore with nothing to restore" problem + // the report below describes (#402 round 2). + if (_ignoredGsPushes > 0) + { + _ignoredGsPushes--; + return; + } + // An unbalanced 'q' still open at end of stream is fine (nothing downstream needs the // state restored past the last operator this interpreter saw); an unbalanced 'Q' is the // opposite problem (a restore with nothing to restore), so this one is reported. Inside @@ -832,6 +923,10 @@ private void PushMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, in { if (_markedContentDepth >= MaxMarkedContentDepth) { + // See PushGraphicsState's own remarks: the same over-cap-push credit, tracked + // separately here since marked-content nesting and the graphics-state stack are + // independent depths. + _ignoredMcPushes++; diagnostics.Report( PdfReaderDiagnosticCode.ContentLimitExceeded, $"Marked-content nesting exceeded {MaxMarkedContentDepth} levels; further " @@ -846,6 +941,13 @@ private void PopMarkedContent(StreamContext ctx, DiagnosticSink diagnostics, int { if (_markedContentDepth <= _markedContentFloor) { + // See PopGraphicsState's own remarks: spend an over-cap push credit before reporting. + if (_ignoredMcPushes > 0) + { + _ignoredMcPushes--; + return; + } + diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, "'EMC' with no matching 'BMC'/'BDC'; it was ignored.", @@ -953,6 +1055,24 @@ private void HandleDo( var xobjectNameOperand = _operands.Count == 1 ? _operands[0] : null; EmitAndClear("Do", offset, visitor); + if (_inTextObject) + { + // §8.2 Figure 9 / Table 50: 'Do' is a General Graphics State category operator, not + // one Table 50 lists among the operators legal inside a text object; a producer that + // invokes it there is wrong regardless of what the named XObject turns out to be. The + // recursion below still runs (a text-object violation is not itself a reason to skip + // an otherwise-resolvable Form), but the shared _textState instance a form's own + // content may disturb (BT/Td/ET, unbracketed by any q/Q-style save) is saved and + // restored around that recursion below regardless of this check, so the report here is + // purely informational: nothing downstream depends on _inTextObject to avoid the leak + // (#402 round 2). + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + "'Do' occurred inside a text object (ISO 32000-2 §8.2 Table 50); 'Do' is not one " + + "of the operators a text object's own state permits.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + if (xobjectNameOperand is not PdfName xobjectName) return; @@ -967,14 +1087,49 @@ private void HandleDo( } if (entryRaw is not PdfIndirectReference xobjectRef) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'Do' names '/{xobjectName.Value}', present in the applicable /Resources " + + "/XObject dictionary but not as an indirect reference to a stream.", + ctx.DiagObjectNumber, pageIndex: pageIndex); return; + } var stream = _reader.ResolveStream(xobjectRef); if (stream is null) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'Do' names '/{xobjectName.Value}', but object {xobjectRef.ObjectNumber} does not " + + "resolve to a stream.", + ctx.DiagObjectNumber, pageIndex: pageIndex); return; + } + + if (stream.Dictionary.Get(PdfName.Subtype) is not PdfName subtype) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'Do' names '/{xobjectName.Value}', object {stream.ObjectNumber}, whose " + + "/Subtype is missing or is not a name, so it cannot be used as an XObject.", + stream.ObjectNumber, pageIndex: pageIndex); + return; + } - if (stream.Dictionary.Get(PdfName.Subtype) is not PdfName subtype || !subtype.Equals(XObjectSubtypeForm)) - return; // An Image XObject, or anything else: no recursion; the caller already got Do. + if (subtype.Equals(XObjectSubtypeImage)) + return; // An Image XObject: no recursion; the caller already got Do. + + if (!subtype.Equals(XObjectSubtypeForm)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ResourceMissing, + $"'Do' names '/{xobjectName.Value}', object {stream.ObjectNumber}, whose /Subtype " + + $"'/{subtype.Value}' is neither /Form nor /Image, so it cannot be used as an " + + "XObject.", + stream.ObjectNumber, pageIndex: pageIndex); + return; + } var objectNumber = stream.ObjectNumber; @@ -988,26 +1143,32 @@ private void HandleDo( return; } - if (_formDepth >= _limits.MaxFormXObjectDepth) + // The cycle check runs BEFORE the depth cap: with MaxFormXObjectDepth set low (1, say), a + // self-referencing form would otherwise hit the depth cap first and report + // FormXObjectDepthExceeded, which is technically true but strictly less informative than + // FormXObjectCycle, the code that names the actual reason recursion cannot continue + // (#402 round 2). + if (_openForms.Contains(objectNumber)) { diagnostics.Report( - PdfReaderDiagnosticCode.FormXObjectDepthExceeded, - $"Form XObject recursion exceeded {_limits.MaxFormXObjectDepth} levels; this 'Do' " - + "was not followed.", + PdfReaderDiagnosticCode.FormXObjectCycle, + $"Form XObject {objectNumber} invokes itself, directly or through a chain of nested " + + "'Do' operators; the recursive invocation was skipped.", objectNumber, pageIndex: pageIndex); return; } - if (!_openForms.Add(objectNumber)) + if (_formDepth >= _limits.MaxFormXObjectDepth) { diagnostics.Report( - PdfReaderDiagnosticCode.FormXObjectCycle, - $"Form XObject {objectNumber} invokes itself, directly or through a chain of nested " - + "'Do' operators; the recursive invocation was skipped.", + PdfReaderDiagnosticCode.FormXObjectDepthExceeded, + $"Form XObject recursion exceeded {_limits.MaxFormXObjectDepth} levels; this 'Do' " + + "was not followed.", objectNumber, pageIndex: pageIndex); return; } + _openForms.Add(objectNumber); _formInvocations++; _formDepth++; try @@ -1022,47 +1183,55 @@ private void HandleDo( visitor.OnFormBegin(formDict, matrix, bbox, objectNumber, offset); try { - byte[]? decoded; - try + // Checked BEFORE decoding, not after: decoding this invocation's content only to + // discard it once the budget already stood at zero still pays the full decode cost + // (allocation, filter work) for nothing, every single invocation. Against an 8 + // MiB-decoding form drawn 256 times from a 43 KB file, checking after the decode + // measured 6.9 GiB allocated once the budget was already spent on the first few + // invocations; at the 4096-invocation cap with a 512 MiB form, roughly 2 TiB + // (#402 round 2). Every invocation of a form still counts its bytes again against + // this Run's own shared budget when it DOES decode: the cost being bounded is + // interpretation WORK, and a form drawn many times is interpreted that many times, + // not decoded-and-cached once. + byte[]? decoded = null; + if (_contentBytesRemaining > 0) { - decoded = _reader.GetDecodedStreamData(stream); - } - catch (InvalidDataException) - { - diagnostics.Report( - PdfReaderDiagnosticCode.ContentStreamLexError, - $"Form XObject {objectNumber}'s content stream failed to decode.", - objectNumber, pageIndex: pageIndex); - decoded = null; - } - - if (decoded is not null) - { - // Every invocation of a form counts its bytes again against this Run's own - // shared budget: the cost being bounded is interpretation WORK, and a form - // drawn many times is interpreted that many times, not decoded-and-cached once. - if (_contentBytesRemaining <= 0) + try { - decoded = null; // Budget already spent; skip this invocation's content. + decoded = _reader.GetDecodedStreamData(stream); } - else if (decoded.Length > _contentBytesRemaining) + catch (InvalidDataException) { - var take = TruncateAtWhitespaceBoundary( - decoded, (int)Math.Min(decoded.Length, _contentBytesRemaining)); - decoded = decoded.AsSpan(0, take).ToArray(); - diagnostics.ReportRetained( - PdfReaderDiagnosticCode.ContentStreamTooLarge, - $"Form XObject {objectNumber}'s content pushed this Run's combined " - + $"page-and-forms budget past {MaxContentBytes / (1024 * 1024)} MiB; " - + "interpretation of it stopped there.", + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + $"Form XObject {objectNumber}'s content stream failed to decode.", objectNumber, pageIndex: pageIndex); - // See BuildPageContentBuffer's own remark on why this is forced to exactly - // zero rather than left at whatever the whitespace back-off did not use. - _contentBytesRemaining = 0; + decoded = null; } - else + + if (decoded is not null) { - _contentBytesRemaining -= decoded.Length; + ContentStreamsDecoded++; + if (decoded.Length > _contentBytesRemaining) + { + var take = TruncateAtWhitespaceBoundary( + decoded, (int)Math.Min(decoded.Length, _contentBytesRemaining)); + decoded = decoded.AsSpan(0, take).ToArray(); + diagnostics.ReportRetained( + PdfReaderDiagnosticCode.ContentStreamTooLarge, + $"Form XObject {objectNumber}'s content pushed this Run's combined " + + $"page-and-forms budget past {MaxContentBytes / (1024 * 1024)} MiB; " + + "interpretation of it stopped there.", + objectNumber, pageIndex: pageIndex); + // See BuildPageContentBuffer's own remark on why this is forced to + // exactly zero rather than left at whatever the whitespace back-off did + // not use. + _contentBytesRemaining = 0; + } + else + { + _contentBytesRemaining -= decoded.Length; + } } } @@ -1078,11 +1247,24 @@ private void HandleDo( // MaxGraphicsStateDepth budget, and the form's own content cannot 'Q' past its // own entry point back into the invoker's own saves. Marked-content nesting // (§14.6.1) and BX/EX depth get the same bracketing: nothing a form does to - // either may leak into the invoker once Do returns. Text state (_textState, - // the Tm/Tlm pair) is deliberately NOT saved here: Do is not itself a - // text-showing operator and is not permitted inside a text object (§8.2 - // Figure 9 / Table 51's own allowed-operator sets), so there is no open text - // object for a form's own content to disturb in the first place. + // either may leak into the invoker once Do returns. The over-cap push credits + // (_ignoredGsPushes/_ignoredMcPushes) are reset to 0 for the form's own scope + // and restored afterward too, so neither side of the boundary can consume a + // credit that belongs to the other's own over-cap pushes (#402 round 2). + // + // §9.4.1/§9.4.2's text matrices (_textState) are ALSO saved and restored here, + // even though 'Do' is not itself a text-showing operator and, per §8.2 Table + // 50, is not one of the operators a text object's own state permits at all (see + // the _inTextObject check above): that only says the INVOKING content is wrong + // to call 'Do' from inside a text object, not that the form's OWN content + // cannot open its own, entirely independent text object (a 'BT' resets + // TextMatrix/TextLineMatrix unconditionally, with no check against whatever the + // invoker's own text state happened to be). Nothing else brackets _textState + // off from a form's content the way the floors above do for the graphics and + // marked-content stacks, so without this save/restore a form whose own content + // opens and moves a text object (even one that itself never leaves it open, + // e.g. 'BT 999 888 Td ET') silently overwrote the invoker's own TextMatrix once + // Do returned, with no diagnostic at all (#402 round 2). var savedGs = _gs; var savedGsStackCount = _gsStack.Count; var savedMarkedContentDepth = _markedContentDepth; @@ -1090,12 +1272,27 @@ private void HandleDo( var savedGsFloor = _gsFloor; var savedMarkedContentFloor = _markedContentFloor; var savedBxFloor = _bxFloor; + var savedIgnoredGsPushes = _ignoredGsPushes; + var savedIgnoredMcPushes = _ignoredMcPushes; + var savedTextMatrix = _textState.TextMatrix; + var savedTextLineMatrix = _textState.TextLineMatrix; _gsFloor = savedGsStackCount; _markedContentFloor = savedMarkedContentDepth; _bxFloor = savedBxDepth; + _ignoredGsPushes = 0; + _ignoredMcPushes = 0; _gs = _gs.Clone(); + // §8.10.1 b): "Concatenates the matrix specified by the form dictionary's + // Matrix entry with the current transformation matrix". Applied to the CLONE + // above, not the invoker's own _gs, so a visitor reading GraphicsState.Ctm from + // inside the form's own first operator sees the composed matrix while the + // invoker's own CTM, restored in the finally below, is never touched by it + // (#402 round 2: before this, a visitor had no way to recover the composed CTM + // at all, since C0^-1 x M x C0 is undefined for a singular form /Matrix). + _gs.Ctm = matrix.Concat(_gs.Ctm); + // §7.8.2: operands never carry across an operator, and Do is the operator // here, so neither an operand left over from before this Do nor one trailing // the form's own content should reach the operator that follows on either side. @@ -1115,6 +1312,10 @@ private void HandleDo( _gsFloor = savedGsFloor; _markedContentFloor = savedMarkedContentFloor; _bxFloor = savedBxFloor; + _ignoredGsPushes = savedIgnoredGsPushes; + _ignoredMcPushes = savedIgnoredMcPushes; + _textState.TextMatrix = savedTextMatrix; + _textState.TextLineMatrix = savedTextLineMatrix; } } } @@ -1246,14 +1447,34 @@ private bool HandleInlineImage( dict.Set(key, value); } - // §8.9.7: "Unless the image uses ASCIIHexDecode or ASCII85Decode ..., the ID operator shall - // be followed by a single white-space character, and the next character shall be - // interpreted as the first byte of image data." Consuming at most one whitespace byte here - // is correct for every filter, including AHx/A85, since a producer is free to include that - // one separating byte regardless (the spec exempts them from being REQUIRED to, not from - // being ALLOWED to). §7.2.3: "The combination of a CARRIAGE RETURN followed immediately by - // a LINE FEED shall be treated as one EOL marker", so a CR immediately followed by an LF - // is consumed as that ONE separator, not as the separator plus a data byte. + // Computed before the separator-skip logic below (not after, as an earlier version did), + // since the whitespace rule itself depends on which filters are in play (#402 round 2): a + // producer that names ASCIIHexDecode/ASCII85Decode anywhere in a /Filter array gets extra + // whitespace skipped per NOTE 2 below, and deciding that requires already knowing the + // (Table 91/92-expanded) filter names dict.Set left behind while the key/value loop above + // ran. + var filterNames = CollectFilterNames(dict); + var hasDisallowedFilter = filterNames.Any(f => + f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); + + // §8.9.7: "Unless the image uses ASCIIHexDecode or ASCII85Decode as one of its filters, the + // ID operator shall be followed by a single white-space character, and the next character + // shall be interpreted as the first byte of image data." "As ONE OF its filters" (not + // merely "as its final filter", the narrower phrasing NOTE 2 below happens to use for its + // own skip-without-decoding shortcut) is what decides this: a filter array may name either + // one anywhere, not only last. NOTE 2: "if the final or only filter is ASCIIHexDecode or + // ASCII85Decode skip any further white-space [after the first]" before counting /L's own + // bytes; applied here to "as one of its filters" (matching the normative sentence, not + // NOTE 2's narrower "final or only") for the same reason this reader treats every position + // in a /Filter array as eligible elsewhere (CollectFilterNames itself does not distinguish + // position either). Before this fix, at most one whitespace byte was consumed for every + // filter shape, so 'BI /F /A85 /L 48 ID <48-byte payload> EI' (two spaces after ID) came + // out 3 bytes short: the fixed-at-one skip left the payload's own second byte behind as + // data, and the rest re-lexed as content instead. §7.2.3: "The combination of a CARRIAGE + // RETURN followed immediately by a LINE FEED shall be treated as one EOL marker", so a CR + // immediately followed by an LF is consumed as that ONE separator, not as the separator + // plus a data byte, before any of the above. + var skipsExtraWhitespace = filterNames.Any(f => f.Value is "ASCIIHexDecode" or "ASCII85Decode"); var consumedCrLf = false; if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 && PdfLexer.IsWhitespaceByte((byte)separatorByte)) @@ -1268,12 +1489,16 @@ private bool HandleInlineImage( { lexer.Seek(lexer.Position + 1); } + + if (skipsExtraWhitespace) + { + while (lexer.TryPeek() is var extraByte && extraByte >= 0 + && PdfLexer.IsWhitespaceByte((byte)extraByte)) + lexer.Seek(lexer.Position + 1); + } } var dataStart = lexer.Position; - var filterNames = CollectFilterNames(dict); - var hasDisallowedFilter = filterNames.Any(f => - f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); var length = TryLengthFromDictionary(dict, dataStart, ctx, diagnostics, pageIndex, out var lengthPastEnd); var usedTierA = length is not null; @@ -1319,19 +1544,21 @@ private bool HandleInlineImage( // (lengthFromScan) since it already IS that fallback. if (resyncPos is null && !lengthFromScan) { - var tierName = usedTierA ? "/L" : "the unfiltered image's computed length"; - ReportInlineImageMalformed( - $"the image data length from {tierName} did not land on an 'EI' operator", ctx, - diagnostics, pageIndex); - + // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for binary + // image data that reading is ambiguous: a producer may have meant only the CR as the ID + // separator, with the LF as the image's own first byte. Retry once with the data window + // shifted one byte earlier, since a payload that happens to begin with LF right after a + // CR separator is exactly the case the CR-LF-as-one-marker choice above would otherwise + // misjudge. The malformed report just below is skipped when this retry alone is what + // recovers the image: a conforming file whose payload happens to start with LF right + // after a lone CR separator must not carry a warning it recovered from cleanly + // (#402 round 2; reporting unconditionally before the retry even ran is what made a + // correctly-recovered file carry one anyway). The EI-scan fallback below is a + // DIFFERENT case: reaching it at all means the declared or computed length was wrong + // outright, not merely ambiguous, so recovering through IT still reports. + var recoveredViaCrRetry = false; if (consumedCrLf) { - // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for - // binary image data that reading is ambiguous: a producer may have meant only the - // CR as the ID separator, with the LF as the image's own first byte. Retry once - // with the data window shifted one byte earlier, before falling back to the scan, - // since a payload that happens to begin with LF right after a CR separator is - // exactly the case the CR-LF-as-one-marker choice above would otherwise misjudge. var retryStart = dataStart - 1; var retryEnd = retryStart + length.Value; if (retryStart >= 0 && retryEnd <= _currentBuffer.Length) @@ -1342,10 +1569,19 @@ private bool HandleInlineImage( dataStart = retryStart; data = _currentBuffer.Slice(dataStart, length.Value); resyncPos = retryResync; + recoveredViaCrRetry = true; } } } + if (!recoveredViaCrRetry) + { + var tierName = usedTierA ? "/L" : "the unfiltered image's computed length"; + ReportInlineImageMalformed( + $"the image data length from {tierName} did not land on an 'EI' operator", ctx, + diagnostics, pageIndex); + } + if (resyncPos is null) { var scanEnd = ScanForEi(dataStart); @@ -1412,8 +1648,20 @@ private List CollectFilterNames(PdfDictionary dict) out bool pastEnd) { pastEnd = false; - if (dict.Get(PdfName.Length) is not PdfInteger lengthObj) + var lengthRaw = dict.Get(PdfName.Length); + if (lengthRaw is null) + return null; // Table 91: /L is optional; falls through to tier b or the EI scan. + + if (lengthRaw is not PdfInteger lengthObj) + { + // Present but the wrong type (a PdfReal, say): reported the same way an invalid /W, + // /H, or /BPC is (#402 round 2), rather than silently falling through to tier b/c as + // if /L had never been written at all. + ReportInlineImageMalformed( + "'/L' is missing, or carries an invalid, non-integer value", ctx, diagnostics, + pageIndex); return null; + } if (lengthObj.Value < 0) { @@ -1441,10 +1689,15 @@ private List CollectFilterNames(PdfDictionary dict) var height = ReadIntEntry(dict, HeightKey); var bpc = isMask ? 1 : ReadIntEntry(dict, BitsPerComponentKey); - // Table 87 types Width, Height, and BitsPerComponent as positive integers. ReadIntEntry - // already turns a non-integer or an out-of-int-range value into "missing" (null); zero or - // negative is the one shape left for this method itself to reject. - if (width is null || height is null || bpc is null || width <= 0 || height <= 0 || bpc <= 0) + // Table 87 types Width and Height as integer, with no stated sign restriction of its own; + // "positive" is this reader's own requirement (a zero or negative sample count computes + // nothing meaningful), not the table's own wording. BitsPerComponent is different: Table 87 + // restricts its VALUE outright to 1, 2, 4, 8, or (from PDF 1.5) 16, so a value outside that + // set is invalid regardless of sign, not merely non-positive; ImageMask forces bpc to the + // one legal value for a mask (1) above, bypassing this check entirely. ReadIntEntry already + // turns a non-integer or an out-of-int-range /W or /H into "missing" (null). + if (width is null || height is null || bpc is null || width <= 0 || height <= 0 + || (!isMask && bpc is not (1 or 2 or 4 or 8 or 16))) { ReportInlineImageMalformed( "an unfiltered image is missing, or carries an invalid, /W, /H or /BPC needed to " @@ -1563,12 +1816,13 @@ private int ResolveComponentCount( } // Bounded lookahead: lexes at most ProbeTokens tokens from a candidate resync point, over at - // most ProbeWindowBytes bytes, and accepts it unless one of those tokens is a keyword this - // probe cannot justify as legitimate content-stream syntax. Bounding both the token count and - // the byte window is what keeps ScanForEi linear in the content length: an earlier version of - // this probe lexed all the way to the end of the buffer from every candidate, which cost O(N) - // work per candidate and made a content stream built from many false 'EI' candidates (a - // filtered image followed by literal " EI (" text repeated many times, say) quadratic overall. + // most ProbeWindowBytes bytes (ExtendedProbeWindowBytes on the retry, see LooksLikeResyncPoint + // below), and accepts unless one of those tokens is a keyword this probe cannot justify as + // legitimate content-stream syntax. Bounding both the token count and the byte window is what + // keeps ScanForEi linear in the content length: an earlier version of this probe lexed all the + // way to the end of the buffer from every candidate, which cost O(N) work per candidate and + // made a content stream built from many false 'EI' candidates (a filtered image followed by + // literal " EI (" text repeated many times, say) quadratic overall. // // A non-keyword token (a number, name, string, array or dictionary delimiter, whatever its // bytes) is neutral: it neither accepts nor rejects the candidate on its own. A keyword is @@ -1581,7 +1835,9 @@ private int ResolveComponentCount( // data would otherwise be mistaken for legitimate syntax. A 'BI' keyword ends the probe with // acceptance immediately, without lexing further: the bytes after ITS OWN following 'ID' are // raw image data and must not be judged as tokens at all. - private bool LooksLikeResyncPoint(int pos) + private enum ProbeOutcome { Accept, Reject, RanOffClippedWindow } + + private ProbeOutcome ProbeOnce(int pos, int windowBytes) { // Whether this window is itself an artificial cap, i.e. more buffer exists beyond it that // this probe deliberately does not look at. Only THAT case makes "ran off the window" @@ -1591,15 +1847,15 @@ private bool LooksLikeResyncPoint(int pos) // there. Matters for a short fixture, or a content stream that stops mid-token, where the // window and "the rest of the buffer" happen to coincide. var remaining = _currentBuffer.Length - pos; - var windowClipped = remaining > ProbeWindowBytes; - var windowLength = Math.Min(ProbeWindowBytes, remaining); + var windowClipped = remaining > windowBytes; + var windowLength = Math.Min(windowBytes, remaining); var window = _currentBuffer.Slice(pos, windowLength); var probe = new PdfLexer(window, contentStreamMode: true); for (var i = 0; i < ProbeTokens; i++) { if (probe.AtEnd) - return true; + return ProbeOutcome.Accept; Token token; try @@ -1615,30 +1871,53 @@ private bool LooksLikeResyncPoint(int pos) // bytes the probe saw, PROVIDED the window was clipped in the first place (see // above). Anything else, including running off a window that already reached the // buffer's own true end, rejects instead. - return windowClipped && probe.Position >= window.Length; + return windowClipped && probe.Position >= window.Length + ? ProbeOutcome.RanOffClippedWindow + : ProbeOutcome.Reject; } if (token.Kind == TokenKind.EndOfInput) - return true; + return ProbeOutcome.Accept; if (token.Kind != TokenKind.Keyword) continue; var raw = token.Raw.Span; if (raw.SequenceEqual("BI"u8)) - return true; + return ProbeOutcome.Accept; if (raw.SequenceEqual("true"u8) || raw.SequenceEqual("false"u8) || raw.SequenceEqual("null"u8)) continue; if (raw.Length == 1 && (raw[0] == (byte)'{' || raw[0] == (byte)'}' || raw[0] == (byte)'>')) continue; - if (ContentOperators.IsKnown(System.Text.Encoding.Latin1.GetString(raw))) + if (ContentOperators.IsKnown(raw)) continue; foreach (var b in raw) { if (b is < (byte)'!' or > (byte)'~') - return false; + return ProbeOutcome.Reject; } } - return true; + return ProbeOutcome.Accept; + } + + private bool LooksLikeResyncPoint(int pos) + { + var first = ProbeOnce(pos, ProbeWindowBytes); + if (first != ProbeOutcome.RanOffClippedWindow) + return first == ProbeOutcome.Accept; + + // A token ran off the clipped ProbeWindowBytes window: re-probe once with the much larger + // ExtendedProbeWindowBytes window before accepting OR rejecting, since a legitimate token + // following a false 'EI' candidate (a long literal string, say) can easily run past 128 + // bytes without being image data at all. Accepting on the first window's own inconclusive + // result alone let a sufficiently long unterminated token mask a malformed stream, since + // "ran off a clipped window" and "the token never closes" are indistinguishable from a + // 128-byte window alone (#402 round 2: a DCT image without /L whose data contained an + // unclosed literal string straddling the 128-byte mark, followed much later by the real + // 'EI', reported ContentStreamLexError instead of InlineImageMalformed once the straddle + // point crossed the window, and lost the trailing 'Q' to the caller's visitor). If a token + // also runs off this larger window, the candidate is rejected outright rather than accepted + // a second time. + return ProbeOnce(pos, ExtendedProbeWindowBytes) == ProbeOutcome.Accept; } } diff --git a/src/VellumPdf.Reader/Content/ContentOperators.cs b/src/VellumPdf.Reader/Content/ContentOperators.cs index fb92c6d8..d31d6705 100644 --- a/src/VellumPdf.Reader/Content/ContentOperators.cs +++ b/src/VellumPdf.Reader/Content/ContentOperators.cs @@ -7,10 +7,12 @@ namespace VellumPdf.Reader.Content; /// The 73 content-stream operators ISO 32000-2 Annex A Table A.1 lists, alphabetically, with each /// one's operand count for 's stack-discipline check. A count of /// marks the four colour operators (SC, sc, SCN, -/// scn) whose own tables (§8.6.8) give them 1 to 4 numeric operands plus, for the N -/// suffix, an optional trailing pattern name: no single fixed count describes them, so this -/// interpreter accepts any operand count for them rather than reporting -/// on a legitimately variable call. +/// scn): Table 73 (§8.6.8) gives them one numeric operand per colourant of the current +/// colour space, plus, for the N suffix, an optional trailing pattern name, and §8.6.6.5 +/// lets a DeviceN space name "an arbitrary number" of colourants, so no single fixed count (nor +/// even a small fixed range) describes them; this interpreter accepts any operand count for them +/// rather than reporting on a +/// legitimately variable call. /// internal static class ContentOperators { @@ -35,7 +37,10 @@ internal static class ContentOperators ["BX"] = 0, ["EX"] = 0, - // Table 351/352: marked-content operators. + // Table 352: marked-content operators. Annex A Table A.1's own cross-reference for BMC + // names Table 351 ("Entries in a data dictionary") instead, an error in the standard: BDC, + // DP, EMC, and MP all cite Table 352 the way BMC itself should, and Table 351 is the + // unrelated marked-content PROPERTY LIST shape, not the operator table). ["BDC"] = 2, ["BMC"] = 1, ["DP"] = 2, @@ -129,11 +134,37 @@ internal static class ContentOperators /// A.1 lists. internal static bool IsKnown(string operatorName) => _arity.ContainsKey(operatorName); + // Backs the ReadOnlySpan overload below without re-hashing through a second dictionary: + // StringComparer.Ordinal implements IAlternateEqualityComparer, string>, so + // this alternate lookup shares _arity's own buckets. + private static readonly Dictionary.AlternateLookup> _arityByChars = + _arity.GetAlternateLookup>(); + + /// + /// Span-based overload of for a caller holding a keyword as raw + /// Latin-1 bytes rather than an already-allocated string. The inline-image resync probe + /// (ContentInterpreter.ProbeOnce) checks one candidate keyword per token it lexes, and + /// allocating a string for each one only to discard it after one ContainsKey call was + /// avoidable work on that hot path (#402 round 2). + /// + internal static bool IsKnown(ReadOnlySpan operatorName) + { + // No operator this table lists is longer than a couple of characters; a byte run this much + // longer can never match one, so this bails out before stack-allocating for a length a + // hostile or corrupted stream fully controls. + if (operatorName.Length > 8) + return false; + + Span chars = stackalloc char[operatorName.Length]; + var written = System.Text.Encoding.Latin1.GetChars(operatorName, chars); + return _arityByChars.ContainsKey(chars[..written]); + } + /// /// The operand count expects, or for the /// four colour operators this interpreter does not arity-check. Throws - /// for a name would report false for; - /// every call site checks that first. + /// for a name would report + /// false for; every call site checks that first. /// internal static int ExpectedOperandCount(string operatorName) => _arity[operatorName]; } diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs index 3055eb7f..69cf9b41 100644 --- a/src/VellumPdf.Reader/Content/IContentVisitor.cs +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -17,9 +17,13 @@ internal interface IContentVisitor { /// /// Called for every recognised operator (Annex A Table A.1) the interpreter accepts, i.e. after - /// its own operand-count and stack-discipline checks pass; a malformed or unrecognised operator - /// never reaches this callback (see - /// and ). + /// its own operand-count and stack-discipline checks pass. An unrecognised operator (see + /// ) and an operator whose own arity or + /// operand-type check fails (see ) + /// never reach this callback. The one exception is an unbalanced Q or EMC: the + /// interpreter still reaches this callback for it, with no operands, once it has reported the + /// missing q/BMC/BDC to pop, since ignoring the pop is itself the + /// operator's only effect and still needs reporting to a caller tracking operator sequence. /// /// The operator keyword, e.g. "Tj" or "re". /// @@ -56,8 +60,13 @@ internal interface IContentVisitor /// /// The form XObject's own stream dictionary. /// The form's /Matrix (ISO 32000-2 §8.10.2 Table 93), or - /// when absent or malformed. The interpreter does not compose this - /// with the CTM itself; that is left to the caller. + /// when absent or malformed. The interpreter itself concatenates + /// this into 's own CTM (§8.10.1 b)) before + /// interpreting the form's own content, so a callback reading GraphicsState.Ctm from + /// inside for the form's own first operator already sees the composed + /// value. At the time THIS callback itself runs, GraphicsState.Ctm still holds the + /// invoker's own CTM, since the concatenation happens only after + /// returns. /// The form's /BBox (Table 93, Required), or /// when the entry is absent (Table 93 marks it Required, so a /// here is itself a malformation the visitor may report) or does not diff --git a/src/VellumPdf.Reader/Content/Matrix.cs b/src/VellumPdf.Reader/Content/Matrix.cs index 887fea27..598e7c6c 100644 --- a/src/VellumPdf.Reader/Content/Matrix.cs +++ b/src/VellumPdf.Reader/Content/Matrix.cs @@ -19,8 +19,9 @@ internal readonly record struct Matrix(double A, double B, double C, double D, d /// Composes this matrix with so that applying the result to a point is /// the same as applying this matrix first, then , i.e. /// this.Concat(other) == this × other in row-vector convention. This is the operation - /// cm uses to fold its operand matrix into the CTM (§8.3.4: "the new matrix shall be the - /// result of premultiplying the specified matrix with the current matrix", + /// cm uses to fold its operand matrix into the CTM (§8.3.4: "when a new transformation + /// is concatenated with an existing one, the matrix representing it shall be multiplied + /// before (premultiplied with) the existing transformation matrix", /// CTM_new = M × CTM_old, i.e. m.Concat(ctm)), and the one Td/TD use /// to fold a translation into the text line matrix (§9.4.2). /// diff --git a/src/VellumPdf.Reader/Content/TextState.cs b/src/VellumPdf.Reader/Content/TextState.cs index 9b447168..a6f93040 100644 --- a/src/VellumPdf.Reader/Content/TextState.cs +++ b/src/VellumPdf.Reader/Content/TextState.cs @@ -5,10 +5,13 @@ namespace VellumPdf.Reader.Content; /// /// The text-positioning matrices ISO 32000-2 §9.4.2 tracks per text object: the text matrix and -/// text line matrix. Deliberately separate from : §9.4.2 says these two -/// matrices are "not part of the graphics state" and are not saved or restored by q/Q. -/// Only BT resets them (to identity), and only Td, TD, Tm, and -/// T* update them. Not stacked; the interpreter owns exactly one live instance. +/// text line matrix. Deliberately separate from : §9.4.1 says these two +/// matrices (along with the derived text rendering matrix this reader does not track) "may be +/// specified only within a text object and shall not persist from one text object to the +/// next", so they are never saved or restored by q/Q the way §8.4.4's own +/// graphics-state parameters are. Only BT resets them (to identity), and only Td, +/// TD, Tm, and T* update them. Not stacked; the interpreter owns exactly +/// one live instance. /// internal sealed class TextState { diff --git a/src/VellumPdf.Reader/DiagnosticSink.cs b/src/VellumPdf.Reader/DiagnosticSink.cs index 7b5d8da9..93dfd33e 100644 --- a/src/VellumPdf.Reader/DiagnosticSink.cs +++ b/src/VellumPdf.Reader/DiagnosticSink.cs @@ -13,9 +13,13 @@ namespace VellumPdf.Reader; /// exception to that cap: a caller uses it for the rare condition that must stay visible even on a /// document engineered to exhaust the cap on unrelated conditions first, since /// alone offers no way to tell such a condition apart from an ordinary one once the cap is full. -/// is the one caller today, and bounds itself to at most two -/// retained entries per walk, not three, even though three distinct codes each reach -/// (see its own remarks for why). +/// is one caller, and bounds itself to at most two retained entries +/// per walk, not three, even though three distinct codes each reach +/// (see its own remarks for why). ContentInterpreter (#98) is the other: it bounds itself +/// to at most two retained entries per Run the same way, one each for +/// and +/// (see that type's own remarks +/// for why each of those fires at most once). /// /// /// The maximum number of ordinary diagnostics this sink holds before it starts recording diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 79d8b336..10311f22 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -388,11 +388,14 @@ public enum PdfReaderDiagnosticCode /// second sign character (--5, -+5: §7.3.3 allows only "an optional sign"), a /// dictionary operand on an operator other than BDC/DP (§7.8.2: "Dictionaries /// shall be permitted as operands only by certain specific operators"), a known operator - /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an unbalanced - /// Q with no matching q on the graphics-state stack, or an unbalanced EMC - /// with no matching BMC/BDC (§14.6.1). An unbalanced q still open at the - /// end of a content stream is not reported: nothing downstream of this interpreter needs the - /// graphics state restored past the last operator it saw. + /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an operand of + /// the wrong type where the arity is otherwise right (a TJ whose single operand is not + /// an array, §9.4.3), a Do that occurred inside a text object (§8.2 Table 50, which does + /// not list Do among the operators a text object permits), an unbalanced Q with + /// no matching q on the graphics-state stack, or an unbalanced EMC with no + /// matching BMC/BDC (§14.6.1). An unbalanced q still open at the end of a + /// content stream is not reported: nothing downstream of this interpreter needs the graphics + /// state restored past the last operator it saw. /// OperandStackMalformed = 302, @@ -440,14 +443,24 @@ public enum PdfReaderDiagnosticCode /// dictionary entries was itself invalid: a filter this interpreter never applies to inline /// image data (JBIG2Decode, JPXDecode, or Crypt: §8.9.7 itself excludes /// all three from inline-image use), a missing ID or EI operator, a missing, - /// non-integer, or non-positive /W, /H, or /BPC where the image's shape - /// requires one to compute the data length (Table 87 types all three as positive integers), a - /// negative /L (§8.9.7, Table 91; PDF 2.0), an /L or computed length past the end - /// of the stream, or a computed length (from /L or from the image's own shape) that does - /// not land on the following EI operator, in which case this reader retries the EI scan - /// before giving up. The image is skipped (its data is still delimited well enough for - /// interpretation of the rest of the content stream to continue), and no inline-image callback - /// is raised for it. + /// non-integer, or non-positive /W or /H, a missing /BPC or one whose + /// value is not 1, 2, 4, or 8 (or, from PDF 1.5, 16) where the image's shape requires one to + /// compute the data length (Table 87 types /W and /H as integer and restricts + /// /BPC's own value to that fixed set; "positive" for /W//H is this + /// reader's own requirement, not the table's own wording), an /L present but not an + /// integer, or negative (§8.9.7, Table 91; PDF 2.0), an /L or computed length past the + /// end of the stream, or a computed length (from /L or from the image's own shape) that + /// does not land on the following EI operator, in which case this reader retries the EI + /// scan before giving up. No inline-image callback is raised for it. When the data was still + /// delimited (a disallowed filter, a length that the EI scan recovered) only the image + /// is skipped and interpretation of the rest of the content stream continues; when it could not + /// be delimited at all (no ID, no EI, a length past the end of the stream) + /// interpretation of that stream stops there, since nothing past that point can be + /// resynchronised reliably. Reported at most once per page: the sink's dedupe key is (code, + /// object, page), and every image on one content stream reports against that same object + /// (or, for the page's own top-level content specifically, the same ), + /// so a second inline image on the same page with its own, different malformation is not + /// listed separately. /// InlineImageMalformed = 307, diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 0b550b5a..2f1e4b29 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -257,6 +257,25 @@ public void UnknownOperatorInsideBX_dropsItsOperands_perTable33() Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.UnknownOperator); } + [Fact] + public void UnknownOperatorOutsideBX_alsoDropsItsOperands() + { + // §7.8.2: "operands shall not be left over when an operator finishes execution" applies to + // an unrecognised keyword's own (no-op) execution too, not only to a recognised one + // (#402 round 2). Before the fix, the operands preceding an unrecognised operator outside a + // BX/EX section survived to be misread by whatever operator followed: '20' here was fed + // into 'w' as its own operand instead of the '1' that belongs to it. + const string content = "10 20 Zork\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + var w = Assert.Single(visitor.Operators); + Assert.Equal(1, ((PdfInteger)w.Operands[0]).Value); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.UnknownOperator); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + } + // ── Operand-stack and graphics-state caps ─────────────────────────────────────────────────── [Fact] @@ -317,6 +336,46 @@ public void UnbalancedQ_isIgnoredWithADiagnostic() Assert.Contains(visitor.Operators, o => o.Op == "w"); // interpretation continued } + // ── Over-cap 'q'/'BMC' pushes must not desync a later balanced pop (#402 round 2) ───────────── + + [Theory] + [InlineData(65)] + [InlineData(70)] + public void BalancedQAndQ_pastTheGraphicsStateCap_reportsOnlyTheLimit_andStaysBalanced(int depth) + { + // Before the fix, the pushes past MaxGraphicsStateDepth (64) were dropped but their + // matching 'Q's still popped anyway, so a balanced 'q'...'Q' sequence deeper than the cap + // reported BOTH ContentLimitExceeded (for the dropped pushes) AND OperandStackMalformed + // (for the "Q"s that found nothing real left on the stack once the real pushes were + // exhausted), and desynchronised GraphicsState.Ctm from the actual nesting besides. + var content = string.Concat(Enumerable.Repeat("q\n", depth)) + + "2 0 0 2 10 20 cm\n" + + string.Concat(Enumerable.Repeat("Q\n", depth)); + + var interpreter = RunAndKeepInterpreter(BuildPageDoc(content), out var reader); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + // Every 'q' was balanced by a 'Q', so the CTM the 'cm' set inside is restored to identity, + // whether or not this reader's own stack held every one of those saves. + Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + } + + [Fact] + public void BalancedBmcAndEmc_pastTheMarkedContentCap_reportsOnlyTheLimit_andStaysBalanced() + { + const int depth = 65; + var content = string.Concat(Enumerable.Repeat("/Tag BMC\n", depth)) + + string.Concat(Enumerable.Repeat("EMC\n", depth)) + + "1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + // ── q/Q/cm matrix state, text state ────────────────────────────────────────────────────────── [Fact] @@ -473,6 +532,28 @@ public void SelfReferencingForm_reportsCycleOnce() Assert.Single(visitor.FormEnds); } + [Fact] + public void SelfReferencingForm_withMaxDepthOne_stillReportsCycle_notDepthExceeded() + { + // With MaxFormXObjectDepth this tight, the outer 'Do' (page -> Form11, depth 0 -> 1) + // succeeds, so Form11's own self-'Do' hits _formDepth == the cap on the very same + // invocation that also closes the cycle. Before the fix, the depth check ran BEFORE the + // cycle check, so this reported FormXObjectDepthExceeded (technically true, but strictly + // less informative than the actual reason: the recursion is not merely deep, it never + // terminates at all) (#402 round 2). + var doc = BuildPageDoc( + "/F1 Do\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] " + + "/Resources << /XObject << /F1 11 0 R >> >> >>", + "/F1 Do"u8.ToArray())); + + var (reader, _, _) = Run(doc, new PdfReaderOptions { MaxFormXObjectDepth = 1 }); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.FormXObjectCycle); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.FormXObjectDepthExceeded); + } + [Fact] public void FormDrawn4097Times_reportsBudgetExceeded_andThePageStillCompletes() { @@ -547,6 +628,109 @@ public void Form_matrixAndBBox_resolveThroughAnIndirectReference() Assert.Equal(4, begin.BBox.UrY); } + // ── §8.10.1 b): the form's own /Matrix is concatenated into the CTM (#402 round 2) ─────────── + + [Fact] + public void Do_concatenatesTheFormsOwnMatrixIntoTheCtm_forOperatorsInsideTheForm() + { + var doc = BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Matrix [2 0 0 2 10 10] >>", + "0 0 1 1 re"u8.ToArray())); + + Matrix? ctmInsideForm = null; + var reader = PdfReader.Open(doc); + var interpreter = new ContentInterpreter(reader); + var probe = new CtmProbeVisitor(() => ctmInsideForm ??= interpreter.GraphicsState.Ctm); + interpreter.Run(reader.GetPage(0), probe); + + // §8.3.4: CTM_new = M x CTM_old; the invoker's own CTM at the point of Do is identity, so + // the composed value equals the form's own /Matrix exactly. + Assert.Equal(new Matrix(2, 0, 0, 2, 10, 10), ctmInsideForm); + // The invoker's own CTM is untouched once Do returns. + Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + } + + private sealed class CtmProbeVisitor(Action onFirstOperatorInsideForm) : IContentVisitor + { + private bool _insideForm; + + public void OnOperator(string operatorName, IReadOnlyList operands, int offset) + { + if (_insideForm) + onFirstOperatorInsideForm(); + } + + public void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset) { } + + public void OnFormBegin( + PdfDictionary formDictionary, Matrix formMatrix, PdfRectangle? boundingBox, int objectNumber, + int offset) => + _insideForm = true; + + public void OnFormEnd(int objectNumber) => _insideForm = false; + } + + // ── HandleDo reports a resource that resolves but is not a usable XObject (#402 round 2) ───── + + [Fact] + public void Do_namingAnEntryThatIsNotAnIndirectReference_reportsResourceMissing() + { + var (reader, _, visitor) = Run(BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 /NotAReference >> >>")); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // interpretation continued past Do + } + + [Fact] + public void Do_namingAReferenceThatDoesNotResolveToAStream_reportsResourceMissing() + { + var (reader, _, visitor) = Run(BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form >>"))); // no stream body: not a stream object + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void Do_namingAStreamWithNoSubtype_reportsResourceMissing() + { + var (reader, _, visitor) = Run(BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject >>", []))); // /Subtype absent + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void Do_namingAStreamWithASubtypeThatIsNeitherFormNorImage_reportsResourceMissing() + { + var (reader, _, visitor) = Run(BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Foo >>", []))); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void Do_namingAnImageXObject_reportsNoResourceMissing_andNoRecursion() + { + var (reader, _, visitor) = Run(BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /BitsPerComponent 8 " + + "/ColorSpace /DeviceGray >>", [0]))); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + Assert.Contains(visitor.Operators, o => o.Op == "Do"); + Assert.Empty(visitor.FormBegins); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + // ── Do brackets a form's content in an implicit q/Q (ISO 32000-2 §8.10.1) ─────────────────── [Fact] @@ -564,6 +748,47 @@ public void Do_onAFormThatChangesTheCtm_doesNotLeakTheChangeIntoTheInvoker() Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); } + [Fact] + public void Do_onAFormThatOpensItsOwnTextObject_doesNotLeakTextMatricesIntoTheInvoker() + { + // Before the fix, _textState was never saved/restored around a form invocation (the class + // doc reasoned Do could never appear inside a text object, so there was nothing to + // disturb; that reasoning missed that the FORM's own content can open an entirely + // independent text object regardless of the invoker's own state). A form whose own content + // opens, moves, and closes its own text object silently overwrote the invoker's own + // TextMatrix once Do returned, with no diagnostic at all (#402 round 2). + var doc = BuildPageDoc( + "/F1 Do\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", + "BT 999 888 Td ET"u8.ToArray())); + + var interpreter = RunAndKeepInterpreter(doc, out _); + + Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); + Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + } + + [Fact] + public void Do_insideATextObject_reportsOperandStackMalformedOnce_andStillRecurses() + { + // §8.2 Table 50 does not list 'Do' among the operators a text object permits. This does + // not itself stop the recursion (the form is still resolvable and drawn); it is purely a + // producer-side diagnostic (#402 round 2). + var doc = BuildPageDoc( + "BT\n/F1 Do\nET\n1 w\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", [])); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics.Where(d => + d.Code == PdfReaderDiagnosticCode.OperandStackMalformed + && d.Message.Contains("text object", StringComparison.Ordinal)).ToList(); + Assert.Single(reports); + Assert.Single(visitor.FormBegins); + Assert.Contains(visitor.Operators, o => o.Op == "w"); // interpretation continued + } + [Fact] public void Do_onAFormWithUnbalancedQ_doesNotPopThePagesOwnSave() { @@ -673,6 +898,90 @@ private static class ContentInterpreterBudget internal const long MaxContentBytes = 64L * 1024 * 1024; } + // ── The per-Run content budget is charged per element as chunks arrive (#402 round 2) ──────── + + [Fact] + public void ContentsArray_referencingTheSameOversizedStreamTenTimes_decodesOnlyFourElements() + { + // Before the fix, BuildPageContentBuffer decoded and held EVERY /Contents array element in + // memory before Concatenate ever got a chance to apply the budget: peak heap scaled with + // element count x MaxDecodedStreamBytes rather than with the budget itself. Ten references + // to the same ~20 MiB-decoding stream reproduces that (one 20 MiB Flate stream referenced + // 128 times from a 103 KB file measured 4662 MiB peak managed heap while interpreting + // exactly the same operators a four-element array would); charging the budget as each + // chunk arrives means only the first four references are ever decoded at all. Three fit + // comfortably inside the 64 MiB budget, and the fourth is what pushes the running total + // over it, so ContentStreamsDecoded pins that count directly rather than asserting on + // wall-clock time or process memory. + var unit = "0 0 1 1 re\n"u8.ToArray(); + var repeatsPerChunk = (20 * 1024 * 1024) / unit.Length; + var chunkBody = new byte[unit.Length * repeatsPerChunk]; + for (var i = 0; i < repeatsPerChunk; i++) + unit.CopyTo(chunkBody, i * unit.Length); + var encoded = Flate(chunkBody); + + var tenRefsDoc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [4 0 R 4 0 R 4 0 R 4 0 R 4 0 R 4 0 R 4 0 R 4 0 R " + + "4 0 R 4 0 R] >>"), + new Obj(4, "<< /Filter /FlateDecode >>", encoded)); + + var (reader, interpreter, visitor) = Run(tenRefsDoc); + + Assert.Equal(4, interpreter.ContentStreamsDecoded); + var tooLarge = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge).ToList(); + Assert.Single(tooLarge); + + var fourRefsDoc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [4 0 R 4 0 R 4 0 R 4 0 R] >>"), + new Obj(4, "<< /Filter /FlateDecode >>", encoded)); + var (_, _, fourVisitor) = Run(fourRefsDoc); + + Assert.Equal(fourVisitor.Operators.Count, visitor.Operators.Count); + } + + [Fact] + public void FormDecodingToEightMiB_drawn64Times_decodesExactlyEightTimes() + { + // A2 (#402 round 2): HandleDo checked _contentBytesRemaining AFTER decoding a form's + // content, so a decode whose result was about to be discarded (budget already spent) still + // paid the full allocation and filter cost every single invocation. 64 MiB / 8 MiB = 8: the + // budget check hoisted above the decode means invocations past the 8th skip decoding + // entirely rather than decoding (and counting) a 9th time only to discard it, the way an + // inexactly-sized form still would (a sliver of budget left over after 8 whole decodes + // still reads as "budget available" to the entry check, so a 9th decode still runs and only + // gets truncated afterward). + var unit = "0 0 1 1 re\n"u8.ToArray(); + const int formSize = 8 * 1024 * 1024; + var repeats = formSize / unit.Length; + var formBody = new byte[formSize]; + for (var i = 0; i < repeats; i++) + unit.CopyTo(formBody, i * unit.Length); + Array.Fill(formBody, (byte)'\n', repeats * unit.Length, formSize - repeats * unit.Length); + + var pageContent = string.Concat(Enumerable.Repeat("/F1 Do\n", 64)); + var doc = BuildPageDoc( + pageContent, "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", formBody)); + + var (_, interpreter, _) = Run(doc); + + // 9, not 8: ContentStreamsDecoded also counts the page's own /Contents decode + // (BuildPageContentBuffer.AddElement increments it too, per A1), which runs once before + // any 'Do' and spends a small slice of the same budget on the page's own "'/F1 Do' x 64" + // bytes; the 8 form decodes below are what this test pins. + Assert.Equal(9, interpreter.ContentStreamsDecoded); + } + // ── Inline images ──────────────────────────────────────────────────────────────────────────── [Fact] @@ -755,6 +1064,98 @@ public void InvalidW_takesTheEiScanPath_withTheMissingOrInvalidReport(string inv Assert.True(img.Data.AsSpan().SequenceEqual(data)); } + // ── NOTE 2's "skip any further white-space" for ASCIIHexDecode/ASCII85Decode (#402 round 2) ── + + [Fact] + public void A85WithTwoSpacesAfterId_andExplicitL_skipsBothSeparatorBytes() + { + // §8.9.7 NOTE 2: once the single required separator is consumed, a filter of + // ASCIIHexDecode or ASCII85Decode skips any FURTHER white-space too, before /L's own bytes + // are counted. Before this fix, at most one whitespace byte was ever consumed regardless of + // filter, so the second space here was misread as the payload's own first byte and /L=4 + // came out 3 bytes short of the real data. + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /F /A85 /L {data.Length} ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void A85WithTwoSpacesAfterId_andNoL_stillSkipsBothSeparatorBytes() + { + // Same as above but falling all the way to the EI scan (tier c) instead of an explicit /L: + // before this fix, the extra space landed inside the scanned data too, so the recovered + // bytes were " ABCD" (5, with the stray leading space) rather than "ABCD" (4). + byte[] data = "ABCD"u8.ToArray(); + var content = "BI /F /A85 ID " + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void A85WithCrLfThenTwoMoreSpaces_skipsAllOfThem() + { + // The CR-LF-as-one-EOL-marker separator (§7.2.3) is consumed first, then the extra + // whitespace loop below still applies on top of it for an ASCII85Decode filter. + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /F /A85 /L {data.Length} ID\r\n " + + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void AHxWithTwoSpacesAfterId_andExplicitL_skipsBothSeparatorBytes() + { + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /F /AHx /L {data.Length} ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void RawImageWithTwoSpacesAfterId_keepsTheSecondSpaceAsData() + { + // Control: a filter NOT in {ASCIIHexDecode, ASCII85Decode} (here, no filter at all) still + // consumes exactly one separator byte, per the normative sentence's own "Unless ...". The + // second space is the payload's own first sample byte, not something to skip. + var content = "BI /W 5 /H 1 /BPC 8 /CS /G ID ABCD EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(" ABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + [Fact] public void FilteredAHx_usesTheEiScan() { @@ -921,10 +1322,11 @@ public void TwoConsecutiveInlineImages_theSecondWithAShortDictionary_bothDelimit [Fact] public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindow_isAccepted() { - // The probe's own bounded lexer runs off its 128-byte window mid-string here (the literal - // is 200 bytes, longer than the window), the inconclusive case this reader accepts rather - // than rejects: PdfLexer's string readers advance Position as they read, so reaching the - // window's own end here means the window ran out, not that the bytes were malformed. + // The probe's first, 128-byte window runs off mid-string here (the literal is 200 bytes, + // longer than that window), so the probe re-tries once against the 4096-byte extended + // window (#402 round 2); the string closes well inside THAT one, so the candidate is + // accepted the same way it was before the two-window redesign, just through the second + // window rather than an inconclusive-accept on the first. var longLiteral = new string('X', 200); byte[] data = [0xFF, 0xD8, 0xFF]; var content = "BI /F /DCT ID "u8.ToArray() @@ -943,6 +1345,96 @@ public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindo Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); } + // ── The extended (4096-byte) probe window still rejects a token that never closes at all, + // regardless of how far past the first window that non-closure runs (#402 round 2) ────────── + + [Theory] + [InlineData(100)] // inside the 128-byte first window: already correct before this fix + [InlineData(110)] + [InlineData(200)] // past the first window, inside the second + [InlineData(5000)] // past BOTH windows + public void FalseEiCandidate_insideAnUnterminatedLiteralString_isRejected_regardlessOfLength(int n) + { + // Reproduces the round-1 regression this round's reviewers found: a false ' EI' candidate + // immediately followed by an unterminated literal string that never closes ANYWHERE in the + // rest of the buffer, then N filler bytes, then the real ' EI\nQ\n'. The OLD single-window + // probe treated "ran off a clipped window" as accept-by-default, so once N pushed the false + // candidate's own unterminated string past the 128-byte window (but the buffer still had + // more bytes beyond it), the false candidate was WRONGLY accepted as the resync point, + // truncating the image to 3 bytes and losing 'Q' to the caller's visitor entirely, with no + // InlineImageMalformed to explain why. The two-window redesign keeps the correct outcome at + // every N tested here: a string that never closes anywhere is rejected at either window, + // never accepted just because it ran past the first one. + var falseCandidate = " EI (never closed"u8.ToArray(); + var filler = new byte[n]; + Array.Fill(filler, (byte)'z'); + byte[] jpegNoise = [0x01, 0x02, 0xFF, 0xD8, 0xFF]; + var data = jpegNoise.Concat(falseCandidate).Concat(filler).ToArray(); + + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void FalseEiCandidate_insideAnUnterminatedHexString_isRejected() + { + // The hex-string counterpart of the literal-string regression above: an unclosed '<' run + // longer than the first probe window, never closed anywhere in the rest of the buffer. + var falseCandidate = " EI <414243"u8.ToArray(); // '<' opens a hex string that never closes + var filler = new byte[200]; + Array.Fill(filler, (byte)'0'); + byte[] jpegNoise = [0x01, 0x02, 0xFF, 0xD8, 0xFF]; + var data = jpegNoise.Concat(falseCandidate).Concat(filler).ToArray(); + + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void RealEi_followedByATerminated4000ByteLiteral_isAccepted() + { + // The true terminating 'EI' must still be accepted even when what follows it is a long + // (past the first window, inside the second) but well-formed token: rejecting a resync + // point that IS the real one just because legitimate content after it happens to be long + // would be exactly as wrong as accepting one that never closes at all. + var longLiteral = new string('Y', 4000); + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(Encoding.ASCII.GetBytes($" EI ({longLiteral}) Tj\nQ\n")) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + var tj = Assert.Single(visitor.Operators, o => o.Op == "Tj"); + var str = (PdfLiteralString)tj.Operands[0]; + Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); + } + // ── A failed tier-a/tier-b end falls back to the scan instead of losing the stream (#402) ── [Fact] @@ -989,6 +1481,32 @@ public void IdFollowedByCrLf_withExplicitL_treatsBothBytesAsOneSeparator() Assert.Contains(visitor.Operators, o => o.Op == "w"); } + [Fact] + public void CrAsSingleSeparator_withDataStartingWithLf_recoversWithoutAWarning() + { + // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for binary image + // data that reading is ambiguous: here the separator is a lone CR, and the image's own + // first byte IS LF, so wrongly consuming both as the marker shifts the whole data window + // one byte late. With no whitespace between the (wrongly windowed) data and 'EI', + // the shifted window's own SkipToEi check fails outright (it lands one byte INTO 'EI' + // rather than in front of it), so the one-byte-earlier retry runs and recovers the correct + // 5-byte window (LF,'A','B','C','D'). Before the fix, InlineImageMalformed was reported + // unconditionally before that retry ever ran, so this conforming file carried a warning + // even though the retry recovered it cleanly (#402 round 2). + var raw = "BI /W 5 /H 1 /BPC 8 /CS /G ID\r"u8.ToArray() + .Concat("\nABCDEI\nQ\n1 w\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(raw); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("\nABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + [Fact] public void W_zero_reportsMalformed_andRecoversViaTheEiScan() { @@ -1003,6 +1521,66 @@ public void W_zero_reportsMalformed_andRecoversViaTheEiScan() Assert.Contains(visitor.Operators, o => o.Op == "w"); } + [Fact] + public void L_asARealNumber_reportsMalformed_andRecoversViaTheEiScan() + { + // Table 91 types /L as an integer; a PdfReal there (dropped silently before this fix) is + // reported the same way an invalid /W, /H, or /BPC already is (#402 round 2). + var content = "BI /F /AHx /L 4.0 ID ABCD EI\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed + && d.Message.Contains("missing, or carries an invalid", StringComparison.Ordinal)); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("ABCD"u8.ToArray())); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Theory] + [InlineData(3)] + [InlineData(7)] + [InlineData(9)] + public void InvalidBpc_notInTheFixedSet_takesTheEiScanPath(int bpc) + { + // Table 87 restricts /BPC's own value to 1, 2, 4, 8, or 16, not merely "a positive + // integer"; a value like 3 was accepted before this fix (#402 round 2). + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /W 2 /H 2 /BPC {bpc} /CS /G ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed + && d.Message.Contains("missing, or carries an invalid", StringComparison.Ordinal)); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + } + + [Fact] + public void WidthHeightBpc_allIntMaxValue_recoversViaTheEiScan_withoutOverflowing() + { + // int.MaxValue is rejected by the /BPC set-membership check before it ever reaches + // TryComputeUnfilteredLength's own (long) width * bpc * components multiplication, closing + // the theoretical overflow that check guards against (#402 round 2): this must recover + // through the EI scan rather than throwing or hanging. + byte[] data = "ABCD"u8.ToArray(); + var content = $"BI /W {int.MaxValue} /H {int.MaxValue} /BPC {int.MaxValue} /CS /G ID " + + Encoding.ASCII.GetString(data) + " EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + // ── Table 92 abbreviations inside a /CS array (#402) ──────────────────────────────────────── [Fact] diff --git a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs index 7e769858..87c3e030 100644 --- a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs @@ -115,6 +115,28 @@ public void ContentStreamMode_seekingPastAOneByteKeyword_lexesTheFollowingTokenN Assert.Equal("Tj", Encoding.ASCII.GetString(next.Raw.Span)); } + // TokenKind (internal) named by string here too, for the same CS0051 reason as + // AllTokenKindFixtures below. + [Theory] + [InlineData("}5", nameof(TokenKind.Integer), "5")] + [InlineData("}/N", nameof(TokenKind.Name), "/N")] + [InlineData("}(s)", nameof(TokenKind.LiteralString), "(s)")] + public void ContentStreamMode_aOneByteKeyword_abuttingANonKeywordToken_lexesBothSeparately( + string input, string expectedSecondKindName, string expectedSecondRaw) + { + var expectedSecondKind = Enum.Parse(expectedSecondKindName); + var lexer = new PdfLexer(Encoding.ASCII.GetBytes(input), contentStreamMode: true); + + var first = lexer.NextToken(); + var second = lexer.NextToken(); + + Assert.Equal(TokenKind.Keyword, first.Kind); + Assert.Equal(1, first.Raw.Length); + Assert.Equal((byte)'}', first.Raw.Span[0]); + Assert.Equal(expectedSecondKind, second.Kind); + Assert.Equal(expectedSecondRaw, Encoding.ASCII.GetString(second.Raw.Span)); + } + [Fact] public void ContentStreamMode_insideACompatibilitySection_lexesAWholeBxToExSequenceWithoutThrowing() { From 5f5f84b6f33fe9433474ab6b81067bfe3b74e645 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Thu, 3 Sep 2026 20:09:55 +0200 Subject: [PATCH 06/14] Bracket the text-object flag around Do and fix the Table 50 category MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-2 fix-up saved the invoker's text matrices around a Form XObject invocation but not the "inside a text object" flag, so a form invoked from within BT/ET had its own 'Do' judged against the invoker's state, and a form whose content opened BT without ET left the invoker looking as if it were still inside a text object. Save the flag, start the form's content outside any text object, and restore on return; two tests fail without the bracket and pass with it. The same fix-up described 'Do' as a general graphics state operator. ISO 32000-2 Table 50 lists it under XObjects; what forbids it inside a text object is §8.2 Figure 9, which admits no operator of that category there. Correct the comments, the diagnostic message and the code doc, and replace the paraphrased §8.10.1 b) quotation with the standard's own words. --- .../Content/ContentInterpreter.cs | 29 +++++++----- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 4 +- .../ContentInterpreterTests.cs | 46 ++++++++++++++++++- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 36fb709b..4ecd1de1 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -1057,9 +1057,10 @@ private void HandleDo( if (_inTextObject) { - // §8.2 Figure 9 / Table 50: 'Do' is a General Graphics State category operator, not - // one Table 50 lists among the operators legal inside a text object; a producer that - // invokes it there is wrong regardless of what the named XObject turns out to be. The + // §8.2 Figure 9 admits only the general graphics state, colour, text state, + // text-positioning, text-showing and marked-content categories of Table 50 inside a + // text object; 'Do' sits in the XObjects category, so a producer that invokes it + // there is wrong regardless of what the named XObject turns out to be. The // recursion below still runs (a text-object violation is not itself a reason to skip // an otherwise-resolvable Form), but the shared _textState instance a form's own // content may disturb (BT/Td/ET, unbracketed by any q/Q-style save) is saved and @@ -1068,8 +1069,8 @@ private void HandleDo( // (#402 round 2). diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, - "'Do' occurred inside a text object (ISO 32000-2 §8.2 Table 50); 'Do' is not one " - + "of the operators a text object's own state permits.", + "'Do' occurred inside a text object (ISO 32000-2 §8.2 Figure 9 admits no XObjects " + + "category operator there).", ctx.DiagObjectNumber, pageIndex: pageIndex); } @@ -1252,10 +1253,10 @@ private void HandleDo( // and restored afterward too, so neither side of the boundary can consume a // credit that belongs to the other's own over-cap pushes (#402 round 2). // - // §9.4.1/§9.4.2's text matrices (_textState) are ALSO saved and restored here, - // even though 'Do' is not itself a text-showing operator and, per §8.2 Table - // 50, is not one of the operators a text object's own state permits at all (see - // the _inTextObject check above): that only says the INVOKING content is wrong + // §9.4.1's text matrices (_textState) are ALSO saved and restored here, even + // though 'Do' is not itself a text-showing operator and §8.2 Figure 9 admits no + // XObjects-category operator inside a text object at all (see the + // _inTextObject check above): that only says the INVOKING content is wrong // to call 'Do' from inside a text object, not that the form's OWN content // cannot open its own, entirely independent text object (a 'BT' resets // TextMatrix/TextLineMatrix unconditionally, with no check against whatever the @@ -1276,16 +1277,21 @@ private void HandleDo( var savedIgnoredMcPushes = _ignoredMcPushes; var savedTextMatrix = _textState.TextMatrix; var savedTextLineMatrix = _textState.TextLineMatrix; + var savedInTextObject = _inTextObject; _gsFloor = savedGsStackCount; _markedContentFloor = savedMarkedContentDepth; _bxFloor = savedBxDepth; _ignoredGsPushes = 0; _ignoredMcPushes = 0; + // The form's own content starts outside any text object whatever the invoker + // was doing: its 'Do' is judged against its own BT/ET, and its ET must not + // close the invoker's text object once Do returns. + _inTextObject = false; _gs = _gs.Clone(); - // §8.10.1 b): "Concatenates the matrix specified by the form dictionary's - // Matrix entry with the current transformation matrix". Applied to the CLONE + // §8.10.1 b): "Concatenates the matrix from the form dictionary's Matrix entry + // with the current transformation matrix (CTM)". Applied to the CLONE // above, not the invoker's own _gs, so a visitor reading GraphicsState.Ctm from // inside the form's own first operator sees the composed matrix while the // invoker's own CTM, restored in the finally below, is never touched by it @@ -1316,6 +1322,7 @@ private void HandleDo( _ignoredMcPushes = savedIgnoredMcPushes; _textState.TextMatrix = savedTextMatrix; _textState.TextLineMatrix = savedTextLineMatrix; + _inTextObject = savedInTextObject; } } } diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 10311f22..19205761 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -390,8 +390,8 @@ public enum PdfReaderDiagnosticCode /// shall be permitted as operands only by certain specific operators"), a known operator /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an operand of /// the wrong type where the arity is otherwise right (a TJ whose single operand is not - /// an array, §9.4.3), a Do that occurred inside a text object (§8.2 Table 50, which does - /// not list Do among the operators a text object permits), an unbalanced Q with + /// an array, §9.4.3), a Do that occurred inside a text object (§8.2 Figure 9 admits no + /// operator of Table 50's XObjects category there), an unbalanced Q with /// no matching q on the graphics-state stack, or an unbalanced EMC with no /// matching BMC/BDC (§14.6.1). An unbalanced q still open at the end of a /// content stream is not reported: nothing downstream of this interpreter needs the graphics diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 2f1e4b29..efffaea9 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -772,7 +772,7 @@ public void Do_onAFormThatOpensItsOwnTextObject_doesNotLeakTextMatricesIntoTheIn [Fact] public void Do_insideATextObject_reportsOperandStackMalformedOnce_andStillRecurses() { - // §8.2 Table 50 does not list 'Do' among the operators a text object permits. This does + // §8.2 Figure 9 admits no XObjects-category (Table 50) operator inside a text object. This does // not itself stop the recursion (the form is still resolvable and drawn); it is purely a // producer-side diagnostic (#402 round 2). var doc = BuildPageDoc( @@ -789,6 +789,50 @@ public void Do_insideATextObject_reportsOperandStackMalformedOnce_andStillRecurs Assert.Contains(visitor.Operators, o => o.Op == "w"); // interpretation continued } + [Fact] + public void Do_insideATextObject_judgesTheFormsOwnDoAgainstTheFormsOwnTextObjectState() + { + // The form's content starts outside any text object whatever the invoker was doing, so a + // 'Do' in a form that opens no BT of its own is not a violation even when the invoking + // 'Do' sat inside one. Before the HandleDo bracket saved the flag, the form's 'Do' was + // reported against the form's object number as well (#402 round 2). + var doc = BuildPageDoc( + "BT\n/F1 Do\nET\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj( + 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << /XObject << /F2 12 0 R >> >> >>", + "/F2 Do\n"u8.ToArray()), + new Obj(12, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", [])); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics.Where(d => + d.Code == PdfReaderDiagnosticCode.OperandStackMalformed + && d.Message.Contains("text object", StringComparison.Ordinal)).ToList(); + var report = Assert.Single(reports); + Assert.NotEqual(11, report.ObjectNumber); + Assert.Equal(2, visitor.FormBegins.Count); + } + + [Fact] + public void Do_onAFormWithAnUnbalancedBT_doesNotLeaveTheInvokerInsideATextObject() + { + // The flag is restored when the form returns, so a form whose content opens BT and never + // closes it (a §9.4.1 violation of the form's own) cannot make the invoker's next 'Do' + // look like it sits inside a text object. + var doc = BuildPageDoc( + "/F1 Do\n/F2 Do\n1 w\n", "<< /XObject << /F1 11 0 R /F2 12 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "BT\n"u8.ToArray()), + new Obj(12, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", [])); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => + d.Code == PdfReaderDiagnosticCode.OperandStackMalformed + && d.Message.Contains("text object", StringComparison.Ordinal)); + Assert.Equal(2, visitor.FormBegins.Count); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + [Fact] public void Do_onAFormWithUnbalancedQ_doesNotPopThePagesOwnSave() { From 53b02cae146489f1193f47c328627fa0038a2119 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 02:02:34 +0200 Subject: [PATCH 07/14] Fix content-interpreter review findings from PR #402 round 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bound the inline-image resync probe by a shared per-Run byte budget instead of a per-candidate window: many false 'EI' candidates each paying a full window's own lexing cost drove the " EI (" repeated shape to 16.6 s per decoded MiB, and a fixed extended window still rejected the terminating 'EI' whose own legitimate follow-on token happened to be longer than it. Require a positive reason to accept a candidate (a Table A.1 operator, 'BI', or the buffer's own end) rather than merely the absence of a reject, closing the same gap for neutral token kinds (arrays, dictionaries, closed strings) that round 2 closed only for tokens PdfLexer itself throws on. Scope the AHx/A85 extra-whitespace skip to the first /Filter array element (or the sole name), the one reading the bytes after ID, per §7.4.1's own decode-order example, not any position the two filters happen to occupy. Strip a CR LF pair as one delimiter before 'EI', matching the ID side. Type-check every operand this interpreter reads for its own graphics or text state (cm, Tc, Tw, Tz, TL, Tf, Tr, Ts, Td, TD, Tm, Do) before using it, reporting 302 and dropping the operator instead of letting a wrongly-typed operand silently coerce to 0 or no-op. Cap a content- stream array or dictionary operand's element count with the lexer alone, before PdfObjectParser ever materialises it, so a 20,000,000- element TJ array no longer allocates the whole array (and every boxed element in it) before the cap gets a chance to stop it. Reset a Form XObject's own BX/EX depth to 0 rather than inheriting the invoker's, since Table 33 scopes one compatibility section to one content stream. Check BX/EX's own arity like every other operator, without dropping the section transition on a mismatch. Move the "not an indirect reference" check in BuildPageContentBuffer ahead of the per-Run budget short-circuit, since it costs nothing to run either way. Correct the 300, 302, 305, 306, and 307 diagnostic docs to match what this interpreter reports: a lexer or parser failure ends interpretation for the rest of the page, not just the failing stream, unlike a resolve-or-decode failure; 302 now names the newly type-checked operators; 305 counts every Do that reaches a form, not only one that decodes; 306 covers a present-but-unusable XObject entry; and 307's inline-image callback is raised whenever the image was still delimited, including after an /L recovery or a probe-budget-exhausted acceptance. Fix a handful of clause and table citations off by one subclause (Tf, Tr, Ts, a form's own /Matrix default) or naming the wrong table (BMC's own Table 351/352 mixup, an invented Tc clause). Add a second fuzz generator that mutates the page's own content-stream body in place, since the existing file-wide mutator reaches Run on only 5% of its samples (most mutations land in the xref or trailer); this one reaches Run on nearly every sample instead. Bind the existing generator's own Sample call through a block-bodied lambda: the shared assertion helper now returns whether Run was reached (for the new generator's own majority check), and an expression-bodied lambda binds to Sample's Func overload by exact delegate-type match, reinterpreting the ~95% of samples that never reach Run as failing property cases instead of the acceptable outcome they are. --- .../Content/ContentInterpreter.cs | 597 +++++++++++++----- .../Content/ContentOperators.cs | 10 +- src/VellumPdf.Reader/Content/GraphicsState.cs | 20 +- .../Content/IContentVisitor.cs | 16 +- src/VellumPdf.Reader/Content/Matrix.cs | 4 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 93 ++- src/VellumPdf.Reader/ReaderLimits.cs | 5 +- .../ContentInterpreterTests.cs | 582 +++++++++++++++-- .../PdfLexerContentModeTests.cs | 19 + 9 files changed, 1073 insertions(+), 273 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 4ecd1de1..e8601e65 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -44,9 +44,15 @@ internal sealed class ContentInterpreter // 32 as this reader's own ceiling would reject a legal call, not just a hostile one. private const int MaxOperandsPerOperator = 64; - // §9.4.3's own TJ array holds a mix of strings and numeric adjustments; this reader's own - // ceiling on how many elements one such array may carry. - private const int MaxTjElements = 8192; + // This reader's own ceiling on how many depth-1 elements a single array or dictionary operand + // may carry (an array's own elements, or a dictionary's own keys and values counted together). + // §9.4.3's own TJ array is what most directly exercises it, but CompositeOperandWithinCap + // applies it to every composite operand this interpreter parses, not only TJ's, and enforces it + // BEFORE PdfObjectParser ever materialises the composite (#402 round 3): a 20,000,000-element TJ + // array (about 40 MiB of source text) used to allocate the whole PdfArray, and every boxed + // PdfInteger in it, before this cap was consulted at all, measured at 1,784 MiB allocated and + // 977 MiB committed for one dropped operator and one 309. + private const int MaxCompositeOperandElements = 8192; // §8.4.4's q/Q pair; this reader's own ceiling on how deep a legitimate document nests them. private const int MaxGraphicsStateDepth = 64; @@ -67,21 +73,19 @@ internal sealed class ContentInterpreter // can interpret far more total content than its own /Contents ever declares. private const long MaxContentBytes = 64L * 1024 * 1024; - // The bounded resync probe (see LooksLikeResyncPoint) lexes at most this many bytes, and at - // most ProbeTokens tokens, from each 'EI' candidate: bounding the work per candidate is what - // keeps ScanForEi linear in the content length rather than quadratic (a probe that lexed to - // the end of the buffer from every candidate cost O(N) per candidate, O(N^2) overall). - private const int ProbeWindowBytes = 128; - private const int ProbeTokens = 8; - - // The second, larger window LooksLikeResyncPoint re-probes with when a token runs off the - // clipped ProbeWindowBytes window: a legitimate token following a false 'EI' candidate (a long - // literal string, say) can run well past 128 bytes without being image data at all, so a - // candidate is rejected only once a token also runs off THIS window (#402 round 2; the - // previous single-window probe treated running off a clipped window as inconclusive and - // accepted the candidate either way, which let a sufficiently long unterminated token past the - // window mask a malformed stream). - private const int ExtendedProbeWindowBytes = 4096; + // The bounded resync probe (see ProbeOnce/LooksLikeResyncPoint) spends at most this many bytes + // of lexing across an ENTIRE Run, shared by every 'EI' candidate ScanForEi tries: bounding the + // RUN's own total work, not each candidate's, is what keeps ScanForEi's amortised cost linear in + // the content length rather than quadratic (#402 round 3; a per-candidate byte/token cap alone + // still let a probe reject the terminating 'EI' whose own legitimate follow-on token happened to be + // longer than the cap, and a two-window retry to fix THAT still paid, in the worst case, one + // full window's own lexing cost per false candidate: an adversarial run of false candidates + // each immediately followed by an unterminated string, "\" EI (\"" repeated, drove that cost to + // 16.6 s per decoded MiB with no diagnostic to explain it). Once this budget is spent, every + // later candidate in the Run is accepted unverified rather than probed at all (see + // ProbeOutcome.Exhausted): the total probe lexing one Run can ever do is bounded to this many + // bytes plus at most one further window, whatever the content size or candidate count. + private const long MaxProbeBytesPerRun = 16L * 1024 * 1024; private static readonly PdfName XObjectSubtypeForm = new("Form"); private static readonly PdfName XObjectSubtypeImage = new("Image"); @@ -112,6 +116,14 @@ internal sealed class ContentInterpreter private long _contentBytesRemaining; private ReadOnlyMemory _currentBuffer; + // The resync probe's own remaining share of MaxProbeBytesPerRun, and whether it has already + // been spent this Run. Once _probeBudgetExhausted is true, LooksLikeResyncPoint accepts every + // later 'EI' candidate without probing it at all (see ProbeOnce's Exhausted outcome), and + // HandleInlineImage reports that once, at the offset it first happened. + private long _probeBytesRemaining; + private bool _probeBudgetExhausted; + private int _probeBudgetExhaustedAtOffset; + // Pushes PushGraphicsState/PushMarkedContent dropped for being over MaxGraphicsStateDepth or // MaxMarkedContentDepth: a matching 'Q'/'EMC' consumes one of these before it may report an // unbalanced pop (see PopGraphicsState/PopMarkedContent), so a producer that legitimately @@ -151,6 +163,14 @@ internal sealed class ContentInterpreter /// internal int ContentStreamsDecoded { get; private set; } + /// + /// Test-only visibility into how many bytes the resync probe (ProbeOnce) has lexed this + /// , so a test can pin how the probe's own MaxProbeBytesPerRun budget + /// bounds its work directly, the same way pins the content + /// budget (#402 round 3). + /// + internal long ProbeBytesConsumed { get; private set; } + /// Creates an interpreter that resolves resources and streams through /// , under that reader's own . internal ContentInterpreter(PdfDocumentReader reader) @@ -189,6 +209,10 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _ignoredGsPushes = 0; _ignoredMcPushes = 0; ContentStreamsDecoded = 0; + _probeBytesRemaining = MaxProbeBytesPerRun; + _probeBudgetExhausted = false; + _probeBudgetExhaustedAtOffset = 0; + ProbeBytesConsumed = 0; var diagnostics = _reader.CreateContentDiagnosticScope(); var pageIndex = page.Index; @@ -249,9 +273,13 @@ private ReadOnlyMemory BuildPageContentBuffer( void AddElement(PdfObject element) { - if (budgetSpent) - return; - + // Checked ahead of the budgetSpent short-circuit below, unlike the resolve/decode work + // further down: a type check against an object already in hand costs nothing, so a + // /Contents element that is not even an indirect reference gets its own 300 report + // whether or not the budget already ran out on an earlier element, while resolving and + // decoding (costly work, gated on the budget) stay behind the short-circuit (#402 + // round 3; the 300 doc's own "resumes with the next stream" clause covers only the + // resolve-or-decode failure this short-circuit still gates). if (element is not PdfIndirectReference elementRef) { diagnostics.Report( @@ -262,6 +290,9 @@ void AddElement(PdfObject element) return; } + if (budgetSpent) + return; + var stream = _reader.ResolveStream(elementRef); if (stream is null) { @@ -488,14 +519,28 @@ private void InterpretStream( PushOperand(PdfObjectParser.ParseName(token), ctx, diagnostics, pageIndex); break; - case TokenKind.ArrayBegin: - lexer.Seek(offset); - PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); - break; - - case TokenKind.DictBegin: - lexer.Seek(offset); - PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); + case TokenKind.ArrayBegin or TokenKind.DictBegin: + // Pre-scanned with the lexer alone (no PdfObject allocation) before + // ParseObject ever materialises it: an adversarial TJ array can name + // millions of elements in a source text small enough to decode well + // within the content budget, so the element-count cap has to be + // consulted before the allocation it exists to bound, not after (#402 + // round 3; see CompositeOperandWithinCap and MaxCompositeOperandElements). + if (CompositeOperandWithinCap(lexer)) + { + lexer.Seek(offset); + PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); + } + else + { + _operandOverflow = true; + var shape = token.Kind == TokenKind.ArrayBegin ? "An array" : "A dictionary"; + diagnostics.Report( + PdfReaderDiagnosticCode.ContentLimitExceeded, + $"{shape} operand exceeds {MaxCompositeOperandElements} elements; " + + "the operator taking it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } break; case TokenKind.Keyword: @@ -637,6 +682,58 @@ private void ClearOperands() _operandOverflow = false; } + // Pre-scans an array or dictionary operand with the LEXER ALONE, starting right after its + // already-consumed opening token, to decide whether it stays within + // MaxCompositeOperandElements before PdfObjectParser ever allocates a PdfObject for it. Counts + // only DEPTH-1 tokens: the ones that are this composite's own direct elements (an array) or + // keys and values (a dictionary), the same items PdfArray.Count/PdfDictionary.Count would count + // once materialised. A nested array or dictionary counts once, at its own opening token, not + // once per token it itself contains. Leaves the lexer positioned right after the matching close + // either way, so the caller can either seek back to re-parse it (within cap) or simply continue + // with the next token (over cap: nothing more from this composite is needed). + private static bool CompositeOperandWithinCap(PdfLexer lexer) + { + var depth = 1; + var count = 0; + while (depth > 0) + { + Token token; + try + { + token = lexer.NextToken(); + } + catch (InvalidDataException) + { + // No matching close inside this composite: not this method's problem to diagnose. + // The caller seeks back to the opening token and lets ParseObject re-derive the + // same failure and report it (ContentStreamLexError, #300), the way it always has. + return true; + } + + if (token.Kind == TokenKind.EndOfInput) + return true; // Same: unterminated composite, ParseObject's own retry reports it. + + switch (token.Kind) + { + case TokenKind.ArrayBegin or TokenKind.DictBegin: + if (depth == 1) + count++; + depth++; + break; + + case TokenKind.ArrayEnd or TokenKind.DictEnd: + depth--; + break; + + default: + if (depth == 1) + count++; + break; + } + } + return count <= MaxCompositeOperandElements; + } + // ── Operator dispatch ──────────────────────────────────────────────────────────────────────── private void HandleOperator( @@ -673,6 +770,31 @@ private void HandleOperator( return; } + // BX/EX's own arity (0) is checked the same way as every other operator's below, unlike the + // dispatch this used to short-circuit through before any check ran at all: '1 2 3 BX 4 EX' + // used to hand the visitor BX with 3 leftover operands and EX with 1, reporting nothing even + // though _arity["BX"] is 0 (#402 round 3). A mismatch here still only drops the OPERANDS, + // not the operator itself: Table 33 opens or closes a compatibility section regardless of + // what garbage operands preceded BX/EX, so the section transition below still runs either + // way, unlike an ordinary operator's mismatch, which drops the whole call. + var expected = ContentOperators.ExpectedOperandCount(name); + var arityOk = true; + if (_operandOverflow) + { + arityOk = false; + ClearOperands(); // Already reported when the overflow itself happened. + } + else if (expected != ContentOperators.Variable && _operands.Count != expected) + { + arityOk = false; + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"'{name}' expects {expected} operand(s) but {_operands.Count} were on the stack; " + + "it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + } + if (name is "BX") { _bxDepth++; @@ -687,22 +809,8 @@ private void HandleOperator( return; } - var expected = ContentOperators.ExpectedOperandCount(name); - if (_operandOverflow) - { - ClearOperands(); - return; // Already reported when the overflow itself happened. - } - if (expected != ContentOperators.Variable && _operands.Count != expected) - { - diagnostics.Report( - PdfReaderDiagnosticCode.OperandStackMalformed, - $"'{name}' expects {expected} operand(s) but {_operands.Count} were on the stack; " - + "it was dropped.", - ctx.DiagObjectNumber, pageIndex: pageIndex); - ClearOperands(); + if (!arityOk) return; - } // ISO 32000-2 §7.8.2: "Dictionaries shall be permitted as operands only by certain specific // operators". BDC and DP (§14.6.2) are the only two in Annex A Table A.1 that take one. @@ -722,10 +830,20 @@ private void HandleOperator( } } + // A numeric or name operand this interpreter reads for its OWN state (cm/Tf/Td/etc. below) + // used to be handed to NumberOperand or a bare pattern match unchecked: a wrongly typed + // operand at the right arity silently substituted 0, or (for Do) silently dropped the + // invocation, with no diagnostic at all (#402 round 3). An operator this interpreter only + // forwards to the visitor untouched (w, J, the colour operators, ...) is exempt: its own + // operand types are the visitor's to type-check, not this interpreter's, since this + // interpreter never reads them for its own state. + if (!ValidateOperandTypes(name, ctx, diagnostics, pageIndex)) + return; + switch (name) { case "TJ": - if (_operands[0] is not PdfArray tjArray) + if (_operands[0] is not PdfArray) { diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, @@ -734,15 +852,6 @@ private void HandleOperator( ClearOperands(); return; } - if (tjArray.Count > MaxTjElements) - { - diagnostics.Report( - PdfReaderDiagnosticCode.ContentLimitExceeded, - $"TJ's array operand exceeds {MaxTjElements} elements; it was dropped.", - ctx.DiagObjectNumber, pageIndex: pageIndex); - ClearOperands(); - return; - } break; case "q": @@ -864,6 +973,59 @@ private void EmitAndClear(string name, int offset, IContentVisitor visitor) _ => 0, }; + private static bool IsNumericOperand(PdfObject obj) => obj is PdfInteger or PdfReal; + + // Type-checks the operands of every operator this interpreter reads for its own graphics or + // text state, ahead of the switch below that reads them: by the time this runs, the + // arity check above already guarantees _operands.Count matches each name's own Table A.1 arity, + // so only the TYPE of each operand is in question here. A mismatch reports + // OperandStackMalformed (the same code the 302 doc already covers "an operand of the wrong type + // where the arity is otherwise right" under) and drops the operator entirely, the way the + // dictionary-operand check just above does, rather than letting NumberOperand's own 0-default + // or a silent Do no-op mask the malformation (#402 round 3). + private bool ValidateOperandTypes(string name, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + bool ok; + switch (name) + { + case "cm" or "Tm": + ok = IsNumericOperand(_operands[0]) && IsNumericOperand(_operands[1]) + && IsNumericOperand(_operands[2]) && IsNumericOperand(_operands[3]) + && IsNumericOperand(_operands[4]) && IsNumericOperand(_operands[5]); + break; + + case "Tc" or "Tw" or "Tz" or "TL" or "Tr" or "Ts": + ok = IsNumericOperand(_operands[0]); + break; + + case "Td" or "TD": + ok = IsNumericOperand(_operands[0]) && IsNumericOperand(_operands[1]); + break; + + case "Tf": + ok = _operands[0] is PdfName && IsNumericOperand(_operands[1]); + break; + + case "Do": + ok = _operands[0] is PdfName; + break; + + default: + return true; // Forwarded to the visitor untouched; not this interpreter's to check. + } + + if (ok) + return true; + + diagnostics.Report( + PdfReaderDiagnosticCode.OperandStackMalformed, + $"'{name}' operand is the wrong type for the arity ISO 32000-2 Annex A Table A.1 gives " + + "it; it was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + ClearOperands(); + return false; + } + // ── q/Q, BMC/BDC/EMC ───────────────────────────────────────────────────────────────────────── private void PushGraphicsState(StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) @@ -1058,9 +1220,10 @@ private void HandleDo( if (_inTextObject) { // §8.2 Figure 9 admits only the general graphics state, colour, text state, - // text-positioning, text-showing and marked-content categories of Table 50 inside a - // text object; 'Do' sits in the XObjects category, so a producer that invokes it - // there is wrong regardless of what the named XObject turns out to be. The + // text-positioning, text-showing, marked-content, and compatibility categories of + // Table 50 inside a text object (seven, not six: #402 round 3); 'Do' sits in the + // XObjects category, so a producer that invokes it there is wrong regardless of what + // the named XObject turns out to be. The // recursion below still runs (a text-object violation is not itself a reason to skip // an otherwise-resolvable Form), but the shared _textState instance a form's own // content may disturb (BT/Td/ET, unbracketed by any q/Q-style save) is saved and @@ -1170,6 +1333,10 @@ private void HandleDo( } _openForms.Add(objectNumber); + // Counts every 'Do' that reaches this point, i.e. every invocation past the recursion + // guards above, not only one whose content goes on to decode successfully (the 305 doc + // says so; #402 round 3 fixed the doc to match, since this counter itself already counted + // this way from the start). _formInvocations++; _formDepth++; try @@ -1281,9 +1448,20 @@ private void HandleDo( _gsFloor = savedGsStackCount; _markedContentFloor = savedMarkedContentDepth; - _bxFloor = savedBxDepth; _ignoredGsPushes = 0; _ignoredMcPushes = 0; + // Unlike the graphics-state and marked-content floors just above, BX/EX depth + // does not carry the invoker's own state into the form's content: Table 33 + // scopes one compatibility section to ONE content stream, and a form is its own + // content stream, so it has nothing of the invoker's BX/EX nesting to inherit or + // protect against. Both start at 0 here rather than at the invoker's current + // depth (#402 round 3; without this, a form invoked from inside 'BX ... Do ... + // EX' inherited _bxDepth > 0 from the invoker, which made the form's own unknown + // operators look like they were still inside the INVOKER's compatibility + // section and silently swallowed them, even though the same form invoked as a + // bare 'Do' correctly reported them). + _bxDepth = 0; + _bxFloor = 0; // The form's own content starts outside any text object whatever the invoker // was doing: its 'Do' is judged against its own BT/ET, and its ET must not // close the invoker's text object once Do returns. @@ -1464,24 +1642,33 @@ private bool HandleInlineImage( var hasDisallowedFilter = filterNames.Any(f => f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); - // §8.9.7: "Unless the image uses ASCIIHexDecode or ASCII85Decode as one of its filters, the - // ID operator shall be followed by a single white-space character, and the next character - // shall be interpreted as the first byte of image data." "As ONE OF its filters" (not - // merely "as its final filter", the narrower phrasing NOTE 2 below happens to use for its - // own skip-without-decoding shortcut) is what decides this: a filter array may name either - // one anywhere, not only last. NOTE 2: "if the final or only filter is ASCIIHexDecode or - // ASCII85Decode skip any further white-space [after the first]" before counting /L's own - // bytes; applied here to "as one of its filters" (matching the normative sentence, not - // NOTE 2's narrower "final or only") for the same reason this reader treats every position - // in a /Filter array as eligible elsewhere (CollectFilterNames itself does not distinguish - // position either). Before this fix, at most one whitespace byte was consumed for every - // filter shape, so 'BI /F /A85 /L 48 ID <48-byte payload> EI' (two spaces after ID) came - // out 3 bytes short: the fixed-at-one skip left the payload's own second byte behind as - // data, and the rest re-lexed as content instead. §7.2.3: "The combination of a CARRIAGE - // RETURN followed immediately by a LINE FEED shall be treated as one EOL marker", so a CR - // immediately followed by an LF is consumed as that ONE separator, not as the separator - // plus a data byte, before any of the above. - var skipsExtraWhitespace = filterNames.Any(f => f.Value is "ASCIIHexDecode" or "ASCII85Decode"); + // §8.9.7's normative sentence excludes ASCIIHexDecode/ASCII85Decode "as one of its filters" + // from the single-white-space rule; NOTE 2 gives the actual skip-without-decoding recipe, + // scoped narrower, to "the final or only filter": "if the final or only filter is + // ASCIIHexDecode or ASCII85Decode skip any further white-space [after the first]" before + // counting /L's own bytes. NOTE 2's own skip only ever touches the raw bytes right after + // ID, so its "final" filter can only mean final in ENCODING order: the filter applied LAST + // when the data was written, and therefore the FIRST one a decoder strips. §7.4.1's own + // EXAMPLE 2 fixes what "order" means for a /Filter array: data "encoded using LZW and ASCII + // base-85 encoding (in that order)" (LZW applied first, A85 applied last) decodes through + // "/Filter [/ASCII85Decode /LZWDecode]", A85 named FIRST, because the array is written in + // DECODE order and A85 is what strips the literal bytes first. So NOTE 2's "final" filter is + // always array position 0 (or the sole name), never any later position (#402 round 3: an + // earlier version skipped whenever AHx/A85 appeared ANYWHERE in the array, which corrupts + // data when a different filter is the one reading the raw bytes: '/F [/Fl /A85]' names A85 + // second, so FlateDecode owns the raw bytes, and skipping past the payload's own leading + // byte as though A85 owned that position ate it). hasDisallowedFilter above stays + // position-independent on purpose: the normative sentence's own "as one of its filters" + // is a different question (any of JBIG2Decode/JPXDecode/Crypt at any position is forbidden) + // from which filter reads the raw bytes first. Before the round-2 fix, at most one + // whitespace byte was consumed for every filter shape, so 'BI /F /A85 /L 48 ID <48-byte + // payload> EI' (two spaces after ID) came out 3 bytes short: the fixed-at-one skip left the + // payload's own second byte behind as data, and the rest re-lexed as content instead. + // §7.2.3: "The combination of a CARRIAGE RETURN followed immediately by a LINE FEED shall be + // treated as one EOL marker", so a CR immediately followed by an LF is consumed as that ONE + // separator, not as the separator plus a data byte, before any of the above. + var skipsExtraWhitespace = filterNames.Count > 0 + && filterNames[0].Value is "ASCIIHexDecode" or "ASCII85Decode"; var consumedCrLf = false; if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 && PdfLexer.IsWhitespaceByte((byte)separatorByte)) @@ -1524,6 +1711,7 @@ private bool HandleInlineImage( var scanEnd = ScanForEi(dataStart); if (scanEnd is null) { + ReportProbeBudgetExhaustedIfNeeded(ctx, diagnostics, pageIndex); ReportInlineImageMalformed( "no 'EI' operator delimiting the image data could be found", ctx, diagnostics, pageIndex); @@ -1533,15 +1721,13 @@ private bool HandleInlineImage( lengthFromScan = true; } + // No bounds check needed here: tier a (TryLengthFromDictionary) already rejects a length + // running past the buffer as pastEnd, tier b (TryComputeUnfilteredLength) returns null on + // the same overrun, and tier c (ScanForEi) can only ever return an offset it found by + // scanning forward from dataStart within the buffer. Every path into `length` already + // guarantees dataStart + length.Value falls inside [dataStart, _currentBuffer.Length] + // (#402 round 3: the equivalent check here was dead code no path could reach). var dataEnd = dataStart + length.Value; - if (dataEnd < dataStart || dataEnd > _currentBuffer.Length) - { - ReportInlineImageMalformed( - "the computed image data length runs past the end of the content stream", ctx, - diagnostics, pageIndex); - return false; - } - var data = _currentBuffer.Slice(dataStart, length.Value); var resyncPos = SkipToEi(dataEnd); @@ -1603,6 +1789,7 @@ private bool HandleInlineImage( if (resyncPos is null) { + ReportProbeBudgetExhaustedIfNeeded(ctx, diagnostics, pageIndex); ReportInlineImageMalformed( "no 'EI' operator was found at the computed end of the image data", ctx, diagnostics, pageIndex); @@ -1610,6 +1797,7 @@ private bool HandleInlineImage( } lexer.Seek(resyncPos.Value); + ReportProbeBudgetExhaustedIfNeeded(ctx, diagnostics, pageIndex); if (hasDisallowedFilter) { @@ -1623,6 +1811,23 @@ private bool HandleInlineImage( return true; } + // Reports, once per Run (the sink's own (code, object, page) dedupe folds every later call), + // that the resync probe's own MaxProbeBytesPerRun budget ran out before a candidate 'EI' could + // be confirmed, so it (and every later candidate this Run) was accepted unverified instead + // (#402 round 3; see ProbeOnce's Exhausted outcome and LooksLikeResyncPoint). + private void ReportProbeBudgetExhaustedIfNeeded( + StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) + { + if (!_probeBudgetExhausted) + return; + + ReportInlineImageMalformed( + $"the resync probe's {MaxProbeBytesPerRun / (1024 * 1024)} MiB per-run byte budget was " + + $"spent before the candidate 'EI' at offset {_probeBudgetExhaustedAtOffset} could be " + + "confirmed; it, and every later candidate this run, was accepted without verification", + ctx, diagnostics, pageIndex); + } + private void ReportInlineImageMalformed( string reason, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) => diagnostics.Report( @@ -1657,16 +1862,20 @@ private List CollectFilterNames(PdfDictionary dict) pastEnd = false; var lengthRaw = dict.Get(PdfName.Length); if (lengthRaw is null) - return null; // Table 91: /L is optional; falls through to tier b or the EI scan. + // §8.9.7 NOTE 1: Length is new to PDF 2.0, so an older file will not carry the key at + // all; that is tolerated here by falling through to tier b or the EI scan, not treated + // as itself a malformation the way a PRESENT-but-wrong-typed /L is just below. + return null; if (lengthRaw is not PdfInteger lengthObj) { // Present but the wrong type (a PdfReal, say): reported the same way an invalid /W, // /H, or /BPC is (#402 round 2), rather than silently falling through to tier b/c as - // if /L had never been written at all. + // if /L had never been written at all. This branch is only reachable once /L IS + // present (the null check above already handled "absent"), so the message names what + // was found wrong with it rather than repeating "missing" (#402 round 3). ReportInlineImageMalformed( - "'/L' is missing, or carries an invalid, non-integer value", ctx, diagnostics, - pageIndex); + "'/L' is present but its value is not an integer", ctx, diagnostics, pageIndex); return null; } @@ -1780,7 +1989,13 @@ private int ResolveComponentCount( } // Tier (c): scan for whitespace-EI-whitespace/EOF, accepted only when the bounded probe just - // past the candidate (LooksLikeResyncPoint) does not reject it. + // past the candidate (LooksLikeResyncPoint) does not reject it. Residual gap this scan cannot + // close on its own (stated here and in the PR body, not fixed: closing it needs decoding the + // image data itself, out of scope for a byte-level scan): a '%' comment after a false 'EI' that + // runs to the end of its line can swallow the terminating 'EI' written on that same line, so + // the operator that follows the comment's own line accepts the false candidate with no + // diagnostic. The bytes are well-formed content either way, so this scan has no way to tell + // the two apart. private int? ScanForEi(int dataStart) { var span = _currentBuffer.Span; @@ -1802,7 +2017,18 @@ private int ResolveComponentCount( if (!LooksLikeResyncPoint(after)) continue; - var dataEnd = i > dataStart && PdfLexer.IsWhitespaceByte(span[i - 1]) ? i - 1 : i; + // Strips the single white-space byte §8.9.7 excludes from the image data (the one + // delimiting 'EI'). §7.2.3 makes a CARRIAGE RETURN immediately followed by a LINE FEED + // ONE EOL marker, not two separate white-space bytes, so when that pair sits right + // before 'EI' both bytes are the delimiter, not just the LF (#402 round 3: stripping + // only the LF left the CR behind as the image's own trailing byte). + var dataEnd = i; + if (i > dataStart && PdfLexer.IsWhitespaceByte(span[i - 1])) + { + dataEnd = i - 1; + if (dataEnd > dataStart && span[dataEnd] == (byte)'\n' && span[dataEnd - 1] == (byte)'\r') + dataEnd--; + } return dataEnd; } return null; @@ -1822,47 +2048,59 @@ private int ResolveComponentCount( return pos + 2; } - // Bounded lookahead: lexes at most ProbeTokens tokens from a candidate resync point, over at - // most ProbeWindowBytes bytes (ExtendedProbeWindowBytes on the retry, see LooksLikeResyncPoint - // below), and accepts unless one of those tokens is a keyword this probe cannot justify as - // legitimate content-stream syntax. Bounding both the token count and the byte window is what - // keeps ScanForEi linear in the content length: an earlier version of this probe lexed all the - // way to the end of the buffer from every candidate, which cost O(N) work per candidate and - // made a content stream built from many false 'EI' candidates (a filtered image followed by - // literal " EI (" text repeated many times, say) quadratic overall. - // - // A non-keyword token (a number, name, string, array or dictionary delimiter, whatever its - // bytes) is neutral: it neither accepts nor rejects the candidate on its own. A keyword is - // accepted when it is a Table A.1 operator (or true/false/null, or the one-byte content-mode - // keywords '{', '}', '>' this lexer's own content-stream mode produces), and otherwise accepted - // only when every one of its bytes is printable ASCII (0x21 to 0x7E): an unknown-but-printable - // keyword is the kind of thing §7.8.2 already tolerates outside a compatibility section (a - // future operator this reader does not know yet, a stray "R"), while a byte run containing - // anything else is binary noise a coincidental "EI" byte pair inside DCT- or JPX-compressed - // data would otherwise be mistaken for legitimate syntax. A 'BI' keyword ends the probe with - // acceptance immediately, without lexing further: the bytes after ITS OWN following 'ID' are - // raw image data and must not be judged as tokens at all. - private enum ProbeOutcome { Accept, Reject, RanOffClippedWindow } - - private ProbeOutcome ProbeOnce(int pos, int windowBytes) + // Lexes forward from a candidate resync point until it finds a positive reason to accept or + // reject, or exhausts this Run's own MaxProbeBytesPerRun budget (#402 round 3 redesign; replaces + // the round-2 two-window, token-capped probe, which let every false candidate in a long run of + // them each pay a full window's own lexing cost, driving the " EI (" repeated-candidate shape to + // 16.6 s per decoded MiB, and separately let an unterminated token that merely ran off the + // second, larger window mask the terminating 'EI' whose own legitimate follow-on token + // happened to be longer still). A candidate needs a POSITIVE reason to accept now, not merely + // the absence of a rejecting keyword: a number, name, string, array/dictionary delimiter, + // true/false/null, or an + // unknown-but-printable keyword is neutral and keeps the probe lexing rather than accepting by + // default, closing the round-2 gap where a straddling but well-formed array or dictionary (never + // itself a keyword) produced the identical wrong "accept" outcome an unterminated string used to. + private enum ProbeOutcome + { + /// A Table A.1 operator other than 'EI'/'ID', 'BI', or the buffer's own true end + /// (not merely this probe's budget) was reached. + Accept, + + /// An 'EI' or 'ID' keyword (still inside image data that continues to a LATER + /// 'EI'), a keyword containing a non-printable byte, or a lex failure that was not merely + /// this probe running out of budget. + Reject, + + /// The probe's own share of MaxProbeBytesPerRun ran out before it reached an + /// Accept or Reject outcome. Treated as an accept by LooksLikeResyncPoint (see its own + /// remarks), but distinctly, so HandleInlineImage can report that this candidate was + /// accepted unverified rather than confirmed. + Exhausted, + } + + private ProbeOutcome ProbeOnce(int pos) { - // Whether this window is itself an artificial cap, i.e. more buffer exists beyond it that - // this probe deliberately does not look at. Only THAT case makes "ran off the window" - // inconclusive rather than an outright rejection: a short buffer where the window reaches - // the buffer's own true end behaves exactly like the unbounded lexer this probe replaces - // (a token that never closes anywhere is malformed, full stop), so it must still reject - // there. Matters for a short fixture, or a content stream that stops mid-token, where the - // window and "the rest of the buffer" happen to coincide. var remaining = _currentBuffer.Length - pos; - var windowClipped = remaining > windowBytes; - var windowLength = Math.Min(windowBytes, remaining); + var windowLength = (int)Math.Min(remaining, _probeBytesRemaining); + // Whether this window is itself an artificial cap, i.e. more buffer exists beyond it that + // the probe's own remaining budget deliberately does not look at. Only THAT case makes + // reaching the window's own end inconclusive (Exhausted) rather than an outright Accept or + // Reject: a window that reaches the buffer's own true end behaves exactly like the + // unbounded lexer this probe conceptually stands in for (a token that never closes anywhere + // is malformed, full stop; the buffer's own end with nothing pending IS a legitimate resync + // point), so those still resolve outright rather than staying inconclusive. + var windowClipped = windowLength < remaining; var window = _currentBuffer.Slice(pos, windowLength); var probe = new PdfLexer(window, contentStreamMode: true); + ProbeOutcome outcome; - for (var i = 0; i < ProbeTokens; i++) + while (true) { if (probe.AtEnd) - return ProbeOutcome.Accept; + { + outcome = windowClipped ? ProbeOutcome.Exhausted : ProbeOutcome.Accept; + break; + } Token token; try @@ -1871,60 +2109,101 @@ private ProbeOutcome ProbeOnce(int pos, int windowBytes) } catch (InvalidDataException) { - // Ran off the end of the window mid-token (an unterminated literal or hex string - // straddling the boundary): inconclusive, since PdfLexer's own string readers - // advance Position as they go, so Position sitting at or past the window's own - // length here means the failure was purely the window's own limit, not malformed - // bytes the probe saw, PROVIDED the window was clipped in the first place (see - // above). Anything else, including running off a window that already reached the - // buffer's own true end, rejects instead. - return windowClipped && probe.Position >= window.Length - ? ProbeOutcome.RanOffClippedWindow + // Ran off the end of the window mid-token (an unterminated literal or hex string, + // say): Exhausted only when that running-off was purely the budget's own limit + // (window clipped, and the lexer's own Position landed at or past the window's own + // length trying to close the token); anything else, including a malformed byte + // found strictly inside an unclipped window, rejects outright. + outcome = windowClipped && probe.Position >= window.Length + ? ProbeOutcome.Exhausted : ProbeOutcome.Reject; + break; } if (token.Kind == TokenKind.EndOfInput) - return ProbeOutcome.Accept; + { + outcome = windowClipped ? ProbeOutcome.Exhausted : ProbeOutcome.Accept; + break; + } + if (token.Kind != TokenKind.Keyword) - continue; + continue; // Neutral: a number, name, string, or array/dictionary delimiter. var raw = token.Raw.Span; + if (raw.SequenceEqual("EI"u8) || raw.SequenceEqual("ID"u8)) + { + // Still inside image data that continues to a LATER 'EI': this candidate is a false + // one, not a resync point. + outcome = ProbeOutcome.Reject; + break; + } if (raw.SequenceEqual("BI"u8)) - return ProbeOutcome.Accept; + { + // The bytes after ITS OWN following 'ID' are raw image data and must not be judged + // as tokens at all, so the probe stops here rather than lexing into them. + outcome = ProbeOutcome.Accept; + break; + } if (raw.SequenceEqual("true"u8) || raw.SequenceEqual("false"u8) || raw.SequenceEqual("null"u8)) - continue; - if (raw.Length == 1 && (raw[0] == (byte)'{' || raw[0] == (byte)'}' || raw[0] == (byte)'>')) - continue; + continue; // Neutral. + if (raw.Length == 1 && raw[0] is (byte)'{' or (byte)'}' or (byte)'>') + continue; // Neutral: this lexer's own one-byte content-mode keywords. if (ContentOperators.IsKnown(raw)) - continue; + { + outcome = ProbeOutcome.Accept; + break; + } + var hasNonPrintableByte = false; foreach (var b in raw) { if (b is < (byte)'!' or > (byte)'~') - return ProbeOutcome.Reject; + { + hasNonPrintableByte = true; + break; + } } + if (hasNonPrintableByte) + { + // Binary noise a coincidental "EI" byte pair inside DCT- or JPX-compressed data + // would otherwise be mistaken for legitimate syntax. + outcome = ProbeOutcome.Reject; + break; + } + // Neutral: an unknown-but-printable keyword, the kind of thing §7.8.2 already tolerates + // outside a compatibility section (a future operator this reader does not know yet, a + // stray "R"). } - return ProbeOutcome.Accept; + + // Charges what this probe spent: the window's own length when it ran out of budget before + // resolving (Exhausted), or however far the lexer got otherwise. A window of length 0 + // (the budget already fully spent when this call started) charges nothing more and + // resolves to Exhausted immediately, which is what makes every candidate after the budget + // runs out cost nothing to probe. + var charged = outcome == ProbeOutcome.Exhausted ? windowLength : Math.Min(probe.Position, windowLength); + _probeBytesRemaining -= charged; + if (_probeBytesRemaining < 0) + _probeBytesRemaining = 0; + ProbeBytesConsumed += charged; + return outcome; } private bool LooksLikeResyncPoint(int pos) { - var first = ProbeOnce(pos, ProbeWindowBytes); - if (first != ProbeOutcome.RanOffClippedWindow) - return first == ProbeOutcome.Accept; - - // A token ran off the clipped ProbeWindowBytes window: re-probe once with the much larger - // ExtendedProbeWindowBytes window before accepting OR rejecting, since a legitimate token - // following a false 'EI' candidate (a long literal string, say) can easily run past 128 - // bytes without being image data at all. Accepting on the first window's own inconclusive - // result alone let a sufficiently long unterminated token mask a malformed stream, since - // "ran off a clipped window" and "the token never closes" are indistinguishable from a - // 128-byte window alone (#402 round 2: a DCT image without /L whose data contained an - // unclosed literal string straddling the 128-byte mark, followed much later by the real - // 'EI', reported ContentStreamLexError instead of InlineImageMalformed once the straddle - // point crossed the window, and lost the trailing 'Q' to the caller's visitor). If a token - // also runs off this larger window, the candidate is rejected outright rather than accepted - // a second time. - return ProbeOnce(pos, ExtendedProbeWindowBytes) == ProbeOutcome.Accept; + var outcome = ProbeOnce(pos); + if (outcome == ProbeOutcome.Exhausted) + { + // Accepted unverified, once and for the rest of this Run: HandleInlineImage reports + // this the first time it happens, naming the offset accepted without verification, so a + // caller can tell "the interpreter confirmed this resync point" apart from "the + // interpreter ran out of budget and took its best guess" (#402 round 3). + if (!_probeBudgetExhausted) + { + _probeBudgetExhausted = true; + _probeBudgetExhaustedAtOffset = pos; + } + return true; + } + return outcome == ProbeOutcome.Accept; } } diff --git a/src/VellumPdf.Reader/Content/ContentOperators.cs b/src/VellumPdf.Reader/Content/ContentOperators.cs index d31d6705..868688ba 100644 --- a/src/VellumPdf.Reader/Content/ContentOperators.cs +++ b/src/VellumPdf.Reader/Content/ContentOperators.cs @@ -9,8 +9,8 @@ namespace VellumPdf.Reader.Content; /// marks the four colour operators (SC, sc, SCN, /// scn): Table 73 (§8.6.8) gives them one numeric operand per colourant of the current /// colour space, plus, for the N suffix, an optional trailing pattern name, and §8.6.6.5 -/// lets a DeviceN space name "an arbitrary number" of colourants, so no single fixed count (nor -/// even a small fixed range) describes them; this interpreter accepts any operand count for them +/// lets a DeviceN space name "an arbitrary number of colour components", so no single fixed count +/// (nor even a small fixed range) describes them; this interpreter accepts any operand count for them /// rather than reporting on a /// legitimately variable call. /// @@ -39,8 +39,10 @@ internal static class ContentOperators // Table 352: marked-content operators. Annex A Table A.1's own cross-reference for BMC // names Table 351 ("Entries in a data dictionary") instead, an error in the standard: BDC, - // DP, EMC, and MP all cite Table 352 the way BMC itself should, and Table 351 is the - // unrelated marked-content PROPERTY LIST shape, not the operator table). + // DP, EMC, and MP all cite Table 352 the way BMC itself should. Table 351 belongs to + // §14.5's page-piece dictionaries (page-piece private data: LastModified, Private) and has + // nothing to do with marked content; a marked-content property list (§14.6.2) is described + // in prose and has no table of its own (#402 round 3). ["BDC"] = 2, ["BMC"] = 1, ["DP"] = 2, diff --git a/src/VellumPdf.Reader/Content/GraphicsState.cs b/src/VellumPdf.Reader/Content/GraphicsState.cs index 49783d5b..d4292feb 100644 --- a/src/VellumPdf.Reader/Content/GraphicsState.cs +++ b/src/VellumPdf.Reader/Content/GraphicsState.cs @@ -20,8 +20,8 @@ internal sealed class GraphicsState /// The current transformation matrix, concatenated by cm (§8.3.4). internal Matrix Ctm { get; set; } = Matrix.Identity; - /// Character spacing, Tc (§9.3.2). Added to the horizontal displacement of every - /// glyph shown, including the byte that follows a stretch of encoded space. + /// Character spacing, Tc (§9.3.2). Added to the horizontal or vertical + /// component of each glyph's displacement, depending on the writing mode. internal double CharSpacing { get; set; } /// Word spacing, Tw (§9.3.3). Added only after a single-byte code 32. @@ -35,12 +35,12 @@ internal sealed class GraphicsState internal double Leading { get; set; } /// - /// The font operand from the last Tf or gs-with-/Font (§9.3.6, §8.4.5 Table - /// 57): a naming a /Resources /Font entry for Tf, or the - /// an ExtGState's own /Font array names as its font directly (Table - /// 57 requires that to be an indirect reference to a font dictionary) for gs. Not resolved - /// by this interpreter either way: font resolution and glyph positioning are for a later caller - /// to do, not this interpreter's own job. + /// The font operand from the last Tf or gs-with-/Font (§9.3.1 Table 103, + /// §8.4.5 Table 57): a naming a /Resources /Font entry for + /// Tf, or the an ExtGState's own /Font array names as its + /// font directly (Table 57 requires that to be an indirect reference to a font dictionary) for + /// gs. Not resolved by this interpreter either way: font resolution and glyph + /// positioning are for a later caller to do, not this interpreter's own job. /// internal PdfObject? Font { get; set; } @@ -48,10 +48,10 @@ internal sealed class GraphicsState /// that set . internal double FontSize { get; set; } - /// Text rendering mode, Tr (§9.3.7): 0–7 per Table 104, not validated here. + /// Text rendering mode, Tr (§9.3.6): 0–7 per Table 104, not validated here. internal int RenderMode { get; set; } - /// Text rise, Ts (§9.3.8): vertical displacement, in unscaled text-space units. + /// Text rise, Ts (§9.3.7): vertical displacement, in unscaled text-space units. internal double Rise { get; set; } /// Deep-copies this state for a q push; Q discards the top and restores diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs index 69cf9b41..c615a89a 100644 --- a/src/VellumPdf.Reader/Content/IContentVisitor.cs +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -51,12 +51,16 @@ internal interface IContentVisitor void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset); /// - /// Called immediately before the interpreter recurses into a Form XObject's own content, once a - /// Do operator (already reported through ) has been resolved to a - /// /Subtype /Form stream and every recursion guard (depth, cycle, per-page budget) has - /// passed. Matched by exactly one call once the form's own content - /// finishes interpreting, even if that content raises further nested - /// calls of its own in between. + /// Called once a Do operator (already reported through ) has + /// been resolved to a /Subtype /Form stream and every recursion guard (depth, cycle, + /// per-page budget) has passed, raised for every Do that reaches that point, including + /// one whose content then fails to decode or is skipped for this run's own content budget + /// (#402 round 3: an earlier version of this doc said the call happens only immediately before + /// the interpreter recurses into the form's own content, which is false; the decode + /// itself, and the budget check ahead of it, both happen AFTER this callback, not before it). + /// Matched by exactly one call once the form's own content finishes + /// interpreting (or is skipped, in the cases above), even if that content raises further + /// nested calls of its own in between. /// /// The form XObject's own stream dictionary. /// The form's /Matrix (ISO 32000-2 §8.10.2 Table 93), or diff --git a/src/VellumPdf.Reader/Content/Matrix.cs b/src/VellumPdf.Reader/Content/Matrix.cs index 598e7c6c..b38e53a4 100644 --- a/src/VellumPdf.Reader/Content/Matrix.cs +++ b/src/VellumPdf.Reader/Content/Matrix.cs @@ -11,8 +11,8 @@ namespace VellumPdf.Reader.Content; /// internal readonly record struct Matrix(double A, double B, double C, double D, double E, double F) { - /// The identity matrix [1 0 0 1 0 0]: §8.3.4's own default for /Matrix - /// and the CTM at the start of every content stream. + /// The identity matrix [1 0 0 1 0 0]: the CTM at the start of every content + /// stream, and Table 93's own default for a form's /Matrix (§8.10.2). internal static readonly Matrix Identity = new(1, 0, 0, 1, 0, 0); /// diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 19205761..919971e0 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -357,10 +357,15 @@ public enum PdfReaderDiagnosticCode /// /Contents array (§7.7.3.3 Table 31) did not resolve to a usable stream at all: a /// non-stream element, a reference that fails to resolve, or a stream whose filter chain could /// not be decoded (including one carrying an image filter, which this interpreter never - /// attempts to decode as content). Interpretation of that stream stops at the point of failure; - /// operators already reported to the caller's visitor before the failure are kept, and, for a - /// multi-stream /Contents array specifically, interpretation resumes with the next - /// stream in the array rather than abandoning the whole page. + /// attempts to decode as content). Operators already reported to the caller's visitor before + /// the failure are kept either way, but the two cases end a different amount of the page's + /// content (#402 round 3): an element that fails to resolve or decode is simply skipped, and + /// interpretation resumes with the NEXT stream in a multi-stream /Contents array, since + /// every element still contributes its own decoded bytes to one buffer built from the whole + /// array before interpretation of any of it begins. A lexer or parser failure happens INSIDE + /// that already-concatenated buffer, so it ends interpretation for the rest of the page's + /// content outright, later /Contents array elements included, not merely the one + /// stream the failure happened to fall in. /// ContentStreamLexError = 300, @@ -389,13 +394,18 @@ public enum PdfReaderDiagnosticCode /// dictionary operand on an operator other than BDC/DP (§7.8.2: "Dictionaries /// shall be permitted as operands only by certain specific operators"), a known operator /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an operand of - /// the wrong type where the arity is otherwise right (a TJ whose single operand is not - /// an array, §9.4.3), a Do that occurred inside a text object (§8.2 Figure 9 admits no - /// operator of Table 50's XObjects category there), an unbalanced Q with - /// no matching q on the graphics-state stack, or an unbalanced EMC with no - /// matching BMC/BDC (§14.6.1). An unbalanced q still open at the end of a - /// content stream is not reported: nothing downstream of this interpreter needs the graphics - /// state restored past the last operator it saw. + /// the wrong type where the arity is otherwise right: a TJ whose single operand is not + /// an array (§9.4.3); a non-numeric operand to cm, Tc, Tw, Tz, + /// TL, Tf's second, Tr, Ts, Td, TD, or Tm; or a + /// non-name operand to Tf's first or to Do (#402 round 3; every OTHER operator + /// this interpreter recognises only forwards its own operands to the visitor untouched, so + /// their operand types are the visitor's to type-check, not this interpreter's); a Do + /// that occurred inside a text object (§8.2 Figure 9 admits no operator of Table 50's XObjects + /// category there), an unbalanced Q with no matching q on the graphics-state + /// stack, or an unbalanced EMC with no matching BMC/BDC (§14.6.1). An + /// unbalanced q still open at the end of a content stream is not reported: nothing + /// downstream of this interpreter needs the graphics state restored past the last operator it + /// saw. /// OperandStackMalformed = 302, @@ -418,13 +428,19 @@ public enum PdfReaderDiagnosticCode FormXObjectCycle = 304, /// - /// A single page invoked Form XObjects (successful Do recursions, counted across the - /// whole page, not per subtree) more than 4096 times. Descent into any further form stops for - /// the rest of the page; operators already reported before the budget was reached are kept, and - /// interpretation of the page's own (non-form) content continues past the point where the - /// budget was hit. Reported through DiagnosticSink.ReportRetained: a condition that ends - /// the page's own form recursion for good is worth surfacing even once - /// is spent on earlier, unrelated conditions. + /// A single page invoked Form XObjects (Do invocations that reached the form, counted + /// across the whole page, not per subtree, and including one whose content then failed to + /// decode or was skipped for the run's own content budget; #402 round 3 fixed this doc, which + /// previously said "successful" recursions, to say so) more than 4096 times. ISO 32000-2 places + /// no limit of its own on how deeply or how often a page may invoke Do (§8.10); this cap + /// is this reader's own bound against a wide, shallow invocation graph a depth cap alone would + /// not catch. Descent + /// into any further form stops for the rest of the page; operators already reported before the + /// budget was reached are kept, and interpretation of the page's own (non-form) content + /// continues past the point where the budget was hit. Reported through + /// DiagnosticSink.ReportRetained: a condition that ends the page's own form recursion for + /// good is worth surfacing even once is spent on + /// earlier, unrelated conditions. /// FormXObjectBudgetExceeded = 305, @@ -433,14 +449,19 @@ public enum PdfReaderDiagnosticCode /// image's /CS (/ColorSpace) entry, named a resource absent from the applicable /// /Resources subdictionary (ISO 32000-2 §7.8.3): the page's own, or, inside a Form /// XObject, that form's own /Resources falling back to its parent's when absent - /// (§8.10.2). The operator is still reported to the caller's visitor; only the interpreter's own - /// attempt to resolve the name failed. + /// (§8.10.2). Also covers Do naming an /XObject entry that IS present but is not + /// a usable XObject stream (#402 round 3): not an indirect reference, does not resolve to a + /// stream, or resolves to one whose /Subtype is missing, is not a name, or names neither + /// /Form nor /Image (Table 86). The operator is still reported to the caller's + /// visitor either way; only the interpreter's own attempt to resolve or use the name failed. /// ResourceMissing = 306, /// - /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded, or one of its - /// dictionary entries was itself invalid: a filter this interpreter never applies to inline + /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded, one of its + /// dictionary entries was itself invalid, or this run's own resync-probe budget ran out before + /// a candidate EI could be confirmed (#402 round 3; see + /// ContentInterpreter.ProbeOnce): a filter this interpreter never applies to inline /// image data (JBIG2Decode, JPXDecode, or Crypt: §8.9.7 itself excludes /// all three from inline-image use), a missing ID or EI operator, a missing, /// non-integer, or non-positive /W or /H, a missing /BPC or one whose @@ -449,18 +470,22 @@ public enum PdfReaderDiagnosticCode /// /BPC's own value to that fixed set; "positive" for /W//H is this /// reader's own requirement, not the table's own wording), an /L present but not an /// integer, or negative (§8.9.7, Table 91; PDF 2.0), an /L or computed length past the - /// end of the stream, or a computed length (from /L or from the image's own shape) that - /// does not land on the following EI operator, in which case this reader retries the EI - /// scan before giving up. No inline-image callback is raised for it. When the data was still - /// delimited (a disallowed filter, a length that the EI scan recovered) only the image - /// is skipped and interpretation of the rest of the content stream continues; when it could not - /// be delimited at all (no ID, no EI, a length past the end of the stream) - /// interpretation of that stream stops there, since nothing past that point can be - /// resynchronised reliably. Reported at most once per page: the sink's dedupe key is (code, - /// object, page), and every image on one content stream reports against that same object - /// (or, for the page's own top-level content specifically, the same ), - /// so a second inline image on the same page with its own, different malformation is not - /// listed separately. + /// end of the stream, a computed length (from /L or from the image's own shape) that + /// does not land on the following EI operator, in which case this reader retries the + /// EI scan before giving up, or the resync probe spending its whole per-run byte budget + /// before it could confirm a candidate EI, in which case that candidate is accepted + /// unverified rather than left unresolved. The inline-image callback IS raised whenever the + /// image was still delimited by the time this reports, including after an /L-past-the-end + /// or did-not-land-on-EI recovery, or a probe-budget-exhausted acceptance; it is not + /// raised only when the image could not be delimited at all. When the data was still delimited + /// (a disallowed filter, a length the EI scan recovered, or a probe-budget-exhausted + /// acceptance) only the image is skipped and interpretation of the rest of the content stream + /// continues; when it could not be delimited at all (no ID, no EI) interpretation + /// of that stream stops there, since nothing past that point can be resynchronised reliably. + /// Reported at most once per page: the sink's dedupe key is (code, object, page), and every + /// image on one content stream reports against that same object (or, for the page's own + /// top-level content specifically, the same ), so a second inline image + /// on the same page with its own, different malformation is not listed separately. /// InlineImageMalformed = 307, diff --git a/src/VellumPdf.Reader/ReaderLimits.cs b/src/VellumPdf.Reader/ReaderLimits.cs index 24fa4c86..d56f5080 100644 --- a/src/VellumPdf.Reader/ReaderLimits.cs +++ b/src/VellumPdf.Reader/ReaderLimits.cs @@ -94,8 +94,9 @@ internal readonly record struct ReaderLimits( /// in mobile devices than desktop computers" — the ceiling is this processor's own choice, not a /// spec requirement, so , /// , , and - /// are each a safe upper bound a caller may only lower, never raise. A value under the - /// corresponding floor is rejected too: below or + /// are each a safe upper bound a caller may only + /// lower, never raise. A value under the corresponding floor is rejected too: below + /// or /// , an otherwise ordinary document routinely /// fails to decode or reconstruct at all; below every report /// would turn into a suppression count, disabling the channel entirely; below diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index efffaea9..85fd4b93 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -276,6 +276,53 @@ public void UnknownOperatorOutsideBX_alsoDropsItsOperands() Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); } + [Fact] + public void BxEx_ownArityIsChecked_butTheSectionStillOpensAndCloses() + { + // BX and EX are both arity-0 (Table 33), but used to be dispatched before the generic + // arity check ran at all, so leftover operands on either one reported nothing even though + // _arity["BX"] and _arity["EX"] are both 0 (#402 round 3). The fix runs the same arity + // check on BX/EX as every other operator, but unlike an ordinary mismatch (which drops the + // whole call), a compatibility section still opens or closes regardless of what garbage + // operands preceded BX/EX: Table 33 does not condition that on anything. Two pages, not + // one: the sink's dedupe key is (code, object, page), so a malformed BX and a malformed EX + // on the SAME page/object would collapse to one recorded diagnostic even though both + // independently fire and independently clear their own operands (see + // UnknownOperatorInsideBX for the same dedupe behaviour on a different code); splitting + // them across two pages is what makes both individually observable in reader.Diagnostics. + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< >>", "1 2 3 BX\nEX\nQ\n"u8.ToArray()), + new Obj(5, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 6 0 R >>"), + new Obj(6, "<< >>", "BX\n4 EX\nQ\n"u8.ToArray())); + + var reader = PdfReader.Open(doc, new PdfReaderOptions()); + var interpreter = new ContentInterpreter(reader); + var visitor0 = new RecordingVisitor(); + var visitor1 = new RecordingVisitor(); + interpreter.Run(reader.GetPage(0), visitor0); + interpreter.Run(reader.GetPage(1), visitor1); + + var mismatches = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Equal(2, mismatches.Count); + Assert.Contains(mismatches, d => d.Message.Contains("'BX'", StringComparison.Ordinal)); + Assert.Contains(mismatches, d => d.Message.Contains("'EX'", StringComparison.Ordinal)); + // Both sections still close on their own page: a form invoked with the SAME unknown + // operator inside it would report UnknownOperator if EX had failed to close the section + // (see UnknownOperatorInsideBX above for the positive case); here Q, well outside BX/EX by + // the time it runs on each page, still reaches the visitor as an ordinary operator. + Assert.Contains(visitor0.Operators, o => o.Op == "Q"); + Assert.Contains(visitor1.Operators, o => o.Op == "Q"); + } + // ── Operand-stack and graphics-state caps ─────────────────────────────────────────────────── [Fact] @@ -327,6 +374,42 @@ public void TjOperand_notAnArray_reportsOperandStackMalformed_notALimit() Assert.Contains(visitor.Operators, o => o.Op == "w"); } + [Fact] + public void HugeArrayOperand_capReportsBeforeMaterialisingIt_boundingAllocation() + { + // An array operand used to be fully materialised as a PdfArray, with one boxed PdfInteger + // per element, before the 8192-element cap was ever consulted (:737-745 against :491-494 at + // 5f5f84b): a 20,000,000-element '[1 1 1 ...] TJ', source text well within the 64 MiB + // content budget, allocated 1,784 MiB and committed 977 MiB before reporting one 309. + // CompositeOperandWithinCap decides the cap with the lexer alone (#402 round 3), so this + // array is never materialised at all. Built unfiltered (BuildPageDocRaw), not through + // FlateDecode: decompressing a ~38 MiB result into a growable buffer is itself a sizeable + // allocation this reader's own filter pipeline pays regardless of this fix, and folding it + // into the same measurement would bound the WRONG thing (measured: 204.7 MiB total with + // FlateDecode in the mix, most of it decompression, none of it this fix's own concern). + const int elementCount = 20_000_000; + var unit = "1 "u8.ToArray(); + var arrayBody = new byte[unit.Length * elementCount]; + for (var i = 0; i < elementCount; i++) + unit.CopyTo(arrayBody, i * unit.Length); + var content = "["u8.ToArray().Concat(arrayBody).Concat("] TJ\n1 w\n"u8.ToArray()).ToArray(); + var doc = BuildPageDocRaw(content); + + var before = GC.GetTotalAllocatedBytes(precise: true); + var (reader, _, visitor) = Run(doc); + var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + // An allocation bound, not a wall-clock one (#400): generous by design, not tight. 96 MiB + // is comfortably above the ~38 MiB source text this reader still reads and lexes, and far + // below the 1,784 MiB the pre-fix path allocated for the identical shape. + Assert.True( + allocated < 96L * 1024 * 1024, + $"expected under 96 MiB allocated for the capped array; measured " + + $"{allocated / (1024.0 * 1024.0):F1} MiB."); + } + [Fact] public void UnbalancedQ_isIgnoredWithADiagnostic() { @@ -442,6 +525,70 @@ private static ContentInterpreter RunAndKeepInterpreter(byte[] pdfBytes, out Pdf return interpreter; } + // ── An operand of the wrong type at the right arity reports 302 and drops the operator, + // rather than NumberOperand silently substituting 0 or Do silently no-op'ing (#402 round 3) ─── + + [Fact] + public void Cm_withANonNumericOperand_reportsOnce_andLeavesTheCtmUntouched() + { + // Before this fix, NumberOperand substituted 0 for the string operand with no diagnostic, + // so this delivered Ctm = [1 0 0 1 0 50] instead of leaving the identity CTM untouched. + var interpreter = RunAndKeepInterpreter( + BuildPageDoc("1 0 0 1 (x) 50 cm\n"), out var reader); + + Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(reports); + Assert.Contains("'cm'", reports[0].Message, StringComparison.Ordinal); + } + + [Fact] + public void TfWithAStringSizeAndTdWithANameOperand_bothDropped_stateUntouched() + { + // Before this fix: Tf's non-numeric size operand ("(12)", a string) silently coerced to a + // FontSize of 0 (already the default, so this alone would not have been visible), but Td's + // non-numeric ty operand ("/N", a name) coerced to 0 while tx=10 was still applied, + // delivering TextMatrix = [1 0 0 1 10 0] instead of leaving it at the identity. Both Tf and + // Td are dropped entirely now; only one 302 is recorded, since the sink dedupes on (code, + // object, page) and both reports share the same key. + var interpreter = RunAndKeepInterpreter( + BuildPageDoc( + "BT\n/F1 (12) Tf\n10 /N Td\nET\n", "<< /Font << /F1 5 0 R >> >>", + new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), + out var reader); + + Assert.Null(interpreter.GraphicsState.Font); + Assert.Equal(0, interpreter.GraphicsState.FontSize); + Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(reports); + } + + [Fact] + public void Do_withANonNameOperand_reportsOnce_andNeverReachesTheVisitor() + { + // Before this fix, a non-name operand to Do silently no-op'd: EmitAndClear had already + // reported "Do" to the visitor before the (missing) type check would have run, and neither + // 302 nor 306 was ever reported. Do is now type-checked BEFORE it is emitted, so a + // malformed invocation like this never reaches the visitor at all. + var content = "42 Do\n"; + var doc = BuildPageDoc( + content, "<< /XObject << /F1 5 0 R >> >>", + new Obj(5, "<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] >>", "1 w\n"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(visitor.Operators, o => o.Op == "Do"); + Assert.Empty(visitor.FormBegins); + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed + && d.Message.Contains("'Do'", StringComparison.Ordinal)); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + } + // ── gs with /Font ──────────────────────────────────────────────────────────────────────────── [Fact] @@ -628,6 +775,27 @@ public void Form_matrixAndBBox_resolveThroughAnIndirectReference() Assert.Equal(4, begin.BBox.UrY); } + [Fact] + public void FormInvokedInsideBX_startsItsOwnCompatibilitySection_notInheritedFromTheInvoker() + { + // Table 33 scopes a BX/EX compatibility section to ONE content stream; a form is its own + // content stream. Before this fix, HandleDo saved and restored _bxDepth but never RESET it + // for the form's own scope, so a form invoked from inside 'BX ... Do ... EX' inherited the + // invoker's _bxDepth > 0, making the form's own unknown operator look like it was still + // inside the INVOKER's compatibility section and silently swallowing it, even though the + // same form invoked as a bare 'Do' correctly reports it (#402 round 3). + var doc = BuildPageDoc( + "BX\n/F1 Do\nEX\n", "<< /XObject << /F1 11 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "10 20 Zork\n1 w\n"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); + Assert.Single(reports); + Assert.Contains("'Zork'", reports[0].Message, StringComparison.Ordinal); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + // ── §8.10.1 b): the form's own /Matrix is concatenated into the CTM (#402 round 2) ─────────── [Fact] @@ -1200,6 +1368,32 @@ public void RawImageWithTwoSpacesAfterId_keepsTheSecondSpaceAsData() Assert.Contains(visitor.Operators, o => o.Op == "w"); } + [Fact] + public void FilterArrayWithA85Second_doesNotSkipExtraWhitespace_flateOwnsThePosition() + { + // §8.9.7 NOTE 2's skip recipe applies only to the filter that reads the raw bytes right + // after ID: the FIRST element of a /Filter array (decode order, §7.4.1's own EXAMPLE 2), + // never a later one. '/F [/Fl /A85]' names FlateDecode first, so FlateDecode owns the raw + // bytes; A85's presence at position 1 must not trigger the extra-whitespace skip meant for + // whichever filter sits at position 0 (#402 round 3: an earlier version skipped whenever + // AHx/A85 appeared ANYWHERE in the array, eating the payload's own leading 0x20 byte here + // and sliding the whole 8-byte window one byte late, delivering "62 63 64 65 66 67 68 20" + // instead of the correct "20 62 63 64 65 66 67 68"). + byte[] payload = [0x20, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68]; + var content = "BI /F [/Fl /A85] /L 8 ID "u8.ToArray() + .Concat(payload) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(payload)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + [Fact] public void FilteredAHx_usesTheEiScan() { @@ -1264,37 +1458,210 @@ public void DctInlineImage_withAFalseEiFollowedByBinaryNoise_isSkippedByTheScan( Assert.Equal("Q", Assert.Single(visitor.Operators).Op); } - // ── The bounded resync probe (#402): linear-time scan, less strict, still rejects noise ──── + // ── The bounded resync probe (#402 round 3 redesign): a per-Run byte budget shared by every + // 'EI' candidate, not a per-candidate window, bounds ScanForEi's total cost regardless of + // content size or candidate count ──────────────────────────────────────────────────────────── + + [Fact] + public void ManyFalseEiCandidates_atOneMebibyte_exhaustsTheProbeBudgetExactlyOnce() + { + // Reproduces the pre-round-3 cost blowup this shape drove into the two-window probe: N + // repeats of " EI (" (a false candidate immediately followed by the start of a literal + // string that never closes anywhere in the rest of the buffer) after a DCT-filtered + // image's own data, then the terminating ' EI'. Measured against the round-2 probe: 1 MiB + // decoded content, 16,608 ms; 4 MiB, 71,320 ms; the 64 MiB content cap extrapolates to + // about 19 minutes, with no diagnostic to explain the cost. Every candidate here is close + // enough to the buffer's own true end that the round-3 probe's window is UNCLIPPED for the + // first several: each of those pays a full scan to the true end trying to close the never- + // closing string (an outright Reject, not a budget artefact) before the 16 MiB run budget + // itself runs out partway through one candidate's own window, which is what turns that one + // candidate's own outcome into Exhausted (accepted unverified) instead. One MiB of content + // is what makes that transition land inside this one test run: about 16 unclipped full- + // buffer rejects (~1 MiB each) exhaust the 16 MiB budget, so the total cost this shape can + // ever impose is bounded by the budget, not by how many further false candidates follow. + const int decodedContentBytes = 1024 * 1024; + var falseCandidate = " EI ("u8.ToArray(); + var prefix = "BI /F /DCT ID "u8.ToArray().Concat([0xFF, 0xD8, 0xFF]).ToArray(); + var suffix = " EI\nQ\n"u8.ToArray(); + var noiseLength = decodedContentBytes - prefix.Length - suffix.Length; + var repeats = noiseLength / falseCandidate.Length; + var noise = new byte[falseCandidate.Length * repeats]; + for (var i = 0; i < repeats; i++) + falseCandidate.CopyTo(noise, i * falseCandidate.Length); + + var content = prefix.Concat(noise).Concat(suffix).ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, interpreter, visitor) = Run(doc); + + const long maxProbeBytesPerRun = 16L * 1024 * 1024; // mirrors ContentInterpreter's own private const + Assert.Equal(maxProbeBytesPerRun, interpreter.ProbeBytesConsumed); + + // Exactly the budget report and the outer lexer's own report on the unterminated literal + // string the accepted-unverified candidate leaves behind (#402 round 3): the accepted + // candidate's own trailing '(' never closes anywhere in the rest of the buffer (every later + // '(' in the noise just nests deeper), so InterpretStream's own main loop hits + // InvalidDataException once it reaches true end of buffer still inside that string. + Assert.Equal(2, reader.Diagnostics.Count); + Assert.Contains( + reader.Diagnostics, + d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed + && d.Message.Contains("accepted without verification", StringComparison.Ordinal)); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Single(visitor.InlineImages); // the (false) accepted-unverified candidate delimits it + } [Fact] - public void ManyFalseEiCandidates_doesNotThrow_andReportsADiagnostic() + public void DctImage_withNoFalseCandidates_atFourMebibytes_probesAtMostTheTrailingOperator() { - // Reproduces the pre-#402 quadratic blowup: N repeats of " EI (" (a false candidate - // followed by the start of a literal string) after a DCT-filtered image's own data. The - // unbounded probe this used to run from EVERY candidate re-lexed all the way to the end of - // the buffer looking for the string's own closing ')', which never comes; O(N) work per - // candidate made the whole scan O(N^2) (measured pre-fix: 100 KB content, 18 s; 400 KB, - // 305 s, from a 1.2 KB Flate-compressed source). The bounded probe caps the per-candidate - // cost, so this reads (with `dotnet test`'s own default timeout as the actual regression - // guard, per this repo's no-wall-clock-assertion rule) rather than hanging. - const int n = 20_000; - var falseCandidate = " EI ("u8.ToArray(); - var noise = new byte[falseCandidate.Length * n]; - for (var i = 0; i < n; i++) - falseCandidate.CopyTo(noise, i * falseCandidate.Length); + // The other half of the round-3 bound: a clean image (no false 'EI' candidates at all) + // must cost the probe almost nothing, however large the image itself is, since ScanForEi + // finds the terminating 'EI' on its very first byte-level match and LooksLikeResyncPoint probes + // forward from there exactly once. + const int imageBytes = 4 * 1024 * 1024; + var data = new byte[imageBytes]; + // 'E'/'I' bytes deliberately excluded so no coincidental candidate exists in the noise. + for (var i = 0; i < data.Length; i++) + data[i] = (byte)(0x01 + (i % 0x40)); var content = "BI /F /DCT ID "u8.ToArray() - .Concat([0xFF, 0xD8, 0xFF]) - .Concat(noise) + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) .ToArray(); var doc = BuildPageDocRaw(content); - var (reader, _, _) = Run(doc); + var (reader, interpreter, visitor) = Run(doc); - Assert.Contains( - reader.Diagnostics, - d => d.Code is PdfReaderDiagnosticCode.ContentStreamLexError - or PdfReaderDiagnosticCode.InlineImageMalformed); + // <= 2: the bytes of 'Q' the single ProbeOnce call from just past the terminating 'EI' consumes. + Assert.True( + interpreter.ProbeBytesConsumed <= 2, + $"expected the probe to spend at most 2 bytes on a candidate-free image; spent " + + $"{interpreter.ProbeBytesConsumed}."); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Theory] + [InlineData(4093)] // token exactly 4096 bytes: already correct under the old two-window probe + [InlineData(4094)] + [InlineData(5000)] + [InlineData(40_000)] + public void RealEi_followedByALiteralOfAnyLength_isAlwaysAccepted_noExtendedWindowLeft(int litLen) + { + // Round-3 HIGH 2: the round-2 probe's extended (4096-byte) window rejected the terminating + // 'EI' when the legitimate token right after it was longer than that window, losing the + // image and the rest of the stream (litLen 4094 and up reported InlineImageMalformed and + // delivered nothing). The round-3 probe has no per-candidate window at all, only the shared + // per-Run budget (far larger than any of these), so every length here is accepted the same + // way. + var longLiteral = new string('Y', litLen); + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(Encoding.ASCII.GetBytes($" EI ({longLiteral}) Tj\nQ\n")) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + var tj = Assert.Single(visitor.Operators, o => o.Op == "Tj"); + var str = (PdfLiteralString)tj.Operands[0]; + Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + public static IEnumerable NeutralStraddlingShapes() + { + yield return ["[1 2 3"]; // unterminated array + yield return ["< d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.InRange(img.Data.Length, 315, 335); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void StreamEndingRightAfterEi_stillDelimitsTheImage(bool trailingWhitespace) + { + // A candidate probed at the buffer's own true end (not merely a budget-clipped window) is + // a legitimate resync point on its own: AtEnd on an unclipped window accepts. + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat(trailingWhitespace ? " EI \n"u8.ToArray() : " EI"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + } + + [Fact] + public void ScanForEi_dataEndingInCrLfBeforeEi_stripsBothBytes() + { + // §8.9.7 excludes "the white-space delimiting those operators" from the image data; §7.2.3 + // makes a CARRIAGE RETURN immediately followed by a LINE FEED ONE EOL marker. Before this + // fix, ScanForEi stripped only the LF, leaving a trailing 0D byte in the delivered data + // (#402 round 3; the ID side already consumed a CR LF pair as one separator). + byte[] data = [0xFF, 0xD8, 0xFF]; + var content = "BI /F /DCT ID "u8.ToArray() + .Concat(data) + .Concat("\r\nEI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); } [Fact] @@ -1366,11 +1733,11 @@ public void TwoConsecutiveInlineImages_theSecondWithAShortDictionary_bothDelimit [Fact] public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindow_isAccepted() { - // The probe's first, 128-byte window runs off mid-string here (the literal is 200 bytes, - // longer than that window), so the probe re-tries once against the 4096-byte extended - // window (#402 round 2); the string closes well inside THAT one, so the candidate is - // accepted the same way it was before the two-window redesign, just through the second - // window rather than an inconclusive-accept on the first. + // The round-2 probe's own first, 128-byte window ran off mid-string here (the literal is + // 200 bytes, longer than that window) and needed a second, larger window to accept it. The + // round-3 probe has no fixed window at all, only the shared per-Run budget (#402 round 3), + // so a 200-byte literal that closes properly is neutral and simply keeps the probe lexing + // until it reaches 'Tj', a known operator: accepted the same way, just without a retry. var longLiteral = new string('X', 200); byte[] data = [0xFF, 0xD8, 0xFF]; var content = "BI /F /DCT ID "u8.ToArray() @@ -1389,26 +1756,25 @@ public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindo Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); } - // ── The extended (4096-byte) probe window still rejects a token that never closes at all, - // regardless of how far past the first window that non-closure runs (#402 round 2) ────────── + // ── A token that never closes at all is rejected regardless of how far it runs, since the + // shared per-Run budget only turns "ran out of budget" into an accept, never "ran off a fixed + // per-candidate window" (#402 round 3; see ProbeOnce's windowClipped distinction) ──────────── [Theory] - [InlineData(100)] // inside the 128-byte first window: already correct before this fix + [InlineData(100)] [InlineData(110)] - [InlineData(200)] // past the first window, inside the second - [InlineData(5000)] // past BOTH windows + [InlineData(200)] + [InlineData(5000)] public void FalseEiCandidate_insideAnUnterminatedLiteralString_isRejected_regardlessOfLength(int n) { - // Reproduces the round-1 regression this round's reviewers found: a false ' EI' candidate - // immediately followed by an unterminated literal string that never closes ANYWHERE in the - // rest of the buffer, then N filler bytes, then the real ' EI\nQ\n'. The OLD single-window - // probe treated "ran off a clipped window" as accept-by-default, so once N pushed the false - // candidate's own unterminated string past the 128-byte window (but the buffer still had - // more bytes beyond it), the false candidate was WRONGLY accepted as the resync point, - // truncating the image to 3 bytes and losing 'Q' to the caller's visitor entirely, with no - // InlineImageMalformed to explain why. The two-window redesign keeps the correct outcome at - // every N tested here: a string that never closes anywhere is rejected at either window, - // never accepted just because it ran past the first one. + // A false ' EI' candidate immediately followed by an unterminated literal string that + // never closes ANYWHERE in the rest of the buffer, then N filler bytes, then the + // terminating ' EI\nQ\n'. The remaining buffer here (a few KB at most) is always far + // smaller than the 16 MiB per-Run probe budget, so the probe's window is never clipped by + // that budget for any of these: the string never closes at all, so it is rejected outright + // at every N tested, the same way a round-1 regression (accepting a token that merely ran + // past a FIXED window) never had a chance to reappear once that fixed window itself was + // removed. var falseCandidate = " EI (never closed"u8.ToArray(); var filler = new byte[n]; Array.Fill(filler, (byte)'z'); @@ -1432,8 +1798,8 @@ public void FalseEiCandidate_insideAnUnterminatedLiteralString_isRejected_regard [Fact] public void FalseEiCandidate_insideAnUnterminatedHexString_isRejected() { - // The hex-string counterpart of the literal-string regression above: an unclosed '<' run - // longer than the first probe window, never closed anywhere in the rest of the buffer. + // The hex-string counterpart of the literal-string case above: an unclosed '<' run that + // never closes anywhere in the rest of the buffer. var falseCandidate = " EI <414243"u8.ToArray(); // '<' opens a hex string that never closes var filler = new byte[200]; Array.Fill(filler, (byte)'0'); @@ -1457,10 +1823,11 @@ public void FalseEiCandidate_insideAnUnterminatedHexString_isRejected() [Fact] public void RealEi_followedByATerminated4000ByteLiteral_isAccepted() { - // The true terminating 'EI' must still be accepted even when what follows it is a long - // (past the first window, inside the second) but well-formed token: rejecting a resync - // point that IS the real one just because legitimate content after it happens to be long - // would be exactly as wrong as accepting one that never closes at all. + // The true terminating 'EI' must still be accepted even when what follows it is a long but + // well-formed token: rejecting the resync point that terminates the image just because + // legitimate content after it happens to be long would be exactly as wrong as accepting one + // that never closes at all. See RealEi_followedByALiteralOfAnyLength_isAlwaysAccepted_noExtendedWindowLeft + // for the round-3 boundary cases the old two-window probe got wrong (litLen >= 4094). var longLiteral = new string('Y', 4000); byte[] data = [0xFF, 0xD8, 0xFF]; var content = "BI /F /DCT ID "u8.ToArray() @@ -1569,7 +1936,9 @@ public void W_zero_reportsMalformed_andRecoversViaTheEiScan() public void L_asARealNumber_reportsMalformed_andRecoversViaTheEiScan() { // Table 91 types /L as an integer; a PdfReal there (dropped silently before this fix) is - // reported the same way an invalid /W, /H, or /BPC already is (#402 round 2). + // reported the same way an invalid /W, /H, or /BPC already is (#402 round 2). The message + // says "present but" rather than "missing, or" (#402 round 3: this branch is reachable only + // once /L IS present, so a "missing" message here was never accurate). var content = "BI /F /AHx /L 4.0 ID ABCD EI\nQ\n1 w\n"; var (reader, _, visitor) = Run(BuildPageDoc(content)); @@ -1577,7 +1946,7 @@ public void L_asARealNumber_reportsMalformed_andRecoversViaTheEiScan() Assert.Contains( reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed - && d.Message.Contains("missing, or carries an invalid", StringComparison.Ordinal)); + && d.Message.Contains("present but its value is not an integer", StringComparison.Ordinal)); var img = Assert.Single(visitor.InlineImages); Assert.True(img.Data.AsSpan().SequenceEqual("ABCD"u8.ToArray())); Assert.Contains(visitor.Operators, o => o.Op == "Q"); @@ -1781,6 +2150,34 @@ public void ContentsStreamWithAnImageFilter_reportsLexError() Assert.Empty(visitor.Operators); } + [Fact] + public void LexerFailureInsideOneStream_endsTheWholeConcatenatedBuffer_laterElementsIncluded() + { + // The 300 doc's "resumes with the next stream in the array" clause holds only for an + // element that fails to RESOLVE or DECODE (NonStreamContentsElement_isSkippedWithLexError + // above): every element that DOES decode is concatenated into one buffer before + // interpretation of any of it begins (Table 31: "the division between streams may occur + // only at the boundaries between lexical tokens"), so a lexer or parser failure inside that + // buffer ends interpretation for the rest of the page's content, later array elements + // included, not merely the one stream the failure fell in (#402 round 3). Object 4 decodes + // to a token '1 0 ]' the object parser cannot make sense of (a lone ']' with no opening + // '['); object 5's own 'J'/'j' never run. + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents [4 0 R 5 0 R] >>"), + new Obj(4, "<< >>", "1 w\n1 0 ]\n"u8.ToArray()), + new Obj(5, "<< >>", "2 J\n3 j\n"u8.ToArray())); + + var (reader, _, visitor) = Run(doc); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + } + // ── 64 MiB content cap ─────────────────────────────────────────────────────────────────────── [Fact] @@ -1905,7 +2302,11 @@ public void ContentStreamTooLarge_isRetainedEvenOnceMaxDiagnosticsIsAlreadySpent // ── Fuzzing ────────────────────────────────────────────────────────────────────────────────── - private static readonly byte[] FuzzSeed = BuildPdf( + private static readonly byte[] FuzzContent = Encoding.ASCII.GetBytes( + "q\n2 0 0 2 10 20 cm\nBT\n/F1 12 Tf\n(Hi) Tj\n[(A) -10 (B)] TJ\nET\n" + + "/G1 gs\n/X1 Do\nBI /W 2 /H 2 /BPC 8 /CS /G ID \x01\x02\x03\x04 EI\nQ\n"); + + private static byte[] BuildFuzzDoc(byte[] content) => BuildPdf( 1, new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), @@ -1913,15 +2314,15 @@ public void ContentStreamTooLarge_isRetainedEvenOnceMaxDiagnosticsIsAlreadySpent "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + "/Resources << /Font << /F1 6 0 R >> /XObject << /X1 7 0 R >> " + "/ExtGState << /G1 8 0 R >> >> /Contents 4 0 R >>"), - new Obj(4, "<< >>", Encoding.ASCII.GetBytes( - "q\n2 0 0 2 10 20 cm\nBT\n/F1 12 Tf\n(Hi) Tj\n[(A) -10 (B)] TJ\nET\n" - + "/G1 gs\n/X1 Do\nBI /W 2 /H 2 /BPC 8 /CS /G ID \x01\x02\x03\x04 EI\nQ\n")), + new Obj(4, "<< >>", content), new Obj(6, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"), new Obj( 7, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Matrix [1 0 0 1 0 0] >>", "1 w\n"u8.ToArray()), new Obj(8, "<< /Type /ExtGState /Font [6 0 R 10] >>")); + private static readonly byte[] FuzzSeed = BuildFuzzDoc(FuzzContent); + private readonly record struct MutationOp(int Kind, int Position, byte Value, int Length); private static readonly Gen MutationOpGen = @@ -1929,9 +2330,23 @@ public void ContentStreamTooLarge_isRetainedEvenOnceMaxDiagnosticsIsAlreadySpent Gen.Int[0, 5], Gen.Int[0, int.MaxValue], Gen.Byte, Gen.Int[1, 32], (kind, position, value, length) => new MutationOp(kind, position, value, length)); + // Mutates the whole file's own bytes: exercises reconstruction, object resolution, and + // structural malformation, but a mutation landing in the xref, trailer, or object headers + // (most of the file, by byte count) usually keeps the interpreter from ever being reached at + // all: see ContentBodyFuzzInputGen below for the generator that exists to reach it reliably + // (#402 round 3). private static readonly Gen FuzzInputGen = MutationOpGen.Array[1, 8].Select(ops => Mutate(FuzzSeed, ops)); + // Mutates only the page's own content-stream BODY, then rebuilds a structurally valid document + // around the result: every byte this generator can touch is one ContentInterpreter itself + // walks, so PdfReader.Open and GetPage keep succeeding on nearly every sample regardless of + // what the mutations did to operator syntax, giving Run deep, repeated coverage of the + // interpreter's own malformed-content recovery paths rather than mostly exercising xref repair + // (#402 round 3; FuzzInputGenReachesRunOnTheMajorityOfSamples pins the reach-rate difference). + private static readonly Gen ContentBodyFuzzInputGen = + MutationOpGen.Array[1, 8].Select(ops => BuildFuzzDoc(Mutate(FuzzContent, ops))); + private static byte[] Mutate(byte[] seed, MutationOp[] ops) { var buffer = new List(seed); @@ -1964,9 +2379,62 @@ private static byte[] Mutate(byte[] seed, MutationOp[] ops) [Fact] public void Fuzz_run_neverThrowsOutsideTheDeclaredVocabulary_andAlwaysTerminates() - => FuzzInputGen.Sample(AssertInterpreterIsRobust, iter: FuzzBudget.Iterations); + // A block-bodied lambda, not `bytes => AssertInterpreterIsRobust(bytes)`: CsCheck.Sample has + // both an Action overload (fails only on a thrown exception) and a Func one + // (fails whenever the delegate returns false). AssertInterpreterIsRobust now returns + // whether Run was reached (#402 round 3, for the majority-of-samples test below), and an + // expression-bodied lambda binds to the Func overload by exact delegate-type match, + // silently reinterpreting "this sample never reached Run" (true for ~95% of this generator's + // own samples) as a FAILING property case instead of the acceptable outcome it is. A + // block-bodied lambda has no expression value to match Func against, so it can only + // bind to Action, restoring the original semantics. + => FuzzInputGen.Sample(bytes => { AssertInterpreterIsRobust(bytes); }, iter: FuzzBudget.Iterations); + + [Fact] + public void ContentBodyFuzzGenerator_isRobust_andReachesRunOnTheMajorityOfSamples() + { + // One Sample() call, not two: it both runs the same robustness assertions + // Fuzz_run_neverThrowsOutsideTheDeclaredVocabulary_andAlwaysTerminates pins for the + // file-wide generator AND counts how many samples reach ContentInterpreter.Run at all. A + // separate Fact duplicating the robustness-only pass would double this generator's own + // CsCheck cost (every core, per the class doc's own #398/#399 contention lesson) for no + // extra coverage. + // + // The file-wide generator's own mutations mostly land in the xref, trailer, or object + // headers (most of the file, by byte count), so most samples never resolve a page at all: + // measured at 20,000 iterations, 990 of 19,628 samples (5.0%) reached Run: the interpreter + // itself is exercised on only a sliver of that generator's own budget. The content-body + // generator exists to fix that: every byte it can touch is inside the one content stream + // ContentInterpreter itself walks, so the surrounding file structure stays intact and Run + // is reached on nearly every sample instead (#402 round 3). Pinned by COUNT, not time + // (#400): a majority is the bar, generous against the near-100% this generator reaches in + // practice. threads: 1: this call does not need parallelism of its own to finish well + // within a test run, and a second CsCheck.Sample call in this same class independently + // trying to claim every core is needless oversubscription alongside + // Fuzz_run_neverThrowsOutsideTheDeclaredVocabulary_andAlwaysTerminates's own (the class + // doc's own #398/#399 note is why this file treats that as worth avoiding, not adding to). + var reached = 0L; + var total = 0L; + ContentBodyFuzzInputGen.Sample( + bytes => + { + total++; + if (AssertInterpreterIsRobust(bytes)) + reached++; + }, + iter: FuzzBudget.Iterations, threads: 1); + + Assert.True(total > 0); + Assert.True( + reached * 2 >= total, + $"the content-body fuzz generator reached Run on {reached}/{total} samples; expected a " + + "majority."); + } - private static void AssertInterpreterIsRobust(byte[] bytes) + /// Runs the same robustness assertions + /// pins, and reports whether this sample reached ContentInterpreter.Run at all, so a + /// caller can separately measure how often each generator gets that far (#402 round 3). + private static bool AssertInterpreterIsRobust(byte[] bytes) { // Two separate try blocks, not one covering Open/GetPage/Run together: ContentInterpreter's // own class doc promises InvalidDataException never escapes Run (UnsupportedPdfFeatureException @@ -1991,6 +2459,7 @@ private static void AssertInterpreterIsRobust(byte[] bytes) // interpreter follows: a robustness oracle, not a conformance one. } + var reachedRun = page is not null; if (page is not null) { try @@ -2008,6 +2477,7 @@ private static void AssertInterpreterIsRobust(byte[] bytes) Assert.True( stopwatch.Elapsed <= TimeSpan.FromSeconds(4), $"content interpretation took {stopwatch.Elapsed} on a {bytes.Length}-byte input."); + return reachedRun; } private static class FuzzBudget diff --git a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs index 87c3e030..3b6fc711 100644 --- a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs @@ -160,6 +160,25 @@ public void ContentStreamMode_insideACompatibilitySection_lexesAWholeBxToExSeque kinds); } + [Fact] + public void ContentStreamMode_stillThrowsOnALoneCloseParen() + { + // Content-stream mode relaxes exactly three bytes ('{', '}', a lone '>'), per this file's + // own class doc; a lone ')' is not one of them and must keep throwing (#402 round 3, C5: a + // boundary pin so a later change to what this mode relaxes cannot widen it here silently). + var lexer = new PdfLexer(")"u8.ToArray(), contentStreamMode: true); + + Assert.Throws(() => lexer.NextToken()); + } + + [Fact] + public void ContentStreamMode_stillThrowsOnAnUnterminatedLiteralString() + { + var lexer = new PdfLexer("(never closed"u8.ToArray(), contentStreamMode: true); + + Assert.Throws(() => lexer.NextToken()); + } + // ── Every other token kind lexes identically in both modes ───────────────────────────────────── // MemberData parameters must be public-accessible types (CS0051), so TokenKind (internal, via From 7672a39a8441436813b1cadb4165b602cf8ab420 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 03:08:15 +0200 Subject: [PATCH 08/14] Count composite-operand tokens at every depth before the cap CompositeOperandWithinCap judged an array or dictionary operand by its top-level element count alone, so a TJ array nesting twenty million numbers one level down passed the 8192 cap at depth one and was still materialised in full by PdfObjectParser, and an unterminated array of the same size was parsed all the way to the lexer error before the cap saw it. Both shapes cost the 1.7 GiB the round-3 fix was meant to bound. Count every token at every nesting depth (closing delimiters aside), stop on the lexer's own failure or the end of input, and decide from that count in every exit: an over-cap composite, terminated or not, is dropped with one ContentLimitExceeded and never parsed, while a within-cap unterminated one keeps reporting ContentStreamLexError through ParseObject as before. The flat, nested and unterminated shapes each allocate 76.3 MiB now; the nested and unterminated tests fail against the previous commit, which reports no 309 for either. --- CHANGELOG.md | 4 +- .../Content/ContentInterpreter.cs | 39 ++++++----- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 5 +- .../ContentInterpreterTests.cs | 67 +++++++++++++++++++ 4 files changed, 91 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43035454..ebc0adf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). what the other nine codes describe: `ContentStreamLexError`, `UnknownOperator` (`Warning` severity: an operator this reader skipped is a best-effort reading, not proof the output is correct), `OperandStackMalformed` for a producer-side malformation, `ContentLimitExceeded` for - this reader's own processing ceilings instead (an operand-count, `TJ`-array, `q`-depth, or - marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, + this reader's own processing ceilings instead (an operand-count, array-or-dictionary-operand + token, `q`-depth, or marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, `InlineImageMalformed`, and `ContentStreamTooLarge` (one 64 MiB decoded-content budget per page, shared between its own `/Contents` and every Form XObject it draws; each invocation of a form counts again, since the interpretation cost this bounds scales with how many times a form is diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index e8601e65..e7c9c640 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -44,8 +44,10 @@ internal sealed class ContentInterpreter // 32 as this reader's own ceiling would reject a legal call, not just a hostile one. private const int MaxOperandsPerOperator = 64; - // This reader's own ceiling on how many depth-1 elements a single array or dictionary operand - // may carry (an array's own elements, or a dictionary's own keys and values counted together). + // This reader's own ceiling on how many tokens a single array or dictionary operand may carry, + // counted at every nesting depth (each element, key, value and nested opening delimiter is one + // token), so the count bounds what PdfObjectParser would allocate for the composite as a whole. + // Counting only the top level would let '[[1 1 1 ...]]' hide millions of elements behind one. // §9.4.3's own TJ array is what most directly exercises it, but CompositeOperandWithinCap // applies it to every composite operand this interpreter parses, not only TJ's, and enforces it // BEFORE PdfObjectParser ever materialises the composite (#402 round 3): a 20,000,000-element TJ @@ -523,7 +525,7 @@ private void InterpretStream( // Pre-scanned with the lexer alone (no PdfObject allocation) before // ParseObject ever materialises it: an adversarial TJ array can name // millions of elements in a source text small enough to decode well - // within the content budget, so the element-count cap has to be + // within the content budget, so the token-count cap has to be // consulted before the allocation it exists to bound, not after (#402 // round 3; see CompositeOperandWithinCap and MaxCompositeOperandElements). if (CompositeOperandWithinCap(lexer)) @@ -537,7 +539,7 @@ private void InterpretStream( var shape = token.Kind == TokenKind.ArrayBegin ? "An array" : "A dictionary"; diagnostics.Report( PdfReaderDiagnosticCode.ContentLimitExceeded, - $"{shape} operand exceeds {MaxCompositeOperandElements} elements; " + $"{shape} operand exceeds {MaxCompositeOperandElements} tokens; " + "the operator taking it was dropped.", ctx.DiagObjectNumber, pageIndex: pageIndex); } @@ -684,13 +686,15 @@ private void ClearOperands() // Pre-scans an array or dictionary operand with the LEXER ALONE, starting right after its // already-consumed opening token, to decide whether it stays within - // MaxCompositeOperandElements before PdfObjectParser ever allocates a PdfObject for it. Counts - // only DEPTH-1 tokens: the ones that are this composite's own direct elements (an array) or - // keys and values (a dictionary), the same items PdfArray.Count/PdfDictionary.Count would count - // once materialised. A nested array or dictionary counts once, at its own opening token, not - // once per token it itself contains. Leaves the lexer positioned right after the matching close - // either way, so the caller can either seek back to re-parse it (within cap) or simply continue - // with the next token (over cap: nothing more from this composite is needed). + // MaxCompositeOperandElements before PdfObjectParser ever allocates a PdfObject for it. Every + // token inside the composite counts, at any depth, because every one of them becomes an + // allocation once materialised; closing delimiters are the exception, since they allocate + // nothing. Leaves the lexer positioned right after the matching close, so the caller can either + // seek back to re-parse it (within cap) or simply continue with the next token (over cap: + // nothing more from this composite is needed). An unterminated composite is judged by the + // same count: within the cap, the caller's ParseObject re-derives the failure and reports it + // (ContentStreamLexError, 300) the way it always has; over the cap it is dropped as over-cap + // without ever being parsed, since parsing it would allocate everything before failing. private static bool CompositeOperandWithinCap(PdfLexer lexer) { var depth = 1; @@ -704,20 +708,16 @@ private static bool CompositeOperandWithinCap(PdfLexer lexer) } catch (InvalidDataException) { - // No matching close inside this composite: not this method's problem to diagnose. - // The caller seeks back to the opening token and lets ParseObject re-derive the - // same failure and report it (ContentStreamLexError, #300), the way it always has. - return true; + break; } if (token.Kind == TokenKind.EndOfInput) - return true; // Same: unterminated composite, ParseObject's own retry reports it. + break; switch (token.Kind) { case TokenKind.ArrayBegin or TokenKind.DictBegin: - if (depth == 1) - count++; + count++; depth++; break; @@ -726,8 +726,7 @@ private static bool CompositeOperandWithinCap(PdfLexer lexer) break; default: - if (depth == 1) - count++; + count++; break; } } diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 919971e0..55aead60 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -507,8 +507,9 @@ public enum PdfReaderDiagnosticCode /// /// A content stream hit one of this reader's own processing ceilings rather than being /// malformed by its producer: more than 64 operands accumulated before an operator (§7.8.2 - /// gives an operator's own operand count no declared bound of its own), a TJ array - /// (§9.4.3) with more than 8192 elements, more than 64 nested q saves, or marked-content + /// gives an operator's own operand count no declared bound of its own), an array or dictionary + /// operand (a TJ array, §9.4.3, being the usual one) carrying more than 8192 tokens + /// counted at every nesting depth, more than 64 nested q saves, or marked-content /// nesting (§14.6.1) past the same 64-deep cap. Split out from /// (#402) so a caller can tell "this file hit a limit of this reader" apart from "this file is /// malformed", a distinction the two codes sharing one value made impossible to draw. The diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 85fd4b93..84209090 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -410,6 +410,73 @@ public void HugeArrayOperand_capReportsBeforeMaterialisingIt_boundingAllocation( + $"{allocated / (1024.0 * 1024.0):F1} MiB."); } + [Fact] + public void HugeNestedArrayOperand_countsEveryDepth_boundingAllocation() + { + // The first version of CompositeOperandWithinCap counted depth-1 tokens only, so + // '[[1 1 1 ...]] TJ' counted as ONE element, passed the cap, and was materialised in full: + // the same 1,784 MiB shape as the flat array above, hidden behind a single pair of + // brackets. Every token at every depth counts now. + const int elementCount = 20_000_000; + var unit = "1 "u8.ToArray(); + var arrayBody = new byte[unit.Length * elementCount]; + for (var i = 0; i < elementCount; i++) + unit.CopyTo(arrayBody, i * unit.Length); + var content = "[["u8.ToArray().Concat(arrayBody).Concat("]] TJ\n1 w\n"u8.ToArray()).ToArray(); + var doc = BuildPageDocRaw(content); + + var before = GC.GetTotalAllocatedBytes(precise: true); + var (reader, _, visitor) = Run(doc); + var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + Assert.True( + allocated < 96L * 1024 * 1024, + $"expected under 96 MiB allocated for the capped nested array; measured " + + $"{allocated / (1024.0 * 1024.0):F1} MiB."); + } + + [Fact] + public void HugeUnterminatedArrayOperand_isDroppedAsOverCap_notMaterialisedBeforeTheLexError() + { + // An unterminated composite used to return "within cap" from the pre-scan so that + // ParseObject could re-derive the ContentStreamLexError, which it did, but only after + // allocating every element up to the point of failure. Over the cap, the composite is now + // dropped with one 309 and never parsed; the lexer is left at end of input, so no 300 + // follows and nothing else is reported. + const int elementCount = 20_000_000; + var unit = "1 "u8.ToArray(); + var arrayBody = new byte[unit.Length * elementCount]; + for (var i = 0; i < elementCount; i++) + unit.CopyTo(arrayBody, i * unit.Length); + var content = "1 w\n["u8.ToArray().Concat(arrayBody).ToArray(); + var doc = BuildPageDocRaw(content); + + var before = GC.GetTotalAllocatedBytes(precise: true); + var (reader, _, visitor) = Run(doc); + var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + Assert.True( + allocated < 96L * 1024 * 1024, + $"expected under 96 MiB allocated for the unterminated array; measured " + + $"{allocated / (1024.0 * 1024.0):F1} MiB."); + } + + [Fact] + public void SmallUnterminatedArrayOperand_stillReportsTheLexError_notTheCap() + { + // Within the cap, an unterminated composite keeps its original outcome: ParseObject + // re-derives the failure and the stream ends with ContentStreamLexError, not 309. + var (reader, _, visitor) = Run(BuildPageDoc("1 w\n[1 2 3\n")); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); + } + [Fact] public void UnbalancedQ_isIgnoredWithADiagnostic() { From 2d3c04db99f8fafc303554ea4de830af2ed971af Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 03:34:18 +0200 Subject: [PATCH 09/14] Measure the composite-cap allocation bound per thread The three allocation-bound tests read GC.GetTotalAllocatedBytes, which is process-wide. The test host runs classes in parallel, so on the CI runner the delta included every other class's allocations: 1125.1 MiB for the flat array and 373.1 MiB for the unterminated one against the 96 MiB bound, for a shape that costs 76.3 MiB when the class runs alone (run 33824666885 at 7672a39). Run interprets synchronously on the calling thread, so GC.GetAllocatedBytesForCurrentThread sees exactly this call's allocations and nothing else's. --- .../ContentInterpreterTests.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 84209090..1aba9638 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -395,9 +395,14 @@ public void HugeArrayOperand_capReportsBeforeMaterialisingIt_boundingAllocation( var content = "["u8.ToArray().Concat(arrayBody).Concat("] TJ\n1 w\n"u8.ToArray()).ToArray(); var doc = BuildPageDocRaw(content); - var before = GC.GetTotalAllocatedBytes(precise: true); + // Measured per thread, not process-wide: the test host runs classes in parallel, and the + // process-wide counter charged this delta with whatever the other classes allocated at the + // same moment (1125.1 MiB and 373.1 MiB on the CI runner for a shape that costs 76.3 MiB + // alone). Run interprets synchronously on the calling thread, so the thread-local counter + // sees exactly this call's allocations. + var before = GC.GetAllocatedBytesForCurrentThread(); var (reader, _, visitor) = Run(doc); - var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); @@ -425,9 +430,9 @@ public void HugeNestedArrayOperand_countsEveryDepth_boundingAllocation() var content = "[["u8.ToArray().Concat(arrayBody).Concat("]] TJ\n1 w\n"u8.ToArray()).ToArray(); var doc = BuildPageDocRaw(content); - var before = GC.GetTotalAllocatedBytes(precise: true); + var before = GC.GetAllocatedBytesForCurrentThread(); var (reader, _, visitor) = Run(doc); - var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); @@ -453,9 +458,9 @@ public void HugeUnterminatedArrayOperand_isDroppedAsOverCap_notMaterialisedBefor var content = "1 w\n["u8.ToArray().Concat(arrayBody).ToArray(); var doc = BuildPageDocRaw(content); - var before = GC.GetTotalAllocatedBytes(precise: true); + var before = GC.GetAllocatedBytesForCurrentThread(); var (reader, _, visitor) = Run(doc); - var allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); From cb5280e262caaaa96303eadb391dba3092a5a460 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 05:35:17 +0200 Subject: [PATCH 10/14] Fix content-interpreter review findings from PR #402 round 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Table 107's ' and " operators never advanced the text line matrix or set word/character spacing: they fell through to the default, state-inert case, so a page ending a line with either operator kept the wrong text position with no diagnostic. Both are now type-checked and update state the way Tj/T*/Tw/Tc already do. gs, cs, CS, and sh each read their own operand for a resource lookup (HandleExtGState, ValidateColorSpaceResource, ValidateNamedResource) but were exempt from ValidateOperandTypes, so a wrong-typed operand silently no-op'd the lookup instead of reporting 302, the same gap Do had before round 3. A 'q', 'BMC', or 'BDC' dropped for this reader's own operand-count or composite-element ceiling left no push credit, so a conformant document whose over-cap push was still correctly balanced by a later pop got an unbalanced-pop diagnostic it did not deserve. The depth caps already solved this for their own case; this extends the same credit to the operand-count and composite-element ceilings. The inline-image resync probe (ProbeOnce) treated an exception at the buffer's own true end exactly like a malformed byte found strictly inside the probe window: both were an outright reject. That is weaker evidence than a malformed byte inside the window, since it cannot tell "the file ends mid-token" apart from "this candidate sits inside image data whose next token happens to run to the file's end without closing." A new WeakReject outcome lets ScanForEi keep such a candidate as a fallback instead of losing an image and reporting a false 307. The CR-LF-after-ID handling preferred the wrong reading first: it read a CR immediately followed by LF as one two-byte separator, per §7.2.3's own EOL rule, then retried the one-byte reading only if that failed to land on 'EI'. §8.9.7 says only "a single white-space character," so the one-byte reading is now the primary one for the two length-verifiable tiers, with the two-byte fold as the retry; the unverifiable EI-scan tier keeps the fold, since it has no length to arbitrate the ambiguity with. Also: CollectFilterNames drops a non-name /Filter array element rather than keeping its position, so a wrong element could slide into the "final filter" check meant for position 0; an over-cap composite operand whose count pass hit a lex failure reported only the cap diagnostic, dropping the fact that the composite was also malformed; and several diagnostic-doc paragraphs described behaviour the code no longer matched (the inline-image callback for a disallowed filter, dedupe scope across Form XObjects, which content-stream failures the 300 code covers). --- .../Content/ContentInterpreter.cs | 425 +++++++++++++----- src/VellumPdf.Reader/Content/GraphicsState.cs | 12 +- src/VellumPdf.Reader/Content/TextState.cs | 5 +- .../PdfDocumentReader.Content.cs | 2 +- src/VellumPdf.Reader/PdfLexer.cs | 4 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 85 ++-- .../ContentInterpreterTests.cs | 405 ++++++++++++++++- 7 files changed, 769 insertions(+), 169 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index e7c9c640..287e3c69 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -75,7 +75,7 @@ internal sealed class ContentInterpreter // can interpret far more total content than its own /Contents ever declares. private const long MaxContentBytes = 64L * 1024 * 1024; - // The bounded resync probe (see ProbeOnce/LooksLikeResyncPoint) spends at most this many bytes + // The bounded resync probe (see ProbeOnce/ClassifyResyncPoint) spends at most this many bytes // of lexing across an ENTIRE Run, shared by every 'EI' candidate ScanForEi tries: bounding the // RUN's own total work, not each candidate's, is what keeps ScanForEi's amortised cost linear in // the content length rather than quadratic (#402 round 3; a per-candidate byte/token cap alone @@ -85,8 +85,11 @@ internal sealed class ContentInterpreter // each immediately followed by an unterminated string, "\" EI (\"" repeated, drove that cost to // 16.6 s per decoded MiB with no diagnostic to explain it). Once this budget is spent, every // later candidate in the Run is accepted unverified rather than probed at all (see - // ProbeOutcome.Exhausted): the total probe lexing one Run can ever do is bounded to this many - // bytes plus at most one further window, whatever the content size or candidate count. + // ProbeOutcome.Exhausted): the total probe lexing one Run can ever do is bounded to exactly + // this many bytes, whatever the content size or candidate count (#402 round 4: ProbeOnce's own + // window length is Math.Min(remaining, _probeBytesRemaining), so a window can never itself run + // past what is left of the budget; measured at exactly 16,777,216 charged at the 1, 4, and + // 16 MiB settings alike, not that plus a further window's own worth). private const long MaxProbeBytesPerRun = 16L * 1024 * 1024; private static readonly PdfName XObjectSubtypeForm = new("Form"); @@ -119,12 +122,19 @@ internal sealed class ContentInterpreter private ReadOnlyMemory _currentBuffer; // The resync probe's own remaining share of MaxProbeBytesPerRun, and whether it has already - // been spent this Run. Once _probeBudgetExhausted is true, LooksLikeResyncPoint accepts every + // been spent this Run. Once _probeBudgetExhausted is true, ClassifyResyncPoint accepts every // later 'EI' candidate without probing it at all (see ProbeOnce's Exhausted outcome), and - // HandleInlineImage reports that once, at the offset it first happened. + // HandleInlineImage reports that against EVERY later inline image this Run delimits, not only + // the one whose own scan spent the budget: the sink's own (code, object, page) dedupe key means + // a later image inside a DIFFERENT content stream (a different Form XObject, or the page's own + // content once a form already spent it) is not deduped against the first report at all, so the + // message names the offset AND the object number of the FIRST occurrence explicitly + // (_probeBudgetExhaustedAtObjectNumber), rather than letting a later report's own ctx attribute + // an offset from a DIFFERENT buffer to itself (#402 round 4). private long _probeBytesRemaining; private bool _probeBudgetExhausted; private int _probeBudgetExhaustedAtOffset; + private int? _probeBudgetExhaustedAtObjectNumber; // Pushes PushGraphicsState/PushMarkedContent dropped for being over MaxGraphicsStateDepth or // MaxMarkedContentDepth: a matching 'Q'/'EMC' consumes one of these before it may report an @@ -214,6 +224,7 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) _probeBytesRemaining = MaxProbeBytesPerRun; _probeBudgetExhausted = false; _probeBudgetExhaustedAtOffset = 0; + _probeBudgetExhaustedAtObjectNumber = null; ProbeBytesConsumed = 0; var diagnostics = _reader.CreateContentDiagnosticScope(); @@ -528,7 +539,7 @@ private void InterpretStream( // within the content budget, so the token-count cap has to be // consulted before the allocation it exists to bound, not after (#402 // round 3; see CompositeOperandWithinCap and MaxCompositeOperandElements). - if (CompositeOperandWithinCap(lexer)) + if (CompositeOperandWithinCap(lexer, out var countPassLexerFailed)) { lexer.Seek(offset); PushOperand(parser.ParseObject(), ctx, diagnostics, pageIndex); @@ -542,6 +553,24 @@ private void InterpretStream( $"{shape} operand exceeds {MaxCompositeOperandElements} tokens; " + "the operator taking it was dropped.", ctx.DiagObjectNumber, pageIndex: pageIndex); + if (countPassLexerFailed) + { + // The count pass itself hit a malformed byte inside the + // composite (an unterminated string, say) before it ever + // reached the cap-comparison this branch reports 309 for; that + // failure did not merely bail the count pass out early on count + // alone, so it gets its own 300 too (#402 round 4: an + // over-cap composite whose count pass failed this way used to + // report only the 309, silently ending interpretation of the + // rest of the stream with nothing to explain why, where the + // identical failure on an UNDER-cap composite already reported + // 300 through ParseObject's own re-parse). + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + "The content stream's syntax could not be interpreted past " + + "this point; interpretation of it stopped here.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } } break; @@ -689,16 +718,24 @@ private void ClearOperands() // MaxCompositeOperandElements before PdfObjectParser ever allocates a PdfObject for it. Every // token inside the composite counts, at any depth, because every one of them becomes an // allocation once materialised; closing delimiters are the exception, since they allocate - // nothing. Leaves the lexer positioned right after the matching close, so the caller can either - // seek back to re-parse it (within cap) or simply continue with the next token (over cap: - // nothing more from this composite is needed). An unterminated composite is judged by the - // same count: within the cap, the caller's ParseObject re-derives the failure and reports it - // (ContentStreamLexError, 300) the way it always has; over the cap it is dropped as over-cap - // without ever being parsed, since parsing it would allocate everything before failing. - private static bool CompositeOperandWithinCap(PdfLexer lexer) + // nothing. When the composite terminates cleanly, this leaves the lexer positioned right after + // the matching close, so the caller can either seek back to re-parse it (within cap) or move on + // to the next token (over cap: nothing more from this composite is needed); an + // unterminated composite (see lexerFailed below) leaves the lexer wherever the failed token + // left it instead, which is NOT necessarily right after any close (#402 round 4: qualifying + // this to the terminated case, since the unterminated one two paragraphs below already + // contradicted an unqualified claim here). An unterminated composite is judged by the same + // count: within the cap, the caller's ParseObject re-derives the failure and reports it + // (ContentStreamLexError, 300) the way it always has; over the cap, lexerFailed tells the + // caller to report that same 300 directly, since nothing will re-parse this composite to + // derive it the way the within-cap path does (#402 round 4: over the cap used to report only + // ContentLimitExceeded, silently dropping the fact that the composite was ALSO malformed and + // not merely oversized). + private static bool CompositeOperandWithinCap(PdfLexer lexer, out bool lexerFailed) { var depth = 1; var count = 0; + lexerFailed = false; while (depth > 0) { Token token; @@ -708,6 +745,7 @@ private static bool CompositeOperandWithinCap(PdfLexer lexer) } catch (InvalidDataException) { + lexerFailed = true; break; } @@ -781,6 +819,25 @@ private void HandleOperator( if (_operandOverflow) { arityOk = false; + // A 'q', 'BMC', or 'BDC' dropped here for one of this reader's own ceilings (an + // over-cap composite operand, or the 64-operand-per-operator cap in PushOperand) is + // still credited toward its own matching pop, the same as an over-DEPTH push already + // is in PushGraphicsState/PushMarkedContent: §14.6.2 puts no size bound on a property + // list, so a producer whose BDC's own dictionary operand happens to exceed this + // reader's own MaxCompositeOperandElements, and who then balances that BDC with an + // EMC, must not ALSO be accused of an unbalanced EMC purely because this reader + // declined to push the marked-content nesting it was asked to (#402 round 4). An + // arity-mismatch drop (e.g. '1 q') is producer-side, not this reader's own ceiling, so + // it keeps reporting through the branch below instead. + switch (name) + { + case "q": + _ignoredGsPushes++; + break; + case "BMC" or "BDC": + _ignoredMcPushes++; + break; + } ClearOperands(); // Already reported when the overflow itself happened. } else if (expected != ContentOperators.Variable && _operands.Count != expected) @@ -829,13 +886,17 @@ private void HandleOperator( } } - // A numeric or name operand this interpreter reads for its OWN state (cm/Tf/Td/etc. below) - // used to be handed to NumberOperand or a bare pattern match unchecked: a wrongly typed - // operand at the right arity silently substituted 0, or (for Do) silently dropped the - // invocation, with no diagnostic at all (#402 round 3). An operator this interpreter only - // forwards to the visitor untouched (w, J, the colour operators, ...) is exempt: its own - // operand types are the visitor's to type-check, not this interpreter's, since this - // interpreter never reads them for its own state. + // A numeric, name, or string operand this interpreter reads for its OWN state, or reads to + // look up a resource on its own behalf (cm/Tf/Td/'/"/gs/cs/CS/sh/Do/etc. below), used to be + // handed to NumberOperand or a bare pattern match unchecked: a wrongly typed operand at the + // right arity silently substituted 0, silently no-oped a resource lookup, or (for Do) + // silently dropped the invocation, with no diagnostic at all (#402 round 3; the resource- + // lookup operators gs/cs/CS/sh joined this check in round 4, since ValidateNamedResource and + // ValidateColorSpaceResource already read _operands[0] as a name and silently no-op on a + // wrong type otherwise). An operator this interpreter only forwards to the visitor untouched + // (w, J, the colour-setting operators, ...) is exempt: its own operand types are the + // visitor's to type-check, not this interpreter's, since this interpreter never reads them + // for its own state or its own resource lookups. if (!ValidateOperandTypes(name, ctx, diagnostics, pageIndex)) return; @@ -928,6 +989,22 @@ private void HandleOperator( _textState.MoveTextPosition(0, -_gs.Leading); break; + case "'": + // Table 107: "This operator shall have the same effect as the code: T* string Tj". + // T*'s own move (§9.4.3) runs here; the text-showing half stays the visitor's, the + // same as Tj's own string operand, which this interpreter never reads (#402 round 4). + _textState.MoveTextPosition(0, -_gs.Leading); + break; + + case "\"": + // Table 107: "This operator shall have the same effect as this code: aw Tw ac Tc + // string '". aw and ac land in the text state before the T*-equivalent move that + // "'" itself performs (#402 round 4). + _gs.WordSpacing = NumberOperand(0); + _gs.CharSpacing = NumberOperand(1); + _textState.MoveTextPosition(0, -_gs.Leading); + break; + case "BDC" or "BMC": PushMarkedContent(ctx, diagnostics, pageIndex); break; @@ -975,13 +1052,14 @@ private void EmitAndClear(string name, int offset, IContentVisitor visitor) private static bool IsNumericOperand(PdfObject obj) => obj is PdfInteger or PdfReal; // Type-checks the operands of every operator this interpreter reads for its own graphics or - // text state, ahead of the switch below that reads them: by the time this runs, the - // arity check above already guarantees _operands.Count matches each name's own Table A.1 arity, - // so only the TYPE of each operand is in question here. A mismatch reports - // OperandStackMalformed (the same code the 302 doc already covers "an operand of the wrong type - // where the arity is otherwise right" under) and drops the operator entirely, the way the - // dictionary-operand check just above does, rather than letting NumberOperand's own 0-default - // or a silent Do no-op mask the malformation (#402 round 3). + // text state, or for its own resource lookup, ahead of the switch below (or, for gs/cs/CS/sh, + // the resource-lookup helpers) that read them: by the time this runs, the arity check above + // already guarantees _operands.Count matches each name's own Table A.1 arity, so only the TYPE + // of each operand is in question here. A mismatch reports OperandStackMalformed (the same code + // the 302 doc already covers "an operand of the wrong type where the arity is otherwise right" + // under) and drops the operator entirely, the way the dictionary-operand check just above does, + // rather than letting NumberOperand's own 0-default or a silent Do/gs/cs/CS/sh no-op mask the + // malformation (#402 round 3; gs/cs/CS/sh joined this method in round 4). private bool ValidateOperandTypes(string name, StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) { bool ok; @@ -1009,6 +1087,23 @@ private bool ValidateOperandTypes(string name, StreamContext ctx, DiagnosticSink ok = _operands[0] is PdfName; break; + case "'": + ok = _operands[0] is PdfLiteralString or PdfHexString; + break; + + case "\"": + ok = IsNumericOperand(_operands[0]) && IsNumericOperand(_operands[1]) + && _operands[2] is PdfLiteralString or PdfHexString; + break; + + // Table 73 (§8.6.8): CS/cs's own operand is "name". Table 56 (§8.4.4): gs's own operand + // is "dictName". Table 76 (§8.7.4.2): sh's own operand is "name". All three are read for + // a resource lookup below (ValidateColorSpaceResource/HandleExtGState/ValidateNamedResource), + // the same reason Do's own name operand is checked here rather than left to the visitor. + case "cs" or "CS" or "gs" or "sh": + ok = _operands[0] is PdfName; + break; + default: return true; // Forwarded to the visitor untouched; not this interpreter's to check. } @@ -1236,6 +1331,8 @@ private void HandleDo( ctx.DiagObjectNumber, pageIndex: pageIndex); } + // Defensive only: ValidateOperandTypes (~:839) already guarantees a one-element, PdfName + // operand for 'Do' by the time HandleOperator dispatches here (#402 round 4). if (xobjectNameOperand is not PdfName xobjectName) return; @@ -1663,35 +1760,53 @@ private bool HandleInlineImage( // whitespace byte was consumed for every filter shape, so 'BI /F /A85 /L 48 ID <48-byte // payload> EI' (two spaces after ID) came out 3 bytes short: the fixed-at-one skip left the // payload's own second byte behind as data, and the rest re-lexed as content instead. - // §7.2.3: "The combination of a CARRIAGE RETURN followed immediately by a LINE FEED shall be - // treated as one EOL marker", so a CR immediately followed by an LF is consumed as that ONE - // separator, not as the separator plus a data byte, before any of the above. - var skipsExtraWhitespace = filterNames.Count > 0 - && filterNames[0].Value is "ASCIIHexDecode" or "ASCII85Decode"; - var consumedCrLf = false; + // Decided from the RAW /Filter array's own element 0, not filterNames[0]: CollectFilterNames + // drops a non-name element rather than keeping its place in the array, so '/F [null /A85]' + // would otherwise promote /A85 into filterNames[0] and skip whitespace as though IT were + // position 0, when the array's own position 0 is neither AHx nor A85 at all (#402 round 4). + var skipsExtraWhitespace = FirstFilterIsAsciiHexOrAscii85(dict); + + // §8.9.7: "the ID operator shall be followed by a single white-space character, and the + // next character shall be interpreted as the first byte of image data." Exactly ONE byte is + // consumed as that mandated separator below, even when it is a CR immediately followed by + // an LF, which §7.2.3's own EOL rule folds into one two-byte marker for LINE-ENDING + // purposes only; §8.9.7 does not import that fold, and unlike §7.3.8.1 (which names + // "CARRIAGE RETURN LINE FEED" outright on the one construct, a stream's own + // keyword-to-data boundary, where a spec author who wanted the fold applied there wrote it + // in), §8.9.7 says only "a single white-space character". isCrLf records whether the + // mandated byte happened to be a CR immediately followed by an LF, so tiers a/b below (each + // of which has a declared or computed length to verify a reading against) can retry the + // two-byte reading once should the one-byte reading fail to land on 'EI' (#402 round 4: an + // earlier version preferred the two-byte reading first and retried the one-byte reading + // second, which fed a payload's own leading LF byte to the ID separator instead of to the + // image data whenever a producer wrote a lone CR before a payload that happened to start + // with LF). + var isCrLf = false; + var oneByteSeparatorPos = lexer.Position; if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 && PdfLexer.IsWhitespaceByte((byte)separatorByte)) { - if (separatorByte == (byte)'\r' && lexer.Position + 1 < _currentBuffer.Length - && _currentBuffer.Span[lexer.Position + 1] == (byte)'\n') - { - lexer.Seek(lexer.Position + 2); - consumedCrLf = true; - } - else - { - lexer.Seek(lexer.Position + 1); - } + isCrLf = separatorByte == (byte)'\r' && lexer.Position + 1 < _currentBuffer.Length + && _currentBuffer.Span[lexer.Position + 1] == (byte)'\n'; + lexer.Seek(lexer.Position + 1); + oneByteSeparatorPos = lexer.Position; if (skipsExtraWhitespace) { while (lexer.TryPeek() is var extraByte && extraByte >= 0 && PdfLexer.IsWhitespaceByte((byte)extraByte)) lexer.Seek(lexer.Position + 1); + // NOTE 2's own "skip any further white-space [after the first]" already consumes a + // following LF the same way whether the CR alone or the CR LF pair is read as the + // mandated separator, so the two readings converge on the same position here: there + // is nothing left for tiers a/b's own retry, below, to try a second time. + oneByteSeparatorPos = lexer.Position; + isCrLf = false; } } + var foldedSeparatorPos = isCrLf ? oneByteSeparatorPos + 1 : oneByteSeparatorPos; - var dataStart = lexer.Position; + var dataStart = oneByteSeparatorPos; var length = TryLengthFromDictionary(dict, dataStart, ctx, diagnostics, pageIndex, out var lengthPastEnd); var usedTierA = length is not null; @@ -1707,7 +1822,15 @@ private bool HandleInlineImage( var lengthFromScan = false; if (length is null) { - var scanEnd = ScanForEi(dataStart); + // Tier c has no declared or computed length to verify either ID-separator reading + // against, so unlike tiers a/b just above it cannot retry between the two; it keeps + // §7.2.3's own CR-LF fold as its one reading instead (#402 round 4): the fold misjudges + // only a lone-CR separator immediately followed by an LF payload byte, a narrower + // failure mode than the one-byte reading's own would be here, which would prepend a + // spurious LF onto every ordinary CR LF producer's data, the common case, whenever this + // scan has no length to tell the two readings apart with. + dataStart = foldedSeparatorPos; + var scanEnd = ScanForEi(dataStart, ctx.DiagObjectNumber); if (scanEnd is null) { ReportProbeBudgetExhaustedIfNeeded(ctx, diagnostics, pageIndex); @@ -1736,24 +1859,26 @@ private bool HandleInlineImage( // (lengthFromScan) since it already IS that fallback. if (resyncPos is null && !lengthFromScan) { - // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for binary - // image data that reading is ambiguous: a producer may have meant only the CR as the ID - // separator, with the LF as the image's own first byte. Retry once with the data window - // shifted one byte earlier, since a payload that happens to begin with LF right after a - // CR separator is exactly the case the CR-LF-as-one-marker choice above would otherwise - // misjudge. The malformed report just below is skipped when this retry alone is what - // recovers the image: a conforming file whose payload happens to start with LF right - // after a lone CR separator must not carry a warning it recovered from cleanly - // (#402 round 2; reporting unconditionally before the retry even ran is what made a - // correctly-recovered file carry one anyway). The EI-scan fallback below is a - // DIFFERENT case: reaching it at all means the declared or computed length was wrong - // outright, not merely ambiguous, so recovering through IT still reports. + // dataStart is still the one-byte-separator reading here (lengthFromScan is false, so + // tier c's own reassignment above never ran). §8.9.7 gives that one-byte reading no way + // to tell a lone-CR separator immediately followed by an LF payload byte apart from a + // two-byte CR-LF separator, so when it fails to land on 'EI' and the mandated byte was a + // CR immediately followed by an LF, retry once with §7.2.3's own fold: both bytes + // consumed as the separator instead, since a producer that wrote a two-byte CR-LF + // separator is exactly the case the one-byte reading above would otherwise misjudge + // (#402 round 4; this mirrors, in the opposite direction, an earlier version that tried + // the fold FIRST and retried the one-byte reading second). The malformed report just + // below is skipped when this retry alone is what recovers the image: a conforming file + // recovered from cleanly must not carry a warning about it (#402 round 2). The EI-scan + // fallback below is a DIFFERENT case: reaching it at all means neither reading's + // declared or computed length landed on 'EI', not merely one of the two, so recovering + // through IT still reports. var recoveredViaCrRetry = false; - if (consumedCrLf) + if (isCrLf) { - var retryStart = dataStart - 1; + var retryStart = foldedSeparatorPos; var retryEnd = retryStart + length.Value; - if (retryStart >= 0 && retryEnd <= _currentBuffer.Length) + if (retryEnd <= _currentBuffer.Length) { var retryResync = SkipToEi(retryEnd); if (retryResync is not null) @@ -1776,7 +1901,7 @@ private bool HandleInlineImage( if (resyncPos is null) { - var scanEnd = ScanForEi(dataStart); + var scanEnd = ScanForEi(dataStart, ctx.DiagObjectNumber); if (scanEnd is not null) { length = scanEnd.Value - dataStart; @@ -1810,20 +1935,31 @@ private bool HandleInlineImage( return true; } - // Reports, once per Run (the sink's own (code, object, page) dedupe folds every later call), - // that the resync probe's own MaxProbeBytesPerRun budget ran out before a candidate 'EI' could - // be confirmed, so it (and every later candidate this Run) was accepted unverified instead - // (#402 round 3; see ProbeOnce's Exhausted outcome and LooksLikeResyncPoint). + // Reports that the resync probe's own MaxProbeBytesPerRun budget ran out before a candidate + // 'EI' could be confirmed, so it (and every later candidate this Run) was accepted unverified + // instead (#402 round 3; see ProbeOnce's Exhausted outcome and ClassifyResyncPoint). Called for + // every inline image this Run still delimits once the budget is spent, not only the one whose + // own scan spent it: the sink's own (code, object, page) dedupe collapses every call against + // the SAME object into one, but a later inline image inside a DIFFERENT content stream (a + // different Form XObject, or the page's own content once a form already spent the budget) is + // not deduped against that first report at all, so this names the offset AND the object number + // (or "the page's own content" when that object number is null) of the FIRST occurrence + // explicitly, rather than pairing the first offset with whatever object happens to be current + // on a later, separately-reported call (#402 round 4). private void ReportProbeBudgetExhaustedIfNeeded( StreamContext ctx, DiagnosticSink diagnostics, int pageIndex) { if (!_probeBudgetExhausted) return; + var firstSpentIn = _probeBudgetExhaustedAtObjectNumber is { } objectNumber + ? $"object {objectNumber}" + : "the page's own content"; ReportInlineImageMalformed( $"the resync probe's {MaxProbeBytesPerRun / (1024 * 1024)} MiB per-run byte budget was " - + $"spent before the candidate 'EI' at offset {_probeBudgetExhaustedAtOffset} could be " - + "confirmed; it, and every later candidate this run, was accepted without verification", + + $"first spent at offset {_probeBudgetExhaustedAtOffset} of {firstSpentIn}, before a " + + "candidate 'EI' there could be confirmed; that candidate, and every later candidate " + + "this run, was accepted without verification", ctx, diagnostics, pageIndex); } @@ -1851,6 +1987,22 @@ private List CollectFilterNames(PdfDictionary dict) return names; } + // NOTE 2's "final or only filter" is array position 0 (see the remarks above + // skipsExtraWhitespace's own assignment); this reads that position directly off the RAW + // /Filter value rather than off CollectFilterNames' own result, since a non-name element there + // is dropped rather than kept in place, which would otherwise let a later name slide into + // position 0 (#402 round 4). + private static bool FirstFilterIsAsciiHexOrAscii85(PdfDictionary dict) + { + var first = dict.Get(PdfName.Filter) switch + { + PdfName n => n, + PdfArray { Count: > 0 } arr => arr[0] as PdfName, + _ => null, + }; + return first?.Value is "ASCIIHexDecode" or "ASCII85Decode"; + } + // Tier (a): /L, PDF 2.0's own Table 91 entry (§8.9.7). See InlineImageAbbreviations' own remarks // for why this cites Table 91 rather than Table 93, which ISO 32000-2 reserves for a Form // XObject dictionary's unrelated entries (§8.10.2). @@ -1988,16 +2140,27 @@ private int ResolveComponentCount( } // Tier (c): scan for whitespace-EI-whitespace/EOF, accepted only when the bounded probe just - // past the candidate (LooksLikeResyncPoint) does not reject it. Residual gap this scan cannot - // close on its own (stated here and in the PR body, not fixed: closing it needs decoding the - // image data itself, out of scope for a byte-level scan): a '%' comment after a false 'EI' that - // runs to the end of its line can swallow the terminating 'EI' written on that same line, so - // the operator that follows the comment's own line accepts the false candidate with no - // diagnostic. The bytes are well-formed content either way, so this scan has no way to tell - // the two apart. - private int? ScanForEi(int dataStart) + // past the candidate (ClassifyResyncPoint) does not reject it outright. A WeakReject verdict + // (see ProbeOutcome's own remarks) is remembered as a fallback rather than rejected on the + // spot: the FIRST such candidate is kept while the scan keeps looking for a stronger one, and + // is returned only once the scan finds no Accept-or-Exhausted candidate at all (#402 round 4). + // Residual gap this scan cannot close on its own (stated here and in the PR body, not fixed: + // closing it needs decoding the image data itself, out of scope for a byte-level scan): any + // whitespace-delimited 'EI' followed by bytes that lex as a Table A.1 operator, as 'BI', or as + // the buffer's own end is indistinguishable, at the byte level, from a resync point the + // terminating 'EI' itself sets, so a false 'EI' immediately followed by any of those three + // shapes is accepted with no diagnostic. A '%' comment is the easiest worked example: one + // running from right after a false 'EI' to the end of its own line can swallow the LATER, + // terminating 'EI' written on that same line, so the operator that follows the comment's own + // line is what the probe actually sees, and it accepts through the ordinary Table A.1 rule with + // nothing to tell the two apart. The same blind spot applies to a coincidental operator inside + // compressed noise (" EI n ", " EI W ", " EI f " each + // truncate the image data with no 307 and hand the visitor a spurious operator) and to " EI BI " + // (the BI accept rule below): the bytes are well-formed content either way. + private int? ScanForEi(int dataStart, int? diagObjectNumber) { var span = _currentBuffer.Span; + int? weakCandidate = null; for (var i = dataStart; i + 1 < span.Length; i++) { if (span[i] != (byte)'E' || span[i + 1] != (byte)'I') @@ -2013,24 +2176,40 @@ private int ResolveComponentCount( if (!followedOk) continue; - if (!LooksLikeResyncPoint(after)) + var verdict = ClassifyResyncPoint(after, diagObjectNumber); + if (verdict == ResyncVerdict.Reject) continue; - // Strips the single white-space byte §8.9.7 excludes from the image data (the one - // delimiting 'EI'). §7.2.3 makes a CARRIAGE RETURN immediately followed by a LINE FEED - // ONE EOL marker, not two separate white-space bytes, so when that pair sits right - // before 'EI' both bytes are the delimiter, not just the LF (#402 round 3: stripping - // only the LF left the CR behind as the image's own trailing byte). - var dataEnd = i; - if (i > dataStart && PdfLexer.IsWhitespaceByte(span[i - 1])) - { - dataEnd = i - 1; - if (dataEnd > dataStart && span[dataEnd] == (byte)'\n' && span[dataEnd - 1] == (byte)'\r') - dataEnd--; - } - return dataEnd; + var dataEnd = TrimEiDelimiter(span, dataStart, i); + if (verdict == ResyncVerdict.Accept) + return dataEnd; + + // WeakReject: this candidate's own probe ran off the buffer's TRUE end mid-token + // (§8.9.7 gives this scan no way to tell "the file ends with a malformed token" apart + // from "this false 'EI' sits inside image data whose next token happens to run to the + // file's own end without closing"), which is weaker evidence than a malformed byte + // found strictly inside the probe window (still a Reject, above). Only the FIRST one is + // kept: a later, stronger candidate always wins over an earlier weak one. + weakCandidate ??= dataEnd; } - return null; + return weakCandidate; + } + + // Strips the single white-space byte §8.9.7 excludes from the image data (the one delimiting + // 'EI'). §7.2.3 makes a CARRIAGE RETURN immediately followed by a LINE FEED ONE EOL marker, not + // two separate white-space bytes, so when that pair sits right before 'EI' both bytes are the + // delimiter, not just the LF (#402 round 3: stripping only the LF left the CR behind as the + // image's own trailing byte). + private static int TrimEiDelimiter(ReadOnlySpan span, int dataStart, int eiOffset) + { + var dataEnd = eiOffset; + if (eiOffset > dataStart && PdfLexer.IsWhitespaceByte(span[eiOffset - 1])) + { + dataEnd = eiOffset - 1; + if (dataEnd > dataStart && span[dataEnd] == (byte)'\n' && span[dataEnd - 1] == (byte)'\r') + dataEnd--; + } + return dataEnd; } // Confirms an 'EI' candidate at exactly a known offset (used once tier a/b already computed a @@ -2066,13 +2245,22 @@ private enum ProbeOutcome Accept, /// An 'EI' or 'ID' keyword (still inside image data that continues to a LATER - /// 'EI'), a keyword containing a non-printable byte, or a lex failure that was not merely - /// this probe running out of budget. + /// 'EI'), a keyword containing a non-printable byte, or a lex failure found strictly inside + /// an unclipped window (a malformed byte the buffer's true end had nothing to do with). Reject, + /// A lex failure that ran off the buffer's own TRUE end (not this probe's own + /// window clip) trying to close a token: an unterminated literal or hex string with no more + /// buffer left to find its closing delimiter in, say. Weaker evidence than : + /// this scan cannot tell "the file genuinely ends mid-token" apart from "this false 'EI' sits + /// inside image data whose next token happens to run to the file's own end without closing" + /// (#402 round 4; see ScanForEi's own remarks on how this outcome is used as a fallback + /// rather than rejected outright). + WeakReject, + /// The probe's own share of MaxProbeBytesPerRun ran out before it reached an - /// Accept or Reject outcome. Treated as an accept by LooksLikeResyncPoint (see its own - /// remarks), but distinctly, so HandleInlineImage can report that this candidate was + /// Accept, Reject, or WeakReject outcome. Treated as an accept by ClassifyResyncPoint (see + /// its own remarks), but distinctly, so HandleInlineImage can report that this candidate was /// accepted unverified rather than confirmed. Exhausted, } @@ -2109,13 +2297,18 @@ private ProbeOutcome ProbeOnce(int pos) catch (InvalidDataException) { // Ran off the end of the window mid-token (an unterminated literal or hex string, - // say): Exhausted only when that running-off was purely the budget's own limit - // (window clipped, and the lexer's own Position landed at or past the window's own - // length trying to close the token); anything else, including a malformed byte - // found strictly inside an unclipped window, rejects outright. - outcome = windowClipped && probe.Position >= window.Length - ? ProbeOutcome.Exhausted - : ProbeOutcome.Reject; + // say). Three cases share this one catch block, distinguished by WHICH end the + // token ran off: the budget's own artificial clip (Exhausted, windowClipped and the + // lexer's own Position landed at or past the window's own length), the buffer's own + // TRUE end with nothing left to close the token (WeakReject, the same Position + // check but an unclipped window: #402 round 4), or a malformed byte found strictly + // inside an unclipped window, short of either end (Reject outright). + outcome = (windowClipped, probe.Position >= window.Length) switch + { + (true, true) => ProbeOutcome.Exhausted, + (false, true) => ProbeOutcome.WeakReject, + _ => ProbeOutcome.Reject, + }; break; } @@ -2187,22 +2380,42 @@ private ProbeOutcome ProbeOnce(int pos) return outcome; } - private bool LooksLikeResyncPoint(int pos) + // ScanForEi's own verdict on one candidate: Accept ends the scan immediately, Reject moves on + // to the next candidate with nothing kept, and WeakReject moves on too but leaves the candidate + // behind as a fallback ScanForEi returns if nothing stronger ever turns up (#402 round 4). + private enum ResyncVerdict + { + Accept, + Reject, + WeakReject, + } + + private ResyncVerdict ClassifyResyncPoint(int pos, int? diagObjectNumber) { var outcome = ProbeOnce(pos); if (outcome == ProbeOutcome.Exhausted) { // Accepted unverified, once and for the rest of this Run: HandleInlineImage reports - // this the first time it happens, naming the offset accepted without verification, so a - // caller can tell "the interpreter confirmed this resync point" apart from "the + // this the first time it happens, naming the offset AND object number this candidate + // was accepted at without verification (#402 round 4: recording diagObjectNumber here, + // alongside the offset, is what lets a LATER report against a different content + // stream's own object number still name where the budget actually ran out, rather than + // pairing this offset with whatever object happens to be current when it is reported), + // so a caller can tell "the interpreter confirmed this resync point" apart from "the // interpreter ran out of budget and took its best guess" (#402 round 3). if (!_probeBudgetExhausted) { _probeBudgetExhausted = true; _probeBudgetExhaustedAtOffset = pos; + _probeBudgetExhaustedAtObjectNumber = diagObjectNumber; } - return true; + return ResyncVerdict.Accept; } - return outcome == ProbeOutcome.Accept; + return outcome switch + { + ProbeOutcome.Accept => ResyncVerdict.Accept, + ProbeOutcome.WeakReject => ResyncVerdict.WeakReject, + _ => ResyncVerdict.Reject, + }; } } diff --git a/src/VellumPdf.Reader/Content/GraphicsState.cs b/src/VellumPdf.Reader/Content/GraphicsState.cs index d4292feb..75b019ee 100644 --- a/src/VellumPdf.Reader/Content/GraphicsState.cs +++ b/src/VellumPdf.Reader/Content/GraphicsState.cs @@ -20,18 +20,20 @@ internal sealed class GraphicsState /// The current transformation matrix, concatenated by cm (§8.3.4). internal Matrix Ctm { get; set; } = Matrix.Identity; - /// Character spacing, Tc (§9.3.2). Added to the horizontal or vertical - /// component of each glyph's displacement, depending on the writing mode. + /// Character spacing, Tc (§9.3.2), also settable by "'s own ac + /// operand (Table 107). Added to the horizontal or vertical component of each glyph's + /// displacement, depending on the writing mode. internal double CharSpacing { get; set; } - /// Word spacing, Tw (§9.3.3). Added only after a single-byte code 32. + /// Word spacing, Tw (§9.3.3), also settable by "'s own aw operand + /// (Table 107). Added only after a single-byte code 32. internal double WordSpacing { get; set; } /// Horizontal scaling, Tz (§9.3.4), as a percentage; 100 is unscaled. internal double HorizontalScaling { get; set; } = 100; - /// Leading, TL (§9.3.5): the line-to-line advance T* and ' use, and - /// what TD sets from its own ty operand. + /// Leading, TL (§9.3.5): the line-to-line advance T*, ', and + /// " use, and what TD sets from its own ty operand. internal double Leading { get; set; } /// diff --git a/src/VellumPdf.Reader/Content/TextState.cs b/src/VellumPdf.Reader/Content/TextState.cs index a6f93040..a4018cf3 100644 --- a/src/VellumPdf.Reader/Content/TextState.cs +++ b/src/VellumPdf.Reader/Content/TextState.cs @@ -10,8 +10,9 @@ namespace VellumPdf.Reader.Content; /// specified only within a text object and shall not persist from one text object to the /// next", so they are never saved or restored by q/Q the way §8.4.4's own /// graphics-state parameters are. Only BT resets them (to identity), and only Td, -/// TD, Tm, and T* update them. Not stacked; the interpreter owns exactly -/// one live instance. +/// TD, Tm, T*, and the two Table 107 line-showing operators ' and +/// " (each defined as having the same effect as T* followed by a text-showing +/// operator) update them. Not stacked; the interpreter owns exactly one live instance. /// internal sealed class TextState { diff --git a/src/VellumPdf.Reader/PdfDocumentReader.Content.cs b/src/VellumPdf.Reader/PdfDocumentReader.Content.cs index 64e2b4ee..7408edb1 100644 --- a/src/VellumPdf.Reader/PdfDocumentReader.Content.cs +++ b/src/VellumPdf.Reader/PdfDocumentReader.Content.cs @@ -7,7 +7,7 @@ public sealed partial class PdfDocumentReader { /// /// Creates a fresh scope (see ) - /// forwarding into this reader's own diagnostics. ContentInterpreter.Run is the first real + /// forwarding into this reader's own diagnostics. ContentInterpreter.Run is the first /// caller of (see that method's own remarks), creating /// one scope per page interpreted so a caller who runs the interpreter over the same page more /// than once (text extraction, then image extraction, per #98) gets a fresh, page-scoped diff --git a/src/VellumPdf.Reader/PdfLexer.cs b/src/VellumPdf.Reader/PdfLexer.cs index 28b9576c..b653809d 100644 --- a/src/VellumPdf.Reader/PdfLexer.cs +++ b/src/VellumPdf.Reader/PdfLexer.cs @@ -121,11 +121,11 @@ internal PdfLexer(ReadOnlyMemory data, bool contentStreamMode) _contentStreamMode = contentStreamMode; } - // ── ISO 32000-2 §7.2.2 — whitespace bytes ───────────────────────────── + // ── ISO 32000-2 §7.2.3 Table 1: whitespace bytes ────────────────────── private static bool IsWhitespace(byte b) => b is 0 or 9 or 10 or 12 or 13 or 32; - // ── ISO 32000-2 §7.2.2 — delimiter bytes ────────────────────────────── + // ── ISO 32000-2 §7.2.3 Table 2: delimiter bytes ─────────────────────── private static bool IsDelimiter(byte b) => b is (byte)'(' or (byte)')' or (byte)'<' or (byte)'>' or (byte)'[' or (byte)']' or (byte)'{' or (byte)'}' diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 55aead60..6c0d85a9 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -366,6 +366,16 @@ public enum PdfReaderDiagnosticCode /// that already-concatenated buffer, so it ends interpretation for the rest of the page's /// content outright, later /Contents array elements included, not merely the one /// stream the failure happened to fall in. + /// + /// Two further cases report this same code but concern neither a /Contents element nor + /// the page's own concatenated buffer (#402 round 4): a Form XObject's own content stream + /// (§8.10) failing to decode, reported against that form's own object number and skipping only + /// that one Do invocation's descent, the rest of the page's content (and any sibling + /// invocation of the same form) unaffected; and an reaching + /// ContentInterpreter.Run's own outer catch from a malformed indirect-reference chain met + /// while resolving a resource, XObject, or nested Form XObject, which ends interpretation of the + /// whole page (nothing narrower than the page itself is available to resume from at that point). + /// /// ContentStreamLexError = 300, @@ -396,16 +406,19 @@ public enum PdfReaderDiagnosticCode /// invoked with the wrong operand count for its own arity (Annex A Table A.1), an operand of /// the wrong type where the arity is otherwise right: a TJ whose single operand is not /// an array (§9.4.3); a non-numeric operand to cm, Tc, Tw, Tz, - /// TL, Tf's second, Tr, Ts, Td, TD, or Tm; or a - /// non-name operand to Tf's first or to Do (#402 round 3; every OTHER operator - /// this interpreter recognises only forwards its own operands to the visitor untouched, so - /// their operand types are the visitor's to type-check, not this interpreter's); a Do - /// that occurred inside a text object (§8.2 Figure 9 admits no operator of Table 50's XObjects - /// category there), an unbalanced Q with no matching q on the graphics-state - /// stack, or an unbalanced EMC with no matching BMC/BDC (§14.6.1). An - /// unbalanced q still open at the end of a content stream is not reported: nothing - /// downstream of this interpreter needs the graphics state restored past the last operator it - /// saw. + /// TL, Tf's second, Tr, Ts, Td, TD, or Tm; a + /// non-name operand to Tf's first, to Do, or to gs/cs/CS/sh + /// (#402 round 4: this reader reads that operand for its own resource lookup, the same reason + /// Do's own name operand is checked); a non-string operand to ', or a non-numeric + /// first or second or non-string third operand to " (#402 round 4; Table 107) (#402 + /// round 3 for the rest of this list; every operator this interpreter recognises but does not + /// name above only forwards its own operands to the visitor untouched, so their operand types + /// are the visitor's to type-check, not this interpreter's); a Do that occurred inside a + /// text object (§8.2 Figure 9 admits no operator of Table 50's XObjects category there), an + /// unbalanced Q with no matching q on the graphics-state stack, or an unbalanced + /// EMC with no matching BMC/BDC (§14.6.1). An unbalanced q still + /// open at the end of a content stream is not reported: nothing downstream of this interpreter + /// needs the graphics state restored past the last operator it saw. /// OperandStackMalformed = 302, @@ -430,17 +443,15 @@ public enum PdfReaderDiagnosticCode /// /// A single page invoked Form XObjects (Do invocations that reached the form, counted /// across the whole page, not per subtree, and including one whose content then failed to - /// decode or was skipped for the run's own content budget; #402 round 3 fixed this doc, which - /// previously said "successful" recursions, to say so) more than 4096 times. ISO 32000-2 places - /// no limit of its own on how deeply or how often a page may invoke Do (§8.10); this cap - /// is this reader's own bound against a wide, shallow invocation graph a depth cap alone would - /// not catch. Descent - /// into any further form stops for the rest of the page; operators already reported before the - /// budget was reached are kept, and interpretation of the page's own (non-form) content - /// continues past the point where the budget was hit. Reported through - /// DiagnosticSink.ReportRetained: a condition that ends the page's own form recursion for - /// good is worth surfacing even once is spent on - /// earlier, unrelated conditions. + /// decode or was skipped for the run's own content budget (#402 round 3)) more than 4096 + /// times. ISO 32000-2 places no limit of its own on how deeply or how often a page may invoke + /// Do (§8.10); this cap is this reader's own bound against a wide, shallow invocation + /// graph a depth cap alone would not catch. Descent into any further form stops for the rest of + /// the page; operators already reported before the budget was reached are kept, and + /// interpretation of the page's own (non-form) content continues past the point where the + /// budget was hit. Reported through DiagnosticSink.ReportRetained: a condition that ends + /// the page's own form recursion for good is worth surfacing even once + /// is spent on earlier, unrelated conditions. /// FormXObjectBudgetExceeded = 305, @@ -474,18 +485,26 @@ public enum PdfReaderDiagnosticCode /// does not land on the following EI operator, in which case this reader retries the /// EI scan before giving up, or the resync probe spending its whole per-run byte budget /// before it could confirm a candidate EI, in which case that candidate is accepted - /// unverified rather than left unresolved. The inline-image callback IS raised whenever the - /// image was still delimited by the time this reports, including after an /L-past-the-end - /// or did-not-land-on-EI recovery, or a probe-budget-exhausted acceptance; it is not - /// raised only when the image could not be delimited at all. When the data was still delimited - /// (a disallowed filter, a length the EI scan recovered, or a probe-budget-exhausted - /// acceptance) only the image is skipped and interpretation of the rest of the content stream - /// continues; when it could not be delimited at all (no ID, no EI) interpretation - /// of that stream stops there, since nothing past that point can be resynchronised reliably. - /// Reported at most once per page: the sink's dedupe key is (code, object, page), and every - /// image on one content stream reports against that same object (or, for the page's own - /// top-level content specifically, the same ), so a second inline image - /// on the same page with its own, different malformation is not listed separately. + /// unverified rather than left unresolved. The inline-image callback is raised whenever the + /// image was delimited AND its filter chain is one this reader accepts on an inline image, + /// including after an /L-past-the-end or did-not-land-on-EI recovery, or a + /// probe-budget-exhausted acceptance (#402 round 4: a disallowed filter delimits the image just + /// as successfully as an accepted one, so "was delimited" alone is not what decides whether the + /// callback fires). A disallowed filter (JBIG2Decode, JPXDecode, or Crypt) + /// still delimits the image, so interpretation of the rest of the content stream continues past + /// its EI, but skips the callback for that image; when the image could not be delimited + /// at all (no ID, no EI) interpretation of that stream stops there instead, since + /// nothing past that point can be resynchronised reliably. + /// Reported at most once per content stream (the page's own content, or one Form XObject) per + /// page: the sink's dedupe key is (code, object, page), and every image on one content stream + /// reports against that stream's own object number (or, for the page's own top-level content + /// specifically, ), so a second inline image on the SAME content stream + /// with its own, different malformation is not listed separately, but the same page's own + /// content and each Form XObject it draws dedupe independently, since each carries a distinct + /// object number here (#402 round 4: an earlier version of this doc said "per page" outright, + /// which undercounts a page invoking two forms that each carry their own malformed inline + /// image, since the two reports land against two different object numbers and are not deduped + /// against each other at all). /// InlineImageMalformed = 307, diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 1aba9638..5f6011be 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -156,7 +156,11 @@ public void OperandTypes_areParsedWithExactValuesAndShapes() + "[(A) -120 (B) [1 2] 5] TJ\n" + "/Span << /MCID 1 /Foo (bar) >> BDC\n" + "EMC\n" - + "true false null \"\n"; + // 'sc' (Table 73) takes a variable operand count and forwards it to the visitor + // untouched (#402 round 4: moved off '\"', once '\"' gained its own Table 107 operand + // type check, so this line still pins boolean/null keyword parsing on an operator that + // does not type-check them). + + "true false null sc\n"; var (_, _, visitor) = Run(BuildPageDoc(content)); @@ -204,11 +208,11 @@ public void OperandTypes_areParsedWithExactValuesAndShapes() Assert.Contains(visitor.Operators, o => o.Op == "EMC" && o.Operands.Count == 0); - var quote = visitor.Operators.Single(o => o.Op == "\""); - Assert.Equal(3, quote.Operands.Count); - Assert.Same(PdfBoolean.True, quote.Operands[0]); - Assert.Same(PdfBoolean.False, quote.Operands[1]); - Assert.Same(PdfNull.Instance, quote.Operands[2]); + var sc = visitor.Operators.Single(o => o.Op == "sc"); + Assert.Equal(3, sc.Operands.Count); + Assert.Same(PdfBoolean.True, sc.Operands[0]); + Assert.Same(PdfBoolean.False, sc.Operands[1]); + Assert.Same(PdfNull.Instance, sc.Operands[2]); } // ── BX/EX compatibility sections ──────────────────────────────────────────────────────────── @@ -482,6 +486,38 @@ public void SmallUnterminatedArrayOperand_stillReportsTheLexError_notTheCap() Assert.Equal(["w"], visitor.Operators.Select(o => o.Op)); } + [Fact] + public void OverCapArrayOperand_withAMalformedTokenInside_reportsTheLexErrorAlongsideTheCap() + { + // 9000 elements is over MaxCompositeOperandElements (8192), and the unterminated string + // right after them is its own lex failure (no closing ')' anywhere in the rest of the + // buffer), not merely the count pass bailing out at the cap. Before this fix, the count + // pass's own catch swallowed that failure silently: only 309 was reported, with nothing to + // explain why interpretation also stopped dead right there, even though the identical + // failure UNDER the cap (see the next test) already reported 300 through ParseObject's own + // re-parse (#402 round 4). + var content = "[" + string.Concat(Enumerable.Repeat("1 ", 9000)) + "(unterminated\n] TJ\n1 w\n"; + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Empty(visitor.Operators); + } + + [Fact] + public void UnderCapArrayOperand_withAMalformedTokenInside_reportsOnlyTheLexError() + { + // Same malformed string, well under the cap: ParseObject's own re-parse already derives + // this as ContentStreamLexError, with no ContentLimitExceeded at all, since the count pass + // never came near the cap. + const string content = "[1 1 (unterminated\n] TJ\n2 w\n"; + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Empty(visitor.Operators); + } + [Fact] public void UnbalancedQ_isIgnoredWithADiagnostic() { @@ -531,6 +567,63 @@ public void BalancedBmcAndEmc_pastTheMarkedContentCap_reportsOnlyTheLimit_andSta Assert.Contains(visitor.Operators, o => o.Op == "w"); } + // ── A push dropped for one of this reader's own ceilings (not the depth cap above, which + // already credits its own drops) still credits its matching pop, so 'EMC'/'Q' does not report + // an unbalanced pop purely because this reader declined to push the state it was asked to + // (#402 round 4) ───────────────────────────────────────────────────────────────────────────── + + [Fact] + public void BdcDroppedForAnOverCapPropertiesDictionary_creditsTheMatchingEmc() + { + // §14.6.2 puts no size bound on a property list. 5000 key/value pairs is 10,000 tokens, + // over MaxCompositeOperandElements (8192), so BDC's own dictionary operand is dropped and + // BDC itself never reaches the visitor; before this fix, the balancing EMC then reported an + // unbalanced pop on top of the 309, even though the document itself is conformant. + var props = string.Concat(Enumerable.Range(0, 5000).Select(i => $"/K{i} {i} ")); + var content = $"/Tag << {props}>> BDC\nEMC\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(visitor.Operators, o => o.Op == "BDC"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void QDroppedForTheOperandCountCap_creditsTheMatchingQ() + { + // 65 numeric operands ahead of 'q' overflow MaxOperandsPerOperator (64), so 'q' itself is + // dropped for the SAME reason PushOperand already reported (no second diagnostic); the + // matching 'Q' must still consume that credit silently rather than report an unbalanced pop. + var content = string.Join(' ', Enumerable.Repeat("1", 65)) + " q\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + Assert.DoesNotContain(visitor.Operators, o => o.Op == "q"); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Fact] + public void QDroppedForTheOperandCountCap_creditsOnlyOneQ_aSecondQStillReportsUnbalanced() + { + // Only ONE 'q' was dropped, so only one credit exists: a second 'Q' past it finds nothing + // pushed and no credit left, and reports the ordinary unbalanced-pop diagnostic. + var content = string.Join(' ', Enumerable.Repeat("1", 65)) + " q\nQ\nQ\n1 w\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + var unbalanced = reader.Diagnostics + .Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(unbalanced); + Assert.Contains("'Q'", unbalanced[0].Message, StringComparison.Ordinal); + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + // ── q/Q/cm matrix state, text state ────────────────────────────────────────────────────────── [Fact] @@ -661,6 +754,75 @@ public void Do_withANonNameOperand_reportsOnce_andNeverReachesTheVisitor() Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); } + // ── ' and " (Table 107 line-showing operators, #402 round 4) ─────────────────────────────────── + + [Fact] + public void Quote_movesToTheNextLine_andForwardsTheStringOperandUntouched() + { + // Table 107: "' ... shall have the same effect as the code: T* string Tj". Before this fix, + // ' fell through to the default (state-inert) case: the text matrices were left untouched + // and the string operand still reached the visitor, since forwarding happens either way. + const string content = "BT\n100 TL\n5 0 Td\n(a) '\nET\n"; + var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + + // Td moves the line matrix to [1 0 0 1 5 0]; ' then moves by (0, -TL=-100): + // Tlm_new = [1 0 0 1 0 -100] x [1 0 0 1 5 0] = [1 0 0 1 5 -100]. + Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), interpreter.TextState.TextLineMatrix); + Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), interpreter.TextState.TextMatrix); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + + var quote = visitor.Operators.Single(o => o.Op == "'"); + var str = Assert.Single(quote.Operands); + Assert.Equal("a", Encoding.ASCII.GetString(((PdfLiteralString)str).Bytes.Span)); + } + + [Fact] + public void DoubleQuote_setsWordAndCharSpacing_thenMovesToTheNextLine() + { + // Table 107: "\" ... shall have the same effect as this code: aw Tw ac Tc string '": aw and + // ac land in the text state before the T*-equivalent move. + const string content = "BT\n50 TL\n7 8 (x) \"\nET\n"; + var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + + Assert.Equal(7, interpreter.GraphicsState.WordSpacing); + Assert.Equal(8, interpreter.GraphicsState.CharSpacing); + // TextLineMatrix starts at identity (no Td/Tm ran); " moves by (0, -TL=-50): + // Tlm_new = [1 0 0 1 0 -50] x [1 0 0 1 0 0] = [1 0 0 1 0 -50]. + Assert.Equal(new Matrix(1, 0, 0, 1, 0, -50), interpreter.TextState.TextLineMatrix); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); + + var quote = visitor.Operators.Single(o => o.Op == "\""); + Assert.Equal(3, quote.Operands.Count); + } + + [Fact] + public void DoubleQuote_withNonNumericSpacingOperands_reportsOnce_andIsDropped() + { + const string content = "BT\n(a) (b) (x) \"\nET\n"; + var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(visitor.Operators, o => o.Op == "\""); + Assert.Equal(0, interpreter.GraphicsState.WordSpacing); + Assert.Equal(0, interpreter.GraphicsState.CharSpacing); + Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(reports); + } + + [Fact] + public void Quote_withANonStringOperand_reportsOnce_andIsDropped() + { + const string content = "BT\n5 '\nET\n"; + var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + + Assert.DoesNotContain(visitor.Operators, o => o.Op == "'"); + Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(reports); + } + // ── gs with /Font ──────────────────────────────────────────────────────────────────────────── [Fact] @@ -687,6 +849,29 @@ public void Gs_namingAMissingExtGState_reportsResourceMissing() Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); } + // ── gs/cs/CS/sh read their own operand for a resource lookup, so a wrong type reports 302 the + // same way Do's own does, rather than the lookup silently no-op'ing (#402 round 4) ──────────── + + [Theory] + [InlineData("12 gs\n")] + [InlineData("5 sh\n")] + [InlineData("(x) cs\n")] + [InlineData("7 CS\n")] + public void ResourceLookupOperator_withANonNameOperand_reportsOnce_andNeverReachesTheVisitor( + string content) + { + // Before this fix: HandleExtGState/ValidateColorSpaceResource/ValidateNamedResource each + // read _operands[0] as a name unchecked, so a numeric or string operand there silently + // no-op'd the resource lookup with no diagnostic at all, the same gap Do had before round 3. + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Empty(visitor.Operators); + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) + .ToList(); + Assert.Single(reports); + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ResourceMissing); + } + // ── Form XObjects ──────────────────────────────────────────────────────────────────────────── private static Obj[] BuildFormChain(int count, string leafExtra = "") @@ -1466,6 +1651,31 @@ public void FilterArrayWithA85Second_doesNotSkipExtraWhitespace_flateOwnsThePosi Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void FilterArrayWithANonNameElementZero_doesNotSkipExtraWhitespace() + { + // '/F [null /A85]': CollectFilterNames drops the non-name element rather than keeping its + // place in the array, so a naive filterNames[0] read would see /A85 there and wrongly skip + // extra whitespace as though A85 itself sat at position 0. The raw array's own position 0 + // is null, neither AHx nor A85, so no extra whitespace should be skipped here at all + // (#402 round 4). The payload's own leading byte is a space (0x20): if the bug were still + // present, the extra-whitespace-skip loop would eat it as though it followed a + // single-white-space-then-skip-more rule that does not apply to this filter shape. + byte[] payload = [0x20, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68]; + var content = "BI /F [null /A85] /L 8 ID "u8.ToArray() + .Concat(payload) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(payload)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + [Fact] public void FilteredAHx_usesTheEiScan() { @@ -1506,10 +1716,10 @@ public void DctInlineImage_withAFalseEiInsideItsData_isSkippedByTheScan() [Fact] public void DctInlineImage_withAFalseEiFollowedByBinaryNoise_isSkippedByTheScan() { - // The harder false candidate: " EI " followed by bytes outside ISO 32000-2 §7.2.2's - // whitespace and delimiter sets, which the lexer accepts as one Keyword token without - // throwing. Only the probe's operator check (the keyword is not in Annex A Table A.1) - // rejects this one; the real EI follows and is accepted. + // The harder false candidate: " EI " followed by bytes outside ISO 32000-2 §7.2.3 Tables 1 + // and 2's whitespace and delimiter sets, which the lexer accepts as one Keyword token + // without throwing. Only the probe's operator check (the keyword is not in Annex A Table + // A.1) rejects this one; the terminating EI follows and is accepted. var falseCandidate = " EI "u8.ToArray(); byte[] noiseAfter = [0x8F, 0x12, 0xC4, 0x7A, 0x20, 0xFE, 0x01]; byte[] jpegNoise = [0xFF, 0xD8, 0xFF, 0xE0, 0x00]; @@ -1918,6 +2128,42 @@ public void RealEi_followedByATerminated4000ByteLiteral_isAccepted() Assert.Equal(longLiteral, Encoding.ASCII.GetString(str.Bytes.Span)); } + // ── The terminating 'EI' followed by a token that runs off the buffer's own TRUE end (not + // merely a probe window) is weaker evidence than a malformed byte found strictly inside an + // unclipped window, so it is accepted as a fallback rather than rejected outright the way that + // stronger case still is (#402 round 4; see ProbeOutcome.WeakReject and ScanForEi) ───────────── + + [Theory] + [InlineData("(abc")] // unterminated literal string, runs to the buffer's true end + [InlineData(" d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(jpeg)); + Assert.Empty(visitor.Operators); + } + // ── A failed tier-a/tier-b end falls back to the scan instead of losing the stream (#402) ── [Fact] @@ -1964,18 +2210,63 @@ public void IdFollowedByCrLf_withExplicitL_treatsBothBytesAsOneSeparator() Assert.Contains(visitor.Operators, o => o.Op == "w"); } + [Fact] + public void CrAsSingleSeparator_withDataStartingWithLf_landsOnEiWithNoRetry() + { + // §8.9.7's own single-white-space-byte reading, taken as this reader's primary one (#402 + // round 4): the separator is a lone CR, and the image's own declared length (/L 4) starts + // right after it, at the LF the producer meant as the image's own first data byte. Folding + // CR LF together here (the old primary reading) would have eaten that LF as part of the + // separator instead, dropping it from the delivered data. + byte[] data = [0x0A, 0x41, 0x42, 0x43]; + var raw = "BI /W 4 /H 1 /BPC 8 /CS /G /L 4 ID\r"u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(raw); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void CrLfSeparator_whoseOneByteReadingMissesEi_recoversViaTheRetry() + { + // The twin of the repro just above: here the producer wrote a two-byte CR LF separator + // (§7.2.3's own fold), so the one-byte-first primary reading (CR alone) lands one + // byte short of 'EI' and must retry with both bytes consumed, the mirror image of the retry + // 'CrAsSingleSeparator_withDataStartingWithLf_landsOnEiWithNoRetry' does not need. + byte[] data = "ABCD"u8.ToArray(); + var raw = "BI /W 4 /H 1 /BPC 8 /CS /G /L 4 ID\r\n"u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(raw); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + [Fact] public void CrAsSingleSeparator_withDataStartingWithLf_recoversWithoutAWarning() { - // §7.2.3 treats a CR immediately followed by an LF as one EOL marker, but for binary image - // data that reading is ambiguous: here the separator is a lone CR, and the image's own - // first byte IS LF, so wrongly consuming both as the marker shifts the whole data window - // one byte late. With no whitespace between the (wrongly windowed) data and 'EI', - // the shifted window's own SkipToEi check fails outright (it lands one byte INTO 'EI' - // rather than in front of it), so the one-byte-earlier retry runs and recovers the correct - // 5-byte window (LF,'A','B','C','D'). Before the fix, InlineImageMalformed was reported - // unconditionally before that retry ever ran, so this conforming file carried a warning - // even though the retry recovered it cleanly (#402 round 2). + // §8.9.7 mandates exactly ONE white-space byte after 'ID'; here that byte is a lone CR, and + // the image's own first byte happens to be LF. §7.2.3's CR-LF-as-one-EOL-marker fold has no + // role here (#402 round 4): this reader's own primary reading takes only the CR as the + // separator, so the 5-byte window (LF,'A','B','C','D') this repro needs is exactly what the + // primary reading produces on the first try now, landing directly on 'EI' with no retry at + // all. Before round 4's inversion, the primary reading folded CR LF together, shifting the + // window one byte late and needing a retry to recover; before round 2, that retry existed + // but InlineImageMalformed was reported unconditionally ahead of it regardless of outcome, + // so a conforming file recovered cleanly still carried a warning about it. var raw = "BI /W 5 /H 1 /BPC 8 /CS /G ID\r"u8.ToArray() .Concat("\nABCDEI\nQ\n1 w\n"u8.ToArray()) .ToArray(); @@ -2145,6 +2436,70 @@ public void JpxDecodeInlineImage_reportsMalformed_andTheStreamContinuesAfterIt() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + // §8.9.7 excludes JBIG2Decode, JPXDecode, and Crypt from inline-image use; the disallowed + // filter still delimits the image (§8.9.7 gives this reader everything it needs to find the + // 'EI'), so interpretation continues past it, but the callback is skipped for that one image + // (#402 round 4: pinning all three filter names, not JPXDecode alone, and pinning that the + // callback itself, not merely the diagnostic, is skipped). + + [Fact] + public void Jbig2DecodeInlineImage_reportsMalformed_andTheStreamContinuesAfterIt() + { + var content = "BI /F /JBIG2Decode /L 3 ID abc EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Empty(visitor.InlineImages); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + [Fact] + public void CryptInlineImage_reportsMalformed_andTheStreamContinuesAfterIt() + { + var content = "BI /F /Crypt /L 3 ID abc EI\nQ\n"; + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Empty(visitor.InlineImages); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + // ── 307's own (code, object, page) dedupe is per CONTENT STREAM, not per page: a page invoking + // two forms, each carrying its own malformed inline image, reports two 307s, one against each + // form's own object number, since the sink never sees the same key twice (#402 round 4) ────── + + [Fact] + public void InlineImageMalformed_inTwoDifferentForms_reportsTwice() + { + var formBody = "BI /F /JPXDecode /L 3 ID abc EI\nQ\n"u8.ToArray(); + var content = "/A Do\n/B Do\n"; + var doc = BuildPageDoc( + content, "<< /XObject << /A 11 0 R /B 12 0 R >> >>", + new Obj(11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", formBody), + new Obj(12, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", formBody)); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics + .Where(d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed) + .ToList(); + Assert.Equal(2, reports.Count); + Assert.Empty(visitor.InlineImages); + Assert.Equal(2, visitor.Operators.Count(o => o.Op == "Q")); + } + + [Fact] + public void InlineImageMalformed_onThePagesOwnContentAlone_reportsOnce() + { + var content = "BI /F /JPXDecode /L 3 ID abc EI\nQ\n"; + + var (reader, _, _) = Run(BuildPageDoc(content)); + + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + } + [Fact] public void MissingId_reportsMalformed() { @@ -2415,7 +2770,8 @@ private static byte[] BuildFuzzDoc(byte[] content) => BuildPdf( // walks, so PdfReader.Open and GetPage keep succeeding on nearly every sample regardless of // what the mutations did to operator syntax, giving Run deep, repeated coverage of the // interpreter's own malformed-content recovery paths rather than mostly exercising xref repair - // (#402 round 3; FuzzInputGenReachesRunOnTheMajorityOfSamples pins the reach-rate difference). + // (#402 round 3; ContentBodyFuzzGenerator_isRobust_andReachesRunOnTheMajorityOfSamples pins + // this generator's own reach rate). private static readonly Gen ContentBodyFuzzInputGen = MutationOpGen.Array[1, 8].Select(ops => BuildFuzzDoc(Mutate(FuzzContent, ops))); @@ -2556,6 +2912,15 @@ private static class FuzzBudget { private const long DefaultIterations = 3_000; + /// + /// Iterations per fuzz / case. Overridable via + /// VELLUMPDF_FUZZ_ITER so the nightly workflow can run a much larger budget without + /// the PR-gating default paying for it. A copy of ParserFuzzTests.FuzzBudget (#402 + /// round 4: restored the doc a prior copy of this type dropped), kept as two separate + /// private nested types rather than one shared internal one, since the two test classes' + /// own fuzz targets (the lexer/parser/reader pipeline there, this interpreter here) are + /// unrelated enough that sharing state between them is not worth the coupling. + /// internal static long Iterations { get From 745d81b4c3add75a124c87b7cd225030d523e91c Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 05:48:46 +0200 Subject: [PATCH 11/14] Use the CR-LF fold for the EI scan after a wrong /L When a tier-a or tier-b length failed to land on 'EI' under both ID-separator readings, the fallback EI scan started from the one-byte reading that tiers a/b begin with. That scan has no length to verify a reading against, so it is in tier c's situation, and tier c reads a CR LF pair as one separator; starting one byte earlier prepended the separator's LF to the recovered data of every CR LF producer whose /L was wrong. The scan now starts from the folded position (a no-op when the separator was not CR LF); one test pins the recovered bytes. Also strips the review-history narration and the hollow intensifiers that the round-4 fix-up left in comments and the 302/307 diagnostic docs, and states the 307 dedupe scope directly. --- .../Content/ContentInterpreter.cs | 24 ++++++++++-------- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 18 ++++++------- .../ContentInterpreterTests.cs | 25 ++++++++++++++++++- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 287e3c69..343750f9 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -1776,11 +1776,10 @@ private bool HandleInlineImage( // in), §8.9.7 says only "a single white-space character". isCrLf records whether the // mandated byte happened to be a CR immediately followed by an LF, so tiers a/b below (each // of which has a declared or computed length to verify a reading against) can retry the - // two-byte reading once should the one-byte reading fail to land on 'EI' (#402 round 4: an - // earlier version preferred the two-byte reading first and retried the one-byte reading - // second, which fed a payload's own leading LF byte to the ID separator instead of to the - // image data whenever a producer wrote a lone CR before a payload that happened to start - // with LF). + // two-byte reading once should the one-byte reading fail to land on 'EI' (#402 round 4: + // reading the pair first fed a payload's own leading LF byte to the ID separator instead of + // to the image data whenever a producer wrote a lone CR before a payload that happened to + // start with LF). var isCrLf = false; var oneByteSeparatorPos = lexer.Position; if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 @@ -1866,8 +1865,7 @@ private bool HandleInlineImage( // CR immediately followed by an LF, retry once with §7.2.3's own fold: both bytes // consumed as the separator instead, since a producer that wrote a two-byte CR-LF // separator is exactly the case the one-byte reading above would otherwise misjudge - // (#402 round 4; this mirrors, in the opposite direction, an earlier version that tried - // the fold FIRST and retried the one-byte reading second). The malformed report just + // (#402 round 4). The malformed report just // below is skipped when this retry alone is what recovers the image: a conforming file // recovered from cleanly must not carry a warning about it (#402 round 2). The EI-scan // fallback below is a DIFFERENT case: reaching it at all means neither reading's @@ -1901,6 +1899,12 @@ private bool HandleInlineImage( if (resyncPos is null) { + // Neither reading's length landed on 'EI', so from here on this is tier c's own + // situation: a scan with no length to verify a reading against. It takes tier c's + // reading too (the CR-LF fold; see the comment on the tier-c branch above) rather + // than keeping the one-byte reading tiers a/b started from, so a CR LF producer + // whose /L is wrong does not get a spurious LF prepended to its recovered data. + dataStart = foldedSeparatorPos; var scanEnd = ScanForEi(dataStart, ctx.DiagObjectNumber); if (scanEnd is not null) { @@ -2152,7 +2156,7 @@ private int ResolveComponentCount( // shapes is accepted with no diagnostic. A '%' comment is the easiest worked example: one // running from right after a false 'EI' to the end of its own line can swallow the LATER, // terminating 'EI' written on that same line, so the operator that follows the comment's own - // line is what the probe actually sees, and it accepts through the ordinary Table A.1 rule with + // line is what the probe sees, and it accepts through the ordinary Table A.1 rule with // nothing to tell the two apart. The same blind spot applies to a coincidental operator inside // compressed noise (" EI n ", " EI W ", " EI f " each // truncate the image data with no 307 and hand the visitor a spurious operator) and to " EI BI " @@ -2252,7 +2256,7 @@ private enum ProbeOutcome /// A lex failure that ran off the buffer's own TRUE end (not this probe's own /// window clip) trying to close a token: an unterminated literal or hex string with no more /// buffer left to find its closing delimiter in, say. Weaker evidence than : - /// this scan cannot tell "the file genuinely ends mid-token" apart from "this false 'EI' sits + /// this scan cannot tell "the file ends mid-token" apart from "this false 'EI' sits /// inside image data whose next token happens to run to the file's own end without closing" /// (#402 round 4; see ScanForEi's own remarks on how this outcome is used as a fallback /// rather than rejected outright). @@ -2399,7 +2403,7 @@ private ResyncVerdict ClassifyResyncPoint(int pos, int? diagObjectNumber) // this the first time it happens, naming the offset AND object number this candidate // was accepted at without verification (#402 round 4: recording diagObjectNumber here, // alongside the offset, is what lets a LATER report against a different content - // stream's own object number still name where the budget actually ran out, rather than + // stream's own object number still name where the budget ran out, rather than // pairing this offset with whatever object happens to be current when it is reported), // so a caller can tell "the interpreter confirmed this resync point" apart from "the // interpreter ran out of budget and took its best guess" (#402 round 3). diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 6c0d85a9..9dafe1e7 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -408,12 +408,12 @@ public enum PdfReaderDiagnosticCode /// an array (§9.4.3); a non-numeric operand to cm, Tc, Tw, Tz, /// TL, Tf's second, Tr, Ts, Td, TD, or Tm; a /// non-name operand to Tf's first, to Do, or to gs/cs/CS/sh - /// (#402 round 4: this reader reads that operand for its own resource lookup, the same reason - /// Do's own name operand is checked); a non-string operand to ', or a non-numeric - /// first or second or non-string third operand to " (#402 round 4; Table 107) (#402 - /// round 3 for the rest of this list; every operator this interpreter recognises but does not - /// name above only forwards its own operands to the visitor untouched, so their operand types - /// are the visitor's to type-check, not this interpreter's); a Do that occurred inside a + /// (this reader reads that operand for its own resource lookup, the same reason Do's + /// own name operand is checked); a non-string operand to ', or a non-numeric first or + /// second or non-string third operand to " (Table 107) (#402 rounds 3 and 4; every + /// operator this interpreter recognises but does not name above only forwards its own operands + /// to the visitor untouched, so their operand types are the visitor's to type-check, not this + /// interpreter's); a Do that occurred inside a /// text object (§8.2 Figure 9 admits no operator of Table 50's XObjects category there), an /// unbalanced Q with no matching q on the graphics-state stack, or an unbalanced /// EMC with no matching BMC/BDC (§14.6.1). An unbalanced q still @@ -501,10 +501,8 @@ public enum PdfReaderDiagnosticCode /// specifically, ), so a second inline image on the SAME content stream /// with its own, different malformation is not listed separately, but the same page's own /// content and each Form XObject it draws dedupe independently, since each carries a distinct - /// object number here (#402 round 4: an earlier version of this doc said "per page" outright, - /// which undercounts a page invoking two forms that each carry their own malformed inline - /// image, since the two reports land against two different object numbers and are not deduped - /// against each other at all). + /// object number here: a page invoking two forms that each carry their own malformed inline + /// image lists two reports (#402 round 4). /// InlineImageMalformed = 307, diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 5f6011be..b456e6d9 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -2143,7 +2143,7 @@ public void TerminatingEi_followedByATokenThatNeverCloses_isStillAccepted(string { // Before this fix, an exception thrown at the buffer's own TRUE end (not an artificial // probe-window clip) was treated exactly like a malformed byte found strictly inside the - // window: an outright Reject. That misjudged the ONE 'EI' this content actually has as a + // window: an outright Reject. That misjudged the ONE 'EI' this content has as a // false candidate, losing the image and reporting a false InlineImageMalformed, when the // 'EI' was correctly delimited all along, and the unterminated token after it belongs to // the OUTER lexer's own ContentStreamLexError instead. @@ -2255,6 +2255,29 @@ public void CrLfSeparator_whoseOneByteReadingMissesEi_recoversViaTheRetry() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void CrLfSeparator_whoseLengthIsWrongUnderBothReadings_recoversThroughTheScanWithoutALeadingLf() + { + // /L 2 is wrong for a 4-byte payload, so neither the one-byte reading nor the CR LF retry + // lands on 'EI' and the image falls through to the EI scan. That scan has no length to + // verify a reading against, so it takes tier c's own reading, the CR LF fold: the recovered + // data must be the payload alone, not the payload with the separator's LF prepended. + byte[] data = "ABCD"u8.ToArray(); + var raw = "BI /W 4 /H 1 /BPC 8 /CS /G /L 2 ID\r\n"u8.ToArray() + .Concat(data) + .Concat(" EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(raw); + + var (reader, _, visitor) = Run(doc); + + var diag = Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + Assert.Contains("did not land on an 'EI' operator", diag.Message); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(data)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + [Fact] public void CrAsSingleSeparator_withDataStartingWithLf_recoversWithoutAWarning() { From cdc94a9befe3bfe908c492fe6f8820c90cb60a38 Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 07:36:49 +0200 Subject: [PATCH 12/14] Bound the operand-number stackalloc and fix round-5 findings TryParseOperandNumber sized its padding buffer with a raw stackalloc keyed on the numeric token's own length. PdfLexer.ReadNumeric puts no bound on that length, so a floating-point operand of about 1.5 million digits overflowed the stack outright, an uncatchable crash the interpreter's own contract says a malformed or absurd construct must not cause. Mirror the sibling guard in PdfObjectParser: stackalloc only for a padded length up to 1024 bytes, fall back to a heap array above that. Utf8Parser still parses the long literal exactly on the heap, so a conformant-if-absurd operand keeps its value instead of being capped or rejected. Added a value-level test pinning the parsed value for a two-million-digit operand. The CR-LF retry comment overstated its own reach: it only recovers a two-byte CR-LF separator when the strict one-byte reading fails to land on 'EI'. When the payload's own last byte is itself white space, the strict reading lands on 'EI' regardless (SkipToEi skips leading white space before the delimiter), so the retry never runs and the data comes back shifted one byte with no diagnostic. That is the reading section 8.9.7 mandates, not a defect, so the fix is disclosure: the comment now says so plainly, and a new test pins the behavior against the ambiguous byte sequence that exposes it. Also scoped the 307 diagnostic's doc comment to what the code reports: an /L past the end of the stream reports, but a computed length that overruns the buffer falls back to the EI scan silently. The rest of the round-5 findings were prose corrections across ContentInterpreter, PdfReaderDiagnostic, TextState, and PdfLexer: verbatim spec quotes, a stray colon in a Table 107 quote, an unchecked long multiply's wrap behavior documented as harmless, a stale count of the Conformance types that build their own PdfLexer, and several ragged comment lines reflowed. --- .../Content/ContentInterpreter.cs | 91 +++++++++++-------- src/VellumPdf.Reader/Content/TextState.cs | 7 +- src/VellumPdf.Reader/PdfLexer.cs | 32 ++++--- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 53 ++++++----- .../ContentInterpreterTests.cs | 61 ++++++++++++- 5 files changed, 163 insertions(+), 81 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 343750f9..7da5acb7 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -668,7 +668,10 @@ private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, o return true; } - Span padded = stackalloc byte[span.Length + 2]; + // The token length is attacker-controlled, so only stackalloc for short literals; an + // operand of a million digits would otherwise overflow the stack (an uncatchable crash). + var paddedLength = span.Length + 2; + Span padded = paddedLength <= 1024 ? stackalloc byte[paddedLength] : new byte[paddedLength]; var len = 0; if (span.IsEmpty || span[0] == (byte)'.') padded[len++] = (byte)'0'; @@ -990,7 +993,7 @@ private void HandleOperator( break; case "'": - // Table 107: "This operator shall have the same effect as the code: T* string Tj". + // Table 107: "This operator shall have the same effect as the code T* string Tj". // T*'s own move (§9.4.3) runs here; the text-showing half stays the visitor's, the // same as Tj's own string operand, which this interpreter never reads (#402 round 4). _textState.MoveTextPosition(0, -_gs.Leading); @@ -1317,13 +1320,12 @@ private void HandleDo( // text-positioning, text-showing, marked-content, and compatibility categories of // Table 50 inside a text object (seven, not six: #402 round 3); 'Do' sits in the // XObjects category, so a producer that invokes it there is wrong regardless of what - // the named XObject turns out to be. The - // recursion below still runs (a text-object violation is not itself a reason to skip - // an otherwise-resolvable Form), but the shared _textState instance a form's own - // content may disturb (BT/Td/ET, unbracketed by any q/Q-style save) is saved and - // restored around that recursion below regardless of this check, so the report here is - // purely informational: nothing downstream depends on _inTextObject to avoid the leak - // (#402 round 2). + // the named XObject turns out to be. The recursion below still runs (a text-object + // violation is not itself a reason to skip an otherwise-resolvable Form), but the + // shared _textState instance a form's own content may disturb (BT/Td/ET, unbracketed by + // any q/Q-style save) is saved and restored around that recursion below regardless of + // this check, so the report here is purely informational: nothing downstream depends on + // _inTextObject to avoid the leak (#402 round 2). diagnostics.Report( PdfReaderDiagnosticCode.OperandStackMalformed, "'Do' occurred inside a text object (ISO 32000-2 §8.2 Figure 9 admits no XObjects " @@ -1739,8 +1741,8 @@ private bool HandleInlineImage( f.Value is "JBIG2Decode" or "JPXDecode" or "Crypt"); // §8.9.7's normative sentence excludes ASCIIHexDecode/ASCII85Decode "as one of its filters" - // from the single-white-space rule; NOTE 2 gives the actual skip-without-decoding recipe, - // scoped narrower, to "the final or only filter": "if the final or only filter is + // from the single-white-space rule; NOTE 2 gives the skip-without-decoding recipe, scoped + // narrower, to "the final or only filter": "if the final or only filter is // ASCIIHexDecode or ASCII85Decode skip any further white-space [after the first]" before // counting /L's own bytes. NOTE 2's own skip only ever touches the raw bytes right after // ID, so its "final" filter can only mean final in ENCODING order: the filter applied LAST @@ -1770,16 +1772,16 @@ private bool HandleInlineImage( // next character shall be interpreted as the first byte of image data." Exactly ONE byte is // consumed as that mandated separator below, even when it is a CR immediately followed by // an LF, which §7.2.3's own EOL rule folds into one two-byte marker for LINE-ENDING - // purposes only; §8.9.7 does not import that fold, and unlike §7.3.8.1 (which names - // "CARRIAGE RETURN LINE FEED" outright on the one construct, a stream's own - // keyword-to-data boundary, where a spec author who wanted the fold applied there wrote it - // in), §8.9.7 says only "a single white-space character". isCrLf records whether the - // mandated byte happened to be a CR immediately followed by an LF, so tiers a/b below (each - // of which has a declared or computed length to verify a reading against) can retry the - // two-byte reading once should the one-byte reading fail to land on 'EI' (#402 round 4: - // reading the pair first fed a payload's own leading LF byte to the ID separator instead of - // to the image data whenever a producer wrote a lone CR before a payload that happened to - // start with LF). + // purposes only; §8.9.7 does not import that fold, and unlike §7.3.8.1 (which requires "an + // end-of-line marker consisting of either a CARRIAGE RETURN and a LINE FEED or just a LINE + // FEED" outright on the one construct, a stream's own keyword-to-data boundary, where a spec + // author who wanted the fold applied there wrote it in), §8.9.7 says only "a single + // white-space character". isCrLf records whether the mandated byte happened to be a CR + // immediately followed by an LF, so tiers a/b below (each of which has a declared or + // computed length to verify a reading against) can retry the two-byte reading once should + // the one-byte reading fail to land on 'EI' (#402 round 4: reading the pair first fed a + // payload's own leading LF byte to the ID separator instead of to the image data whenever a + // producer wrote a lone CR before a payload that happened to start with LF). var isCrLf = false; var oneByteSeparatorPos = lexer.Position; if (lexer.TryPeek() is var separatorByte && separatorByte >= 0 @@ -1865,12 +1867,21 @@ private bool HandleInlineImage( // CR immediately followed by an LF, retry once with §7.2.3's own fold: both bytes // consumed as the separator instead, since a producer that wrote a two-byte CR-LF // separator is exactly the case the one-byte reading above would otherwise misjudge - // (#402 round 4). The malformed report just - // below is skipped when this retry alone is what recovers the image: a conforming file - // recovered from cleanly must not carry a warning about it (#402 round 2). The EI-scan - // fallback below is a DIFFERENT case: reaching it at all means neither reading's - // declared or computed length landed on 'EI', not merely one of the two, so recovering - // through IT still reports. + // (#402 round 4). This retry only runs when the one-byte reading's own resync above + // already failed to land on 'EI'; it does not cover every CR-LF producer. When the + // payload's own last byte happens to be white space, the one-byte reading is off by one + // (it left the payload's true first byte, the LF, at the front of the data) but still + // lands on 'EI' regardless, because SkipToEi skips leading white space before checking + // for 'EI' and that displaced trailing byte gets skipped the same way. The retry never + // runs in that case, and the visitor receives the data shifted one byte, with no + // diagnostic. That is the reading §8.9.7 mandates ("a single white-space character"), + // not a defect this retry is meant to close. + // + // The malformed report just below is skipped when this retry alone is what recovers the + // image: a conforming file recovered from cleanly must not carry a warning about it + // (#402 round 2). The EI-scan fallback below is a DIFFERENT case: reaching it at all + // means neither reading's declared or computed length landed on 'EI', not merely one of + // the two, so recovering through IT still reports. var recoveredViaCrRetry = false; if (isCrLf) { @@ -2081,6 +2092,11 @@ private static bool FirstFilterIsAsciiHexOrAscii85(PdfDictionary dict) return null; // Unknown colour space: fall back to the EI scan (tier c). var rowBytes = ((long)width.Value * bpc.Value * components + 7) / 8; + // rowBytes can reach roughly 2^34 (width up to int.MaxValue, bpc up to 16) and height up to + // int.MaxValue - 1 (~2^31), so this multiply can wrap a signed 64-bit long. The wrap is + // harmless: the total < 0 check below and the bounds check that follows it both still run, + // and a surviving wrapped value that passes both merely fails to land on 'EI' and recovers + // through the scan (tier c), the same recovery an ordinary overrun takes. var total = rowBytes * height.Value; if (total < 0 || dataStart + total > _currentBuffer.Length) return null; // Fall back to the EI scan rather than trusting a runaway computed size. @@ -2152,15 +2168,15 @@ private int ResolveComponentCount( // closing it needs decoding the image data itself, out of scope for a byte-level scan): any // whitespace-delimited 'EI' followed by bytes that lex as a Table A.1 operator, as 'BI', or as // the buffer's own end is indistinguishable, at the byte level, from a resync point the - // terminating 'EI' itself sets, so a false 'EI' immediately followed by any of those three - // shapes is accepted with no diagnostic. A '%' comment is the easiest worked example: one - // running from right after a false 'EI' to the end of its own line can swallow the LATER, - // terminating 'EI' written on that same line, so the operator that follows the comment's own - // line is what the probe sees, and it accepts through the ordinary Table A.1 rule with + // terminating 'EI' itself sets, so a false 'EI' followed, after any run of neutral tokens, by + // any of those three shapes is accepted with no diagnostic. A '%' comment is the easiest worked + // example: one running from right after a false 'EI' to the end of its own line can swallow the + // LATER, terminating 'EI' written on that same line, so the operator that follows the comment's + // own line is what the probe sees, and it accepts through the ordinary Table A.1 rule with // nothing to tell the two apart. The same blind spot applies to a coincidental operator inside - // compressed noise (" EI n ", " EI W ", " EI f " each - // truncate the image data with no 307 and hand the visitor a spurious operator) and to " EI BI " - // (the BI accept rule below): the bytes are well-formed content either way. + // compressed noise (" EI n ", " EI W ", " EI f " each truncate the image data with no 307 and + // hand the visitor a spurious operator) and to " EI BI " (the BI accept rule below): the bytes + // are well-formed content either way. private int? ScanForEi(int dataStart, int? diagObjectNumber) { var span = _currentBuffer.Span; @@ -2250,7 +2266,8 @@ private enum ProbeOutcome /// An 'EI' or 'ID' keyword (still inside image data that continues to a LATER /// 'EI'), a keyword containing a non-printable byte, or a lex failure found strictly inside - /// an unclipped window (a malformed byte the buffer's true end had nothing to do with). + /// the window, clipped or not (a malformed byte neither the window's own clip nor the + /// buffer's true end had anything to do with). Reject, /// A lex failure that ran off the buffer's own TRUE end (not this probe's own @@ -2306,7 +2323,7 @@ private ProbeOutcome ProbeOnce(int pos) // lexer's own Position landed at or past the window's own length), the buffer's own // TRUE end with nothing left to close the token (WeakReject, the same Position // check but an unclipped window: #402 round 4), or a malformed byte found strictly - // inside an unclipped window, short of either end (Reject outright). + // inside the window, short of either end, clipped or not (Reject outright). outcome = (windowClipped, probe.Position >= window.Length) switch { (true, true) => ProbeOutcome.Exhausted, diff --git a/src/VellumPdf.Reader/Content/TextState.cs b/src/VellumPdf.Reader/Content/TextState.cs index a4018cf3..1af7731d 100644 --- a/src/VellumPdf.Reader/Content/TextState.cs +++ b/src/VellumPdf.Reader/Content/TextState.cs @@ -10,9 +10,10 @@ namespace VellumPdf.Reader.Content; /// specified only within a text object and shall not persist from one text object to the /// next", so they are never saved or restored by q/Q the way §8.4.4's own /// graphics-state parameters are. Only BT resets them (to identity), and only Td, -/// TD, Tm, T*, and the two Table 107 line-showing operators ' and -/// " (each defined as having the same effect as T* followed by a text-showing -/// operator) update them. Not stacked; the interpreter owns exactly one live instance. +/// TD, Tm, T*, and the two Table 107 text-showing operators that also move +/// to the next line, ' and ", update them: ' has the same effect as +/// T* followed by Tj, and " sets Tw and Tc and then behaves as +/// '. Not stacked; the interpreter owns exactly one live instance. /// internal sealed class TextState { diff --git a/src/VellumPdf.Reader/PdfLexer.cs b/src/VellumPdf.Reader/PdfLexer.cs index b653809d..54be9818 100644 --- a/src/VellumPdf.Reader/PdfLexer.cs +++ b/src/VellumPdf.Reader/PdfLexer.cs @@ -77,19 +77,19 @@ internal sealed class PdfLexer { private readonly ReadOnlyMemory _data; - // Off by default: every existing consumer (the object parser, the 11 Conformance rules that - // build their own PdfLexer, and every non-content test) relies on the file-structure lexer's - // stricter rule that a lone '{', '}', or unmatched '>' is malformed PDF. §7.2.3 Table 2 lists - // '{' and '}' as delimiters with no syntax of their own in the file-structure grammar; the one - // place they mean anything is inside a Type 4 PostScript calculator function body (§7.10.5), - // which this lexer is never pointed at, so the file-structure grammar itself has no production - // for a lone one and it stays malformed in default mode. A content stream is a different - // grammar: those same bytes are junk with no meaning of their own there either, but §7.8.2's - // BX/EX rule is the reason to tolerate junk inside a compatibility section rather than abort - // the whole page over it, so ContentInterpreter needs to lex them as harmless one-byte - // unknown-operator keywords instead. Opt-in, not a runtime detection of "is this a content - // stream", keeps that decision with the caller that already knows which grammar applies - // instead of guessing from the bytes. + // Off by default: every existing consumer (the object parser, the 13 Conformance types (11 + // rules and two helpers) that build their own PdfLexer, and every non-content test) relies on + // the file-structure lexer's stricter rule that a lone '{', '}', or unmatched '>' is malformed + // PDF. §7.2.3 Table 2 lists '{' and '}' as delimiters with no syntax of their own in the + // file-structure grammar; the one place they mean anything is inside a Type 4 PostScript + // calculator function body (§7.10.5), which this lexer is never pointed at, so the + // file-structure grammar itself has no production for a lone one and it stays malformed in + // default mode. A content stream is a different grammar: those same bytes are junk with no + // meaning of their own there either, but §7.8.2's BX/EX rule is the reason to tolerate junk + // inside a compatibility section rather than abort the whole page over it, so ContentInterpreter + // needs to lex them as harmless one-byte unknown-operator keywords instead. Opt-in, not a + // runtime detection of "is this a content stream", keeps that decision with the caller that + // already knows which grammar applies instead of guessing from the bytes. private readonly bool _contentStreamMode; /// Current byte offset within . @@ -203,8 +203,10 @@ public ReadOnlyMemory Slice(int offset, int length) } /// - /// Advances the cursor to . - /// Must be >= current and <= . + /// Moves the cursor to , forward or backward. Only + /// 0 <= position <= is enforced; several callers deliberately + /// rewind past a position already read (a lookahead scan that backs off after deciding not to + /// consume it, say) and re-lex from there. /// public void Seek(int position) { diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 9dafe1e7..555b7c9d 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -469,9 +469,9 @@ public enum PdfReaderDiagnosticCode ResourceMissing = 306, /// - /// An inline image (ISO 32000-2 §8.9.7) could not be delimited or decoded, one of its - /// dictionary entries was itself invalid, or this run's own resync-probe budget ran out before - /// a candidate EI could be confirmed (#402 round 3; see + /// An inline image (ISO 32000-2 §8.9.7) could not be delimited, one of its dictionary entries + /// was itself invalid, or this run's own resync-probe budget ran out before a candidate + /// EI could be confirmed (#402 round 3; see /// ContentInterpreter.ProbeOnce): a filter this interpreter never applies to inline /// image data (JBIG2Decode, JPXDecode, or Crypt: §8.9.7 itself excludes /// all three from inline-image use), a missing ID or EI operator, a missing, @@ -480,29 +480,31 @@ public enum PdfReaderDiagnosticCode /// compute the data length (Table 87 types /W and /H as integer and restricts /// /BPC's own value to that fixed set; "positive" for /W//H is this /// reader's own requirement, not the table's own wording), an /L present but not an - /// integer, or negative (§8.9.7, Table 91; PDF 2.0), an /L or computed length past the - /// end of the stream, a computed length (from /L or from the image's own shape) that - /// does not land on the following EI operator, in which case this reader retries the - /// EI scan before giving up, or the resync probe spending its whole per-run byte budget - /// before it could confirm a candidate EI, in which case that candidate is accepted - /// unverified rather than left unresolved. The inline-image callback is raised whenever the - /// image was delimited AND its filter chain is one this reader accepts on an inline image, - /// including after an /L-past-the-end or did-not-land-on-EI recovery, or a - /// probe-budget-exhausted acceptance (#402 round 4: a disallowed filter delimits the image just - /// as successfully as an accepted one, so "was delimited" alone is not what decides whether the - /// callback fires). A disallowed filter (JBIG2Decode, JPXDecode, or Crypt) - /// still delimits the image, so interpretation of the rest of the content stream continues past - /// its EI, but skips the callback for that image; when the image could not be delimited - /// at all (no ID, no EI) interpretation of that stream stops there instead, since - /// nothing past that point can be resynchronised reliably. + /// integer, or negative (§8.9.7, Table 91; PDF 2.0), an /L naming a length past the + /// end of the stream (a computed length that overruns the stream instead falls back to the + /// EI scan below with no report of its own), a computed length (from /L or from + /// the image's own shape) that does not land on the following EI operator, in which + /// case this reader retries the EI scan before giving up, or the resync probe spending + /// its whole per-run byte budget before it could confirm a candidate EI, in which case + /// that candidate is accepted unverified rather than left unresolved. The inline-image + /// callback is raised whenever the image was delimited AND its filter chain is one this reader + /// accepts on an inline image, including after an /L-past-the-end or + /// did-not-land-on-EI recovery, or a probe-budget-exhausted acceptance (#402 round 4: a + /// disallowed filter delimits the image just as successfully as an accepted one, so "was + /// delimited" alone is not what decides whether the callback fires). A disallowed filter + /// (JBIG2Decode, JPXDecode, or Crypt) still delimits the image, so + /// interpretation of the rest of the content stream continues past its EI, but skips the + /// callback for that image; when the image could not be delimited at all (no ID, no + /// EI) interpretation of that stream stops there instead, since nothing past that point + /// can be resynchronised reliably. /// Reported at most once per content stream (the page's own content, or one Form XObject) per /// page: the sink's dedupe key is (code, object, page), and every image on one content stream /// reports against that stream's own object number (or, for the page's own top-level content - /// specifically, ), so a second inline image on the SAME content stream - /// with its own, different malformation is not listed separately, but the same page's own - /// content and each Form XObject it draws dedupe independently, since each carries a distinct - /// object number here: a page invoking two forms that each carry their own malformed inline - /// image lists two reports (#402 round 4). + /// specifically, ), so a second report of this code against the same + /// content stream, whether from another image or from a second malformation of the same one, + /// is not listed separately, but the same page's own content and each Form XObject it draws + /// dedupe independently, since each carries a distinct object number here: a page invoking two + /// forms that each carry their own malformed inline image lists two reports (#402 round 4). /// InlineImageMalformed = 307, @@ -531,7 +533,10 @@ public enum PdfReaderDiagnosticCode /// (#402) so a caller can tell "this file hit a limit of this reader" apart from "this file is /// malformed", a distinction the two codes sharing one value made impossible to draw. The /// offending operator, or push, is dropped and interpretation continues, the same recovery - /// uses. + /// uses. A q, BMC, or BDC dropped for + /// one of these ceilings still consumes its matching Q or EMC silently, so a + /// conformant file is not also charged an for this + /// reader's own ceiling. /// ContentLimitExceeded = 309, diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index b456e6d9..89bea1ab 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -215,6 +215,33 @@ public void OperandTypes_areParsedWithExactValuesAndShapes() Assert.Same(PdfNull.Instance, sc.Operands[2]); } + [Fact] + public void RealOperandOfTwoMillionDigits_isParsedOnTheHeap_andRunReturns() + { + // TryParseOperandNumber used to size its padding buffer with a raw stackalloc keyed on the + // numeric token's own length. PdfLexer.ReadNumeric puts no bound on that length (it is a + // content-stream operand, not a value this reader's own resource caps ever see), so an + // operand of about 1.5 million digits overflowed the stack outright, an uncatchable crash + // this type's own contract (above) says a malformed or absurd construct must not cause. On + // the heap Utf8Parser still parses this exactly: a two-million-digit literal is conformant, + // if absurd, §7.3.3 syntax, so its value survives rather than falling back to a malformed + // report. This must not run against the pre-fix binary: there it kills the whole test host. + var digits = new string('1', 2_000_000); + var content = "0." + digits + " w\n1 w\n"; + // About 2 MiB of content, far under the default MaxDecodedStreamBytes the Run helper leaves + // in force, so a limit diagnostic cannot be what Assert.Empty below is looking at. + Assert.True(content.Length < ReaderLimits.DefaultMaxDecodedBytes); + + var (reader, _, visitor) = Run(BuildPageDoc(content)); + + var wCalls = visitor.Operators.Where(o => o.Op == "w").ToList(); + Assert.Equal(2, wCalls.Count); + var real = (PdfReal)wCalls[0].Operands[0]; + Assert.Equal(0.1111111111, real.Value, 9); + Assert.Equal(1, ((PdfInteger)wCalls[1].Operands[0]).Value); + Assert.Empty(reader.Diagnostics); + } + // ── BX/EX compatibility sections ──────────────────────────────────────────────────────────── [Fact] @@ -610,7 +637,10 @@ public void QDroppedForTheOperandCountCap_creditsTheMatchingQ() public void QDroppedForTheOperandCountCap_creditsOnlyOneQ_aSecondQStillReportsUnbalanced() { // Only ONE 'q' was dropped, so only one credit exists: a second 'Q' past it finds nothing - // pushed and no credit left, and reports the ordinary unbalanced-pop diagnostic. + // pushed and no credit left, and reports the ordinary unbalanced-pop diagnostic. This test + // guards over-crediting rather than pinning the round-4 fix itself (the two sibling tests + // do that): a credit of 2 for the one dropped 'q' would silence both 'Q's, leaving + // `unbalanced` empty and failing Assert.Single below. var content = string.Join(' ', Enumerable.Repeat("1", 65)) + " q\nQ\nQ\n1 w\n"; var (reader, _, visitor) = Run(BuildPageDoc(content)); @@ -759,7 +789,7 @@ public void Do_withANonNameOperand_reportsOnce_andNeverReachesTheVisitor() [Fact] public void Quote_movesToTheNextLine_andForwardsTheStringOperandUntouched() { - // Table 107: "' ... shall have the same effect as the code: T* string Tj". Before this fix, + // Table 107: "' ... shall have the same effect as the code T* string Tj". Before this fix, // ' fell through to the default (state-inert) case: the text matrices were left untouched // and the string operand still reached the visitor, since forwarding happens either way. const string content = "BT\n100 TL\n5 0 Td\n(a) '\nET\n"; @@ -2255,6 +2285,33 @@ public void CrLfSeparator_whoseOneByteReadingMissesEi_recoversViaTheRetry() Assert.Contains(visitor.Operators, o => o.Op == "Q"); } + [Fact] + public void CrLfSeparator_whosePayloadEndsInWhiteSpace_isReadTheStrictWayWithNoRetry() + { + // Pins the §8.9.7 one-byte reading for this ambiguous file, not a defect. The producer + // wrote a two-byte CR LF separator, and the payload's own true last byte is itself white + // space (LF), so the strict one-byte reading, which leaves the payload's true first byte + // (the LF from the CR-LF pair) at the front of the data instead of consuming it as part of + // the separator, still lands on 'EI': SkipToEi skips leading white space before 'EI', and + // that displaced trailing byte gets skipped the same way. The CR-LF retry above this test + // never runs, so the visitor receives the data shifted one byte with no diagnostic; §8.9.7 + // gives the reader no way to tell this apart from a producer that meant a lone CR instead. + byte[] delivered = [0x0A, 0x41, 0x42, 0x43]; + byte[] payload = [0x41, 0x42, 0x43, 0x0A]; + var raw = "BI /W 4 /H 1 /BPC 8 /CS /G /L 4 ID\r\n"u8.ToArray() + .Concat(payload) + .Concat("EI\nQ\n"u8.ToArray()) + .ToArray(); + var doc = BuildPageDocRaw(raw); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.InlineImageMalformed); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual(delivered)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + [Fact] public void CrLfSeparator_whoseLengthIsWrongUnderBothReadings_recoversThroughTheScanWithoutALeadingLf() { From 483e903af8cdb20bb5c7354f0afc8001c652cd7e Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 09:22:42 +0200 Subject: [PATCH 13/14] Bound diagnostic messages to stop retaining whole tokens An unrecognised operator keyword or a missing named resource used to be decoded and quoted into its own diagnostic message whole. Neither PdfLexer.ReadKeyword nor PdfName bounds a token's own length, and a Diagnostic is retained for the reader's lifetime, so an attacker-sized token turned into a comparably sized permanent allocation, amplified further by page count through the sink's (code, object, page) dedupe key. QuoteExcerpt now caps every diagnostic-bound name or keyword at 32 characters, appending the token's own byte length past that point. The unrecognised-keyword site also stops decoding past that bound, since nothing past it can change operator dispatch. Also folds in the remaining round 6 findings: the 302 doc's scoping sentence, an inverted byte-role description in the inline image disclosure, and several wording and line-wrap fixes. --- .../Content/ContentInterpreter.cs | 127 ++++++++++++------ .../Content/IContentVisitor.cs | 7 +- src/VellumPdf.Reader/PdfReaderDiagnostic.cs | 30 +++-- .../ContentInterpreterTests.cs | 101 ++++++++++++-- 4 files changed, 194 insertions(+), 71 deletions(-) diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index 7da5acb7..c3833073 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -92,6 +92,16 @@ internal sealed class ContentInterpreter // 16 MiB settings alike, not that plus a further window's own worth). private const long MaxProbeBytesPerRun = 16L * 1024 * 1024; + // A diagnostic's job is to identify a malformed keyword or name, not to carry the whole thing: + // PdfLexer.ReadKeyword bounds neither a keyword's own length nor, per PdfName, a name's, and + // Annex C.1 puts no bound on either ("this PDF standard does not restrict the size or quantity + // of things described in the PDF file format"; Table C.1's 127-byte name length is only + // informative). A Diagnostic is retained for the reader's own lifetime (DiagnosticSink), so + // quoting an oversized token whole would turn one attacker- or corruption-controlled byte run + // into a comparably sized permanent allocation, once per (code, object, page) the sink's dedupe + // key admits (#402 round 6). + private const int MaxQuotedTokenChars = 32; + private static readonly PdfName XObjectSubtypeForm = new("Form"); private static readonly PdfName XObjectSubtypeImage = new("Image"); private static readonly PdfName ImageMaskKey = new("ImageMask"); @@ -590,8 +600,21 @@ private void InterpretStream( } else { - var name = System.Text.Encoding.Latin1.GetString(raw); - HandleOperator(name, offset, ctx, visitor, diagnostics, pageIndex); + // Decoded only far enough to name the operator or, for an + // unrecognised one, excerpt it in the 301 below (QuoteExcerpt + // truncates past MaxQuotedTokenChars anyway); ReadKeyword puts + // no bound on a keyword's own length, so materialising the + // whole thing here for an attacker-sized token would allocate + // what the diagnostic then discards most of (#402 round 6). + // ContentOperators.IsKnown rejects anything over 8 characters + // (ContentOperators.cs), so a keyword truncated to + // MaxQuotedTokenChars + 1 bytes still fails dispatch exactly + // the way the whole one did, and every recognised operator, + // never more than 3 characters, is decoded in full either way. + var decodeLength = Math.Min(raw.Length, MaxQuotedTokenChars + 1); + var name = System.Text.Encoding.Latin1.GetString(raw[..decodeLength]); + HandleOperator( + name, raw.Length, offset, ctx, visitor, diagnostics, pageIndex); } break; } @@ -634,11 +657,11 @@ private void HandleNumber(Token token, StreamContext ctx, DiagnosticSink diagnos ctx.DiagObjectNumber, pageIndex: pageIndex); } - // System.Buffers.Text.Utf8Parser backs this, but against a normalised copy of the token's - // bytes, not the raw span, because PDF's own numeric grammar - // (ISO 32000-2 §7.3.3) allows a bare leading or trailing decimal point ("-.5", "6.") that the - // BCL's own double formats do not universally accept the same way across runtimes; padding a - // missing digit on either side of '.' sidesteps that without reimplementing number parsing. + // System.Buffers.Text.Utf8Parser backs this, but against a normalised copy of the token's bytes, + // not the raw span, because PDF's own numeric grammar (ISO 32000-2 §7.3.3) allows a bare leading + // or trailing decimal point ("-.5", "6.") that the BCL's own double formats do not universally + // accept the same way across runtimes; padding a missing digit on either side of '.' sidesteps + // that without reimplementing number parsing. private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, out PdfObject? result) { result = null; @@ -669,7 +692,8 @@ private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, o } // The token length is attacker-controlled, so only stackalloc for short literals; an - // operand of a million digits would otherwise overflow the stack (an uncatchable crash). + // operand of a million digits or more would otherwise overflow the stack (an uncatchable + // crash). var paddedLength = span.Length + 2; Span padded = paddedLength <= 1024 ? stackalloc byte[paddedLength] : new byte[paddedLength]; var len = 0; @@ -774,11 +798,23 @@ private static bool CompositeOperandWithinCap(PdfLexer lexer, out bool lexerFail return count <= MaxCompositeOperandElements; } + // Quotes at most MaxQuotedTokenChars of a diagnostic-bound name or keyword; see that constant + // for why. byteLength is the token's own full length, which for a PdfName's Value is just + // text.Length (Latin1: one char per byte); the one caller that decodes only far enough to + // excerpt an oversized keyword (HandleOperator's own dispatch site) passes the raw token's + // length separately, since text itself is already truncated there. + private static string QuoteExcerpt(string text) => QuoteExcerpt(text, text.Length); + + private static string QuoteExcerpt(string text, int byteLength) => + byteLength <= MaxQuotedTokenChars + ? text + : $"{text[..MaxQuotedTokenChars]}... ({byteLength} bytes)"; + // ── Operator dispatch ──────────────────────────────────────────────────────────────────────── private void HandleOperator( - string name, int offset, StreamContext ctx, IContentVisitor visitor, DiagnosticSink diagnostics, - int pageIndex) + string name, int keywordByteLength, int offset, StreamContext ctx, IContentVisitor visitor, + DiagnosticSink diagnostics, int pageIndex) { if (!ContentOperators.IsKnown(name)) { @@ -802,8 +838,8 @@ private void HandleOperator( { diagnostics.Report( PdfReaderDiagnosticCode.UnknownOperator, - $"'{name}' is not one of the operators ISO 32000-2 Annex A Table A.1 defines; " - + "it was ignored.", + $"'{QuoteExcerpt(name, keywordByteLength)}' is not one of the operators ISO " + + "32000-2 Annex A Table A.1 defines; it was ignored.", pageIndex: pageIndex); } ClearOperands(); @@ -1245,8 +1281,8 @@ private void ValidateNamedResource( diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'{op}' names '/{name.Value}', absent from the applicable /Resources /{category.Value} " - + "dictionary.", + $"'{op}' names '/{QuoteExcerpt(name.Value)}', absent from the applicable /Resources " + + $"/{category.Value} dictionary.", ctx.DiagObjectNumber, pageIndex: pageIndex); } @@ -1276,8 +1312,8 @@ private void HandleExtGState(StreamContext ctx, DiagnosticSink diagnostics, int { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'gs' names '/{gsName.Value}', absent from the applicable /Resources /ExtGState " - + "dictionary.", + $"'gs' names '/{QuoteExcerpt(gsName.Value)}', absent from the applicable /Resources " + + "/ExtGState dictionary.", ctx.DiagObjectNumber, pageIndex: pageIndex); return; } @@ -1342,8 +1378,8 @@ private void HandleDo( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'Do' names '/{xobjectName.Value}', absent from the applicable /Resources /XObject " - + "dictionary.", + $"'Do' names '/{QuoteExcerpt(xobjectName.Value)}', absent from the applicable " + + "/Resources /XObject dictionary.", ctx.DiagObjectNumber, pageIndex: pageIndex); return; } @@ -1352,8 +1388,8 @@ private void HandleDo( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'Do' names '/{xobjectName.Value}', present in the applicable /Resources " - + "/XObject dictionary but not as an indirect reference to a stream.", + $"'Do' names '/{QuoteExcerpt(xobjectName.Value)}', present in the applicable " + + "/Resources /XObject dictionary but not as an indirect reference to a stream.", ctx.DiagObjectNumber, pageIndex: pageIndex); return; } @@ -1363,8 +1399,8 @@ private void HandleDo( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'Do' names '/{xobjectName.Value}', but object {xobjectRef.ObjectNumber} does not " - + "resolve to a stream.", + $"'Do' names '/{QuoteExcerpt(xobjectName.Value)}', but object " + + $"{xobjectRef.ObjectNumber} does not resolve to a stream.", ctx.DiagObjectNumber, pageIndex: pageIndex); return; } @@ -1373,8 +1409,8 @@ private void HandleDo( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'Do' names '/{xobjectName.Value}', object {stream.ObjectNumber}, whose " - + "/Subtype is missing or is not a name, so it cannot be used as an XObject.", + $"'Do' names '/{QuoteExcerpt(xobjectName.Value)}', object {stream.ObjectNumber}, " + + "whose /Subtype is missing or is not a name, so it cannot be used as an XObject.", stream.ObjectNumber, pageIndex: pageIndex); return; } @@ -1386,9 +1422,9 @@ private void HandleDo( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"'Do' names '/{xobjectName.Value}', object {stream.ObjectNumber}, whose /Subtype " - + $"'/{subtype.Value}' is neither /Form nor /Image, so it cannot be used as an " - + "XObject.", + $"'Do' names '/{QuoteExcerpt(xobjectName.Value)}', object {stream.ObjectNumber}, " + + $"whose /Subtype '/{QuoteExcerpt(subtype.Value)}' is neither /Form nor /Image, " + + "so it cannot be used as an XObject.", stream.ObjectNumber, pageIndex: pageIndex); return; } @@ -1870,12 +1906,12 @@ private bool HandleInlineImage( // (#402 round 4). This retry only runs when the one-byte reading's own resync above // already failed to land on 'EI'; it does not cover every CR-LF producer. When the // payload's own last byte happens to be white space, the one-byte reading is off by one - // (it left the payload's true first byte, the LF, at the front of the data) but still - // lands on 'EI' regardless, because SkipToEi skips leading white space before checking - // for 'EI' and that displaced trailing byte gets skipped the same way. The retry never - // runs in that case, and the visitor receives the data shifted one byte, with no - // diagnostic. That is the reading §8.9.7 mandates ("a single white-space character"), - // not a defect this retry is meant to close. + // (it left the LF of the CR LF pair at the front of the data) but still lands on 'EI' + // regardless, because SkipToEi skips leading white space before checking for 'EI' and + // that displaced trailing byte gets skipped the same way. The retry never runs in that + // case, and the visitor receives the data shifted one byte, with no diagnostic. That is + // the reading §8.9.7 mandates ("a single white-space character"), not a defect this + // retry is meant to close. // // The malformed report just below is skipped when this retry alone is what recovers the // image: a conforming file recovered from cleanly must not carry a warning about it @@ -2095,8 +2131,10 @@ private static bool FirstFilterIsAsciiHexOrAscii85(PdfDictionary dict) // rowBytes can reach roughly 2^34 (width up to int.MaxValue, bpc up to 16) and height up to // int.MaxValue - 1 (~2^31), so this multiply can wrap a signed 64-bit long. The wrap is // harmless: the total < 0 check below and the bounds check that follows it both still run, - // and a surviving wrapped value that passes both merely fails to land on 'EI' and recovers - // through the scan (tier c), the same recovery an ordinary overrun takes. + // and a surviving wrapped value that passes both takes the did-not-land-on-'EI' path + // instead: a 307 reports first, then the scan (tier c) recovers the image. Measured with + // /W 1824726041 /H 1263665316 /BPC 16 /CS /CMYK (rowBytes * height wraps to 32): one image + // delivered, the operators that followed it reached the visitor, and exactly one 307. var total = rowBytes * height.Value; if (total < 0 || dataStart + total > _currentBuffer.Length) return null; // Fall back to the EI scan rather than trusting a runaway computed size. @@ -2152,8 +2190,8 @@ private int ResolveComponentCount( { diagnostics.Report( PdfReaderDiagnosticCode.ResourceMissing, - $"An inline image's /CS names '/{csName.Value}', absent from the applicable " - + "/Resources /ColorSpace dictionary.", + $"An inline image's /CS names '/{QuoteExcerpt(csName.Value)}', absent from the " + + "applicable /Resources /ColorSpace dictionary.", ctx.DiagObjectNumber, pageIndex: pageIndex); } return -1; @@ -2233,8 +2271,11 @@ private static int TrimEiDelimiter(ReadOnlySpan span, int dataStart, int e } // Confirms an 'EI' candidate at exactly a known offset (used once tier a/b already computed a - // length) by requiring the bytes there literally spell "EI" preceded and followed the way §8.9.7 - // describes; unlike ScanForEi this does not search, it verifies one position. + // length) by requiring the bytes there literally spell "EI" preceded the way §8.9.7 describes. + // Unlike ScanForEi (tier c), this does not search (it verifies one position only) and checks + // nothing about what follows 'EI': ScanForEi's own followedOk check (whitespace or a delimiter + // right after 'EI') has no counterpart here, so a tier a/b image is delivered even when the byte + // immediately after 'EI' is neither. private int? SkipToEi(int dataEnd) { var pos = dataEnd; @@ -2254,10 +2295,10 @@ private static int TrimEiDelimiter(ReadOnlySpan span, int dataStart, int e // second, larger window mask the terminating 'EI' whose own legitimate follow-on token // happened to be longer still). A candidate needs a POSITIVE reason to accept now, not merely // the absence of a rejecting keyword: a number, name, string, array/dictionary delimiter, - // true/false/null, or an - // unknown-but-printable keyword is neutral and keeps the probe lexing rather than accepting by - // default, closing the round-2 gap where a straddling but well-formed array or dictionary (never - // itself a keyword) produced the identical wrong "accept" outcome an unterminated string used to. + // true/false/null, or an unknown-but-printable keyword is neutral and keeps the probe lexing + // rather than accepting by default, closing the round-2 gap where a straddling but well-formed + // array or dictionary (never itself a keyword) produced the identical wrong "accept" outcome an + // unterminated string used to. private enum ProbeOutcome { /// A Table A.1 operator other than 'EI'/'ID', 'BI', or the buffer's own true end diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs index c615a89a..d2b61875 100644 --- a/src/VellumPdf.Reader/Content/IContentVisitor.cs +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -54,10 +54,9 @@ internal interface IContentVisitor /// Called once a Do operator (already reported through ) has /// been resolved to a /Subtype /Form stream and every recursion guard (depth, cycle, /// per-page budget) has passed, raised for every Do that reaches that point, including - /// one whose content then fails to decode or is skipped for this run's own content budget - /// (#402 round 3: an earlier version of this doc said the call happens only immediately before - /// the interpreter recurses into the form's own content, which is false; the decode - /// itself, and the budget check ahead of it, both happen AFTER this callback, not before it). + /// one whose content then fails to decode or is skipped for this run's own content budget: the + /// decode itself, and the budget check ahead of it, both happen AFTER this callback runs, not + /// before it (#402 round 3). /// Matched by exactly one call once the form's own content finishes /// interpreting (or is skipped, in the cases above), even if that content raises further /// nested calls of its own in between. diff --git a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs index 555b7c9d..82d534f8 100644 --- a/src/VellumPdf.Reader/PdfReaderDiagnostic.cs +++ b/src/VellumPdf.Reader/PdfReaderDiagnostic.cs @@ -359,7 +359,7 @@ public enum PdfReaderDiagnosticCode /// not be decoded (including one carrying an image filter, which this interpreter never /// attempts to decode as content). Operators already reported to the caller's visitor before /// the failure are kept either way, but the two cases end a different amount of the page's - /// content (#402 round 3): an element that fails to resolve or decode is simply skipped, and + /// content (#402 round 3): an element that fails to resolve or decode is skipped, and /// interpretation resumes with the NEXT stream in a multi-stream /Contents array, since /// every element still contributes its own decoded bytes to one buffer built from the whole /// array before interpretation of any of it begins. A lexer or parser failure happens INSIDE @@ -396,10 +396,12 @@ public enum PdfReaderDiagnosticCode /// A content stream's operand-stack discipline broke down in one of several ways this /// interpreter groups under one code rather than one each, since every case has the same /// remedy: drop the offending operator (or, for an unbalanced Q/EMC, drop the - /// pop) and keep interpreting. Every case here is a producer-side malformation (the document - /// itself is wrong, not merely bigger than this reader is willing to process); see - /// for the four cases that are this reader's own processing - /// ceiling instead. Covers: a number token that does not parse, is not finite, or carries a + /// pop) and keep interpreting. Every case here is producer-side except a numeric literal whose + /// value exceeds this reader's own or range, which is + /// reported here, rather than as , because what gets dropped + /// is the operand itself, not an operator or a push; see for + /// the four cases that are this reader's own processing ceiling instead. Covers: a number token + /// that does not parse, is not finite, or carries a /// second sign character (--5, -+5: §7.3.3 allows only "an optional sign"), a /// dictionary operand on an operator other than BDC/DP (§7.8.2: "Dictionaries /// shall be permitted as operands only by certain specific operators"), a known operator @@ -413,10 +415,10 @@ public enum PdfReaderDiagnosticCode /// second or non-string third operand to " (Table 107) (#402 rounds 3 and 4; every /// operator this interpreter recognises but does not name above only forwards its own operands /// to the visitor untouched, so their operand types are the visitor's to type-check, not this - /// interpreter's); a Do that occurred inside a - /// text object (§8.2 Figure 9 admits no operator of Table 50's XObjects category there), an - /// unbalanced Q with no matching q on the graphics-state stack, or an unbalanced - /// EMC with no matching BMC/BDC (§14.6.1). An unbalanced q still + /// interpreter's); a Do that occurred inside a text object (§8.2 Figure 9 admits no + /// operator of Table 50's XObjects category there), an unbalanced Q with no matching + /// q on the graphics-state stack, or an unbalanced EMC with no matching + /// BMC/BDC (§14.6.1). An unbalanced q still /// open at the end of a content stream is not reported: nothing downstream of this interpreter /// needs the graphics state restored past the last operator it saw. /// @@ -471,11 +473,11 @@ public enum PdfReaderDiagnosticCode /// /// An inline image (ISO 32000-2 §8.9.7) could not be delimited, one of its dictionary entries /// was itself invalid, or this run's own resync-probe budget ran out before a candidate - /// EI could be confirmed (#402 round 3; see - /// ContentInterpreter.ProbeOnce): a filter this interpreter never applies to inline - /// image data (JBIG2Decode, JPXDecode, or Crypt: §8.9.7 itself excludes - /// all three from inline-image use), a missing ID or EI operator, a missing, - /// non-integer, or non-positive /W or /H, a missing /BPC or one whose + /// EI could be confirmed (#402 round 3; see ContentInterpreter.ProbeOnce): a + /// filter this interpreter never applies to inline image data (JBIG2Decode, + /// JPXDecode, or Crypt: §8.9.7 itself excludes all three from inline-image use), + /// a missing ID or EI operator, a missing, non-integer, or non-positive + /// /W or /H, a missing /BPC or one whose /// value is not 1, 2, 4, or 8 (or, from PDF 1.5, 16) where the image's shape requires one to /// compute the data length (Table 87 types /W and /H as integer and restricts /// /BPC's own value to that fixed set; "positive" for /W//H is this diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index 89bea1ab..eb7472c3 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -2048,8 +2048,8 @@ public void FalseEiCandidate_followedByALongLiteralStringStraddlingTheProbeWindo // The round-2 probe's own first, 128-byte window ran off mid-string here (the literal is // 200 bytes, longer than that window) and needed a second, larger window to accept it. The // round-3 probe has no fixed window at all, only the shared per-Run budget (#402 round 3), - // so a 200-byte literal that closes properly is neutral and simply keeps the probe lexing - // until it reaches 'Tj', a known operator: accepted the same way, just without a retry. + // so a 200-byte literal that closes properly is neutral and keeps the probe lexing until it + // reaches 'Tj', a known operator: accepted the same way, just without a retry. var longLiteral = new string('X', 200); byte[] data = [0xFF, 0xD8, 0xFF]; var content = "BI /F /DCT ID "u8.ToArray() @@ -2288,14 +2288,14 @@ public void CrLfSeparator_whoseOneByteReadingMissesEi_recoversViaTheRetry() [Fact] public void CrLfSeparator_whosePayloadEndsInWhiteSpace_isReadTheStrictWayWithNoRetry() { - // Pins the §8.9.7 one-byte reading for this ambiguous file, not a defect. The producer - // wrote a two-byte CR LF separator, and the payload's own true last byte is itself white - // space (LF), so the strict one-byte reading, which leaves the payload's true first byte - // (the LF from the CR-LF pair) at the front of the data instead of consuming it as part of - // the separator, still lands on 'EI': SkipToEi skips leading white space before 'EI', and - // that displaced trailing byte gets skipped the same way. The CR-LF retry above this test - // never runs, so the visitor receives the data shifted one byte with no diagnostic; §8.9.7 - // gives the reader no way to tell this apart from a producer that meant a lone CR instead. + // Pins the §8.9.7 one-byte reading for this ambiguous file, not a defect. The producer wrote a + // two-byte CR LF separator, and the payload's own true last byte is itself white space (LF), so + // the strict one-byte reading, which leaves the LF from the CR LF pair at the front of the data + // instead of consuming it as part of the separator, still lands on 'EI': SkipToEi skips leading + // white space before 'EI', and that displaced trailing byte gets skipped the same way. The CR-LF + // retry above this test never runs, so the visitor receives the data shifted one byte with no + // diagnostic; §8.9.7 gives the reader no way to tell this apart from a producer that meant a + // lone CR instead. byte[] delivered = [0x0A, 0x41, 0x42, 0x43]; byte[] payload = [0x41, 0x42, 0x43, 0x0A]; var raw = "BI /W 4 /H 1 /BPC 8 /CS /G /L 4 ID\r\n"u8.ToArray() @@ -2807,6 +2807,87 @@ public void ContentStreamTooLarge_isRetainedEvenOnceMaxDiagnosticsIsAlreadySpent Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamTooLarge); } + // ── An unrecognised token is excerpted, not quoted whole (#402 round 6) ───────────────────── + // + // PdfLexer.ReadKeyword bounds neither a keyword's own length nor, per PdfName, a name's, so an + // unrecognised operator or a missing named resource used to decode and interpolate the whole + // token into its own diagnostic Message. A Diagnostic is retained for the reader's own + // lifetime, so a multi-megabyte token produced a comparably sized permanent allocation; these + // pin the fixed-size excerpt QuoteExcerpt reports instead. 4 MiB is well under both the 64 MiB + // content budget and PdfReaderOptions.MaxDecodedStreamBytes's own 512 MiB default, so neither + // needs raising for these fixtures. + + [Fact] + public void UnknownOperator_ofAttackerControlledLength_reportsOnlyAFixedExcerpt() + { + var keyword = new string('A', 4_194_304); + var content = Encoding.ASCII.GetBytes(keyword + "\n1 w\n"); + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< /Filter /FlateDecode >>", Flate(content))); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); + var report = Assert.Single(reports); + Assert.True( + report.Message.Length < 200, + $"expected a message under 200 chars, got {report.Message.Length}."); + var expectedExcerpt = $"'{new string('A', 32)}... (4194304 bytes)'"; + Assert.StartsWith(expectedExcerpt, report.Message, StringComparison.Ordinal); + + var w = Assert.Single(visitor.Operators); + Assert.Equal("w", w.Op); + } + + [Fact] + public void ResourceMissing_forADoNameOfAttackerControlledLength_reportsOnlyAFixedExcerpt() + { + var xobjectName = new string('B', 4_194_304); + var content = Encoding.ASCII.GetBytes($"/{xobjectName} Do\n1 w\n"); + var doc = BuildPdf( + 1, + new Obj(1, "<< /Type /Catalog /Pages 2 0 R >>"), + new Obj(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + new Obj(3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>"), + new Obj(4, "<< /Filter /FlateDecode >>", Flate(content))); + + var (reader, _, visitor) = Run(doc); + + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.ResourceMissing).ToList(); + var report = Assert.Single(reports); + var expectedExcerpt = $"/{new string('B', 32)}... (4194304 bytes)'"; + Assert.Contains(expectedExcerpt, report.Message, StringComparison.Ordinal); + + Assert.Contains(visitor.Operators, o => o.Op == "w"); + } + + [Theory] + [InlineData(32, false)] + [InlineData(33, true)] + public void UnknownOperator_atTheExcerptBoundary_truncatesOnlyPastMaxQuotedTokenChars( + int keywordLength, bool expectTruncation) + { + var keyword = new string('A', keywordLength); + var doc = BuildPageDocRaw(Encoding.ASCII.GetBytes(keyword + "\n")); + + var (reader, _, _) = Run(doc); + + var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.UnknownOperator).ToList(); + var report = Assert.Single(reports); + var expectedQuoted = expectTruncation + ? $"'{keyword[..32]}... ({keywordLength} bytes)'" + : $"'{keyword}'"; + Assert.Contains(expectedQuoted, report.Message, StringComparison.Ordinal); + } + // ── Fuzzing ────────────────────────────────────────────────────────────────────────────────── private static readonly byte[] FuzzContent = Encoding.ASCII.GetBytes( From 582db83cf04a669da4428536304c65512954243c Mon Sep 17 00:00:00 2001 From: Timothy van der Ham Date: Fri, 4 Sep 2026 11:52:32 +0200 Subject: [PATCH 14/14] Bound the content interpreter's remaining retention gaps Round 7 found the same class of defect rounds 5 and 6 each found one more site of: a diagnostic or a live interpreter field retaining an attacker-sized, content-derived allocation with no bound tied to its own size. Run's own outer catch used to forward the caught InvalidDataException's Message verbatim. PdfObjectParser quotes an offending object-header keyword or numeric literal whole with no bound of its own, and a Diagnostic is retained for the reader's lifetime, so a crafted object header or a multi-million-digit real literal in a /Contents stream's own dictionary turned into a comparably sized permanent allocation. The catch now reports the condition instead of the exception text. HandleInlineImage's key/value loop handed every dictionary value to parser.ParseObject() with no CompositeOperandWithinCap pre-scan, so an array or dictionary value inside BI...ID bypassed MaxCompositeOperandElements entirely, and the loop admitted an unbounded number of key/value pairs. Both are now capped the same way the main operand loop already caps a TJ array, dropping the image with a 309 rather than materialising the value first. Run reset its per-run state on entry only, so an operand pushed but never consumed (or set into GraphicsState.Font by Tf/gs and never overwritten) stayed referenced by the interpreter instance for as long as it lived, not just for the duration of one Run call. Run's body now clears that state on exit too. Sweeping the rest of src/VellumPdf.Reader for the same shape found one more site, in PageTreeWalker.cs, untouched by rounds 1-6: a /Type name interpolated whole into PageTreeMissing (the tree's own root) and PageTreeNodeMalformed (any other node) via PdfName.ToString(), with no excerpt bound the way ContentInterpreter's own QuoteExcerpt already gives every content-derived name. Fixed the same shape: a local QuoteType, 32-char excerpt. Sweep table (site; what sizes it; bound; pinning test): - ContentInterpreter.Run outer catch; ex.Message from PdfObjectParser's object-header/numeric-literal exceptions; fixed text now, no interpolation (fixed this commit); tests: UnparsableContentsObjectHeader_reportsAFixedMessage_notTheWholeToken TwoPagesSharingAMalformedContentsDictionary_eachReportOneBoundedMessage - HandleInlineImage array/general ParseObject() calls; an inline image dictionary array or dictionary value; now MaxCompositeOperandElements (8192) via CompositeOperandWithinCap (fixed this commit); tests: InlineImageDictionaryArrayValue_overTheCompositeCap_dropsTheImage_boundingAllocation InlineImageDictionaryFilterArrayValue_overTheCompositeCap_dropsTheImage InlineImageDictionaryArrayValue_atTheCompositeCapBoundary - HandleInlineImage key/value loop; number of dictionary entries; now MaxInlineImageDictionaryEntries, 64 (fixed this commit); tests (Theories pinning the boundary at 64 delivered and 65 dropped): InlineImageDictionary_withMoreEntriesThanTheCap_reportsOnce_andDropsTheImage InlineImageDictionary_withEntriesUpToTheCap_deliversTheImage - GraphicsState.Font / _gsStack past Run's own return; a Tf/gs font operand, or a q-pushed clone holding one; now cleared in Run's own finally (fixed this commit); test: Run_dropsAnUnconsumedFontOperandOnExit_soItBecomesCollectable - PageTreeWalker.Walk root-node /Type diagnostic (PageTreeMissing); the root's own /Type name; now QuoteType, 32-char excerpt (found by this sweep, fixed this commit); test: RootWithAnOversizedType_reportsOnlyAFixedExcerpt - PageTreeWalker.ClassifyNode Skip-branch /Type diagnostic (PageTreeNodeMalformed), not deduped across distinct object numbers; a non-root node's own /Type name; now QuoteType (found by this sweep, fixed this commit); test: KidWithAnOversizedType_reportsOnlyAFixedExcerpt - HandleOperator's arity-mismatch / dictionary-operand / wrong-type diagnostics quoting the operator name; bounded by construction, reachable only once ContentOperators.IsKnown(name) is true and every table key is 3 characters or fewer; no fix needed. - Filters.cs decompression-limit messages (Flate/LZW/RunLength); a fixed template plus the configured MaxDecodedBytes limit, never a producer-controlled string; already bounded, no fix needed. - TryParseOperandNumber's padded real-number buffer; the numeric token's own length; heap-allocated past 1024 bytes but a local, never retained past the one ParseReal call (round 3, unchanged); no fix needed. - TryReadNumbers for a form's /Matrix and /BBox; array.Count; bounded to 6 or 4 by a precondition check before the call; no fix needed. - Concatenate's own buffer allocations; total/cappedLength; bounded by MaxContentBytes, 64 MiB, before allocation; no fix needed. - gs/BDC resource-dictionary reads, form /Matrix and /BBox; not applicable, these resolve through the object graph (_reader.ResolveValue), not content-stream lexing, so this interpreter's own composite cap does not apply; PdfDocumentReader's MaxResolveDepth and PdfObjectParser's MaxNestingDepth are the relevant bounds instead; out of scope by design. - PdfDocumentReader.cs's own Report call sites; object/generation numbers only, every message fixed text or an int; already bounded, no fix needed. - TextState; holds only Matrix (double fields), no PdfObject reference at all; nothing to bound. Declined nothing from the brief; the PageTreeWalker.cs fix and its two tests are this sweep's own addition beyond the three findings. --- CHANGELOG.md | 8 +- .../Content/ContentInterpreter.cs | 162 ++++-- .../Content/IContentVisitor.cs | 12 +- src/VellumPdf.Reader/Pages/PageTreeWalker.cs | 23 +- .../ContentInterpreterTests.cs | 522 +++++++++++++++--- tests/VellumPdf.Reader.Tests/PageTreeTests.cs | 52 ++ .../PdfLexerContentModeTests.cs | 2 +- 7 files changed, 665 insertions(+), 116 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc0adf6..1dd550e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,9 +77,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). unsupported construct is reported, not thrown, and interpretation continues past it. That is what the other nine codes describe: `ContentStreamLexError`, `UnknownOperator` (`Warning` severity: an operator this reader skipped is a best-effort reading, not proof the output is - correct), `OperandStackMalformed` for a producer-side malformation, `ContentLimitExceeded` for - this reader's own processing ceilings instead (an operand-count, array-or-dictionary-operand - token, `q`-depth, or marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, + correct), `OperandStackMalformed` for a producer-side malformation (also covering a numeric + literal past this reader's own `long`/`double` range, since what gets dropped there is the + operand itself, not an operator or a push), `ContentLimitExceeded` for this reader's own + processing ceilings instead (an operand-count, array-or-dictionary-operand token, `q`-depth, or + marked-content-depth cap), `FormXObjectCycle`, `FormXObjectBudgetExceeded`, `ResourceMissing`, `InlineImageMalformed`, and `ContentStreamTooLarge` (one 64 MiB decoded-content budget per page, shared between its own `/Contents` and every Form XObject it draws; each invocation of a form counts again, since the interpretation cost this bounds scales with how many times a form is diff --git a/src/VellumPdf.Reader/Content/ContentInterpreter.cs b/src/VellumPdf.Reader/Content/ContentInterpreter.cs index c3833073..9ca4c9bc 100644 --- a/src/VellumPdf.Reader/Content/ContentInterpreter.cs +++ b/src/VellumPdf.Reader/Content/ContentInterpreter.cs @@ -56,6 +56,17 @@ internal sealed class ContentInterpreter // 977 MiB committed for one dropped operator and one 309. private const int MaxCompositeOperandElements = 8192; + // §8.9.7's Table 91 lists eleven inline image dictionary entries (BitsPerComponent, ColorSpace, + // Decode, DecodeParms, Filter, Height, ImageMask, Intent, Interpolate, Length, Width; Table 92 + // layers abbreviations onto some of their VALUES, not further entries of its own), and §8.9.7 + // itself says "Entries other than those listed shall be ignored", so this reader's own ceiling + // on how many key/value pairs one inline image dictionary may carry, before HandleInlineImage + // gives up on it, is generous by construction: no conformant producer's dictionary comes close. + // It exists only to bound how much a hostile BI...ID section, one that never reaches ID at all, + // can make this reader allocate one PdfName key (and, per MaxCompositeOperandElements above, one + // capped value) at a time (#402 round 7). + private const int MaxInlineImageDictionaryEntries = 64; + // §8.4.4's q/Q pair; this reader's own ceiling on how deep a legitimate document nests them. private const int MaxGraphicsStateDepth = 64; @@ -249,7 +260,7 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) var ctx = new StreamContext(page.Resources, soleObjectNumber); InterpretStream(buffer, ctx, visitor, pageIndex, diagnostics); } - catch (InvalidDataException ex) + catch (InvalidDataException) { // The outermost guard for a malformed indirect-reference chain reached through // resource, XObject, or Form XObject resolution (a corrupt cross-reference offset, @@ -258,11 +269,36 @@ internal void Run(PdfReadPage page, IContentVisitor visitor) // handles those). Consistent with this type's own class doc promise that // InvalidDataException never escapes Run, and with the notify-and-continue policy // every other diagnostic in this channel follows. + // + // The exception's own Message is not forwarded: PdfObjectParser quotes the offending + // header keyword or numeric literal whole, with no bound of its own, and a diagnostic + // is retained for the reader's own lifetime (DiagnosticSink), so an attacker- or + // corruption-sized token would become a comparably sized permanent allocation once per + // (code, object, page) the sink's own dedupe key admits (#402 round 7). diagnostics.Report( PdfReaderDiagnosticCode.ContentStreamLexError, - $"The page's content could not be fully resolved: {ex.Message}", + "The page's content could not be fully resolved: an object it references could " + + "not be parsed.", pageIndex: pageIndex); } + finally + { + // Clears every content-derived reference this Run's own per-Run state may still hold, + // so an attacker-sized operand pushed but never consumed by a later operator (no + // closing operator at all, or one that never sets a new GraphicsState field to + // overwrite it) does not stay pinned on this interpreter for the rest of its own + // lifetime: the entry resets above already give the NEXT Run a clean slate, but an + // interpreter that is reused only after a long delay, or never reused again, would + // otherwise keep the LAST Run's own content alive regardless. ProbeBytesConsumed is + // left alone: a test reads it after Run returns as telemetry, not as content-derived + // state (#402 round 7). + _operands.Clear(); + _operandOverflow = false; + _gsStack.Clear(); + _gs = new GraphicsState(); + _openForms.Clear(); + _textState.BeginText(); + } } // ── /Contents resolution and concatenation (ISO 32000-2 §7.7.3.3 Table 31) ───────────────────── @@ -606,11 +642,14 @@ private void InterpretStream( // no bound on a keyword's own length, so materialising the // whole thing here for an attacker-sized token would allocate // what the diagnostic then discards most of (#402 round 6). - // ContentOperators.IsKnown rejects anything over 8 characters - // (ContentOperators.cs), so a keyword truncated to - // MaxQuotedTokenChars + 1 bytes still fails dispatch exactly - // the way the whole one did, and every recognised operator, - // never more than 3 characters, is decoded in full either way. + // HandleOperator's own dispatch below goes through + // ContentOperators.IsKnown(string), a bare dictionary lookup + // with no length guard of its own (only the ReadOnlySpan + // overload the resync probe uses, in ContentOperators.cs, bails + // out past 8 bytes); a keyword truncated to MaxQuotedTokenChars + // + 1 bytes still fails that lookup exactly the way the whole + // one did, since no key in the table is longer than 3 characters, + // and every recognised operator is decoded in full either way. var decodeLength = Math.Min(raw.Length, MaxQuotedTokenChars + 1); var name = System.Text.Encoding.Latin1.GetString(raw[..decodeLength]); HandleOperator( @@ -692,8 +731,8 @@ private static bool TryParseOperandNumber(ReadOnlySpan raw, bool isReal, o } // The token length is attacker-controlled, so only stackalloc for short literals; an - // operand of a million digits or more would otherwise overflow the stack (an uncatchable - // crash). + // operand of about 1.5 million digits or more would otherwise overflow the stack (an + // uncatchable crash); one million digits alone still returns normally. var paddedLength = span.Length + 2; Span padded = paddedLength <= 1024 ? stackalloc byte[paddedLength] : new byte[paddedLength]; var len = 0; @@ -799,10 +838,13 @@ private static bool CompositeOperandWithinCap(PdfLexer lexer, out bool lexerFail } // Quotes at most MaxQuotedTokenChars of a diagnostic-bound name or keyword; see that constant - // for why. byteLength is the token's own full length, which for a PdfName's Value is just - // text.Length (Latin1: one char per byte); the one caller that decodes only far enough to - // excerpt an oversized keyword (HandleOperator's own dispatch site) passes the raw token's - // length separately, since text itself is already truncated there. + // for why. byteLength is the DECODED value's own byte length (Latin1: one char per byte), not + // necessarily the raw token's: for a PdfName it is just text.Length, but a name whose raw token + // used one or more '#xx' escapes (§7.3.5) decodes to fewer bytes than it was written in, so the + // raw token can run longer than byteLength reports ('/' + 40 'B' + '#20' x10 is a 71-byte raw + // token whose decoded Value is 50 bytes, and this reports "(50 bytes)"). The one caller that + // decodes only far enough to excerpt an oversized keyword (HandleOperator's own dispatch site) + // passes the raw token's own length separately, since text itself is already truncated there. private static string QuoteExcerpt(string text) => QuoteExcerpt(text, text.Length); private static string QuoteExcerpt(string text, int byteLength) => @@ -831,7 +873,7 @@ private void HandleOperator( // (no-op) execution just as much as to a recognised one (#402 round 2; an earlier // version kept the operands outside BX/EX on the theory that a stray "R" left over // from indirect-reference syntax §7.8.2 forbids in content streams usually belonged to - // whatever REAL operator followed rather than to "R" itself, but that leniency broke a + // whatever OPERATOR followed rather than to "R" itself, but that leniency broke a // differently-shaped input just as easily: '10 20 Zork' ahead of '1 w' silently fed // the leftover 20 into 'w' as its own operand instead of 1). if (_bxDepth <= 0) @@ -1699,6 +1741,7 @@ private bool HandleInlineImage( DiagnosticSink diagnostics, int pageIndex, int biOffset) { var dict = new PdfDictionary(); + var entryCount = 0; while (true) { @@ -1723,6 +1766,21 @@ private bool HandleInlineImage( return false; } + // Checked before this key is even decoded into a PdfName, not after: an over-cap + // dictionary must not keep paying per-pair allocation cost for entries this reader is + // about to drop the whole image over anyway (#402 round 7; see + // MaxInlineImageDictionaryEntries for why 64 rejects nothing conformant). + if (entryCount >= MaxInlineImageDictionaryEntries) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentLimitExceeded, + $"An inline image dictionary has more than {MaxInlineImageDictionaryEntries} " + + "entries; the image was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + return false; + } + entryCount++; + var key = InlineImageAbbreviations.ExpandKey(PdfObjectParser.ParseName(keyTok)); var isColorSpaceKey = key.Equals(PdfName.ColorSpace); var isFilterKey = key.Equals(PdfName.Filter); @@ -1731,6 +1789,41 @@ private bool HandleInlineImage( var valueStart = lexer.Position; var valueTok = lexer.NextToken(); + // Pre-scanned with the lexer alone, exactly the way the main operand loop's own + // ArrayBegin/DictBegin case does (see CompositeOperandWithinCap's own remarks), before + // either PdfObjectParser branch below ever materialises the value: an inline image + // dictionary value has no operator-level arity check of its own to fall back on for + // this, so without this pre-scan a value here bypassed MaxCompositeOperandElements + // entirely, even though every other array or dictionary this interpreter parses from + // content is bounded by it (#402 round 7). + if (valueTok.Kind is TokenKind.ArrayBegin or TokenKind.DictBegin) + { + if (!CompositeOperandWithinCap(lexer, out var valueLexerFailed)) + { + diagnostics.Report( + PdfReaderDiagnosticCode.ContentLimitExceeded, + $"An inline image dictionary value exceeds {MaxCompositeOperandElements} " + + "tokens; the image was dropped.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + if (valueLexerFailed) + { + // Same reasoning as the identical branch in the main operand loop's own + // ArrayBegin/DictBegin case above: the count pass itself hit a malformed + // byte before it ever reached the cap comparison this 309 already covers, + // so that failure gets its own 300 too rather than silently ending + // interpretation with nothing to explain why. + diagnostics.Report( + PdfReaderDiagnosticCode.ContentStreamLexError, + "The content stream's syntax could not be interpreted past this " + + "point; interpretation of it stopped here.", + ctx.DiagObjectNumber, pageIndex: pageIndex); + } + return false; + } + lexer.Seek(valueStart); + valueTok = lexer.NextToken(); + } + PdfObject value; if (valueTok.Kind == TokenKind.Name && (isColorSpaceKey || isFilterKey)) { @@ -1904,14 +1997,15 @@ private bool HandleInlineImage( // consumed as the separator instead, since a producer that wrote a two-byte CR-LF // separator is exactly the case the one-byte reading above would otherwise misjudge // (#402 round 4). This retry only runs when the one-byte reading's own resync above - // already failed to land on 'EI'; it does not cover every CR-LF producer. When the - // payload's own last byte happens to be white space, the one-byte reading is off by one - // (it left the LF of the CR LF pair at the front of the data) but still lands on 'EI' - // regardless, because SkipToEi skips leading white space before checking for 'EI' and - // that displaced trailing byte gets skipped the same way. The retry never runs in that - // case, and the visitor receives the data shifted one byte, with no diagnostic. That is - // the reading §8.9.7 mandates ("a single white-space character"), not a defect this - // retry is meant to close. + // already failed to land on 'EI'; it does not cover every CR-LF producer. The CR LF + // pair at the mandated separator is what shifts the one-byte reading off by one in the + // first place (it leaves the LF of the pair at the front of the data instead of + // consuming it as part of the separator); when the payload's own last byte then happens + // to be white space, that shifted reading still lands on 'EI' regardless, because + // SkipToEi skips leading white space before checking for 'EI', so the displaced trailing + // byte gets skipped the same way. The retry never runs in that case, and the visitor + // receives the data shifted one byte, with no diagnostic. That is the reading §8.9.7 + // mandates ("a single white-space character"), not a defect this retry is meant to close. // // The malformed report just below is skipped when this retry alone is what recovers the // image: a conforming file recovered from cleanly must not carry a warning about it @@ -2131,10 +2225,14 @@ private static bool FirstFilterIsAsciiHexOrAscii85(PdfDictionary dict) // rowBytes can reach roughly 2^34 (width up to int.MaxValue, bpc up to 16) and height up to // int.MaxValue - 1 (~2^31), so this multiply can wrap a signed 64-bit long. The wrap is // harmless: the total < 0 check below and the bounds check that follows it both still run, - // and a surviving wrapped value that passes both takes the did-not-land-on-'EI' path - // instead: a 307 reports first, then the scan (tier c) recovers the image. Measured with - // /W 1824726041 /H 1263665316 /BPC 16 /CS /CMYK (rowBytes * height wraps to 32): one image - // delivered, the operators that followed it reached the visitor, and exactly one 307. + // and a surviving wrapped value that passes both usually takes the did-not-land-on-'EI' + // path instead (a 307 reports first, then the scan, tier c, recovers the image), UNLESS the + // wrapped value happens to land exactly on an 'EI', in which case it is silently accepted + // with no diagnostic at all, the same as any other length that happens to be right. Measured + // with /W 1824726041 /H 1263665316 /BPC 16 /CS /CMYK (rowBytes * height wraps to 32): one + // image delivered, the operators that followed it reached the visitor, and exactly one 307. + // With 32 bytes of image data instead of a mismatched payload, the same wrapped total (32) + // lands squarely on 'EI': one 32-byte image delivered, zero 307s. var total = rowBytes * height.Value; if (total < 0 || dataStart + total > _currentBuffer.Length) return null; // Fall back to the EI scan rather than trusting a runaway computed size. @@ -2271,11 +2369,13 @@ private static int TrimEiDelimiter(ReadOnlySpan span, int dataStart, int e } // Confirms an 'EI' candidate at exactly a known offset (used once tier a/b already computed a - // length) by requiring the bytes there literally spell "EI" preceded the way §8.9.7 describes. - // Unlike ScanForEi (tier c), this does not search (it verifies one position only) and checks - // nothing about what follows 'EI': ScanForEi's own followedOk check (whitespace or a delimiter - // right after 'EI') has no counterpart here, so a tier a/b image is delivered even when the byte - // immediately after 'EI' is neither. + // length): skips zero or more §8.9.7 white-space bytes, then requires the bytes right after to + // literally spell "EI"; nothing before 'EI' is REQUIRED to be white space, only tolerated when + // present ('/L 4 ID ABCDEIQ ' delimits with no white-space byte immediately before 'EI' and no + // 307). Unlike ScanForEi (tier c), this does not search (it verifies one position only) and + // checks nothing about what follows 'EI': ScanForEi's own followedOk check (whitespace or a + // delimiter right after 'EI') has no counterpart here, so a tier a/b image is delivered even + // when the byte immediately after 'EI' is neither. private int? SkipToEi(int dataEnd) { var pos = dataEnd; diff --git a/src/VellumPdf.Reader/Content/IContentVisitor.cs b/src/VellumPdf.Reader/Content/IContentVisitor.cs index d2b61875..ee87afee 100644 --- a/src/VellumPdf.Reader/Content/IContentVisitor.cs +++ b/src/VellumPdf.Reader/Content/IContentVisitor.cs @@ -46,7 +46,10 @@ internal interface IContentVisitor /// abbreviation already expanded (e.g. /W to /Width, /CS /G to /// /ColorSpace /DeviceGray). /// The image's own (still filtered, undecoded) sample data: the bytes between - /// ID and EI, excluding the delimiting white space. + /// ID and EI, excluding the delimiting white space. A slice over the interpreter's + /// own content buffer (up to the 64 MiB per-run budget), valid only for the duration of this + /// callback, so a callback that needs it after returning must copy it out, not hold the slice: + /// holding it pins the whole buffer alive, not just this image's own share of it. /// The byte offset of the BI operator that began this image. void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset); @@ -56,10 +59,9 @@ internal interface IContentVisitor /// per-page budget) has passed, raised for every Do that reaches that point, including /// one whose content then fails to decode or is skipped for this run's own content budget: the /// decode itself, and the budget check ahead of it, both happen AFTER this callback runs, not - /// before it (#402 round 3). - /// Matched by exactly one call once the form's own content finishes - /// interpreting (or is skipped, in the cases above), even if that content raises further - /// nested calls of its own in between. + /// before it (#402 round 3). Matched by exactly one call once the + /// form's own content finishes interpreting (or is skipped, in the cases above), even if that + /// content raises further nested calls of its own in between. /// /// The form XObject's own stream dictionary. /// The form's /Matrix (ISO 32000-2 §8.10.2 Table 93), or diff --git a/src/VellumPdf.Reader/Pages/PageTreeWalker.cs b/src/VellumPdf.Reader/Pages/PageTreeWalker.cs index 25b9cd60..5feecda0 100644 --- a/src/VellumPdf.Reader/Pages/PageTreeWalker.cs +++ b/src/VellumPdf.Reader/Pages/PageTreeWalker.cs @@ -93,6 +93,23 @@ internal static class PageTreeWalker // with no usable /Kids of its own: contributes zero children. private static readonly PdfArray EmptyKids = new(); + // A /Type value has no length bound of its own (Annex C.1: "this PDF standard does not + // restrict the size or quantity of things described in the PDF file format"), and both + // diagnostics that quote it (PageTreeMissing on the tree's own root, PageTreeNodeMalformed on + // any other node) are retained for the reader's own lifetime (DiagnosticSink), so interpolating + // an oversized /Type name whole would turn one attacker- or corruption-controlled byte run into + // a comparably sized permanent allocation, once per (code, object) the sink's own dedupe key + // admits: the same class of defect ContentInterpreter.QuoteExcerpt exists to bound (#402 + // round 7 sweep). + private const int MaxQuotedTypeChars = 32; + + private static string QuoteType(PdfName? type) => + type is null + ? "no /Type" + : type.Value.Length <= MaxQuotedTypeChars + ? type.ToString() + : $"/{type.Value[..MaxQuotedTypeChars]}... ({type.Value.Length} chars)"; + /// Walks 's page tree, reporting shape problems to /// along the way, and returns the pages found in tree order. internal static List Walk(PdfDocumentReader reader, DiagnosticSink diagnostics) @@ -158,7 +175,9 @@ internal static List Walk(PdfDocumentReader reader, DiagnosticSink var rootKind = ClassifyByType(reader, cache, rootDict, rootKidsArray, out var rootType); if (rootKind != NodeKind.Node) { - var found = rootType is not null ? $"/Type {rootType}" : "no /Type and no /Kids array"; + var found = rootType is not null + ? $"/Type {QuoteType(rootType)}" + : "no /Type and no /Kids array"; diagnostics.Report( PdfReaderDiagnosticCode.PageTreeMissing, $"The page tree root is not a page-tree node ({found}); ISO 32000-2 §7.7.2 Table " @@ -480,7 +499,7 @@ private static NodeKind ClassifyNode( case NodeKind.Skip: diagnostics.Report( PdfReaderDiagnosticCode.PageTreeNodeMalformed, - $"Object declares {type} where a page-tree node or page object was expected " + $"Object declares {QuoteType(type)} where a page-tree node or page object was expected " + "(ISO 32000-2 §7.7.3.2 Table 30, §7.7.3.3 Table 31); it was skipped.", NullIfZero(objectNumber)); kids = EmptyKids; diff --git a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs index eb7472c3..457475c0 100644 --- a/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs +++ b/tests/VellumPdf.Reader.Tests/ContentInterpreterTests.cs @@ -63,6 +63,40 @@ private static byte[] BuildPdf(int rootObjectNumber, params Obj[] objects) return ms.ToArray(); } + /// Same layout as , but writes each object's bytes verbatim + /// rather than through the "N 0 obj\n{dict}\nendobj\n" template: needed when a fixture + /// deliberately writes an object whose own header does not parse (#402 round 7). + private static byte[] BuildPdfWithRawObjectBytes( + int rootObjectNumber, params (int Num, byte[] Bytes)[] objects) + { + var ms = new MemoryStream(); + void W(string s) => ms.Write(Encoding.ASCII.GetBytes(s)); + + W("%PDF-1.7\n"); + + var maxNum = objects.Max(o => o.Num); + var offsets = new int?[maxNum + 1]; + foreach (var obj in objects.OrderBy(o => o.Num)) + { + offsets[obj.Num] = (int)ms.Position; + ms.Write(obj.Bytes); + } + + var xrefOffset = (int)ms.Position; + W($"xref\n0 {maxNum + 1}\n"); + W("0000000000 65535 f \n"); + for (var i = 1; i <= maxNum; i++) + { + W(offsets[i] is { } offset + ? $"{offset:D10} 00000 n \n" + : "0000000000 65535 f \n"); + } + W($"trailer\n<< /Size {maxNum + 1} /Root {rootObjectNumber} 0 R >>\n"); + W($"startxref\n{xrefOffset}\n%%EOF\n"); + + return ms.ToArray(); + } + private static byte[] Flate(byte[] raw) { var ms = new MemoryStream(); @@ -130,6 +164,56 @@ public void OnFormBegin( public void OnFormEnd(int objectNumber) => FormEnds.Add(objectNumber); } + /// A copy of the / + /// field values as of one OnOperator call. + /// 's own exit-time reset (#402 round 7) means neither is readable once + /// itself returns, so a test that wants "the state after + /// the page's own last operator" reads it here instead, captured DURING that operator's own + /// callback the way 's own class doc already says both are meant + /// to be read. + private sealed record ContentStateSnapshot( + Matrix Ctm, double CharSpacing, double WordSpacing, double HorizontalScaling, double Leading, + PdfObject? Font, double FontSize, int RenderMode, double Rise, Matrix TextMatrix, + Matrix TextLineMatrix); + + private sealed class StateSnapshotVisitor(ContentInterpreter interpreter) : IContentVisitor + { + public List<(string Op, List Operands, int Offset)> Operators { get; } = []; + + /// The state snapshot as of the MOST RECENT OnOperator call, updated on + /// every one; still if no operator ever reached this visitor. + public ContentStateSnapshot? LastState { get; private set; } + + public void OnOperator(string operatorName, IReadOnlyList operands, int offset) + { + Operators.Add((operatorName, [.. operands], offset)); + var gs = interpreter.GraphicsState; + var ts = interpreter.TextState; + LastState = new ContentStateSnapshot( + gs.Ctm, gs.CharSpacing, gs.WordSpacing, gs.HorizontalScaling, gs.Leading, gs.Font, + gs.FontSize, gs.RenderMode, gs.Rise, ts.TextMatrix, ts.TextLineMatrix); + } + + public void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset) { } + + public void OnFormBegin( + PdfDictionary formDictionary, Matrix formMatrix, PdfRectangle? boundingBox, int objectNumber, + int offset) + { } + + public void OnFormEnd(int objectNumber) { } + } + + private static (PdfDocumentReader Reader, ContentStateSnapshot? State, StateSnapshotVisitor Visitor) + RunAndCaptureFinalState(byte[] pdfBytes) + { + var reader = PdfReader.Open(pdfBytes); + var interpreter = new ContentInterpreter(reader); + var visitor = new StateSnapshotVisitor(interpreter); + interpreter.Run(reader.GetPage(0), visitor); + return (reader, visitor.LastState, visitor); + } + private static (PdfDocumentReader Reader, ContentInterpreter Interpreter, RecordingVisitor Visitor) Run( byte[] pdfBytes, PdfReaderOptions? options = null) { @@ -564,19 +648,20 @@ public void BalancedQAndQ_pastTheGraphicsStateCap_reportsOnlyTheLimit_andStaysBa // Before the fix, the pushes past MaxGraphicsStateDepth (64) were dropped but their // matching 'Q's still popped anyway, so a balanced 'q'...'Q' sequence deeper than the cap // reported BOTH ContentLimitExceeded (for the dropped pushes) AND OperandStackMalformed - // (for the "Q"s that found nothing real left on the stack once the real pushes were + // (for the "Q"s that found nothing left on the stack once the producer's own pushes were // exhausted), and desynchronised GraphicsState.Ctm from the actual nesting besides. var content = string.Concat(Enumerable.Repeat("q\n", depth)) + "2 0 0 2 10 20 cm\n" + string.Concat(Enumerable.Repeat("Q\n", depth)); - var interpreter = RunAndKeepInterpreter(BuildPageDoc(content), out var reader); + var (reader, state, _) = RunAndCaptureFinalState(BuildPageDoc(content)); Assert.Contains(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); // Every 'q' was balanced by a 'Q', so the CTM the 'cm' set inside is restored to identity, - // whether or not this reader's own stack held every one of those saves. - Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + // whether or not this reader's own stack held every one of those saves. The LAST 'Q' is + // what the snapshot reflects, so this is exactly the state the page itself ends on. + Assert.Equal(Matrix.Identity, state!.Ctm); } [Fact] @@ -659,19 +744,17 @@ public void QDroppedForTheOperandCountCap_creditsOnlyOneQ_aSecondQStillReportsUn [Fact] public void GraphicsStateStack_savesAndRestoresTheCtm() { - var interpreter = RunAndKeepInterpreter( - BuildPageDoc("q\n2 0 0 2 10 20 cm\nQ\n"), out _); + var (_, state, _) = RunAndCaptureFinalState(BuildPageDoc("q\n2 0 0 2 10 20 cm\nQ\n")); - Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + Assert.Equal(Matrix.Identity, state!.Ctm); } [Fact] public void Cm_concatenatesOntoTheCurrentCtm() { - var interpreter = RunAndKeepInterpreter( - BuildPageDoc("2 0 0 2 10 20 cm\n"), out _); + var (_, state, _) = RunAndCaptureFinalState(BuildPageDoc("2 0 0 2 10 20 cm\n")); - Assert.Equal(new Matrix(2, 0, 0, 2, 10, 20), interpreter.GraphicsState.Ctm); + Assert.Equal(new Matrix(2, 0, 0, 2, 10, 20), state!.Ctm); } [Fact] @@ -683,41 +766,32 @@ public void TextStateOperators_setTheExpectedFields() + "5 -6 TD\n" + "1 0 0 1 100 200 Tm\n" + "T*\n"; - var interpreter = RunAndKeepInterpreter( + var (_, state, _) = RunAndCaptureFinalState( BuildPageDoc(content, "<< /Font << /F1 5 0 R >> >>", - new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), - out _); - - Assert.Equal(1, interpreter.GraphicsState.CharSpacing); - Assert.Equal(2, interpreter.GraphicsState.WordSpacing); - Assert.Equal(150, interpreter.GraphicsState.HorizontalScaling); - Assert.Equal(6, interpreter.GraphicsState.Leading); // TD's ty=-6 sets TL=-(-6)=6 - Assert.Equal("F1", ((PdfName)interpreter.GraphicsState.Font!).Value); - Assert.Equal(24, interpreter.GraphicsState.FontSize); - Assert.Equal(2, interpreter.GraphicsState.RenderMode); - Assert.Equal(3, interpreter.GraphicsState.Rise); + new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))); + + Assert.Equal(1, state!.CharSpacing); + Assert.Equal(2, state.WordSpacing); + Assert.Equal(150, state.HorizontalScaling); + Assert.Equal(6, state.Leading); // TD's ty=-6 sets TL=-(-6)=6 + Assert.Equal("F1", ((PdfName)state.Font!).Value); + Assert.Equal(24, state.FontSize); + Assert.Equal(2, state.RenderMode); + Assert.Equal(3, state.Rise); // After Tm replaces both matrices with [1 0 0 1 100 200], T* moves by (0, -TL=-6): // Tlm_new = [1 0 0 1 0 -6] x [1 0 0 1 100 200] = [1 0 0 1 100 194]. - Assert.Equal(new Matrix(1, 0, 0, 1, 100, 194), interpreter.TextState.TextMatrix); + Assert.Equal(new Matrix(1, 0, 0, 1, 100, 194), state.TextMatrix); } [Fact] public void BT_resetsTheTextMatrices() { const string content = "1 0 0 1 100 200 Tm\nBT\n"; - var interpreter = RunAndKeepInterpreter(BuildPageDoc(content), out _); + var (_, state, _) = RunAndCaptureFinalState(BuildPageDoc(content)); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); - } - - private static ContentInterpreter RunAndKeepInterpreter(byte[] pdfBytes, out PdfDocumentReader reader) - { - reader = PdfReader.Open(pdfBytes); - var interpreter = new ContentInterpreter(reader); - interpreter.Run(reader.GetPage(0), new RecordingVisitor()); - return interpreter; + Assert.Equal(Matrix.Identity, state!.TextMatrix); + Assert.Equal(Matrix.Identity, state.TextLineMatrix); } // ── An operand of the wrong type at the right arity reports 302 and drops the operator, @@ -728,10 +802,11 @@ public void Cm_withANonNumericOperand_reportsOnce_andLeavesTheCtmUntouched() { // Before this fix, NumberOperand substituted 0 for the string operand with no diagnostic, // so this delivered Ctm = [1 0 0 1 0 50] instead of leaving the identity CTM untouched. - var interpreter = RunAndKeepInterpreter( - BuildPageDoc("1 0 0 1 (x) 50 cm\n"), out var reader); + // 'cm' itself never reaches the visitor (the type check drops it before EmitAndClear), so + // the state snapshot below is taken at the trailing 'w' instead. + var (reader, state, _) = RunAndCaptureFinalState(BuildPageDoc("1 0 0 1 (x) 50 cm\n1 w\n")); - Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + Assert.Equal(Matrix.Identity, state!.Ctm); var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) .ToList(); Assert.Single(reports); @@ -746,16 +821,17 @@ public void TfWithAStringSizeAndTdWithANameOperand_bothDropped_stateUntouched() // non-numeric ty operand ("/N", a name) coerced to 0 while tx=10 was still applied, // delivering TextMatrix = [1 0 0 1 10 0] instead of leaving it at the identity. Both Tf and // Td are dropped entirely now; only one 302 is recorded, since the sink dedupes on (code, - // object, page) and both reports share the same key. - var interpreter = RunAndKeepInterpreter( + // object, page) and both reports share the same key. 'ET' is the last operator to reach + // the visitor (Tf and Td are both dropped), so the snapshot at 'ET' is exactly the state + // this page ends on. + var (reader, state, _) = RunAndCaptureFinalState( BuildPageDoc( "BT\n/F1 (12) Tf\n10 /N Td\nET\n", "<< /Font << /F1 5 0 R >> >>", - new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), - out var reader); + new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))); - Assert.Null(interpreter.GraphicsState.Font); - Assert.Equal(0, interpreter.GraphicsState.FontSize); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); + Assert.Null(state!.Font); + Assert.Equal(0, state.FontSize); + Assert.Equal(Matrix.Identity, state.TextMatrix); var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) .ToList(); Assert.Single(reports); @@ -793,12 +869,13 @@ public void Quote_movesToTheNextLine_andForwardsTheStringOperandUntouched() // ' fell through to the default (state-inert) case: the text matrices were left untouched // and the string operand still reached the visitor, since forwarding happens either way. const string content = "BT\n100 TL\n5 0 Td\n(a) '\nET\n"; - var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + var (reader, state, visitor) = RunAndCaptureFinalState(BuildPageDoc(content)); // Td moves the line matrix to [1 0 0 1 5 0]; ' then moves by (0, -TL=-100): - // Tlm_new = [1 0 0 1 0 -100] x [1 0 0 1 5 0] = [1 0 0 1 5 -100]. - Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), interpreter.TextState.TextLineMatrix); - Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), interpreter.TextState.TextMatrix); + // Tlm_new = [1 0 0 1 0 -100] x [1 0 0 1 5 0] = [1 0 0 1 5 -100]. 'ET' follows and does not + // disturb either matrix, so the snapshot at 'ET' (the last operator) still reflects it. + Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), state!.TextLineMatrix); + Assert.Equal(new Matrix(1, 0, 0, 1, 5, -100), state.TextMatrix); Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); var quote = visitor.Operators.Single(o => o.Op == "'"); @@ -812,13 +889,13 @@ public void DoubleQuote_setsWordAndCharSpacing_thenMovesToTheNextLine() // Table 107: "\" ... shall have the same effect as this code: aw Tw ac Tc string '": aw and // ac land in the text state before the T*-equivalent move. const string content = "BT\n50 TL\n7 8 (x) \"\nET\n"; - var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + var (reader, state, visitor) = RunAndCaptureFinalState(BuildPageDoc(content)); - Assert.Equal(7, interpreter.GraphicsState.WordSpacing); - Assert.Equal(8, interpreter.GraphicsState.CharSpacing); + Assert.Equal(7, state!.WordSpacing); + Assert.Equal(8, state.CharSpacing); // TextLineMatrix starts at identity (no Td/Tm ran); " moves by (0, -TL=-50): // Tlm_new = [1 0 0 1 0 -50] x [1 0 0 1 0 0] = [1 0 0 1 0 -50]. - Assert.Equal(new Matrix(1, 0, 0, 1, 0, -50), interpreter.TextState.TextLineMatrix); + Assert.Equal(new Matrix(1, 0, 0, 1, 0, -50), state.TextLineMatrix); Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); var quote = visitor.Operators.Single(o => o.Op == "\""); @@ -829,12 +906,12 @@ public void DoubleQuote_setsWordAndCharSpacing_thenMovesToTheNextLine() public void DoubleQuote_withNonNumericSpacingOperands_reportsOnce_andIsDropped() { const string content = "BT\n(a) (b) (x) \"\nET\n"; - var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + var (reader, state, visitor) = RunAndCaptureFinalState(BuildPageDoc(content)); Assert.DoesNotContain(visitor.Operators, o => o.Op == "\""); - Assert.Equal(0, interpreter.GraphicsState.WordSpacing); - Assert.Equal(0, interpreter.GraphicsState.CharSpacing); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + Assert.Equal(0, state!.WordSpacing); + Assert.Equal(0, state.CharSpacing); + Assert.Equal(Matrix.Identity, state.TextLineMatrix); var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) .ToList(); Assert.Single(reports); @@ -844,10 +921,10 @@ public void DoubleQuote_withNonNumericSpacingOperands_reportsOnce_andIsDropped() public void Quote_withANonStringOperand_reportsOnce_andIsDropped() { const string content = "BT\n5 '\nET\n"; - var (reader, interpreter, visitor) = Run(BuildPageDoc(content)); + var (reader, state, visitor) = RunAndCaptureFinalState(BuildPageDoc(content)); Assert.DoesNotContain(visitor.Operators, o => o.Op == "'"); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + Assert.Equal(Matrix.Identity, state!.TextLineMatrix); var reports = reader.Diagnostics.Where(d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed) .ToList(); Assert.Single(reports); @@ -858,17 +935,16 @@ public void Quote_withANonStringOperand_reportsOnce_andIsDropped() [Fact] public void Gs_withFont_surfacesTheFontSelectionToTheState() { - var interpreter = RunAndKeepInterpreter( + var (_, state, _) = RunAndCaptureFinalState( BuildPageDoc( "/G1 gs\n", "<< /ExtGState << /G1 6 0 R >> >>", new Obj(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"), - new Obj(6, "<< /Type /ExtGState /Font [5 0 R 18] >>")), - out _); + new Obj(6, "<< /Type /ExtGState /Font [5 0 R 18] >>"))); - var fontRef = Assert.IsType(interpreter.GraphicsState.Font); + var fontRef = Assert.IsType(state!.Font); Assert.Equal(5, fontRef.ObjectNumber); - Assert.Equal(18, interpreter.GraphicsState.FontSize); + Assert.Equal(18, state.FontSize); } [Fact] @@ -1095,26 +1171,37 @@ public void Do_concatenatesTheFormsOwnMatrixIntoTheCtm_forOperatorsInsideTheForm "0 0 1 1 re"u8.ToArray())); Matrix? ctmInsideForm = null; + Matrix? ctmAfterForm = null; var reader = PdfReader.Open(doc); var interpreter = new ContentInterpreter(reader); - var probe = new CtmProbeVisitor(() => ctmInsideForm ??= interpreter.GraphicsState.Ctm); + // Both captured DURING their own callback (#402 round 7: Run's own exit-time reset means + // GraphicsState is not readable once Run itself returns), not just the inside-the-form one: + // the invoker's own restored CTM is read from 'w', the operator right after 'Do' returns, + // rather than from the interpreter after Run. + var probe = new CtmProbeVisitor( + () => ctmInsideForm ??= interpreter.GraphicsState.Ctm, + () => ctmAfterForm ??= interpreter.GraphicsState.Ctm); interpreter.Run(reader.GetPage(0), probe); // §8.3.4: CTM_new = M x CTM_old; the invoker's own CTM at the point of Do is identity, so // the composed value equals the form's own /Matrix exactly. Assert.Equal(new Matrix(2, 0, 0, 2, 10, 10), ctmInsideForm); // The invoker's own CTM is untouched once Do returns. - Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + Assert.Equal(Matrix.Identity, ctmAfterForm); } - private sealed class CtmProbeVisitor(Action onFirstOperatorInsideForm) : IContentVisitor + private sealed class CtmProbeVisitor( + Action onFirstOperatorInsideForm, Action onFirstOperatorAfterForm) : IContentVisitor { private bool _insideForm; + private bool _formEnded; public void OnOperator(string operatorName, IReadOnlyList operands, int offset) { if (_insideForm) onFirstOperatorInsideForm(); + else if (_formEnded) + onFirstOperatorAfterForm(); } public void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset) { } @@ -1124,7 +1211,11 @@ public void OnFormBegin( int offset) => _insideForm = true; - public void OnFormEnd(int objectNumber) => _insideForm = false; + public void OnFormEnd(int objectNumber) + { + _insideForm = false; + _formEnded = true; + } } // ── HandleDo reports a resource that resolves but is not a usable XObject (#402 round 2) ───── @@ -1197,9 +1288,11 @@ public void Do_onAFormThatChangesTheCtm_doesNotLeakTheChangeIntoTheInvoker() 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "3 0 0 3 0 0 cm"u8.ToArray())); - var interpreter = RunAndKeepInterpreter(doc, out var reader); + var (reader, state, _) = RunAndCaptureFinalState(doc); - Assert.Equal(Matrix.Identity, interpreter.GraphicsState.Ctm); + // 'w', the trailing top-level operator, is state-neutral (line width is not tracked), so + // the snapshot at 'w' still reflects the CTM exactly as Do's own finally restored it. + Assert.Equal(Matrix.Identity, state!.Ctm); Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.OperandStackMalformed); } @@ -1218,10 +1311,11 @@ public void Do_onAFormThatOpensItsOwnTextObject_doesNotLeakTextMatricesIntoTheIn 11, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", "BT 999 888 Td ET"u8.ToArray())); - var interpreter = RunAndKeepInterpreter(doc, out _); + var (_, state, _) = RunAndCaptureFinalState(doc); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextMatrix); - Assert.Equal(Matrix.Identity, interpreter.TextState.TextLineMatrix); + // Same reasoning as the CTM test above: 'w' does not disturb either text matrix. + Assert.Equal(Matrix.Identity, state!.TextMatrix); + Assert.Equal(Matrix.Identity, state.TextLineMatrix); } [Fact] @@ -1572,7 +1666,7 @@ public void A85WithTwoSpacesAfterId_andExplicitL_skipsBothSeparatorBytes() // ASCIIHexDecode or ASCII85Decode skips any FURTHER white-space too, before /L's own bytes // are counted. Before this fix, at most one whitespace byte was ever consumed regardless of // filter, so the second space here was misread as the payload's own first byte and /L=4 - // came out 3 bytes short of the real data. + // came out 3 bytes short of the image data. byte[] data = "ABCD"u8.ToArray(); var content = $"BI /F /A85 /L {data.Length} ID " + Encoding.ASCII.GetString(data) + " EI\nQ\n1 w\n"; @@ -1724,7 +1818,7 @@ public void DctInlineImage_withAFalseEiInsideItsData_isSkippedByTheScan() { // A whitespace-delimited "EI" followed by an unterminated literal string, which never // closes anywhere in the rest of the content stream. The resync probe's lex attempt from - // that point throws, so ScanForEi rejects it and keeps looking for the real one. + // that point throws, so ScanForEi rejects it and keeps looking for the terminating 'EI'. var falseCandidate = " EI (unterminated "u8.ToArray(); byte[] jpegNoise = [0x01, 0x02, 0xFF, 0xD8, 0xFF]; var data = jpegNoise.Concat(falseCandidate).Concat(new byte[] { 3, 4, 5 }).ToArray(); @@ -2888,6 +2982,286 @@ public void UnknownOperator_atTheExcerptBoundary_truncatesOnlyPastMaxQuotedToken Assert.Contains(expectedQuoted, report.Message, StringComparison.Ordinal); } + // ── Run's own outer catch no longer forwards ex.Message (#402 round 7) ────────────────────── + // + // PdfObjectParser's own "N G obj" header check and its numeric-range checks interpolate the + // whole malformed token into their own exception's Message, with no bound of its own (out of + // this PR's scope: PdfDocumentReader.Open's own callers read that Message directly too). + // BuildPageContentBuffer's ResolveStream/ResolveValue calls have no local catch for either, so + // an InvalidDataException from either used to reach Run's own outer catch and get interpolated + // into a diagnostic Message retained for the reader's own lifetime, in full. + + [Fact] + public void UnparsableContentsObjectHeader_reportsAFixedMessage_notTheWholeToken() + { + // Object 4's own "N G obj" header names no "obj" keyword at all, just 4,194,304 'A' bytes: + // PdfObjectParser.ParseIndirectObject's own ExpectToken(Keyword, "'obj' keyword") accepts + // that as A Keyword token, then IsKeyword(objKw.Raw, "obj") rejects it and throws with the + // whole token quoted. ResolveStream (PdfDocumentReader.cs) calls ParseIndirectObject with + // no local catch, so this reaches BuildPageContentBuffer's own AddElement (which only + // catches around GetDecodedStreamData, not around ResolveStream itself) and then Run's own + // outer catch, before InterpretStream is ever entered. + var badHeader = "4 0 " + new string('A', 4_194_304) + "\n"; + var doc = BuildPdfWithRawObjectBytes( + 1, + (1, Encoding.ASCII.GetBytes("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")), + (2, Encoding.ASCII.GetBytes("2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n")), + (3, Encoding.ASCII.GetBytes( + "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>\nendobj\n")), + (4, Encoding.ASCII.GetBytes(badHeader))); + + var (reader, _, visitor) = Run(doc); + + var report = Assert.Single( + reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.True( + report.Message.Length < 200, $"expected under 200 chars, got {report.Message.Length}."); + Assert.Equal( + "The page's content could not be fully resolved: an object it references could not " + + "be parsed.", + report.Message); + Assert.Equal(0, report.PageIndex); + Assert.Empty(visitor.Operators); + } + + [Fact] + public void TwoPagesSharingAMalformedContentsDictionary_eachReportOneBoundedMessage() + { + // Same bug, reached through PdfObjectParser.ParseReal's own "Real number out of range" + // exception instead: a 2,000,000-digit real literal in the STREAM'S OWN dictionary parses + // to +Infinity under double.TryParse and is rejected there, while ParseDictionary is still + // parsing the dictionary itself, well before "stream" is ever looked for. Two pages share + // object 4, so each page's own Run independently re-parses it and independently throws: + // Run's own catch reports no object number, so the sink's (code, object, page) dedupe key + // does not collapse the two pages' reports into one the way an object-scoped report would. + var digits = new string('1', 2_000_000); + var doc = BuildPdfWithRawObjectBytes( + 1, + (1, Encoding.ASCII.GetBytes("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")), + (2, Encoding.ASCII.GetBytes( + "2 0 obj\n<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>\nendobj\n")), + (3, Encoding.ASCII.GetBytes( + "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>\nendobj\n")), + (5, Encoding.ASCII.GetBytes( + "5 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + "/Resources << >> /Contents 4 0 R >>\nendobj\n")), + (4, Encoding.ASCII.GetBytes($"4 0 obj\n<< /Bogus {digits}.0 >>\nendobj\n"))); + + var reader = PdfReader.Open(doc, new PdfReaderOptions()); + var interpreter = new ContentInterpreter(reader); + interpreter.Run(reader.GetPage(0), new RecordingVisitor()); + interpreter.Run(reader.GetPage(1), new RecordingVisitor()); + + var reports = reader.Diagnostics + .Where(d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError).ToList(); + Assert.Equal(2, reports.Count); + foreach (var report in reports) + { + Assert.True( + report.Message.Length < 200, $"expected under 200 chars, got {report.Message.Length}."); + Assert.Equal( + "The page's content could not be fully resolved: an object it references could " + + "not be parsed.", + report.Message); + } + Assert.Equal([0, 1], reports.Select(r => r.PageIndex).OrderBy(p => p)); + var totalMessageLength = reader.Diagnostics.Sum(d => d.Message.Length); + Assert.True( + totalMessageLength < 1000, + $"expected under 1000 total chars across every diagnostic, got {totalMessageLength}."); + } + + // ── Inline image dictionary values bypass the composite cap (#402 round 7) ───────────────────── + // + // The key/value loop in HandleInlineImage used to hand every value straight to + // parser.ParseObject() with no CompositeOperandWithinCap pre-scan, so an array or dictionary + // value here bypassed MaxCompositeOperandElements entirely, and the number of key/value pairs + // the loop admitted was uncapped too. + + [Fact] + public void InlineImageDictionaryArrayValue_overTheCompositeCap_dropsTheImage_boundingAllocation() + { + // /D [1 1 1 ...] used to be fully materialised as a PdfArray, one boxed PdfInteger per + // element, before any cap was consulted: measured (pre-fix, 10,000,000 elements) 892.4 MiB + // allocated for one dropped image and no diagnostic. 20,000 elements is already well over + // MaxCompositeOperandElements (8192), so this is over the cap on a ~40 KB content stream, + // not a stress case. + var content = "BI /D [" + string.Concat(Enumerable.Repeat("1 ", 20_000)) + "] ID ABC EI\n1 w\n"; + var doc = BuildPageDoc(content); + + var before = GC.GetAllocatedBytesForCurrentThread(); + var (reader, _, visitor) = Run(doc); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + var report = Assert.Single( + reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal( + "An inline image dictionary value exceeds 8192 tokens; the image was dropped.", + report.Message); + Assert.DoesNotContain( + reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentStreamLexError); + Assert.Empty(visitor.InlineImages); + // HandleInlineImage's own "return false" ends the stream the same way every other + // malformed-dictionary path does (see HandleInlineImage's own class doc): '1 w', after the + // image's own 'EI', is never reached. + Assert.Empty(visitor.Operators); + // An allocation bound, not a wall-clock one (#400): generous by design. Measured on the + // fixed code: 96,952 bytes (94.7 KiB) for this ~40 KB content stream (the cap is decided by + // the lexer alone, so the array is never materialised); 16 MiB leaves ample margin. + Assert.True( + allocated < 16L * 1024 * 1024, + $"expected under 16 MiB allocated; measured {allocated / (1024.0 * 1024.0):F2} MiB."); + } + + [Fact] + public void InlineImageDictionaryFilterArrayValue_overTheCompositeCap_dropsTheImage() + { + // The /F-array branch (the abbreviation-expanding one) copies every element into a fresh + // List and PdfArray on top of parser.ParseObject()'s own allocation, so it is + // checked separately from the general /D case above: both call sites need the same + // pre-scan, not just one of them. + var content = "BI /F [" + string.Concat(Enumerable.Repeat("/AHx ", 20_000)) + "] ID ABC EI\n1 w\n"; + var doc = BuildPageDoc(content); + + var (reader, _, visitor) = Run(doc); + + var report = Assert.Single( + reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal( + "An inline image dictionary value exceeds 8192 tokens; the image was dropped.", + report.Message); + Assert.Empty(visitor.InlineImages); + Assert.Empty(visitor.Operators); + } + + [Theory] + [InlineData(8192, false)] + [InlineData(8193, true)] + public void InlineImageDictionaryArrayValue_atTheCompositeCapBoundary( + int elementCount, bool expectDropped) + { + var content = "BI /D [" + string.Concat(Enumerable.Repeat("1 ", elementCount)) + "] ID ABC EI\nQ\n"; + var doc = BuildPageDoc(content); + + var (reader, _, visitor) = Run(doc); + + if (expectDropped) + { + Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Empty(visitor.InlineImages); + Assert.DoesNotContain(visitor.Operators, o => o.Op == "Q"); + } + else + { + Assert.DoesNotContain( + reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + var img = Assert.Single(visitor.InlineImages); + var decodeArray = (PdfArray)img.Dict.Get(new PdfName("Decode"))!; + Assert.Equal(elementCount, decodeArray.Count); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + } + + [Theory] + [InlineData(65)] + [InlineData(100)] + public void InlineImageDictionary_withMoreEntriesThanTheCap_reportsOnce_andDropsTheImage(int entryCount) + { + // Table 91 lists eleven entries; a producer's own dictionary never comes close to 64, so + // this covers only a hostile BI...ID section that never reaches ID at all. 65 is the first + // count over the cap; 100 shows the report stays a single one however far past it. + var keys = string.Concat(Enumerable.Range(0, entryCount).Select(i => $"/K{i:D3} 1 ")); + var content = "BI " + keys + "ID ABC EI\nQ\n"; + var doc = BuildPageDoc(content); + + var (reader, _, visitor) = Run(doc); + + var report = Assert.Single(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + Assert.Equal( + "An inline image dictionary has more than 64 entries; the image was dropped.", + report.Message); + Assert.Empty(visitor.InlineImages); + Assert.DoesNotContain(visitor.Operators, o => o.Op == "Q"); + } + + [Theory] + [InlineData(60)] + [InlineData(64)] + public void InlineImageDictionary_withEntriesUpToTheCap_deliversTheImage(int entryCount) + { + // 64 is the cap itself and must still deliver: the check rejects the 65th entry, not the + // 64th. + var keys = string.Concat(Enumerable.Range(0, entryCount).Select(i => $"/K{i:D3} 1 ")); + var content = "BI " + keys + "ID ABC EI\nQ\n"; + var doc = BuildPageDoc(content); + + var (reader, _, visitor) = Run(doc); + + Assert.DoesNotContain(reader.Diagnostics, d => d.Code == PdfReaderDiagnosticCode.ContentLimitExceeded); + var img = Assert.Single(visitor.InlineImages); + Assert.True(img.Data.AsSpan().SequenceEqual("ABC"u8)); + Assert.Contains(visitor.Operators, o => o.Op == "Q"); + } + + // ── Run resets its own state on exit, not only entry (#402 round 7) ──────────────────────────── + + private sealed class FontOperandCapturingVisitor : IContentVisitor + { + public WeakReference? FontOperand { get; private set; } + + public void OnOperator(string operatorName, IReadOnlyList operands, int offset) + { + if (operatorName == "Tf" && FontOperand is null) + FontOperand = new WeakReference(operands[0]); + } + + public void OnInlineImage(PdfDictionary dictionary, ReadOnlyMemory data, int offset) { } + + public void OnFormBegin( + PdfDictionary formDictionary, Matrix formMatrix, PdfRectangle? boundingBox, int objectNumber, + int offset) + { } + + public void OnFormEnd(int objectNumber) { } + } + + [Fact] + public void Run_dropsAnUnconsumedFontOperandOnExit_soItBecomesCollectable() + { + // GraphicsState.Font (set by Tf) is the one PdfObject field this interpreter's own state + // keeps past the operator that set it. Without Run's own exit-time reset, an + // attacker-sized name operand pushed through Tf, with no later Tf or 'gs' to overwrite + // Font and no Q to pop it, stayed referenced by _gs for as long as this interpreter + // instance itself lived, not merely for the duration of Run (measured, pre-fix: a + // 4,194,304-byte name operand retained 33,562,352 bytes from a 16,779-byte file, 2000x; a + // second Run on the same interpreter did not release it either). A throwaway visitor is + // used here, not RecordingVisitor: RecordingVisitor's own Operators list keeps a strong + // reference to every operand it has ever seen, by design (#98), which would keep this + // WeakReference alive regardless of what Run itself does. + var content = "/" + new string('B', 4_194_304) + " 12 Tf\n1 w\n"; + var doc = BuildPageDoc(content); + var reader = PdfReader.Open(doc, new PdfReaderOptions()); + var page = reader.GetPage(0); + var interpreter = new ContentInterpreter(reader); + var visitor = new FontOperandCapturingVisitor(); + + interpreter.Run(page, visitor); + + var weakRef = visitor.FontOperand; + Assert.NotNull(weakRef); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + Assert.False( + weakRef!.IsAlive, + "expected the Tf font operand to be collectable once Run has returned."); + GC.KeepAlive(interpreter); + } + // ── Fuzzing ────────────────────────────────────────────────────────────────────────────────── private static readonly byte[] FuzzContent = Encoding.ASCII.GetBytes( diff --git a/tests/VellumPdf.Reader.Tests/PageTreeTests.cs b/tests/VellumPdf.Reader.Tests/PageTreeTests.cs index 560c2c7c..676ac156 100644 --- a/tests/VellumPdf.Reader.Tests/PageTreeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PageTreeTests.cs @@ -584,6 +584,33 @@ public void RootIsTheCatalogItself_withKidsBoltedOn_reportsPageTreeMissing() Assert.Equal(1, d.ObjectNumber); } + [Fact] + public void RootWithAnOversizedType_reportsOnlyAFixedExcerpt() + { + // A /Type value has no length bound of its own (Annex C.1), and PageTreeMissing is retained + // for the reader's own lifetime, so before this fix the root's own bogus /Type name was + // interpolated whole via PdfName.ToString(): the same class of defect ContentInterpreter's + // own QuoteExcerpt exists to bound, found sweeping this PR's other diagnostic sites for it + // (#402 round 7). + var hugeType = new string('A', 1_000_000); + var bytes = BuildPdf( + rootObjectNumber: 1, + (1, "<< /Type /Catalog /Pages 2 0 R >>"), + (2, $"<< /Type /{hugeType} /Kids [3 0 R] /Count 1 >>"), + (3, "<< /Type /Page /MediaBox [0 0 100 100] >>")); + + using var reader = Open(bytes); + + Assert.Equal(0, reader.PageCount); + var d = Assert.Single(reader.Diagnostics, x => x.Code == PdfReaderDiagnosticCode.PageTreeMissing); + // The fixed sentence this excerpt sits inside is itself close to 200 chars (the ISO + // citation), so the bound here is against the 1,000,000-char /Type value this message used + // to carry whole, not against PageTreeMissing's own fixed text. + Assert.True(d.Message.Length < 300, $"expected under 300 chars, got {d.Message.Length}."); + var expectedExcerpt = $"/Type /{new string('A', 32)}... (1000000 chars)"; + Assert.Contains(expectedExcerpt, d.Message, StringComparison.Ordinal); + } + [Fact] public void RootWithEmptyKids_yieldsZeroPages_withNoDiagnosticAtAll() { @@ -642,6 +669,31 @@ public void KidClassifiedByType_wrongTypeIsSkipped_neitherNodeNorPage() Assert.Contains(malformed, d => d.ObjectNumber == 3); } + [Fact] + public void KidWithAnOversizedType_reportsOnlyAFixedExcerpt() + { + // Same defect as RootWithAnOversizedType_reportsOnlyAFixedExcerpt above, on the OTHER + // ClassifyByType Skip site (a non-root node reached through /Kids): the sink's own (code, + // object, page) dedupe key does not collapse this against a DIFFERENT object's own report, + // so several oversized-/Type kids each retain their own excerpt-bounded copy rather than + // sharing amplification the way a page-scoped report would (#402 round 7). + var hugeType = new string('B', 1_000_000); + var bytes = BuildPdf( + rootObjectNumber: 1, + (1, "<< /Type /Catalog /Pages 2 0 R >>"), + (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + (3, $"<< /Type /{hugeType} >>")); + + using var reader = Open(bytes); + + Assert.Equal(0, reader.PageCount); + var d = Assert.Single(reader.Diagnostics, x => x.Code == PdfReaderDiagnosticCode.PageTreeNodeMalformed); + Assert.Equal(3, d.ObjectNumber); + Assert.True(d.Message.Length < 200, $"expected under 200 chars, got {d.Message.Length}."); + var expectedExcerpt = $"Object declares /{new string('B', 32)}... (1000000 chars)"; + Assert.Contains(expectedExcerpt, d.Message, StringComparison.Ordinal); + } + [Fact] public void TypePagesWithNoUsableKids_reportsNodeMalformed_contributesNoChildren() { diff --git a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs index 3b6fc711..081e4611 100644 --- a/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs +++ b/tests/VellumPdf.Reader.Tests/PdfLexerContentModeTests.cs @@ -60,7 +60,7 @@ public void ContentStreamMode_lexesPostScriptHeritageBytesAsOneByteKeywordTokens [Fact] public void ContentStreamMode_lexesADictionaryEndImmediatelyAfterALoneGreaterThan() { - // "> >>": a lone '>' keyword token, then a real dictionary-end token right after it. + // "> >>": a lone '>' keyword token, then a two-byte '>>' dictionary-end token right after it. var lexer = new PdfLexer(Encoding.ASCII.GetBytes("> >>"), contentStreamMode: true); var first = lexer.NextToken();