diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Common.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Common.cs index d9629a2..1be0b2a 100644 --- a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Common.cs +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Common.cs @@ -91,19 +91,13 @@ SearchValues arbitraryAllowed private Utf8TokenBuffer DoctypeSystem => _doctypeSystem ??= new(64); public Utf8HtmlTokenizer(IUtf8HtmlTokenSink sink) - : this(sink, null, HtmlStreamingLimits.Default, countInputBytes: true) - { - } + : this(sink, null, HtmlStreamingLimits.Default, countInputBytes: true) { } public Utf8HtmlTokenizer(IUtf8HtmlTokenSink sink, HtmlStreamingLimits limits) - : this(sink, null, limits, countInputBytes: true) - { - } + : this(sink, null, limits, countInputBytes: true) { } public Utf8HtmlTokenizer(IUtf8HtmlTokenSink sink, Utf8HtmlTokenizerStateMetrics? stateMetrics) - : this(sink, stateMetrics, HtmlStreamingLimits.Default, countInputBytes: true) - { - } + : this(sink, stateMetrics, HtmlStreamingLimits.Default, countInputBytes: true) { } public Utf8HtmlTokenizer( IUtf8HtmlTokenSink sink, @@ -284,858 +278,4 @@ internal void RefreshStartTagSourceRangeSink() => /// Enters the CDATA section state after the tree constructor accepts a CDATA declaration in foreign content. /// public void EnterCDataSection() => _state = State.CDataSection; - - /// - /// Consumes complete, well-formed UTF-8. Use for arbitrary - /// byte chunks or malformed-input replacement. - /// - public void Write(ReadOnlyMemory utf8) - { - RecordInputSegment(); - WriteCore(utf8.Span, yieldOnRequest: false); - } - - /// - public void Write(ReadOnlySpan utf8) - { - RecordInputSegment(); - WriteCore(utf8, yieldOnRequest: false); - } - - /// - /// Consumes input until the sink requests a yield. The caller must resubmit the unconsumed suffix before offering - /// unrelated input. - /// - /// The number of bytes consumed from . - internal Int32 WriteUntilYield(ReadOnlySpan utf8) - { - ResetYieldRequest(); - return WriteCore(utf8, yieldOnRequest: true); - } - - internal void RequestYield() => _yieldRequested = true; - - internal void ResetYieldRequest() => _yieldRequested = false; - - internal void RecordInputSegment() => _segments++; - - private Int32 WriteCore(ReadOnlySpan utf8, Boolean yieldOnRequest) - { - ThrowIfCompleted(); - var previousBytesConsumed = 0L; - if (TResourceLimits.Enabled) - { - previousBytesConsumed = _inputBytesConsumed; - var observedInputBytes = SaturatingAdd(previousBytesConsumed, utf8.Length); - if (observedInputBytes > _maximumInputBytesAllowed) - { - throw new HtmlStreamingLimitExceededException( - HtmlStreamingLimit.InputBytes, - _maximumInputBytesAllowed, - observedInputBytes - ); - } - } - - var consumed = WriteTrustedUtf8(utf8, yieldOnRequest); - if (TResourceLimits.Enabled) - { - _inputBytesConsumed = SaturatingAdd(previousBytesConsumed, consumed); - } - - return consumed; - } - - internal Boolean IsYieldRequested => _yieldRequested; - - internal Int32 WriteTrustedUtf8(ReadOnlySpan utf8, Boolean yieldOnRequest) => - _stateMetrics is null - ? WriteUtf8(utf8, yieldOnRequest) - : WriteUtf8(utf8, yieldOnRequest); - - internal Boolean TracksStartTagSourceRanges => _startTagSourceRangeSink is not null; - - /// - /// The normalized-input offset before which no future tag edit can land. While a tag is open, - /// the offset pins to its '<'; every insertion, replacement, and separator look-back sits at or above - /// the returned value. Meaningful only while is set, - /// and only at quiescent points (between calls). - /// - internal Int64 RewritePublishableOffset - { - get - { - if (_completed) - { - return _normalizedBytesConsumed; - } - - var wantsEndTagRange = _startTagSourceRangeSink?.WantsEndTagSourceRanges == true; - switch (_state) - { - case State.TagOpen: - return _lastLessThanSourceOffset; - case State.EndTagOpen: - case State.RawLessThan: - case State.RawEndTagOpen: - case State.RawEndTagName: - case State.ScriptLessThan: - case State.ScriptEndTagName: - case State.ScriptEscapedLessThan: - case State.ScriptEscapedEndTagName: - return wantsEndTagRange ? _lastLessThanSourceOffset : _currentSourceOffset; - case State.TagName: - case State.BeforeAttributeName: - case State.AttributeName: - case State.AfterAttributeName: - case State.BeforeAttributeValue: - case State.AttributeValueDoubleQuoted: - case State.AttributeValueSingleQuoted: - case State.AttributeValueUnquoted: - case State.AfterAttributeValueQuoted: - case State.SelfClosingStartTag: - return _isEndTag && !wantsEndTagRange ? _currentSourceOffset : _currentTagSourceOffset; - case State.CharacterReference: - // A reference inside a captured attribute value keeps the start tag open. - return IsTagTailState(_returnState) && (!_isEndTag || wantsEndTagRange) - ? _currentTagSourceOffset - : _currentSourceOffset; - default: - // Comments, doctypes, raw text, and script data never produce edits. - return _normalizedBytesConsumed; - } - } - } - - /// - /// Consumes input that skipped UTF-8 validation, stopping before the first byte that would - /// need it. Returns the number of bytes consumed; the byte at that position, if any, is - /// non-ASCII and must be validated by the caller before re-entry via - /// . Must not be used while - /// is set: discarded text swallows unvalidated bytes - /// raw, but an observing sink republishes the stream, which must be normalized UTF-8. - /// - internal Int32 WriteArbitraryAscii(ReadOnlySpan utf8, Boolean yieldOnRequest) => - _stateMetrics is null - ? WriteUtf8(utf8, yieldOnRequest) - : WriteUtf8(utf8, yieldOnRequest); - - private Int32 WriteUtf8(ReadOnlySpan utf8, Boolean yieldOnRequest) - where TMetrics : struct, IStateMetricsPolicy - where TTrust : struct, IInputTrustPolicy - { - var trackSourceRanges = _startTagSourceRangeSink is not null; - var sourceBase = trackSourceRanges ? _normalizedBytesConsumed : 0; - var index = 0; - - try - { - while (index < utf8.Length) - { - if (!_pendingCarriageReturn) - { - if (IsTagTailState(_state) && (_isEndTag || _startTagEmitted)) - { - var sourceOffset = trackSourceRanges ? sourceBase + index : 0; - var consumed = !_isEndTag && _captureStartTagAttributes - ? ScanTagTail( - utf8[index..], - sourceOffset, - trackSourceRanges, - yieldOnRequest - ) - : ScanTagTail( - utf8[index..], - sourceOffset, - trackSourceRanges, - yieldOnRequest - ); - - if (consumed > 0) - { - index += consumed; - if (yieldOnRequest && _yieldRequested) - { - return index; - } - - continue; - } - } - else if (_state == State.TagName) - { - var remaining = utf8.Slice(index); - var stop = IndexOfTagNameStop(remaining); - var run = stop < 0 ? remaining.Length : stop; - - if (run > 0) - { - // _state is TagName by the test above; naming the constant lets it fold - // instead of reloading the field in the hot path. - RecordState((Int32)State.TagName, run); - AppendTagName(remaining[..run]); - index += run; - if (stop < 0) - { - // The name continues past this span; nothing to fuse. - continue; - } - } - - var stopByte = remaining[run]; - // Tag-name stop fusion: the byte that ended the name is in-span and its whole - // effect here is one of three transitions, so take them instead of a per-byte - // round-trip. Not the full terminator set - '\0' becomes a replacement - // character, '\r' starts CR normalization, non-ASCII must bounce to the caller. - if (stopByte is (Byte)'\t' or (Byte)'\n' or (Byte)'\f' or (Byte)' ' or (Byte)'/' or (Byte)'>') - { - index++; - RecordFusedTagNameStopIf(); - if (trackSourceRanges) - { - _currentSourceOffset = sourceBase + index; - } - - if (stopByte == (Byte)'>') - { - FinishTag(selfClosing: false); - } - else - { - // Mirrors the ProcessTagState TagName arm: the start tag is emitted - // before the state changes, and end tags no-op inside EmitTagStart. - EmitTagStart(); - _state = stopByte == (Byte)'/' ? State.SelfClosingStartTag : State.BeforeAttributeName; - } - - if (yieldOnRequest && _yieldRequested) - { - return index; - } - - continue; - } - } - else if (_state == State.Data && _textUtf8CarryLength == 0) - { - var remaining = utf8.Slice(index); - Int32 run; - - if (remaining[0] == (Byte)'<') - { - run = 0; - } - else if (!_captureText) - { - run = remaining.IndexOf((Byte)'<'); - } - else - { - run = IndexOfCaptureStop(remaining, DataTextTerminators, DataTextArbitraryAllowed); - } - - if (run < 0) - { - run = remaining.Length; - } - - if (run > 0) - { - RecordState((Int32)_state, run); - if (_captureText) - { - EmitText(utf8.Slice(index, run)); - - if (RawTextEnabled) - { - EmitRawText(sourceBase + index, utf8.Slice(index, run), CurrentRawTextType()); - } - - if (yieldOnRequest && _yieldRequested) - { - index += run; - return index; - } - } - - index += run; - continue; - } - - // Stop-byte fusion: a '<' followed by an ASCII letter (or "/" + letter) in - // data state always begins a tag, so consume through the first name byte here - // instead of surrendering '<', the follower, and the letter to three per-byte - // dispatcher round-trips. Only data state qualifies: raw text, RCDATA, and - // script data route '<' through the end-tag candidate machinery below. All - // fused bytes are ASCII by test, so the trust policy is satisfied. - if (remaining[0] == (Byte)'<' && remaining.Length >= 2) - { - Int32 fused; - Boolean isEndTag; - if (IsAsciiLetter(remaining[1])) - { - fused = 2; - isEndTag = false; - } - else if (remaining[1] == (Byte)'/' && remaining.Length >= 3 && IsAsciiLetter(remaining[2])) - { - fused = 3; - isEndTag = true; - } - else - { - goto PerByteStateMachine; - } - - if (TMetrics.Enabled) - { - RecordFusedTagOpen(isEndTag); - } - - index += fused; - if (trackSourceRanges) - { - _currentSourceOffset = sourceBase + index; - _lastLessThanSourceOffset = sourceBase + index - fused; - } - - BeginTag(isEndTag, remaining[fused - 1]); - continue; - } - } - else if (_state is State.RawText or State.ScriptData && _textUtf8CarryLength == 0) - { - var consumed = _captureText - ? ScanRawTextContent( - utf8[index..], - sourceBase + index, - trackSourceRanges, - yieldOnRequest - ) - : ScanRawTextContent( - utf8[index..], - sourceBase + index, - trackSourceRanges, - yieldOnRequest - ); - if (consumed > 0) - { - index += consumed; - if (yieldOnRequest && _yieldRequested) - { - return index; - } - - continue; - } - } - else if (_state == State.Comment) - { - var consumed = ScanCommentContent(utf8[index..]); - if (consumed > 0) - { - index += consumed; - continue; - } - } - else if (_state == State.Plaintext && _textUtf8CarryLength == 0) - { - var consumed = ScanPlaintextContent(utf8[index..], sourceBase + index); - if (consumed > 0) - { - index += consumed; - if (yieldOnRequest && _yieldRequested) - { - return index; - } - - continue; - } - } - } - - PerByteStateMachine: - var value = utf8[index]; - if (TTrust.StopAtNonAscii && value >= 0x80) - { - // Unvalidated non-ASCII: hand back to the caller, which validates the run and - // re-feeds it through the trusted entry point. - return index; - } - - index++; - if (trackSourceRanges) - { - _currentSourceOffset = sourceBase + index; - if (value == (Byte)'<') - { - _lastLessThanSourceOffset = _currentSourceOffset - 1; - } - } - - if (_pendingCarriageReturn) - { - _pendingCarriageReturn = false; - if (value == (Byte)'\n') - { - if (RawTextEnabled && IsRawTextInputState(_state)) - EmitRawCurrentByte(value, CurrentRawTextType()); - continue; - } - } - - if (value == (Byte)'\r') - { - _pendingCarriageReturn = true; - value = (Byte)'\n'; - } - - if (IsScriptState(_state)) - { - ProcessScriptInput(value, utf8, ref index); - } - else - { - Process(value); - } - - if (yieldOnRequest && _yieldRequested) - { - return index; - } - } - - return index; - } - finally - { - if (trackSourceRanges) - { - _normalizedBytesConsumed = sourceBase + index; - // Only the consumed slice is reported, so partial consumption (yield, or the fused - // ASCII path handing back at a non-ASCII byte) never double-observes the tail. - _startTagSourceRangeSink!.ObserveNormalizedUtf8End(sourceBase, utf8[..index], RewritePublishableOffset); - } - } - } - - public void Complete() - { - if (_completed) - { - return; - } - - Utf8AttributeNameIndex.Reset(ref _seenAttributeIndex); - switch (_state) - { - case State.TagOpen: - EmitText("<"u8); - EmitRawText(_normalizedBytesConsumed - 1, "<"u8, Utf8HtmlTextType.Data); - break; - case State.EndTagOpen: - EmitText(" _captureText = (_sink.Capture & Utf8HtmlTokenCapture.Text) != 0; - - /// - /// True while a raw-text sink is attached. Callers test this before doing any work that only a - /// raw-text consumer needs - classifying the text type, materializing a single-byte span - so a - /// parse with no raw-text sink pays one field test per site and nothing else. - /// - private Boolean RawTextEnabled => (_streamingFlags & RawTextEnabledFlag) != 0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void EmitRawCurrentByte(Byte value, Utf8HtmlTextType textType) - { - if (RawTextEnabled) - EmitRawCurrentByteCore(value, textType); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void EmitRawCurrentByteCore(Byte value, Utf8HtmlTextType textType) - { - Span source = stackalloc Byte[1]; - source[0] = _pendingCarriageReturn ? (Byte)'\r' : value; - EmitRawTextCore(_currentSourceOffset - 1, source, textType); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void EmitRawText(Int64 sourceStart, ReadOnlySpan utf8, Utf8HtmlTextType textType) - { - if (RawTextEnabled && !utf8.IsEmpty) - EmitRawTextCore(sourceStart, utf8, textType); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void EmitRawTextCore(Int64 sourceStart, ReadOnlySpan utf8, Utf8HtmlTextType textType) - { - if (utf8.IsEmpty || _streamingCommentSink is not IUtf8HtmlRawTextSink { WantsRawText: true } rawTextSink) - return; - rawTextSink.RawText(sourceStart, utf8, textType, isLastInTextNode: false); - _streamingFlags = (Byte)( - (_streamingFlags & (CommentStartedFlag | CaptureCommentFlag | RawTextEnabledFlag)) - | RawTextNodeOpenFlag - | ((Byte)textType << RawTextTypeShift) - ); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void EndRawText(Int64 sourceOffset) - { - if ((_streamingFlags & RawTextNodeOpenFlag) != 0) - EndRawTextCore(sourceOffset); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void EndRawTextCore(Int64 sourceOffset) - { - ((IUtf8HtmlRawTextSink)_streamingCommentSink!).RawText( - sourceOffset, - [], - (Utf8HtmlTextType)((_streamingFlags & RawTextTypeMask) >> RawTextTypeShift), - isLastInTextNode: true - ); - _streamingFlags &= CommentStartedFlag | CaptureCommentFlag | RawTextEnabledFlag; - } - - private Utf8HtmlTextType CurrentRawTextType() => - _state switch - { - State.Plaintext => Utf8HtmlTextType.PlainText, - >= State.ScriptData and <= State.ScriptDoubleEscapeEnd => Utf8HtmlTextType.ScriptData, - >= State.CDataSection and <= State.CDataSectionEnd => Utf8HtmlTextType.CDataSection, - >= State.RawText and <= State.RawEndTagName => IsRcData() - ? Utf8HtmlTextType.RcData - : Utf8HtmlTextType.RawText, - State.CharacterReference when _returnState == State.RawText => IsRcData() - ? Utf8HtmlTextType.RcData - : Utf8HtmlTextType.RawText, - _ => Utf8HtmlTextType.Data, - }; - - private static Boolean IsRawTextInputState(State state) => - state - is State.Data - or State.Plaintext - or State.RawText - or State.CDataSection - or >= State.ScriptData - and <= State.ScriptDoubleEscapeEnd; - - private Boolean StreamingCommentStarted - { - get => (_streamingFlags & CommentStartedFlag) != 0; - set => - _streamingFlags = value - ? (Byte)(_streamingFlags | CommentStartedFlag) - : (Byte)(_streamingFlags & ~CommentStartedFlag); - } - - private Boolean CapturesStreamingComment - { - get => (_streamingFlags & CaptureCommentFlag) != 0; - set => - _streamingFlags = value - ? (Byte)(_streamingFlags | CaptureCommentFlag) - : (Byte)(_streamingFlags & ~CaptureCommentFlag); - } - - private const Byte CommentStartedFlag = 0x01; - private const Byte CaptureCommentFlag = 0x02; - private const Byte RawTextEnabledFlag = 0x80; - private const Byte RawTextNodeOpenFlag = 0x40; - private const Byte RawTextTypeMask = 0x1C; - private const Int32 RawTextTypeShift = 2; - - private void EmitText(ReadOnlySpan utf8) - { - if (_captureText) - { - _sink.Text(utf8); - } - } - - private void EmitReplacementCharacter() => EmitText("\uFFFD"u8); - - private void EmitCDataText(ReadOnlySpan utf8) - { - if (!SkipCDATA) - { - EmitText(utf8); - } - } - - private void EmitCDataByte(Byte value) - { - if (!SkipCDATA) - { - EmitByte(value); - } - } - - private void EmitByte(Byte value) - { - if (!_captureText) - { - return; - } - - if (_textUtf8CarryLength != 0) - { - _textUtf8Carry |= (UInt32)value << (_textUtf8CarryLength++ * 8); - if (_textUtf8CarryLength == _textUtf8ExpectedLength) - { - Span scalar = stackalloc Byte[4]; - for (var index = 0; index < _textUtf8CarryLength; index++) - { - scalar[index] = (Byte)(_textUtf8Carry >> (index * 8)); - } - - EmitText(scalar[.._textUtf8CarryLength]); - _textUtf8Carry = 0; - _textUtf8CarryLength = 0; - _textUtf8ExpectedLength = 0; - } - - return; - } - - if (value >= 0x80) - { - _textUtf8Carry = value; - _textUtf8CarryLength = 1; - _textUtf8ExpectedLength = Utf8SequenceLength(value); - return; - } - - Span single = stackalloc Byte[1]; - single[0] = value; - EmitText(single); - } - - private void Reconsume(ref Boolean reconsume, State state) - { - _state = state; - reconsume = true; - _reconsumes++; - } - - private void Append(Utf8TokenBuffer buffer, Byte value) - { - if (TResourceLimits.Enabled) - { - EnsureBufferedTokenCapacity(1); - } - - buffer.Append(value); - if (TResourceLimits.Enabled) - { - ObserveBufferAppend(1); - } - } - - private void Append(Utf8TokenBuffer buffer, ReadOnlySpan value) - { - if (TResourceLimits.Enabled) - { - EnsureBufferedTokenCapacity(value.Length); - } - - buffer.Append(value); - if (TResourceLimits.Enabled) - { - ObserveBufferAppend(value.Length); - } - } - - private void ObserveBufferAppend(Int32 count) - { - _bufferedTokenBytes += count; - if (_bufferedTokenBytes > _maximumBufferedTokenBytes) - { - _maximumBufferedTokenBytes = (Int32)Math.Min(_bufferedTokenBytes, Int32.MaxValue); - } - } - - private void EnsureBufferedTokenCapacity(Int32 additional) - { - var observed = SaturatingAdd(_bufferedTokenBytes, additional); - if (observed > _maximumBufferedTokenBytesAllowed) - { - ThrowLimitExceeded(HtmlStreamingLimit.BufferedTokenBytes, _maximumBufferedTokenBytesAllowed, observed); - } - } - - private static ReadOnlySpan WrittenSpan(Utf8TokenBuffer? buffer) => - buffer is null ? ReadOnlySpan.Empty : buffer.WrittenSpan; - - private void Clear(Utf8TokenBuffer? buffer) - { - if (buffer is null) - { - return; - } - - if (TResourceLimits.Enabled) - { - _bufferedTokenBytes -= buffer.WrittenCount; - } - - buffer.ResetWrittenCount(); - } - - private static Int64 SaturatingAdd(Int64 value, Int32 additional) => - value > Int64.MaxValue - additional ? Int64.MaxValue : value + additional; - - private static void ThrowLimitExceeded(HtmlStreamingLimit limit, Int64 allowed, Int64 observed) => - throw new HtmlStreamingLimitExceededException(limit, allowed, observed); - - private static Int32 Utf8SequenceLength(Byte lead) => - lead switch - { - < 0x80 => 1, - < 0xE0 => 2, - < 0xF0 => 3, - < 0xF8 => 4, - _ => 1, - }; - - private void AppendTagName(Byte value) => Append(_name, value); - - private void AppendTagName(ReadOnlySpan value) => Append(_name, value); - - private void AppendTagNameReplacedNull(Byte value) - { - if (value == 0) - { - AppendTagName("\uFFFD"u8); - } - else - { - AppendTagName(value); - } - } - - private Utf8HtmlName CurrentTagName() => new(_name.WrittenSpan, ref _tagNameIdentityCache); - - private Utf8HtmlName CurrentAttributeName() => new(_attributeName.WrittenSpan, ref _attributeNameIdentityCache); - - private Boolean IsAttributeReturnState() => _returnState is not State.Data and not State.RawText; - - private static Boolean StartsWithAsciiIgnoreCase(ReadOnlySpan expected, ReadOnlySpan candidate) - { - if (candidate.Length > expected.Length) - { - return false; - } - - for (var i = 0; i < candidate.Length; i++) - { - if (AsciiLower(expected[i]) != AsciiLower(candidate[i])) - { - return false; - } - } - - return true; - } - - // Measured 2026-08-09: rewriting this as an explicit 64-bit mask test moved nothing - // (news.google -0.13%, linkedin +0.16%, stackoverflow -1.79% retired instructions, inside the - // run-to-run spread). The JIT already lowers a constant or-pattern over a byte to a bit test, - // so there is no chain to shorten here. Left as the readable form on purpose. - private static Boolean IsSpace(Byte value) => value is 0x09 or 0x0A or 0x0C or 0x0D or 0x20; - - private static Boolean IsAsciiLetter(Byte value) => - (UInt32)(value - 'A') <= 'Z' - 'A' || (UInt32)(value - 'a') <= 'z' - 'a'; - - private static Boolean IsAsciiAlphaNumeric(Byte value) => IsAsciiLetter(value) || (UInt32)(value - '0') <= 9; - - private static Boolean IsTagDelimiter(Byte value) => value is (Byte)'>' or (Byte)'/' || IsSpace(value); - - private static Byte AsciiLower(Byte value) => Utf8NameHash.ToLowerAscii(value); - - private void ThrowIfCompleted() - { - if (_completed) - { - throw new InvalidOperationException("The tokenizer is already complete."); - } - } -} \ No newline at end of file +} diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Output.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Output.cs new file mode 100644 index 0000000..52c056d --- /dev/null +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Output.cs @@ -0,0 +1,335 @@ +using System.Runtime.CompilerServices; + +namespace AngleSharp.ReadOnlyDom.Streaming.Tokenization; + +internal partial class Utf8HtmlTokenizer + where TResourceLimits : struct, IResourceLimitPolicy +{ + private void RefreshCapture() => _captureText = (_sink.Capture & Utf8HtmlTokenCapture.Text) != 0; + + /// + /// True while a raw-text sink is attached. Callers test this before doing any work that only a + /// raw-text consumer needs - classifying the text type, materializing a single-byte span - so a + /// parse with no raw-text sink pays one field test per site and nothing else. + /// + private Boolean RawTextEnabled => (_streamingFlags & RawTextEnabledFlag) != 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EmitRawCurrentByte(Byte value, Utf8HtmlTextType textType) + { + if (RawTextEnabled) + EmitRawCurrentByteCore(value, textType); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EmitRawCurrentByteCore(Byte value, Utf8HtmlTextType textType) + { + Span source = stackalloc Byte[1]; + source[0] = _pendingCarriageReturn ? (Byte)'\r' : value; + EmitRawTextCore(_currentSourceOffset - 1, source, textType); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EmitRawText(Int64 sourceStart, ReadOnlySpan utf8, Utf8HtmlTextType textType) + { + if (RawTextEnabled && !utf8.IsEmpty) + EmitRawTextCore(sourceStart, utf8, textType); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EmitRawTextCore(Int64 sourceStart, ReadOnlySpan utf8, Utf8HtmlTextType textType) + { + if (utf8.IsEmpty || _streamingCommentSink is not IUtf8HtmlRawTextSink { WantsRawText: true } rawTextSink) + return; + rawTextSink.RawText(sourceStart, utf8, textType, isLastInTextNode: false); + _streamingFlags = (Byte)( + (_streamingFlags & (CommentStartedFlag | CaptureCommentFlag | RawTextEnabledFlag)) + | RawTextNodeOpenFlag + | ((Byte)textType << RawTextTypeShift) + ); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EndRawText(Int64 sourceOffset) + { + if ((_streamingFlags & RawTextNodeOpenFlag) != 0) + EndRawTextCore(sourceOffset); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EndRawTextCore(Int64 sourceOffset) + { + ((IUtf8HtmlRawTextSink)_streamingCommentSink!).RawText( + sourceOffset, + [], + (Utf8HtmlTextType)((_streamingFlags & RawTextTypeMask) >> RawTextTypeShift), + isLastInTextNode: true + ); + _streamingFlags &= CommentStartedFlag | CaptureCommentFlag | RawTextEnabledFlag; + } + + private Utf8HtmlTextType CurrentRawTextType() => + _state switch + { + State.Plaintext => Utf8HtmlTextType.PlainText, + >= State.ScriptData and <= State.ScriptDoubleEscapeEnd => Utf8HtmlTextType.ScriptData, + >= State.CDataSection and <= State.CDataSectionEnd => Utf8HtmlTextType.CDataSection, + >= State.RawText and <= State.RawEndTagName => IsRcData() + ? Utf8HtmlTextType.RcData + : Utf8HtmlTextType.RawText, + State.CharacterReference when _returnState == State.RawText => IsRcData() + ? Utf8HtmlTextType.RcData + : Utf8HtmlTextType.RawText, + _ => Utf8HtmlTextType.Data, + }; + + private static Boolean IsRawTextInputState(State state) => + state + is State.Data + or State.Plaintext + or State.RawText + or State.CDataSection + or >= State.ScriptData + and <= State.ScriptDoubleEscapeEnd; + + private Boolean StreamingCommentStarted + { + get => (_streamingFlags & CommentStartedFlag) != 0; + set => + _streamingFlags = value + ? (Byte)(_streamingFlags | CommentStartedFlag) + : (Byte)(_streamingFlags & ~CommentStartedFlag); + } + + private Boolean CapturesStreamingComment + { + get => (_streamingFlags & CaptureCommentFlag) != 0; + set => + _streamingFlags = value + ? (Byte)(_streamingFlags | CaptureCommentFlag) + : (Byte)(_streamingFlags & ~CaptureCommentFlag); + } + + private const Byte CommentStartedFlag = 0x01; + private const Byte CaptureCommentFlag = 0x02; + private const Byte RawTextEnabledFlag = 0x80; + private const Byte RawTextNodeOpenFlag = 0x40; + private const Byte RawTextTypeMask = 0x1C; + private const Int32 RawTextTypeShift = 2; + + private void EmitText(ReadOnlySpan utf8) + { + if (_captureText) + { + _sink.Text(utf8); + } + } + + private void EmitReplacementCharacter() => EmitText("\uFFFD"u8); + + private void EmitCDataText(ReadOnlySpan utf8) + { + if (!SkipCDATA) + { + EmitText(utf8); + } + } + + private void EmitCDataByte(Byte value) + { + if (!SkipCDATA) + { + EmitByte(value); + } + } + + private void EmitByte(Byte value) + { + if (!_captureText) + { + return; + } + + if (_textUtf8CarryLength != 0) + { + _textUtf8Carry |= (UInt32)value << (_textUtf8CarryLength++ * 8); + if (_textUtf8CarryLength == _textUtf8ExpectedLength) + { + Span scalar = stackalloc Byte[4]; + for (var index = 0; index < _textUtf8CarryLength; index++) + { + scalar[index] = (Byte)(_textUtf8Carry >> (index * 8)); + } + + EmitText(scalar[.._textUtf8CarryLength]); + _textUtf8Carry = 0; + _textUtf8CarryLength = 0; + _textUtf8ExpectedLength = 0; + } + + return; + } + + if (value >= 0x80) + { + _textUtf8Carry = value; + _textUtf8CarryLength = 1; + _textUtf8ExpectedLength = Utf8SequenceLength(value); + return; + } + + Span single = stackalloc Byte[1]; + single[0] = value; + EmitText(single); + } + + private void Reconsume(ref Boolean reconsume, State state) + { + _state = state; + reconsume = true; + _reconsumes++; + } + + private void Append(Utf8TokenBuffer buffer, Byte value) + { + if (TResourceLimits.Enabled) + { + EnsureBufferedTokenCapacity(1); + } + + buffer.Append(value); + if (TResourceLimits.Enabled) + { + ObserveBufferAppend(1); + } + } + + private void Append(Utf8TokenBuffer buffer, ReadOnlySpan value) + { + if (TResourceLimits.Enabled) + { + EnsureBufferedTokenCapacity(value.Length); + } + + buffer.Append(value); + if (TResourceLimits.Enabled) + { + ObserveBufferAppend(value.Length); + } + } + + private void ObserveBufferAppend(Int32 count) + { + _bufferedTokenBytes += count; + if (_bufferedTokenBytes > _maximumBufferedTokenBytes) + { + _maximumBufferedTokenBytes = (Int32)Math.Min(_bufferedTokenBytes, Int32.MaxValue); + } + } + + private void EnsureBufferedTokenCapacity(Int32 additional) + { + var observed = SaturatingAdd(_bufferedTokenBytes, additional); + if (observed > _maximumBufferedTokenBytesAllowed) + { + ThrowLimitExceeded(HtmlStreamingLimit.BufferedTokenBytes, _maximumBufferedTokenBytesAllowed, observed); + } + } + + private static ReadOnlySpan WrittenSpan(Utf8TokenBuffer? buffer) => + buffer is null ? ReadOnlySpan.Empty : buffer.WrittenSpan; + + private void Clear(Utf8TokenBuffer? buffer) + { + if (buffer is null) + { + return; + } + + if (TResourceLimits.Enabled) + { + _bufferedTokenBytes -= buffer.WrittenCount; + } + + buffer.ResetWrittenCount(); + } + + private static Int64 SaturatingAdd(Int64 value, Int32 additional) => + value > Int64.MaxValue - additional ? Int64.MaxValue : value + additional; + + private static void ThrowLimitExceeded(HtmlStreamingLimit limit, Int64 allowed, Int64 observed) => + throw new HtmlStreamingLimitExceededException(limit, allowed, observed); + + private static Int32 Utf8SequenceLength(Byte lead) => + lead switch + { + < 0x80 => 1, + < 0xE0 => 2, + < 0xF0 => 3, + < 0xF8 => 4, + _ => 1, + }; + + private void AppendTagName(Byte value) => Append(_name, value); + + private void AppendTagName(ReadOnlySpan value) => Append(_name, value); + + private void AppendTagNameReplacedNull(Byte value) + { + if (value == 0) + { + AppendTagName("\uFFFD"u8); + } + else + { + AppendTagName(value); + } + } + + private Utf8HtmlName CurrentTagName() => new(_name.WrittenSpan, ref _tagNameIdentityCache); + + private Utf8HtmlName CurrentAttributeName() => new(_attributeName.WrittenSpan, ref _attributeNameIdentityCache); + + private Boolean IsAttributeReturnState() => _returnState is not State.Data and not State.RawText; + + private static Boolean StartsWithAsciiIgnoreCase(ReadOnlySpan expected, ReadOnlySpan candidate) + { + if (candidate.Length > expected.Length) + { + return false; + } + + for (var i = 0; i < candidate.Length; i++) + { + if (AsciiLower(expected[i]) != AsciiLower(candidate[i])) + { + return false; + } + } + + return true; + } + + // Measured 2026-08-09: rewriting this as an explicit 64-bit mask test moved nothing + // (news.google -0.13%, linkedin +0.16%, stackoverflow -1.79% retired instructions, inside the + // run-to-run spread). The JIT already lowers a constant or-pattern over a byte to a bit test, + // so there is no chain to shorten here. Left as the readable form on purpose. + private static Boolean IsSpace(Byte value) => value is 0x09 or 0x0A or 0x0C or 0x0D or 0x20; + + private static Boolean IsAsciiLetter(Byte value) => + (UInt32)(value - 'A') <= 'Z' - 'A' || (UInt32)(value - 'a') <= 'z' - 'a'; + + private static Boolean IsAsciiAlphaNumeric(Byte value) => IsAsciiLetter(value) || (UInt32)(value - '0') <= 9; + + private static Boolean IsTagDelimiter(Byte value) => value is (Byte)'>' or (Byte)'/' || IsSpace(value); + + private static Byte AsciiLower(Byte value) => Utf8NameHash.ToLowerAscii(value); + + private void ThrowIfCompleted() + { + if (_completed) + { + throw new InvalidOperationException("The tokenizer is already complete."); + } + } +} diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagStates.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagStates.cs new file mode 100644 index 0000000..1d17939 --- /dev/null +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagStates.cs @@ -0,0 +1,252 @@ +namespace AngleSharp.ReadOnlyDom.Streaming.Tokenization; + +internal partial class Utf8HtmlTokenizer + where TResourceLimits : struct, IResourceLimitPolicy +{ + // Every start tag used to pay for a compact-key computation just to discover it is not + private void ProcessTagState(Byte value, ref Boolean reconsume) + { + switch (_state) + { + case State.TagOpen: + if (value == (Byte)'/') + { + _state = State.EndTagOpen; + } + else if (value == (Byte)'!') + { + EndRawText(_lastLessThanSourceOffset); + Clear(_candidate); + _state = State.MarkupDeclaration; + } + else if (value == (Byte)'?' && IsSupportingProcessingInstructions) + { + EndRawText(_lastLessThanSourceOffset); + Clear(_candidate); + if (!SkipProcessingInstructions) + { + Append(_candidate, value); + } + _state = State.ProcessingInstruction; + } + else if (value == (Byte)'?') + { + EndRawText(_lastLessThanSourceOffset); + Clear(_candidate); + Append(_candidate, value); + _state = State.BogusComment; + } + else if (IsAsciiLetter(value)) + { + BeginTag(isEndTag: false, value); + } + else + { + EmitText("<"u8); + EmitRawText(_currentSourceOffset - 2, "<"u8, Utf8HtmlTextType.Data); + Reconsume(ref reconsume, State.Data); + } + break; + case State.EndTagOpen: + if (IsAsciiLetter(value)) + { + BeginTag(isEndTag: true, value); + } + else if (value == (Byte)'>') + { + EndRawText(_lastLessThanSourceOffset); + _state = State.Data; + } + else + { + EndRawText(_lastLessThanSourceOffset); + Clear(_candidate); + Reconsume(ref reconsume, State.BogusComment); + } + break; + case State.TagName: + if (IsSpace(value)) + { + EmitTagStart(); + _state = State.BeforeAttributeName; + } + else if (value == (Byte)'/') + { + EmitTagStart(); + _state = State.SelfClosingStartTag; + } + else if (value == (Byte)'>') + { + FinishTag(selfClosing: false); + } + else + { + AppendTagNameReplacedNull(value); + } + + break; + case State.BeforeAttributeName: + if (IsSpace(value)) + { + break; + } + + if (value == (Byte)'/') + { + _state = State.SelfClosingStartTag; + break; + } + if (value == (Byte)'>') + { + FinishTag(selfClosing: false); + break; + } + if (_captureStartTagAttributes) + { + Clear(_attributeName); + Clear(_attributeValue); + _attributeNameIdentityCache.Reset(); + AppendReplacedNull(_attributeName, value, lowerAscii: false); + } + else + { + _attributeCapture = AttributeCapture.Discard; + } + _state = State.AttributeName; + break; + case State.AttributeName: + if (IsSpace(value)) + { + _state = State.AfterAttributeName; + } + else if (value == (Byte)'=') + { + DecideAttributeCapture(); + _state = State.BeforeAttributeValue; + } + else if (value is (Byte)'/' or (Byte)'>') + { + CommitAttribute(); + Reconsume(ref reconsume, State.BeforeAttributeName); + } + else + { + if (_captureStartTagAttributes) + { + AppendReplacedNull(_attributeName, value, lowerAscii: false); + } + } + + break; + case State.AfterAttributeName: + if (IsSpace(value)) + { + break; + } + + if (value == (Byte)'=') + { + DecideAttributeCapture(); + _state = State.BeforeAttributeValue; + break; + } + CommitAttribute(); + Reconsume(ref reconsume, State.BeforeAttributeName); + break; + case State.BeforeAttributeValue: + if (IsSpace(value)) + { + break; + } + + if (value == (Byte)'"') + { + _state = State.AttributeValueDoubleQuoted; + } + else if (value == (Byte)'\'') + { + _state = State.AttributeValueSingleQuoted; + } + else if (value == (Byte)'>') + { + CommitAttribute(); + FinishTag(selfClosing: false); + } + else + { + _state = State.AttributeValueUnquoted; + Reconsume(ref reconsume, _state); + } + break; + case State.AttributeValueDoubleQuoted: + case State.AttributeValueSingleQuoted: + var quote = _state == State.AttributeValueDoubleQuoted ? (Byte)'"' : (Byte)'\''; + if (value == quote) + { + _state = State.AfterAttributeValueQuoted; + } + else + { + // '&' is appended raw here: attribute character references are + // decoded over the buffered value when the attribute commits. + if (_attributeCapture == AttributeCapture.Capture) + { + AppendReplacedNull(AttributeValue, value, lowerAscii: false); + } + } + break; + case State.AttributeValueUnquoted: + if (IsSpace(value)) + { + CommitAttribute(); + _state = State.BeforeAttributeName; + } + else if (value == (Byte)'>') + { + CommitAttribute(); + FinishTag(selfClosing: false); + } + else + { + if (_attributeCapture == AttributeCapture.Capture) + { + AppendReplacedNull(AttributeValue, value, lowerAscii: false); + } + } + break; + case State.AfterAttributeValueQuoted: + CommitAttribute(); + if (IsSpace(value)) + { + _state = State.BeforeAttributeName; + } + else if (value == (Byte)'/') + { + _state = State.SelfClosingStartTag; + } + else if (value == (Byte)'>') + { + FinishTag(selfClosing: false); + } + else + { + Reconsume(ref reconsume, State.BeforeAttributeName); + } + + break; + case State.SelfClosingStartTag: + if (value == (Byte)'>') + { + FinishTag(selfClosing: true); + } + else + { + Reconsume(ref reconsume, State.BeforeAttributeName); + } + + break; + default: + throw new InvalidOperationException($"Unexpected {nameof(State)} value: {_state}"); + } + } +} diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagTailScanner.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagTailScanner.cs new file mode 100644 index 0000000..f77fbe0 --- /dev/null +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.TagTailScanner.cs @@ -0,0 +1,505 @@ +using System.Runtime.CompilerServices; + +namespace AngleSharp.ReadOnlyDom.Streaming.Tokenization; + +internal partial class Utf8HtmlTokenizer + where TResourceLimits : struct, IResourceLimitPolicy +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Boolean IsTagTailState(State state) => + state + is State.BeforeAttributeName + or State.AttributeName + or State.AfterAttributeName + or State.BeforeAttributeValue + or State.AttributeValueDoubleQuoted + or State.AttributeValueSingleQuoted + or State.AttributeValueUnquoted + or State.AfterAttributeValueQuoted + or State.SelfClosingStartTag; + + private Int32 ScanTagTail( + ReadOnlySpan utf8, + Int64 sourceOffset, + Boolean trackSourceRanges, + Boolean yieldOnRequest + ) + where TMetrics : struct, IStateMetricsPolicy + where TTrust : struct, IInputTrustPolicy + where TCapture : struct, IAttributeCapturePolicy + { + // Threaded form of the tag-tail states: the entry state dispatches once, control then + // transfers directly between the labelled constructs, and the current state lives in + // the program counter instead of the _state field until the span runs out or the tag + // finishes. Discarded tails dominate dense markup, so the per-transition dispatch and + // field traffic this removes is multiplied by the attribute count of the whole input. + // The CaptureOn instantiation runs the same shape over captured start tags, folding the + // name/value appends and commit calls into the transfers; bytes the scanner cannot + // resolve locally ('\0' replacement, '\r' normalization inside values, unvalidated + // non-ASCII) hand back to the per-byte fallback by writing _state and returning the + // consumed count, after which the outer loop re-enters the scanner. + var index = 0; + // A callback may request a yield while the old per-byte machine is still reconsuming the + // current delimiter. Finish that transition before returning at the same observable boundary. + var yieldAfterTransition = false; + Byte value; + switch (_state) + { + case State.BeforeAttributeName: + goto BeforeAttributeName; + case State.AttributeName: + goto AttributeName; + case State.AfterAttributeName: + goto AfterAttributeName; + case State.BeforeAttributeValue: + goto BeforeAttributeValue; + case State.AttributeValueDoubleQuoted: + goto AttributeValueDoubleQuoted; + case State.AttributeValueSingleQuoted: + goto AttributeValueSingleQuoted; + case State.AttributeValueUnquoted: + goto AttributeValueUnquoted; + case State.AfterAttributeValueQuoted: + goto AfterAttributeValueQuoted; + case State.SelfClosingStartTag: + goto SelfClosingStartTag; + default: + return 0; + } + + BeforeAttributeName: + while (true) + { + if ((UInt32)index >= (UInt32)utf8.Length) + { + _state = State.BeforeAttributeName; + return index; + } + value = utf8[index]; + RecordState((Int32)State.BeforeAttributeName, 1); + if (IsSpace(value)) + { + index++; + continue; + } + if (value == (Byte)'>') + { + _state = State.BeforeAttributeName; + FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); + return index; + } + if (value == (Byte)'/') + { + index++; + if (yieldAfterTransition) + { + _state = State.SelfClosingStartTag; + return index; + } + goto SelfClosingStartTag; + } + if (TCapture.Enabled) + { + if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) + { + // '\0' starts the name with a replacement character and unvalidated + // non-ASCII must bounce to the caller: per-byte fallback for both. + _state = State.BeforeAttributeName; + return index; + } + Clear(_attributeName); + Clear(_attributeValue); + _attributeNameIdentityCache.Reset(); + Append(_attributeName, value); + } + // CaptureOff still has to consume the byte that starts the discarded name. Leaving + // it for AttributeName changes its meaning when the byte is also a scanner delimiter + // (notably '=' after an unexpected solidus on an end tag). + index++; + if (yieldAfterTransition) + { + _state = State.AttributeName; + return index; + } + goto AttributeName; + } + + AttributeName: + { + var remaining = utf8[index..]; + var run = TCapture.Enabled + ? IndexOfCaptureStop(remaining, AttributeNameTerminators, AttributeNameArbitraryAllowed) + : IndexOfDiscardedAttributeNameStop(remaining); + if (run < 0) + { + RecordState((Int32)State.AttributeName, remaining.Length); + if (TCapture.Enabled && !remaining.IsEmpty) + { + Append(_attributeName, remaining); + } + _state = State.AttributeName; + return utf8.Length; + } + value = remaining[run]; + if (TCapture.Enabled) + { + if (run > 0) + { + Append(_attributeName, remaining[..run]); + } + if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) + { + RecordState((Int32)State.AttributeName, run); + _state = State.AttributeName; + return index + run; + } + } + RecordState((Int32)State.AttributeName, run + 1); + index += run; + if (value == (Byte)'=') + { + index++; + if (TCapture.Enabled) + { + DecideAttributeCapture(); + if (yieldOnRequest && _yieldRequested) + { + _state = State.BeforeAttributeValue; + return index; + } + } + goto BeforeAttributeValue; + } + if (IsSpace(value)) + { + index++; + goto AfterAttributeName; + } + // '/' or '>': the pending attribute commits, then the byte is reconsumed by the + // attribute-start handler, as in the general machine. + if (TCapture.Enabled) + { + CommitAttribute(); + yieldAfterTransition = yieldOnRequest && _yieldRequested; + } + goto BeforeAttributeName; + } + + AfterAttributeName: + while (true) + { + if ((UInt32)index >= (UInt32)utf8.Length) + { + _state = State.AfterAttributeName; + return index; + } + value = utf8[index]; + RecordState((Int32)State.AfterAttributeName, 1); + if (IsSpace(value)) + { + index++; + continue; + } + if (value == (Byte)'=') + { + index++; + if (TCapture.Enabled) + { + DecideAttributeCapture(); + if (yieldOnRequest && _yieldRequested) + { + _state = State.BeforeAttributeValue; + return index; + } + } + goto BeforeAttributeValue; + } + // Anything else ends the name-only attribute; the byte is reconsumed as an + // attribute-name starter (or '/', '>'). + if (TCapture.Enabled) + { + CommitAttribute(); + yieldAfterTransition = yieldOnRequest && _yieldRequested; + } + goto BeforeAttributeName; + } + + BeforeAttributeValue: + while (true) + { + if ((UInt32)index >= (UInt32)utf8.Length) + { + _state = State.BeforeAttributeValue; + return index; + } + value = utf8[index]; + RecordState((Int32)State.BeforeAttributeValue, 1); + if (IsSpace(value)) + { + index++; + continue; + } + if (value == (Byte)'"') + { + index++; + goto AttributeValueDoubleQuoted; + } + if (value == (Byte)'\'') + { + index++; + goto AttributeValueSingleQuoted; + } + if (value == (Byte)'>') + { + // FinishTag commits the pending missing-value attribute before closing. + _state = State.BeforeAttributeValue; + FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); + return index; + } + goto AttributeValueUnquoted; + } + + AttributeValueDoubleQuoted: + { + var remaining = utf8[index..]; + if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) + { + var run = IndexOfCaptureStop( + remaining, + DoubleQuotedAttributeValueTerminators, + DoubleQuotedAttributeValueArbitraryAllowed + ); + if (run < 0) + { + RecordState((Int32)State.AttributeValueDoubleQuoted, remaining.Length); + if (!remaining.IsEmpty) + { + Append(AttributeValue, remaining); + } + _state = State.AttributeValueDoubleQuoted; + return utf8.Length; + } + if (run > 0) + { + Append(AttributeValue, remaining[..run]); + } + if (remaining[run] != (Byte)'"') + { + // '\0' replacement, '\r' normalization, or unvalidated non-ASCII. + RecordState((Int32)State.AttributeValueDoubleQuoted, run); + _state = State.AttributeValueDoubleQuoted; + return index + run; + } + RecordState((Int32)State.AttributeValueDoubleQuoted, run + 1); + index += run + 1; + goto AfterAttributeValueQuoted; + } + else + { + var run = remaining.IndexOf((Byte)'"'); + if (run < 0) + { + RecordState((Int32)State.AttributeValueDoubleQuoted, remaining.Length); + _state = State.AttributeValueDoubleQuoted; + return utf8.Length; + } + RecordState((Int32)State.AttributeValueDoubleQuoted, run + 1); + index += run + 1; + goto AfterAttributeValueQuoted; + } + } + + AttributeValueSingleQuoted: + { + var remaining = utf8[index..]; + if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) + { + var run = IndexOfCaptureStop( + remaining, + SingleQuotedAttributeValueTerminators, + SingleQuotedAttributeValueArbitraryAllowed + ); + if (run < 0) + { + RecordState((Int32)State.AttributeValueSingleQuoted, remaining.Length); + if (!remaining.IsEmpty) + { + Append(AttributeValue, remaining); + } + _state = State.AttributeValueSingleQuoted; + return utf8.Length; + } + if (run > 0) + { + Append(AttributeValue, remaining[..run]); + } + if (remaining[run] != (Byte)'\'') + { + RecordState((Int32)State.AttributeValueSingleQuoted, run); + _state = State.AttributeValueSingleQuoted; + return index + run; + } + RecordState((Int32)State.AttributeValueSingleQuoted, run + 1); + index += run + 1; + goto AfterAttributeValueQuoted; + } + else + { + var run = remaining.IndexOf((Byte)'\''); + if (run < 0) + { + RecordState((Int32)State.AttributeValueSingleQuoted, remaining.Length); + _state = State.AttributeValueSingleQuoted; + return utf8.Length; + } + RecordState((Int32)State.AttributeValueSingleQuoted, run + 1); + index += run + 1; + goto AfterAttributeValueQuoted; + } + } + + AttributeValueUnquoted: + { + var remaining = utf8[index..]; + Int32 run; + if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) + { + run = IndexOfCaptureStop( + remaining, + UnquotedAttributeValueTerminators, + UnquotedAttributeValueArbitraryAllowed + ); + if (run < 0) + { + RecordState((Int32)State.AttributeValueUnquoted, remaining.Length); + if (!remaining.IsEmpty) + { + Append(AttributeValue, remaining); + } + _state = State.AttributeValueUnquoted; + return utf8.Length; + } + value = remaining[run]; + if (run > 0) + { + Append(AttributeValue, remaining[..run]); + } + if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) + { + RecordState((Int32)State.AttributeValueUnquoted, run); + _state = State.AttributeValueUnquoted; + return index + run; + } + } + else + { + run = remaining.IndexOfAny(DiscardedUnquotedAttributeValueTerminators); + if (run < 0) + { + RecordState((Int32)State.AttributeValueUnquoted, remaining.Length); + _state = State.AttributeValueUnquoted; + return utf8.Length; + } + value = remaining[run]; + } + RecordState((Int32)State.AttributeValueUnquoted, run + 1); + index += run; + if (value == (Byte)'>') + { + // FinishTag commits the pending attribute before closing. + _state = State.AttributeValueUnquoted; + FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); + return index; + } + // Whitespace ends the unquoted value. + if (TCapture.Enabled) + { + CommitAttribute(); + } + index++; + if (yieldOnRequest && _yieldRequested) + { + _state = State.BeforeAttributeName; + return index; + } + goto BeforeAttributeName; + } + + AfterAttributeValueQuoted: + { + if ((UInt32)index >= (UInt32)utf8.Length) + { + _state = State.AfterAttributeValueQuoted; + return index; + } + value = utf8[index]; + RecordState((Int32)State.AfterAttributeValueQuoted, 1); + if (TCapture.Enabled) + { + // The general machine commits the closed value before dispatching on the byte + // after the quote; a span ending here defers the commit the same way. + CommitAttribute(); + } + if (IsSpace(value)) + { + index++; + if (yieldOnRequest && _yieldRequested) + { + _state = State.BeforeAttributeName; + return index; + } + goto BeforeAttributeName; + } + if (value == (Byte)'/') + { + index++; + if (yieldOnRequest && _yieldRequested) + { + _state = State.SelfClosingStartTag; + return index; + } + goto SelfClosingStartTag; + } + if (value == (Byte)'>') + { + _state = State.AfterAttributeValueQuoted; + FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); + return index; + } + if (yieldOnRequest && _yieldRequested) + { + _state = State.BeforeAttributeName; + return index; + } + goto BeforeAttributeName; + } + + SelfClosingStartTag: + { + if ((UInt32)index >= (UInt32)utf8.Length) + { + _state = State.SelfClosingStartTag; + return index; + } + value = utf8[index]; + RecordState((Int32)State.SelfClosingStartTag, 1); + if (value == (Byte)'>') + { + _state = State.SelfClosingStartTag; + FinishScannedTag(ref index, selfClosing: true, sourceOffset, trackSourceRanges); + return index; + } + goto BeforeAttributeName; + } + } + + private void FinishScannedTag(ref Int32 index, Boolean selfClosing, Int64 sourceOffset, Boolean trackSourceRanges) + { + index++; + if (trackSourceRanges) + { + _currentSourceOffset = sourceOffset + index; + } + FinishTag(selfClosing); + } +} diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Tags.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Tags.cs index 8db638b..307d09d 100644 --- a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Tags.cs +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.Tags.cs @@ -94,9 +94,7 @@ private static Int32 IndexOfTagNameStop(ReadOnlySpan utf8) private static Int32 IndexOfDiscardedAttributeNameStop(ReadOnlySpan utf8) { var scan = - utf8.Length <= DiscardedAttributeNameScalarScanLimit - ? utf8 - : utf8[..DiscardedAttributeNameScalarScanLimit]; + utf8.Length <= DiscardedAttributeNameScalarScanLimit ? utf8 : utf8[..DiscardedAttributeNameScalarScanLimit]; var index = 0; while ((UInt32)index < (UInt32)scan.Length) { @@ -214,10 +212,7 @@ private void DecideAttributeCapture() // length cannot be wanted, so a rejected occurrence can never be the first-seen occurrence of // an accepted name, and duplicate tracking stays observationally identical. var lengths = _attributeNameLengths; - if ( - lengths != UInt64.MaxValue - && (lengths & (1UL << Math.Min(_attributeName.WrittenCount, 63))) == 0 - ) + if (lengths != UInt64.MaxValue && (lengths & (1UL << Math.Min(_attributeName.WrittenCount, 63))) == 0) { _attributeCapture = AttributeCapture.Discard; return; @@ -428,748 +423,4 @@ private void FinishTagCore(Boolean selfClosing) _state = _rawEndTag is null ? State.Data : State.RawText; } } - - // Every start tag used to pay for a compact-key computation just to discover it is not - private void ProcessTagState(Byte value, ref Boolean reconsume) - { - switch (_state) - { - case State.TagOpen: - if (value == (Byte)'/') - { - _state = State.EndTagOpen; - } - else if (value == (Byte)'!') - { - EndRawText(_lastLessThanSourceOffset); - Clear(_candidate); - _state = State.MarkupDeclaration; - } - else if (value == (Byte)'?' && IsSupportingProcessingInstructions) - { - EndRawText(_lastLessThanSourceOffset); - Clear(_candidate); - if (!SkipProcessingInstructions) - { - Append(_candidate, value); - } - _state = State.ProcessingInstruction; - } - else if (value == (Byte)'?') - { - EndRawText(_lastLessThanSourceOffset); - Clear(_candidate); - Append(_candidate, value); - _state = State.BogusComment; - } - else if (IsAsciiLetter(value)) - { - BeginTag(isEndTag: false, value); - } - else - { - EmitText("<"u8); - EmitRawText(_currentSourceOffset - 2, "<"u8, Utf8HtmlTextType.Data); - Reconsume(ref reconsume, State.Data); - } - break; - case State.EndTagOpen: - if (IsAsciiLetter(value)) - { - BeginTag(isEndTag: true, value); - } - else if (value == (Byte)'>') - { - EndRawText(_lastLessThanSourceOffset); - _state = State.Data; - } - else - { - EndRawText(_lastLessThanSourceOffset); - Clear(_candidate); - Reconsume(ref reconsume, State.BogusComment); - } - break; - case State.TagName: - if (IsSpace(value)) - { - EmitTagStart(); - _state = State.BeforeAttributeName; - } - else if (value == (Byte)'/') - { - EmitTagStart(); - _state = State.SelfClosingStartTag; - } - else if (value == (Byte)'>') - { - FinishTag(selfClosing: false); - } - else - { - AppendTagNameReplacedNull(value); - } - - break; - case State.BeforeAttributeName: - if (IsSpace(value)) - { - break; - } - - if (value == (Byte)'/') - { - _state = State.SelfClosingStartTag; - break; - } - if (value == (Byte)'>') - { - FinishTag(selfClosing: false); - break; - } - if (_captureStartTagAttributes) - { - Clear(_attributeName); - Clear(_attributeValue); - _attributeNameIdentityCache.Reset(); - AppendReplacedNull(_attributeName, value, lowerAscii: false); - } - else - { - _attributeCapture = AttributeCapture.Discard; - } - _state = State.AttributeName; - break; - case State.AttributeName: - if (IsSpace(value)) - { - _state = State.AfterAttributeName; - } - else if (value == (Byte)'=') - { - DecideAttributeCapture(); - _state = State.BeforeAttributeValue; - } - else if (value is (Byte)'/' or (Byte)'>') - { - CommitAttribute(); - Reconsume(ref reconsume, State.BeforeAttributeName); - } - else - { - if (_captureStartTagAttributes) - { - AppendReplacedNull(_attributeName, value, lowerAscii: false); - } - } - - break; - case State.AfterAttributeName: - if (IsSpace(value)) - { - break; - } - - if (value == (Byte)'=') - { - DecideAttributeCapture(); - _state = State.BeforeAttributeValue; - break; - } - CommitAttribute(); - Reconsume(ref reconsume, State.BeforeAttributeName); - break; - case State.BeforeAttributeValue: - if (IsSpace(value)) - { - break; - } - - if (value == (Byte)'"') - { - _state = State.AttributeValueDoubleQuoted; - } - else if (value == (Byte)'\'') - { - _state = State.AttributeValueSingleQuoted; - } - else if (value == (Byte)'>') - { - CommitAttribute(); - FinishTag(selfClosing: false); - } - else - { - _state = State.AttributeValueUnquoted; - Reconsume(ref reconsume, _state); - } - break; - case State.AttributeValueDoubleQuoted: - case State.AttributeValueSingleQuoted: - var quote = _state == State.AttributeValueDoubleQuoted ? (Byte)'"' : (Byte)'\''; - if (value == quote) - { - _state = State.AfterAttributeValueQuoted; - } - else - { - // '&' is appended raw here: attribute character references are - // decoded over the buffered value when the attribute commits. - if (_attributeCapture == AttributeCapture.Capture) - { - AppendReplacedNull(AttributeValue, value, lowerAscii: false); - } - } - break; - case State.AttributeValueUnquoted: - if (IsSpace(value)) - { - CommitAttribute(); - _state = State.BeforeAttributeName; - } - else if (value == (Byte)'>') - { - CommitAttribute(); - FinishTag(selfClosing: false); - } - else - { - if (_attributeCapture == AttributeCapture.Capture) - { - AppendReplacedNull(AttributeValue, value, lowerAscii: false); - } - } - break; - case State.AfterAttributeValueQuoted: - CommitAttribute(); - if (IsSpace(value)) - { - _state = State.BeforeAttributeName; - } - else if (value == (Byte)'/') - { - _state = State.SelfClosingStartTag; - } - else if (value == (Byte)'>') - { - FinishTag(selfClosing: false); - } - else - { - Reconsume(ref reconsume, State.BeforeAttributeName); - } - - break; - case State.SelfClosingStartTag: - if (value == (Byte)'>') - { - FinishTag(selfClosing: true); - } - else - { - Reconsume(ref reconsume, State.BeforeAttributeName); - } - - break; - default: - throw new InvalidOperationException($"Unexpected {nameof(State)} value: {_state}"); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Boolean IsTagTailState(State state) => - state is State.BeforeAttributeName - or State.AttributeName - or State.AfterAttributeName - or State.BeforeAttributeValue - or State.AttributeValueDoubleQuoted - or State.AttributeValueSingleQuoted - or State.AttributeValueUnquoted - or State.AfterAttributeValueQuoted - or State.SelfClosingStartTag; - - private Int32 ScanTagTail( - ReadOnlySpan utf8, - Int64 sourceOffset, - Boolean trackSourceRanges, - Boolean yieldOnRequest - ) - where TMetrics : struct, IStateMetricsPolicy - where TTrust : struct, IInputTrustPolicy - where TCapture : struct, IAttributeCapturePolicy - { - // Threaded form of the tag-tail states: the entry state dispatches once, control then - // transfers directly between the labelled constructs, and the current state lives in - // the program counter instead of the _state field until the span runs out or the tag - // finishes. Discarded tails dominate dense markup, so the per-transition dispatch and - // field traffic this removes is multiplied by the attribute count of the whole input. - // The CaptureOn instantiation runs the same shape over captured start tags, folding the - // name/value appends and commit calls into the transfers; bytes the scanner cannot - // resolve locally ('\0' replacement, '\r' normalization inside values, unvalidated - // non-ASCII) hand back to the per-byte fallback by writing _state and returning the - // consumed count, after which the outer loop re-enters the scanner. - var index = 0; - // A callback may request a yield while the old per-byte machine is still reconsuming the - // current delimiter. Finish that transition before returning at the same observable boundary. - var yieldAfterTransition = false; - Byte value; - switch (_state) - { - case State.BeforeAttributeName: - goto BeforeAttributeName; - case State.AttributeName: - goto AttributeName; - case State.AfterAttributeName: - goto AfterAttributeName; - case State.BeforeAttributeValue: - goto BeforeAttributeValue; - case State.AttributeValueDoubleQuoted: - goto AttributeValueDoubleQuoted; - case State.AttributeValueSingleQuoted: - goto AttributeValueSingleQuoted; - case State.AttributeValueUnquoted: - goto AttributeValueUnquoted; - case State.AfterAttributeValueQuoted: - goto AfterAttributeValueQuoted; - case State.SelfClosingStartTag: - goto SelfClosingStartTag; - default: - return 0; - } - - BeforeAttributeName: - while (true) - { - if ((UInt32)index >= (UInt32)utf8.Length) - { - _state = State.BeforeAttributeName; - return index; - } - value = utf8[index]; - RecordState((Int32)State.BeforeAttributeName, 1); - if (IsSpace(value)) - { - index++; - continue; - } - if (value == (Byte)'>') - { - _state = State.BeforeAttributeName; - FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); - return index; - } - if (value == (Byte)'/') - { - index++; - if (yieldAfterTransition) - { - _state = State.SelfClosingStartTag; - return index; - } - goto SelfClosingStartTag; - } - if (TCapture.Enabled) - { - if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) - { - // '\0' starts the name with a replacement character and unvalidated - // non-ASCII must bounce to the caller: per-byte fallback for both. - _state = State.BeforeAttributeName; - return index; - } - Clear(_attributeName); - Clear(_attributeValue); - _attributeNameIdentityCache.Reset(); - Append(_attributeName, value); - } - // CaptureOff still has to consume the byte that starts the discarded name. Leaving - // it for AttributeName changes its meaning when the byte is also a scanner delimiter - // (notably '=' after an unexpected solidus on an end tag). - index++; - if (yieldAfterTransition) - { - _state = State.AttributeName; - return index; - } - goto AttributeName; - } - - AttributeName: - { - var remaining = utf8[index..]; - var run = TCapture.Enabled - ? IndexOfCaptureStop(remaining, AttributeNameTerminators, AttributeNameArbitraryAllowed) - : IndexOfDiscardedAttributeNameStop(remaining); - if (run < 0) - { - RecordState((Int32)State.AttributeName, remaining.Length); - if (TCapture.Enabled && !remaining.IsEmpty) - { - Append(_attributeName, remaining); - } - _state = State.AttributeName; - return utf8.Length; - } - value = remaining[run]; - if (TCapture.Enabled) - { - if (run > 0) - { - Append(_attributeName, remaining[..run]); - } - if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) - { - RecordState((Int32)State.AttributeName, run); - _state = State.AttributeName; - return index + run; - } - } - RecordState((Int32)State.AttributeName, run + 1); - index += run; - if (value == (Byte)'=') - { - index++; - if (TCapture.Enabled) - { - DecideAttributeCapture(); - if (yieldOnRequest && _yieldRequested) - { - _state = State.BeforeAttributeValue; - return index; - } - } - goto BeforeAttributeValue; - } - if (IsSpace(value)) - { - index++; - goto AfterAttributeName; - } - // '/' or '>': the pending attribute commits, then the byte is reconsumed by the - // attribute-start handler, as in the general machine. - if (TCapture.Enabled) - { - CommitAttribute(); - yieldAfterTransition = yieldOnRequest && _yieldRequested; - } - goto BeforeAttributeName; - } - - AfterAttributeName: - while (true) - { - if ((UInt32)index >= (UInt32)utf8.Length) - { - _state = State.AfterAttributeName; - return index; - } - value = utf8[index]; - RecordState((Int32)State.AfterAttributeName, 1); - if (IsSpace(value)) - { - index++; - continue; - } - if (value == (Byte)'=') - { - index++; - if (TCapture.Enabled) - { - DecideAttributeCapture(); - if (yieldOnRequest && _yieldRequested) - { - _state = State.BeforeAttributeValue; - return index; - } - } - goto BeforeAttributeValue; - } - // Anything else ends the name-only attribute; the byte is reconsumed as an - // attribute-name starter (or '/', '>'). - if (TCapture.Enabled) - { - CommitAttribute(); - yieldAfterTransition = yieldOnRequest && _yieldRequested; - } - goto BeforeAttributeName; - } - - BeforeAttributeValue: - while (true) - { - if ((UInt32)index >= (UInt32)utf8.Length) - { - _state = State.BeforeAttributeValue; - return index; - } - value = utf8[index]; - RecordState((Int32)State.BeforeAttributeValue, 1); - if (IsSpace(value)) - { - index++; - continue; - } - if (value == (Byte)'"') - { - index++; - goto AttributeValueDoubleQuoted; - } - if (value == (Byte)'\'') - { - index++; - goto AttributeValueSingleQuoted; - } - if (value == (Byte)'>') - { - // FinishTag commits the pending missing-value attribute before closing. - _state = State.BeforeAttributeValue; - FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); - return index; - } - goto AttributeValueUnquoted; - } - - AttributeValueDoubleQuoted: - { - var remaining = utf8[index..]; - if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) - { - var run = IndexOfCaptureStop( - remaining, - DoubleQuotedAttributeValueTerminators, - DoubleQuotedAttributeValueArbitraryAllowed - ); - if (run < 0) - { - RecordState((Int32)State.AttributeValueDoubleQuoted, remaining.Length); - if (!remaining.IsEmpty) - { - Append(AttributeValue, remaining); - } - _state = State.AttributeValueDoubleQuoted; - return utf8.Length; - } - if (run > 0) - { - Append(AttributeValue, remaining[..run]); - } - if (remaining[run] != (Byte)'"') - { - // '\0' replacement, '\r' normalization, or unvalidated non-ASCII. - RecordState((Int32)State.AttributeValueDoubleQuoted, run); - _state = State.AttributeValueDoubleQuoted; - return index + run; - } - RecordState((Int32)State.AttributeValueDoubleQuoted, run + 1); - index += run + 1; - goto AfterAttributeValueQuoted; - } - else - { - var run = remaining.IndexOf((Byte)'"'); - if (run < 0) - { - RecordState((Int32)State.AttributeValueDoubleQuoted, remaining.Length); - _state = State.AttributeValueDoubleQuoted; - return utf8.Length; - } - RecordState((Int32)State.AttributeValueDoubleQuoted, run + 1); - index += run + 1; - goto AfterAttributeValueQuoted; - } - } - - AttributeValueSingleQuoted: - { - var remaining = utf8[index..]; - if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) - { - var run = IndexOfCaptureStop( - remaining, - SingleQuotedAttributeValueTerminators, - SingleQuotedAttributeValueArbitraryAllowed - ); - if (run < 0) - { - RecordState((Int32)State.AttributeValueSingleQuoted, remaining.Length); - if (!remaining.IsEmpty) - { - Append(AttributeValue, remaining); - } - _state = State.AttributeValueSingleQuoted; - return utf8.Length; - } - if (run > 0) - { - Append(AttributeValue, remaining[..run]); - } - if (remaining[run] != (Byte)'\'') - { - RecordState((Int32)State.AttributeValueSingleQuoted, run); - _state = State.AttributeValueSingleQuoted; - return index + run; - } - RecordState((Int32)State.AttributeValueSingleQuoted, run + 1); - index += run + 1; - goto AfterAttributeValueQuoted; - } - else - { - var run = remaining.IndexOf((Byte)'\''); - if (run < 0) - { - RecordState((Int32)State.AttributeValueSingleQuoted, remaining.Length); - _state = State.AttributeValueSingleQuoted; - return utf8.Length; - } - RecordState((Int32)State.AttributeValueSingleQuoted, run + 1); - index += run + 1; - goto AfterAttributeValueQuoted; - } - } - - AttributeValueUnquoted: - { - var remaining = utf8[index..]; - Int32 run; - if (TCapture.Enabled && _attributeCapture == AttributeCapture.Capture) - { - run = IndexOfCaptureStop( - remaining, - UnquotedAttributeValueTerminators, - UnquotedAttributeValueArbitraryAllowed - ); - if (run < 0) - { - RecordState((Int32)State.AttributeValueUnquoted, remaining.Length); - if (!remaining.IsEmpty) - { - Append(AttributeValue, remaining); - } - _state = State.AttributeValueUnquoted; - return utf8.Length; - } - value = remaining[run]; - if (run > 0) - { - Append(AttributeValue, remaining[..run]); - } - if (value == 0 || (TTrust.StopAtNonAscii && value >= 0x80)) - { - RecordState((Int32)State.AttributeValueUnquoted, run); - _state = State.AttributeValueUnquoted; - return index + run; - } - } - else - { - run = remaining.IndexOfAny(DiscardedUnquotedAttributeValueTerminators); - if (run < 0) - { - RecordState((Int32)State.AttributeValueUnquoted, remaining.Length); - _state = State.AttributeValueUnquoted; - return utf8.Length; - } - value = remaining[run]; - } - RecordState((Int32)State.AttributeValueUnquoted, run + 1); - index += run; - if (value == (Byte)'>') - { - // FinishTag commits the pending attribute before closing. - _state = State.AttributeValueUnquoted; - FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); - return index; - } - // Whitespace ends the unquoted value. - if (TCapture.Enabled) - { - CommitAttribute(); - } - index++; - if (yieldOnRequest && _yieldRequested) - { - _state = State.BeforeAttributeName; - return index; - } - goto BeforeAttributeName; - } - - AfterAttributeValueQuoted: - { - if ((UInt32)index >= (UInt32)utf8.Length) - { - _state = State.AfterAttributeValueQuoted; - return index; - } - value = utf8[index]; - RecordState((Int32)State.AfterAttributeValueQuoted, 1); - if (TCapture.Enabled) - { - // The general machine commits the closed value before dispatching on the byte - // after the quote; a span ending here defers the commit the same way. - CommitAttribute(); - } - if (IsSpace(value)) - { - index++; - if (yieldOnRequest && _yieldRequested) - { - _state = State.BeforeAttributeName; - return index; - } - goto BeforeAttributeName; - } - if (value == (Byte)'/') - { - index++; - if (yieldOnRequest && _yieldRequested) - { - _state = State.SelfClosingStartTag; - return index; - } - goto SelfClosingStartTag; - } - if (value == (Byte)'>') - { - _state = State.AfterAttributeValueQuoted; - FinishScannedTag(ref index, selfClosing: false, sourceOffset, trackSourceRanges); - return index; - } - if (yieldOnRequest && _yieldRequested) - { - _state = State.BeforeAttributeName; - return index; - } - goto BeforeAttributeName; - } - - SelfClosingStartTag: - { - if ((UInt32)index >= (UInt32)utf8.Length) - { - _state = State.SelfClosingStartTag; - return index; - } - value = utf8[index]; - RecordState((Int32)State.SelfClosingStartTag, 1); - if (value == (Byte)'>') - { - _state = State.SelfClosingStartTag; - FinishScannedTag(ref index, selfClosing: true, sourceOffset, trackSourceRanges); - return index; - } - goto BeforeAttributeName; - } - } - - private void FinishScannedTag(ref Int32 index, Boolean selfClosing, Int64 sourceOffset, Boolean trackSourceRanges) - { - index++; - if (trackSourceRanges) - { - _currentSourceOffset = sourceOffset + index; - } - FinishTag(selfClosing); - } } diff --git a/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.WriteLoop.cs b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.WriteLoop.cs new file mode 100644 index 0000000..f8d377a --- /dev/null +++ b/src/AngleSharp.ReadOnlyDom.Streaming/Tokenization/Tokenizer/Utf8HtmlTokenizer.WriteLoop.cs @@ -0,0 +1,532 @@ +namespace AngleSharp.ReadOnlyDom.Streaming.Tokenization; + +internal partial class Utf8HtmlTokenizer + where TResourceLimits : struct, IResourceLimitPolicy +{ + /// + /// Consumes complete, well-formed UTF-8. Use for arbitrary + /// byte chunks or malformed-input replacement. + /// + public void Write(ReadOnlyMemory utf8) + { + RecordInputSegment(); + WriteCore(utf8.Span, yieldOnRequest: false); + } + + /// + public void Write(ReadOnlySpan utf8) + { + RecordInputSegment(); + WriteCore(utf8, yieldOnRequest: false); + } + + /// + /// Consumes input until the sink requests a yield. The caller must resubmit the unconsumed suffix before offering + /// unrelated input. + /// + /// The number of bytes consumed from . + internal Int32 WriteUntilYield(ReadOnlySpan utf8) + { + ResetYieldRequest(); + return WriteCore(utf8, yieldOnRequest: true); + } + + internal void RequestYield() => _yieldRequested = true; + + internal void ResetYieldRequest() => _yieldRequested = false; + + internal void RecordInputSegment() => _segments++; + + private Int32 WriteCore(ReadOnlySpan utf8, Boolean yieldOnRequest) + { + ThrowIfCompleted(); + var previousBytesConsumed = 0L; + if (TResourceLimits.Enabled) + { + previousBytesConsumed = _inputBytesConsumed; + var observedInputBytes = SaturatingAdd(previousBytesConsumed, utf8.Length); + if (observedInputBytes > _maximumInputBytesAllowed) + { + throw new HtmlStreamingLimitExceededException( + HtmlStreamingLimit.InputBytes, + _maximumInputBytesAllowed, + observedInputBytes + ); + } + } + + var consumed = WriteTrustedUtf8(utf8, yieldOnRequest); + if (TResourceLimits.Enabled) + { + _inputBytesConsumed = SaturatingAdd(previousBytesConsumed, consumed); + } + + return consumed; + } + + internal Boolean IsYieldRequested => _yieldRequested; + + internal Int32 WriteTrustedUtf8(ReadOnlySpan utf8, Boolean yieldOnRequest) => + _stateMetrics is null + ? WriteUtf8(utf8, yieldOnRequest) + : WriteUtf8(utf8, yieldOnRequest); + + internal Boolean TracksStartTagSourceRanges => _startTagSourceRangeSink is not null; + + /// + /// The normalized-input offset before which no future tag edit can land. While a tag is open, + /// the offset pins to its '<'; every insertion, replacement, and separator look-back sits at or above + /// the returned value. Meaningful only while is set, + /// and only at quiescent points (between calls). + /// + internal Int64 RewritePublishableOffset + { + get + { + if (_completed) + { + return _normalizedBytesConsumed; + } + + var wantsEndTagRange = _startTagSourceRangeSink?.WantsEndTagSourceRanges == true; + switch (_state) + { + case State.TagOpen: + return _lastLessThanSourceOffset; + case State.EndTagOpen: + case State.RawLessThan: + case State.RawEndTagOpen: + case State.RawEndTagName: + case State.ScriptLessThan: + case State.ScriptEndTagName: + case State.ScriptEscapedLessThan: + case State.ScriptEscapedEndTagName: + return wantsEndTagRange ? _lastLessThanSourceOffset : _currentSourceOffset; + case State.TagName: + case State.BeforeAttributeName: + case State.AttributeName: + case State.AfterAttributeName: + case State.BeforeAttributeValue: + case State.AttributeValueDoubleQuoted: + case State.AttributeValueSingleQuoted: + case State.AttributeValueUnquoted: + case State.AfterAttributeValueQuoted: + case State.SelfClosingStartTag: + return _isEndTag && !wantsEndTagRange ? _currentSourceOffset : _currentTagSourceOffset; + case State.CharacterReference: + // A reference inside a captured attribute value keeps the start tag open. + return IsTagTailState(_returnState) && (!_isEndTag || wantsEndTagRange) + ? _currentTagSourceOffset + : _currentSourceOffset; + default: + // Comments, doctypes, raw text, and script data never produce edits. + return _normalizedBytesConsumed; + } + } + } + + /// + /// Consumes input that skipped UTF-8 validation, stopping before the first byte that would + /// need it. Returns the number of bytes consumed; the byte at that position, if any, is + /// non-ASCII and must be validated by the caller before re-entry via + /// . Must not be used while + /// is set: discarded text swallows unvalidated bytes + /// raw, but an observing sink republishes the stream, which must be normalized UTF-8. + /// + internal Int32 WriteArbitraryAscii(ReadOnlySpan utf8, Boolean yieldOnRequest) => + _stateMetrics is null + ? WriteUtf8(utf8, yieldOnRequest) + : WriteUtf8(utf8, yieldOnRequest); + + private Int32 WriteUtf8(ReadOnlySpan utf8, Boolean yieldOnRequest) + where TMetrics : struct, IStateMetricsPolicy + where TTrust : struct, IInputTrustPolicy + { + var trackSourceRanges = _startTagSourceRangeSink is not null; + var sourceBase = trackSourceRanges ? _normalizedBytesConsumed : 0; + var index = 0; + + try + { + while (index < utf8.Length) + { + if (!_pendingCarriageReturn) + { + if (IsTagTailState(_state) && (_isEndTag || _startTagEmitted)) + { + var sourceOffset = trackSourceRanges ? sourceBase + index : 0; + var consumed = + !_isEndTag && _captureStartTagAttributes + ? ScanTagTail( + utf8[index..], + sourceOffset, + trackSourceRanges, + yieldOnRequest + ) + : ScanTagTail( + utf8[index..], + sourceOffset, + trackSourceRanges, + yieldOnRequest + ); + + if (consumed > 0) + { + index += consumed; + if (yieldOnRequest && _yieldRequested) + { + return index; + } + + continue; + } + } + else if (_state == State.TagName) + { + var remaining = utf8.Slice(index); + var stop = IndexOfTagNameStop(remaining); + var run = stop < 0 ? remaining.Length : stop; + + if (run > 0) + { + // _state is TagName by the test above; naming the constant lets it fold + // instead of reloading the field in the hot path. + RecordState((Int32)State.TagName, run); + AppendTagName(remaining[..run]); + index += run; + if (stop < 0) + { + // The name continues past this span; nothing to fuse. + continue; + } + } + + var stopByte = remaining[run]; + // Tag-name stop fusion: the byte that ended the name is in-span and its whole + // effect here is one of three transitions, so take them instead of a per-byte + // round-trip. Not the full terminator set - '\0' becomes a replacement + // character, '\r' starts CR normalization, non-ASCII must bounce to the caller. + if (stopByte is (Byte)'\t' or (Byte)'\n' or (Byte)'\f' or (Byte)' ' or (Byte)'/' or (Byte)'>') + { + index++; + RecordFusedTagNameStopIf(); + if (trackSourceRanges) + { + _currentSourceOffset = sourceBase + index; + } + + if (stopByte == (Byte)'>') + { + FinishTag(selfClosing: false); + } + else + { + // Mirrors the ProcessTagState TagName arm: the start tag is emitted + // before the state changes, and end tags no-op inside EmitTagStart. + EmitTagStart(); + _state = stopByte == (Byte)'/' ? State.SelfClosingStartTag : State.BeforeAttributeName; + } + + if (yieldOnRequest && _yieldRequested) + { + return index; + } + + continue; + } + } + else if (_state == State.Data && _textUtf8CarryLength == 0) + { + var remaining = utf8.Slice(index); + Int32 run; + + if (remaining[0] == (Byte)'<') + { + run = 0; + } + else if (!_captureText) + { + run = remaining.IndexOf((Byte)'<'); + } + else + { + run = IndexOfCaptureStop(remaining, DataTextTerminators, DataTextArbitraryAllowed); + } + + if (run < 0) + { + run = remaining.Length; + } + + if (run > 0) + { + RecordState((Int32)_state, run); + if (_captureText) + { + EmitText(utf8.Slice(index, run)); + + if (RawTextEnabled) + { + EmitRawText(sourceBase + index, utf8.Slice(index, run), CurrentRawTextType()); + } + + if (yieldOnRequest && _yieldRequested) + { + index += run; + return index; + } + } + + index += run; + continue; + } + + // Stop-byte fusion: a '<' followed by an ASCII letter (or "/" + letter) in + // data state always begins a tag, so consume through the first name byte here + // instead of surrendering '<', the follower, and the letter to three per-byte + // dispatcher round-trips. Only data state qualifies: raw text, RCDATA, and + // script data route '<' through the end-tag candidate machinery below. All + // fused bytes are ASCII by test, so the trust policy is satisfied. + if (remaining[0] == (Byte)'<' && remaining.Length >= 2) + { + Int32 fused; + Boolean isEndTag; + if (IsAsciiLetter(remaining[1])) + { + fused = 2; + isEndTag = false; + } + else if (remaining[1] == (Byte)'/' && remaining.Length >= 3 && IsAsciiLetter(remaining[2])) + { + fused = 3; + isEndTag = true; + } + else + { + goto PerByteStateMachine; + } + + if (TMetrics.Enabled) + { + RecordFusedTagOpen(isEndTag); + } + + index += fused; + if (trackSourceRanges) + { + _currentSourceOffset = sourceBase + index; + _lastLessThanSourceOffset = sourceBase + index - fused; + } + + BeginTag(isEndTag, remaining[fused - 1]); + continue; + } + } + else if (_state is State.RawText or State.ScriptData && _textUtf8CarryLength == 0) + { + var consumed = _captureText + ? ScanRawTextContent( + utf8[index..], + sourceBase + index, + trackSourceRanges, + yieldOnRequest + ) + : ScanRawTextContent( + utf8[index..], + sourceBase + index, + trackSourceRanges, + yieldOnRequest + ); + if (consumed > 0) + { + index += consumed; + if (yieldOnRequest && _yieldRequested) + { + return index; + } + + continue; + } + } + else if (_state == State.Comment) + { + var consumed = ScanCommentContent(utf8[index..]); + if (consumed > 0) + { + index += consumed; + continue; + } + } + else if (_state == State.Plaintext && _textUtf8CarryLength == 0) + { + var consumed = ScanPlaintextContent(utf8[index..], sourceBase + index); + if (consumed > 0) + { + index += consumed; + if (yieldOnRequest && _yieldRequested) + { + return index; + } + + continue; + } + } + } + + PerByteStateMachine: + var value = utf8[index]; + if (TTrust.StopAtNonAscii && value >= 0x80) + { + // Unvalidated non-ASCII: hand back to the caller, which validates the run and + // re-feeds it through the trusted entry point. + return index; + } + + index++; + if (trackSourceRanges) + { + _currentSourceOffset = sourceBase + index; + if (value == (Byte)'<') + { + _lastLessThanSourceOffset = _currentSourceOffset - 1; + } + } + + if (_pendingCarriageReturn) + { + _pendingCarriageReturn = false; + if (value == (Byte)'\n') + { + if (RawTextEnabled && IsRawTextInputState(_state)) + EmitRawCurrentByte(value, CurrentRawTextType()); + continue; + } + } + + if (value == (Byte)'\r') + { + _pendingCarriageReturn = true; + value = (Byte)'\n'; + } + + if (IsScriptState(_state)) + { + ProcessScriptInput(value, utf8, ref index); + } + else + { + Process(value); + } + + if (yieldOnRequest && _yieldRequested) + { + return index; + } + } + + return index; + } + finally + { + if (trackSourceRanges) + { + _normalizedBytesConsumed = sourceBase + index; + // Only the consumed slice is reported, so partial consumption (yield, or the fused + // ASCII path handing back at a non-ASCII byte) never double-observes the tail. + _startTagSourceRangeSink!.ObserveNormalizedUtf8End(sourceBase, utf8[..index], RewritePublishableOffset); + } + } + } + + public void Complete() + { + if (_completed) + { + return; + } + + Utf8AttributeNameIndex.Reset(ref _seenAttributeIndex); + switch (_state) + { + case State.TagOpen: + EmitText("<"u8); + EmitRawText(_normalizedBytesConsumed - 1, "<"u8, Utf8HtmlTextType.Data); + break; + case State.EndTagOpen: + EmitText("