From 9d622f6cb14e096bad1159252439a058434fe0cc Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet Date: Tue, 7 Jul 2026 14:51:39 +0200 Subject: [PATCH 1/3] Failing test for issue #44 --- stream_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/stream_test.go b/stream_test.go index 80d7582..16a7625 100644 --- a/stream_test.go +++ b/stream_test.go @@ -309,6 +309,27 @@ func TestIssue30(t *testing.T) { } } +// TestIssue44 The parser stops parsing the stream after the first token if +// its size greater or equal to the size of the parser's buffer. +func TestIssue44(t *testing.T) { + parser := New() + parser.AllowKeywordSymbols(Underscore, Numbers) + + // stream with first token close to the size of the parser's internal buffer + buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67") + stream := parser.ParseStream(buf, 40) + defer stream.Close() + + require.True(t, stream.IsValid()) + require.True(t, stream.CurrentToken().IsKeyword()) + require.Equal(t, "this_token_is_exactly_40_characters_long", stream.CurrentToken().ValueString()) + + stream.GoNext() + require.True(t, stream.IsValid(), "stream should still be valid after first token") + require.True(t, stream.CurrentToken().IsInteger()) + require.Equal(t, int64(67), stream.CurrentToken().ValueInt64()) +} + func TestStreamOverflow(t *testing.T) { parser := New() buf := bytes.NewBuffer([]byte("a b c")) From 3a325fa903dd22204245d5eafebc7c151534bd37 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet Date: Tue, 7 Jul 2026 18:18:58 +0200 Subject: [PATCH 2/3] Fix parsing stream with first token >= to parser's buffer When (first token size) >= (parser's chunk size), the next chunks are loaded during the parsing of the first token. This triggers the end of the parsing loop, leaving the frist token as the only one parsed, and thus having its "next" field set to nil. This in turn, lead Stream.GoNext() to detect the end of the stream. To fix this, Stream.GoNext() now starts by checking the next field of the current token, parsing the current chunk if needed, before deciding if the end of the stream is reached. Fixes #44 --- stream.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stream.go b/stream.go index 3b8d19f..169d5ac 100644 --- a/stream.go +++ b/stream.go @@ -103,13 +103,14 @@ func (s *Stream) GetParsedLength() int { // If there is no token, it initiates the parsing of the next chunk of data. // If there is no data, the pointer will point to the TokenUndef token. func (s *Stream) GoNext() *Stream { + // The next token might not have been parsed yet, try to do it now. + if s.current.next == nil && s.p != nil { + n := s.p.n + s.p.parse() + s.len += s.p.n - n + } if s.current.next != nil { s.current = s.current.next - if s.current.next == nil && s.p != nil { // lazy load and parse next data-chunk - n := s.p.n - s.p.parse() - s.len += s.p.n - n - } if s.historySize != 0 && s.current.id-s.head.id > s.historySize { t := s.head s.head = s.head.unlink() From 2a49f870fce8f78d217c21cec2df1f2e1dc9df55 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet Date: Thu, 9 Jul 2026 14:23:01 +0200 Subject: [PATCH 3/3] Also fix NextToken() when first token size >= chunk size NextToken() also assumed that the next token was always parsed, which is not the case. Add a new ensureParseNext() function that can be used in both GoNext() and NextToken(), and that makes sure the next token is parsed if the end of the stream has not been reached yet. --- stream.go | 17 +++++++++++------ stream_test.go | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/stream.go b/stream.go index 169d5ac..b279282 100644 --- a/stream.go +++ b/stream.go @@ -99,16 +99,20 @@ func (s *Stream) GetParsedLength() int { } } -// GoNext moves the stream pointer to the next token. -// If there is no token, it initiates the parsing of the next chunk of data. -// If there is no data, the pointer will point to the TokenUndef token. -func (s *Stream) GoNext() *Stream { - // The next token might not have been parsed yet, try to do it now. - if s.current.next == nil && s.p != nil { +// ensureParseNext lazily parses the next chunk of data when the token list ends mid-stream. +func (s *Stream) ensureParseNext() { + if s.current.next == nil && s.current != undefToken && s.p != nil { n := s.p.n s.p.parse() s.len += s.p.n - n } +} + +// GoNext moves the stream pointer to the next token. +// If there is no token, it initiates the parsing of the next chunk of data. +// If there is no data, the pointer will point to the TokenUndef token. +func (s *Stream) GoNext() *Stream { + s.ensureParseNext() if s.current.next != nil { s.current = s.current.next if s.historySize != 0 && s.current.id-s.head.id > s.historySize { @@ -263,6 +267,7 @@ func (s *Stream) PrevToken() *Token { // If next token doesn't exist, the method returns TypeUndef token. // Do not save a result (Token) into variables — the next token may be changed at any time. func (s *Stream) NextToken() *Token { + s.ensureParseNext() if s.current.next != nil { return s.current.next } diff --git a/stream_test.go b/stream_test.go index 16a7625..2ef3b32 100644 --- a/stream_test.go +++ b/stream_test.go @@ -330,6 +330,25 @@ func TestIssue44(t *testing.T) { require.Equal(t, int64(67), stream.CurrentToken().ValueInt64()) } +// TestIssue44NextToken NextToken should be able to peek the next token even +// when the first token's size is grreater or equal to the size of the parser's +// buffer. +func TestIssue44NextToken(t *testing.T) { + parser := New() + parser.AllowKeywordSymbols(Underscore, Numbers) + + // stream with first token close to the size of the parser's internal buffer + buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67") + stream := parser.ParseStream(buf, 40) + defer stream.Close() + + require.True(t, stream.IsValid()) + next := stream.NextToken() + require.NotEqual(t, undefToken, next, "NextToken should not return undefToken") + require.True(t, next.IsInteger()) + require.Equal(t, int64(67), next.ValueInt64()) +} + func TestStreamOverflow(t *testing.T) { parser := New() buf := bytes.NewBuffer([]byte("a b c"))