diff --git a/stream.go b/stream.go index 3b8d19f..b279282 100644 --- a/stream.go +++ b/stream.go @@ -99,17 +99,22 @@ func (s *Stream) GetParsedLength() int { } } +// ensureParseNext lazily parses the next chunk of data when the token list ends mid-stream. +func (s *Stream) ensureParseNext() { + if s.current.next == nil && s.current != undefToken && s.p != nil { + n := s.p.n + s.p.parse() + s.len += s.p.n - n + } +} + // GoNext moves the stream pointer to the next token. // If there is no token, it initiates the parsing of the next chunk of data. // If there is no data, the pointer will point to the TokenUndef token. func (s *Stream) GoNext() *Stream { + s.ensureParseNext() if s.current.next != nil { s.current = s.current.next - if s.current.next == nil && s.p != nil { // lazy load and parse next data-chunk - n := s.p.n - s.p.parse() - s.len += s.p.n - n - } if s.historySize != 0 && s.current.id-s.head.id > s.historySize { t := s.head s.head = s.head.unlink() @@ -262,6 +267,7 @@ func (s *Stream) PrevToken() *Token { // If next token doesn't exist, the method returns TypeUndef token. // Do not save a result (Token) into variables — the next token may be changed at any time. func (s *Stream) NextToken() *Token { + s.ensureParseNext() if s.current.next != nil { return s.current.next } diff --git a/stream_test.go b/stream_test.go index 80d7582..2ef3b32 100644 --- a/stream_test.go +++ b/stream_test.go @@ -309,6 +309,46 @@ func TestIssue30(t *testing.T) { } } +// TestIssue44 The parser stops parsing the stream after the first token if +// its size greater or equal to the size of the parser's buffer. +func TestIssue44(t *testing.T) { + parser := New() + parser.AllowKeywordSymbols(Underscore, Numbers) + + // stream with first token close to the size of the parser's internal buffer + buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67") + stream := parser.ParseStream(buf, 40) + defer stream.Close() + + require.True(t, stream.IsValid()) + require.True(t, stream.CurrentToken().IsKeyword()) + require.Equal(t, "this_token_is_exactly_40_characters_long", stream.CurrentToken().ValueString()) + + stream.GoNext() + require.True(t, stream.IsValid(), "stream should still be valid after first token") + require.True(t, stream.CurrentToken().IsInteger()) + require.Equal(t, int64(67), stream.CurrentToken().ValueInt64()) +} + +// TestIssue44NextToken NextToken should be able to peek the next token even +// when the first token's size is grreater or equal to the size of the parser's +// buffer. +func TestIssue44NextToken(t *testing.T) { + parser := New() + parser.AllowKeywordSymbols(Underscore, Numbers) + + // stream with first token close to the size of the parser's internal buffer + buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67") + stream := parser.ParseStream(buf, 40) + defer stream.Close() + + require.True(t, stream.IsValid()) + next := stream.NextToken() + require.NotEqual(t, undefToken, next, "NextToken should not return undefToken") + require.True(t, next.IsInteger()) + require.Equal(t, int64(67), next.ValueInt64()) +} + func TestStreamOverflow(t *testing.T) { parser := New() buf := bytes.NewBuffer([]byte("a b c"))