Fix parsing stream with first token size >= parser's chunk size

stream.go

@@ -99,17 +99,22 @@ func (s *Stream) GetParsedLength() int {
 	}
 }
 
+// ensureParseNext lazily parses the next chunk of data when the token list ends mid-stream.
+func (s *Stream) ensureParseNext() {
+	if s.current.next == nil && s.current != undefToken && s.p != nil {
+		n := s.p.n
+		s.p.parse()
+		s.len += s.p.n - n
+	}
+}
+
 // GoNext moves the stream pointer to the next token.
 // If there is no token, it initiates the parsing of the next chunk of data.
 // If there is no data, the pointer will point to the TokenUndef token.
 func (s *Stream) GoNext() *Stream {
+	s.ensureParseNext()
 	if s.current.next != nil {
 		s.current = s.current.next
-		if s.current.next == nil && s.p != nil { // lazy load and parse next data-chunk
-			n := s.p.n
-			s.p.parse()
-			s.len += s.p.n - n
-		}
 		if s.historySize != 0 && s.current.id-s.head.id > s.historySize {
 			t := s.head
 			s.head = s.head.unlink()
@@ -262,6 +267,7 @@ func (s *Stream) PrevToken() *Token {
 // If next token doesn't exist, the method returns TypeUndef token.
 // Do not save a result (Token) into variables — the next token may be changed at any time.
 func (s *Stream) NextToken() *Token {
+	s.ensureParseNext()
 	if s.current.next != nil {
 		return s.current.next
 	}

stream_test.go

@@ -309,6 +309,46 @@ func TestIssue30(t *testing.T) {
 	}
 }
 
+// TestIssue44 The parser stops parsing the stream after the first token if
+// its size greater or equal to the size of the parser's buffer.
+func TestIssue44(t *testing.T) {
+	parser := New()
+	parser.AllowKeywordSymbols(Underscore, Numbers)
+
+	// stream with first token close to the size of the parser's internal buffer
+	buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67")
+	stream := parser.ParseStream(buf, 40)
+	defer stream.Close()
+
+	require.True(t, stream.IsValid())
+	require.True(t, stream.CurrentToken().IsKeyword())
+	require.Equal(t, "this_token_is_exactly_40_characters_long", stream.CurrentToken().ValueString())
+
+	stream.GoNext()
+	require.True(t, stream.IsValid(), "stream should still be valid after first token")
+	require.True(t, stream.CurrentToken().IsInteger())
+	require.Equal(t, int64(67), stream.CurrentToken().ValueInt64())
+}
+
+// TestIssue44NextToken NextToken should be able to peek the next token even
+// when the first token's size is grreater or equal to the size of the parser's
+// buffer.
+func TestIssue44NextToken(t *testing.T) {
+	parser := New()
+	parser.AllowKeywordSymbols(Underscore, Numbers)
+
+	// stream with first token close to the size of the parser's internal buffer
+	buf := bytes.NewBufferString("this_token_is_exactly_40_characters_long 67")
+	stream := parser.ParseStream(buf, 40)
+	defer stream.Close()
+
+	require.True(t, stream.IsValid())
+	next := stream.NextToken()
+	require.NotEqual(t, undefToken, next, "NextToken should not return undefToken")
+	require.True(t, next.IsInteger())
+	require.Equal(t, int64(67), next.ValueInt64())
+}
+
 func TestStreamOverflow(t *testing.T) {
 	parser := New()
 	buf := bytes.NewBuffer([]byte("a b c"))