diff --git a/parser.go b/parser.go index 6db0e8b..56efab8 100644 --- a/parser.go +++ b/parser.go @@ -297,7 +297,7 @@ func (p *parsing) parseNumber() bool { if start == -1 { // floats can be started from a pointer start = p.pos } - } else if !(nextByte == 'e' || nextByte == 'E' || nextByte == 0) { + } else if !((nextByte == 'e' || nextByte == 'E') && hasNumber) && nextByte != 0 { break } floatTraitPos = p.pos diff --git a/testdata/fuzz/FuzzStream/0989a2a2ef3fb8ff b/testdata/fuzz/FuzzStream/0989a2a2ef3fb8ff new file mode 100644 index 0000000..de70513 --- /dev/null +++ b/testdata/fuzz/FuzzStream/0989a2a2ef3fb8ff @@ -0,0 +1,2 @@ +go test fuzz v1 +string(".E0") diff --git a/tokenizer_test.go b/tokenizer_test.go index 196b630..e302b16 100644 --- a/tokenizer_test.go +++ b/tokenizer_test.go @@ -158,6 +158,11 @@ func TestTokenizeEdgeCases(t *testing.T) { }}, {"\x00", []Token{ // https://github.com/bzick/tokenizer/issues/28 }}, + {".E0", []Token{ // https://github.com/bzick/tokenizer/issues/42 + {key: TokenUnknown, value: s2b("."), offset: 0, line: 1, id: 0}, + {key: TokenKeyword, value: s2b("E"), offset: 1, line: 1, id: 1}, + {key: TokenInteger, value: s2b("0"), offset: 2, line: 1, id: 2}, + }}, } for _, v := range data1 { t.Run(v.str, func(t *testing.T) { @@ -389,14 +394,14 @@ func FuzzStream(f *testing.F) { stream := tokenizer.ParseStream(buffer, 100) var actual []byte + var tokens []*Token for stream.IsValid() { current := stream.CurrentToken() - // t.Logf("%#v", current) + tokens = append(tokens, current) actual = append(actual, current.Indent()...) actual = append(actual, current.Value()...) stream.GoNext() } - // t.Logf("%#v", stream.CurrentToken()) // As we only concatenate the indents of each token, the trailing // whitespaces and token separators are lost, so we trim these @@ -405,6 +410,7 @@ func FuzzStream(f *testing.F) { expected := bytes.TrimRight(origBytes, trimset) actual = bytes.TrimRight(actual, trimset) if !bytes.Equal(expected, actual) { + t.Log(tokens) t.Errorf("input:\n%q\nexpected:\n%q\nactual:\n%q", orig, expected, actual) } })