Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ func (p *parsing) parseNumber() bool {
if start == -1 { // floats can be started from a pointer
start = p.pos
}
} else if !(nextByte == 'e' || nextByte == 'E' || nextByte == 0) {
} else if !((nextByte == 'e' || nextByte == 'E') && hasNumber) && nextByte != 0 {
break
}
floatTraitPos = p.pos
Expand Down
2 changes: 2 additions & 0 deletions testdata/fuzz/FuzzStream/0989a2a2ef3fb8ff
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
go test fuzz v1
string(".E0")
10 changes: 8 additions & 2 deletions tokenizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,11 @@ func TestTokenizeEdgeCases(t *testing.T) {
}},
{"\x00", []Token{ // https://github.com/bzick/tokenizer/issues/28
}},
{".E0", []Token{ // https://github.com/bzick/tokenizer/issues/42
{key: TokenUnknown, value: s2b("."), offset: 0, line: 1, id: 0},
{key: TokenKeyword, value: s2b("E"), offset: 1, line: 1, id: 1},
{key: TokenInteger, value: s2b("0"), offset: 2, line: 1, id: 2},
}},
}
for _, v := range data1 {
t.Run(v.str, func(t *testing.T) {
Expand Down Expand Up @@ -389,14 +394,14 @@ func FuzzStream(f *testing.F) {

stream := tokenizer.ParseStream(buffer, 100)
var actual []byte
var tokens []*Token
for stream.IsValid() {
current := stream.CurrentToken()
// t.Logf("%#v", current)
tokens = append(tokens, current)
actual = append(actual, current.Indent()...)
actual = append(actual, current.Value()...)
stream.GoNext()
}
// t.Logf("%#v", stream.CurrentToken())

// As we only concatenate the indents of each token, the trailing
// whitespaces and token separators are lost, so we trim these
Expand All @@ -405,6 +410,7 @@ func FuzzStream(f *testing.F) {
expected := bytes.TrimRight(origBytes, trimset)
actual = bytes.TrimRight(actual, trimset)
if !bytes.Equal(expected, actual) {
t.Log(tokens)
t.Errorf("input:\n%q\nexpected:\n%q\nactual:\n%q", orig, expected, actual)
}
})
Expand Down
Loading