diff --git a/lexer.go b/lexer.go index 104f3b1..db3ab4e 100644 --- a/lexer.go +++ b/lexer.go @@ -6,6 +6,7 @@ package toml import ( + "bytes" "errors" "fmt" "io" @@ -24,7 +25,7 @@ type tomlLexStateFn func() tomlLexStateFn // Define lexer type tomlLexer struct { input *buffruneio.Reader // Textual source - buffer []rune // Runes composing the current token + buffer bytes.Buffer // Runes composing the current token tokens chan token depth int line int @@ -53,13 +54,13 @@ func (l *tomlLexer) next() rune { r := l.read() if r != eof { - l.buffer = append(l.buffer, r) + l.buffer.WriteRune(r) } return r } func (l *tomlLexer) ignore() { - l.buffer = make([]rune, 0) + l.buffer.Reset() l.line = l.endbufferLine l.col = l.endbufferCol } @@ -85,7 +86,7 @@ func (l *tomlLexer) emitWithValue(t tokenType, value string) { } func (l *tomlLexer) emit(t tokenType) { - l.emitWithValue(t, string(l.buffer)) + l.emitWithValue(t, l.buffer.String()) } func (l *tomlLexer) peek() rune { @@ -536,7 +537,7 @@ func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.buffer.Len() > 0 { l.emit(tokenKeyGroupArray) } l.next() @@ -559,7 +560,7 @@ func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.buffer.Len() > 0 { l.emit(tokenKeyGroup) } l.next() diff --git a/lexer_test.go b/lexer_test.go index 6b324ea..dce7a63 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -1,6 +1,7 @@ package toml import ( + "os" "strings" "testing" ) @@ -748,3 +749,31 @@ func TestLexUnknownRvalue(t *testing.T) { {Position{1, 5}, tokenError, `no value can start with \`}, }) } + +func BenchmarkLexer(b *testing.B) { + sample := `title = "Hugo: A Fast and Flexible Website Generator" +baseurl = "http://gohugo.io/" +MetaDataFormat = "yaml" +pluralizeListTitles = false + +[params] + description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go" + author = "Steve Francia (spf13) and friends" + release = "0.22-DEV" + +[[menu.main]] + name = "Download Hugo" + pre = "" + url = "https://github.com/spf13/hugo/releases" + weight = -200 +` + rd := strings.NewReader(sample) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + rd.Seek(0, os.SEEK_SET) + ch := lexToml(rd) + for _ = range ch { + } + } +}