Lex performance improvement (#176)

* Use []token instead of chan token

name             old time/op    new time/op    delta
ParseToml-8        1.18ms ± 0%    0.91ms ± 0%  -22.98%
UnmarshalToml-8    1.29ms ± 0%    0.95ms ± 0%  -25.96%

name             old alloc/op   new alloc/op   delta
ParseToml-8         429kB ± 0%     444kB ± 0%   +3.49%
UnmarshalToml-8     451kB ± 0%     466kB ± 0%   +3.32%

name             old allocs/op  new allocs/op  delta
ParseToml-8         14.1k ± 0%     13.7k ± 0%   -2.31%
UnmarshalToml-8     15.1k ± 0%     14.7k ± 0%   -2.16%

* Lex on []byte instead of io.Reader

name             old time/op    new time/op    delta
ParseToml-8        1.18ms ± 0%    0.29ms ± 0%  -75.18%
UnmarshalToml-8    1.27ms ± 0%    0.38ms ± 0%  -70.38%

name             old alloc/op   new alloc/op   delta
ParseToml-8         429kB ± 0%     135kB ± 0%  -68.53%
UnmarshalToml-8     451kB ± 0%     157kB ± 0%  -65.22%

name             old allocs/op  new allocs/op  delta
ParseToml-8         14.1k ± 0%      3.2k ± 0%  -77.20%
UnmarshalToml-8     15.1k ± 0%      4.2k ± 0%  -72.00%

lexer_test.go

@@ -1,38 +1,14 @@
 package toml
 
 import (
-	"os"
-	"strings"
+	"reflect"
 	"testing"
 )
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	ch := lexToml(strings.NewReader(input))
-	for _, expected := range expectedFlow {
-		token := <-ch
-		if token != expected {
-			t.Log("While testing: ", input)
-			t.Log("compared (got)", token, "to (expected)", expected)
-			t.Log("\tvalue:", token.val, "<->", expected.val)
-			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
-			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
-			t.Log("\tline:", token.Line, "<->", expected.Line)
-			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
-			t.Log("compared", token, "to", expected)
-			t.FailNow()
-		}
-	}
-
-	tok, ok := <-ch
-	if ok {
-		t.Log("channel is not closed!")
-		t.Log(len(ch)+1, "tokens remaining:")
-
-		t.Log("token ->", tok)
-		for token := range ch {
-			t.Log("token ->", token)
-		}
-		t.FailNow()
+	tokens := lexToml([]byte(input))
+	if !reflect.DeepEqual(tokens, expectedFlow) {
+		t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens)
 	}
 }
 
@@ -767,13 +743,8 @@ pluralizeListTitles = false
 	url = "https://github.com/spf13/hugo/releases"
 	weight = -200
 `
-	rd := strings.NewReader(sample)
-
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		rd.Seek(0, os.SEEK_SET)
-		ch := lexToml(rd)
-		for _ = range ch {
-		}
+		lexToml([]byte(sample))
 	}
 }