Fix unmarshal error with tab in multi-line basic string (#355)

Fixes #354

lexer.go

@@ -313,7 +313,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
 			l.next()
-			str, err := l.lexStringAsString(`"`, false, true)
+			str, err := l.lexStringAsString(`"`, false, true, false)
 			if err != nil {
 				return l.errorf(err.Error())
 			}
@@ -419,7 +419,7 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
 // Lex a string and return the results as a string.
 // Terminator is the substring indicating the end of the token.
 // The resulting string does not include the terminator.
-func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool, acceptTab bool) (string, error) {
 	growingString := ""
 
 	if discardLeadingNewLine {
@@ -512,7 +512,8 @@ func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine,
 		} else {
 			r := l.peek()
 
-			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) {
+			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) &&
+				!(acceptTab && r == '\t') {
 				return "", fmt.Errorf("unescaped control character %U", r)
 			}
 			l.next()
@@ -534,15 +535,17 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 	terminator := `"`
 	discardLeadingNewLine := false
 	acceptNewLines := false
+	acceptTab := false
 	if l.follow(`""`) {
 		l.skip()
 		l.skip()
 		terminator = `"""`
 		discardLeadingNewLine = true
 		acceptNewLines = true
+		acceptTab = true
 	}
 
-	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines, acceptTab)
 
 	if err != nil {
 		return l.errorf(err.Error())

lexer_test.go

@@ -654,6 +654,13 @@ func TestMultilineString(t *testing.T) {
 		{Position{6, 9}, tokenEOF, ""},
 	})
 
+	testFlow(t, `foo = """hello	world"""`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 10}, tokenString, "hello\tworld"},
+		{Position{1, 24}, tokenEOF, ""},
+	})
+
 	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
 		{Position{1, 1}, tokenKey, "key2"},
 		{Position{1, 6}, tokenEqual, "="},

marshal_test.go

@@ -1419,6 +1419,29 @@ func TestMarshalDirectMultilineString(t *testing.T) {
 	}
 }
 
+//issue 354
+func TestUnmarshalMultilineStringWithTab(t *testing.T) {
+	input := []byte(`
+Field = """
+hello	world"""
+`)
+
+	type Test struct {
+		Field string
+	}
+
+	expected := Test{"hello\tworld"}
+	result := Test{}
+	err := Unmarshal(input, &result)
+	if err != nil {
+		t.Fatal("unmarshal should not error:", err)
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unmarshal: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", expected, result)
+	}
+}
+
 var customMultilineTagTestToml = []byte(`int_slice = [
   1,
   2,