From e872682c7887a7442c78554abdbfb4176867a008 Mon Sep 17 00:00:00 2001
From: Allen <leloucharcher@163.com>
Date: Wed, 15 Apr 2020 20:46:56 +0800
Subject: [PATCH] Fix unmarshal error with tab in multi-line basic string
 (#355)

Fixes #354
---
 lexer.go        | 11 +++++++----
 lexer_test.go   |  7 +++++++
 marshal_test.go | 23 +++++++++++++++++++++++
 3 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/lexer.go b/lexer.go
index 88fd91e..54b0969 100644
--- a/lexer.go
+++ b/lexer.go
@@ -313,7 +313,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
 			l.next()
-			str, err := l.lexStringAsString(`"`, false, true)
+			str, err := l.lexStringAsString(`"`, false, true, false)
 			if err != nil {
 				return l.errorf(err.Error())
 			}
@@ -419,7 +419,7 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
 // Lex a string and return the results as a string.
 // Terminator is the substring indicating the end of the token.
 // The resulting string does not include the terminator.
-func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool, acceptTab bool) (string, error) {
 	growingString := ""
 
 	if discardLeadingNewLine {
@@ -512,7 +512,8 @@ func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine,
 		} else {
 			r := l.peek()
 
-			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) {
+			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) &&
+				!(acceptTab && r == '\t') {
 				return "", fmt.Errorf("unescaped control character %U", r)
 			}
 			l.next()
@@ -534,15 +535,17 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 	terminator := `"`
 	discardLeadingNewLine := false
 	acceptNewLines := false
+	acceptTab := false
 	if l.follow(`""`) {
 		l.skip()
 		l.skip()
 		terminator = `"""`
 		discardLeadingNewLine = true
 		acceptNewLines = true
+		acceptTab = true
 	}
 
-	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines, acceptTab)
 
 	if err != nil {
 		return l.errorf(err.Error())
diff --git a/lexer_test.go b/lexer_test.go
index 6a33f57..2d06f4e 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -654,6 +654,13 @@ func TestMultilineString(t *testing.T) {
 		{Position{6, 9}, tokenEOF, ""},
 	})
 
+	testFlow(t, `foo = """hello	world"""`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 10}, tokenString, "hello\tworld"},
+		{Position{1, 24}, tokenEOF, ""},
+	})
+
 	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
 		{Position{1, 1}, tokenKey, "key2"},
 		{Position{1, 6}, tokenEqual, "="},
diff --git a/marshal_test.go b/marshal_test.go
index 33ab64a..7f4fdc6 100644
--- a/marshal_test.go
+++ b/marshal_test.go
@@ -1419,6 +1419,29 @@ func TestMarshalDirectMultilineString(t *testing.T) {
 	}
 }
 
+//issue 354
+func TestUnmarshalMultilineStringWithTab(t *testing.T) {
+	input := []byte(`
+Field = """
+hello	world"""
+`)
+
+	type Test struct {
+		Field string
+	}
+
+	expected := Test{"hello\tworld"}
+	result := Test{}
+	err := Unmarshal(input, &result)
+	if err != nil {
+		t.Fatal("unmarshal should not error:", err)
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unmarshal: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", expected, result)
+	}
+}
+
 var customMultilineTagTestToml = []byte(`int_slice = [
   1,
   2,