Fix inline table loading errors (#381)

Fixes #379 #380

lexer.go

@@ -26,7 +26,7 @@ type tomlLexer struct {
 	currentTokenStart int
 	currentTokenStop  int
 	tokens            []token
-	depth             int
+	brackets          []rune
 	line              int
 	col               int
 	endbufferLine     int
@@ -123,6 +123,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
+		case '}': // after '{'
+			return l.lexRightCurlyBrace
 		case '[':
 			return l.lexTableKey
 		case '#':
@@ -140,10 +142,6 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			l.skip()
 		}
 
-		if l.depth > 0 {
-			return l.lexRvalue
-		}
-
 		if isKeyStartChar(next) {
 			return l.lexKey
 		}
@@ -167,10 +165,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 		case '=':
 			return l.lexEqual
 		case '[':
-			l.depth++
 			return l.lexLeftBracket
 		case ']':
-			l.depth--
 			return l.lexRightBracket
 		case '{':
 			return l.lexLeftCurlyBrace
@@ -188,12 +184,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			fallthrough
 		case '\n':
 			l.skip()
-			if l.depth == 0 {
+			if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '[' {
-				return l.lexVoid
+				return l.lexRvalue
 			}
-			return l.lexRvalue
+			return l.lexVoid
-		case '_':
-			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
@@ -236,10 +230,6 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexNumber
 		}
 
-		if isAlphanumeric(next) {
-			return l.lexKey
-		}
-
 		return l.errorf("no value can start with %c", next)
 	}
 
@@ -250,12 +240,17 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
 	l.next()
 	l.emit(tokenLeftCurlyBrace)
+	l.brackets = append(l.brackets, '{')
 	return l.lexVoid
 }
 
 func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
 	l.next()
 	l.emit(tokenRightCurlyBrace)
+	if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '{' {
+		return l.errorf("cannot have '}' here")
+	}
+	l.brackets = l.brackets[:len(l.brackets)-1]
 	return l.lexRvalue
 }
 
@@ -302,6 +297,9 @@ func (l *tomlLexer) lexEqual() tomlLexStateFn {
 func (l *tomlLexer) lexComma() tomlLexStateFn {
 	l.next()
 	l.emit(tokenComma)
+	if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '{' {
+		return l.lexVoid
+	}
 	return l.lexRvalue
 }
 
@@ -361,6 +359,7 @@ func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn {
 func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
 	l.next()
 	l.emit(tokenLeftBracket)
+	l.brackets = append(l.brackets, '[')
 	return l.lexRvalue
 }
 
@@ -543,7 +542,6 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 	}
 
 	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
-
 	if err != nil {
 		return l.errorf(err.Error())
 	}
@@ -615,6 +613,10 @@ func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn {
 func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
 	l.next()
 	l.emit(tokenRightBracket)
+	if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '[' {
+		return l.errorf("cannot have ']' here")
+	}
+	l.brackets = l.brackets[:len(l.brackets)-1]
 	return l.lexRvalue
 }
 

lexer_test.go

@@ -698,6 +698,7 @@ func TestUnicodeString(t *testing.T) {
 		{Position{1, 22}, tokenEOF, ""},
 	})
 }
+
 func TestEscapeInString(t *testing.T) {
 	testFlow(t, `foo = "\b\f\/"`, []token{
 		{Position{1, 1}, tokenKey, "foo"},
@@ -772,6 +773,16 @@ func TestLexUnknownRvalue(t *testing.T) {
 	})
 }
 
+func TestLexInlineTableEmpty(t *testing.T) {
+	testFlow(t, `foo = {}`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 8}, tokenRightCurlyBrace, "}"},
+		{Position{1, 9}, tokenEOF, ""},
+	})
+}
+
 func TestLexInlineTableBareKey(t *testing.T) {
 	testFlow(t, `foo = { bar = "baz" }`, []token{
 		{Position{1, 1}, tokenKey, "foo"},
@@ -798,6 +809,116 @@ func TestLexInlineTableBareKeyDash(t *testing.T) {
 	})
 }
 
+func TestLexInlineTableBareKeyInArray(t *testing.T) {
+	testFlow(t, `foo = [{ -bar_ = "baz" }]`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 8}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 10}, tokenKey, "-bar_"},
+		{Position{1, 16}, tokenEqual, "="},
+		{Position{1, 19}, tokenString, "baz"},
+		{Position{1, 24}, tokenRightCurlyBrace, "}"},
+		{Position{1, 25}, tokenRightBracket, "]"},
+		{Position{1, 26}, tokenEOF, ""},
+	})
+}
+
+func TestLexInlineTableError1(t *testing.T) {
+	testFlow(t, `foo = { 123 = 0 ]`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "123"},
+		{Position{1, 13}, tokenEqual, "="},
+		{Position{1, 15}, tokenInteger, "0"},
+		{Position{1, 17}, tokenRightBracket, "]"},
+		{Position{1, 18}, tokenError, "cannot have ']' here"},
+	})
+}
+
+func TestLexInlineTableError2(t *testing.T) {
+	testFlow(t, `foo = { 123 = 0 }}`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "123"},
+		{Position{1, 13}, tokenEqual, "="},
+		{Position{1, 15}, tokenInteger, "0"},
+		{Position{1, 17}, tokenRightCurlyBrace, "}"},
+		{Position{1, 18}, tokenRightCurlyBrace, "}"},
+		{Position{1, 19}, tokenError, "cannot have '}' here"},
+	})
+}
+
+func TestLexInlineTableDottedKey1(t *testing.T) {
+	testFlow(t, `foo = { a = 0, 123.45abc = 0 }`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "a"},
+		{Position{1, 11}, tokenEqual, "="},
+		{Position{1, 13}, tokenInteger, "0"},
+		{Position{1, 14}, tokenComma, ","},
+		{Position{1, 16}, tokenKey, "123.45abc"},
+		{Position{1, 26}, tokenEqual, "="},
+		{Position{1, 28}, tokenInteger, "0"},
+		{Position{1, 30}, tokenRightCurlyBrace, "}"},
+		{Position{1, 31}, tokenEOF, ""},
+	})
+}
+
+func TestLexInlineTableDottedKey2(t *testing.T) {
+	testFlow(t, `foo = { a = 0, '123'.'45abc' = 0 }`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "a"},
+		{Position{1, 11}, tokenEqual, "="},
+		{Position{1, 13}, tokenInteger, "0"},
+		{Position{1, 14}, tokenComma, ","},
+		{Position{1, 16}, tokenKey, "'123'.'45abc'"},
+		{Position{1, 30}, tokenEqual, "="},
+		{Position{1, 32}, tokenInteger, "0"},
+		{Position{1, 34}, tokenRightCurlyBrace, "}"},
+		{Position{1, 35}, tokenEOF, ""},
+	})
+}
+
+func TestLexInlineTableDottedKey3(t *testing.T) {
+	testFlow(t, `foo = { a = 0, "123"."45ʎǝʞ" = 0 }`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "a"},
+		{Position{1, 11}, tokenEqual, "="},
+		{Position{1, 13}, tokenInteger, "0"},
+		{Position{1, 14}, tokenComma, ","},
+		{Position{1, 16}, tokenKey, `"123"."45ʎǝʞ"`},
+		{Position{1, 30}, tokenEqual, "="},
+		{Position{1, 32}, tokenInteger, "0"},
+		{Position{1, 34}, tokenRightCurlyBrace, "}"},
+		{Position{1, 35}, tokenEOF, ""},
+	})
+}
+
+func TestLexInlineTableBareKeyWithComma(t *testing.T) {
+	testFlow(t, `foo = { -bar1 = "baz", -bar_ = "baz" }`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
+		{Position{1, 9}, tokenKey, "-bar1"},
+		{Position{1, 15}, tokenEqual, "="},
+		{Position{1, 18}, tokenString, "baz"},
+		{Position{1, 22}, tokenComma, ","},
+		{Position{1, 24}, tokenKey, "-bar_"},
+		{Position{1, 30}, tokenEqual, "="},
+		{Position{1, 33}, tokenString, "baz"},
+		{Position{1, 38}, tokenRightCurlyBrace, "}"},
+		{Position{1, 39}, tokenEOF, ""},
+	})
+}
+
 func TestLexInlineTableBareKeyUnderscore(t *testing.T) {
 	testFlow(t, `foo = { _bar = "baz" }`, []token{
 		{Position{1, 1}, tokenKey, "foo"},

parser_test.go

@@ -239,7 +239,8 @@ func TestLocalDateTime(t *testing.T) {
 				Minute:     32,
 				Second:     0,
 				Nanosecond: 0,
-			}},
+			},
+		},
 	})
 }
 
@@ -257,7 +258,8 @@ func TestLocalDateTimeNano(t *testing.T) {
 				Minute:     32,
 				Second:     0,
 				Nanosecond: 999999000,
-			}},
+			},
+		},
 	})
 }
 
@@ -677,7 +679,7 @@ func TestInlineTableUnterminated(t *testing.T) {
 
 func TestInlineTableCommaExpected(t *testing.T) {
 	_, err := Load("foo = {hello = 53 test = foo}")
-	if err.Error() != "(1, 19): comma expected between fields in inline table" {
+	if err.Error() != "(1, 19): unexpected token type in inline table: no value can start with t" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -691,7 +693,7 @@ func TestInlineTableCommaStart(t *testing.T) {
 
 func TestInlineTableDoubleComma(t *testing.T) {
 	_, err := Load("foo = {hello = 53,, foo = 17}")
-	if err.Error() != "(1, 19): need field between two commas in inline table" {
+	if err.Error() != "(1, 19): unexpected token type in inline table: keys cannot contain , character" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -928,10 +930,8 @@ func TestTomlValueStringRepresentation(t *testing.T) {
 		{"hello world", "\"hello world\""},
 		{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
 		{"\x05", "\"\\u0005\""},
-		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), "1979-05-27T07:32:00Z"},
-			"1979-05-27T07:32:00Z"},
+		{[]interface{}{"gamma", "delta"}, "[\"gamma\",\"delta\"]"},
-		{[]interface{}{"gamma", "delta"},
-			"[\"gamma\",\"delta\"]"},
 		{nil, ""},
 	} {
 		result, err := tomlValueStringRepresentation(item.Value, "", "", false)
@@ -1061,7 +1061,7 @@ func TestInvalidFloatParsing(t *testing.T) {
 	}
 
 	_, err = Load("a=_1_2")
-	if err.Error() != "(1, 3): cannot start number with underscore" {
+	if err.Error() != "(1, 3): no value can start with _" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -1125,11 +1125,10 @@ The quick brown \
     the lazy dog."""
 
 str3 = """\
-       The quick brown \ 
+       The quick brown \
-       fox jumps over \ 
+       fox jumps over \
-       the lazy dog.\  
+       the lazy dog.\
-       """`)
+	   """`)
-
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}