diff --git a/lexer.go b/lexer.go index 0bde0a1..a9dd080 100644 --- a/lexer.go +++ b/lexer.go @@ -26,7 +26,7 @@ type tomlLexer struct { currentTokenStart int currentTokenStop int tokens []token - depth int + brackets []rune line int col int endbufferLine int @@ -123,6 +123,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn { for { next := l.peek() switch next { + case '}': // after '{' + return l.lexRightCurlyBrace case '[': return l.lexTableKey case '#': @@ -140,10 +142,6 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn { l.skip() } - if l.depth > 0 { - return l.lexRvalue - } - if isKeyStartChar(next) { return l.lexKey } @@ -167,10 +165,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { case '=': return l.lexEqual case '[': - l.depth++ return l.lexLeftBracket case ']': - l.depth-- return l.lexRightBracket case '{': return l.lexLeftCurlyBrace @@ -188,12 +184,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { fallthrough case '\n': l.skip() - if l.depth == 0 { - return l.lexVoid + if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '[' { + return l.lexRvalue } - return l.lexRvalue - case '_': - return l.errorf("cannot start number with underscore") + return l.lexVoid } if l.follow("true") { @@ -236,10 +230,6 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { return l.lexNumber } - if isAlphanumeric(next) { - return l.lexKey - } - return l.errorf("no value can start with %c", next) } @@ -250,12 +240,17 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn { l.next() l.emit(tokenLeftCurlyBrace) + l.brackets = append(l.brackets, '{') return l.lexVoid } func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn { l.next() l.emit(tokenRightCurlyBrace) + if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '{' { + return l.errorf("cannot have '}' here") + } + l.brackets = l.brackets[:len(l.brackets)-1] return l.lexRvalue } @@ -302,6 +297,9 @@ func (l *tomlLexer) lexEqual() tomlLexStateFn { func (l *tomlLexer) lexComma() tomlLexStateFn { l.next() l.emit(tokenComma) + if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '{' { + return l.lexVoid + } return l.lexRvalue } @@ -361,6 +359,7 @@ func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn { func (l *tomlLexer) lexLeftBracket() tomlLexStateFn { l.next() l.emit(tokenLeftBracket) + l.brackets = append(l.brackets, '[') return l.lexRvalue } @@ -543,7 +542,6 @@ func (l *tomlLexer) lexString() tomlLexStateFn { } str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines) - if err != nil { return l.errorf(err.Error()) } @@ -615,6 +613,10 @@ func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { func (l *tomlLexer) lexRightBracket() tomlLexStateFn { l.next() l.emit(tokenRightBracket) + if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '[' { + return l.errorf("cannot have ']' here") + } + l.brackets = l.brackets[:len(l.brackets)-1] return l.lexRvalue } diff --git a/lexer_test.go b/lexer_test.go index ff826e9..0608ba8 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -698,6 +698,7 @@ func TestUnicodeString(t *testing.T) { {Position{1, 22}, tokenEOF, ""}, }) } + func TestEscapeInString(t *testing.T) { testFlow(t, `foo = "\b\f\/"`, []token{ {Position{1, 1}, tokenKey, "foo"}, @@ -772,6 +773,16 @@ func TestLexUnknownRvalue(t *testing.T) { }) } +func TestLexInlineTableEmpty(t *testing.T) { + testFlow(t, `foo = {}`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 8}, tokenRightCurlyBrace, "}"}, + {Position{1, 9}, tokenEOF, ""}, + }) +} + func TestLexInlineTableBareKey(t *testing.T) { testFlow(t, `foo = { bar = "baz" }`, []token{ {Position{1, 1}, tokenKey, "foo"}, @@ -798,6 +809,116 @@ func TestLexInlineTableBareKeyDash(t *testing.T) { }) } +func TestLexInlineTableBareKeyInArray(t *testing.T) { + testFlow(t, `foo = [{ -bar_ = "baz" }]`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftBracket, "["}, + {Position{1, 8}, tokenLeftCurlyBrace, "{"}, + {Position{1, 10}, tokenKey, "-bar_"}, + {Position{1, 16}, tokenEqual, "="}, + {Position{1, 19}, tokenString, "baz"}, + {Position{1, 24}, tokenRightCurlyBrace, "}"}, + {Position{1, 25}, tokenRightBracket, "]"}, + {Position{1, 26}, tokenEOF, ""}, + }) +} + +func TestLexInlineTableError1(t *testing.T) { + testFlow(t, `foo = { 123 = 0 ]`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "123"}, + {Position{1, 13}, tokenEqual, "="}, + {Position{1, 15}, tokenInteger, "0"}, + {Position{1, 17}, tokenRightBracket, "]"}, + {Position{1, 18}, tokenError, "cannot have ']' here"}, + }) +} + +func TestLexInlineTableError2(t *testing.T) { + testFlow(t, `foo = { 123 = 0 }}`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "123"}, + {Position{1, 13}, tokenEqual, "="}, + {Position{1, 15}, tokenInteger, "0"}, + {Position{1, 17}, tokenRightCurlyBrace, "}"}, + {Position{1, 18}, tokenRightCurlyBrace, "}"}, + {Position{1, 19}, tokenError, "cannot have '}' here"}, + }) +} + +func TestLexInlineTableDottedKey1(t *testing.T) { + testFlow(t, `foo = { a = 0, 123.45abc = 0 }`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "a"}, + {Position{1, 11}, tokenEqual, "="}, + {Position{1, 13}, tokenInteger, "0"}, + {Position{1, 14}, tokenComma, ","}, + {Position{1, 16}, tokenKey, "123.45abc"}, + {Position{1, 26}, tokenEqual, "="}, + {Position{1, 28}, tokenInteger, "0"}, + {Position{1, 30}, tokenRightCurlyBrace, "}"}, + {Position{1, 31}, tokenEOF, ""}, + }) +} + +func TestLexInlineTableDottedKey2(t *testing.T) { + testFlow(t, `foo = { a = 0, '123'.'45abc' = 0 }`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "a"}, + {Position{1, 11}, tokenEqual, "="}, + {Position{1, 13}, tokenInteger, "0"}, + {Position{1, 14}, tokenComma, ","}, + {Position{1, 16}, tokenKey, "'123'.'45abc'"}, + {Position{1, 30}, tokenEqual, "="}, + {Position{1, 32}, tokenInteger, "0"}, + {Position{1, 34}, tokenRightCurlyBrace, "}"}, + {Position{1, 35}, tokenEOF, ""}, + }) +} + +func TestLexInlineTableDottedKey3(t *testing.T) { + testFlow(t, `foo = { a = 0, "123"."45ʎǝʞ" = 0 }`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "a"}, + {Position{1, 11}, tokenEqual, "="}, + {Position{1, 13}, tokenInteger, "0"}, + {Position{1, 14}, tokenComma, ","}, + {Position{1, 16}, tokenKey, `"123"."45ʎǝʞ"`}, + {Position{1, 30}, tokenEqual, "="}, + {Position{1, 32}, tokenInteger, "0"}, + {Position{1, 34}, tokenRightCurlyBrace, "}"}, + {Position{1, 35}, tokenEOF, ""}, + }) +} + +func TestLexInlineTableBareKeyWithComma(t *testing.T) { + testFlow(t, `foo = { -bar1 = "baz", -bar_ = "baz" }`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 7}, tokenLeftCurlyBrace, "{"}, + {Position{1, 9}, tokenKey, "-bar1"}, + {Position{1, 15}, tokenEqual, "="}, + {Position{1, 18}, tokenString, "baz"}, + {Position{1, 22}, tokenComma, ","}, + {Position{1, 24}, tokenKey, "-bar_"}, + {Position{1, 30}, tokenEqual, "="}, + {Position{1, 33}, tokenString, "baz"}, + {Position{1, 38}, tokenRightCurlyBrace, "}"}, + {Position{1, 39}, tokenEOF, ""}, + }) +} + func TestLexInlineTableBareKeyUnderscore(t *testing.T) { testFlow(t, `foo = { _bar = "baz" }`, []token{ {Position{1, 1}, tokenKey, "foo"}, diff --git a/parser_test.go b/parser_test.go index b33c776..5c81f2b 100644 --- a/parser_test.go +++ b/parser_test.go @@ -239,7 +239,8 @@ func TestLocalDateTime(t *testing.T) { Minute: 32, Second: 0, Nanosecond: 0, - }}, + }, + }, }) } @@ -257,7 +258,8 @@ func TestLocalDateTimeNano(t *testing.T) { Minute: 32, Second: 0, Nanosecond: 999999000, - }}, + }, + }, }) } @@ -677,7 +679,7 @@ func TestInlineTableUnterminated(t *testing.T) { func TestInlineTableCommaExpected(t *testing.T) { _, err := Load("foo = {hello = 53 test = foo}") - if err.Error() != "(1, 19): comma expected between fields in inline table" { + if err.Error() != "(1, 19): unexpected token type in inline table: no value can start with t" { t.Error("Bad error message:", err.Error()) } } @@ -691,7 +693,7 @@ func TestInlineTableCommaStart(t *testing.T) { func TestInlineTableDoubleComma(t *testing.T) { _, err := Load("foo = {hello = 53,, foo = 17}") - if err.Error() != "(1, 19): need field between two commas in inline table" { + if err.Error() != "(1, 19): unexpected token type in inline table: keys cannot contain , character" { t.Error("Bad error message:", err.Error()) } } @@ -928,10 +930,8 @@ func TestTomlValueStringRepresentation(t *testing.T) { {"hello world", "\"hello world\""}, {"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""}, {"\x05", "\"\\u0005\""}, - {time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), - "1979-05-27T07:32:00Z"}, - {[]interface{}{"gamma", "delta"}, - "[\"gamma\",\"delta\"]"}, + {time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), "1979-05-27T07:32:00Z"}, + {[]interface{}{"gamma", "delta"}, "[\"gamma\",\"delta\"]"}, {nil, ""}, } { result, err := tomlValueStringRepresentation(item.Value, "", "", false) @@ -1061,7 +1061,7 @@ func TestInvalidFloatParsing(t *testing.T) { } _, err = Load("a=_1_2") - if err.Error() != "(1, 3): cannot start number with underscore" { + if err.Error() != "(1, 3): no value can start with _" { t.Error("Bad error message:", err.Error()) } } @@ -1125,11 +1125,10 @@ The quick brown \ the lazy dog.""" str3 = """\ - The quick brown \ - fox jumps over \ - the lazy dog.\ - """`) - + The quick brown \ + fox jumps over \ + the lazy dog.\ + """`) if err != nil { t.Fatalf("unexpected error: %v", err) }