ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix inline table loading errors (#381)

Browse Source 
Fixes #379 #380
This commit is contained in:
x-hgg-x
2020-05-04 19:13:55 +02:00
committed by GitHub
parent d05497900e
commit f1ba6388fb
3 changed files with 153 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer.go
+19 -17
View File
@@ -26,7 +26,7 @@ type tomlLexer struct {
currentTokenStart int
currentTokenStop int
tokens []token
depth int
brackets []rune
line int
col int
endbufferLine int
@@ -123,6 +123,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
for {
next := l.peek()
switch next {
case '}': // after '{'
return l.lexRightCurlyBrace
case '[':
return l.lexTableKey
case '#':
@@ -140,10 +142,6 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
l.skip()
}
if l.depth > 0 {
return l.lexRvalue
}
if isKeyStartChar(next) {
return l.lexKey
}
@@ -167,10 +165,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
case '=':
return l.lexEqual
case '[':
l.depth++
return l.lexLeftBracket
case ']':
l.depth--
return l.lexRightBracket
case '{':
return l.lexLeftCurlyBrace
@@ -188,12 +184,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
fallthrough
case '\n':
l.skip()
if l.depth == 0 {
return l.lexVoid
if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '[' {
return l.lexRvalue
}
return l.lexRvalue
case '_':
return l.errorf("cannot start number with underscore")
return l.lexVoid
}
if l.follow("true") {
@@ -236,10 +230,6 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
return l.lexNumber
}
if isAlphanumeric(next) {
return l.lexKey
}
return l.errorf("no value can start with %c", next)
}
@@ -250,12 +240,17 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
l.next()
l.emit(tokenLeftCurlyBrace)
l.brackets = append(l.brackets, '{')
return l.lexVoid
}
func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
l.next()
l.emit(tokenRightCurlyBrace)
if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '{' {
return l.errorf("cannot have '}' here")
}
l.brackets = l.brackets[:len(l.brackets)-1]
return l.lexRvalue
}
@@ -302,6 +297,9 @@ func (l *tomlLexer) lexEqual() tomlLexStateFn {
func (l *tomlLexer) lexComma() tomlLexStateFn {
l.next()
l.emit(tokenComma)
if len(l.brackets) > 0 && l.brackets[len(l.brackets)-1] == '{' {
return l.lexVoid
}
return l.lexRvalue
}
@@ -361,6 +359,7 @@ func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn {
func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
l.next()
l.emit(tokenLeftBracket)
l.brackets = append(l.brackets, '[')
return l.lexRvalue
}
@@ -543,7 +542,6 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
}
str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
if err != nil {
return l.errorf(err.Error())
}
@@ -615,6 +613,10 @@ func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn {
func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
l.next()
l.emit(tokenRightBracket)
if len(l.brackets) == 0 || l.brackets[len(l.brackets)-1] != '[' {
return l.errorf("cannot have ']' here")
}
l.brackets = l.brackets[:len(l.brackets)-1]
return l.lexRvalue
}
lexer_test.go
+121
View File
@@ -698,6 +698,7 @@ func TestUnicodeString(t *testing.T) {
{Position{1, 22}, tokenEOF, ""},
})
}
func TestEscapeInString(t *testing.T) {
testFlow(t, `foo = "\b\f\/"`, []token{
{Position{1, 1}, tokenKey, "foo"},
@@ -772,6 +773,16 @@ func TestLexUnknownRvalue(t *testing.T) {
})
}
func TestLexInlineTableEmpty(t *testing.T) {
testFlow(t, `foo = {}`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 8}, tokenRightCurlyBrace, "}"},
{Position{1, 9}, tokenEOF, ""},
})
}
func TestLexInlineTableBareKey(t *testing.T) {
testFlow(t, `foo = { bar = "baz" }`, []token{
{Position{1, 1}, tokenKey, "foo"},
@@ -798,6 +809,116 @@ func TestLexInlineTableBareKeyDash(t *testing.T) {
})
}
func TestLexInlineTableBareKeyInArray(t *testing.T) {
testFlow(t, `foo = [{ -bar_ = "baz" }]`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftBracket, "["},
{Position{1, 8}, tokenLeftCurlyBrace, "{"},
{Position{1, 10}, tokenKey, "-bar_"},
{Position{1, 16}, tokenEqual, "="},
{Position{1, 19}, tokenString, "baz"},
{Position{1, 24}, tokenRightCurlyBrace, "}"},
{Position{1, 25}, tokenRightBracket, "]"},
{Position{1, 26}, tokenEOF, ""},
})
}
func TestLexInlineTableError1(t *testing.T) {
testFlow(t, `foo = { 123 = 0 ]`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "123"},
{Position{1, 13}, tokenEqual, "="},
{Position{1, 15}, tokenInteger, "0"},
{Position{1, 17}, tokenRightBracket, "]"},
{Position{1, 18}, tokenError, "cannot have ']' here"},
})
}
func TestLexInlineTableError2(t *testing.T) {
testFlow(t, `foo = { 123 = 0 }}`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "123"},
{Position{1, 13}, tokenEqual, "="},
{Position{1, 15}, tokenInteger, "0"},
{Position{1, 17}, tokenRightCurlyBrace, "}"},
{Position{1, 18}, tokenRightCurlyBrace, "}"},
{Position{1, 19}, tokenError, "cannot have '}' here"},
})
}
func TestLexInlineTableDottedKey1(t *testing.T) {
testFlow(t, `foo = { a = 0, 123.45abc = 0 }`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "a"},
{Position{1, 11}, tokenEqual, "="},
{Position{1, 13}, tokenInteger, "0"},
{Position{1, 14}, tokenComma, ","},
{Position{1, 16}, tokenKey, "123.45abc"},
{Position{1, 26}, tokenEqual, "="},
{Position{1, 28}, tokenInteger, "0"},
{Position{1, 30}, tokenRightCurlyBrace, "}"},
{Position{1, 31}, tokenEOF, ""},
})
}
func TestLexInlineTableDottedKey2(t *testing.T) {
testFlow(t, `foo = { a = 0, '123'.'45abc' = 0 }`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "a"},
{Position{1, 11}, tokenEqual, "="},
{Position{1, 13}, tokenInteger, "0"},
{Position{1, 14}, tokenComma, ","},
{Position{1, 16}, tokenKey, "'123'.'45abc'"},
{Position{1, 30}, tokenEqual, "="},
{Position{1, 32}, tokenInteger, "0"},
{Position{1, 34}, tokenRightCurlyBrace, "}"},
{Position{1, 35}, tokenEOF, ""},
})
}
func TestLexInlineTableDottedKey3(t *testing.T) {
testFlow(t, `foo = { a = 0, "123"."45ʎǝʞ" = 0 }`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "a"},
{Position{1, 11}, tokenEqual, "="},
{Position{1, 13}, tokenInteger, "0"},
{Position{1, 14}, tokenComma, ","},
{Position{1, 16}, tokenKey, `"123"."45ʎǝʞ"`},
{Position{1, 30}, tokenEqual, "="},
{Position{1, 32}, tokenInteger, "0"},
{Position{1, 34}, tokenRightCurlyBrace, "}"},
{Position{1, 35}, tokenEOF, ""},
})
}
func TestLexInlineTableBareKeyWithComma(t *testing.T) {
testFlow(t, `foo = { -bar1 = "baz", -bar_ = "baz" }`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 7}, tokenLeftCurlyBrace, "{"},
{Position{1, 9}, tokenKey, "-bar1"},
{Position{1, 15}, tokenEqual, "="},
{Position{1, 18}, tokenString, "baz"},
{Position{1, 22}, tokenComma, ","},
{Position{1, 24}, tokenKey, "-bar_"},
{Position{1, 30}, tokenEqual, "="},
{Position{1, 33}, tokenString, "baz"},
{Position{1, 38}, tokenRightCurlyBrace, "}"},
{Position{1, 39}, tokenEOF, ""},
})
}
func TestLexInlineTableBareKeyUnderscore(t *testing.T) {
testFlow(t, `foo = { _bar = "baz" }`, []token{
{Position{1, 1}, tokenKey, "foo"},
parser_test.go
+13 -14
View File
@@ -239,7 +239,8 @@ func TestLocalDateTime(t *testing.T) {
Minute: 32,
Second: 0,
Nanosecond: 0,
}},
},
},
})
}
@@ -257,7 +258,8 @@ func TestLocalDateTimeNano(t *testing.T) {
Minute: 32,
Second: 0,
Nanosecond: 999999000,
}},
},
},
})
}
@@ -677,7 +679,7 @@ func TestInlineTableUnterminated(t *testing.T) {
func TestInlineTableCommaExpected(t *testing.T) {
_, err := Load("foo = {hello = 53 test = foo}")
if err.Error() != "(1, 19): comma expected between fields in inline table" {
if err.Error() != "(1, 19): unexpected token type in inline table: no value can start with t" {
t.Error("Bad error message:", err.Error())
}
}
@@ -691,7 +693,7 @@ func TestInlineTableCommaStart(t *testing.T) {
func TestInlineTableDoubleComma(t *testing.T) {
_, err := Load("foo = {hello = 53,, foo = 17}")
if err.Error() != "(1, 19): need field between two commas in inline table" {
if err.Error() != "(1, 19): unexpected token type in inline table: keys cannot contain , character" {
t.Error("Bad error message:", err.Error())
}
}
@@ -928,10 +930,8 @@ func TestTomlValueStringRepresentation(t *testing.T) {
{"hello world", "\"hello world\""},
{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
{"\x05", "\"\\u0005\""},
{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
"1979-05-27T07:32:00Z"},
{[]interface{}{"gamma", "delta"},
"[\"gamma\",\"delta\"]"},
{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), "1979-05-27T07:32:00Z"},
{[]interface{}{"gamma", "delta"}, "[\"gamma\",\"delta\"]"},
{nil, ""},
} {
result, err := tomlValueStringRepresentation(item.Value, "", "", false)
@@ -1061,7 +1061,7 @@ func TestInvalidFloatParsing(t *testing.T) {
}
_, err = Load("a=_1_2")
if err.Error() != "(1, 3): cannot start number with underscore" {
if err.Error() != "(1, 3): no value can start with _" {
t.Error("Bad error message:", err.Error())
}
}
@@ -1125,11 +1125,10 @@ The quick brown \
the lazy dog."""
str3 = """\
The quick brown \
fox jumps over \
the lazy dog.\
"""`)
The quick brown \
fox jumps over \
the lazy dog.\
"""`)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}