From dd4c4ffc2b26140e77e475e24894eef727374851 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 19 Aug 2015 10:24:53 -0700 Subject: [PATCH 1/4] Implement inline tables --- .travis.yml | 2 +- lexer.go | 20 +++++++++++++++++- lexer_test.go | 17 +++++----------- parser.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++- parser_test.go | 46 +++++++++++++++++++++++++++++++++++++++++ token.go | 7 ++++++- 6 files changed, 131 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 83e84ad..eeaf72b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,10 @@ language: go script: "./test.sh" sudo: false go: - - 1.1 - 1.2 - 1.3 - 1.4.1 + - 1.5 - tip before_install: - go get github.com/axw/gocov/gocov diff --git a/lexer.go b/lexer.go index 95beb09..25635b4 100644 --- a/lexer.go +++ b/lexer.go @@ -158,13 +158,17 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { case '.': return l.errorf("cannot start float with a dot") case '=': - return l.errorf("cannot have multiple equals for the same key") + return l.lexEqual case '[': l.depth++ return l.lexLeftBracket case ']': l.depth-- return l.lexRightBracket + case '{': + return l.lexLeftCurlyBrace + case '}': + return l.lexRightCurlyBrace case '#': return l.lexComment case '"': @@ -218,6 +222,20 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { return nil } +func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn { + l.ignore() + l.pos++ + l.emit(tokenLeftCurlyBrace) + return l.lexRvalue +} + +func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn { + l.ignore() + l.pos++ + l.emit(tokenRightCurlyBrace) + return l.lexRvalue +} + func (l *tomlLexer) lexDate() tomlLexStateFn { l.emit(tokenDate) return l.lexRvalue diff --git a/lexer_test.go b/lexer_test.go index 82946bf..3509367 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -8,11 +8,12 @@ func testFlow(t *testing.T, input string, expectedFlow []token) { token := <-ch if token != expected { t.Log("While testing: ", input) + t.Log("compared (got)", token, "to (expected)", expected) + t.Log("\tvalue:", token.val, "<->", expected.val) + t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String()) + t.Log("\tline:", token.Line, "<->", expected.Line) + t.Log("\tcolumn:", token.Col, "<->", expected.Col) t.Log("compared", token, "to", expected) - t.Log(token.val, "<->", expected.val) - t.Log(token.typ, "<->", expected.typ) - t.Log(token.Line, "<->", expected.Line) - t.Log(token.Col, "<->", expected.Col) t.FailNow() } } @@ -371,14 +372,6 @@ func TestFloatWithExponent5(t *testing.T) { }) } -func TestDoubleEqualKey(t *testing.T) { - testFlow(t, "foo= = 2", []token{ - token{Position{1, 1}, tokenKey, "foo"}, - token{Position{1, 4}, tokenEqual, "="}, - token{Position{1, 5}, tokenError, "cannot have multiple equals for the same key"}, - }) -} - func TestInvalidEsquapeSequence(t *testing.T) { testFlow(t, `foo = "\x"`, []token{ token{Position{1, 1}, tokenKey, "foo"}, diff --git a/parser.go b/parser.go index 59f513b..9075002 100644 --- a/parser.go +++ b/parser.go @@ -244,6 +244,8 @@ func (p *tomlParser) parseRvalue() interface{} { return val case tokenLeftBracket: return p.parseArray() + case tokenLeftCurlyBrace: + return p.parseInlineTable() case tokenError: p.raiseError(tok, "%s", tok) } @@ -253,7 +255,51 @@ func (p *tomlParser) parseRvalue() interface{} { return nil } -func (p *tomlParser) parseArray() []interface{} { +func tokenIsComma(t *token) bool { + return t != nil && t.typ == tokenComma +} + +func (p *tomlParser) parseInlineTable() *TomlTree { + tree := newTomlTree() + var previous *token +Loop: + for { + follow := p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated inline table") + } + switch follow.typ { + case tokenRightCurlyBrace: + p.getToken() + break Loop + case tokenKey: + if !tokenIsComma(previous) && previous != nil { + p.raiseError(follow, "comma expected between fields in inline table") + } + key := p.getToken() + p.assume(tokenEqual) + value := p.parseRvalue() + tree.Set(key.val, value) + case tokenComma: + if previous == nil { + p.raiseError(follow, "inline table cannot start with a comma") + } + if tokenIsComma(previous) { + p.raiseError(follow, "need field between two commas in inline table") + } + p.getToken() + default: + p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String()) + } + previous = follow + } + if tokenIsComma(previous) { + p.raiseError(previous, "trailing comma at the end of inline table") + } + return tree +} + +func (p *tomlParser) parseArray() interface{} { var array []interface{} arrayType := reflect.TypeOf(nil) for { @@ -263,6 +309,13 @@ func (p *tomlParser) parseArray() []interface{} { } if follow.typ == tokenRightBracket { p.getToken() + if arrayType == reflect.TypeOf(newTomlTree()) { + tomlArray := make([]*TomlTree, len(array)) + for i, v := range array { + tomlArray[i] = v.(*TomlTree) + } + return tomlArray + } return array } val := p.parseRvalue() diff --git a/parser_test.go b/parser_test.go index ebf1626..c507a23 100644 --- a/parser_test.go +++ b/parser_test.go @@ -310,6 +310,52 @@ func TestArrayWithExtraCommaComment(t *testing.T) { }) } +func TestSimpleInlineGroup(t *testing.T) { + tree, err := Load("key = {a = 42}") + assertTree(t, tree, err, map[string]interface{}{ + "key": map[string]interface{}{ + "a": int64(42), + }, + }) +} + +func TestDoubleInlineGroup(t *testing.T) { + tree, err := Load("key = {a = 42, b = \"foo\"}") + assertTree(t, tree, err, map[string]interface{}{ + "key": map[string]interface{}{ + "a": int64(42), + "b": "foo", + }, + }) +} + +func TestExampleInlineGroup(t *testing.T) { + tree, err := Load(`name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 }`) + assertTree(t, tree, err, map[string]interface{}{ + "name": map[string]interface{}{ + "first": "Tom", + "last": "Preston-Werner", + }, + "point": map[string]interface{}{ + "x": int64(1), + "y": int64(2), + }, + }) +} + +func TestExampleInlineGroupInArray(t *testing.T) { + tree, err := Load(`points = [{ x = 1, y = 2 }]`) + assertTree(t, tree, err, map[string]interface{}{ + "points": []map[string]interface{}{ + map[string]interface{}{ + "x": int64(1), + "y": int64(2), + }, + }, + }) +} + func TestDuplicateGroups(t *testing.T) { _, err := Load("[foo]\na=2\n[foo]b=3") if err.Error() != "(3, 2): duplicated tables" { diff --git a/token.go b/token.go index f0fbac9..ebbf960 100644 --- a/token.go +++ b/token.go @@ -26,6 +26,8 @@ const ( tokenEqual tokenLeftBracket tokenRightBracket + tokenLeftCurlyBrace + tokenRightCurlyBrace tokenLeftParen tokenRightParen tokenDoubleLeftBracket @@ -44,6 +46,7 @@ const ( ) var tokenTypeNames = []string{ + "Error", "EOF", "Comment", "Key", @@ -54,7 +57,9 @@ var tokenTypeNames = []string{ "Float", "=", "[", - "[", + "]", + "{", + "}", "(", ")", "]]", From 821a80e6350768891e8419c83e628675de74eba4 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 9 Sep 2015 17:01:05 +0100 Subject: [PATCH 2/4] Add removed test --- parser.go | 3 +++ parser_test.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/parser.go b/parser.go index 9075002..5751632 100644 --- a/parser.go +++ b/parser.go @@ -171,6 +171,7 @@ func (p *tomlParser) parseGroup() tomlParserStateFn { func (p *tomlParser) parseAssign() tomlParserStateFn { key := p.getToken() p.assume(tokenEqual) + value := p.parseRvalue() var groupKey []string if len(p.currentGroup) > 0 { @@ -246,6 +247,8 @@ func (p *tomlParser) parseRvalue() interface{} { return p.parseArray() case tokenLeftCurlyBrace: return p.parseInlineTable() + case tokenEqual: + p.raiseError(tok, "cannot have multiple equals for the same key") case tokenError: p.raiseError(tok, "%s", tok) } diff --git a/parser_test.go b/parser_test.go index c507a23..93d0e78 100644 --- a/parser_test.go +++ b/parser_test.go @@ -571,3 +571,10 @@ func TestInvalidGroupArray(t *testing.T) { t.Error("Should error") } } + +func TestDoubleEqual(t *testing.T) { + _, err := Load("foo= = 2") + if err.Error() != "(1, 6): cannot have multiple equals for the same key" { + t.Error("Bad error message:", err.Error()) + } +} From d467309bdd1be55c84d9e48efd94d5cc2fe91c22 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 9 Sep 2015 17:04:36 +0100 Subject: [PATCH 3/4] Add comment to justify this madness --- parser.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parser.go b/parser.go index 5751632..0fe0457 100644 --- a/parser.go +++ b/parser.go @@ -312,6 +312,10 @@ func (p *tomlParser) parseArray() interface{} { } if follow.typ == tokenRightBracket { p.getToken() + // An array of TomlTrees is actually an array of inline + // tables, which is a shorthand for a table array. If the + // array was not converted from []interface{} to []*TomlTree, + // the two notations would not be equivalent. if arrayType == reflect.TypeOf(newTomlTree()) { tomlArray := make([]*TomlTree, len(array)) for i, v := range array { From 7d69e5a5c55a1f03cc93e955c549e34bcdeed690 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 9 Sep 2015 17:40:27 +0100 Subject: [PATCH 4/4] Tests for erroneous inline tables --- parser_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/parser_test.go b/parser_test.go index 365c247..0532d61 100644 --- a/parser_test.go +++ b/parser_test.go @@ -376,6 +376,34 @@ func TestExampleInlineGroupInArray(t *testing.T) { }) } +func TestInlineTableUnterminated(t *testing.T) { + _, err := Load("foo = {") + if err.Error() != "(1, 8): unterminated inline table" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestInlineTableCommaExpected(t *testing.T) { + _, err := Load("foo = {hello = 53 test = foo}") + if err.Error() != "(1, 19): comma expected between fields in inline table" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestInlineTableCommaStart(t *testing.T) { + _, err := Load("foo = {, hello = 53}") + if err.Error() != "(1, 8): inline table cannot start with a comma" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestInlineTableDoubleComma(t *testing.T) { + _, err := Load("foo = {hello = 53,, foo = 17}") + if err.Error() != "(1, 19): need field between two commas in inline table" { + t.Error("Bad error message:", err.Error()) + } +} + func TestDuplicateGroups(t *testing.T) { _, err := Load("[foo]\na=2\n[foo]b=3") if err.Error() != "(3, 2): duplicated tables" {