From 40a44dc51f81f6911fe3e0adf24cf6040cd77701 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 14:43:27 +0100 Subject: [PATCH 01/19] Add BurntSushi's test suite --- .gitignore | 1 + test_program/go-test.sh | 6 ++++++ test_program/test_program.go | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 .gitignore create mode 100755 test_program/go-test.sh create mode 100644 test_program/test_program.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1b6190 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test_program/test_program_bin diff --git a/test_program/go-test.sh b/test_program/go-test.sh new file mode 100755 index 0000000..10043aa --- /dev/null +++ b/test_program/go-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +go get github.com/BurntSushi/toml-test # install test suite +go get github.com/BurntSushi/toml/toml-test-go # install my parser +go build -o test_program_bin github.com/pelletier/go-toml/test_program +$GOPATH/bin/toml-test ./test_program_bin # run tests on my parser diff --git a/test_program/test_program.go b/test_program/test_program.go new file mode 100644 index 0000000..8f9da83 --- /dev/null +++ b/test_program/test_program.go @@ -0,0 +1,20 @@ +package main + +import ( + "io/ioutil" + "os" + "github.com/pelletier/go-toml" +) + +func main() { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + os.Exit(2) + } + _, err = toml.Load(string(bytes)) + if err == nil { + os.Exit(0) + } else { + os.Exit(1) + } +} From bbe45c63f2f2ba70d4bd8fa81058d75d45fc8141 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 14:50:52 +0100 Subject: [PATCH 02/19] Add test script to run both unit and example tests --- .travis.yml | 1 + test.sh | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100755 test.sh diff --git a/.travis.yml b/.travis.yml index 17a616a..ea2966c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: go +script: "./test.sh" go: - 1.0 - 1.1 diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..aa829ee --- /dev/null +++ b/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Run basic go unit tests +go test -v ./... + +# Run example-based toml tests +cd test_program && ./go-test.sh From c163b3f68b43eecf9a2593378f18e8d35da74747 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 15:00:31 +0100 Subject: [PATCH 03/19] Add wrapper to find the test binary --- test_program/go-test.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test_program/go-test.sh b/test_program/go-test.sh index 10043aa..3c65d15 100755 --- a/test_program/go-test.sh +++ b/test_program/go-test.sh @@ -3,4 +3,18 @@ go get github.com/BurntSushi/toml-test # install test suite go get github.com/BurntSushi/toml/toml-test-go # install my parser go build -o test_program_bin github.com/pelletier/go-toml/test_program -$GOPATH/bin/toml-test ./test_program_bin # run tests on my parser + +toml_test_wrapper() { + if hash toml-test 2>/dev/null; then # test availability in $PATH + toml-test "$@" + else + p="$HOME/gopath/bin/toml-test" # try in Travi's place + if [ -f "$p" ]; then + "$p" "$@" + else + "$GOPATH/bin/toml-test" "$@" + fi + fi +} + +toml_test_wrapper ./test_program_bin # run tests on my parser From def543355845561d8512c53301fe712b80f32460 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 15:06:32 +0100 Subject: [PATCH 04/19] Add a note on the README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 334d927..756ed51 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,14 @@ Feel free to report bugs and patches using GitHub's pull requests system on [pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be much appreciated! +### Run tests + +You have to make sure two kind of tests run: + +1. The Go unit tests: `go test` +2. The TOML examples base: `./test_program/go-test.sh` + +You can run both of them using `./test.sh`. ## License From 979a0555120452c947b805c0875d78da2a8010dc Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 15:28:52 +0100 Subject: [PATCH 05/19] Retrieve the exit code from the test suites --- test.sh | 4 ++++ test_program/go-test.sh | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test.sh b/test.sh index aa829ee..b6ed4b4 100755 --- a/test.sh +++ b/test.sh @@ -2,6 +2,10 @@ # Run basic go unit tests go test -v ./... +result=$? # Run example-based toml tests cd test_program && ./go-test.sh +result="$(( result || $? ))" + +exit $result diff --git a/test_program/go-test.sh b/test_program/go-test.sh index 3c65d15..71293a3 100755 --- a/test_program/go-test.sh +++ b/test_program/go-test.sh @@ -5,16 +5,22 @@ go get github.com/BurntSushi/toml/toml-test-go # install my parser go build -o test_program_bin github.com/pelletier/go-toml/test_program toml_test_wrapper() { + ret=0 if hash toml-test 2>/dev/null; then # test availability in $PATH toml-test "$@" + ret=$? else p="$HOME/gopath/bin/toml-test" # try in Travi's place if [ -f "$p" ]; then "$p" "$@" + ret=$? else "$GOPATH/bin/toml-test" "$@" + ret=$? fi fi } -toml_test_wrapper ./test_program_bin # run tests on my parser +toml_test_wrapper ./test_program_bin | tee test_out +ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])" +exit $ret From 72f17747a07112b00bf3c6a9997faf882fcca061 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 15:45:50 +0100 Subject: [PATCH 06/19] Prevent mixed types in arrays --- parser.go | 8 ++++++++ parser_test.go | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/parser.go b/parser.go index e9c9f52..54750c3 100644 --- a/parser.go +++ b/parser.go @@ -4,6 +4,7 @@ package toml import ( "fmt" + "reflect" "strconv" "time" ) @@ -146,6 +147,7 @@ func parseRvalue(p *parser) interface{} { func parseArray(p *parser) []interface{} { array := make([]interface{}, 0) + arrayType := reflect.TypeOf(nil) for { follow := p.peek() if follow == nil || follow.typ == tokenEOF { @@ -156,6 +158,12 @@ func parseArray(p *parser) []interface{} { return array } val := parseRvalue(p) + if arrayType == nil { + arrayType = reflect.TypeOf(val) + } + if reflect.TypeOf(val) != arrayType { + panic("mixed types in array") + } array = append(array, val) follow = p.peek() if follow == nil { diff --git a/parser_test.go b/parser_test.go index 629e0a6..1f55faa 100644 --- a/parser_test.go +++ b/parser_test.go @@ -142,6 +142,18 @@ func TestArrayNested(t *testing.T) { }) } +func TestArrayMixedTypes(t *testing.T) { + _, err := Load("a = [42, 16.0]") + if err.Error() != "mixed types in array" { + t.Error("Bad error message:", err.Error()) + } + + _, err = Load("a = [42, \"hello\"]") + if err.Error() != "mixed types in array" { + t.Error("Bad error message:", err.Error()) + } +} + func TestArrayNestedStrings(t *testing.T) { tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]") assertTree(t, tree, err, map[string]interface{}{ From 8081f3cc094f203bf5d6f7264282051118beb7ba Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 16:10:26 +0100 Subject: [PATCH 07/19] Don't allow tables to be redefined --- parser.go | 3 +++ parser_test.go | 7 +++++++ toml.go | 12 ++++++++++++ 3 files changed, 22 insertions(+) diff --git a/parser.go b/parser.go index 54750c3..f91e80c 100644 --- a/parser.go +++ b/parser.go @@ -87,6 +87,9 @@ func parseGroup(p *parser) parserStateFn { if key.typ != tokenKeyGroup { panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key)) } + if p.tree.Has(key.val) { + panic("duplicated tables") + } p.tree.createSubTree(key.val) p.assume(tokenRightBracket) p.currentGroup = key.val diff --git a/parser_test.go b/parser_test.go index 1f55faa..cda0322 100644 --- a/parser_test.go +++ b/parser_test.go @@ -196,6 +196,13 @@ func TestArrayWithExtraCommaComment(t *testing.T) { }) } +func TestDuplicateGroups(t *testing.T) { + _, err := Load("[foo]\na=2\n[foo]b=3") + if err.Error() != "duplicated tables" { + t.Error("Bad error message:", err.Error()) + } +} + func TestMissingFile(t *testing.T) { _, err := LoadFile("foo.toml") if err.Error() != "open foo.toml: no such file or directory" { diff --git a/toml.go b/toml.go index 410bab6..bfd12b2 100644 --- a/toml.go +++ b/toml.go @@ -15,6 +15,18 @@ import ( // This is the result of the parsing of a TOML file. type TomlTree map[string]interface{} +// Has returns a boolean indicating if the toplevel tree contains the given +// key. +func (t *TomlTree) Has(key string) bool { + mp := (map[string]interface{})(*t) + for k, _ := range mp { + if k == key { + return true + } + } + return false +} + // Keys returns the keys of the toplevel tree. // Warning: this is a costly operation. func (t *TomlTree) Keys() []string { From c743c9031558f834131a657c4ad0749080e4d42c Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 16:23:53 +0100 Subject: [PATCH 08/19] Don't allow empty intermediate tables --- parser_test.go | 7 +++++++ toml.go | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/parser_test.go b/parser_test.go index cda0322..441c4e2 100644 --- a/parser_test.go +++ b/parser_test.go @@ -203,6 +203,13 @@ func TestDuplicateGroups(t *testing.T) { } } +func TestEmptyIntermediateTable(t *testing.T) { + _, err := Load("[foo..bar]") + if err.Error() != "empty intermediate table" { + t.Error("Bad error message:", err.Error()) + } +} + func TestMissingFile(t *testing.T) { _, err := LoadFile("foo.toml") if err.Error() != "open foo.toml: no such file or directory" { diff --git a/toml.go b/toml.go index bfd12b2..e1a137f 100644 --- a/toml.go +++ b/toml.go @@ -88,6 +88,9 @@ func (t *TomlTree) Set(key string, value interface{}) { func (t *TomlTree) createSubTree(key string) { subtree := t for _, intermediate_key := range strings.Split(key, ".") { + if intermediate_key == "" { + panic("empty intermediate table") + } _, exists := (*subtree)[intermediate_key] if !exists { var new_tree TomlTree = make(TomlTree) @@ -121,6 +124,5 @@ func LoadFile(path string) (tree *TomlTree, err error) { s := string(buff) tree, err = Load(s) } - return } From 278c4d97ec243b3ff08a6ac13aae0636508f585e Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:17:15 +0100 Subject: [PATCH 09/19] Don't allow floats starting with a dot --- lexer.go | 7 +++++++ parser.go | 2 ++ parser_test.go | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/lexer.go b/lexer.go index 2593466..c4a2bd6 100644 --- a/lexer.go +++ b/lexer.go @@ -234,6 +234,10 @@ func lexRvalue(l *lexer) stateFn { return lexNumber } + if next == '.' { + return l.errorf("cannot start float with a dot") + } + if isSpace(next) { l.ignore() } @@ -416,6 +420,9 @@ func lexNumber(l *lexer) stateFn { l.backup() break } + if point_seen && !digit_seen { + return l.errorf("cannot start float with a dot") + } } if !digit_seen { diff --git a/parser.go b/parser.go index f91e80c..ee5758a 100644 --- a/parser.go +++ b/parser.go @@ -141,6 +141,8 @@ func parseRvalue(p *parser) interface{} { return val case tokenLeftBracket: return parseArray(p) + case tokenError: + panic(tok.val) } panic("never reached") diff --git a/parser_test.go b/parser_test.go index 441c4e2..9beef36 100644 --- a/parser_test.go +++ b/parser_test.go @@ -210,6 +210,18 @@ func TestEmptyIntermediateTable(t *testing.T) { } } +func TestFloatsWithoutLeadingZeros(t *testing.T) { + _, err := Load("a = .42") + if err.Error() != "cannot start float with a dot" { + t.Error("Bad error message:", err.Error()) + } + + _, err = Load("a = -.42") + if err.Error() != "cannot start float with a dot" { + t.Error("Bad error message:", err.Error()) + } +} + func TestMissingFile(t *testing.T) { _, err := LoadFile("foo.toml") if err.Error() != "open foo.toml: no such file or directory" { From 5ffe2e5565cde1ece44217a00227d0e8393041c3 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:24:53 +0100 Subject: [PATCH 10/19] Don't allow float to end with a dot --- lexer.go | 6 ++++++ lexer_test.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lexer.go b/lexer.go index c4a2bd6..0ad27a2 100644 --- a/lexer.go +++ b/lexer.go @@ -413,6 +413,12 @@ func lexNumber(l *lexer) stateFn { for { next := l.next() if next == '.' { + if point_seen { + return l.errorf("cannot have two dots in one float") + } + if !isDigit(l.peek()) { + return l.errorf("float cannot end with a dot") + } point_seen = true } else if isDigit(next) { digit_seen = true diff --git a/lexer_test.go b/lexer_test.go index e5846bd..c2bb2f2 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -268,6 +268,22 @@ func TestKeyEqualDate(t *testing.T) { }) } +func TestFloatEndingWithDot(t *testing.T) { + testFlow(t, "foo = 42.", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "float cannot end with a dot"}, + }) +} + +func TestFloatWithTwoDots(t *testing.T) { + testFlow(t, "foo = 4.2.", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "cannot have two dots in one float"}, + }) +} + func TestKeyEqualNumber(t *testing.T) { testFlow(t, "foo = 42", []token{ token{tokenKey, "foo"}, From e8d5dbf787345a6b87bd57bb8e8923ca51f7d884 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:28:16 +0100 Subject: [PATCH 11/19] Don't allow two equals for the same key --- lexer.go | 8 ++++---- lexer_test.go | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lexer.go b/lexer.go index 0ad27a2..679bb4d 100644 --- a/lexer.go +++ b/lexer.go @@ -192,6 +192,10 @@ func lexRvalue(l *lexer) stateFn { for { next := l.peek() switch next { + case '.': + return l.errorf("cannot start float with a dot") + case '=': + return l.errorf("cannot have multiple equals for the same key") case '[': l.depth += 1 return lexLeftBracket @@ -234,10 +238,6 @@ func lexRvalue(l *lexer) stateFn { return lexNumber } - if next == '.' { - return l.errorf("cannot start float with a dot") - } - if isSpace(next) { l.ignore() } diff --git a/lexer_test.go b/lexer_test.go index c2bb2f2..e3b5daf 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -284,6 +284,14 @@ func TestFloatWithTwoDots(t *testing.T) { }) } +func TestDoubleEqualKey(t *testing.T) { + testFlow(t, "foo= = 2", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "cannot have multiple equals for the same key"}, + }) +} + func TestKeyEqualNumber(t *testing.T) { testFlow(t, "foo = 42", []token{ token{tokenKey, "foo"}, From a34fc5f051b108c25a8bcc0cfa558fd6d65b2cb0 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:34:11 +0100 Subject: [PATCH 12/19] Don't allow invalid escape sequences --- lexer.go | 3 +++ lexer_test.go | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/lexer.go b/lexer.go index 679bb4d..c3a145c 100644 --- a/lexer.go +++ b/lexer.go @@ -358,6 +358,9 @@ func lexString(l *lexer) stateFn { return l.errorf("invalid unicode escape: \\u" + code) } growing_string += string(rune(intcode)) + } else if l.follow("\\") { + l.pos += 1 + return l.errorf("invalid escape sequence: \\" + string(l.peek())) } else { growing_string += string(l.peek()) } diff --git a/lexer_test.go b/lexer_test.go index e3b5daf..b7de9f5 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -292,6 +292,14 @@ func TestDoubleEqualKey(t *testing.T) { }) } +func TestInvalidEsquapeSequence(t *testing.T) { + testFlow(t, "foo = \"\\x\"", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "invalid escape sequence: \\x"}, + }) +} + func TestKeyEqualNumber(t *testing.T) { testFlow(t, "foo = 42", []token{ token{tokenKey, "foo"}, From 01609e0ab7fdd56edc769fc04383eb34489501b6 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:46:30 +0100 Subject: [PATCH 13/19] Add some tests for nested empty arrays --- lexer_test.go | 14 ++++++++++++++ parser_test.go | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/lexer_test.go b/lexer_test.go index b7de9f5..1d7e37b 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -300,6 +300,20 @@ func TestInvalidEsquapeSequence(t *testing.T) { }) } +func TestNestedArrays(t *testing.T) { + testFlow(t, "foo = [[[]]]", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenLeftBracket, "["}, + token{tokenLeftBracket, "["}, + token{tokenLeftBracket, "["}, + token{tokenRightBracket, "]"}, + token{tokenRightBracket, "]"}, + token{tokenRightBracket, "]"}, + token{tokenEOF, ""}, + }) +} + func TestKeyEqualNumber(t *testing.T) { testFlow(t, "foo = 42", []token{ token{tokenKey, "foo"}, diff --git a/parser_test.go b/parser_test.go index 9beef36..9108bab 100644 --- a/parser_test.go +++ b/parser_test.go @@ -142,6 +142,13 @@ func TestArrayNested(t *testing.T) { }) } +func TestNestedEmptyArrays(t *testing.T) { + tree, err := Load("a = [[[]]]") + assertTree(t, tree, err, map[string]interface{}{ + "a": [][][]interface{}{[][]interface{}{[]interface{}{}}}, + }) +} + func TestArrayMixedTypes(t *testing.T) { _, err := Load("a = [42, 16.0]") if err.Error() != "mixed types in array" { From 30854544771ad0be7415f0035e1fc03ac722dd02 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 17:50:59 +0100 Subject: [PATCH 14/19] Don't allow duplicate keys --- parser.go | 3 +++ parser_test.go | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/parser.go b/parser.go index ee5758a..ba78f3d 100644 --- a/parser.go +++ b/parser.go @@ -104,6 +104,9 @@ func parseAssign(p *parser) parserStateFn { if p.currentGroup != "" { final_key = p.currentGroup + "." + key.val } + if p.tree.Get(final_key) != nil { + panic(fmt.Sprintf("the following key was defined twice: %s", final_key)) + } p.tree.Set(final_key, value) return parseStart(p) } diff --git a/parser_test.go b/parser_test.go index 9108bab..66e7eaa 100644 --- a/parser_test.go +++ b/parser_test.go @@ -149,6 +149,7 @@ func TestNestedEmptyArrays(t *testing.T) { }) } + func TestArrayMixedTypes(t *testing.T) { _, err := Load("a = [42, 16.0]") if err.Error() != "mixed types in array" { @@ -210,6 +211,13 @@ func TestDuplicateGroups(t *testing.T) { } } +func TestDuplicateKeys(t *testing.T) { + _, err := Load("foo = 2\nfoo = 3") + if err.Error() != "the following key was defined twice: foo" { + t.Error("Bad error message:", err.Error()) + } +} + func TestEmptyIntermediateTable(t *testing.T) { _, err := Load("[foo..bar]") if err.Error() != "empty intermediate table" { From 4fdde9794ad23b093738fc4d4c9cc84200561520 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 19:00:35 +0100 Subject: [PATCH 15/19] Output JSON for test suite Reused @BurntSushi's --- test_program/test_program.go | 63 +++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/test_program/test_program.go b/test_program/test_program.go index 8f9da83..8912c05 100644 --- a/test_program/test_program.go +++ b/test_program/test_program.go @@ -4,6 +4,10 @@ import ( "io/ioutil" "os" "github.com/pelletier/go-toml" + "encoding/json" + "fmt" + "log" + "time" ) func main() { @@ -11,10 +15,61 @@ func main() { if err != nil { os.Exit(2) } - _, err = toml.Load(string(bytes)) - if err == nil { - os.Exit(0) - } else { + tree, err := toml.Load(string(bytes)) + if err != nil { os.Exit(1) } + + typedTree := translate((map[string]interface{})(*tree)) + + if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil { + log.Fatalf("Error encoding JSON: %s", err) + } + + os.Exit(0) +} + +func translate(tomlData interface{}) interface{} { + + switch orig := tomlData.(type) { + case map[string]interface{}: + typed := make(map[string]interface{}, len(orig)) + for k, v := range orig { + typed[k] = translate(v) + } + return typed + case *toml.TomlTree: + return translate((map[string]interface{})(*orig)) + case []map[string]interface{}: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed + case []interface{}: + typed := make([]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v) + } + return tag("array", typed) + case time.Time: + return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) + case bool: + return tag("bool", fmt.Sprintf("%v", orig)) + case int64: + return tag("integer", fmt.Sprintf("%d", orig)) + case float64: + return tag("float", fmt.Sprintf("%v", orig)) + case string: + return tag("string", orig) + } + + panic(fmt.Sprintf("Unknown type: %T", tomlData)) +} + +func tag(typeName string, data interface{}) map[string]interface{} { + return map[string]interface{}{ + "type": typeName, + "value": data, + } } From a4d623ad05f22b0f18b30158de16693c1575fd13 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 19:29:01 +0100 Subject: [PATCH 16/19] Fix implicit declaration --- parser.go | 25 +++++++++++++++---------- parser_test.go | 26 +++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/parser.go b/parser.go index ba78f3d..6328d12 100644 --- a/parser.go +++ b/parser.go @@ -10,10 +10,11 @@ import ( ) type parser struct { - flow chan token - tree *TomlTree - tokensBuffer []token - currentGroup string + flow chan token + tree *TomlTree + tokensBuffer []token + currentGroup string + seenGroupKeys []string } type parserStateFn func(*parser) parserStateFn @@ -87,9 +88,12 @@ func parseGroup(p *parser) parserStateFn { if key.typ != tokenKeyGroup { panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key)) } - if p.tree.Has(key.val) { - panic("duplicated tables") + for _, item := range p.seenGroupKeys { + if item == key.val { + panic("duplicated tables") + } } + p.seenGroupKeys = append(p.seenGroupKeys, key.val) p.tree.createSubTree(key.val) p.assume(tokenRightBracket) p.currentGroup = key.val @@ -192,10 +196,11 @@ func parseArray(p *parser) []interface{} { func parse(flow chan token) *TomlTree { result := make(TomlTree) parser := &parser{ - flow: flow, - tree: &result, - tokensBuffer: make([]token, 0), - currentGroup: "", + flow: flow, + tree: &result, + tokensBuffer: make([]token, 0), + currentGroup: "", + seenGroupKeys: make([]string, 0), } parser.run() return parser.tree diff --git a/parser_test.go b/parser_test.go index 66e7eaa..238c3c5 100644 --- a/parser_test.go +++ b/parser_test.go @@ -12,9 +12,15 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac return } for k, v := range ref { - if fmt.Sprintf("%v", tree.Get(k)) != fmt.Sprintf("%v", v) { - t.Log("was expecting", v, "at", k, "but got", tree.Get(k)) - t.Error() + node := tree.Get(k) + switch node.(type) { + case *TomlTree: + assertTree(t, node.(*TomlTree), err, v.(map[string]interface{})) + default: + if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) { + t.Log("was expecting", v, "at", k, "but got", node) + t.Error() + } } } } @@ -225,6 +231,20 @@ func TestEmptyIntermediateTable(t *testing.T) { } } +func TestImplicitDeclarationBefore(t *testing.T) { + tree, err := Load("[a.b.c]\nanswer = 42\n[a]\nbetter = 43") + assertTree(t, tree, err, map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": map[string]interface{}{ + "answer": int64(42), + }, + }, + "better": int64(43), + }, + }) +} + func TestFloatsWithoutLeadingZeros(t *testing.T) { _, err := Load("a = .42") if err.Error() != "cannot start float with a dot" { From b1d602f733f9be7810a7634b3d72c6f994860e0c Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 19:35:47 +0100 Subject: [PATCH 17/19] Add missing string escape sequences --- lexer.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lexer.go b/lexer.go index c3a145c..91f93c9 100644 --- a/lexer.go +++ b/lexer.go @@ -332,6 +332,15 @@ func lexString(l *lexer) stateFn { } else if l.follow("\\n") { l.pos += 1 growing_string += "\n" + } else if l.follow("\\b") { + l.pos += 1 + growing_string += "\b" + } else if l.follow("\\f") { + l.pos += 1 + growing_string += "\f" + } else if l.follow("\\/") { + l.pos += 1 + growing_string += "/" } else if l.follow("\\t") { l.pos += 1 growing_string += "\t" From 0c4e891f3e967ca58b7df1951a642799765bb403 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 19:46:56 +0100 Subject: [PATCH 18/19] Handle non-alpha chars in keys --- lexer.go | 4 +++- lexer_test.go | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lexer.go b/lexer.go index 91f93c9..5efbf6b 100644 --- a/lexer.go +++ b/lexer.go @@ -68,7 +68,9 @@ func isAlphanumeric(r rune) bool { } func isKeyChar(r rune) bool { - return isAlphanumeric(r) || r == '-' + // "Keys start with the first non-whitespace character and end with the last + // non-whitespace character before the equals sign." + return !(isSpace(r) || r == '\n' || r == eof || r == '=') } func isDigit(r rune) bool { diff --git a/lexer_test.go b/lexer_test.go index 1d7e37b..6436378 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -113,6 +113,15 @@ func TestBasicKeyAndEqual(t *testing.T) { }) } +func TestKeyWithSharpAndEqual(t *testing.T) { + testFlow(t, "key#name = 5", []token{ + token{tokenKey, "key#name"}, + token{tokenEqual, "="}, + token{tokenInteger, "5"}, + token{tokenEOF, ""}, + }) +} + func TestKeyEqualStringEscape(t *testing.T) { testFlow(t, "foo = \"hello\\\"\"", []token{ token{tokenKey, "foo"}, From dc20c454d74bff190254f5564b2c9b39e7e19108 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 10 Dec 2013 21:51:40 +0100 Subject: [PATCH 19/19] Handle dots in keys --- lexer.go | 8 ++++---- lexer_test.go | 8 ++++++++ parser.go | 24 +++++++++++++----------- toml.go | 10 ++++++++-- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/lexer.go b/lexer.go index 5efbf6b..5dfff38 100644 --- a/lexer.go +++ b/lexer.go @@ -169,10 +169,6 @@ func lexVoid(l *lexer) stateFn { return lexEqual } - if isAlphanumeric(next) { - return lexKey - } - if isSpace(next) { l.ignore() } @@ -181,6 +177,10 @@ func lexVoid(l *lexer) stateFn { return lexRvalue } + if isKeyChar(next) { + return lexKey + } + if l.next() == eof { break } diff --git a/lexer_test.go b/lexer_test.go index 6436378..8582520 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -121,6 +121,14 @@ func TestKeyWithSharpAndEqual(t *testing.T) { token{tokenEOF, ""}, }) } +func TestKeyWithSymbolsAndEqual(t *testing.T) { + testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ + token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"}, + token{tokenEqual, "="}, + token{tokenInteger, "5"}, + token{tokenEOF, ""}, + }) +} func TestKeyEqualStringEscape(t *testing.T) { testFlow(t, "foo = \"hello\\\"\"", []token{ diff --git a/parser.go b/parser.go index 6328d12..672bb35 100644 --- a/parser.go +++ b/parser.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "time" ) @@ -13,7 +14,7 @@ type parser struct { flow chan token tree *TomlTree tokensBuffer []token - currentGroup string + currentGroup []string seenGroupKeys []string } @@ -96,7 +97,7 @@ func parseGroup(p *parser) parserStateFn { p.seenGroupKeys = append(p.seenGroupKeys, key.val) p.tree.createSubTree(key.val) p.assume(tokenRightBracket) - p.currentGroup = key.val + p.currentGroup = strings.Split(key.val, ".") return parseStart(p) } @@ -104,14 +105,17 @@ func parseAssign(p *parser) parserStateFn { key := p.getToken() p.assume(tokenEqual) value := parseRvalue(p) - final_key := key.val - if p.currentGroup != "" { - final_key = p.currentGroup + "." + key.val + var final_key []string + if len(p.currentGroup) > 0 { + final_key = p.currentGroup + } else { + final_key = make([]string, 0) } - if p.tree.Get(final_key) != nil { - panic(fmt.Sprintf("the following key was defined twice: %s", final_key)) + final_key = append(final_key, key.val) + if p.tree.GetPath(final_key) != nil { + panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, "."))) } - p.tree.Set(final_key, value) + p.tree.SetPath(final_key, value) return parseStart(p) } @@ -182,8 +186,6 @@ func parseArray(p *parser) []interface{} { panic("unterminated array") } if follow.typ != tokenRightBracket && follow.typ != tokenComma { - fmt.Println(follow.typ) - fmt.Println(follow.val) panic("missing comma") } if follow.typ == tokenComma { @@ -199,7 +201,7 @@ func parse(flow chan token) *TomlTree { flow: flow, tree: &result, tokensBuffer: make([]token, 0), - currentGroup: "", + currentGroup: make([]string, 0), seenGroupKeys: make([]string, 0), } parser.run() diff --git a/toml.go b/toml.go index e1a137f..dbf8ae8 100644 --- a/toml.go +++ b/toml.go @@ -42,8 +42,11 @@ func (t *TomlTree) Keys() []string { // Key is a dot-separated path (e.g. a.b.c). // Returns nil if the path does not exist in the tree. func (t *TomlTree) Get(key string) interface{} { + return t.GetPath(strings.Split(key, ".")) +} + +func (t *TomlTree) GetPath(keys []string) interface{} { subtree := t - keys := strings.Split(key, ".") for _, intermediate_key := range keys[:len(keys)-1] { _, exists := (*subtree)[intermediate_key] if !exists { @@ -67,8 +70,11 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { // Key is a dot-separated path (e.g. a.b.c). // Creates all necessary intermediates trees, if needed. func (t *TomlTree) Set(key string, value interface{}) { + t.SetPath(strings.Split(key, "."), value) +} + +func (t *TomlTree) SetPath(keys []string, value interface{}) { subtree := t - keys := strings.Split(key, ".") for _, intermediate_key := range keys[:len(keys)-1] { _, exists := (*subtree)[intermediate_key] if !exists {