diff --git a/.travis.yml b/.travis.yml index 23b0214..0c442b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: go script: "./test.sh" go: - - 1.0 - 1.1 - 1.2 - tip diff --git a/README.md b/README.md index 756ed51..d0b574d 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ You can run both of them using `./test.sh`. ## License -Copyright (c) 2013 Thomas Pelletier +Copyright (c) 2013, 2014 Thomas Pelletier, Eric Anderton Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/clean.sh b/clean.sh new file mode 100755 index 0000000..44d49d9 --- /dev/null +++ b/clean.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# fail out of the script if anything here fails +set -e + +# clear out stuff generated by test.sh +rm -rf src test_program_bin toml-test diff --git a/test_program/test_program.go b/cmd/test_program.go similarity index 90% rename from test_program/test_program.go rename to cmd/test_program.go index 8912c05..06ec8b5 100644 --- a/test_program/test_program.go +++ b/cmd/test_program.go @@ -1,12 +1,12 @@ package main import ( - "io/ioutil" - "os" - "github.com/pelletier/go-toml" "encoding/json" "fmt" + "github.com/pelletier/go-toml" + "io/ioutil" "log" + "os" "time" ) @@ -40,6 +40,12 @@ func translate(tomlData interface{}) interface{} { return typed case *toml.TomlTree: return translate((map[string]interface{})(*orig)) + case []*toml.TomlTree: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed case []map[string]interface{}: typed := make([]map[string]interface{}, len(orig)) for i, v := range orig { diff --git a/lexer.go b/lexer.go index 67f9ebc..6b9311c 100644 --- a/lexer.go +++ b/lexer.go @@ -34,8 +34,11 @@ const ( tokenFloat tokenLeftBracket tokenRightBracket + tokenDoubleLeftBracket + tokenDoubleRightBracket tokenDate tokenKeyGroup + tokenKeyGroupArray tokenComma tokenEOL ) @@ -387,8 +390,42 @@ func lexString(l *lexer) stateFn { func lexKeyGroup(l *lexer) stateFn { l.ignore() l.pos += 1 - l.emit(tokenLeftBracket) - return lexInsideKeyGroup + + if l.peek() == '[' { + // token '[[' signifies an array of anonymous key groups + l.pos += 1 + l.emit(tokenDoubleLeftBracket) + return lexInsideKeyGroupArray + } else { + // vanilla key group + l.emit(tokenLeftBracket) + return lexInsideKeyGroup + } +} + +func lexInsideKeyGroupArray(l *lexer) stateFn { + for { + if l.peek() == ']' { + if l.pos > l.start { + l.emit(tokenKeyGroupArray) + } + l.ignore() + l.pos += 1 + if l.peek() != ']' { + break // error + } + l.pos += 1 + l.emit(tokenDoubleRightBracket) + return lexVoid + } else if l.peek() == '[' { + return l.errorf("group name cannot contain ']'") + } + + if l.next() == eof { + break + } + } + return l.errorf("unclosed key group array") } func lexInsideKeyGroup(l *lexer) stateFn { @@ -401,6 +438,8 @@ func lexInsideKeyGroup(l *lexer) stateFn { l.pos += 1 l.emit(tokenRightBracket) return lexVoid + } else if l.peek() == '[' { + return l.errorf("group name cannot contain ']'") } if l.next() == eof { diff --git a/lexer_test.go b/lexer_test.go index 8582520..ebd03a9 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -404,3 +404,12 @@ func TestUnicodeString(t *testing.T) { token{tokenEOF, ""}, }) } + +func TestKeyGroupArray(t *testing.T) { + testFlow(t, "[[foo]]", []token{ + token{tokenDoubleLeftBracket, "[["}, + token{tokenKeyGroupArray, "foo"}, + token{tokenDoubleRightBracket, "]]"}, + token{tokenEOF, ""}, + }) +} diff --git a/parser.go b/parser.go index 672bb35..b9a3f5f 100644 --- a/parser.go +++ b/parser.go @@ -71,6 +71,8 @@ func parseStart(p *parser) parserStateFn { } switch tok.typ { + case tokenDoubleLeftBracket: + return parseGroupArray case tokenLeftBracket: return parseGroup case tokenKey: @@ -83,6 +85,38 @@ func parseStart(p *parser) parserStateFn { return nil } +func parseGroupArray(p *parser) parserStateFn { + p.getToken() // discard the [[ + key := p.getToken() + if key.typ != tokenKeyGroupArray { + panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key)) + } + + // get or create group array element at the indicated part in the path + p.currentGroup = strings.Split(key.val, ".") + dest_tree := p.tree.GetPath(p.currentGroup) + var array []*TomlTree + if dest_tree == nil { + array = make([]*TomlTree, 0) + } else if dest_tree.([]*TomlTree) != nil { + array = dest_tree.([]*TomlTree) + } else { + panic(fmt.Sprintf("key %s is already assigned and not of type group array", key)) + } + + // add a new tree to the end of the group array + new_tree := make(TomlTree) + array = append(array, &new_tree) + p.tree.SetPath(p.currentGroup, array) + + // keep this key name from use by other kinds of assignments + p.seenGroupKeys = append(p.seenGroupKeys, key.val) + + // move to next parser state + p.assume(tokenDoubleRightBracket) + return parseStart(p) +} + func parseGroup(p *parser) parserStateFn { p.getToken() // discard the [ key := p.getToken() @@ -105,17 +139,31 @@ func parseAssign(p *parser) parserStateFn { key := p.getToken() p.assume(tokenEqual) value := parseRvalue(p) - var final_key []string + var group_key []string if len(p.currentGroup) > 0 { - final_key = p.currentGroup + group_key = p.currentGroup } else { - final_key = make([]string, 0) + group_key = make([]string, 0) } - final_key = append(final_key, key.val) - if p.tree.GetPath(final_key) != nil { + + // find the group to assign, looking out for arrays of groups + var target_node *TomlTree + switch node := p.tree.GetPath(group_key).(type) { + case []*TomlTree: + target_node = node[len(node)-1] + case *TomlTree: + target_node = node + default: + panic(fmt.Sprintf("Unknown group type for path %v", group_key)) + } + + // assign value to the found group + local_key := []string{key.val} + final_key := append(group_key, key.val) + if target_node.GetPath(local_key) != nil { panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, "."))) } - p.tree.SetPath(final_key, value) + target_node.SetPath(local_key, value) return parseStart(p) } diff --git a/parser_test.go b/parser_test.go index 238c3c5..dcdd3d1 100644 --- a/parser_test.go +++ b/parser_test.go @@ -13,13 +13,16 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac } for k, v := range ref { node := tree.Get(k) - switch node.(type) { + switch cast_node := node.(type) { + case []*TomlTree: + for idx, item := range cast_node { + assertTree(t, item, err, v.([]map[string]interface{})[idx]) + } case *TomlTree: - assertTree(t, node.(*TomlTree), err, v.(map[string]interface{})) + assertTree(t, cast_node, err, v.(map[string]interface{})) default: if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) { - t.Log("was expecting", v, "at", k, "but got", node) - t.Error() + t.Errorf("was expecting %v at %v but got %v", v, k, node) } } } @@ -155,7 +158,6 @@ func TestNestedEmptyArrays(t *testing.T) { }) } - func TestArrayMixedTypes(t *testing.T) { _, err := Load("a = [42, 16.0]") if err.Error() != "mixed types in array" { @@ -284,3 +286,54 @@ func TestParseFile(t *testing.T) { "clients.data": []interface{}{[]string{"gamma", "delta"}, []int64{1, 2}}, }) } + +func TestParseKeyGroupArray(t *testing.T) { + tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69") + assertTree(t, tree, err, map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": []map[string]interface{}{ + {"a": int64(42)}, + {"a": int64(69)}, + }, + }, + }) +} + +func TestToTomlValue(t *testing.T) { + for idx, item := range []struct { + Value interface{} + Expect string + }{ + {int64(12345), "12345"}, + {float64(123.45), "123.45"}, + {bool(true), "true"}, + {"hello world", "\"hello world\""}, + {"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""}, + {"\x05", "\"\\u0005\""}, + {time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), + "1979-05-27T07:32:00Z"}, + {[]interface{}{"gamma", "delta"}, + "[\n \"gamma\",\n \"delta\",\n]"}, + } { + result := toTomlValue(item.Value, 0) + if result != item.Expect { + t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect) + } + } +} + +func TestToString(t *testing.T) { + tree := &TomlTree{ + "foo": &TomlTree{ + "bar": []*TomlTree{ + {"a": int64(42)}, + {"a": int64(69)}, + }, + }, + } + result := tree.ToString() + expected := "\n[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n" + if result != expected { + t.Errorf("Expected got '%s', expected '%s'", result, expected) + } +} diff --git a/test.sh b/test.sh index b6ed4b4..07aa52e 100755 --- a/test.sh +++ b/test.sh @@ -1,11 +1,28 @@ #!/bin/bash +# fail out of the script if anything here fails +set -e -# Run basic go unit tests -go test -v ./... -result=$? +# set the path to the present working directory +export GOPATH=`pwd` -# Run example-based toml tests -cd test_program && ./go-test.sh -result="$(( result || $? ))" +# Vendorize the BurntSushi test suite +# NOTE: this gets a specific release to avoid versioning issues +if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then + mkdir -p src/github.com/BurntSushi + git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test +fi +pushd src/github.com/BurntSushi/toml-test +git reset --hard '0.2.0' # use the released version, NOT tip +popd +go build -o toml-test github.com/BurntSushi/toml-test -exit $result +# vendorize the current lib for testing +# NOTE: this basically mocks an install without having to go back out to github for code +mkdir -p src/github.com/pelletier/go-toml/cmd +cp *.go *.toml src/github.com/pelletier/go-toml +cp cmd/*.go src/github.com/pelletier/go-toml/cmd +go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go + +# Run basic unit tests and then the BurntSushi test suite +go test -v github.com/pelletier/go-toml +./toml-test ./test_program_bin | tee test_out diff --git a/test_program/go-test.sh b/test_program/go-test.sh deleted file mode 100755 index f34a54d..0000000 --- a/test_program/go-test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -go get github.com/BurntSushi/toml-test # install test suite -go build -o test_program_bin github.com/pelletier/go-toml/test_program - -toml_test_wrapper() { - ret=0 - if hash toml-test 2>/dev/null; then # test availability in $PATH - toml-test "$@" - ret=$? - else - p="$HOME/gopath/bin/toml-test" # try in Travi's place - if [ -f "$p" ]; then - "$p" "$@" - ret=$? - else - "$GOPATH/bin/toml-test" "$@" - ret=$? - fi - fi -} - -toml_test_wrapper ./test_program_bin | tee test_out -ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])" -exit $ret diff --git a/toml.go b/toml.go index dbf8ae8..a7918ba 100644 --- a/toml.go +++ b/toml.go @@ -1,14 +1,17 @@ // TOML markup language parser. // // This version supports the specification as described in -// https://github.com/mojombo/toml/tree/e3656ad493400895f4460f1244a25f8f8e31a32a +// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md package toml import ( "errors" + "fmt" "io/ioutil" "runtime" + "strconv" "strings" + "time" ) // Definition of a TomlTree. @@ -41,29 +44,47 @@ func (t *TomlTree) Keys() []string { // Get the value at key in the TomlTree. // Key is a dot-separated path (e.g. a.b.c). // Returns nil if the path does not exist in the tree. +// If keys is of length zero, the current tree is returned. func (t *TomlTree) Get(key string) interface{} { + if key == "" { + return t + } return t.GetPath(strings.Split(key, ".")) } +// Returns the element in the tree indicated by 'keys'. +// If keys is of length zero, the current tree is returned. func (t *TomlTree) GetPath(keys []string) interface{} { + if len(keys) == 0 { + return t + } subtree := t for _, intermediate_key := range keys[:len(keys)-1] { _, exists := (*subtree)[intermediate_key] if !exists { return nil } - subtree = (*subtree)[intermediate_key].(*TomlTree) + switch node := (*subtree)[intermediate_key].(type) { + case *TomlTree: + subtree = node + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + return nil //(*subtree)[intermediate_key] = append(node, &TomlTree{}) + } + subtree = node[len(node)-1] + } } return (*subtree)[keys[len(keys)-1]] } // Same as Get but with a default value func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { - val := t.Get(key) - if val == nil { - return def - } - return val; + val := t.Get(key) + if val == nil { + return def + } + return val } // Set an element in the tree. @@ -81,7 +102,16 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) { var new_tree TomlTree = make(TomlTree) (*subtree)[intermediate_key] = &new_tree } - subtree = (*subtree)[intermediate_key].(*TomlTree) + switch node := (*subtree)[intermediate_key].(type) { + case *TomlTree: + subtree = node + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + (*subtree)[intermediate_key] = append(node, &TomlTree{}) + } + subtree = node[len(node)-1] + } } (*subtree)[keys[len(keys)-1]] = value } @@ -106,6 +136,104 @@ func (t *TomlTree) createSubTree(key string) { } } +// encodes a string to a TOML-compliant string value +func encodeTomlString(value string) string { + result := "" + for _, rr := range value { + int_rr := uint16(rr) + switch rr { + case '\b': + result += "\\b" + case '\t': + result += "\\t" + case '\n': + result += "\\n" + case '\f': + result += "\\f" + case '\r': + result += "\\r" + case '"': + result += "\\\"" + case '\\': + result += "\\\\" + default: + if int_rr < 0x001F { + result += fmt.Sprintf("\\u%0.4X", int_rr) + } else { + result += string(rr) + } + } + } + return result +} + +// Value print support function for ToString() +// Outputs the TOML compliant string representation of a value +func toTomlValue(item interface{}, indent int) string { + tab := strings.Repeat(" ", indent) + switch value := item.(type) { + case int64: + return tab + strconv.FormatInt(value, 10) + case float64: + return tab + strconv.FormatFloat(value, 'f', -1, 64) + case string: + return tab + "\"" + encodeTomlString(value) + "\"" + case bool: + if value { + return "true" + } else { + return "false" + } + case time.Time: + return tab + value.Format(time.RFC3339) + case []interface{}: + result := tab + "[\n" + for _, item := range value { + result += toTomlValue(item, indent+2) + ",\n" + } + return result + tab + "]" + default: + panic(fmt.Sprintf("unsupported value type: %v", value)) + } +} + +// Recursive support function for ToString() +// Outputs a tree, using the provided keyspace to prefix group names +func (t *TomlTree) toToml(keyspace string) string { + result := "" + for k, v := range (map[string]interface{})(*t) { + // figure out the keyspace + combined_key := k + if keyspace != "" { + combined_key = keyspace + "." + combined_key + } + // output based on type + switch node := v.(type) { + case []*TomlTree: + for _, item := range node { + if len(item.Keys()) > 0 { + result += fmt.Sprintf("\n[[%s]]\n", combined_key) + } + result += item.toToml(combined_key) + } + case *TomlTree: + if len(node.Keys()) > 0 { + result += fmt.Sprintf("\n[%s]\n", combined_key) + } + result += node.toToml(combined_key) + default: + result += fmt.Sprintf("%s = %s\n", k, toTomlValue(node, 0)) + } + } + return result +} + +// Generates a human-readable representation of the current tree. +// Output spans multiple lines, and is suitable for ingest by a TOML parser +func (t *TomlTree) ToString() string { + return t.toToml("") +} + // Create a TomlTree from a string. func Load(content string) (tree *TomlTree, err error) { defer func() { diff --git a/toml_test.go b/toml_test.go index f9fa173..4b6610e 100644 --- a/toml_test.go +++ b/toml_test.go @@ -1 +1,25 @@ package toml + +import ( + "testing" +) + +func TestTomlGetPath(t *testing.T) { + node := make(TomlTree) + //TODO: set other node data + + for idx, item := range []struct { + Path []string + Expected interface{} + }{ + { // empty path test + []string{}, + &node, + }, + } { + result := node.GetPath(item.Path) + if result != item.Expected { + t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result) + } + } +}