Merge pull request #17 from eanderton/master

TOML v0.2.0 Group Array Support and ToString Feature

.travis.yml

@@ -1,7 +1,6 @@
 language: go
 script: "./test.sh"
 go:
-    - 1.0
     - 1.1
     - 1.2
     - tip

README.md

@@ -67,7 +67,7 @@ You can run both of them using `./test.sh`.
 
 ## License
 
-Copyright (c) 2013 Thomas Pelletier
+Copyright (c) 2013, 2014 Thomas Pelletier, Eric Anderton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# fail out of the script if anything here fails
+set -e
+
+# clear out stuff generated by test.sh
+rm -rf src test_program_bin toml-test

cmd/test_program.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
-	"github.com/pelletier/go-toml"
 	"encoding/json"
 	"fmt"
+	"github.com/pelletier/go-toml"
+	"io/ioutil"
 	"log"
+	"os"
 	"time"
 )
 
@@ -40,6 +40,12 @@ func translate(tomlData interface{}) interface{} {
 		return typed
 	case *toml.TomlTree:
 		return translate((map[string]interface{})(*orig))
+	case []*toml.TomlTree:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v).(map[string]interface{})
+		}
+		return typed
 	case []map[string]interface{}:
 		typed := make([]map[string]interface{}, len(orig))
 		for i, v := range orig {

lexer.go

@@ -34,8 +34,11 @@ const (
 	tokenFloat
 	tokenLeftBracket
 	tokenRightBracket
+	tokenDoubleLeftBracket
+	tokenDoubleRightBracket
 	tokenDate
 	tokenKeyGroup
+	tokenKeyGroupArray
 	tokenComma
 	tokenEOL
 )
@@ -387,8 +390,42 @@ func lexString(l *lexer) stateFn {
 func lexKeyGroup(l *lexer) stateFn {
 	l.ignore()
 	l.pos += 1
-	l.emit(tokenLeftBracket)
+
-	return lexInsideKeyGroup
+	if l.peek() == '[' {
+		// token '[[' signifies an array of anonymous key groups
+		l.pos += 1
+		l.emit(tokenDoubleLeftBracket)
+		return lexInsideKeyGroupArray
+	} else {
+		// vanilla key group
+		l.emit(tokenLeftBracket)
+		return lexInsideKeyGroup
+	}
+}
+
+func lexInsideKeyGroupArray(l *lexer) stateFn {
+	for {
+		if l.peek() == ']' {
+			if l.pos > l.start {
+				l.emit(tokenKeyGroupArray)
+			}
+			l.ignore()
+			l.pos += 1
+			if l.peek() != ']' {
+				break // error
+			}
+			l.pos += 1
+			l.emit(tokenDoubleRightBracket)
+			return lexVoid
+		} else if l.peek() == '[' {
+			return l.errorf("group name cannot contain ']'")
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	return l.errorf("unclosed key group array")
 }
 
 func lexInsideKeyGroup(l *lexer) stateFn {
@@ -401,6 +438,8 @@ func lexInsideKeyGroup(l *lexer) stateFn {
 			l.pos += 1
 			l.emit(tokenRightBracket)
 			return lexVoid
+		} else if l.peek() == '[' {
+			return l.errorf("group name cannot contain ']'")
 		}
 
 		if l.next() == eof {

lexer_test.go

@@ -404,3 +404,12 @@ func TestUnicodeString(t *testing.T) {
 		token{tokenEOF, ""},
 	})
 }
+
+func TestKeyGroupArray(t *testing.T) {
+	testFlow(t, "[[foo]]", []token{
+		token{tokenDoubleLeftBracket, "[["},
+		token{tokenKeyGroupArray, "foo"},
+		token{tokenDoubleRightBracket, "]]"},
+		token{tokenEOF, ""},
+	})
+}

parser.go

@@ -71,6 +71,8 @@ func parseStart(p *parser) parserStateFn {
 	}
 
 	switch tok.typ {
+	case tokenDoubleLeftBracket:
+		return parseGroupArray
 	case tokenLeftBracket:
 		return parseGroup
 	case tokenKey:
@@ -83,6 +85,38 @@ func parseStart(p *parser) parserStateFn {
 	return nil
 }
 
+func parseGroupArray(p *parser) parserStateFn {
+	p.getToken() // discard the [[
+	key := p.getToken()
+	if key.typ != tokenKeyGroupArray {
+		panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key))
+	}
+
+	// get or create group array element at the indicated part in the path
+	p.currentGroup = strings.Split(key.val, ".")
+	dest_tree := p.tree.GetPath(p.currentGroup)
+	var array []*TomlTree
+	if dest_tree == nil {
+		array = make([]*TomlTree, 0)
+	} else if dest_tree.([]*TomlTree) != nil {
+		array = dest_tree.([]*TomlTree)
+	} else {
+		panic(fmt.Sprintf("key %s is already assigned and not of type group array", key))
+	}
+
+	// add a new tree to the end of the group array
+	new_tree := make(TomlTree)
+	array = append(array, &new_tree)
+	p.tree.SetPath(p.currentGroup, array)
+
+	// keep this key name from use by other kinds of assignments
+	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+
+	// move to next parser state
+	p.assume(tokenDoubleRightBracket)
+	return parseStart(p)
+}
+
 func parseGroup(p *parser) parserStateFn {
 	p.getToken() // discard the [
 	key := p.getToken()
@@ -105,17 +139,31 @@ func parseAssign(p *parser) parserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
 	value := parseRvalue(p)
-	var final_key []string
+	var group_key []string
 	if len(p.currentGroup) > 0 {
-		final_key = p.currentGroup
+		group_key = p.currentGroup
 	} else {
-		final_key = make([]string, 0)
+		group_key = make([]string, 0)
 	}
-	final_key = append(final_key, key.val)
+
-	if p.tree.GetPath(final_key) != nil {
+	// find the group to assign, looking out for arrays of groups
+	var target_node *TomlTree
+	switch node := p.tree.GetPath(group_key).(type) {
+	case []*TomlTree:
+		target_node = node[len(node)-1]
+	case *TomlTree:
+		target_node = node
+	default:
+		panic(fmt.Sprintf("Unknown group type for path %v", group_key))
+	}
+
+	// assign value to the found group
+	local_key := []string{key.val}
+	final_key := append(group_key, key.val)
+	if target_node.GetPath(local_key) != nil {
 		panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
 	}
-	p.tree.SetPath(final_key, value)
+	target_node.SetPath(local_key, value)
 	return parseStart(p)
 }
 

parser_test.go

@@ -13,13 +13,16 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 	}
 	for k, v := range ref {
 		node := tree.Get(k)
-		switch node.(type) {
+		switch cast_node := node.(type) {
+		case []*TomlTree:
+			for idx, item := range cast_node {
+				assertTree(t, item, err, v.([]map[string]interface{})[idx])
+			}
 		case *TomlTree:
-			assertTree(t, node.(*TomlTree), err, v.(map[string]interface{}))
+			assertTree(t, cast_node, err, v.(map[string]interface{}))
 		default:
 			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
-				t.Log("was expecting", v, "at", k, "but got", node)
+				t.Errorf("was expecting %v at %v but got %v", v, k, node)
-				t.Error()
 			}
 		}
 	}
@@ -155,7 +158,6 @@ func TestNestedEmptyArrays(t *testing.T) {
 	})
 }
 
-
 func TestArrayMixedTypes(t *testing.T) {
 	_, err := Load("a = [42, 16.0]")
 	if err.Error() != "mixed types in array" {
@@ -284,3 +286,54 @@ func TestParseFile(t *testing.T) {
 		"clients.data":            []interface{}{[]string{"gamma", "delta"}, []int64{1, 2}},
 	})
 }
+
+func TestParseKeyGroupArray(t *testing.T) {
+	tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
+	assertTree(t, tree, err, map[string]interface{}{
+		"foo": map[string]interface{}{
+			"bar": []map[string]interface{}{
+				{"a": int64(42)},
+				{"a": int64(69)},
+			},
+		},
+	})
+}
+
+func TestToTomlValue(t *testing.T) {
+	for idx, item := range []struct {
+		Value  interface{}
+		Expect string
+	}{
+		{int64(12345), "12345"},
+		{float64(123.45), "123.45"},
+		{bool(true), "true"},
+		{"hello world", "\"hello world\""},
+		{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
+		{"\x05", "\"\\u0005\""},
+		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+			"1979-05-27T07:32:00Z"},
+		{[]interface{}{"gamma", "delta"},
+			"[\n  \"gamma\",\n  \"delta\",\n]"},
+	} {
+		result := toTomlValue(item.Value, 0)
+		if result != item.Expect {
+			t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect)
+		}
+	}
+}
+
+func TestToString(t *testing.T) {
+	tree := &TomlTree{
+		"foo": &TomlTree{
+			"bar": []*TomlTree{
+				{"a": int64(42)},
+				{"a": int64(69)},
+			},
+		},
+	}
+	result := tree.ToString()
+	expected := "\n[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n"
+	if result != expected {
+		t.Errorf("Expected got '%s', expected '%s'", result, expected)
+	}
+}

test.sh

@@ -1,11 +1,28 @@
 #!/bin/bash
+# fail out of the script if anything here fails
+set -e
 
-# Run basic go unit tests
+# set the path to the present working directory
-go test -v ./...
+export GOPATH=`pwd`
-result=$?
 
-# Run example-based toml tests
+# Vendorize the BurntSushi test suite
-cd test_program && ./go-test.sh
+# NOTE: this gets a specific release to avoid versioning issues
-result="$(( result || $? ))"
+if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then
+  mkdir -p src/github.com/BurntSushi
+  git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test
+fi
+pushd src/github.com/BurntSushi/toml-test
+git reset --hard '0.2.0'  # use the released version, NOT tip
+popd
+go build -o toml-test github.com/BurntSushi/toml-test
 
-exit $result
+# vendorize the current lib for testing
+# NOTE: this basically mocks an install without having to go back out to github for code
+mkdir -p src/github.com/pelletier/go-toml/cmd
+cp *.go *.toml src/github.com/pelletier/go-toml
+cp cmd/*.go src/github.com/pelletier/go-toml/cmd
+go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
+
+# Run basic unit tests and then the BurntSushi test suite
+go test -v github.com/pelletier/go-toml
+./toml-test ./test_program_bin | tee test_out

test_program/go-test.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-go get github.com/BurntSushi/toml-test # install test suite
-go build -o test_program_bin github.com/pelletier/go-toml/test_program
-
-toml_test_wrapper() {
-    ret=0
-    if hash toml-test 2>/dev/null; then # test availability in $PATH
-        toml-test "$@"
-        ret=$?
-    else
-        p="$HOME/gopath/bin/toml-test" # try in Travi's place
-        if [ -f "$p" ]; then
-            "$p" "$@"
-            ret=$?
-        else
-            "$GOPATH/bin/toml-test" "$@"
-            ret=$?
-        fi
-    fi
-}
-
-toml_test_wrapper ./test_program_bin | tee test_out
-ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])"
-exit $ret

toml.go

@@ -1,14 +1,17 @@
 // TOML markup language parser.
 //
 // This version supports the specification as described in
-// https://github.com/mojombo/toml/tree/e3656ad493400895f4460f1244a25f8f8e31a32a
+// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
 package toml
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 )
 
 // Definition of a TomlTree.
@@ -41,29 +44,47 @@ func (t *TomlTree) Keys() []string {
 // Get the value at key in the TomlTree.
 // Key is a dot-separated path (e.g. a.b.c).
 // Returns nil if the path does not exist in the tree.
+// If keys is of length zero, the current tree is returned.
 func (t *TomlTree) Get(key string) interface{} {
+	if key == "" {
+		return t
+	}
 	return t.GetPath(strings.Split(key, "."))
 }
 
+// Returns the element in the tree indicated by 'keys'.
+// If keys is of length zero, the current tree is returned.
 func (t *TomlTree) GetPath(keys []string) interface{} {
+	if len(keys) == 0 {
+		return t
+	}
 	subtree := t
 	for _, intermediate_key := range keys[:len(keys)-1] {
 		_, exists := (*subtree)[intermediate_key]
 		if !exists {
 			return nil
 		}
-		subtree = (*subtree)[intermediate_key].(*TomlTree)
+		switch node := (*subtree)[intermediate_key].(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				return nil //(*subtree)[intermediate_key] = append(node, &TomlTree{})
+			}
+			subtree = node[len(node)-1]
+		}
 	}
 	return (*subtree)[keys[len(keys)-1]]
 }
 
 // Same as Get but with a default value
 func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
-    val := t.Get(key)
+	val := t.Get(key)
-    if val == nil {
+	if val == nil {
-        return def
+		return def
-    }
+	}
-    return val;
+	return val
 }
 
 // Set an element in the tree.
@@ -81,7 +102,16 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 			var new_tree TomlTree = make(TomlTree)
 			(*subtree)[intermediate_key] = &new_tree
 		}
-		subtree = (*subtree)[intermediate_key].(*TomlTree)
+		switch node := (*subtree)[intermediate_key].(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				(*subtree)[intermediate_key] = append(node, &TomlTree{})
+			}
+			subtree = node[len(node)-1]
+		}
 	}
 	(*subtree)[keys[len(keys)-1]] = value
 }
@@ -106,6 +136,104 @@ func (t *TomlTree) createSubTree(key string) {
 	}
 }
 
+// encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	result := ""
+	for _, rr := range value {
+		int_rr := uint16(rr)
+		switch rr {
+		case '\b':
+			result += "\\b"
+		case '\t':
+			result += "\\t"
+		case '\n':
+			result += "\\n"
+		case '\f':
+			result += "\\f"
+		case '\r':
+			result += "\\r"
+		case '"':
+			result += "\\\""
+		case '\\':
+			result += "\\\\"
+		default:
+			if int_rr < 0x001F {
+				result += fmt.Sprintf("\\u%0.4X", int_rr)
+			} else {
+				result += string(rr)
+			}
+		}
+	}
+	return result
+}
+
+// Value print support function for ToString()
+// Outputs the TOML compliant string representation of a value
+func toTomlValue(item interface{}, indent int) string {
+	tab := strings.Repeat(" ", indent)
+	switch value := item.(type) {
+	case int64:
+		return tab + strconv.FormatInt(value, 10)
+	case float64:
+		return tab + strconv.FormatFloat(value, 'f', -1, 64)
+	case string:
+		return tab + "\"" + encodeTomlString(value) + "\""
+	case bool:
+		if value {
+			return "true"
+		} else {
+			return "false"
+		}
+	case time.Time:
+		return tab + value.Format(time.RFC3339)
+	case []interface{}:
+		result := tab + "[\n"
+		for _, item := range value {
+			result += toTomlValue(item, indent+2) + ",\n"
+		}
+		return result + tab + "]"
+	default:
+		panic(fmt.Sprintf("unsupported value type: %v", value))
+	}
+}
+
+// Recursive support function for ToString()
+// Outputs a tree, using the provided keyspace to prefix group names
+func (t *TomlTree) toToml(keyspace string) string {
+	result := ""
+	for k, v := range (map[string]interface{})(*t) {
+		// figure out the keyspace
+		combined_key := k
+		if keyspace != "" {
+			combined_key = keyspace + "." + combined_key
+		}
+		// output based on type
+		switch node := v.(type) {
+		case []*TomlTree:
+			for _, item := range node {
+				if len(item.Keys()) > 0 {
+					result += fmt.Sprintf("\n[[%s]]\n", combined_key)
+				}
+				result += item.toToml(combined_key)
+			}
+		case *TomlTree:
+			if len(node.Keys()) > 0 {
+				result += fmt.Sprintf("\n[%s]\n", combined_key)
+			}
+			result += node.toToml(combined_key)
+		default:
+			result += fmt.Sprintf("%s = %s\n", k, toTomlValue(node, 0))
+		}
+	}
+	return result
+}
+
+// Generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser
+func (t *TomlTree) ToString() string {
+	return t.toToml("")
+}
+
 // Create a TomlTree from a string.
 func Load(content string) (tree *TomlTree, err error) {
 	defer func() {

toml_test.go

@@ -1 +1,25 @@
 package toml
+
+import (
+	"testing"
+)
+
+func TestTomlGetPath(t *testing.T) {
+	node := make(TomlTree)
+	//TODO: set other node data
+
+	for idx, item := range []struct {
+		Path     []string
+		Expected interface{}
+	}{
+		{ // empty path test
+			[]string{},
+			&node,
+		},
+	} {
+		result := node.GetPath(item.Path)
+		if result != item.Expected {
+			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
+		}
+	}
+}