Merge pull request #14 from pelletier/pelletier/go_test_support

Add BurntSushi's test suite

.gitignore

@@ -0,0 +1 @@
+test_program/test_program_bin

.travis.yml

@@ -1,4 +1,5 @@
 language: go
+script: "./test.sh"
 go:
     - 1.0
     - 1.1

README.md

@@ -56,6 +56,14 @@ Feel free to report bugs and patches using GitHub's pull requests system on
 [pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be
 much appreciated!
 
+### Run tests
+
+You have to make sure two kind of tests run:
+
+1. The Go unit tests: `go test`
+2. The TOML examples base: `./test_program/go-test.sh`
+
+You can run both of them using `./test.sh`.
 
 ## License
 

lexer.go

@@ -68,7 +68,9 @@ func isAlphanumeric(r rune) bool {
 }
 
 func isKeyChar(r rune) bool {
-	return isAlphanumeric(r) || r == '-'
+	// "Keys start with the first non-whitespace character and end with the last
+	// non-whitespace character before the equals sign."
+	return !(isSpace(r) || r == '\n' || r == eof || r == '=')
 }
 
 func isDigit(r rune) bool {
@@ -167,10 +169,6 @@ func lexVoid(l *lexer) stateFn {
 			return lexEqual
 		}
 
-		if isAlphanumeric(next) {
-			return lexKey
-		}
-
 		if isSpace(next) {
 			l.ignore()
 		}
@@ -179,6 +177,10 @@ func lexVoid(l *lexer) stateFn {
 			return lexRvalue
 		}
 
+		if isKeyChar(next) {
+			return lexKey
+		}
+
 		if l.next() == eof {
 			break
 		}
@@ -192,6 +194,10 @@ func lexRvalue(l *lexer) stateFn {
 	for {
 		next := l.peek()
 		switch next {
+		case '.':
+			return l.errorf("cannot start float with a dot")
+		case '=':
+			return l.errorf("cannot have multiple equals for the same key")
 		case '[':
 			l.depth += 1
 			return lexLeftBracket
@@ -328,6 +334,15 @@ func lexString(l *lexer) stateFn {
 		} else if l.follow("\\n") {
 			l.pos += 1
 			growing_string += "\n"
+		} else if l.follow("\\b") {
+			l.pos += 1
+			growing_string += "\b"
+		} else if l.follow("\\f") {
+			l.pos += 1
+			growing_string += "\f"
+		} else if l.follow("\\/") {
+			l.pos += 1
+			growing_string += "/"
 		} else if l.follow("\\t") {
 			l.pos += 1
 			growing_string += "\t"
@@ -354,6 +369,9 @@ func lexString(l *lexer) stateFn {
 				return l.errorf("invalid unicode escape: \\u" + code)
 			}
 			growing_string += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos += 1
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
 		} else {
 			growing_string += string(l.peek())
 		}
@@ -409,6 +427,12 @@ func lexNumber(l *lexer) stateFn {
 	for {
 		next := l.next()
 		if next == '.' {
+			if point_seen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
 			point_seen = true
 		} else if isDigit(next) {
 			digit_seen = true
@@ -416,6 +440,9 @@ func lexNumber(l *lexer) stateFn {
 			l.backup()
 			break
 		}
+		if point_seen && !digit_seen {
+			return l.errorf("cannot start float with a dot")
+		}
 	}
 
 	if !digit_seen {

lexer_test.go

@@ -113,6 +113,23 @@ func TestBasicKeyAndEqual(t *testing.T) {
 	})
 }
 
+func TestKeyWithSharpAndEqual(t *testing.T) {
+	testFlow(t, "key#name = 5", []token{
+		token{tokenKey, "key#name"},
+		token{tokenEqual, "="},
+		token{tokenInteger, "5"},
+		token{tokenEOF, ""},
+	})
+}
+func TestKeyWithSymbolsAndEqual(t *testing.T) {
+	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+		token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+		token{tokenEqual, "="},
+		token{tokenInteger, "5"},
+		token{tokenEOF, ""},
+	})
+}
+
 func TestKeyEqualStringEscape(t *testing.T) {
 	testFlow(t, "foo = \"hello\\\"\"", []token{
 		token{tokenKey, "foo"},
@@ -268,6 +285,52 @@ func TestKeyEqualDate(t *testing.T) {
 	})
 }
 
+func TestFloatEndingWithDot(t *testing.T) {
+	testFlow(t, "foo = 42.", []token{
+		token{tokenKey, "foo"},
+		token{tokenEqual, "="},
+		token{tokenError, "float cannot end with a dot"},
+	})
+}
+
+func TestFloatWithTwoDots(t *testing.T) {
+	testFlow(t, "foo = 4.2.", []token{
+		token{tokenKey, "foo"},
+		token{tokenEqual, "="},
+		token{tokenError, "cannot have two dots in one float"},
+	})
+}
+
+func TestDoubleEqualKey(t *testing.T) {
+	testFlow(t, "foo= = 2", []token{
+		token{tokenKey, "foo"},
+		token{tokenEqual, "="},
+		token{tokenError, "cannot have multiple equals for the same key"},
+	})
+}
+
+func TestInvalidEsquapeSequence(t *testing.T) {
+	testFlow(t, "foo = \"\\x\"", []token{
+		token{tokenKey, "foo"},
+		token{tokenEqual, "="},
+		token{tokenError, "invalid escape sequence: \\x"},
+	})
+}
+
+func TestNestedArrays(t *testing.T) {
+	testFlow(t, "foo = [[[]]]", []token{
+		token{tokenKey, "foo"},
+		token{tokenEqual, "="},
+		token{tokenLeftBracket, "["},
+		token{tokenLeftBracket, "["},
+		token{tokenLeftBracket, "["},
+		token{tokenRightBracket, "]"},
+		token{tokenRightBracket, "]"},
+		token{tokenRightBracket, "]"},
+		token{tokenEOF, ""},
+	})
+}
+
 func TestKeyEqualNumber(t *testing.T) {
 	testFlow(t, "foo = 42", []token{
 		token{tokenKey, "foo"},

parser.go

@@ -4,7 +4,9 @@ package toml
 
 import (
 	"fmt"
+	"reflect"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -12,7 +14,8 @@ type parser struct {
 	flow          chan token
 	tree          *TomlTree
 	tokensBuffer  []token
-	currentGroup string
+	currentGroup  []string
+	seenGroupKeys []string
 }
 
 type parserStateFn func(*parser) parserStateFn
@@ -86,9 +89,15 @@ func parseGroup(p *parser) parserStateFn {
 	if key.typ != tokenKeyGroup {
 		panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
 	}
+	for _, item := range p.seenGroupKeys {
+		if item == key.val {
+			panic("duplicated tables")
+		}
+	}
+	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
 	p.tree.createSubTree(key.val)
 	p.assume(tokenRightBracket)
-	p.currentGroup = key.val
+	p.currentGroup = strings.Split(key.val, ".")
 	return parseStart(p)
 }
 
@@ -96,11 +105,17 @@ func parseAssign(p *parser) parserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
 	value := parseRvalue(p)
-	final_key := key.val
+	var final_key []string
-	if p.currentGroup != "" {
+	if len(p.currentGroup) > 0 {
-		final_key = p.currentGroup + "." + key.val
+		final_key = p.currentGroup
+	} else {
+		final_key = make([]string, 0)
 	}
-	p.tree.Set(final_key, value)
+	final_key = append(final_key, key.val)
+	if p.tree.GetPath(final_key) != nil {
+		panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
+	}
+	p.tree.SetPath(final_key, value)
 	return parseStart(p)
 }
 
@@ -137,6 +152,8 @@ func parseRvalue(p *parser) interface{} {
 		return val
 	case tokenLeftBracket:
 		return parseArray(p)
+	case tokenError:
+		panic(tok.val)
 	}
 
 	panic("never reached")
@@ -146,6 +163,7 @@ func parseRvalue(p *parser) interface{} {
 
 func parseArray(p *parser) []interface{} {
 	array := make([]interface{}, 0)
+	arrayType := reflect.TypeOf(nil)
 	for {
 		follow := p.peek()
 		if follow == nil || follow.typ == tokenEOF {
@@ -156,14 +174,18 @@ func parseArray(p *parser) []interface{} {
 			return array
 		}
 		val := parseRvalue(p)
+		if arrayType == nil {
+			arrayType = reflect.TypeOf(val)
+		}
+		if reflect.TypeOf(val) != arrayType {
+			panic("mixed types in array")
+		}
 		array = append(array, val)
 		follow = p.peek()
 		if follow == nil {
 			panic("unterminated array")
 		}
 		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
-			fmt.Println(follow.typ)
-			fmt.Println(follow.val)
 			panic("missing comma")
 		}
 		if follow.typ == tokenComma {
@@ -179,7 +201,8 @@ func parse(flow chan token) *TomlTree {
 		flow:          flow,
 		tree:          &result,
 		tokensBuffer:  make([]token, 0),
-		currentGroup: "",
+		currentGroup:  make([]string, 0),
+		seenGroupKeys: make([]string, 0),
 	}
 	parser.run()
 	return parser.tree

parser_test.go

@@ -12,12 +12,18 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 		return
 	}
 	for k, v := range ref {
-		if fmt.Sprintf("%v", tree.Get(k)) != fmt.Sprintf("%v", v) {
+		node := tree.Get(k)
-			t.Log("was expecting", v, "at", k, "but got", tree.Get(k))
+		switch node.(type) {
+		case *TomlTree:
+			assertTree(t, node.(*TomlTree), err, v.(map[string]interface{}))
+		default:
+			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
+				t.Log("was expecting", v, "at", k, "but got", node)
 				t.Error()
 			}
 		}
 	}
+}
 
 func TestCreateSubTree(t *testing.T) {
 	tree := make(TomlTree)
@@ -142,6 +148,26 @@ func TestArrayNested(t *testing.T) {
 	})
 }
 
+func TestNestedEmptyArrays(t *testing.T) {
+	tree, err := Load("a = [[[]]]")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": [][][]interface{}{[][]interface{}{[]interface{}{}}},
+	})
+}
+
+
+func TestArrayMixedTypes(t *testing.T) {
+	_, err := Load("a = [42, 16.0]")
+	if err.Error() != "mixed types in array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [42, \"hello\"]")
+	if err.Error() != "mixed types in array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestArrayNestedStrings(t *testing.T) {
 	tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -184,6 +210,53 @@ func TestArrayWithExtraCommaComment(t *testing.T) {
 	})
 }
 
+func TestDuplicateGroups(t *testing.T) {
+	_, err := Load("[foo]\na=2\n[foo]b=3")
+	if err.Error() != "duplicated tables" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestDuplicateKeys(t *testing.T) {
+	_, err := Load("foo = 2\nfoo = 3")
+	if err.Error() != "the following key was defined twice: foo" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestEmptyIntermediateTable(t *testing.T) {
+	_, err := Load("[foo..bar]")
+	if err.Error() != "empty intermediate table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestImplicitDeclarationBefore(t *testing.T) {
+	tree, err := Load("[a.b.c]\nanswer = 42\n[a]\nbetter = 43")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": map[string]interface{}{
+			"b": map[string]interface{}{
+				"c": map[string]interface{}{
+					"answer": int64(42),
+				},
+			},
+			"better": int64(43),
+		},
+	})
+}
+
+func TestFloatsWithoutLeadingZeros(t *testing.T) {
+	_, err := Load("a = .42")
+	if err.Error() != "cannot start float with a dot" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = -.42")
+	if err.Error() != "cannot start float with a dot" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestMissingFile(t *testing.T) {
 	_, err := LoadFile("foo.toml")
 	if err.Error() != "open foo.toml: no such file or directory" {

test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Run basic go unit tests
+go test -v ./...
+result=$?
+
+# Run example-based toml tests
+cd test_program && ./go-test.sh
+result="$(( result || $? ))"
+
+exit $result

test_program/go-test.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+go get github.com/BurntSushi/toml-test # install test suite
+go get github.com/BurntSushi/toml/toml-test-go # install my parser
+go build -o test_program_bin github.com/pelletier/go-toml/test_program
+
+toml_test_wrapper() {
+    ret=0
+    if hash toml-test 2>/dev/null; then # test availability in $PATH
+        toml-test "$@"
+        ret=$?
+    else
+        p="$HOME/gopath/bin/toml-test" # try in Travi's place
+        if [ -f "$p" ]; then
+            "$p" "$@"
+            ret=$?
+        else
+            "$GOPATH/bin/toml-test" "$@"
+            ret=$?
+        fi
+    fi
+}
+
+toml_test_wrapper ./test_program_bin | tee test_out
+ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])"
+exit $ret

test_program/test_program.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"github.com/pelletier/go-toml"
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+)
+
+func main() {
+	bytes, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		os.Exit(2)
+	}
+	tree, err := toml.Load(string(bytes))
+	if err != nil {
+		os.Exit(1)
+	}
+
+	typedTree := translate((map[string]interface{})(*tree))
+
+	if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
+		log.Fatalf("Error encoding JSON: %s", err)
+	}
+
+	os.Exit(0)
+}
+
+func translate(tomlData interface{}) interface{} {
+
+	switch orig := tomlData.(type) {
+	case map[string]interface{}:
+		typed := make(map[string]interface{}, len(orig))
+		for k, v := range orig {
+			typed[k] = translate(v)
+		}
+		return typed
+	case *toml.TomlTree:
+		return translate((map[string]interface{})(*orig))
+	case []map[string]interface{}:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v).(map[string]interface{})
+		}
+		return typed
+	case []interface{}:
+		typed := make([]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v)
+		}
+		return tag("array", typed)
+	case time.Time:
+		return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
+	case bool:
+		return tag("bool", fmt.Sprintf("%v", orig))
+	case int64:
+		return tag("integer", fmt.Sprintf("%d", orig))
+	case float64:
+		return tag("float", fmt.Sprintf("%v", orig))
+	case string:
+		return tag("string", orig)
+	}
+
+	panic(fmt.Sprintf("Unknown type: %T", tomlData))
+}
+
+func tag(typeName string, data interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"type":  typeName,
+		"value": data,
+	}
+}

toml.go

@@ -15,6 +15,18 @@ import (
 // This is the result of the parsing of a TOML file.
 type TomlTree map[string]interface{}
 
+// Has returns a boolean indicating if the toplevel tree contains the given
+// key.
+func (t *TomlTree) Has(key string) bool {
+	mp := (map[string]interface{})(*t)
+	for k, _ := range mp {
+		if k == key {
+			return true
+		}
+	}
+	return false
+}
+
 // Keys returns the keys of the toplevel tree.
 // Warning: this is a costly operation.
 func (t *TomlTree) Keys() []string {
@@ -30,8 +42,11 @@ func (t *TomlTree) Keys() []string {
 // Key is a dot-separated path (e.g. a.b.c).
 // Returns nil if the path does not exist in the tree.
 func (t *TomlTree) Get(key string) interface{} {
+	return t.GetPath(strings.Split(key, "."))
+}
+
+func (t *TomlTree) GetPath(keys []string) interface{} {
 	subtree := t
-	keys := strings.Split(key, ".")
 	for _, intermediate_key := range keys[:len(keys)-1] {
 		_, exists := (*subtree)[intermediate_key]
 		if !exists {
@@ -55,8 +70,11 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
 // Key is a dot-separated path (e.g. a.b.c).
 // Creates all necessary intermediates trees, if needed.
 func (t *TomlTree) Set(key string, value interface{}) {
+	t.SetPath(strings.Split(key, "."), value)
+}
+
+func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	subtree := t
-	keys := strings.Split(key, ".")
 	for _, intermediate_key := range keys[:len(keys)-1] {
 		_, exists := (*subtree)[intermediate_key]
 		if !exists {
@@ -76,6 +94,9 @@ func (t *TomlTree) Set(key string, value interface{}) {
 func (t *TomlTree) createSubTree(key string) {
 	subtree := t
 	for _, intermediate_key := range strings.Split(key, ".") {
+		if intermediate_key == "" {
+			panic("empty intermediate table")
+		}
 		_, exists := (*subtree)[intermediate_key]
 		if !exists {
 			var new_tree TomlTree = make(TomlTree)
@@ -109,6 +130,5 @@ func LoadFile(path string) (tree *TomlTree, err error) {
 		s := string(buff)
 		tree, err = Load(s)
 	}
-
 	return
 }