Merge pull request #37 from pelletier/pelletier/better_keys_parsing

Update keys parsing

keysparsing.go

@@ -0,0 +1,56 @@
+// Parsing keys handling both bare and quoted keys.
+
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"unicode"
+)
+
+func parseKey(key string) ([]string, error) {
+	groups := []string{}
+	var buffer bytes.Buffer
+	inQuotes := false
+	escapeNext := false
+	for _, char := range key {
+		if escapeNext {
+			buffer.WriteRune(char)
+			escapeNext = false
+			continue
+		}
+		switch char {
+		case '\\':
+			escapeNext = true
+			continue
+		case '"':
+			inQuotes = !inQuotes
+		case '.':
+			if inQuotes {
+				buffer.WriteRune(char)
+			} else {
+				groups = append(groups, buffer.String())
+				buffer.Reset()
+			}
+		default:
+			if !inQuotes && !isValidBareChar(char) {
+				return nil, fmt.Errorf("invalid bare character: %c", char)
+			}
+			buffer.WriteRune(char)
+		}
+	}
+	if inQuotes {
+		return nil, fmt.Errorf("mismatched quotes")
+	}
+	if escapeNext {
+		return nil, fmt.Errorf("unfinished escape sequence")
+	}
+	if buffer.Len() > 0 {
+		groups = append(groups, buffer.String())
+	}
+	return groups, nil
+}
+
+func isValidBareChar(r rune) bool {
+	return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r)
+}

keysparsing_test.go

@@ -0,0 +1,44 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+func testResult(t *testing.T, key string, expected []string) {
+	parsed, err := parseKey(key)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if len(expected) != len(parsed) {
+		t.Fatal("Expected length", len(expected), "but", len(parsed), "parsed")
+	}
+	for index, expectedKey := range expected {
+		if expectedKey != parsed[index] {
+			t.Fatal("Expected", expectedKey, "at index", index, "but found", parsed[index])
+		}
+	}
+}
+
+func testError(t *testing.T, key string, expectedError string) {
+	_, err := parseKey(key)
+	if fmt.Sprintf("%s", err) != expectedError {
+		t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
+	}
+}
+
+func TestBareKeyBasic(t *testing.T) {
+	testResult(t, "test", []string{"test"})
+}
+
+func TestBareKeyDotted(t *testing.T) {
+	testResult(t, "this.is.a.key", []string{"this", "is", "a", "key"})
+}
+
+func TestDottedKeyBasic(t *testing.T) {
+	testResult(t, "\"a.dotted.key\"", []string{"a.dotted.key"})
+}
+
+func TestBaseKeyPound(t *testing.T) {
+	testError(t, "hello#world", "invalid bare character: #")
+}

lexer.go

@@ -253,9 +253,16 @@ func (l *tomlLexer) lexComma() tomlLexStateFn {
 
 func (l *tomlLexer) lexKey() tomlLexStateFn {
 	l.ignore()
-	for r := l.next(); isKeyChar(r); r = l.next() {
+	inQuotes := false
-		if r == '#' {
+	for r := l.next(); isKeyChar(r) || r == '\n'; r = l.next() {
-			return l.errorf("keys cannot contain # character")
+		if r == '"' {
+			inQuotes = !inQuotes
+		} else if r == '\n' {
+			return l.errorf("keys cannot contain new lines")
+		} else if isSpace(r) && !inQuotes {
+			break
+		} else if !isValidBareChar(r) && !inQuotes {
+			return l.errorf("keys cannot contain %c character", r)
 		}
 	}
 	l.backup()

lexer_test.go

@@ -124,10 +124,7 @@ func TestKeyWithSharpAndEqual(t *testing.T) {
 
 func TestKeyWithSymbolsAndEqual(t *testing.T) {
 	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
-		token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+		token{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
-		token{Position{1, 38}, tokenEqual, "="},
-		token{Position{1, 40}, tokenInteger, "5"},
-		token{Position{1, 41}, tokenEOF, ""},
 	})
 }
 
@@ -549,3 +546,18 @@ func TestKeyGroupArray(t *testing.T) {
 		token{Position{1, 8}, tokenEOF, ""},
 	})
 }
+
+func TestQuotedKey(t *testing.T) {
+	testFlow(t, "\"a b\" = 42", []token{
+		token{Position{1, 1}, tokenKey, "\"a b\""},
+		token{Position{1, 7}, tokenEqual, "="},
+		token{Position{1, 9}, tokenInteger, "42"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestKeyNewline(t *testing.T) {
+	testFlow(t, "a\n= 4", []token{
+		token{Position{1, 1}, tokenError, "keys cannot contain new lines"},
+	})
+}

parser.go

@@ -98,7 +98,10 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn {
 	}
 
 	// get or create group array element at the indicated part in the path
-	keys := strings.Split(key.val, ".")
+	keys, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid group array key: %s", err)
+	}
 	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
 	destTree := p.tree.GetPath(keys)
 	var array []*TomlTree
@@ -153,7 +156,10 @@ func (p *tomlParser) parseGroup() tomlParserStateFn {
 	}
 
 	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
-	keys := strings.Split(key.val, ".")
+	keys, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid group array key: %s", err)
+	}
 	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
 		p.raiseError(key, "%s", err)
 	}
@@ -186,13 +192,21 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	}
 
 	// assign value to the found group
-	localKey := []string{key.val}
+	keyVals, err := parseKey(key.val)
-	finalKey := append(groupKey, key.val)
+	if err != nil {
+		p.raiseError(key, "%s", err)
+	}
+	if len(keyVals) != 1 {
+		p.raiseError(key, "Invalid key")
+	}
+	keyVal := keyVals[0]
+	localKey := []string{keyVal}
+	finalKey := append(groupKey, keyVal)
 	if targetNode.GetPath(localKey) != nil {
 		p.raiseError(key, "The following key was defined twice: %s",
 			strings.Join(finalKey, "."))
 	}
-	targetNode.values[key.val] = &tomlValue{value, key.Position}
+	targetNode.values[keyVal] = &tomlValue{value, key.Position}
 	return p.parseStart
 }
 
@@ -222,7 +236,7 @@ func (p *tomlParser) parseRvalue() interface{} {
 		}
 		return val
 	case tokenDate:
-		val, err := time.Parse(time.RFC3339Nano, tok.val)
+		val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}

parser_test.go

@@ -51,12 +51,10 @@ func TestSimpleKV(t *testing.T) {
 	})
 }
 
-// NOTE: from the BurntSushi test suite
+func TestNumberInKey(t *testing.T) {
-// NOTE: this test is pure evil due to the embedded '.'
+	tree, err := Load("hello2 = 42")
-func TestSpecialKV(t *testing.T) {
-	tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
 	assertTree(t, tree, err, map[string]interface{}{
-		"~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
+		"hello2": int64(42),
 	})
 }
 
@@ -91,14 +89,14 @@ func TestSimpleDate(t *testing.T) {
 func TestDateOffset(t *testing.T) {
 	tree, err := Load("a = 1979-05-27T00:32:00-07:00")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("PDT", -7*60*60)),
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)),
 	})
 }
 
 func TestDateNano(t *testing.T) {
 	tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("PDT", -7*60*60)),
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)),
 	})
 }
 
@@ -109,6 +107,13 @@ func TestSimpleString(t *testing.T) {
 	})
 }
 
+func TestSpaceKey(t *testing.T) {
+	tree, err := Load("\"a b\" = \"hello world\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a b": "hello world",
+	})
+}
+
 func TestStringEscapables(t *testing.T) {
 	tree, err := Load("a = \"a \\n b\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -461,3 +466,10 @@ func TestNestedTreePosition(t *testing.T) {
 			"foo.bar.b": Position{3, 1},
 		})
 }
+
+func TestInvalidGroupArray(t *testing.T) {
+	_, err := Load("[key#group]\nanswer = 42")
+	if err == nil {
+		t.Error("Should error")
+	}
+}

token.go

@@ -120,7 +120,7 @@ func isKeyChar(r rune) bool {
 	// Keys start with the first character that isn't whitespace or [ and end
 	// with the last non-whitespace character before the equals sign. Keys
 	// cannot contain a # character."
-	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
+	return !(r == '\r' || r == '\n' || r == eof || r == '=')
 }
 
 func isKeyStartChar(r rune) bool {

toml.go

@@ -65,7 +65,11 @@ func (t *TomlTree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
-	return t.GetPath(strings.Split(key, "."))
+	comps, err := parseKey(key)
+	if err != nil {
+		return nil
+	}
+	return t.GetPath(comps)
 }
 
 // GetPath returns the element in the tree indicated by 'keys'.