Support dotted-keys (#260)

Implement dotted keys as sequence of bare and quoted keys. Introduced in
TOML 0.5.0.
Fixes #230

keysparsing.go

@@ -3,79 +3,107 @@
 package toml
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"unicode"
 )
 
 // Convert the bare key group string to an array.
-// The input supports double quotation to allow "." inside the key name,
+// The input supports double quotation and single quotation,
 // but escape sequences are not supported. Lexers must unescape them beforehand.
 func parseKey(key string) ([]string, error) {
-	groups := []string{}
+	runes := []rune(key)
-	var buffer bytes.Buffer
+	var groups []string
-	inQuotes := false
-	wasInQuotes := false
-	ignoreSpace := true
-	expectDot := false
 
-	for _, char := range key {
+	if len(key) == 0 {
-		if ignoreSpace {
+		return nil, errors.New("empty key")
-			if char == ' ' {
+	}
-				continue
+
-			}
+	idx := 0
-			ignoreSpace = false
+	for idx < len(runes) {
+		for ; idx < len(runes) && isSpace(runes[idx]); idx++ {
+			// skip leading whitespace
 		}
-		switch char {
+		if idx >= len(runes) {
-		case '"':
+			break
-			if inQuotes {
+		}
-				groups = append(groups, buffer.String())
+		r := runes[idx]
-				buffer.Reset()
+		if isValidBareChar(r) {
-				wasInQuotes = true
+			// parse bare key
-			}
+			startIdx := idx
-			inQuotes = !inQuotes
+			endIdx := -1
-			expectDot = false
+			idx++
-		case '.':
+			for idx < len(runes) {
-			if inQuotes {
+				r = runes[idx]
-				buffer.WriteRune(char)
+				if isValidBareChar(r) {
-			} else {
+					idx++
-				if !wasInQuotes {
+				} else if r == '.' {
-					if buffer.Len() == 0 {
+					endIdx = idx
-						return nil, errors.New("empty table key")
+					break
+				} else if isSpace(r) {
+					endIdx = idx
+					for ; idx < len(runes) && isSpace(runes[idx]); idx++ {
+						// skip trailing whitespace
 					}
-					groups = append(groups, buffer.String())
+					if idx < len(runes) && runes[idx] != '.' {
-					buffer.Reset()
+						return nil, fmt.Errorf("invalid key character after whitespace: %c", runes[idx])
+					}
+					break
+				} else {
+					return nil, fmt.Errorf("invalid bare key character: %c", r)
 				}
-				ignoreSpace = true
-				expectDot = false
-				wasInQuotes = false
 			}
-		case ' ':
+			if endIdx == -1 {
-			if inQuotes {
+				endIdx = idx
-				buffer.WriteRune(char)
-			} else {
-				expectDot = true
 			}
-		default:
+			groups = append(groups, string(runes[startIdx:endIdx]))
-			if !inQuotes && !isValidBareChar(char) {
+		} else if r == '\'' {
-				return nil, fmt.Errorf("invalid bare character: %c", char)
+			// parse single quoted key
+			idx++
+			startIdx := idx
+			for {
+				if idx >= len(runes) {
+					return nil, fmt.Errorf("unclosed single-quoted key")
+				}
+				r = runes[idx]
+				if r == '\'' {
+					groups = append(groups, string(runes[startIdx:idx]))
+					idx++
+					break
+				}
+				idx++
 			}
-			if !inQuotes && expectDot {
+		} else if r == '"' {
-				return nil, errors.New("what?")
+			// parse double quoted key
+			idx++
+			startIdx := idx
+			for {
+				if idx >= len(runes) {
+					return nil, fmt.Errorf("unclosed double-quoted key")
+				}
+				r = runes[idx]
+				if r == '"' {
+					groups = append(groups, string(runes[startIdx:idx]))
+					idx++
+					break
+				}
+				idx++
 			}
-			buffer.WriteRune(char)
+		} else if r == '.' {
-			expectDot = false
+			idx++
+			if idx >= len(runes) {
+				return nil, fmt.Errorf("unexpected end of key")
+			}
+			r = runes[idx]
+			if !isValidBareChar(r) && r != '\'' && r != '"' && r != ' ' {
+				return nil, fmt.Errorf("expecting key part after dot")
+			}
+		} else {
+			return nil, fmt.Errorf("invalid key character: %c", r)
 		}
 	}
-	if inQuotes {
-		return nil, errors.New("mismatched quotes")
-	}
-	if buffer.Len() > 0 {
-		groups = append(groups, buffer.String())
-	}
 	if len(groups) == 0 {
-		return nil, errors.New("empty key")
+		return nil, fmt.Errorf("empty key")
 	}
 	return groups, nil
 }

keysparsing_test.go

@@ -44,7 +44,23 @@ func TestDottedKeyBasic(t *testing.T) {
 }
 
 func TestBaseKeyPound(t *testing.T) {
-	testError(t, "hello#world", "invalid bare character: #")
+	testError(t, "hello#world", "invalid bare key character: #")
+}
+
+func TestUnclosedSingleQuotedKey(t *testing.T) {
+	testError(t, "'", "unclosed single-quoted key")
+}
+
+func TestUnclosedDoubleQuotedKey(t *testing.T) {
+	testError(t, "\"", "unclosed double-quoted key")
+}
+
+func TestInvalidStartKeyCharacter(t *testing.T) {
+	testError(t, "/", "invalid key character: /")
+}
+
+func TestInvalidSpaceInKey(t *testing.T) {
+	testError(t, "invalid key", "invalid key character after whitespace: k")
 }
 
 func TestQuotedKeys(t *testing.T) {
@@ -57,7 +73,7 @@ func TestQuotedKeys(t *testing.T) {
 }
 
 func TestEmptyKey(t *testing.T) {
-	testError(t, "", "empty key")
+	testError(t, ``, "empty key")
-	testError(t, " ", "empty key")
+	testError(t, ` `, "empty key")
 	testResult(t, `""`, []string{""})
 }

lexer.go

@@ -309,7 +309,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 			if err != nil {
 				return l.errorf(err.Error())
 			}
-			growingString += str
+			growingString += "\"" + str + "\""
 			l.next()
 			continue
 		} else if r == '\'' {
@@ -318,13 +318,15 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 			if err != nil {
 				return l.errorf(err.Error())
 			}
-			growingString += str
+			growingString += "'" + str + "'"
 			l.next()
 			continue
 		} else if r == '\n' {
 			return l.errorf("keys cannot contain new lines")
 		} else if isSpace(r) {
 			break
+		} else if r == '.' {
+			// skip
 		} else if !isValidBareChar(r) {
 			return l.errorf("keys cannot contain %c character", r)
 		}

lexer_test.go

@@ -690,7 +690,7 @@ func TestKeyGroupArray(t *testing.T) {
 
 func TestQuotedKey(t *testing.T) {
 	testFlow(t, "\"a b\" = 42", []token{
-		{Position{1, 1}, tokenKey, "a b"},
+		{Position{1, 1}, tokenKey, "\"a b\""},
 		{Position{1, 7}, tokenEqual, "="},
 		{Position{1, 9}, tokenInteger, "42"},
 		{Position{1, 11}, tokenEOF, ""},

parser.go

@@ -77,8 +77,10 @@ func (p *tomlParser) parseStart() tomlParserStateFn {
 		return p.parseAssign
 	case tokenEOF:
 		return nil
+	case tokenError:
+		p.raiseError(tok, "parsing error: %s", tok.String())
 	default:
-		p.raiseError(tok, "unexpected token")
+		p.raiseError(tok, "unexpected token %s", tok.typ)
 	}
 	return nil
 }
@@ -165,6 +167,11 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
 
+	parsedKey, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid key: %s", err.Error())
+	}
+
 	value := p.parseRvalue()
 	var tableKey []string
 	if len(p.currentTable) > 0 {
@@ -173,6 +180,9 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 		tableKey = []string{}
 	}
 
+	prefixKey := parsedKey[0 : len(parsedKey)-1]
+	tableKey = append(tableKey, prefixKey...)
+
 	// find the table to assign, looking out for arrays of tables
 	var targetNode *Tree
 	switch node := p.tree.GetPath(tableKey).(type) {
@@ -180,17 +190,19 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 		targetNode = node[len(node)-1]
 	case *Tree:
 		targetNode = node
+	case nil:
+		// create intermediate
+		if err := p.tree.createSubTree(tableKey, key.Position); err != nil {
+			p.raiseError(key, "could not create intermediate group: %s", err)
+		}
+		targetNode = p.tree.GetPath(tableKey).(*Tree)
 	default:
 		p.raiseError(key, "Unknown table type for path: %s",
 			strings.Join(tableKey, "."))
 	}
 
 	// assign value to the found table
-	keyVals := []string{key.val}
+	keyVal := parsedKey[len(parsedKey)-1]
-	if len(keyVals) != 1 {
-		p.raiseError(key, "Invalid key")
-	}
-	keyVal := keyVals[0]
 	localKey := []string{keyVal}
 	finalKey := append(tableKey, keyVal)
 	if targetNode.GetPath(localKey) != nil {

parser_test.go

@@ -79,7 +79,7 @@ zyx = 42`)
 	if err == nil {
 		t.Error("Error should have been returned.")
 	}
-	if err.Error() != "(1, 4): unexpected token" {
+	if err.Error() != "(1, 4): parsing error: keys cannot contain ] character" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -581,7 +581,7 @@ func TestDuplicateKeys(t *testing.T) {
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "(1, 2): invalid table array key: empty table key" {
+	if err.Error() != "(1, 2): invalid table array key: expecting key part after dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -908,3 +908,32 @@ func TestMapKeyIsNum(t *testing.T) {
 		t.Error("should be passed")
 	}
 }
+
+func TestDottedKeys(t *testing.T) {
+	tree, err := Load(`
+name = "Orange"
+physical.color = "orange"
+physical.shape = "round"
+site."google.com" = true`)
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"name": "Orange",
+		"physical": map[string]interface{}{
+			"color": "orange",
+			"shape": "round",
+		},
+		"site": map[string]interface{}{
+			"google.com": true,
+		},
+	})
+}
+
+func TestInvalidDottedKeyEmptyGroup(t *testing.T) {
+	_, err := Load(`a..b = true`)
+	if err == nil {
+		t.Fatal("should return an error")
+	}
+	if err.Error() != "(1, 1): invalid key: expecting key part after dot" {
+		t.Fatalf("invalid error message: %s", err)
+	}
+}