diff --git a/keysparsing.go b/keysparsing.go index 284db64..e923bc4 100644 --- a/keysparsing.go +++ b/keysparsing.go @@ -3,79 +3,107 @@ package toml import ( - "bytes" "errors" "fmt" "unicode" ) // Convert the bare key group string to an array. -// The input supports double quotation to allow "." inside the key name, +// The input supports double quotation and single quotation, // but escape sequences are not supported. Lexers must unescape them beforehand. func parseKey(key string) ([]string, error) { - groups := []string{} - var buffer bytes.Buffer - inQuotes := false - wasInQuotes := false - ignoreSpace := true - expectDot := false + runes := []rune(key) + var groups []string - for _, char := range key { - if ignoreSpace { - if char == ' ' { - continue - } - ignoreSpace = false + if len(key) == 0 { + return nil, errors.New("empty key") + } + + idx := 0 + for idx < len(runes) { + for ; idx < len(runes) && isSpace(runes[idx]); idx++ { + // skip leading whitespace } - switch char { - case '"': - if inQuotes { - groups = append(groups, buffer.String()) - buffer.Reset() - wasInQuotes = true - } - inQuotes = !inQuotes - expectDot = false - case '.': - if inQuotes { - buffer.WriteRune(char) - } else { - if !wasInQuotes { - if buffer.Len() == 0 { - return nil, errors.New("empty table key") + if idx >= len(runes) { + break + } + r := runes[idx] + if isValidBareChar(r) { + // parse bare key + startIdx := idx + endIdx := -1 + idx++ + for idx < len(runes) { + r = runes[idx] + if isValidBareChar(r) { + idx++ + } else if r == '.' { + endIdx = idx + break + } else if isSpace(r) { + endIdx = idx + for ; idx < len(runes) && isSpace(runes[idx]); idx++ { + // skip trailing whitespace } - groups = append(groups, buffer.String()) - buffer.Reset() + if idx < len(runes) && runes[idx] != '.' { + return nil, fmt.Errorf("invalid key character after whitespace: %c", runes[idx]) + } + break + } else { + return nil, fmt.Errorf("invalid bare key character: %c", r) } - ignoreSpace = true - expectDot = false - wasInQuotes = false } - case ' ': - if inQuotes { - buffer.WriteRune(char) - } else { - expectDot = true + if endIdx == -1 { + endIdx = idx } - default: - if !inQuotes && !isValidBareChar(char) { - return nil, fmt.Errorf("invalid bare character: %c", char) + groups = append(groups, string(runes[startIdx:endIdx])) + } else if r == '\'' { + // parse single quoted key + idx++ + startIdx := idx + for { + if idx >= len(runes) { + return nil, fmt.Errorf("unclosed single-quoted key") + } + r = runes[idx] + if r == '\'' { + groups = append(groups, string(runes[startIdx:idx])) + idx++ + break + } + idx++ } - if !inQuotes && expectDot { - return nil, errors.New("what?") + } else if r == '"' { + // parse double quoted key + idx++ + startIdx := idx + for { + if idx >= len(runes) { + return nil, fmt.Errorf("unclosed double-quoted key") + } + r = runes[idx] + if r == '"' { + groups = append(groups, string(runes[startIdx:idx])) + idx++ + break + } + idx++ } - buffer.WriteRune(char) - expectDot = false + } else if r == '.' { + idx++ + if idx >= len(runes) { + return nil, fmt.Errorf("unexpected end of key") + } + r = runes[idx] + if !isValidBareChar(r) && r != '\'' && r != '"' && r != ' ' { + return nil, fmt.Errorf("expecting key part after dot") + } + } else { + return nil, fmt.Errorf("invalid key character: %c", r) } } - if inQuotes { - return nil, errors.New("mismatched quotes") - } - if buffer.Len() > 0 { - groups = append(groups, buffer.String()) - } if len(groups) == 0 { - return nil, errors.New("empty key") + return nil, fmt.Errorf("empty key") } return groups, nil } diff --git a/keysparsing_test.go b/keysparsing_test.go index 84cb826..5a7530b 100644 --- a/keysparsing_test.go +++ b/keysparsing_test.go @@ -44,7 +44,23 @@ func TestDottedKeyBasic(t *testing.T) { } func TestBaseKeyPound(t *testing.T) { - testError(t, "hello#world", "invalid bare character: #") + testError(t, "hello#world", "invalid bare key character: #") +} + +func TestUnclosedSingleQuotedKey(t *testing.T) { + testError(t, "'", "unclosed single-quoted key") +} + +func TestUnclosedDoubleQuotedKey(t *testing.T) { + testError(t, "\"", "unclosed double-quoted key") +} + +func TestInvalidStartKeyCharacter(t *testing.T) { + testError(t, "/", "invalid key character: /") +} + +func TestInvalidSpaceInKey(t *testing.T) { + testError(t, "invalid key", "invalid key character after whitespace: k") } func TestQuotedKeys(t *testing.T) { @@ -57,7 +73,7 @@ func TestQuotedKeys(t *testing.T) { } func TestEmptyKey(t *testing.T) { - testError(t, "", "empty key") - testError(t, " ", "empty key") + testError(t, ``, "empty key") + testError(t, ` `, "empty key") testResult(t, `""`, []string{""}) } diff --git a/lexer.go b/lexer.go index d11de42..6254d39 100644 --- a/lexer.go +++ b/lexer.go @@ -309,7 +309,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { if err != nil { return l.errorf(err.Error()) } - growingString += str + growingString += "\"" + str + "\"" l.next() continue } else if r == '\'' { @@ -318,13 +318,15 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { if err != nil { return l.errorf(err.Error()) } - growingString += str + growingString += "'" + str + "'" l.next() continue } else if r == '\n' { return l.errorf("keys cannot contain new lines") } else if isSpace(r) { break + } else if r == '.' { + // skip } else if !isValidBareChar(r) { return l.errorf("keys cannot contain %c character", r) } diff --git a/lexer_test.go b/lexer_test.go index cb49130..313b83c 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -690,7 +690,7 @@ func TestKeyGroupArray(t *testing.T) { func TestQuotedKey(t *testing.T) { testFlow(t, "\"a b\" = 42", []token{ - {Position{1, 1}, tokenKey, "a b"}, + {Position{1, 1}, tokenKey, "\"a b\""}, {Position{1, 7}, tokenEqual, "="}, {Position{1, 9}, tokenInteger, "42"}, {Position{1, 11}, tokenEOF, ""}, diff --git a/parser.go b/parser.go index 776353b..a7498e4 100644 --- a/parser.go +++ b/parser.go @@ -77,8 +77,10 @@ func (p *tomlParser) parseStart() tomlParserStateFn { return p.parseAssign case tokenEOF: return nil + case tokenError: + p.raiseError(tok, "parsing error: %s", tok.String()) default: - p.raiseError(tok, "unexpected token") + p.raiseError(tok, "unexpected token %s", tok.typ) } return nil } @@ -165,6 +167,11 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { key := p.getToken() p.assume(tokenEqual) + parsedKey, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid key: %s", err.Error()) + } + value := p.parseRvalue() var tableKey []string if len(p.currentTable) > 0 { @@ -173,6 +180,9 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { tableKey = []string{} } + prefixKey := parsedKey[0 : len(parsedKey)-1] + tableKey = append(tableKey, prefixKey...) + // find the table to assign, looking out for arrays of tables var targetNode *Tree switch node := p.tree.GetPath(tableKey).(type) { @@ -180,17 +190,19 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { targetNode = node[len(node)-1] case *Tree: targetNode = node + case nil: + // create intermediate + if err := p.tree.createSubTree(tableKey, key.Position); err != nil { + p.raiseError(key, "could not create intermediate group: %s", err) + } + targetNode = p.tree.GetPath(tableKey).(*Tree) default: p.raiseError(key, "Unknown table type for path: %s", strings.Join(tableKey, ".")) } // assign value to the found table - keyVals := []string{key.val} - if len(keyVals) != 1 { - p.raiseError(key, "Invalid key") - } - keyVal := keyVals[0] + keyVal := parsedKey[len(parsedKey)-1] localKey := []string{keyVal} finalKey := append(tableKey, keyVal) if targetNode.GetPath(localKey) != nil { diff --git a/parser_test.go b/parser_test.go index 476fd4e..7834575 100644 --- a/parser_test.go +++ b/parser_test.go @@ -79,7 +79,7 @@ zyx = 42`) if err == nil { t.Error("Error should have been returned.") } - if err.Error() != "(1, 4): unexpected token" { + if err.Error() != "(1, 4): parsing error: keys cannot contain ] character" { t.Error("Bad error message:", err.Error()) } } @@ -581,7 +581,7 @@ func TestDuplicateKeys(t *testing.T) { func TestEmptyIntermediateTable(t *testing.T) { _, err := Load("[foo..bar]") - if err.Error() != "(1, 2): invalid table array key: empty table key" { + if err.Error() != "(1, 2): invalid table array key: expecting key part after dot" { t.Error("Bad error message:", err.Error()) } } @@ -908,3 +908,32 @@ func TestMapKeyIsNum(t *testing.T) { t.Error("should be passed") } } + +func TestDottedKeys(t *testing.T) { + tree, err := Load(` +name = "Orange" +physical.color = "orange" +physical.shape = "round" +site."google.com" = true`) + + assertTree(t, tree, err, map[string]interface{}{ + "name": "Orange", + "physical": map[string]interface{}{ + "color": "orange", + "shape": "round", + }, + "site": map[string]interface{}{ + "google.com": true, + }, + }) +} + +func TestInvalidDottedKeyEmptyGroup(t *testing.T) { + _, err := Load(`a..b = true`) + if err == nil { + t.Fatal("should return an error") + } + if err.Error() != "(1, 1): invalid key: expecting key part after dot" { + t.Fatalf("invalid error message: %s", err) + } +}