From 9f36448571799dfcb1814ad0816dc2d1d95e1dfa Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 14 Jul 2015 16:33:33 -0700 Subject: [PATCH 1/6] Basic keys parsing --- keysparsing.go | 55 +++++++++++++++++++++++++++++++++++++++++++++ keysparsing_test.go | 44 ++++++++++++++++++++++++++++++++++++ parser.go | 10 +++++++-- parser_test.go | 7 ++++++ 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 keysparsing.go create mode 100644 keysparsing_test.go diff --git a/keysparsing.go b/keysparsing.go new file mode 100644 index 0000000..6bdb0b8 --- /dev/null +++ b/keysparsing.go @@ -0,0 +1,55 @@ +// Parsing keys handling both bare and quoted keys. + +package toml + +import ( + "bytes" + "fmt" +) + +func parseKey(key string) ([]string, error) { + groups := []string{} + var buffer bytes.Buffer + inQuotes := false + escapeNext := false + for _, char := range key { + if escapeNext { + buffer.WriteRune(char) + escapeNext = false + continue + } + switch char { + case '\\': + escapeNext = true + continue + case '"': + inQuotes = !inQuotes + case '.': + if inQuotes { + buffer.WriteRune(char) + } else { + groups = append(groups, buffer.String()) + buffer.Reset() + } + default: + if !inQuotes && !isValidBareChar(char) { + return nil, fmt.Errorf("invalid bare character: %c", char) + } + buffer.WriteRune(char) + } + } + if inQuotes { + return nil, fmt.Errorf("mismatched quotes") + } + if escapeNext { + return nil, fmt.Errorf("unfinished escape sequence") + } + if buffer.Len() > 0 { + groups = append(groups, buffer.String()) + } + return groups, nil +} + +func isValidBareChar(r rune) bool { + return isAlphanumeric(r) || r == '-' +} diff --git a/keysparsing_test.go b/keysparsing_test.go new file mode 100644 index 0000000..2d5379f --- /dev/null +++ b/keysparsing_test.go @@ -0,0 +1,44 @@ +package toml + +import ( + "fmt" + "testing" +) + +func testResult(t *testing.T, key string, expected []string) { + parsed, err := parseKey(key) + if err != nil { + t.Fatal("Unexpected error:", err) + } + if len(expected) != len(parsed) { + t.Fatal("Expected length", len(expected), "but", len(parsed), "parsed") + } + for index, expectedKey := range expected { + if expectedKey != parsed[index] { + t.Fatal("Expected", expectedKey, "at index", index, "but found", parsed[index]) + } + } +} + +func testError(t *testing.T, key string, expectedError string) { + _, err := parseKey(key) + if fmt.Sprintf("%s", err) != expectedError { + t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err) + } +} + +func TestBareKeyBasic(t *testing.T) { + testResult(t, "test", []string{"test"}) +} + +func TestBareKeyDotted(t *testing.T) { + testResult(t, "this.is.a.key", []string{"this", "is", "a", "key"}) +} + +func TestDottedKeyBasic(t *testing.T) { + testResult(t, "\"a.dotted.key\"", []string{"a.dotted.key"}) +} + +func TestBaseKeyPound(t *testing.T) { + testError(t, "hello#world", "invalid bare character: #") +} diff --git a/parser.go b/parser.go index e83647d..e4d1083 100644 --- a/parser.go +++ b/parser.go @@ -98,7 +98,10 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn { } // get or create group array element at the indicated part in the path - keys := strings.Split(key.val, ".") + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid group array key: %s", err) + } p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries destTree := p.tree.GetPath(keys) var array []*TomlTree @@ -153,7 +156,10 @@ func (p *tomlParser) parseGroup() tomlParserStateFn { } p.seenGroupKeys = append(p.seenGroupKeys, key.val) - keys := strings.Split(key.val, ".") + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid group array key: %s", err) + } if err := p.tree.createSubTree(keys, startToken.Position); err != nil { p.raiseError(key, "%s", err) } diff --git a/parser_test.go b/parser_test.go index dc051ea..d33ddde 100644 --- a/parser_test.go +++ b/parser_test.go @@ -461,3 +461,10 @@ func TestNestedTreePosition(t *testing.T) { "foo.bar.b": Position{3, 1}, }) } + +func TestInvalidGroupArray(t *testing.T) { + _, err := Load("[key#group]\nanswer = 42") + if err == nil { + t.Error("Should error") + } +} From 16a681db2a3606f8477a0d588109bd7de13ed836 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 14 Jul 2015 19:56:28 -0700 Subject: [PATCH 2/6] Allow numbers in keys parsing --- keysparsing.go | 3 ++- lexer.go | 9 +++++++-- lexer_test.go | 14 ++++++++++---- parser.go | 14 +++++++++++--- parser_test.go | 15 ++++++++++----- token.go | 2 +- 6 files changed, 41 insertions(+), 16 deletions(-) diff --git a/keysparsing.go b/keysparsing.go index 6bdb0b8..528f1de 100644 --- a/keysparsing.go +++ b/keysparsing.go @@ -5,6 +5,7 @@ package toml import ( "bytes" "fmt" + "unicode" ) func parseKey(key string) ([]string, error) { @@ -51,5 +52,5 @@ func parseKey(key string) ([]string, error) { } func isValidBareChar(r rune) bool { - return isAlphanumeric(r) || r == '-' + return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r) } diff --git a/lexer.go b/lexer.go index e6c3566..f162565 100644 --- a/lexer.go +++ b/lexer.go @@ -253,9 +253,14 @@ func (l *tomlLexer) lexComma() tomlLexStateFn { func (l *tomlLexer) lexKey() tomlLexStateFn { l.ignore() + inQuotes := false for r := l.next(); isKeyChar(r); r = l.next() { - if r == '#' { - return l.errorf("keys cannot contain # character") + if r == '"' { + inQuotes = !inQuotes + } else if isSpace(r) && !inQuotes { + break + } else if !isValidBareChar(r) && !inQuotes { + return l.errorf("keys cannot contain %c character", r) } } l.backup() diff --git a/lexer_test.go b/lexer_test.go index 683d7f7..6b18a57 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -124,10 +124,7 @@ func TestKeyWithSharpAndEqual(t *testing.T) { func TestKeyWithSymbolsAndEqual(t *testing.T) { testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ - token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"}, - token{Position{1, 38}, tokenEqual, "="}, - token{Position{1, 40}, tokenInteger, "5"}, - token{Position{1, 41}, tokenEOF, ""}, + token{Position{1, 1}, tokenError, "keys cannot contain ~ character"}, }) } @@ -549,3 +546,12 @@ func TestKeyGroupArray(t *testing.T) { token{Position{1, 8}, tokenEOF, ""}, }) } + +func TestQuotedKey(t *testing.T) { + testFlow(t, "\"a b\" = 42", []token{ + token{Position{1, 1}, tokenKey, "\"a b\""}, + token{Position{1, 7}, tokenEqual, "="}, + token{Position{1, 9}, tokenInteger, "42"}, + token{Position{1, 11}, tokenEOF, ""}, + }) +} diff --git a/parser.go b/parser.go index e4d1083..8911261 100644 --- a/parser.go +++ b/parser.go @@ -192,13 +192,21 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { } // assign value to the found group - localKey := []string{key.val} - finalKey := append(groupKey, key.val) + keyVals, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "%s", err) + } + if len(keyVals) != 1 { + p.raiseError(key, "Invalid key") + } + keyVal := keyVals[0] + localKey := []string{keyVal} + finalKey := append(groupKey, keyVal) if targetNode.GetPath(localKey) != nil { p.raiseError(key, "The following key was defined twice: %s", strings.Join(finalKey, ".")) } - targetNode.values[key.val] = &tomlValue{value, key.Position} + targetNode.values[keyVal] = &tomlValue{value, key.Position} return p.parseStart } diff --git a/parser_test.go b/parser_test.go index d33ddde..7a88055 100644 --- a/parser_test.go +++ b/parser_test.go @@ -51,12 +51,10 @@ func TestSimpleKV(t *testing.T) { }) } -// NOTE: from the BurntSushi test suite -// NOTE: this test is pure evil due to the embedded '.' -func TestSpecialKV(t *testing.T) { - tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1") +func TestNumberInKey(t *testing.T) { + tree, err := Load("hello2 = 42") assertTree(t, tree, err, map[string]interface{}{ - "~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1), + "hello2": int64(42), }) } @@ -109,6 +107,13 @@ func TestSimpleString(t *testing.T) { }) } +func TestSpaceKey(t *testing.T) { + tree, err := Load("\"a b\" = \"hello world\"") + assertTree(t, tree, err, map[string]interface{}{ + "a b": "hello world", + }) +} + func TestStringEscapables(t *testing.T) { tree, err := Load("a = \"a \\n b\"") assertTree(t, tree, err, map[string]interface{}{ diff --git a/token.go b/token.go index 6c0bcc6..f0fbac9 100644 --- a/token.go +++ b/token.go @@ -120,7 +120,7 @@ func isKeyChar(r rune) bool { // Keys start with the first character that isn't whitespace or [ and end // with the last non-whitespace character before the equals sign. Keys // cannot contain a # character." - return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=') + return !(r == '\r' || r == '\n' || r == eof || r == '=') } func isKeyStartChar(r rune) bool { From 41a8959f1483973d6e8c77b22d0f060bd1a0605c Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 14 Jul 2015 20:07:43 -0700 Subject: [PATCH 3/6] Reject new lines in keys --- lexer.go | 4 +++- lexer_test.go | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lexer.go b/lexer.go index f162565..7cb789b 100644 --- a/lexer.go +++ b/lexer.go @@ -254,9 +254,11 @@ func (l *tomlLexer) lexComma() tomlLexStateFn { func (l *tomlLexer) lexKey() tomlLexStateFn { l.ignore() inQuotes := false - for r := l.next(); isKeyChar(r); r = l.next() { + for r := l.next(); isKeyChar(r) || r == '\n'; r = l.next() { if r == '"' { inQuotes = !inQuotes + } else if r == '\n' { + return l.errorf("keys cannot contain new lines") } else if isSpace(r) && !inQuotes { break } else if !isValidBareChar(r) && !inQuotes { diff --git a/lexer_test.go b/lexer_test.go index 6b18a57..d1cd130 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -555,3 +555,9 @@ func TestQuotedKey(t *testing.T) { token{Position{1, 11}, tokenEOF, ""}, }) } + +func TestKeyNewline(t *testing.T) { + testFlow(t, "a\n= 4", []token{ + token{Position{1, 1}, tokenError, "keys cannot contain new lines"}, + }) +} From 209315c2af84b0cce997a5c907e603c5fab36125 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Tue, 14 Jul 2015 20:15:02 -0700 Subject: [PATCH 4/6] Fixes #35: Retrieve dotted keys --- toml.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/toml.go b/toml.go index 2896da7..bf35cd3 100644 --- a/toml.go +++ b/toml.go @@ -65,7 +65,11 @@ func (t *TomlTree) Get(key string) interface{} { if key == "" { return t } - return t.GetPath(strings.Split(key, ".")) + comps, err := parseKey(key) + if err != nil { + return nil + } + return t.GetPath(comps) } // GetPath returns the element in the tree indicated by 'keys'. From 36e11971902bed4e1a3236020ca93861a3760460 Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 15 Jul 2015 08:17:28 -0700 Subject: [PATCH 5/6] Test datetimes differently --- parser_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parser_test.go b/parser_test.go index 7a88055..8d55ccd 100644 --- a/parser_test.go +++ b/parser_test.go @@ -87,16 +87,18 @@ func TestSimpleDate(t *testing.T) { } func TestDateOffset(t *testing.T) { + target, _ := time.Parse(time.RFC3339Nano, "1979-05-27T00:32:00-07:00") tree, err := Load("a = 1979-05-27T00:32:00-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("PDT", -7*60*60)), + "a": target, }) } func TestDateNano(t *testing.T) { + target, _ := time.Parse(time.RFC3339Nano, "1979-05-27T00:32:00.999999999-07:00") tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("PDT", -7*60*60)), + "a": target, }) } From 9defd66d3c068b96396e0f24997291fc5b4f2dda Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 15 Jul 2015 10:58:08 -0700 Subject: [PATCH 6/6] Parse datetimes in UTC --- parser.go | 2 +- parser_test.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/parser.go b/parser.go index 8911261..e550967 100644 --- a/parser.go +++ b/parser.go @@ -236,7 +236,7 @@ func (p *tomlParser) parseRvalue() interface{} { } return val case tokenDate: - val, err := time.Parse(time.RFC3339Nano, tok.val) + val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC) if err != nil { p.raiseError(tok, "%s", err) } diff --git a/parser_test.go b/parser_test.go index 8d55ccd..2998913 100644 --- a/parser_test.go +++ b/parser_test.go @@ -87,18 +87,16 @@ func TestSimpleDate(t *testing.T) { } func TestDateOffset(t *testing.T) { - target, _ := time.Parse(time.RFC3339Nano, "1979-05-27T00:32:00-07:00") tree, err := Load("a = 1979-05-27T00:32:00-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": target, + "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)), }) } func TestDateNano(t *testing.T) { - target, _ := time.Parse(time.RFC3339Nano, "1979-05-27T00:32:00.999999999-07:00") tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": target, + "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)), }) }