From 19eb8cf036efc1af977cc37c8b591211bc500d53 Mon Sep 17 00:00:00 2001 From: x-hgg-x <39058530+x-hgg-x@users.noreply.github.com> Date: Thu, 7 May 2020 05:13:18 +0200 Subject: [PATCH] Fix various quoted keys bugs (#400) Fixes #396 #397 #398 #399 --- keysparsing.go | 3 +-- lexer_test.go | 6 +++--- marshal.go | 12 +++++------ marshal_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++- query/match.go | 21 +++++++++--------- query/tokens.go | 8 +++---- token.go | 9 +++----- tomltree_write.go | 2 +- 8 files changed, 82 insertions(+), 33 deletions(-) diff --git a/keysparsing.go b/keysparsing.go index e923bc4..e091500 100644 --- a/keysparsing.go +++ b/keysparsing.go @@ -5,7 +5,6 @@ package toml import ( "errors" "fmt" - "unicode" ) // Convert the bare key group string to an array. @@ -109,5 +108,5 @@ func parseKey(key string) ([]string, error) { } func isValidBareChar(r rune) bool { - return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r) + return isAlphanumeric(r) || r == '-' || isDigit(r) } diff --git a/lexer_test.go b/lexer_test.go index 0608ba8..a54cbb6 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -105,9 +105,9 @@ func TestBasicKeyWithUppercaseMix(t *testing.T) { } func TestBasicKeyWithInternationalCharacters(t *testing.T) { - testFlow(t, "héllÖ", []token{ - {Position{1, 1}, tokenKey, "héllÖ"}, - {Position{1, 6}, tokenEOF, ""}, + testFlow(t, "'héllÖ'", []token{ + {Position{1, 1}, tokenKey, "'héllÖ'"}, + {Position{1, 8}, tokenEOF, ""}, }) } diff --git a/marshal.go b/marshal.go index 230a432..a24ff10 100644 --- a/marshal.go +++ b/marshal.go @@ -436,7 +436,7 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er if tree, ok := val.(*Tree); ok && mtypef.Anonymous && !opts.nameFromTag && !e.promoteAnon { e.appendTree(tval, tree) } else { - tval.SetWithOptions(opts.name, SetOptions{ + tval.SetPathWithOptions([]string{opts.name}, SetOptions{ Comment: opts.comment, Commented: opts.commented, Multiline: opts.multiline, @@ -481,7 +481,7 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er } tval.SetPath([]string{keyStr}, val) } else { - tval.Set(key.String(), val) + tval.SetPath([]string{key.String()}, val) } } } @@ -754,17 +754,17 @@ func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree, mval1 *reflect.V found := false if tval != nil { for _, key := range keysToTry { - exists := tval.Has(key) + exists := tval.HasPath([]string{key}) if !exists { continue } d.visitor.push(key) - val := tval.Get(key) + val := tval.GetPath([]string{key}) fval := mval.Field(i) mvalf, err := d.valueFromToml(mtypef.Type, val, &fval) if err != nil { - return mval, formatError(err, tval.GetPosition(key)) + return mval, formatError(err, tval.GetPositionPath([]string{key})) } mval.Field(i).Set(mvalf) found = true @@ -838,7 +838,7 @@ func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree, mval1 *reflect.V val := tval.GetPath([]string{key}) mvalf, err := d.valueFromToml(mtype.Elem(), val, nil) if err != nil { - return mval, formatError(err, tval.GetPosition(key)) + return mval, formatError(err, tval.GetPositionPath([]string{key})) } mval.SetMapIndex(reflect.ValueOf(key).Convert(mtype.Key()), mvalf) d.visitor.pop() diff --git a/marshal_test.go b/marshal_test.go index 2757ad5..1022000 100644 --- a/marshal_test.go +++ b/marshal_test.go @@ -287,6 +287,59 @@ func TestBasicUnmarshal(t *testing.T) { } } +type quotedKeyMarshalTestStruct struct { + String string `toml:"Z.string-àéù"` + Float float64 `toml:"Yfloat-𝟘"` + Sub basicMarshalTestSubStruct `toml:"Xsubdoc-àéù"` + SubList []basicMarshalTestSubStruct `toml:"W.sublist-𝟘"` +} + +var quotedKeyMarshalTestData = quotedKeyMarshalTestStruct{ + String: "Hello", + Float: 3.5, + Sub: basicMarshalTestSubStruct{"One"}, + SubList: []basicMarshalTestSubStruct{{"Two"}, {"Three"}}, +} + +var quotedKeyMarshalTestToml = []byte(`"Yfloat-𝟘" = 3.5 +"Z.string-àéù" = "Hello" + +[["W.sublist-𝟘"]] + String2 = "Two" + +[["W.sublist-𝟘"]] + String2 = "Three" + +["Xsubdoc-àéù"] + String2 = "One" +`) + +func TestBasicMarshalQuotedKey(t *testing.T) { + result, err := Marshal(quotedKeyMarshalTestData) + if err != nil { + t.Fatal(err) + } + expected := quotedKeyMarshalTestToml + if !bytes.Equal(result, expected) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result) + } +} + +func TestBasicUnmarshalQuotedKey(t *testing.T) { + tree, err := LoadBytes(quotedKeyMarshalTestToml) + if err != nil { + t.Fatal(err) + } + + var q quotedKeyMarshalTestStruct + tree.Unmarshal(&q) + fmt.Println(q) + + if !reflect.DeepEqual(quotedKeyMarshalTestData, q) { + t.Errorf("Bad unmarshal: expected\n-----\n%v\n-----\ngot\n-----\n%v\n-----\n", quotedKeyMarshalTestData, q) + } +} + type testDoc struct { Title string `toml:"title"` BasicLists testDocBasicLists `toml:"basic_lists"` @@ -2070,7 +2123,6 @@ func TestUnmarshalCamelCaseKey(t *testing.T) { } } - func TestUnmarshalNegativeUint(t *testing.T) { type check struct{ U uint } diff --git a/query/match.go b/query/match.go index d7bb15a..d2207ef 100644 --- a/query/match.go +++ b/query/match.go @@ -2,6 +2,7 @@ package query import ( "fmt" + "github.com/pelletier/go-toml" ) @@ -44,16 +45,16 @@ func newMatchKeyFn(name string) *matchKeyFn { func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { if array, ok := node.([]*toml.Tree); ok { for _, tree := range array { - item := tree.Get(f.Name) + item := tree.GetPath([]string{f.Name}) if item != nil { - ctx.lastPosition = tree.GetPosition(f.Name) + ctx.lastPosition = tree.GetPositionPath([]string{f.Name}) f.next.call(item, ctx) } } } else if tree, ok := node.(*toml.Tree); ok { - item := tree.Get(f.Name) + item := tree.GetPath([]string{f.Name}) if item != nil { - ctx.lastPosition = tree.GetPosition(f.Name) + ctx.lastPosition = tree.GetPositionPath([]string{f.Name}) f.next.call(item, ctx) } } @@ -129,8 +130,8 @@ func newMatchAnyFn() *matchAnyFn { func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { if tree, ok := node.(*toml.Tree); ok { for _, k := range tree.Keys() { - v := tree.Get(k) - ctx.lastPosition = tree.GetPosition(k) + v := tree.GetPath([]string{k}) + ctx.lastPosition = tree.GetPositionPath([]string{k}) f.next.call(v, ctx) } } @@ -168,8 +169,8 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { var visit func(tree *toml.Tree) visit = func(tree *toml.Tree) { for _, k := range tree.Keys() { - v := tree.Get(k) - ctx.lastPosition = tree.GetPosition(k) + v := tree.GetPath([]string{k}) + ctx.lastPosition = tree.GetPositionPath([]string{k}) f.next.call(v, ctx) switch node := v.(type) { case *toml.Tree: @@ -207,9 +208,9 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) { switch castNode := node.(type) { case *toml.Tree: for _, k := range castNode.Keys() { - v := castNode.Get(k) + v := castNode.GetPath([]string{k}) if fn(v) { - ctx.lastPosition = castNode.GetPosition(k) + ctx.lastPosition = castNode.GetPositionPath([]string{k}) f.next.call(v, ctx) } } diff --git a/query/tokens.go b/query/tokens.go index 9ae579d..098c856 100644 --- a/query/tokens.go +++ b/query/tokens.go @@ -2,9 +2,9 @@ package query import ( "fmt" - "github.com/pelletier/go-toml" "strconv" - "unicode" + + "github.com/pelletier/go-toml" ) // Define tokens @@ -92,11 +92,11 @@ func isSpace(r rune) bool { } func isAlphanumeric(r rune) bool { - return unicode.IsLetter(r) || r == '_' + return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_' } func isDigit(r rune) bool { - return unicode.IsNumber(r) + return '0' <= r && r <= '9' } func isHexDigit(r rune) bool { diff --git a/token.go b/token.go index 36a3fc8..6af4ec4 100644 --- a/token.go +++ b/token.go @@ -1,9 +1,6 @@ package toml -import ( - "fmt" - "unicode" -) +import "fmt" // Define tokens type tokenType int @@ -112,7 +109,7 @@ func isSpace(r rune) bool { } func isAlphanumeric(r rune) bool { - return unicode.IsLetter(r) || r == '_' + return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_' } func isKeyChar(r rune) bool { @@ -127,7 +124,7 @@ func isKeyStartChar(r rune) bool { } func isDigit(r rune) bool { - return unicode.IsNumber(r) + return '0' <= r && r <= '9' } func isHexDigit(r rune) bool { diff --git a/tomltree_write.go b/tomltree_write.go index 43d195b..c762d4c 100644 --- a/tomltree_write.go +++ b/tomltree_write.go @@ -338,7 +338,7 @@ func (t *Tree) writeToOrdered(w io.Writer, indent, keyspace string, bytesCount i k := node.key v := t.values[k] - combinedKey := k + combinedKey := quoteKeyIfNeeded(k) if keyspace != "" { combinedKey = keyspace + "." + combinedKey }