decoder: strict mode (#512)
This commit is contained in:
Thomas Pelletier
@@ -7,6 +7,7 @@ package imported_tests
|
||||
// marked as skipped until we figure out if that's something we want in v2.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
@@ -1955,66 +1956,80 @@ String2="2"`
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func decoder(doc string) *toml.Decoder {
|
||||
return toml.NewDecoder(bytes.NewReader([]byte(doc)))
|
||||
}
|
||||
|
||||
func strictDecoder(doc string) *toml.Decoder {
|
||||
d := decoder(doc)
|
||||
d.SetStrict(true)
|
||||
return d
|
||||
}
|
||||
|
||||
func TestDecoderStrict(t *testing.T) {
|
||||
t.Skip()
|
||||
// input := `
|
||||
//[decoded]
|
||||
// key = ""
|
||||
//
|
||||
//[undecoded]
|
||||
// key = ""
|
||||
//
|
||||
// [undecoded.inner]
|
||||
// key = ""
|
||||
//
|
||||
// [[undecoded.array]]
|
||||
// key = ""
|
||||
//
|
||||
// [[undecoded.array]]
|
||||
// key = ""
|
||||
//
|
||||
//`
|
||||
// var doc struct {
|
||||
// Decoded struct {
|
||||
// Key string
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// expected := `undecoded keys: ["undecoded.array.0.key" "undecoded.array.1.key" "undecoded.inner.key" "undecoded.key"]`
|
||||
//
|
||||
// err := NewDecoder(bytes.NewReader([]byte(input))).Strict(true).Decode(&doc)
|
||||
// if err == nil {
|
||||
// t.Error("expected error, got none")
|
||||
// } else if err.Error() != expected {
|
||||
// t.Errorf("expect err: %s, got: %s", expected, err.Error())
|
||||
// }
|
||||
//
|
||||
// if err := NewDecoder(bytes.NewReader([]byte(input))).Decode(&doc); err != nil {
|
||||
// t.Errorf("unexpected err: %s", err)
|
||||
// }
|
||||
//
|
||||
// var m map[string]interface{}
|
||||
// if err := NewDecoder(bytes.NewReader([]byte(input))).Decode(&m); err != nil {
|
||||
// t.Errorf("unexpected err: %s", err)
|
||||
// }
|
||||
input := `
|
||||
[decoded]
|
||||
key = ""
|
||||
|
||||
[undecoded]
|
||||
key = ""
|
||||
|
||||
[undecoded.inner]
|
||||
key = ""
|
||||
|
||||
[[undecoded.array]]
|
||||
key = ""
|
||||
|
||||
[[undecoded.array]]
|
||||
key = ""
|
||||
|
||||
`
|
||||
var doc struct {
|
||||
Decoded struct {
|
||||
Key string
|
||||
}
|
||||
}
|
||||
|
||||
err := strictDecoder(input).Decode(&doc)
|
||||
require.Error(t, err)
|
||||
require.IsType(t, &toml.StrictMissingError{}, err)
|
||||
se := err.(*toml.StrictMissingError)
|
||||
|
||||
keys := []toml.Key{}
|
||||
|
||||
for _, e := range se.Errors {
|
||||
keys = append(keys, e.Key())
|
||||
}
|
||||
|
||||
expectedKeys := []toml.Key{
|
||||
{"undecoded"},
|
||||
{"undecoded", "inner"},
|
||||
{"undecoded", "array"},
|
||||
{"undecoded", "array"},
|
||||
}
|
||||
|
||||
require.Equal(t, expectedKeys, keys)
|
||||
|
||||
err = decoder(input).Decode(&doc)
|
||||
require.NoError(t, err)
|
||||
|
||||
var m map[string]interface{}
|
||||
err = decoder(input).Decode(&m)
|
||||
}
|
||||
|
||||
func TestDecoderStrictValid(t *testing.T) {
|
||||
t.Skip()
|
||||
// input := `
|
||||
//[decoded]
|
||||
// key = ""
|
||||
//`
|
||||
// var doc struct {
|
||||
// Decoded struct {
|
||||
// Key string
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// err := NewDecoder(bytes.NewReader([]byte(input))).Strict(true).Decode(&doc)
|
||||
// if err != nil {
|
||||
// t.Fatal("unexpected error:", err)
|
||||
// }
|
||||
input := `
|
||||
[decoded]
|
||||
key = ""
|
||||
`
|
||||
var doc struct {
|
||||
Decoded struct {
|
||||
Key string
|
||||
}
|
||||
}
|
||||
|
||||
err := strictDecoder(input).Decode(&doc)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
type docUnmarshalTOML struct {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package tracker
|
||||
|
||||
import (
|
||||
"github.com/pelletier/go-toml/v2/internal/ast"
|
||||
)
|
||||
|
||||
// KeyTracker is a tracker that keeps track of the current Key as the AST is
|
||||
// walked.
|
||||
type KeyTracker struct {
|
||||
k []string
|
||||
}
|
||||
|
||||
// UpdateTable sets the state of the tracker with the AST table node.
|
||||
func (t *KeyTracker) UpdateTable(node ast.Node) {
|
||||
t.reset()
|
||||
t.Push(node)
|
||||
}
|
||||
|
||||
// UpdateArrayTable sets the state of the tracker with the AST array table node.
|
||||
func (t *KeyTracker) UpdateArrayTable(node ast.Node) {
|
||||
t.reset()
|
||||
t.Push(node)
|
||||
}
|
||||
|
||||
// Push the given key on the stack.
|
||||
func (t *KeyTracker) Push(node ast.Node) {
|
||||
it := node.Key()
|
||||
for it.Next() {
|
||||
t.k = append(t.k, string(it.Node().Data))
|
||||
}
|
||||
}
|
||||
|
||||
// Pop key from stack.
|
||||
func (t *KeyTracker) Pop(node ast.Node) {
|
||||
it := node.Key()
|
||||
for it.Next() {
|
||||
t.k = t.k[:len(t.k)-1]
|
||||
}
|
||||
}
|
||||
|
||||
// Key returns the current key
|
||||
func (t *KeyTracker) Key() []string {
|
||||
k := make([]string, len(t.k))
|
||||
copy(k, t.k)
|
||||
return k
|
||||
}
|
||||
|
||||
func (t *KeyTracker) reset() {
|
||||
t.k = t.k[:0]
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package tracker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pelletier/go-toml/v2/internal/ast"
|
||||
)
|
||||
|
||||
type keyKind uint8
|
||||
|
||||
const (
|
||||
invalidKind keyKind = iota
|
||||
valueKind
|
||||
tableKind
|
||||
arrayTableKind
|
||||
)
|
||||
|
||||
func (k keyKind) String() string {
|
||||
switch k {
|
||||
case invalidKind:
|
||||
return "invalid"
|
||||
case valueKind:
|
||||
return "value"
|
||||
case tableKind:
|
||||
return "table"
|
||||
case arrayTableKind:
|
||||
return "array table"
|
||||
}
|
||||
panic("missing keyKind string mapping")
|
||||
}
|
||||
|
||||
// SeenTracker tracks which keys have been seen with which TOML type to flag duplicates
|
||||
// and mismatches according to the spec.
|
||||
type SeenTracker struct {
|
||||
root *info
|
||||
current *info
|
||||
}
|
||||
|
||||
type info struct {
|
||||
parent *info
|
||||
kind keyKind
|
||||
children map[string]*info
|
||||
explicit bool
|
||||
}
|
||||
|
||||
func (i *info) Clear() {
|
||||
i.children = nil
|
||||
}
|
||||
|
||||
func (i *info) Has(k string) (*info, bool) {
|
||||
c, ok := i.children[k]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
func (i *info) SetKind(kind keyKind) {
|
||||
i.kind = kind
|
||||
}
|
||||
|
||||
func (i *info) CreateTable(k string, explicit bool) *info {
|
||||
return i.createChild(k, tableKind, explicit)
|
||||
}
|
||||
|
||||
func (i *info) CreateArrayTable(k string, explicit bool) *info {
|
||||
return i.createChild(k, arrayTableKind, explicit)
|
||||
}
|
||||
|
||||
func (i *info) createChild(k string, kind keyKind, explicit bool) *info {
|
||||
if i.children == nil {
|
||||
i.children = make(map[string]*info, 1)
|
||||
}
|
||||
|
||||
x := &info{
|
||||
parent: i,
|
||||
kind: kind,
|
||||
explicit: explicit,
|
||||
}
|
||||
i.children[k] = x
|
||||
return x
|
||||
}
|
||||
|
||||
// CheckExpression takes a top-level node and checks that it does not contain keys
|
||||
// that have been seen in previous calls, and validates that types are consistent.
|
||||
func (s *SeenTracker) CheckExpression(node ast.Node) error {
|
||||
if s.root == nil {
|
||||
s.root = &info{
|
||||
kind: tableKind,
|
||||
}
|
||||
s.current = s.root
|
||||
}
|
||||
switch node.Kind {
|
||||
case ast.KeyValue:
|
||||
return s.checkKeyValue(s.current, node)
|
||||
case ast.Table:
|
||||
return s.checkTable(node)
|
||||
case ast.ArrayTable:
|
||||
return s.checkArrayTable(node)
|
||||
default:
|
||||
panic(fmt.Errorf("this should not be a top level node type: %s", node.Kind))
|
||||
}
|
||||
|
||||
}
|
||||
func (s *SeenTracker) checkTable(node ast.Node) error {
|
||||
s.current = s.root
|
||||
|
||||
it := node.Key()
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
if !it.Node().Next().Valid() {
|
||||
break
|
||||
}
|
||||
|
||||
k := string(it.Node().Data)
|
||||
child, found := s.current.Has(k)
|
||||
if !found {
|
||||
child = s.current.CreateTable(k, false)
|
||||
}
|
||||
s.current = child
|
||||
}
|
||||
|
||||
// handle the last part of the key
|
||||
k := string(it.Node().Data)
|
||||
|
||||
i, found := s.current.Has(k)
|
||||
if found {
|
||||
if i.kind != tableKind {
|
||||
return fmt.Errorf("key %s should be a table", k)
|
||||
}
|
||||
if i.explicit {
|
||||
return fmt.Errorf("table %s already exists", k)
|
||||
}
|
||||
i.explicit = true
|
||||
s.current = i
|
||||
} else {
|
||||
s.current = s.current.CreateTable(k, true)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SeenTracker) checkArrayTable(node ast.Node) error {
|
||||
s.current = s.root
|
||||
|
||||
it := node.Key()
|
||||
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
if !it.Node().Next().Valid() {
|
||||
break
|
||||
}
|
||||
|
||||
k := string(it.Node().Data)
|
||||
child, found := s.current.Has(k)
|
||||
if !found {
|
||||
child = s.current.CreateTable(k, false)
|
||||
}
|
||||
s.current = child
|
||||
}
|
||||
|
||||
// handle the last part of the key
|
||||
k := string(it.Node().Data)
|
||||
|
||||
info, found := s.current.Has(k)
|
||||
if found {
|
||||
if info.kind != arrayTableKind {
|
||||
return fmt.Errorf("key %s already exists but is not an array table", k)
|
||||
}
|
||||
info.Clear()
|
||||
} else {
|
||||
info = s.current.CreateArrayTable(k, true)
|
||||
}
|
||||
|
||||
s.current = info
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SeenTracker) checkKeyValue(context *info, node ast.Node) error {
|
||||
it := node.Key()
|
||||
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
k := string(it.Node().Data)
|
||||
child, found := context.Has(k)
|
||||
if found {
|
||||
if child.kind != tableKind {
|
||||
return fmt.Errorf("expected %s to be a table, not a %s", k, child.kind)
|
||||
}
|
||||
} else {
|
||||
child = context.CreateTable(k, false)
|
||||
}
|
||||
context = child
|
||||
}
|
||||
|
||||
if node.Value().Kind == ast.InlineTable {
|
||||
context.SetKind(tableKind)
|
||||
} else {
|
||||
context.SetKind(valueKind)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,200 +1 @@
|
||||
package tracker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pelletier/go-toml/v2/internal/ast"
|
||||
)
|
||||
|
||||
type keyKind uint8
|
||||
|
||||
const (
|
||||
invalidKind keyKind = iota
|
||||
valueKind
|
||||
tableKind
|
||||
arrayTableKind
|
||||
)
|
||||
|
||||
func (k keyKind) String() string {
|
||||
switch k {
|
||||
case invalidKind:
|
||||
return "invalid"
|
||||
case valueKind:
|
||||
return "value"
|
||||
case tableKind:
|
||||
return "table"
|
||||
case arrayTableKind:
|
||||
return "array table"
|
||||
}
|
||||
panic("missing keyKind string mapping")
|
||||
}
|
||||
|
||||
// Tracks which keys have been seen with which TOML type to flag duplicates
|
||||
// and mismatches according to the spec.
|
||||
type Seen struct {
|
||||
root *info
|
||||
current *info
|
||||
}
|
||||
|
||||
type info struct {
|
||||
parent *info
|
||||
kind keyKind
|
||||
children map[string]*info
|
||||
explicit bool
|
||||
}
|
||||
|
||||
func (i *info) Clear() {
|
||||
i.children = nil
|
||||
}
|
||||
|
||||
func (i *info) Has(k string) (*info, bool) {
|
||||
c, ok := i.children[k]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
func (i *info) SetKind(kind keyKind) {
|
||||
i.kind = kind
|
||||
}
|
||||
|
||||
func (i *info) CreateTable(k string, explicit bool) *info {
|
||||
return i.createChild(k, tableKind, explicit)
|
||||
}
|
||||
|
||||
func (i *info) CreateArrayTable(k string, explicit bool) *info {
|
||||
return i.createChild(k, arrayTableKind, explicit)
|
||||
}
|
||||
|
||||
func (i *info) createChild(k string, kind keyKind, explicit bool) *info {
|
||||
if i.children == nil {
|
||||
i.children = make(map[string]*info, 1)
|
||||
}
|
||||
|
||||
x := &info{
|
||||
parent: i,
|
||||
kind: kind,
|
||||
explicit: explicit,
|
||||
}
|
||||
i.children[k] = x
|
||||
return x
|
||||
}
|
||||
|
||||
// CheckExpression takes a top-level node and checks that it does not contain keys
|
||||
// that have been seen in previous calls, and validates that types are consistent.
|
||||
func (s *Seen) CheckExpression(node ast.Node) error {
|
||||
if s.root == nil {
|
||||
s.root = &info{
|
||||
kind: tableKind,
|
||||
}
|
||||
s.current = s.root
|
||||
}
|
||||
switch node.Kind {
|
||||
case ast.KeyValue:
|
||||
return s.checkKeyValue(s.current, node)
|
||||
case ast.Table:
|
||||
return s.checkTable(node)
|
||||
case ast.ArrayTable:
|
||||
return s.checkArrayTable(node)
|
||||
default:
|
||||
panic(fmt.Errorf("this should not be a top level node type: %s", node.Kind))
|
||||
}
|
||||
|
||||
}
|
||||
func (s *Seen) checkTable(node ast.Node) error {
|
||||
s.current = s.root
|
||||
|
||||
it := node.Key()
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
if !it.Node().Next().Valid() {
|
||||
break
|
||||
}
|
||||
|
||||
k := string(it.Node().Data)
|
||||
child, found := s.current.Has(k)
|
||||
if !found {
|
||||
child = s.current.CreateTable(k, false)
|
||||
}
|
||||
s.current = child
|
||||
}
|
||||
|
||||
// handle the last part of the key
|
||||
k := string(it.Node().Data)
|
||||
|
||||
i, found := s.current.Has(k)
|
||||
if found {
|
||||
if i.kind != tableKind {
|
||||
return fmt.Errorf("key %s should be a table", k)
|
||||
}
|
||||
if i.explicit {
|
||||
return fmt.Errorf("table %s already exists", k)
|
||||
}
|
||||
i.explicit = true
|
||||
s.current = i
|
||||
} else {
|
||||
s.current = s.current.CreateTable(k, true)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seen) checkArrayTable(node ast.Node) error {
|
||||
s.current = s.root
|
||||
|
||||
it := node.Key()
|
||||
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
if !it.Node().Next().Valid() {
|
||||
break
|
||||
}
|
||||
|
||||
k := string(it.Node().Data)
|
||||
child, found := s.current.Has(k)
|
||||
if !found {
|
||||
child = s.current.CreateTable(k, false)
|
||||
}
|
||||
s.current = child
|
||||
}
|
||||
|
||||
// handle the last part of the key
|
||||
k := string(it.Node().Data)
|
||||
|
||||
info, found := s.current.Has(k)
|
||||
if found {
|
||||
if info.kind != arrayTableKind {
|
||||
return fmt.Errorf("key %s already exists but is not an array table", k)
|
||||
}
|
||||
info.Clear()
|
||||
} else {
|
||||
info = s.current.CreateArrayTable(k, true)
|
||||
}
|
||||
|
||||
s.current = info
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seen) checkKeyValue(context *info, node ast.Node) error {
|
||||
it := node.Key()
|
||||
|
||||
// handle the first parts of the key, excluding the last one
|
||||
for it.Next() {
|
||||
k := string(it.Node().Data)
|
||||
child, found := context.Has(k)
|
||||
if found {
|
||||
if child.kind != tableKind {
|
||||
return fmt.Errorf("expected %s to be a table, not a %s", k, child.kind)
|
||||
}
|
||||
} else {
|
||||
child = context.CreateTable(k, false)
|
||||
}
|
||||
context = child
|
||||
}
|
||||
|
||||
if node.Value().Kind == ast.InlineTable {
|
||||
context.SetKind(tableKind)
|
||||
} else {
|
||||
context.SetKind(valueKind)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -33,3 +33,27 @@ func SubsliceOffset(data []byte, subslice []byte) int {
|
||||
|
||||
return intoffset
|
||||
}
|
||||
|
||||
func BytesRange(start []byte, end []byte) []byte {
|
||||
if start == nil || end == nil {
|
||||
panic("cannot call BytesRange with nil")
|
||||
}
|
||||
startp := (*reflect.SliceHeader)(unsafe.Pointer(&start))
|
||||
endp := (*reflect.SliceHeader)(unsafe.Pointer(&end))
|
||||
|
||||
if startp.Data > endp.Data {
|
||||
panic(fmt.Errorf("start pointer address (%d) is after end pointer address (%d)", startp.Data, endp.Data))
|
||||
}
|
||||
|
||||
l := startp.Len
|
||||
endLen := int(endp.Data-startp.Data) + endp.Len
|
||||
if endLen > l {
|
||||
l = endLen
|
||||
}
|
||||
|
||||
if l > startp.Cap {
|
||||
panic(fmt.Errorf("range length is larger than capacity"))
|
||||
}
|
||||
|
||||
return start[:l]
|
||||
}
|
||||
|
||||
@@ -77,3 +77,92 @@ func TestUnsafeSubsliceOffsetInvalid(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsafeBytesRange(t *testing.T) {
|
||||
type fn = func() ([]byte, []byte)
|
||||
examples := []struct {
|
||||
desc string
|
||||
test fn
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
desc: "simple",
|
||||
test: func() ([]byte, []byte) {
|
||||
full := []byte("hello world")
|
||||
return full[1:3], full[6:8]
|
||||
},
|
||||
expected: []byte("ello wo"),
|
||||
},
|
||||
{
|
||||
desc: "full",
|
||||
test: func() ([]byte, []byte) {
|
||||
full := []byte("hello world")
|
||||
return full[0:1], full[len(full)-1:]
|
||||
},
|
||||
expected: []byte("hello world"),
|
||||
},
|
||||
{
|
||||
desc: "end before start",
|
||||
test: func() ([]byte, []byte) {
|
||||
full := []byte("hello world")
|
||||
return full[len(full)-1:], full[0:1]
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "nils",
|
||||
test: func() ([]byte, []byte) {
|
||||
return nil, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "nils start",
|
||||
test: func() ([]byte, []byte) {
|
||||
return nil, []byte("foo")
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "nils end",
|
||||
test: func() ([]byte, []byte) {
|
||||
return []byte("foo"), nil
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "start is end",
|
||||
test: func() ([]byte, []byte) {
|
||||
full := []byte("hello world")
|
||||
return full[1:3], full[1:3]
|
||||
},
|
||||
expected: []byte("el"),
|
||||
},
|
||||
{
|
||||
desc: "end contained in start",
|
||||
test: func() ([]byte, []byte) {
|
||||
full := []byte("hello world")
|
||||
return full[1:7], full[2:4]
|
||||
},
|
||||
expected: []byte("ello w"),
|
||||
},
|
||||
{
|
||||
desc: "different backing arrays",
|
||||
test: func() ([]byte, []byte) {
|
||||
one := []byte("hello world")
|
||||
two := []byte("hello world")
|
||||
return one, two
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, e := range examples {
|
||||
t.Run(e.desc, func(t *testing.T) {
|
||||
start, end := e.test()
|
||||
if e.expected == nil {
|
||||
require.Panics(t, func() {
|
||||
unsafe.BytesRange(start, end)
|
||||
})
|
||||
} else {
|
||||
res := unsafe.BytesRange(start, end)
|
||||
require.Equal(t, e.expected, res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user