ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
RiyaJohn
2020-04-27 12:02:06 +05:30
parent 5060c72d94 d1e0fc37ce
commit 4c840f1b8b
7 changed files with 423 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc_test.go
+64
View File
@@ -5,6 +5,7 @@ package toml_test
import (
"fmt"
"log"
"os"
toml "github.com/pelletier/go-toml"
)
@@ -104,3 +105,66 @@ func ExampleUnmarshal() {
// Output:
// user= pelletier
}
func ExampleEncoder_anonymous() {
type Credentials struct {
User string `toml:"user"`
Password string `toml:"password"`
}
type Protocol struct {
Name string `toml:"name"`
}
type Config struct {
Version int `toml:"version"`
Credentials
Protocol `toml:"Protocol"`
}
config := Config{
Version: 2,
Credentials: Credentials{
User: "pelletier",
Password: "mypassword",
},
Protocol: Protocol{
Name: "tcp",
},
}
fmt.Println("Default:")
fmt.Println("---------------")
def := toml.NewEncoder(os.Stdout)
if err := def.Encode(config); err != nil {
log.Fatal(err)
}
fmt.Println("---------------")
fmt.Println("With promotion:")
fmt.Println("---------------")
prom := toml.NewEncoder(os.Stdout).PromoteAnonymous(true)
if err := prom.Encode(config); err != nil {
log.Fatal(err)
}
// Output:
// Default:
// ---------------
// password = "mypassword"
// user = "pelletier"
// version = 2
//
// [Protocol]
// name = "tcp"
// ---------------
// With promotion:
// ---------------
// version = 2
//
// [Credentials]
// password = "mypassword"
// user = "pelletier"
//
// [Protocol]
// name = "tcp"
}
lexer.go
+4 -7
View File
@@ -313,7 +313,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
if r == '"' {
l.next()
str, err := l.lexStringAsString(`"`, false, true, false)
str, err := l.lexStringAsString(`"`, false, true)
if err != nil {
return l.errorf(err.Error())
}
@@ -419,7 +419,7 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
// Lex a string and return the results as a string.
// Terminator is the substring indicating the end of the token.
// The resulting string does not include the terminator.
func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool, acceptTab bool) (string, error) {
func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
growingString := ""
if discardLeadingNewLine {
@@ -512,8 +512,7 @@ func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine,
} else {
r := l.peek()
if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) &&
!(acceptTab && r == '\t') {
if 0x00 <= r && r <= 0x1F && r != '\t' && !(acceptNewLines && (r == '\n' || r == '\r')) {
return "", fmt.Errorf("unescaped control character %U", r)
}
l.next()
@@ -535,17 +534,15 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
terminator := `"`
discardLeadingNewLine := false
acceptNewLines := false
acceptTab := false
if l.follow(`""`) {
l.skip()
l.skip()
terminator = `"""`
discardLeadingNewLine = true
acceptNewLines = true
acceptTab = true
}
str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines, acceptTab)
str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
if err != nil {
return l.errorf(err.Error())
lexer_test.go
+18
View File
@@ -707,6 +707,15 @@ func TestEscapeInString(t *testing.T) {
})
}
func TestTabInString(t *testing.T) {
testFlow(t, `foo = "hello world"`, []token{
{Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenEqual, "="},
{Position{1, 8}, tokenString, "hello\tworld"},
{Position{1, 20}, tokenEOF, ""},
})
}
func TestKeyGroupArray(t *testing.T) {
testFlow(t, "[[foo]]", []token{
{Position{1, 1}, tokenDoubleLeftBracket, "[["},
@@ -725,6 +734,15 @@ func TestQuotedKey(t *testing.T) {
})
}
func TestQuotedKeyTab(t *testing.T) {
testFlow(t, "\"num\tber\" = 123", []token{
{Position{1, 1}, tokenKey, "\"num\tber\""},
{Position{1, 11}, tokenEqual, "="},
{Position{1, 13}, tokenInteger, "123"},
{Position{1, 16}, tokenEOF, ""},
})
}
func TestKeyNewline(t *testing.T) {
testFlow(t, "a\n= 4", []token{
{Position{1, 1}, tokenError, "keys cannot contain new lines"},
marshal.go
+63 -9
View File
@@ -22,6 +22,7 @@ const (
type tomlOpts struct {
name string
nameFromTag bool
comment string
commented bool
multiline bool
@@ -190,9 +191,10 @@ type Encoder struct {
w io.Writer
encOpts
annotation
line int
col int
order marshalOrder
line int
col int
order marshalOrder
promoteAnon bool
}
// NewEncoder returns a new encoder that writes to w.
@@ -279,6 +281,19 @@ func (e *Encoder) SetTagMultiline(v string) *Encoder {
return e
}
// PromoteAnonymous allows to change how anonymous struct fields are marshaled.
// Usually, they are marshaled as if the inner exported fields were fields in
// the outer struct. However, if an anonymous struct field is given a name in
// its TOML tag, it is treated like a regular struct field with that name.
// rather than being anonymous.
//
// In case anonymous promotion is enabled, all anonymous structs are promoted
// and treated like regular struct fields.
func (e *Encoder) PromoteAnonymous(promote bool) *Encoder {
e.promoteAnon = promote
return e
}
func (e *Encoder) marshal(v interface{}) ([]byte, error) {
mtype := reflect.TypeOf(v)
if mtype == nil {
@@ -338,12 +353,15 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er
if err != nil {
return nil, err
}
tval.SetWithOptions(opts.name, SetOptions{
Comment: opts.comment,
Commented: opts.commented,
Multiline: opts.multiline,
}, val)
if tree, ok := val.(*Tree); ok && mtypef.Anonymous && !opts.nameFromTag && !e.promoteAnon {
e.appendTree(tval, tree)
} else {
tval.SetWithOptions(opts.name, SetOptions{
Comment: opts.comment,
Commented: opts.commented,
Multiline: opts.multiline,
}, val)
}
}
}
}
@@ -460,6 +478,19 @@ func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface
}
}
func (e *Encoder) appendTree(t, o *Tree) error {
for key, value := range o.values {
if _, ok := t.values[key]; ok {
continue
}
if tomlValue, ok := value.(*tomlValue); ok {
tomlValue.position.Col = t.position.Col
}
t.values[key] = value
}
return nil
}
// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v.
// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for
// sub-structs, and only definite types can be unmarshaled.
@@ -714,6 +745,22 @@ func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (r
return mval, nil
}
// Convert toml value to marshal primitive slice, using marshal type
func (d *Decoder) valueFromOtherSliceI(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
val := reflect.ValueOf(tval)
lenght := val.Len()
mval := reflect.MakeSlice(mtype, lenght, lenght)
for i := 0; i < lenght; i++ {
val, err := d.valueFromToml(mtype.Elem(), val.Index(i).Interface(), nil)
if err != nil {
return mval, err
}
mval.Index(i).Set(val)
}
return mval, nil
}
// Convert toml value to marshal value, using marshal type. When mval1 is non-nil
// and the given type is a struct value, merge fields into it.
func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *reflect.Value) (reflect.Value, error) {
@@ -857,6 +904,11 @@ func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *ref
ival := mval1.Elem()
return d.valueFromToml(mval1.Elem().Type(), t, &ival)
}
case reflect.Slice:
if isOtherSequence(mtype) && isOtherSequence(reflect.TypeOf(t)) {
return d.valueFromOtherSliceI(mtype, t)
}
return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
default:
return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
}
@@ -892,6 +944,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts {
defaultValue := vf.Tag.Get(tagDefault)
result := tomlOpts{
name: vf.Name,
nameFromTag: false,
comment: comment,
commented: commented,
multiline: multiline,
@@ -904,6 +957,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts {
result.include = false
} else {
result.name = strings.Trim(parse[0], " ")
result.nameFromTag = true
}
}
if vf.PkgPath != "" {
marshal_test.go
+254 -15
View File
@@ -1419,26 +1419,54 @@ func TestMarshalDirectMultilineString(t *testing.T) {
}
}
//issue 354
func TestUnmarshalMultilineStringWithTab(t *testing.T) {
input := []byte(`
Field = """
hello world"""
`)
func TestUnmarshalTabInStringAndQuotedKey(t *testing.T) {
type Test struct {
Field string
Field1 string `toml:"Fie ld1"`
Field2 string
}
expected := Test{"hello\tworld"}
result := Test{}
err := Unmarshal(input, &result)
if err != nil {
t.Fatal("unmarshal should not error:", err)
type TestCase struct {
desc string
input []byte
expected Test
}
if !reflect.DeepEqual(result, expected) {
t.Errorf("Bad unmarshal: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", expected, result)
testCases := []TestCase{
{
desc: "multiline string with tab",
input: []byte("Field2 = \"\"\"\nhello\tworld\"\"\""),
expected: Test{
Field2: "hello\tworld",
},
},
{
desc: "quoted key with tab",
input: []byte("\"Fie\tld1\" = \"key with tab\""),
expected: Test{
Field1: "key with tab",
},
},
{
desc: "basic string tab",
input: []byte("Field2 = \"hello\tworld\""),
expected: Test{
Field2: "hello\tworld",
},
},
}
for i := range testCases {
result := Test{}
err := Unmarshal(testCases[i].input, &result)
if err != nil {
t.Errorf("%s test error:%v", testCases[i].desc, err)
continue
}
if !reflect.DeepEqual(result, testCases[i].expected) {
t.Errorf("%s test error: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n",
testCases[i].desc, testCases[i].expected, result)
}
}
}
@@ -1467,6 +1495,55 @@ func TestMarshalCustomMultiline(t *testing.T) {
}
}
func TestMultilineWithAdjacentQuotationMarks(t *testing.T) {
type testStruct struct {
Str string `multiline:"true"`
}
type testCase struct {
expected []byte
data testStruct
}
testCases := []testCase{
{
expected: []byte(`Str = """
hello\""""
`),
data: testStruct{
Str: "hello\"",
},
},
{
expected: []byte(`Str = """
""\"""\"""\""""
`),
data: testStruct{
Str: "\"\"\"\"\"\"\"\"\"",
},
},
}
for i := range testCases {
result, err := Marshal(testCases[i].data)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(result, testCases[i].expected) {
t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n",
testCases[i].expected, result)
} else {
var data testStruct
if err = Unmarshal(result, &data); err != nil {
t.Fatal(err)
}
if data.Str != testCases[i].data.Str {
t.Errorf("Round trip test fail: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n",
testCases[i].data.Str, data.Str)
}
}
}
}
func TestMarshalEmbedTree(t *testing.T) {
expected := []byte(`OuterField1 = "Out"
OuterField2 = 1024
@@ -1897,6 +1974,99 @@ func TestUnmarshalDefaultFailureUnsupported(t *testing.T) {
}
}
func TestMarshalNestedAnonymousStructs(t *testing.T) {
type Embedded struct {
Value string `toml:"value"`
Top struct {
Value string `toml:"value"`
} `toml:"top"`
}
type Named struct {
Value string `toml:"value"`
}
var doc struct {
Embedded
Named `toml:"named"`
Anonymous struct {
Value string `toml:"value"`
} `toml:"anonymous"`
}
expected := `value = ""
[anonymous]
value = ""
[named]
value = ""
[top]
value = ""
`
result, err := Marshal(doc)
if err != nil {
t.Fatalf("unexpected error: %s", err.Error())
}
if !bytes.Equal(result, []byte(expected)) {
t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result))
}
}
func TestEncoderPromoteNestedAnonymousStructs(t *testing.T) {
type Embedded struct {
Value string `toml:"value"`
}
var doc struct {
Embedded
}
expected := `
[Embedded]
value = ""
`
var buf bytes.Buffer
if err := NewEncoder(&buf).PromoteAnonymous(true).Encode(doc); err != nil {
t.Fatalf("unexpected error: %s", err.Error())
}
if !bytes.Equal(buf.Bytes(), []byte(expected)) {
t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, buf.String())
}
}
func TestMarshalNestedAnonymousStructs_DuplicateField(t *testing.T) {
type Embedded struct {
Value string `toml:"value"`
Top struct {
Value string `toml:"value"`
} `toml:"top"`
}
var doc struct {
Value string `toml:"value"`
Embedded
}
doc.Embedded.Value = "shadowed"
doc.Value = "shadows"
expected := `value = "shadows"
[top]
value = ""
`
result, err := Marshal(doc)
if err != nil {
t.Fatalf("unexpected error: %s", err.Error())
}
if !bytes.Equal(result, []byte(expected)) {
t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result))
}
}
func TestUnmarshalNestedAnonymousStructs(t *testing.T) {
type Nested struct {
Value string `toml:"nested_field"`
@@ -2813,3 +2983,72 @@ func TestUnmarshalNil(t *testing.T) {
t.Errorf("Expected err from nil marshal")
}
}
var sliceTomlDemo = []byte(`str_slice = ["Howdy","Hey There"]
str_slice_ptr= ["Howdy","Hey There"]
int_slice=[1,2]
int_slice_ptr=[1,2]
[[struct_slice]]
String2="1"
[[struct_slice]]
String2="2"
[[struct_slice_ptr]]
String2="1"
[[struct_slice_ptr]]
String2="2"
`)
type sliceStruct struct {
Slice []string ` toml:"str_slice" `
SlicePtr *[]string ` toml:"str_slice_ptr" `
IntSlice []int ` toml:"int_slice" `
IntSlicePtr *[]int ` toml:"int_slice_ptr" `
StructSlice []basicMarshalTestSubStruct ` toml:"struct_slice" `
StructSlicePtr *[]basicMarshalTestSubStruct ` toml:"struct_slice_ptr" `
}
func TestUnmarshalSlice(t *testing.T) {
tree, _ := LoadBytes(sliceTomlDemo)
tree, _ = TreeFromMap(tree.ToMap())
var actual sliceStruct
err := tree.Unmarshal(&actual)
if err != nil {
t.Error("shound not err", err)
}
expected := sliceStruct{
Slice: []string{"Howdy", "Hey There"},
SlicePtr: &[]string{"Howdy", "Hey There"},
IntSlice: []int{1, 2},
IntSlicePtr: &[]int{1, 2},
StructSlice: []basicMarshalTestSubStruct{{"1"}, {"2"}},
StructSlicePtr: &[]basicMarshalTestSubStruct{{"1"}, {"2"}},
}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("Bad unmarshal: expected %v, got %v", expected, actual)
}
}
func TestUnmarshalSliceFail(t *testing.T) {
tree, _ := TreeFromMap(map[string]interface{}{
"str_slice": []int{1, 2},
})
var actual sliceStruct
err := tree.Unmarshal(&actual)
if err.Error() != "(0, 0): Can't convert 1(int64) to string" {
t.Error("expect err:(0, 0): Can't convert 1(int64) to string but got ", err)
}
}
func TestUnmarshalSliceFail2(t *testing.T) {
tree, _ := Load(`str_slice=[1,2]`)
var actual sliceStruct
err := tree.Unmarshal(&actual)
if err.Error() != "(1, 1): Can't convert 1(int64) to string" {
t.Error("expect err:(1, 1): Can't convert 1(int64) to string but got ", err)
}
}
parser_test.go
+7
View File
@@ -696,6 +696,13 @@ func TestInlineTableDoubleComma(t *testing.T) {
}
}
func TestInlineTableTrailingComma(t *testing.T) {
_, err := Load("foo = {hello = 53, foo = 17,}")
if err.Error() != "(1, 28): trailing comma at the end of inline table" {
t.Error("Bad error message:", err.Error())
}
}
func TestDuplicateGroups(t *testing.T) {
_, err := Load("[foo]\na=2\n[foo]b=3")
if err.Error() != "(3, 2): duplicated tables" {
tomltree_write.go
+13 -2
View File
@@ -30,9 +30,15 @@ type sortNode struct {
// are preserved. Quotation marks and backslashes are also not escaped.
func encodeMultilineTomlString(value string, commented string) string {
var b bytes.Buffer
adjacentQuoteCount := 0
b.WriteString(commented)
for _, rr := range value {
for i, rr := range value {
if rr != '"' {
adjacentQuoteCount = 0
} else {
adjacentQuoteCount++
}
switch rr {
case '\b':
b.WriteString(`\b`)
@@ -45,7 +51,12 @@ func encodeMultilineTomlString(value string, commented string) string {
case '\r':
b.WriteString("\r")
case '"':
b.WriteString(`"`)
if adjacentQuoteCount >= 3 || i == len(value)-1 {
adjacentQuoteCount = 0
b.WriteString(`\"`)
} else {
b.WriteString(`"`)
}
case '\\':
b.WriteString(`\`)
default: