Merge remote-tracking branch 'upstream/master'

doc_test.go

@@ -5,6 +5,7 @@ package toml_test
 import (
 	"fmt"
 	"log"
+	"os"
 
 	toml "github.com/pelletier/go-toml"
 )
@@ -104,3 +105,66 @@ func ExampleUnmarshal() {
 	// Output:
 	// user= pelletier
 }
+
+func ExampleEncoder_anonymous() {
+	type Credentials struct {
+		User     string `toml:"user"`
+		Password string `toml:"password"`
+	}
+
+	type Protocol struct {
+		Name string `toml:"name"`
+	}
+
+	type Config struct {
+		Version int `toml:"version"`
+		Credentials
+		Protocol `toml:"Protocol"`
+	}
+	config := Config{
+		Version: 2,
+		Credentials: Credentials{
+			User:     "pelletier",
+			Password: "mypassword",
+		},
+		Protocol: Protocol{
+			Name: "tcp",
+		},
+	}
+	fmt.Println("Default:")
+	fmt.Println("---------------")
+
+	def := toml.NewEncoder(os.Stdout)
+	if err := def.Encode(config); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println("---------------")
+	fmt.Println("With promotion:")
+	fmt.Println("---------------")
+
+	prom := toml.NewEncoder(os.Stdout).PromoteAnonymous(true)
+	if err := prom.Encode(config); err != nil {
+		log.Fatal(err)
+	}
+	// Output:
+	// Default:
+	// ---------------
+	// password = "mypassword"
+	// user = "pelletier"
+	// version = 2
+	//
+	// [Protocol]
+	//   name = "tcp"
+	// ---------------
+	// With promotion:
+	// ---------------
+	// version = 2
+	//
+	// [Credentials]
+	//   password = "mypassword"
+	//   user = "pelletier"
+	//
+	// [Protocol]
+	//   name = "tcp"
+}

lexer.go

@@ -313,7 +313,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
 			l.next()
-			str, err := l.lexStringAsString(`"`, false, true, false)
+			str, err := l.lexStringAsString(`"`, false, true)
 			if err != nil {
 				return l.errorf(err.Error())
 			}
@@ -419,7 +419,7 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
 // Lex a string and return the results as a string.
 // Terminator is the substring indicating the end of the token.
 // The resulting string does not include the terminator.
-func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool, acceptTab bool) (string, error) {
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
 	growingString := ""
 
 	if discardLeadingNewLine {
@@ -512,8 +512,7 @@ func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine,
 		} else {
 			r := l.peek()
 
-			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) &&
+			if 0x00 <= r && r <= 0x1F && r != '\t' && !(acceptNewLines && (r == '\n' || r == '\r')) {
-				!(acceptTab && r == '\t') {
 				return "", fmt.Errorf("unescaped control character %U", r)
 			}
 			l.next()
@@ -535,17 +534,15 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 	terminator := `"`
 	discardLeadingNewLine := false
 	acceptNewLines := false
-	acceptTab := false
 	if l.follow(`""`) {
 		l.skip()
 		l.skip()
 		terminator = `"""`
 		discardLeadingNewLine = true
 		acceptNewLines = true
-		acceptTab = true
 	}
 
-	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines, acceptTab)
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
 
 	if err != nil {
 		return l.errorf(err.Error())

lexer_test.go

@@ -707,6 +707,15 @@ func TestEscapeInString(t *testing.T) {
 	})
 }
 
+func TestTabInString(t *testing.T) {
+	testFlow(t, `foo = "hello	world"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "hello\tworld"},
+		{Position{1, 20}, tokenEOF, ""},
+	})
+}
+
 func TestKeyGroupArray(t *testing.T) {
 	testFlow(t, "[[foo]]", []token{
 		{Position{1, 1}, tokenDoubleLeftBracket, "[["},
@@ -725,6 +734,15 @@ func TestQuotedKey(t *testing.T) {
 	})
 }
 
+func TestQuotedKeyTab(t *testing.T) {
+	testFlow(t, "\"num\tber\" = 123", []token{
+		{Position{1, 1}, tokenKey, "\"num\tber\""},
+		{Position{1, 11}, tokenEqual, "="},
+		{Position{1, 13}, tokenInteger, "123"},
+		{Position{1, 16}, tokenEOF, ""},
+	})
+}
+
 func TestKeyNewline(t *testing.T) {
 	testFlow(t, "a\n= 4", []token{
 		{Position{1, 1}, tokenError, "keys cannot contain new lines"},

marshal.go

@@ -22,6 +22,7 @@ const (
 
 type tomlOpts struct {
 	name         string
+	nameFromTag  bool
 	comment      string
 	commented    bool
 	multiline    bool
@@ -190,9 +191,10 @@ type Encoder struct {
 	w io.Writer
 	encOpts
 	annotation
-	line  int
+	line        int
-	col   int
+	col         int
-	order marshalOrder
+	order       marshalOrder
+	promoteAnon bool
 }
 
 // NewEncoder returns a new encoder that writes to w.
@@ -279,6 +281,19 @@ func (e *Encoder) SetTagMultiline(v string) *Encoder {
 	return e
 }
 
+// PromoteAnonymous allows to change how anonymous struct fields are marshaled.
+// Usually, they are marshaled as if the inner exported fields were fields in
+// the outer struct. However, if an anonymous struct field is given a name in
+// its TOML tag, it is treated like a regular struct field with that name.
+// rather than being anonymous.
+//
+// In case anonymous promotion is enabled, all anonymous structs are promoted
+// and treated like regular struct fields.
+func (e *Encoder) PromoteAnonymous(promote bool) *Encoder {
+	e.promoteAnon = promote
+	return e
+}
+
 func (e *Encoder) marshal(v interface{}) ([]byte, error) {
 	mtype := reflect.TypeOf(v)
 	if mtype == nil {
@@ -338,12 +353,15 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er
 					if err != nil {
 						return nil, err
 					}
-
+					if tree, ok := val.(*Tree); ok && mtypef.Anonymous && !opts.nameFromTag && !e.promoteAnon {
-					tval.SetWithOptions(opts.name, SetOptions{
+						e.appendTree(tval, tree)
-						Comment:   opts.comment,
+					} else {
-						Commented: opts.commented,
+						tval.SetWithOptions(opts.name, SetOptions{
-						Multiline: opts.multiline,
+							Comment:   opts.comment,
-					}, val)
+							Commented: opts.commented,
+							Multiline: opts.multiline,
+						}, val)
+					}
 				}
 			}
 		}
@@ -460,6 +478,19 @@ func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface
 	}
 }
 
+func (e *Encoder) appendTree(t, o *Tree) error {
+	for key, value := range o.values {
+		if _, ok := t.values[key]; ok {
+			continue
+		}
+		if tomlValue, ok := value.(*tomlValue); ok {
+			tomlValue.position.Col = t.position.Col
+		}
+		t.values[key] = value
+	}
+	return nil
+}
+
 // Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v.
 // Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for
 // sub-structs, and only definite types can be unmarshaled.
@@ -714,6 +745,22 @@ func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (r
 	return mval, nil
 }
 
+// Convert toml value to marshal primitive slice, using marshal type
+func (d *Decoder) valueFromOtherSliceI(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+	val := reflect.ValueOf(tval)
+
+	lenght := val.Len()
+	mval := reflect.MakeSlice(mtype, lenght, lenght)
+	for i := 0; i < lenght; i++ {
+		val, err := d.valueFromToml(mtype.Elem(), val.Index(i).Interface(), nil)
+		if err != nil {
+			return mval, err
+		}
+		mval.Index(i).Set(val)
+	}
+	return mval, nil
+}
+
 // Convert toml value to marshal value, using marshal type. When mval1 is non-nil
 // and the given type is a struct value, merge fields into it.
 func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *reflect.Value) (reflect.Value, error) {
@@ -857,6 +904,11 @@ func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *ref
 				ival := mval1.Elem()
 				return d.valueFromToml(mval1.Elem().Type(), t, &ival)
 			}
+		case reflect.Slice:
+			if isOtherSequence(mtype) && isOtherSequence(reflect.TypeOf(t)) {
+				return d.valueFromOtherSliceI(mtype, t)
+			}
+			return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
 		default:
 			return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
 		}
@@ -892,6 +944,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts {
 	defaultValue := vf.Tag.Get(tagDefault)
 	result := tomlOpts{
 		name:         vf.Name,
+		nameFromTag:  false,
 		comment:      comment,
 		commented:    commented,
 		multiline:    multiline,
@@ -904,6 +957,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts {
 			result.include = false
 		} else {
 			result.name = strings.Trim(parse[0], " ")
+			result.nameFromTag = true
 		}
 	}
 	if vf.PkgPath != "" {

marshal_test.go

@@ -1419,26 +1419,54 @@ func TestMarshalDirectMultilineString(t *testing.T) {
 	}
 }
 
-//issue 354
+func TestUnmarshalTabInStringAndQuotedKey(t *testing.T) {
-func TestUnmarshalMultilineStringWithTab(t *testing.T) {
-	input := []byte(`
-Field = """
-hello	world"""
-`)
-
 	type Test struct {
-		Field string
+		Field1 string `toml:"Fie	ld1"`
+		Field2 string
 	}
 
-	expected := Test{"hello\tworld"}
+	type TestCase struct {
-	result := Test{}
+		desc     string
-	err := Unmarshal(input, &result)
+		input    []byte
-	if err != nil {
+		expected Test
-		t.Fatal("unmarshal should not error:", err)
 	}
 
-	if !reflect.DeepEqual(result, expected) {
+	testCases := []TestCase{
-		t.Errorf("Bad unmarshal: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", expected, result)
+		{
+			desc:  "multiline string with tab",
+			input: []byte("Field2 = \"\"\"\nhello\tworld\"\"\""),
+			expected: Test{
+				Field2: "hello\tworld",
+			},
+		},
+		{
+			desc:  "quoted key with tab",
+			input: []byte("\"Fie\tld1\" = \"key with tab\""),
+			expected: Test{
+				Field1: "key with tab",
+			},
+		},
+		{
+			desc:  "basic string tab",
+			input: []byte("Field2 = \"hello\tworld\""),
+			expected: Test{
+				Field2: "hello\tworld",
+			},
+		},
+	}
+
+	for i := range testCases {
+		result := Test{}
+		err := Unmarshal(testCases[i].input, &result)
+		if err != nil {
+			t.Errorf("%s test error:%v", testCases[i].desc, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(result, testCases[i].expected) {
+			t.Errorf("%s test error: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n",
+				testCases[i].desc, testCases[i].expected, result)
+		}
 	}
 }
 
@@ -1467,6 +1495,55 @@ func TestMarshalCustomMultiline(t *testing.T) {
 	}
 }
 
+func TestMultilineWithAdjacentQuotationMarks(t *testing.T) {
+	type testStruct struct {
+		Str string `multiline:"true"`
+	}
+	type testCase struct {
+		expected []byte
+		data     testStruct
+	}
+
+	testCases := []testCase{
+		{
+			expected: []byte(`Str = """
+hello\""""
+`),
+			data: testStruct{
+				Str: "hello\"",
+			},
+		},
+		{
+			expected: []byte(`Str = """
+""\"""\"""\""""
+`),
+			data: testStruct{
+				Str: "\"\"\"\"\"\"\"\"\"",
+			},
+		},
+	}
+	for i := range testCases {
+		result, err := Marshal(testCases[i].data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !bytes.Equal(result, testCases[i].expected) {
+			t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n",
+				testCases[i].expected, result)
+		} else {
+			var data testStruct
+			if err = Unmarshal(result, &data); err != nil {
+				t.Fatal(err)
+			}
+			if data.Str != testCases[i].data.Str {
+				t.Errorf("Round trip test fail: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n",
+					testCases[i].data.Str, data.Str)
+			}
+		}
+	}
+}
+
 func TestMarshalEmbedTree(t *testing.T) {
 	expected := []byte(`OuterField1 = "Out"
 OuterField2 = 1024
@@ -1897,6 +1974,99 @@ func TestUnmarshalDefaultFailureUnsupported(t *testing.T) {
 	}
 }
 
+func TestMarshalNestedAnonymousStructs(t *testing.T) {
+	type Embedded struct {
+		Value string `toml:"value"`
+		Top   struct {
+			Value string `toml:"value"`
+		} `toml:"top"`
+	}
+
+	type Named struct {
+		Value string `toml:"value"`
+	}
+
+	var doc struct {
+		Embedded
+		Named     `toml:"named"`
+		Anonymous struct {
+			Value string `toml:"value"`
+		} `toml:"anonymous"`
+	}
+
+	expected := `value = ""
+
+[anonymous]
+  value = ""
+
+[named]
+  value = ""
+
+[top]
+  value = ""
+`
+
+	result, err := Marshal(doc)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err.Error())
+	}
+	if !bytes.Equal(result, []byte(expected)) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result))
+	}
+}
+
+func TestEncoderPromoteNestedAnonymousStructs(t *testing.T) {
+	type Embedded struct {
+		Value string `toml:"value"`
+	}
+
+	var doc struct {
+		Embedded
+	}
+
+	expected := `
+[Embedded]
+  value = ""
+`
+	var buf bytes.Buffer
+	if err := NewEncoder(&buf).PromoteAnonymous(true).Encode(doc); err != nil {
+		t.Fatalf("unexpected error: %s", err.Error())
+	}
+	if !bytes.Equal(buf.Bytes(), []byte(expected)) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, buf.String())
+	}
+}
+
+func TestMarshalNestedAnonymousStructs_DuplicateField(t *testing.T) {
+	type Embedded struct {
+		Value string `toml:"value"`
+		Top   struct {
+			Value string `toml:"value"`
+		} `toml:"top"`
+	}
+
+	var doc struct {
+		Value string `toml:"value"`
+		Embedded
+	}
+	doc.Embedded.Value = "shadowed"
+	doc.Value = "shadows"
+
+	expected := `value = "shadows"
+
+[top]
+  value = ""
+`
+
+	result, err := Marshal(doc)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err.Error())
+	}
+	if !bytes.Equal(result, []byte(expected)) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result))
+	}
+}
+
 func TestUnmarshalNestedAnonymousStructs(t *testing.T) {
 	type Nested struct {
 		Value string `toml:"nested_field"`
@@ -2813,3 +2983,72 @@ func TestUnmarshalNil(t *testing.T) {
 		t.Errorf("Expected err from nil marshal")
 	}
 }
+
+var sliceTomlDemo = []byte(`str_slice = ["Howdy","Hey There"]
+str_slice_ptr= ["Howdy","Hey There"]
+int_slice=[1,2]
+int_slice_ptr=[1,2]
+[[struct_slice]]
+String2="1"
+[[struct_slice]]
+String2="2"
+[[struct_slice_ptr]]
+String2="1"
+[[struct_slice_ptr]]
+String2="2"
+`)
+
+type sliceStruct struct {
+	Slice          []string                     `  toml:"str_slice"  `
+	SlicePtr       *[]string                    `  toml:"str_slice_ptr"  `
+	IntSlice       []int                        `  toml:"int_slice"  `
+	IntSlicePtr    *[]int                       `  toml:"int_slice_ptr"  `
+	StructSlice    []basicMarshalTestSubStruct  `  toml:"struct_slice"  `
+	StructSlicePtr *[]basicMarshalTestSubStruct `  toml:"struct_slice_ptr"  `
+}
+
+func TestUnmarshalSlice(t *testing.T) {
+	tree, _ := LoadBytes(sliceTomlDemo)
+	tree, _ = TreeFromMap(tree.ToMap())
+
+	var actual sliceStruct
+	err := tree.Unmarshal(&actual)
+	if err != nil {
+		t.Error("shound not err", err)
+	}
+	expected := sliceStruct{
+		Slice:          []string{"Howdy", "Hey There"},
+		SlicePtr:       &[]string{"Howdy", "Hey There"},
+		IntSlice:       []int{1, 2},
+		IntSlicePtr:    &[]int{1, 2},
+		StructSlice:    []basicMarshalTestSubStruct{{"1"}, {"2"}},
+		StructSlicePtr: &[]basicMarshalTestSubStruct{{"1"}, {"2"}},
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Bad unmarshal: expected %v, got %v", expected, actual)
+	}
+
+}
+
+func TestUnmarshalSliceFail(t *testing.T) {
+	tree, _ := TreeFromMap(map[string]interface{}{
+		"str_slice": []int{1, 2},
+	})
+
+	var actual sliceStruct
+	err := tree.Unmarshal(&actual)
+	if err.Error() != "(0, 0): Can't convert 1(int64) to string" {
+		t.Error("expect err:(0, 0): Can't convert 1(int64) to string but got ", err)
+	}
+}
+
+func TestUnmarshalSliceFail2(t *testing.T) {
+	tree, _ := Load(`str_slice=[1,2]`)
+
+	var actual sliceStruct
+	err := tree.Unmarshal(&actual)
+	if err.Error() != "(1, 1): Can't convert 1(int64) to string" {
+		t.Error("expect err:(1, 1): Can't convert 1(int64) to string but got ", err)
+	}
+
+}

parser_test.go

@@ -696,6 +696,13 @@ func TestInlineTableDoubleComma(t *testing.T) {
 	}
 }
 
+func TestInlineTableTrailingComma(t *testing.T) {
+	_, err := Load("foo = {hello = 53, foo = 17,}")
+	if err.Error() != "(1, 28): trailing comma at the end of inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestDuplicateGroups(t *testing.T) {
 	_, err := Load("[foo]\na=2\n[foo]b=3")
 	if err.Error() != "(3, 2): duplicated tables" {

tomltree_write.go

@@ -30,9 +30,15 @@ type sortNode struct {
 // are preserved. Quotation marks and backslashes are also not escaped.
 func encodeMultilineTomlString(value string, commented string) string {
 	var b bytes.Buffer
+	adjacentQuoteCount := 0
 
 	b.WriteString(commented)
-	for _, rr := range value {
+	for i, rr := range value {
+		if rr != '"' {
+			adjacentQuoteCount = 0
+		} else {
+			adjacentQuoteCount++
+		}
 		switch rr {
 		case '\b':
 			b.WriteString(`\b`)
@@ -45,7 +51,12 @@ func encodeMultilineTomlString(value string, commented string) string {
 		case '\r':
 			b.WriteString("\r")
 		case '"':
-			b.WriteString(`"`)
+			if adjacentQuoteCount >= 3 || i == len(value)-1 {
+				adjacentQuoteCount = 0
+				b.WriteString(`\"`)
+			} else {
+				b.WriteString(`"`)
+			}
 		case '\\':
 			b.WriteString(`\`)
 		default: