diff --git a/doc_test.go b/doc_test.go index d64414c..7aaddab 100644 --- a/doc_test.go +++ b/doc_test.go @@ -5,6 +5,7 @@ package toml_test import ( "fmt" "log" + "os" toml "github.com/pelletier/go-toml" ) @@ -104,3 +105,66 @@ func ExampleUnmarshal() { // Output: // user= pelletier } + +func ExampleEncoder_anonymous() { + type Credentials struct { + User string `toml:"user"` + Password string `toml:"password"` + } + + type Protocol struct { + Name string `toml:"name"` + } + + type Config struct { + Version int `toml:"version"` + Credentials + Protocol `toml:"Protocol"` + } + config := Config{ + Version: 2, + Credentials: Credentials{ + User: "pelletier", + Password: "mypassword", + }, + Protocol: Protocol{ + Name: "tcp", + }, + } + fmt.Println("Default:") + fmt.Println("---------------") + + def := toml.NewEncoder(os.Stdout) + if err := def.Encode(config); err != nil { + log.Fatal(err) + } + + fmt.Println("---------------") + fmt.Println("With promotion:") + fmt.Println("---------------") + + prom := toml.NewEncoder(os.Stdout).PromoteAnonymous(true) + if err := prom.Encode(config); err != nil { + log.Fatal(err) + } + // Output: + // Default: + // --------------- + // password = "mypassword" + // user = "pelletier" + // version = 2 + // + // [Protocol] + // name = "tcp" + // --------------- + // With promotion: + // --------------- + // version = 2 + // + // [Credentials] + // password = "mypassword" + // user = "pelletier" + // + // [Protocol] + // name = "tcp" +} diff --git a/lexer.go b/lexer.go index 54b0969..0bde0a1 100644 --- a/lexer.go +++ b/lexer.go @@ -313,7 +313,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() { if r == '"' { l.next() - str, err := l.lexStringAsString(`"`, false, true, false) + str, err := l.lexStringAsString(`"`, false, true) if err != nil { return l.errorf(err.Error()) } @@ -419,7 +419,7 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn { // Lex a string and return the results as a string. // Terminator is the substring indicating the end of the token. // The resulting string does not include the terminator. -func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool, acceptTab bool) (string, error) { +func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) { growingString := "" if discardLeadingNewLine { @@ -512,8 +512,7 @@ func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, } else { r := l.peek() - if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) && - !(acceptTab && r == '\t') { + if 0x00 <= r && r <= 0x1F && r != '\t' && !(acceptNewLines && (r == '\n' || r == '\r')) { return "", fmt.Errorf("unescaped control character %U", r) } l.next() @@ -535,17 +534,15 @@ func (l *tomlLexer) lexString() tomlLexStateFn { terminator := `"` discardLeadingNewLine := false acceptNewLines := false - acceptTab := false if l.follow(`""`) { l.skip() l.skip() terminator = `"""` discardLeadingNewLine = true acceptNewLines = true - acceptTab = true } - str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines, acceptTab) + str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines) if err != nil { return l.errorf(err.Error()) diff --git a/lexer_test.go b/lexer_test.go index 2d06f4e..ff826e9 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -707,6 +707,15 @@ func TestEscapeInString(t *testing.T) { }) } +func TestTabInString(t *testing.T) { + testFlow(t, `foo = "hello world"`, []token{ + {Position{1, 1}, tokenKey, "foo"}, + {Position{1, 5}, tokenEqual, "="}, + {Position{1, 8}, tokenString, "hello\tworld"}, + {Position{1, 20}, tokenEOF, ""}, + }) +} + func TestKeyGroupArray(t *testing.T) { testFlow(t, "[[foo]]", []token{ {Position{1, 1}, tokenDoubleLeftBracket, "[["}, @@ -725,6 +734,15 @@ func TestQuotedKey(t *testing.T) { }) } +func TestQuotedKeyTab(t *testing.T) { + testFlow(t, "\"num\tber\" = 123", []token{ + {Position{1, 1}, tokenKey, "\"num\tber\""}, + {Position{1, 11}, tokenEqual, "="}, + {Position{1, 13}, tokenInteger, "123"}, + {Position{1, 16}, tokenEOF, ""}, + }) +} + func TestKeyNewline(t *testing.T) { testFlow(t, "a\n= 4", []token{ {Position{1, 1}, tokenError, "keys cannot contain new lines"}, diff --git a/marshal.go b/marshal.go index e921a80..0832630 100644 --- a/marshal.go +++ b/marshal.go @@ -22,6 +22,7 @@ const ( type tomlOpts struct { name string + nameFromTag bool comment string commented bool multiline bool @@ -190,9 +191,10 @@ type Encoder struct { w io.Writer encOpts annotation - line int - col int - order marshalOrder + line int + col int + order marshalOrder + promoteAnon bool } // NewEncoder returns a new encoder that writes to w. @@ -279,6 +281,19 @@ func (e *Encoder) SetTagMultiline(v string) *Encoder { return e } +// PromoteAnonymous allows to change how anonymous struct fields are marshaled. +// Usually, they are marshaled as if the inner exported fields were fields in +// the outer struct. However, if an anonymous struct field is given a name in +// its TOML tag, it is treated like a regular struct field with that name. +// rather than being anonymous. +// +// In case anonymous promotion is enabled, all anonymous structs are promoted +// and treated like regular struct fields. +func (e *Encoder) PromoteAnonymous(promote bool) *Encoder { + e.promoteAnon = promote + return e +} + func (e *Encoder) marshal(v interface{}) ([]byte, error) { mtype := reflect.TypeOf(v) if mtype == nil { @@ -338,12 +353,15 @@ func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, er if err != nil { return nil, err } - - tval.SetWithOptions(opts.name, SetOptions{ - Comment: opts.comment, - Commented: opts.commented, - Multiline: opts.multiline, - }, val) + if tree, ok := val.(*Tree); ok && mtypef.Anonymous && !opts.nameFromTag && !e.promoteAnon { + e.appendTree(tval, tree) + } else { + tval.SetWithOptions(opts.name, SetOptions{ + Comment: opts.comment, + Commented: opts.commented, + Multiline: opts.multiline, + }, val) + } } } } @@ -460,6 +478,19 @@ func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface } } +func (e *Encoder) appendTree(t, o *Tree) error { + for key, value := range o.values { + if _, ok := t.values[key]; ok { + continue + } + if tomlValue, ok := value.(*tomlValue); ok { + tomlValue.position.Col = t.position.Col + } + t.values[key] = value + } + return nil +} + // Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v. // Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for // sub-structs, and only definite types can be unmarshaled. @@ -714,6 +745,22 @@ func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (r return mval, nil } +// Convert toml value to marshal primitive slice, using marshal type +func (d *Decoder) valueFromOtherSliceI(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + val := reflect.ValueOf(tval) + + lenght := val.Len() + mval := reflect.MakeSlice(mtype, lenght, lenght) + for i := 0; i < lenght; i++ { + val, err := d.valueFromToml(mtype.Elem(), val.Index(i).Interface(), nil) + if err != nil { + return mval, err + } + mval.Index(i).Set(val) + } + return mval, nil +} + // Convert toml value to marshal value, using marshal type. When mval1 is non-nil // and the given type is a struct value, merge fields into it. func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *reflect.Value) (reflect.Value, error) { @@ -857,6 +904,11 @@ func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}, mval1 *ref ival := mval1.Elem() return d.valueFromToml(mval1.Elem().Type(), t, &ival) } + case reflect.Slice: + if isOtherSequence(mtype) && isOtherSequence(reflect.TypeOf(t)) { + return d.valueFromOtherSliceI(mtype, t) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind()) default: return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind()) } @@ -892,6 +944,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts { defaultValue := vf.Tag.Get(tagDefault) result := tomlOpts{ name: vf.Name, + nameFromTag: false, comment: comment, commented: commented, multiline: multiline, @@ -904,6 +957,7 @@ func tomlOptions(vf reflect.StructField, an annotation) tomlOpts { result.include = false } else { result.name = strings.Trim(parse[0], " ") + result.nameFromTag = true } } if vf.PkgPath != "" { diff --git a/marshal_test.go b/marshal_test.go index 7f4fdc6..04c90ef 100644 --- a/marshal_test.go +++ b/marshal_test.go @@ -1419,26 +1419,54 @@ func TestMarshalDirectMultilineString(t *testing.T) { } } -//issue 354 -func TestUnmarshalMultilineStringWithTab(t *testing.T) { - input := []byte(` -Field = """ -hello world""" -`) - +func TestUnmarshalTabInStringAndQuotedKey(t *testing.T) { type Test struct { - Field string + Field1 string `toml:"Fie ld1"` + Field2 string } - expected := Test{"hello\tworld"} - result := Test{} - err := Unmarshal(input, &result) - if err != nil { - t.Fatal("unmarshal should not error:", err) + type TestCase struct { + desc string + input []byte + expected Test } - if !reflect.DeepEqual(result, expected) { - t.Errorf("Bad unmarshal: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", expected, result) + testCases := []TestCase{ + { + desc: "multiline string with tab", + input: []byte("Field2 = \"\"\"\nhello\tworld\"\"\""), + expected: Test{ + Field2: "hello\tworld", + }, + }, + { + desc: "quoted key with tab", + input: []byte("\"Fie\tld1\" = \"key with tab\""), + expected: Test{ + Field1: "key with tab", + }, + }, + { + desc: "basic string tab", + input: []byte("Field2 = \"hello\tworld\""), + expected: Test{ + Field2: "hello\tworld", + }, + }, + } + + for i := range testCases { + result := Test{} + err := Unmarshal(testCases[i].input, &result) + if err != nil { + t.Errorf("%s test error:%v", testCases[i].desc, err) + continue + } + + if !reflect.DeepEqual(result, testCases[i].expected) { + t.Errorf("%s test error: expected\n-----\n%+v\n-----\ngot\n-----\n%+v\n-----\n", + testCases[i].desc, testCases[i].expected, result) + } } } @@ -1467,6 +1495,55 @@ func TestMarshalCustomMultiline(t *testing.T) { } } +func TestMultilineWithAdjacentQuotationMarks(t *testing.T) { + type testStruct struct { + Str string `multiline:"true"` + } + type testCase struct { + expected []byte + data testStruct + } + + testCases := []testCase{ + { + expected: []byte(`Str = """ +hello\"""" +`), + data: testStruct{ + Str: "hello\"", + }, + }, + { + expected: []byte(`Str = """ +""\"""\"""\"""" +`), + data: testStruct{ + Str: "\"\"\"\"\"\"\"\"\"", + }, + }, + } + for i := range testCases { + result, err := Marshal(testCases[i].data) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(result, testCases[i].expected) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", + testCases[i].expected, result) + } else { + var data testStruct + if err = Unmarshal(result, &data); err != nil { + t.Fatal(err) + } + if data.Str != testCases[i].data.Str { + t.Errorf("Round trip test fail: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", + testCases[i].data.Str, data.Str) + } + } + } +} + func TestMarshalEmbedTree(t *testing.T) { expected := []byte(`OuterField1 = "Out" OuterField2 = 1024 @@ -1897,6 +1974,99 @@ func TestUnmarshalDefaultFailureUnsupported(t *testing.T) { } } +func TestMarshalNestedAnonymousStructs(t *testing.T) { + type Embedded struct { + Value string `toml:"value"` + Top struct { + Value string `toml:"value"` + } `toml:"top"` + } + + type Named struct { + Value string `toml:"value"` + } + + var doc struct { + Embedded + Named `toml:"named"` + Anonymous struct { + Value string `toml:"value"` + } `toml:"anonymous"` + } + + expected := `value = "" + +[anonymous] + value = "" + +[named] + value = "" + +[top] + value = "" +` + + result, err := Marshal(doc) + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + if !bytes.Equal(result, []byte(expected)) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result)) + } +} + +func TestEncoderPromoteNestedAnonymousStructs(t *testing.T) { + type Embedded struct { + Value string `toml:"value"` + } + + var doc struct { + Embedded + } + + expected := ` +[Embedded] + value = "" +` + var buf bytes.Buffer + if err := NewEncoder(&buf).PromoteAnonymous(true).Encode(doc); err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + if !bytes.Equal(buf.Bytes(), []byte(expected)) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, buf.String()) + } +} + +func TestMarshalNestedAnonymousStructs_DuplicateField(t *testing.T) { + type Embedded struct { + Value string `toml:"value"` + Top struct { + Value string `toml:"value"` + } `toml:"top"` + } + + var doc struct { + Value string `toml:"value"` + Embedded + } + doc.Embedded.Value = "shadowed" + doc.Value = "shadows" + + expected := `value = "shadows" + +[top] + value = "" +` + + result, err := Marshal(doc) + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + if !bytes.Equal(result, []byte(expected)) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, string(result)) + } +} + func TestUnmarshalNestedAnonymousStructs(t *testing.T) { type Nested struct { Value string `toml:"nested_field"` @@ -2813,3 +2983,72 @@ func TestUnmarshalNil(t *testing.T) { t.Errorf("Expected err from nil marshal") } } + +var sliceTomlDemo = []byte(`str_slice = ["Howdy","Hey There"] +str_slice_ptr= ["Howdy","Hey There"] +int_slice=[1,2] +int_slice_ptr=[1,2] +[[struct_slice]] +String2="1" +[[struct_slice]] +String2="2" +[[struct_slice_ptr]] +String2="1" +[[struct_slice_ptr]] +String2="2" +`) + +type sliceStruct struct { + Slice []string ` toml:"str_slice" ` + SlicePtr *[]string ` toml:"str_slice_ptr" ` + IntSlice []int ` toml:"int_slice" ` + IntSlicePtr *[]int ` toml:"int_slice_ptr" ` + StructSlice []basicMarshalTestSubStruct ` toml:"struct_slice" ` + StructSlicePtr *[]basicMarshalTestSubStruct ` toml:"struct_slice_ptr" ` +} + +func TestUnmarshalSlice(t *testing.T) { + tree, _ := LoadBytes(sliceTomlDemo) + tree, _ = TreeFromMap(tree.ToMap()) + + var actual sliceStruct + err := tree.Unmarshal(&actual) + if err != nil { + t.Error("shound not err", err) + } + expected := sliceStruct{ + Slice: []string{"Howdy", "Hey There"}, + SlicePtr: &[]string{"Howdy", "Hey There"}, + IntSlice: []int{1, 2}, + IntSlicePtr: &[]int{1, 2}, + StructSlice: []basicMarshalTestSubStruct{{"1"}, {"2"}}, + StructSlicePtr: &[]basicMarshalTestSubStruct{{"1"}, {"2"}}, + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Bad unmarshal: expected %v, got %v", expected, actual) + } + +} + +func TestUnmarshalSliceFail(t *testing.T) { + tree, _ := TreeFromMap(map[string]interface{}{ + "str_slice": []int{1, 2}, + }) + + var actual sliceStruct + err := tree.Unmarshal(&actual) + if err.Error() != "(0, 0): Can't convert 1(int64) to string" { + t.Error("expect err:(0, 0): Can't convert 1(int64) to string but got ", err) + } +} + +func TestUnmarshalSliceFail2(t *testing.T) { + tree, _ := Load(`str_slice=[1,2]`) + + var actual sliceStruct + err := tree.Unmarshal(&actual) + if err.Error() != "(1, 1): Can't convert 1(int64) to string" { + t.Error("expect err:(1, 1): Can't convert 1(int64) to string but got ", err) + } + +} diff --git a/parser_test.go b/parser_test.go index 65d11b0..4c5a65e 100644 --- a/parser_test.go +++ b/parser_test.go @@ -696,6 +696,13 @@ func TestInlineTableDoubleComma(t *testing.T) { } } +func TestInlineTableTrailingComma(t *testing.T) { + _, err := Load("foo = {hello = 53, foo = 17,}") + if err.Error() != "(1, 28): trailing comma at the end of inline table" { + t.Error("Bad error message:", err.Error()) + } +} + func TestDuplicateGroups(t *testing.T) { _, err := Load("[foo]\na=2\n[foo]b=3") if err.Error() != "(3, 2): duplicated tables" { diff --git a/tomltree_write.go b/tomltree_write.go index 16c1986..9acc2f3 100644 --- a/tomltree_write.go +++ b/tomltree_write.go @@ -30,9 +30,15 @@ type sortNode struct { // are preserved. Quotation marks and backslashes are also not escaped. func encodeMultilineTomlString(value string, commented string) string { var b bytes.Buffer + adjacentQuoteCount := 0 b.WriteString(commented) - for _, rr := range value { + for i, rr := range value { + if rr != '"' { + adjacentQuoteCount = 0 + } else { + adjacentQuoteCount++ + } switch rr { case '\b': b.WriteString(`\b`) @@ -45,7 +51,12 @@ func encodeMultilineTomlString(value string, commented string) string { case '\r': b.WriteString("\r") case '"': - b.WriteString(`"`) + if adjacentQuoteCount >= 3 || i == len(value)-1 { + adjacentQuoteCount = 0 + b.WriteString(`\"`) + } else { + b.WriteString(`"`) + } case '\\': b.WriteString(`\`) default: