From 7e6e4b1314c49d42d1bff74c340894921f0ccfac Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Thu, 2 Mar 2017 09:17:06 -0800 Subject: [PATCH] Rewrite TomlTree encoding (#133) * Rewrite `TomlTree` encoding * Introduce `TomlTree.WriteTo` --- parser_test.go | 46 +--- toml.go | 4 +- tomltree_conversions.go | 227 ------------------ tomltree_write.go | 212 ++++++++++++++++ ...versions_test.go => tomltree_write_test.go | 121 +++++++++- 5 files changed, 331 insertions(+), 279 deletions(-) delete mode 100644 tomltree_conversions.go create mode 100644 tomltree_write.go rename tomltree_conversions_test.go => tomltree_write_test.go (50%) diff --git a/parser_test.go b/parser_test.go index 188876b..0d7e68f 100644 --- a/parser_test.go +++ b/parser_test.go @@ -633,22 +633,13 @@ func TestParseKeyGroupArraySpec(t *testing.T) { }) } -func TestToTomlValue(t *testing.T) { +func TestTomlValueStringRepresentation(t *testing.T) { for idx, item := range []struct { Value interface{} Expect string }{ - {int(1), "1"}, - {int8(2), "2"}, - {int16(3), "3"}, - {int32(4), "4"}, {int64(12345), "12345"}, - {uint(10), "10"}, - {uint8(20), "20"}, - {uint16(30), "30"}, - {uint32(40), "40"}, {uint64(50), "50"}, - {float32(12.456), "12.456"}, {float64(123.45), "123.45"}, {bool(true), "true"}, {"hello world", "\"hello world\""}, @@ -660,42 +651,19 @@ func TestToTomlValue(t *testing.T) { "[\"gamma\",\"delta\"]"}, {nil, ""}, } { - result := toTomlValue(item.Value, 0) + result, err := tomlValueStringRepresentation(item.Value) + if err != nil { + t.Errorf("Test %d - unexpected error: %s", idx, err) + } if result != item.Expect { t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect) } } } -func TestToString(t *testing.T) { - tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n") - if err != nil { - t.Errorf("Test failed to parse: %v", err) - return - } - result, err := tree.ToString() - if err != nil { - t.Errorf("Unexpected error: %s", err) - } - expected := "\n[foo]\n\n [[foo.bar]]\n a = 42\n\n [[foo.bar]]\n a = 69\n" - if result != expected { - t.Errorf("Expected got '%s', expected '%s'", result, expected) - } -} - func TestToStringMapStringString(t *testing.T) { - in := map[string]interface{}{"m": map[string]string{"v": "abc"}} - want := "\n[m]\n v = \"abc\"\n" - tree := TreeFromMap(in) - got := tree.String() - - if got != want { - t.Errorf("want:\n%q\ngot:\n%q", want, got) - } -} - -func TestToStringMapInterfaceInterface(t *testing.T) { - in := map[string]interface{}{"m": map[interface{}]interface{}{"v": "abc"}} + in := map[string]interface{}{"m": TreeFromMap(map[string]interface{}{ + "v": &tomlValue{"abc", Position{0, 0}}})} want := "\n[m]\n v = \"abc\"\n" tree := TreeFromMap(in) got := tree.String() diff --git a/toml.go b/toml.go index ad23fe8..8596091 100644 --- a/toml.go +++ b/toml.go @@ -10,13 +10,13 @@ import ( ) type tomlValue struct { - value interface{} + value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list position Position } // TomlTree is the result of the parsing of a TOML file. type TomlTree struct { - values map[string]interface{} + values map[string]interface{} // string -> *tomlValue, *TomlTree, []*TomlTree position Position } diff --git a/tomltree_conversions.go b/tomltree_conversions.go deleted file mode 100644 index fc8f22b..0000000 --- a/tomltree_conversions.go +++ /dev/null @@ -1,227 +0,0 @@ -package toml - -// Tools to convert a TomlTree to different representations - -import ( - "fmt" - "strconv" - "strings" - "time" -) - -// encodes a string to a TOML-compliant string value -func encodeTomlString(value string) string { - result := "" - for _, rr := range value { - intRr := uint16(rr) - switch rr { - case '\b': - result += "\\b" - case '\t': - result += "\\t" - case '\n': - result += "\\n" - case '\f': - result += "\\f" - case '\r': - result += "\\r" - case '"': - result += "\\\"" - case '\\': - result += "\\\\" - default: - if intRr < 0x001F { - result += fmt.Sprintf("\\u%0.4X", intRr) - } else { - result += string(rr) - } - } - } - return result -} - -// Value print support function for ToString() -// Outputs the TOML compliant string representation of a value -func toTomlValue(item interface{}, indent int) string { - tab := strings.Repeat(" ", indent) - switch value := item.(type) { - case int: - return tab + strconv.FormatInt(int64(value), 10) - case int8: - return tab + strconv.FormatInt(int64(value), 10) - case int16: - return tab + strconv.FormatInt(int64(value), 10) - case int32: - return tab + strconv.FormatInt(int64(value), 10) - case int64: - return tab + strconv.FormatInt(value, 10) - case uint: - return tab + strconv.FormatUint(uint64(value), 10) - case uint8: - return tab + strconv.FormatUint(uint64(value), 10) - case uint16: - return tab + strconv.FormatUint(uint64(value), 10) - case uint32: - return tab + strconv.FormatUint(uint64(value), 10) - case uint64: - return tab + strconv.FormatUint(value, 10) - case float32: - return tab + strconv.FormatFloat(float64(value), 'f', -1, 32) - case float64: - return tab + strconv.FormatFloat(value, 'f', -1, 64) - case string: - return tab + "\"" + encodeTomlString(value) + "\"" - case bool: - if value { - return "true" - } - return "false" - case time.Time: - return tab + value.Format(time.RFC3339) - case []interface{}: - values := []string{} - for _, item := range value { - values = append(values, toTomlValue(item, 0)) - } - return "[" + strings.Join(values, ",") + "]" - case nil: - return "" - default: - panic(fmt.Errorf("unsupported value type %T: %v", value, value)) - } -} - -// Recursive support function for ToString() -// Outputs a tree, using the provided keyspace to prefix table names -func (t *TomlTree) toToml(indent, keyspace string) string { - resultChunks := []string{} - for k, v := range t.values { - // figure out the keyspace - combinedKey := k - if keyspace != "" { - combinedKey = keyspace + "." + combinedKey - } - resultChunk := "" - // output based on type - switch node := v.(type) { - case []*TomlTree: - for _, item := range node { - if len(item.Keys()) > 0 { - resultChunk += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey) - } - resultChunk += item.toToml(indent+" ", combinedKey) - } - resultChunks = append(resultChunks, resultChunk) - case *TomlTree: - if len(node.Keys()) > 0 { - resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - } - resultChunk += node.toToml(indent+" ", combinedKey) - resultChunks = append(resultChunks, resultChunk) - case map[string]interface{}: - sub := TreeFromMap(node) - - if len(sub.Keys()) > 0 { - resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - } - resultChunk += sub.toToml(indent+" ", combinedKey) - resultChunks = append(resultChunks, resultChunk) - case map[string]string: - sub := TreeFromMap(convertMapStringString(node)) - - if len(sub.Keys()) > 0 { - resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - } - resultChunk += sub.toToml(indent+" ", combinedKey) - resultChunks = append(resultChunks, resultChunk) - case map[interface{}]interface{}: - sub := TreeFromMap(convertMapInterfaceInterface(node)) - - if len(sub.Keys()) > 0 { - resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - } - resultChunk += sub.toToml(indent+" ", combinedKey) - resultChunks = append(resultChunks, resultChunk) - case *tomlValue: - resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0)) - resultChunks = append([]string{resultChunk}, resultChunks...) - default: - resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0)) - resultChunks = append([]string{resultChunk}, resultChunks...) - } - - } - return strings.Join(resultChunks, "") -} - -// Same as ToToml(), but does not panic and returns an error -func (t *TomlTree) toTomlSafe(indent, keyspace string) (result string, err error) { - defer func() { - if r := recover(); r != nil { - result = "" - switch x := r.(type) { - case error: - err = x - default: - err = fmt.Errorf("unknown panic: %s", r) - } - } - }() - result = t.toToml(indent, keyspace) - return -} - -func convertMapStringString(in map[string]string) map[string]interface{} { - result := make(map[string]interface{}, len(in)) - for k, v := range in { - result[k] = v - } - return result -} - -func convertMapInterfaceInterface(in map[interface{}]interface{}) map[string]interface{} { - result := make(map[string]interface{}, len(in)) - for k, v := range in { - result[k.(string)] = v - } - return result -} - -// ToString generates a human-readable representation of the current tree. -// Output spans multiple lines, and is suitable for ingest by a TOML parser. -// If the conversion cannot be performed, ToString returns a non-nil error. -func (t *TomlTree) ToString() (string, error) { - return t.toTomlSafe("", "") -} - -// String generates a human-readable representation of the current tree. -// Alias of ToString. -func (t *TomlTree) String() string { - result, _ := t.ToString() - return result -} - -// ToMap recursively generates a representation of the current tree using map[string]interface{}. -func (t *TomlTree) ToMap() map[string]interface{} { - result := map[string]interface{}{} - - for k, v := range t.values { - switch node := v.(type) { - case []*TomlTree: - var array []interface{} - for _, item := range node { - array = append(array, item.ToMap()) - } - result[k] = array - case *TomlTree: - result[k] = node.ToMap() - case map[string]interface{}: - sub := TreeFromMap(node) - result[k] = sub.ToMap() - case *tomlValue: - result[k] = node.value - } - } - - return result -} diff --git a/tomltree_write.go b/tomltree_write.go new file mode 100644 index 0000000..a042a1c --- /dev/null +++ b/tomltree_write.go @@ -0,0 +1,212 @@ +package toml + +import ( + "bytes" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" +) + +// encodes a string to a TOML-compliant string value +func encodeTomlString(value string) string { + result := "" + for _, rr := range value { + switch rr { + case '\b': + result += "\\b" + case '\t': + result += "\\t" + case '\n': + result += "\\n" + case '\f': + result += "\\f" + case '\r': + result += "\\r" + case '"': + result += "\\\"" + case '\\': + result += "\\\\" + default: + intRr := uint16(rr) + if intRr < 0x001F { + result += fmt.Sprintf("\\u%0.4X", intRr) + } else { + result += string(rr) + } + } + } + return result +} + +func tomlValueStringRepresentation(v interface{}) (string, error) { + switch value := v.(type) { + case uint64: + return strconv.FormatUint(value, 10), nil + case int64: + return strconv.FormatInt(value, 10), nil + case float64: + return strconv.FormatFloat(value, 'f', -1, 32), nil + case string: + return "\"" + encodeTomlString(value) + "\"", nil + case bool: + if value { + return "true", nil + } + return "false", nil + case time.Time: + return value.Format(time.RFC3339), nil + case nil: + return "", nil + case []interface{}: + values := []string{} + for _, item := range value { + itemRepr, err := tomlValueStringRepresentation(item) + if err != nil { + return "", err + } + values = append(values, itemRepr) + } + return "[" + strings.Join(values, ",") + "]", nil + default: + return "", fmt.Errorf("unsupported value type %T: %v", value, value) + } +} + +func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) { + simpleValuesKeys := make([]string, 0) + complexValuesKeys := make([]string, 0) + + for k := range t.values { + v := t.values[k] + switch v.(type) { + case *TomlTree, []*TomlTree: + complexValuesKeys = append(complexValuesKeys, k) + default: + simpleValuesKeys = append(simpleValuesKeys, k) + } + } + + sort.Strings(simpleValuesKeys) + sort.Strings(complexValuesKeys) + + for _, k := range simpleValuesKeys { + v, ok := t.values[k].(*tomlValue) + if !ok { + return bytesCount, fmt.Errorf("invalid key type at %s: %T", k, t.values[k]) + } + + repr, err := tomlValueStringRepresentation(v.value) + if err != nil { + return bytesCount, err + } + + kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr) + writtenBytesCount, err := w.Write([]byte(kvRepr)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + } + + for _, k := range complexValuesKeys { + v := t.values[k] + + combinedKey := k + if keyspace != "" { + combinedKey = keyspace + "." + combinedKey + } + + switch node := v.(type) { + // node has to be of those two types given how keys are sorted above + case *TomlTree: + tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) + writtenBytesCount, err := w.Write([]byte(tableName)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount) + if err != nil { + return bytesCount, err + } + case []*TomlTree: + for _, subTree := range node { + if len(subTree.values) > 0 { + tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey) + writtenBytesCount, err := w.Write([]byte(tableArrayName)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + + bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount) + if err != nil { + return bytesCount, err + } + } + } + } + } + + return bytesCount, nil +} + +// WriteTo encode the TomlTree as Toml and writes it to the writer w. +// Returns the number of bytes written in case of success, or an error if anything happened. +func (t *TomlTree) WriteTo(w io.Writer) (int64, error) { + return t.writeTo(w, "", "", 0) +} + +// ToTomlString generates a human-readable representation of the current tree. +// Output spans multiple lines, and is suitable for ingest by a TOML parser. +// If the conversion cannot be performed, ToString returns a non-nil error. +func (t *TomlTree) ToTomlString() (string, error) { + var buf bytes.Buffer + _, err := t.WriteTo(&buf) + if err != nil { + return "", err + } + return buf.String(), nil +} + +// String generates a human-readable representation of the current tree. +// Alias of ToString. Present to implement the fmt.Stringer interface. +func (t *TomlTree) String() string { + result, _ := t.ToTomlString() + return result +} + +// ToMap recursively generates a representation of the tree using Go built-in structures. +// The following types are used: +// * uint64 +// * int64 +// * bool +// * string +// * time.Time +// * map[string]interface{} (where interface{} is any of this list) +// * []interface{} (where interface{} is any of this list) +func (t *TomlTree) ToMap() map[string]interface{} { + result := map[string]interface{}{} + + for k, v := range t.values { + switch node := v.(type) { + case []*TomlTree: + var array []interface{} + for _, item := range node { + array = append(array, item.ToMap()) + } + result[k] = array + case *TomlTree: + result[k] = node.ToMap() + case map[string]interface{}: + sub := TreeFromMap(node) + result[k] = sub.ToMap() + case *tomlValue: + result[k] = node.value + } + } + return result +} diff --git a/tomltree_conversions_test.go b/tomltree_write_test.go similarity index 50% rename from tomltree_conversions_test.go rename to tomltree_write_test.go index 40b29b7..2cf3577 100644 --- a/tomltree_conversions_test.go +++ b/tomltree_write_test.go @@ -1,14 +1,46 @@ package toml import ( + "bytes" "errors" + "fmt" "reflect" "strings" "testing" "time" ) -func TestTomlTreeConversionToString(t *testing.T) { +type failingWriter struct { + failAt int + written int + buffer bytes.Buffer +} + +func (f failingWriter) Write(p []byte) (n int, err error) { + count := len(p) + toWrite := f.failAt - count + f.written + if toWrite < 0 { + toWrite = 0 + } + if toWrite > count { + f.written += count + f.buffer.WriteString(string(p)) + return count, nil + } + + f.buffer.WriteString(string(p[:toWrite])) + f.written = f.failAt + return f.written, fmt.Errorf("failingWriter failed after writting %d bytes", f.written) +} + +func assertErrorString(t *testing.T, expected string, err error) { + expectedErr := errors.New(expected) + if err.Error() != expectedErr.Error() { + t.Errorf("expecting error %s, but got %s instead", expected, err) + } +} + +func TestTomlTreeWriteToTomlString(t *testing.T) { toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" } points = { x = 1, y = 2 }`) @@ -16,7 +48,7 @@ points = { x = 1, y = 2 }`) t.Fatal("Unexpected error:", err) } - tomlString, _ := toml.ToString() + tomlString, _ := toml.ToTomlString() reparsedTree, err := Load(tomlString) assertTree(t, reparsedTree, err, map[string]interface{}{ @@ -31,7 +63,23 @@ points = { x = 1, y = 2 }`) }) } -func TestTomlTreeConversionToStringKeysOrders(t *testing.T) { +func TestTomlTreeWriteToTomlStringSimple(t *testing.T) { + tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n") + if err != nil { + t.Errorf("Test failed to parse: %v", err) + return + } + result, err := tree.ToTomlString() + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + expected := "\n[foo]\n\n [[foo.bar]]\n a = 42\n\n [[foo.bar]]\n a = 69\n" + if result != expected { + t.Errorf("Expected got '%s', expected '%s'", result, expected) + } +} + +func TestTomlTreeWriteToTomlStringKeysOrders(t *testing.T) { for i := 0; i < 100; i++ { tree, _ := Load(` foobar = true @@ -41,7 +89,7 @@ func TestTomlTreeConversionToStringKeysOrders(t *testing.T) { foo = 1 bar = "baz2"`) - stringRepr, _ := tree.ToString() + stringRepr, _ := tree.ToTomlString() t.Log("Intermediate string representation:") t.Log(stringRepr) @@ -71,20 +119,20 @@ func testMaps(t *testing.T, actual, expected map[string]interface{}) { } } -func TestToStringTypeConversionError(t *testing.T) { +func TestToTomlStringTypeConversionError(t *testing.T) { tree := TomlTree{ values: map[string]interface{}{ - "thing": []string{"unsupported"}, + "thing": &tomlValue{[]string{"unsupported"}, Position{}}, }, } - _, err := tree.ToString() + _, err := tree.ToTomlString() expected := errors.New("unsupported value type []string: [unsupported]") if err.Error() != expected.Error() { t.Errorf("expecting error %s, but got %s instead", expected, err) } } -func TestTomlTreeConversionToMapSimple(t *testing.T) { +func TestTomlTreeWriteToMapSimple(t *testing.T) { tree, _ := Load("a = 42\nb = 17") expected := map[string]interface{}{ @@ -95,7 +143,58 @@ func TestTomlTreeConversionToMapSimple(t *testing.T) { testMaps(t, tree.ToMap(), expected) } -func TestTomlTreeConversionToMapExampleFile(t *testing.T) { +func TestTomlTreeWriteToInvalidTreeSimpleValue(t *testing.T) { + tree := TomlTree{values: map[string]interface{}{"foo": int8(1)}} + _, err := tree.ToTomlString() + assertErrorString(t, "invalid key type at foo: int8", err) +} + +func TestTomlTreeWriteToInvalidTreeTomlValue(t *testing.T) { + tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{int8(1), Position{}}}} + _, err := tree.ToTomlString() + assertErrorString(t, "unsupported value type int8: 1", err) +} + +func TestTomlTreeWriteToInvalidTreeTomlValueArray(t *testing.T) { + tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{[]interface{}{int8(1)}, Position{}}}} + _, err := tree.ToTomlString() + assertErrorString(t, "unsupported value type int8: 1", err) +} + +func TestTomlTreeWriteToFailingWriterInSimpleValue(t *testing.T) { + toml, _ := Load(`a = 2`) + writer := failingWriter{failAt: 0, written: 0} + _, err := toml.WriteTo(writer) + assertErrorString(t, "failingWriter failed after writting 0 bytes", err) +} + +func TestTomlTreeWriteToFailingWriterInTable(t *testing.T) { + toml, _ := Load(` +[b] +a = 2`) + writer := failingWriter{failAt: 2, written: 0} + _, err := toml.WriteTo(writer) + assertErrorString(t, "failingWriter failed after writting 2 bytes", err) + + writer = failingWriter{failAt: 13, written: 0} + _, err = toml.WriteTo(writer) + assertErrorString(t, "failingWriter failed after writting 13 bytes", err) +} + +func TestTomlTreeWriteToFailingWriterInArray(t *testing.T) { + toml, _ := Load(` +[[b]] +a = 2`) + writer := failingWriter{failAt: 2, written: 0} + _, err := toml.WriteTo(writer) + assertErrorString(t, "failingWriter failed after writting 2 bytes", err) + + writer = failingWriter{failAt: 15, written: 0} + _, err = toml.WriteTo(writer) + assertErrorString(t, "failingWriter failed after writting 15 bytes", err) +} + +func TestTomlTreeWriteToMapExampleFile(t *testing.T) { tree, _ := LoadFile("example.toml") expected := map[string]interface{}{ "title": "TOML Example", @@ -131,7 +230,7 @@ func TestTomlTreeConversionToMapExampleFile(t *testing.T) { testMaps(t, tree.ToMap(), expected) } -func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) { +func TestTomlTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) { tree, _ := Load(` [[menu.main]] a = "menu 1" @@ -152,7 +251,7 @@ func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) { testMaps(t, treeMap, expected) } -func TestTomlTreeConversionToMapWithArrayOfInlineTables(t *testing.T) { +func TestTomlTreeWriteToMapWithArrayOfInlineTables(t *testing.T) { tree, _ := Load(` [params] language_tabs = [