Rewrite TomlTree encoding (#133)

* Rewrite `TomlTree` encoding
* Introduce `TomlTree.WriteTo`
This commit is contained in:
Thomas Pelletier
2017-03-02 09:17:06 -08:00
committed by GitHub
parent 3616783228
commit 7e6e4b1314
5 changed files with 331 additions and 279 deletions
+7 -39
View File
@@ -633,22 +633,13 @@ func TestParseKeyGroupArraySpec(t *testing.T) {
})
}
func TestToTomlValue(t *testing.T) {
func TestTomlValueStringRepresentation(t *testing.T) {
for idx, item := range []struct {
Value interface{}
Expect string
}{
{int(1), "1"},
{int8(2), "2"},
{int16(3), "3"},
{int32(4), "4"},
{int64(12345), "12345"},
{uint(10), "10"},
{uint8(20), "20"},
{uint16(30), "30"},
{uint32(40), "40"},
{uint64(50), "50"},
{float32(12.456), "12.456"},
{float64(123.45), "123.45"},
{bool(true), "true"},
{"hello world", "\"hello world\""},
@@ -660,42 +651,19 @@ func TestToTomlValue(t *testing.T) {
"[\"gamma\",\"delta\"]"},
{nil, ""},
} {
result := toTomlValue(item.Value, 0)
result, err := tomlValueStringRepresentation(item.Value)
if err != nil {
t.Errorf("Test %d - unexpected error: %s", idx, err)
}
if result != item.Expect {
t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect)
}
}
}
func TestToString(t *testing.T) {
tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
if err != nil {
t.Errorf("Test failed to parse: %v", err)
return
}
result, err := tree.ToString()
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
expected := "\n[foo]\n\n [[foo.bar]]\n a = 42\n\n [[foo.bar]]\n a = 69\n"
if result != expected {
t.Errorf("Expected got '%s', expected '%s'", result, expected)
}
}
func TestToStringMapStringString(t *testing.T) {
in := map[string]interface{}{"m": map[string]string{"v": "abc"}}
want := "\n[m]\n v = \"abc\"\n"
tree := TreeFromMap(in)
got := tree.String()
if got != want {
t.Errorf("want:\n%q\ngot:\n%q", want, got)
}
}
func TestToStringMapInterfaceInterface(t *testing.T) {
in := map[string]interface{}{"m": map[interface{}]interface{}{"v": "abc"}}
in := map[string]interface{}{"m": TreeFromMap(map[string]interface{}{
"v": &tomlValue{"abc", Position{0, 0}}})}
want := "\n[m]\n v = \"abc\"\n"
tree := TreeFromMap(in)
got := tree.String()
+2 -2
View File
@@ -10,13 +10,13 @@ import (
)
type tomlValue struct {
value interface{}
value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list
position Position
}
// TomlTree is the result of the parsing of a TOML file.
type TomlTree struct {
values map[string]interface{}
values map[string]interface{} // string -> *tomlValue, *TomlTree, []*TomlTree
position Position
}
-227
View File
@@ -1,227 +0,0 @@
package toml
// Tools to convert a TomlTree to different representations
import (
"fmt"
"strconv"
"strings"
"time"
)
// encodes a string to a TOML-compliant string value
func encodeTomlString(value string) string {
result := ""
for _, rr := range value {
intRr := uint16(rr)
switch rr {
case '\b':
result += "\\b"
case '\t':
result += "\\t"
case '\n':
result += "\\n"
case '\f':
result += "\\f"
case '\r':
result += "\\r"
case '"':
result += "\\\""
case '\\':
result += "\\\\"
default:
if intRr < 0x001F {
result += fmt.Sprintf("\\u%0.4X", intRr)
} else {
result += string(rr)
}
}
}
return result
}
// Value print support function for ToString()
// Outputs the TOML compliant string representation of a value
func toTomlValue(item interface{}, indent int) string {
tab := strings.Repeat(" ", indent)
switch value := item.(type) {
case int:
return tab + strconv.FormatInt(int64(value), 10)
case int8:
return tab + strconv.FormatInt(int64(value), 10)
case int16:
return tab + strconv.FormatInt(int64(value), 10)
case int32:
return tab + strconv.FormatInt(int64(value), 10)
case int64:
return tab + strconv.FormatInt(value, 10)
case uint:
return tab + strconv.FormatUint(uint64(value), 10)
case uint8:
return tab + strconv.FormatUint(uint64(value), 10)
case uint16:
return tab + strconv.FormatUint(uint64(value), 10)
case uint32:
return tab + strconv.FormatUint(uint64(value), 10)
case uint64:
return tab + strconv.FormatUint(value, 10)
case float32:
return tab + strconv.FormatFloat(float64(value), 'f', -1, 32)
case float64:
return tab + strconv.FormatFloat(value, 'f', -1, 64)
case string:
return tab + "\"" + encodeTomlString(value) + "\""
case bool:
if value {
return "true"
}
return "false"
case time.Time:
return tab + value.Format(time.RFC3339)
case []interface{}:
values := []string{}
for _, item := range value {
values = append(values, toTomlValue(item, 0))
}
return "[" + strings.Join(values, ",") + "]"
case nil:
return ""
default:
panic(fmt.Errorf("unsupported value type %T: %v", value, value))
}
}
// Recursive support function for ToString()
// Outputs a tree, using the provided keyspace to prefix table names
func (t *TomlTree) toToml(indent, keyspace string) string {
resultChunks := []string{}
for k, v := range t.values {
// figure out the keyspace
combinedKey := k
if keyspace != "" {
combinedKey = keyspace + "." + combinedKey
}
resultChunk := ""
// output based on type
switch node := v.(type) {
case []*TomlTree:
for _, item := range node {
if len(item.Keys()) > 0 {
resultChunk += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
}
resultChunk += item.toToml(indent+" ", combinedKey)
}
resultChunks = append(resultChunks, resultChunk)
case *TomlTree:
if len(node.Keys()) > 0 {
resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
}
resultChunk += node.toToml(indent+" ", combinedKey)
resultChunks = append(resultChunks, resultChunk)
case map[string]interface{}:
sub := TreeFromMap(node)
if len(sub.Keys()) > 0 {
resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
}
resultChunk += sub.toToml(indent+" ", combinedKey)
resultChunks = append(resultChunks, resultChunk)
case map[string]string:
sub := TreeFromMap(convertMapStringString(node))
if len(sub.Keys()) > 0 {
resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
}
resultChunk += sub.toToml(indent+" ", combinedKey)
resultChunks = append(resultChunks, resultChunk)
case map[interface{}]interface{}:
sub := TreeFromMap(convertMapInterfaceInterface(node))
if len(sub.Keys()) > 0 {
resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
}
resultChunk += sub.toToml(indent+" ", combinedKey)
resultChunks = append(resultChunks, resultChunk)
case *tomlValue:
resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
resultChunks = append([]string{resultChunk}, resultChunks...)
default:
resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
resultChunks = append([]string{resultChunk}, resultChunks...)
}
}
return strings.Join(resultChunks, "")
}
// Same as ToToml(), but does not panic and returns an error
func (t *TomlTree) toTomlSafe(indent, keyspace string) (result string, err error) {
defer func() {
if r := recover(); r != nil {
result = ""
switch x := r.(type) {
case error:
err = x
default:
err = fmt.Errorf("unknown panic: %s", r)
}
}
}()
result = t.toToml(indent, keyspace)
return
}
func convertMapStringString(in map[string]string) map[string]interface{} {
result := make(map[string]interface{}, len(in))
for k, v := range in {
result[k] = v
}
return result
}
func convertMapInterfaceInterface(in map[interface{}]interface{}) map[string]interface{} {
result := make(map[string]interface{}, len(in))
for k, v := range in {
result[k.(string)] = v
}
return result
}
// ToString generates a human-readable representation of the current tree.
// Output spans multiple lines, and is suitable for ingest by a TOML parser.
// If the conversion cannot be performed, ToString returns a non-nil error.
func (t *TomlTree) ToString() (string, error) {
return t.toTomlSafe("", "")
}
// String generates a human-readable representation of the current tree.
// Alias of ToString.
func (t *TomlTree) String() string {
result, _ := t.ToString()
return result
}
// ToMap recursively generates a representation of the current tree using map[string]interface{}.
func (t *TomlTree) ToMap() map[string]interface{} {
result := map[string]interface{}{}
for k, v := range t.values {
switch node := v.(type) {
case []*TomlTree:
var array []interface{}
for _, item := range node {
array = append(array, item.ToMap())
}
result[k] = array
case *TomlTree:
result[k] = node.ToMap()
case map[string]interface{}:
sub := TreeFromMap(node)
result[k] = sub.ToMap()
case *tomlValue:
result[k] = node.value
}
}
return result
}
+212
View File
@@ -0,0 +1,212 @@
package toml
import (
"bytes"
"fmt"
"io"
"sort"
"strconv"
"strings"
"time"
)
// encodes a string to a TOML-compliant string value
func encodeTomlString(value string) string {
result := ""
for _, rr := range value {
switch rr {
case '\b':
result += "\\b"
case '\t':
result += "\\t"
case '\n':
result += "\\n"
case '\f':
result += "\\f"
case '\r':
result += "\\r"
case '"':
result += "\\\""
case '\\':
result += "\\\\"
default:
intRr := uint16(rr)
if intRr < 0x001F {
result += fmt.Sprintf("\\u%0.4X", intRr)
} else {
result += string(rr)
}
}
}
return result
}
func tomlValueStringRepresentation(v interface{}) (string, error) {
switch value := v.(type) {
case uint64:
return strconv.FormatUint(value, 10), nil
case int64:
return strconv.FormatInt(value, 10), nil
case float64:
return strconv.FormatFloat(value, 'f', -1, 32), nil
case string:
return "\"" + encodeTomlString(value) + "\"", nil
case bool:
if value {
return "true", nil
}
return "false", nil
case time.Time:
return value.Format(time.RFC3339), nil
case nil:
return "", nil
case []interface{}:
values := []string{}
for _, item := range value {
itemRepr, err := tomlValueStringRepresentation(item)
if err != nil {
return "", err
}
values = append(values, itemRepr)
}
return "[" + strings.Join(values, ",") + "]", nil
default:
return "", fmt.Errorf("unsupported value type %T: %v", value, value)
}
}
func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) {
simpleValuesKeys := make([]string, 0)
complexValuesKeys := make([]string, 0)
for k := range t.values {
v := t.values[k]
switch v.(type) {
case *TomlTree, []*TomlTree:
complexValuesKeys = append(complexValuesKeys, k)
default:
simpleValuesKeys = append(simpleValuesKeys, k)
}
}
sort.Strings(simpleValuesKeys)
sort.Strings(complexValuesKeys)
for _, k := range simpleValuesKeys {
v, ok := t.values[k].(*tomlValue)
if !ok {
return bytesCount, fmt.Errorf("invalid key type at %s: %T", k, t.values[k])
}
repr, err := tomlValueStringRepresentation(v.value)
if err != nil {
return bytesCount, err
}
kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr)
writtenBytesCount, err := w.Write([]byte(kvRepr))
bytesCount += int64(writtenBytesCount)
if err != nil {
return bytesCount, err
}
}
for _, k := range complexValuesKeys {
v := t.values[k]
combinedKey := k
if keyspace != "" {
combinedKey = keyspace + "." + combinedKey
}
switch node := v.(type) {
// node has to be of those two types given how keys are sorted above
case *TomlTree:
tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
writtenBytesCount, err := w.Write([]byte(tableName))
bytesCount += int64(writtenBytesCount)
if err != nil {
return bytesCount, err
}
bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount)
if err != nil {
return bytesCount, err
}
case []*TomlTree:
for _, subTree := range node {
if len(subTree.values) > 0 {
tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
writtenBytesCount, err := w.Write([]byte(tableArrayName))
bytesCount += int64(writtenBytesCount)
if err != nil {
return bytesCount, err
}
bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount)
if err != nil {
return bytesCount, err
}
}
}
}
}
return bytesCount, nil
}
// WriteTo encode the TomlTree as Toml and writes it to the writer w.
// Returns the number of bytes written in case of success, or an error if anything happened.
func (t *TomlTree) WriteTo(w io.Writer) (int64, error) {
return t.writeTo(w, "", "", 0)
}
// ToTomlString generates a human-readable representation of the current tree.
// Output spans multiple lines, and is suitable for ingest by a TOML parser.
// If the conversion cannot be performed, ToString returns a non-nil error.
func (t *TomlTree) ToTomlString() (string, error) {
var buf bytes.Buffer
_, err := t.WriteTo(&buf)
if err != nil {
return "", err
}
return buf.String(), nil
}
// String generates a human-readable representation of the current tree.
// Alias of ToString. Present to implement the fmt.Stringer interface.
func (t *TomlTree) String() string {
result, _ := t.ToTomlString()
return result
}
// ToMap recursively generates a representation of the tree using Go built-in structures.
// The following types are used:
// * uint64
// * int64
// * bool
// * string
// * time.Time
// * map[string]interface{} (where interface{} is any of this list)
// * []interface{} (where interface{} is any of this list)
func (t *TomlTree) ToMap() map[string]interface{} {
result := map[string]interface{}{}
for k, v := range t.values {
switch node := v.(type) {
case []*TomlTree:
var array []interface{}
for _, item := range node {
array = append(array, item.ToMap())
}
result[k] = array
case *TomlTree:
result[k] = node.ToMap()
case map[string]interface{}:
sub := TreeFromMap(node)
result[k] = sub.ToMap()
case *tomlValue:
result[k] = node.value
}
}
return result
}
@@ -1,14 +1,46 @@
package toml
import (
"bytes"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
)
func TestTomlTreeConversionToString(t *testing.T) {
type failingWriter struct {
failAt int
written int
buffer bytes.Buffer
}
func (f failingWriter) Write(p []byte) (n int, err error) {
count := len(p)
toWrite := f.failAt - count + f.written
if toWrite < 0 {
toWrite = 0
}
if toWrite > count {
f.written += count
f.buffer.WriteString(string(p))
return count, nil
}
f.buffer.WriteString(string(p[:toWrite]))
f.written = f.failAt
return f.written, fmt.Errorf("failingWriter failed after writting %d bytes", f.written)
}
func assertErrorString(t *testing.T, expected string, err error) {
expectedErr := errors.New(expected)
if err.Error() != expectedErr.Error() {
t.Errorf("expecting error %s, but got %s instead", expected, err)
}
}
func TestTomlTreeWriteToTomlString(t *testing.T) {
toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
points = { x = 1, y = 2 }`)
@@ -16,7 +48,7 @@ points = { x = 1, y = 2 }`)
t.Fatal("Unexpected error:", err)
}
tomlString, _ := toml.ToString()
tomlString, _ := toml.ToTomlString()
reparsedTree, err := Load(tomlString)
assertTree(t, reparsedTree, err, map[string]interface{}{
@@ -31,7 +63,23 @@ points = { x = 1, y = 2 }`)
})
}
func TestTomlTreeConversionToStringKeysOrders(t *testing.T) {
func TestTomlTreeWriteToTomlStringSimple(t *testing.T) {
tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
if err != nil {
t.Errorf("Test failed to parse: %v", err)
return
}
result, err := tree.ToTomlString()
if err != nil {
t.Errorf("Unexpected error: %s", err)
}
expected := "\n[foo]\n\n [[foo.bar]]\n a = 42\n\n [[foo.bar]]\n a = 69\n"
if result != expected {
t.Errorf("Expected got '%s', expected '%s'", result, expected)
}
}
func TestTomlTreeWriteToTomlStringKeysOrders(t *testing.T) {
for i := 0; i < 100; i++ {
tree, _ := Load(`
foobar = true
@@ -41,7 +89,7 @@ func TestTomlTreeConversionToStringKeysOrders(t *testing.T) {
foo = 1
bar = "baz2"`)
stringRepr, _ := tree.ToString()
stringRepr, _ := tree.ToTomlString()
t.Log("Intermediate string representation:")
t.Log(stringRepr)
@@ -71,20 +119,20 @@ func testMaps(t *testing.T, actual, expected map[string]interface{}) {
}
}
func TestToStringTypeConversionError(t *testing.T) {
func TestToTomlStringTypeConversionError(t *testing.T) {
tree := TomlTree{
values: map[string]interface{}{
"thing": []string{"unsupported"},
"thing": &tomlValue{[]string{"unsupported"}, Position{}},
},
}
_, err := tree.ToString()
_, err := tree.ToTomlString()
expected := errors.New("unsupported value type []string: [unsupported]")
if err.Error() != expected.Error() {
t.Errorf("expecting error %s, but got %s instead", expected, err)
}
}
func TestTomlTreeConversionToMapSimple(t *testing.T) {
func TestTomlTreeWriteToMapSimple(t *testing.T) {
tree, _ := Load("a = 42\nb = 17")
expected := map[string]interface{}{
@@ -95,7 +143,58 @@ func TestTomlTreeConversionToMapSimple(t *testing.T) {
testMaps(t, tree.ToMap(), expected)
}
func TestTomlTreeConversionToMapExampleFile(t *testing.T) {
func TestTomlTreeWriteToInvalidTreeSimpleValue(t *testing.T) {
tree := TomlTree{values: map[string]interface{}{"foo": int8(1)}}
_, err := tree.ToTomlString()
assertErrorString(t, "invalid key type at foo: int8", err)
}
func TestTomlTreeWriteToInvalidTreeTomlValue(t *testing.T) {
tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{int8(1), Position{}}}}
_, err := tree.ToTomlString()
assertErrorString(t, "unsupported value type int8: 1", err)
}
func TestTomlTreeWriteToInvalidTreeTomlValueArray(t *testing.T) {
tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{[]interface{}{int8(1)}, Position{}}}}
_, err := tree.ToTomlString()
assertErrorString(t, "unsupported value type int8: 1", err)
}
func TestTomlTreeWriteToFailingWriterInSimpleValue(t *testing.T) {
toml, _ := Load(`a = 2`)
writer := failingWriter{failAt: 0, written: 0}
_, err := toml.WriteTo(writer)
assertErrorString(t, "failingWriter failed after writting 0 bytes", err)
}
func TestTomlTreeWriteToFailingWriterInTable(t *testing.T) {
toml, _ := Load(`
[b]
a = 2`)
writer := failingWriter{failAt: 2, written: 0}
_, err := toml.WriteTo(writer)
assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
writer = failingWriter{failAt: 13, written: 0}
_, err = toml.WriteTo(writer)
assertErrorString(t, "failingWriter failed after writting 13 bytes", err)
}
func TestTomlTreeWriteToFailingWriterInArray(t *testing.T) {
toml, _ := Load(`
[[b]]
a = 2`)
writer := failingWriter{failAt: 2, written: 0}
_, err := toml.WriteTo(writer)
assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
writer = failingWriter{failAt: 15, written: 0}
_, err = toml.WriteTo(writer)
assertErrorString(t, "failingWriter failed after writting 15 bytes", err)
}
func TestTomlTreeWriteToMapExampleFile(t *testing.T) {
tree, _ := LoadFile("example.toml")
expected := map[string]interface{}{
"title": "TOML Example",
@@ -131,7 +230,7 @@ func TestTomlTreeConversionToMapExampleFile(t *testing.T) {
testMaps(t, tree.ToMap(), expected)
}
func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) {
func TestTomlTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) {
tree, _ := Load(`
[[menu.main]]
a = "menu 1"
@@ -152,7 +251,7 @@ func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) {
testMaps(t, treeMap, expected)
}
func TestTomlTreeConversionToMapWithArrayOfInlineTables(t *testing.T) {
func TestTomlTreeWriteToMapWithArrayOfInlineTables(t *testing.T) {
tree, _ := Load(`
[params]
language_tabs = [