Fuzzing setup and fixes (#755)

* encode: fix localdate formatting
* encode: fix empty key marshaling
* encode: fix invalid quotation of time.Time
* encode: ensure control chars are escaped
* decode: always use UTC for zero tz
* encode: check for invalid characters in keys
* encode: always construct map for empty array tables
* fuzz: add go 1.18 fuzz test
* encode: handle NaNs
* encode: allow new lines in quoted keys
* encode: never emit table inside array
* encode: don't capitalize inf

.gitattributes

@@ -1,3 +1,4 @@
 * text=auto
 
 benchmark/benchmark.toml text eol=lf
+testdata/** text eol=lf

ci.sh

@@ -76,7 +76,8 @@ cover() {
     fi
 
     pushd "$dir"
-    go test -covermode=atomic -coverprofile=coverage.out ./...
+    go test -covermode=atomic  -coverpkg=./... -coverprofile=coverage.out.tmp ./...
+    cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
     go tool cover -func=coverage.out
     popd
 
@@ -103,8 +104,8 @@ coverage() {
 
 	    echo ""
 
-	    target_pct="$(cat ${target_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
+	    target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
-	    head_pct="$(cat ${head_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
+	    head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
 	    echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
 
 	    delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
@@ -112,6 +113,13 @@ coverage() {
 
 	    if [[ $delta_pct = \-* ]]; then
 		    echo "Regression!";
+
+            target_diff="${output_dir}/target.diff.txt"
+            head_diff="${output_dir}/head.diff.txt"
+            cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
+            cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
+
+            diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
 		    return 1
 	    fi
 	    return 0

decode.go

@@ -130,7 +130,11 @@ func parseDateTime(b []byte) (time.Time, error) {
 		}
 
 		seconds := direction * (hours*3600 + minutes*60)
+		if seconds == 0 {
+			zone = time.UTC
+		} else {
 			zone = time.FixedZone("", seconds)
+		}
 		b = b[dateTimeByteLen:]
 	}
 

fuzz_test.go

@@ -0,0 +1,56 @@
+//go:build go1.18
+// +build go1.18
+
+package toml_test
+
+import (
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func FuzzUnmarshal(f *testing.F) {
+	file, err := ioutil.ReadFile("benchmark/benchmark.toml")
+	if err != nil {
+		panic(err)
+	}
+	f.Add(file)
+
+	f.Fuzz(func(t *testing.T, b []byte) {
+		if strings.Contains(string(b), "nan") {
+			// Current limitation of testify.
+			// https://github.com/stretchr/testify/issues/624
+			t.Skip("can't compare NaNs")
+		}
+
+		t.Log("INITIAL DOCUMENT ===========================")
+		t.Log(string(b))
+
+		var v interface{}
+		err := toml.Unmarshal(b, &v)
+		if err != nil {
+			return
+		}
+
+		t.Log("DECODED VALUE ===========================")
+		t.Logf("%#+v", v)
+
+		encoded, err := toml.Marshal(v)
+		if err != nil {
+			t.Fatalf("cannot marshal unmarshaled document: %s", err)
+		}
+
+		t.Log("ENCODED DOCUMENT ===========================")
+		t.Log(string(encoded))
+
+		var v2 interface{}
+		err = toml.Unmarshal(encoded, &v2)
+		if err != nil {
+			t.Fatalf("failed round trip: %s", err)
+		}
+		require.Equal(t, v, v2)
+	})
+}

marshaler.go

@@ -208,11 +208,20 @@ func (ctx *encoderCtx) isRoot() bool {
 }
 
 func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
-	if !v.IsZero() {
+	i := v.Interface()
-		i, ok := v.Interface().(time.Time)
+
-		if ok {
+	switch x := i.(type) {
-			return i.AppendFormat(b, time.RFC3339), nil
+	case time.Time:
+		if x.Nanosecond() > 0 {
+			return x.AppendFormat(b, time.RFC3339Nano), nil
 		}
+		return x.AppendFormat(b, time.RFC3339), nil
+	case LocalTime:
+		return append(b, x.String()...), nil
+	case LocalDate:
+		return append(b, x.String()...), nil
+	case LocalDateTime:
+		return append(b, x.String()...), nil
 	}
 
 	hasTextMarshaler := v.Type().Implements(textMarshalerType)
@@ -260,16 +269,31 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
 	case reflect.String:
 		b = enc.encodeString(b, v.String(), ctx.options)
 	case reflect.Float32:
-		if math.Trunc(v.Float()) == v.Float() {
+		f := v.Float()
-			b = strconv.AppendFloat(b, v.Float(), 'f', 1, 32)
+
+		if math.IsNaN(f) {
+			b = append(b, "nan"...)
+		} else if f > math.MaxFloat32 {
+			b = append(b, "inf"...)
+		} else if f < -math.MaxFloat32 {
+			b = append(b, "-inf"...)
+		} else if math.Trunc(f) == f {
+			b = strconv.AppendFloat(b, f, 'f', 1, 32)
 		} else {
-			b = strconv.AppendFloat(b, v.Float(), 'f', -1, 32)
+			b = strconv.AppendFloat(b, f, 'f', -1, 32)
 		}
 	case reflect.Float64:
-		if math.Trunc(v.Float()) == v.Float() {
+		f := v.Float()
-			b = strconv.AppendFloat(b, v.Float(), 'f', 1, 64)
+		if math.IsNaN(f) {
+			b = append(b, "nan"...)
+		} else if f > math.MaxFloat64 {
+			b = append(b, "inf"...)
+		} else if f < -math.MaxFloat64 {
+			b = append(b, "-inf"...)
+		} else if math.Trunc(f) == f {
+			b = strconv.AppendFloat(b, f, 'f', 1, 64)
 		} else {
-			b = strconv.AppendFloat(b, v.Float(), 'f', -1, 64)
+			b = strconv.AppendFloat(b, f, 'f', -1, 64)
 		}
 	case reflect.Bool:
 		if v.Bool() {
@@ -300,10 +324,6 @@ func isNil(v reflect.Value) bool {
 func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v reflect.Value) ([]byte, error) {
 	var err error
 
-	if !ctx.hasKey {
-		panic("caller of encodeKv should have set the key in the context")
-	}
-
 	if (ctx.options.omitempty || options.omitempty) && isEmptyValue(v) {
 		return b, nil
 	}
@@ -313,12 +333,7 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
 	}
 
 	b = enc.indent(ctx.indent, b)
-
+	b = enc.encodeKey(b, ctx.key)
-	b, err = enc.encodeKey(b, ctx.key)
-	if err != nil {
-		return nil, err
-	}
-
 	b = append(b, " = "...)
 
 	// create a copy of the context because the value of a KV shouldn't
@@ -365,7 +380,13 @@ func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byt
 }
 
 func needsQuoting(v string) bool {
-	return strings.ContainsAny(v, "'\b\f\n\r\t")
+	// TODO: vectorize
+	for _, b := range []byte(v) {
+		if b == '\'' || b == '\r' || b == '\n' || invalidAscii(b) {
+			return true
+		}
+	}
+	return false
 }
 
 // caller should have checked that the string does not contain new lines or ' .
@@ -437,7 +458,7 @@ func (enc *Encoder) encodeQuotedString(multiline bool, b []byte, v string) []byt
 	return b
 }
 
-// called should have checked that the string is in A-Z / a-z / 0-9 / - / _ .
+// caller should have checked that the string is in A-Z / a-z / 0-9 / - / _ .
 func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte {
 	return append(b, v...)
 }
@@ -453,20 +474,11 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
 
 	b = append(b, '[')
 
-	var err error
+	b = enc.encodeKey(b, ctx.parentKey[0])
-
-	b, err = enc.encodeKey(b, ctx.parentKey[0])
-	if err != nil {
-		return nil, err
-	}
 
 	for _, k := range ctx.parentKey[1:] {
 		b = append(b, '.')
-
+		b = enc.encodeKey(b, k)
-		b, err = enc.encodeKey(b, k)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	b = append(b, "]\n"...)
@@ -475,19 +487,19 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
 }
 
 //nolint:cyclop
-func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
+func (enc *Encoder) encodeKey(b []byte, k string) []byte {
 	needsQuotation := false
 	cannotUseLiteral := false
 
+	if len(k) == 0 {
+		return append(b, "''"...)
+	}
+
 	for _, c := range k {
 		if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
 			continue
 		}
 
-		if c == '\n' {
-			return nil, fmt.Errorf("toml: new line characters in keys are not supported")
-		}
-
 		if c == literalQuote {
 			cannotUseLiteral = true
 		}
@@ -495,13 +507,17 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
 		needsQuotation = true
 	}
 
+	if needsQuotation && needsQuoting(k) {
+		cannotUseLiteral = true
+	}
+
 	switch {
 	case cannotUseLiteral:
-		return enc.encodeQuotedString(false, b, k), nil
+		return enc.encodeQuotedString(false, b, k)
 	case needsQuotation:
-		return enc.encodeLiteralString(b, k), nil
+		return enc.encodeLiteralString(b, k)
 	default:
-		return enc.encodeUnquotedKey(b, k), nil
+		return enc.encodeUnquotedKey(b, k)
 	}
 }
 
@@ -803,6 +819,9 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
 }
 
 func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool {
+	if ctx.insideKv {
+		return false
+	}
 	t := v.Type()
 
 	if t.Kind() == reflect.Interface {
@@ -848,7 +867,6 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by
 func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
 	ctx.shiftKey()
 
-	var err error
 	scratch := make([]byte, 0, 64)
 	scratch = append(scratch, "[["...)
 
@@ -857,10 +875,7 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
 			scratch = append(scratch, '.')
 		}
 
-		scratch, err = enc.encodeKey(scratch, k)
+		scratch = enc.encodeKey(scratch, k)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	scratch = append(scratch, "]]\n"...)
@@ -869,6 +884,7 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
 	for i := 0; i < v.Len(); i++ {
 		b = append(b, scratch...)
 
+		var err error
 		b, err = enc.encode(b, ctx, v.Index(i))
 		if err != nil {
 			return nil, err

marshaler_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math"
 	"math/big"
 	"strings"
 	"testing"
@@ -45,7 +46,7 @@ func TestMarshal(t *testing.T) {
 			v: map[string]string{
 				"hel\nlo": "world",
 			},
-			err: true,
+			expected: `"hel\nlo" = 'world'`,
 		},
 		{
 			desc: `map with " in key`,
@@ -380,7 +381,8 @@ hello = 'world'`,
 			v: map[string][]map[string]string{
 				"a\n": {{"hello": "world"}},
 			},
-			err: true,
+			expected: `[["a\n"]]
+hello = 'world'`,
 		},
 		{
 			desc: "newline in map in slice",
@@ -440,7 +442,7 @@ hello = 'world'`,
 			v: map[string]interface{}{
 				"hello\nworld": 42,
 			},
-			err: true,
+			expected: `"hello\nworld" = 42`,
 		},
 		{
 			desc: "new line in parent of nested table key",
@@ -449,7 +451,8 @@ hello = 'world'`,
 					"inner": 42,
 				},
 			},
-			err: true,
+			expected: `["hello\nworld"]
+inner = 42`,
 		},
 		{
 			desc: "new line in nested table key",
@@ -460,7 +463,9 @@ hello = 'world'`,
 					},
 				},
 			},
-			err: true,
+			expected: `[parent]
+[parent."in\ner"]
+foo = 42`,
 		},
 		{
 			desc: "invalid map key",
@@ -483,7 +488,16 @@ hello = 'world'`,
 			}{
 				T: time.Time{},
 			},
-			expected: `T = '0001-01-01T00:00:00Z'`,
+			expected: `T = 0001-01-01T00:00:00Z`,
+		},
+		{
+			desc: "time nano",
+			v: struct {
+				T time.Time
+			}{
+				T: time.Date(1979, time.May, 27, 0, 32, 0, 999999000, time.UTC),
+			},
+			expected: `T = 1979-05-27T00:32:00.999999Z`,
 		},
 		{
 			desc: "bool",
@@ -656,6 +670,33 @@ func equalStringsIgnoreNewlines(t *testing.T, expected string, actual string) {
 	assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset))
 }
 
+func TestMarshalFloats(t *testing.T) {
+	v := map[string]float32{
+		"nan":  float32(math.NaN()),
+		"+inf": float32(math.Inf(1)),
+		"-inf": float32(math.Inf(-1)),
+	}
+
+	expected := `'+inf' = inf
+-inf = -inf
+nan = nan
+`
+
+	actual, err := toml.Marshal(v)
+	require.NoError(t, err)
+	require.Equal(t, expected, string(actual))
+
+	v64 := map[string]float64{
+		"nan":  math.NaN(),
+		"+inf": math.Inf(1),
+		"-inf": math.Inf(-1),
+	}
+
+	actual, err = toml.Marshal(v64)
+	require.NoError(t, err)
+	require.Equal(t, expected, string(actual))
+}
+
 //nolint:funlen
 func TestMarshalIndentTables(t *testing.T) {
 	examples := []struct {
@@ -1027,6 +1068,24 @@ value = ''
 	require.Equal(t, expected, string(result))
 }
 
+func TestLocalTime(t *testing.T) {
+	v := map[string]toml.LocalTime{
+		"a": toml.LocalTime{
+			Hour:       1,
+			Minute:     2,
+			Second:     3,
+			Nanosecond: 4,
+		},
+	}
+
+	expected := `a = 01:02:03.000000004
+`
+
+	out, err := toml.Marshal(v)
+	require.NoError(t, err)
+	require.Equal(t, expected, string(out))
+}
+
 func ExampleMarshal() {
 	type MyConfig struct {
 		Version int

testdata/fuzz/FuzzUnmarshal/0c08b924aa26af23d16b77029b4f5567a96af93402e0472fef7b837b37580d7c

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=0000-01-01 00:00:00")

testdata/fuzz/FuzzUnmarshal/249a017d8ffa0d88d41c594ada5399f764833f64050180cb39f106d12666853f

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\"\\n\"=\"\"")

testdata/fuzz/FuzzUnmarshal/391ce67b866d58464d067c5981e99951fd499896453a2fe2779a6bfe7df11bf5

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("''=0")

testdata/fuzz/FuzzUnmarshal/49ae83283c5cfbed874bcacbfb342de107b69c2a3d3c35c2d99f67345fe89946

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=0000-01-01")

testdata/fuzz/FuzzUnmarshal/69ea82e85bb4ad966ae86bb05eea3af69ed1a826c846f1bf2282f329b4d8fb36

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=\"\"\"\\U00000000\"\"\"")

testdata/fuzz/FuzzUnmarshal/746910fac6deb42e7e62dce60b333c81c39d844a122c9075894c023400e8fdbf

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=[[{}]]")

testdata/fuzz/FuzzUnmarshal/844d1171a3adc95e51627f0e9378bbaa193fe2d0284f4ba8ed19a5226a8cbc74

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\"\\b\"=\"\"")

testdata/fuzz/FuzzUnmarshal/900c54b5071113ae2a5365b3156377ddd775b9d38add61629544903cb9c94b00

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=inf")

testdata/fuzz/FuzzUnmarshal/c87e8eb4e8fa40fd3c89d8ed16a049ce81592ded710965ee29a4e506eb5427b3

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=0000-01-01 00:00:00+00:00")

testdata/fuzz/FuzzUnmarshal/d0e5d6fc3b865cfae5480c01f301b87c932722ae0c1d692deea413349ea028be

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=[{}]")

testdata/fuzz/FuzzUnmarshal/d2cecea5b8ee5f148419671cef0644178cdba1ee91d3f295be38da576e89ef40

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0=nan")

unmarshaler.go

@@ -322,10 +322,12 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val
 		return v, nil
 	case reflect.Slice:
 		elemType := v.Type().Elem()
+		var elem reflect.Value
 		if elemType.Kind() == reflect.Interface {
-			elemType = mapStringInterfaceType
+			elem = makeMapStringInterface()
+		} else {
+			elem = reflect.New(elemType).Elem()
 		}
-		elem := reflect.New(elemType).Elem()
 		elem2, err := d.handleArrayTable(key, elem)
 		if err != nil {
 			return reflect.Value{}, err

unmarshaler_test.go

@@ -971,7 +971,7 @@ B = "data"`,
 								"Name": "Hammer",
 								"Sku":  int64(738594937),
 							},
-							map[string]interface{}(nil),
+							map[string]interface{}{},
 							map[string]interface{}{
 								"Name":  "Nail",
 								"Sku":   int64(284758393),
@@ -1505,7 +1505,7 @@ B = "data"`,
 					target: &map[string]interface{}{},
 					expected: &map[string]interface{}{
 						"products": []interface{}{
-							map[string]interface{}(nil),
+							map[string]interface{}{},
 						},
 					},
 				}