Use PtrTo to not require Go 1.18 (#874)

Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.2 to 1.8.3.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.8.2...v1.8.3)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/coverage.yml

@@ -13,7 +13,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "1.20"
       - name: Run tests with coverage

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "1.20"
       - name: Login to GitHub Container Registry

.github/workflows/workflow.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup go ${{ matrix.go }}
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
       - name: Run unit tests

ci.sh

@@ -77,7 +77,7 @@ cover() {
 
     pushd "$dir"
     go test -covermode=atomic  -coverpkg=./... -coverprofile=coverage.out.tmp ./...
-    cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
+    cat coverage.out.tmp | grep -v fuzz | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
     go tool cover -func=coverage.out
     popd
 

example_text_marshaling_test.go

@@ -0,0 +1,37 @@
+package toml_test
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+type customInt int
+
+func (i *customInt) UnmarshalText(b []byte) error {
+	x, err := strconv.ParseInt(string(b), 10, 32)
+	if err != nil {
+		return err
+	}
+	*i = customInt(x * 100)
+	return nil
+}
+
+type doc struct {
+	Value customInt
+}
+
+func ExampleUnmarshal_textUnmarshal() {
+	var x doc
+
+	data := []byte(`value  = "42"`)
+	err := toml.Unmarshal(data, &x)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(x)
+	// Output:
+	// {4200}
+}

go.mod

@@ -2,4 +2,4 @@ module github.com/pelletier/go-toml/v2
 
 go 1.16
 
-require github.com/stretchr/testify v1.8.1
+require github.com/stretchr/testify v1.8.3

go.sum

@@ -8,8 +8,8 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

marshaler.go

@@ -577,11 +577,23 @@ func (enc *Encoder) encodeKey(b []byte, k string) []byte {
 	}
 }
 
-func (enc *Encoder) encodeMap(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
-	if v.Type().Key().Kind() != reflect.String {
-		return nil, fmt.Errorf("toml: type %s is not supported as a map key", v.Type().Key().Kind())
-	}
+func (enc *Encoder) keyToString(k reflect.Value) (string, error) {
+	keyType := k.Type()
+	switch {
+	case keyType.Kind() == reflect.String:
+		return k.String(), nil
 
+	case keyType.Implements(textMarshalerType):
+		keyB, err := k.Interface().(encoding.TextMarshaler).MarshalText()
+		if err != nil {
+			return "", fmt.Errorf("toml: error marshalling key %v from text: %w", k, err)
+		}
+		return string(keyB), nil
+	}
+	return "", fmt.Errorf("toml: type %s is not supported as a map key", keyType.Kind())
+}
+
+func (enc *Encoder) encodeMap(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
 	var (
 		t                 table
 		emptyValueOptions valueOptions
@@ -589,13 +601,17 @@ func (enc *Encoder) encodeMap(b []byte, ctx encoderCtx, v reflect.Value) ([]byte
 
 	iter := v.MapRange()
 	for iter.Next() {
-		k := iter.Key().String()
 		v := iter.Value()
 
 		if isNil(v) {
 			continue
 		}
 
+		k, err := enc.keyToString(iter.Key())
+		if err != nil {
+			return nil, err
+		}
+
 		if willConvertToTableOrArrayTable(ctx, v) {
 			t.pushTable(k, v, emptyValueOptions)
 		} else {

marshaler_test.go

@@ -15,6 +15,21 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type marshalTextKey struct {
+	A string
+	B string
+}
+
+func (k marshalTextKey) MarshalText() ([]byte, error) {
+	return []byte(k.A + "-" + k.B), nil
+}
+
+type marshalBadTextKey struct{}
+
+func (k marshalBadTextKey) MarshalText() ([]byte, error) {
+	return nil, fmt.Errorf("error")
+}
+
 func TestMarshal(t *testing.T) {
 	someInt := 42
 
@@ -97,6 +112,53 @@ also = 'that'
 a = 'test'
 `,
 		},
+		{
+			desc: `map with text key`,
+			v: map[marshalTextKey]string{
+				{A: "a", B: "1"}: "value 1",
+				{A: "a", B: "2"}: "value 2",
+				{A: "b", B: "1"}: "value 3",
+			},
+			expected: `a-1 = 'value 1'
+a-2 = 'value 2'
+b-1 = 'value 3'
+`,
+		},
+		{
+			desc: `table with text key`,
+			v: map[marshalTextKey]map[string]string{
+				{A: "a", B: "1"}: {"value": "foo"},
+			},
+			expected: `[a-1]
+value = 'foo'
+`,
+		},
+		{
+			desc: `map with ptr text key`,
+			v: map[*marshalTextKey]string{
+				{A: "a", B: "1"}: "value 1",
+				{A: "a", B: "2"}: "value 2",
+				{A: "b", B: "1"}: "value 3",
+			},
+			expected: `a-1 = 'value 1'
+a-2 = 'value 2'
+b-1 = 'value 3'
+`,
+		},
+		{
+			desc: `map with bad text key`,
+			v: map[marshalBadTextKey]string{
+				{}: "value 1",
+			},
+			err: true,
+		},
+		{
+			desc: `map with bad ptr text key`,
+			v: map[*marshalBadTextKey]string{
+				{}: "value 1",
+			},
+			err: true,
+		},
 		{
 			desc: "simple string array",
 			v: map[string][]string{
@@ -487,9 +549,14 @@ foo = 42
 		},
 		{
 			desc: "invalid map key",
-			v:    map[int]interface{}{},
+			v:    map[int]interface{}{1: "a"},
 			err:  true,
 		},
+		{
+			desc:     "invalid map key but empty",
+			v:        map[int]interface{}{},
+			expected: "",
+		},
 		{
 			desc: "unhandled type",
 			v: struct {

ossfuzz/fuzz.go

@@ -0,0 +1,45 @@
+//go:build go1.18 || go1.19 || go1.20
+// +build go1.18 go1.19 go1.20
+
+package ossfuzz
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+func FuzzToml(data []byte) int {
+	if len(data) >= 2048 {
+		return 0
+	}
+
+	if strings.Contains(string(data), "nan") {
+		return 0
+	}
+
+	var v interface{}
+	err := toml.Unmarshal(data, &v)
+	if err != nil {
+		return 0
+	}
+
+	encoded, err := toml.Marshal(v)
+	if err != nil {
+		panic(fmt.Sprintf("failed to marshal unmarshaled document: %s", err))
+	}
+
+	var v2 interface{}
+	err = toml.Unmarshal(encoded, &v2)
+	if err != nil {
+		panic(fmt.Sprintf("failed round trip: %s", err))
+	}
+
+	if !reflect.DeepEqual(v, v2) {
+		panic(fmt.Sprintf("not equal: %#+v %#+v", v, v2))
+	}
+
+	return 1
+}

unmarshaler.go

@@ -417,7 +417,10 @@ func (d *decoder) handleKeyPart(key unstable.Iterator, v reflect.Value, nextFn h
 		vt := v.Type()
 
 		// Create the key for the map element. Convert to key type.
-		mk := reflect.ValueOf(string(key.Node().Data)).Convert(vt.Key())
+		mk, err := d.keyFromData(vt.Key(), key.Node().Data)
+		if err != nil {
+			return reflect.Value{}, err
+		}
 
 		// If the map does not exist, create it.
 		if v.IsNil() {
@@ -1009,6 +1012,31 @@ func (d *decoder) handleKeyValueInner(key unstable.Iterator, value *unstable.Nod
 	return reflect.Value{}, d.handleValue(value, v)
 }
 
+func (d *decoder) keyFromData(keyType reflect.Type, data []byte) (reflect.Value, error) {
+	switch {
+	case stringType.AssignableTo(keyType):
+		return reflect.ValueOf(string(data)), nil
+
+	case stringType.ConvertibleTo(keyType):
+		return reflect.ValueOf(string(data)).Convert(keyType), nil
+
+	case keyType.Implements(textUnmarshalerType):
+		mk := reflect.New(keyType.Elem())
+		if err := mk.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil {
+			return reflect.Value{}, fmt.Errorf("toml: error unmarshalling key type %s from text: %w", stringType, err)
+		}
+		return mk, nil
+
+	case reflect.PtrTo(keyType).Implements(textUnmarshalerType):
+		mk := reflect.New(keyType)
+		if err := mk.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil {
+			return reflect.Value{}, fmt.Errorf("toml: error unmarshalling key type %s from text: %w", stringType, err)
+		}
+		return mk.Elem(), nil
+	}
+	return reflect.Value{}, fmt.Errorf("toml: cannot convert map key of type %s to expected type %s", stringType, keyType)
+}
+
 func (d *decoder) handleKeyValuePart(key unstable.Iterator, value *unstable.Node, v reflect.Value) (reflect.Value, error) {
 	// contains the replacement for v
 	var rv reflect.Value
@@ -1019,16 +1047,9 @@ func (d *decoder) handleKeyValuePart(key unstable.Iterator, value *unstable.Node
 	case reflect.Map:
 		vt := v.Type()
 
-		mk := reflect.ValueOf(string(key.Node().Data))
-		mkt := stringType
-
-		keyType := vt.Key()
-		if !mkt.AssignableTo(keyType) {
-			if !mkt.ConvertibleTo(keyType) {
-				return reflect.Value{}, fmt.Errorf("toml: cannot convert map key of type %s to expected type %s", mkt, keyType)
-			}
-
-			mk = mk.Convert(keyType)
+		mk, err := d.keyFromData(vt.Key(), key.Node().Data)
+		if err != nil {
+			return reflect.Value{}, err
 		}
 
 		// If the map does not exist, create it.
@@ -1071,6 +1092,19 @@ func (d *decoder) handleKeyValuePart(key unstable.Iterator, value *unstable.Node
 		d.errorContext.Field = path
 
 		f := fieldByIndex(v, path)
+
+		if !f.CanSet() {
+			// If the field is not settable, need to take a slower path and make a copy of
+			// the struct itself to a new location.
+			nvp := reflect.New(v.Type())
+			nvp.Elem().Set(v)
+			v = nvp.Elem()
+			_, err := d.handleKeyValuePart(key, value, v)
+			if err != nil {
+				return reflect.Value{}, err
+			}
+			return nvp.Elem(), nil
+		}
 		x, err := d.handleKeyValueInner(key, value, f)
 		if err != nil {
 			return reflect.Value{}, err

unmarshaler_test.go

@@ -16,6 +16,27 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type unmarshalTextKey struct {
+	A string
+	B string
+}
+
+func (k *unmarshalTextKey) UnmarshalText(text []byte) error {
+	parts := strings.Split(string(text), "-")
+	if len(parts) != 2 {
+		return fmt.Errorf("invalid text key: %s", text)
+	}
+	k.A = parts[0]
+	k.B = parts[1]
+	return nil
+}
+
+type unmarshalBadTextKey struct{}
+
+func (k *unmarshalBadTextKey) UnmarshalText(text []byte) error {
+	return fmt.Errorf("error")
+}
+
 func ExampleDecoder_DisallowUnknownFields() {
 	type S struct {
 		Key1 string
@@ -69,7 +90,6 @@ func ExampleUnmarshal() {
 	fmt.Println("version:", cfg.Version)
 	fmt.Println("name:", cfg.Name)
 	fmt.Println("tags:", cfg.Tags)
-
 	// Output:
 	// version: 2
 	// name: go-toml
@@ -315,6 +335,7 @@ func TestUnmarshal(t *testing.T) {
 		target   interface{}
 		expected interface{}
 		err      bool
+		assert   func(t *testing.T, test test)
 	}
 	examples := []struct {
 		skip  bool
@@ -350,6 +371,96 @@ func TestUnmarshal(t *testing.T) {
 				}
 			},
 		},
+		{
+			desc:  "kv text key",
+			input: `a-1 = "foo"`,
+			gen: func() test {
+				type doc = map[unmarshalTextKey]string
+
+				return test{
+					target:   &doc{},
+					expected: &doc{{A: "a", B: "1"}: "foo"},
+				}
+			},
+		},
+		{
+			desc: "table text key",
+			input: `["a-1"]
+foo = "bar"`,
+			gen: func() test {
+				type doc = map[unmarshalTextKey]map[string]string
+
+				return test{
+					target:   &doc{},
+					expected: &doc{{A: "a", B: "1"}: map[string]string{"foo": "bar"}},
+				}
+			},
+		},
+		{
+			desc:  "kv ptr text key",
+			input: `a-1 = "foo"`,
+			gen: func() test {
+				type doc = map[*unmarshalTextKey]string
+
+				return test{
+					target:   &doc{},
+					expected: &doc{{A: "a", B: "1"}: "foo"},
+					assert: func(t *testing.T, test test) {
+						// Despite the documentation:
+						//     Pointer variable equality is determined based on the equality of the
+						// 		 referenced values (as opposed to the memory addresses).
+						// assert.Equal does not work properly with maps with pointer keys
+						// https://github.com/stretchr/testify/issues/1143
+						expected := make(map[unmarshalTextKey]string)
+						for k, v := range *(test.expected.(*doc)) {
+							expected[*k] = v
+						}
+						got := make(map[unmarshalTextKey]string)
+						for k, v := range *(test.target.(*doc)) {
+							got[*k] = v
+						}
+						assert.Equal(t, expected, got)
+					},
+				}
+			},
+		},
+		{
+			desc:  "kv bad text key",
+			input: `a-1 = "foo"`,
+			gen: func() test {
+				type doc = map[unmarshalBadTextKey]string
+
+				return test{
+					target: &doc{},
+					err:    true,
+				}
+			},
+		},
+		{
+			desc:  "kv bad ptr text key",
+			input: `a-1 = "foo"`,
+			gen: func() test {
+				type doc = map[*unmarshalBadTextKey]string
+
+				return test{
+					target: &doc{},
+					err:    true,
+				}
+			},
+		},
+		{
+			desc: "table bad text key",
+			input: `["a-1"]
+foo = "bar"`,
+			gen: func() test {
+				type doc = map[unmarshalBadTextKey]map[string]string
+
+				return test{
+					target: &doc{},
+					err:    true,
+				}
+			},
+		},
 		{
 			desc:  "time.time with negative zone",
 			input: `a = 1979-05-27T00:32:00-07:00 `, // space intentional
@@ -1521,6 +1632,16 @@ B = "data"`,
 				}
 			},
 		},
+		{
+			desc:  "empty map into map with invalid key type",
+			input: ``,
+			gen: func() test {
+				return test{
+					target:   &map[int]string{},
+					expected: &map[int]string{},
+				}
+			},
+		},
 		{
 			desc:  "into map with convertible key type",
 			input: `A = "hello"`,
@@ -1777,7 +1898,11 @@ B = "data"`,
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
-				assert.Equal(t, test.expected, test.target)
+				if test.assert != nil {
+					test.assert(t, test)
+				} else {
+					assert.Equal(t, test.expected, test.target)
+				}
 			}
 		})
 	}
@@ -2487,6 +2612,70 @@ func TestIssue851(t *testing.T) {
 	require.Equal(t, map[string]string{"a": "1", "b": "2"}, target.Params)
 }
 
+func TestIssue866(t *testing.T) {
+	type Pipeline struct {
+		Mapping map[string]struct {
+			Req [][]string `toml:"req"`
+			Res [][]string `toml:"res"`
+		} `toml:"mapping"`
+	}
+
+	type Pipelines struct {
+		PipelineMapping map[string]*Pipeline `toml:"pipelines"`
+	}
+
+	var badToml = `
+[pipelines.register]
+mapping.inst.req = [
+    ["param1", "value1"],
+]
+mapping.inst.res = [
+    ["param2", "value2"],
+]
+`
+
+	pipelines := new(Pipelines)
+	if err := toml.NewDecoder(bytes.NewBufferString(badToml)).DisallowUnknownFields().Decode(pipelines); err != nil {
+		t.Fatal(err)
+	}
+	if pipelines.PipelineMapping["register"].Mapping["inst"].Req[0][0] != "param1" {
+		t.Fatal("unmarshal failed with mismatch value")
+	}
+
+	var goodTooToml = `
+[pipelines.register]
+mapping.inst.req = [
+    ["param1", "value1"],
+]
+`
+
+	pipelines = new(Pipelines)
+	if err := toml.NewDecoder(bytes.NewBufferString(goodTooToml)).DisallowUnknownFields().Decode(pipelines); err != nil {
+		t.Fatal(err)
+	}
+	if pipelines.PipelineMapping["register"].Mapping["inst"].Req[0][0] != "param1" {
+		t.Fatal("unmarshal failed with mismatch value")
+	}
+
+	var goodToml = `
+[pipelines.register.mapping.inst]
+req = [
+    ["param1", "value1"],
+]
+res = [
+    ["param2", "value2"],
+]
+`
+
+	pipelines = new(Pipelines)
+	if err := toml.NewDecoder(bytes.NewBufferString(goodToml)).DisallowUnknownFields().Decode(pipelines); err != nil {
+		t.Fatal(err)
+	}
+	if pipelines.PipelineMapping["register"].Mapping["inst"].Req[0][0] != "param1" {
+		t.Fatal("unmarshal failed with mismatch value")
+	}
+}
+
 func TestUnmarshalDecodeErrors(t *testing.T) {
 	examples := []struct {
 		desc string

unstable/parser.go

@@ -49,8 +49,6 @@ func NewParserError(highlight []byte, format string, args ...interface{}) error
 // For performance reasons, go-toml doesn't make a copy of the input bytes to
 // the parser. Make sure to copy all the bytes you need to outlive the slice
 // given to the parser.
-//
-// The parser doesn't provide nodes for comments yet, nor for whitespace.
 type Parser struct {
 	data    []byte
 	builder builder
@@ -58,6 +56,8 @@ type Parser struct {
 	left    []byte
 	err     error
 	first   bool
+
+	KeepComments bool
 }
 
 // Data returns the slice provided to the last call to Reset.
@@ -142,6 +142,44 @@ func (p *Parser) Error() error {
 	return p.err
 }
 
+// Position describes a position in the input.
+type Position struct {
+	// Number of bytes from the beginning of the input.
+	Offset int
+	// Line number, starting at 1.
+	Line int
+	// Column number, starting at 1.
+	Column int
+}
+
+// Shape describes the position of a range in the input.
+type Shape struct {
+	Start Position
+	End   Position
+}
+
+func (p *Parser) position(b []byte) Position {
+	offset := danger.SubsliceOffset(p.data, b)
+
+	lead := p.data[:offset]
+
+	return Position{
+		Offset: offset,
+		Line:   bytes.Count(lead, []byte{'\n'}) + 1,
+		Column: len(lead) - bytes.LastIndex(lead, []byte{'\n'}),
+	}
+}
+
+// Shape returns the shape of the given range in the input.  Will
+// panic if the range is not a subslice of the input.
+func (p *Parser) Shape(r Range) Shape {
+	raw := p.Raw(r)
+	return Shape{
+		Start: p.position(raw),
+		End:   p.position(raw[r.Length:]),
+	}
+}
+
 func (p *Parser) parseNewline(b []byte) ([]byte, error) {
 	if b[0] == '\n' {
 		return b[1:], nil
@@ -155,6 +193,19 @@ func (p *Parser) parseNewline(b []byte) ([]byte, error) {
 	return nil, NewParserError(b[0:1], "expected newline but got %#U", b[0])
 }
 
+func (p *Parser) parseComment(b []byte) (reference, []byte, error) {
+	ref := invalidReference
+	data, rest, err := scanComment(b)
+	if p.KeepComments && err == nil {
+		ref = p.builder.Push(Node{
+			Kind: Comment,
+			Raw:  p.Range(data),
+			Data: data,
+		})
+	}
+	return ref, rest, err
+}
+
 func (p *Parser) parseExpression(b []byte) (reference, []byte, error) {
 	// expression =  ws [ comment ]
 	// expression =/ ws keyval ws [ comment ]
@@ -168,7 +219,7 @@ func (p *Parser) parseExpression(b []byte) (reference, []byte, error) {
 	}
 
 	if b[0] == '#' {
-		_, rest, err := scanComment(b)
+		ref, rest, err := p.parseComment(b)
 		return ref, rest, err
 	}
 
@@ -190,7 +241,10 @@ func (p *Parser) parseExpression(b []byte) (reference, []byte, error) {
 	b = p.parseWhitespace(b)
 
 	if len(b) > 0 && b[0] == '#' {
-		_, rest, err := scanComment(b)
+		cref, rest, err := p.parseComment(b)
+		if cref != invalidReference {
+			p.builder.Chain(ref, cref)
+		}
 		return ref, rest, err
 	}
 
@@ -471,17 +525,33 @@ func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
 		Kind: Array,
 	})
 
+	// First indicates whether the parser is looking for the first element
+	// (non-comment) of the array.
 	first := true
 
-	var lastChild reference
+	lastChild := invalidReference
+
+	addChild := func(valueRef reference) {
+		if lastChild == invalidReference {
+			p.builder.AttachChild(parent, valueRef)
+		} else {
+			p.builder.Chain(lastChild, valueRef)
+		}
+		lastChild = valueRef
+	}
 
 	var err error
 	for len(b) > 0 {
-		b, err = p.parseOptionalWhitespaceCommentNewline(b)
+		cref := invalidReference
+		cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
 		if err != nil {
 			return parent, nil, err
 		}
 
+		if cref != invalidReference {
+			addChild(cref)
+		}
+
 		if len(b) == 0 {
 			return parent, nil, NewParserError(arrayStart[:1], "array is incomplete")
 		}
@@ -496,10 +566,13 @@ func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
 			}
 			b = b[1:]
 
-			b, err = p.parseOptionalWhitespaceCommentNewline(b)
+			cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
 			if err != nil {
 				return parent, nil, err
 			}
+			if cref != invalidReference {
+				addChild(cref)
+			}
 		} else if !first {
 			return parent, nil, NewParserError(b[0:1], "array elements must be separated by commas")
 		}
@@ -515,17 +588,16 @@ func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
 			return parent, nil, err
 		}
 
-		if first {
-			p.builder.AttachChild(parent, valueRef)
-		} else {
-			p.builder.Chain(lastChild, valueRef)
-		}
-		lastChild = valueRef
+		addChild(valueRef)
 
-		b, err = p.parseOptionalWhitespaceCommentNewline(b)
+		cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
 		if err != nil {
 			return parent, nil, err
 		}
+		if cref != invalidReference {
+			addChild(cref)
+		}
+
 		first = false
 	}
 
@@ -534,15 +606,34 @@ func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
 	return parent, rest, err
 }
 
-func (p *Parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error) {
+func (p *Parser) parseOptionalWhitespaceCommentNewline(b []byte) (reference, []byte, error) {
+	rootCommentRef := invalidReference
+	latestCommentRef := invalidReference
+
+	addComment := func(ref reference) {
+		if rootCommentRef == invalidReference {
+			rootCommentRef = ref
+		} else if latestCommentRef == invalidReference {
+			p.builder.AttachChild(rootCommentRef, ref)
+			latestCommentRef = ref
+		} else {
+			p.builder.Chain(latestCommentRef, ref)
+			latestCommentRef = ref
+		}
+	}
+
 	for len(b) > 0 {
 		var err error
 		b = p.parseWhitespace(b)
 
 		if len(b) > 0 && b[0] == '#' {
-			_, b, err = scanComment(b)
+			var ref reference
+			ref, b, err = p.parseComment(b)
 			if err != nil {
-				return nil, err
+				return invalidReference, nil, err
+			}
+			if ref != invalidReference {
+				addComment(ref)
 			}
 		}
 
@@ -553,14 +644,14 @@ func (p *Parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
 		if b[0] == '\n' || b[0] == '\r' {
 			b, err = p.parseNewline(b)
 			if err != nil {
-				return nil, err
+				return invalidReference, nil, err
 			}
 		} else {
 			break
 		}
 	}
 
-	return b, nil
+	return rootCommentRef, b, nil
 }
 
 func (p *Parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) {

unstable/parser_test.go

@@ -448,6 +448,163 @@ func TestParser_AST_DateTimes(t *testing.T) {
 	}
 }
 
+// This example demonstrates how to parse a TOML document and preserving
+// comments.  Comments are stored in the AST as Comment nodes. This example
+// displays the structure of the full AST generated by the parser using the
+// following structure:
+//
+//  1. Each root-level expression is separated by three dashes.
+//  2. Bytes associated to a node are displayed in square brackets.
+//  3. Siblings have the same indentation.
+//  4. Children of a node are indented one level.
+func ExampleParser_comments() {
+	doc := `# Top of the document comment.
+# Optional, any amount of lines.
+
+# Above table.
+[table] # Next to table.
+# Above simple value.
+key = "value" # Next to simple value.
+# Below simple value.
+
+# Some comment alone.
+
+# Multiple comments, on multiple lines.
+
+# Above inline table.
+name = { first = "Tom", last = "Preston-Werner" } # Next to inline table.
+# Below inline table.
+
+# Above array.
+array = [ 1, 2, 3 ] # Next to one-line array.
+# Below array.
+
+# Above multi-line array.
+key5 = [ # Next to start of inline array.
+  # Second line before array content.
+  1, # Next to first element.
+  # After first element.
+  # Before second element.
+  2,
+  3, # Next to last element
+  # After last element.
+] # Next to end of array.
+# Below multi-line array.
+
+# Before array table.
+[[products]] # Next to array table.
+# After array table.
+`
+
+	var printGeneric func(*Parser, int, *Node)
+	printGeneric = func(p *Parser, indent int, e *Node) {
+		if e == nil {
+			return
+		}
+		s := p.Shape(e.Raw)
+		x := fmt.Sprintf("%d:%d->%d:%d (%d->%d)", s.Start.Line, s.Start.Column, s.End.Line, s.End.Column, s.Start.Offset, s.End.Offset)
+		fmt.Printf("%-25s | %s%s [%s]\n", x, strings.Repeat("  ", indent), e.Kind, e.Data)
+		printGeneric(p, indent+1, e.Child())
+		printGeneric(p, indent, e.Next())
+	}
+
+	printTree := func(p *Parser) {
+		for p.NextExpression() {
+			e := p.Expression()
+			fmt.Println("---")
+			printGeneric(p, 0, e)
+		}
+		if err := p.Error(); err != nil {
+			panic(err)
+		}
+	}
+
+	p := &Parser{
+		KeepComments: true,
+	}
+	p.Reset([]byte(doc))
+	printTree(p)
+
+	// Output:
+	// ---
+	// 1:1->1:31 (0->30)         | Comment [# Top of the document comment.]
+	// ---
+	// 2:1->2:33 (31->63)        | Comment [# Optional, any amount of lines.]
+	// ---
+	// 4:1->4:15 (65->79)        | Comment [# Above table.]
+	// ---
+	// 1:1->1:1 (0->0)           | Table []
+	// 5:2->5:7 (81->86)         |   Key [table]
+	// 5:9->5:25 (88->104)       | Comment [# Next to table.]
+	// ---
+	// 6:1->6:22 (105->126)      | Comment [# Above simple value.]
+	// ---
+	// 1:1->1:1 (0->0)           | KeyValue []
+	// 7:7->7:14 (133->140)      |   String [value]
+	// 7:1->7:4 (127->130)       |   Key [key]
+	// 7:15->7:38 (141->164)     | Comment [# Next to simple value.]
+	// ---
+	// 8:1->8:22 (165->186)      | Comment [# Below simple value.]
+	// ---
+	// 10:1->10:22 (188->209)    | Comment [# Some comment alone.]
+	// ---
+	// 12:1->12:40 (211->250)    | Comment [# Multiple comments, on multiple lines.]
+	// ---
+	// 14:1->14:22 (252->273)    | Comment [# Above inline table.]
+	// ---
+	// 1:1->1:1 (0->0)           | KeyValue []
+	// 15:8->15:9 (281->282)     |   InlineTable []
+	// 1:1->1:1 (0->0)           |     KeyValue []
+	// 15:18->15:23 (291->296)   |       String [Tom]
+	// 15:10->15:15 (283->288)   |       Key [first]
+	// 1:1->1:1 (0->0)           |     KeyValue []
+	// 15:32->15:48 (305->321)   |       String [Preston-Werner]
+	// 15:25->15:29 (298->302)   |       Key [last]
+	// 15:1->15:5 (274->278)     |   Key [name]
+	// 15:51->15:74 (324->347)   | Comment [# Next to inline table.]
+	// ---
+	// 16:1->16:22 (348->369)    | Comment [# Below inline table.]
+	// ---
+	// 18:1->18:15 (371->385)    | Comment [# Above array.]
+	// ---
+	// 1:1->1:1 (0->0)           | KeyValue []
+	// 1:1->1:1 (0->0)           |   Array []
+	// 1:1->1:1 (0->0)           |     Integer [1]
+	// 1:1->1:1 (0->0)           |     Integer [2]
+	// 1:1->1:1 (0->0)           |     Integer [3]
+	// 19:1->19:6 (386->391)     |   Key [array]
+	// 19:21->19:46 (406->431)   | Comment [# Next to one-line array.]
+	// ---
+	// 20:1->20:15 (432->446)    | Comment [# Below array.]
+	// ---
+	// 22:1->22:26 (448->473)    | Comment [# Above multi-line array.]
+	// ---
+	// 1:1->1:1 (0->0)           | KeyValue []
+	// 1:1->1:1 (0->0)           |   Array []
+	// 23:10->23:42 (483->515)   |     Comment [# Next to start of inline array.]
+	// 24:3->24:38 (518->553)    |       Comment [# Second line before array content.]
+	// 1:1->1:1 (0->0)           |     Integer [1]
+	// 25:6->25:30 (559->583)    |     Comment [# Next to first element.]
+	// 26:3->26:25 (586->608)    |       Comment [# After first element.]
+	// 27:3->27:27 (611->635)    |       Comment [# Before second element.]
+	// 1:1->1:1 (0->0)           |     Integer [2]
+	// 1:1->1:1 (0->0)           |     Integer [3]
+	// 29:6->29:28 (646->668)    |     Comment [# Next to last element]
+	// 30:3->30:24 (671->692)    |       Comment [# After last element.]
+	// 23:1->23:5 (474->478)     |   Key [key5]
+	// 31:3->31:26 (695->718)    | Comment [# Next to end of array.]
+	// ---
+	// 32:1->32:26 (719->744)    | Comment [# Below multi-line array.]
+	// ---
+	// 34:1->34:22 (746->767)    | Comment [# Before array table.]
+	// ---
+	// 1:1->1:1 (0->0)           | ArrayTable []
+	// 35:3->35:11 (770->778)    |   Key [products]
+	// 35:14->35:36 (781->803)   | Comment [# Next to array table.]
+	// ---
+	// 36:1->36:21 (804->824)    | Comment [# After array table.]
+}
+
 func ExampleParser() {
 	doc := `
 	hello = "world"

unstable/scanner.go

@@ -151,7 +151,6 @@ func scanWhitespace(b []byte) ([]byte, []byte) {
 	return b, b[len(b):]
 }
 
-//nolint:unparam
 func scanComment(b []byte) ([]byte, []byte, error) {
 	// comment-start-symbol = %x23 ; #
 	// non-ascii = %x80-D7FF / %xE000-10FFFF