Compare commits

..

18 Commits

Author SHA1 Message Date
Thomas Pelletier 0a422e3dbd Decoder: check max uint on 32 bit platforms (#778)
Fixes #777
2022-05-10 15:43:26 +02:00
Thomas Pelletier 627dade0c7 Encode: support comment on array tables (#776)
Fixes #774
2022-05-10 15:17:36 +02:00
Thomas Pelletier b2e0231cc9 Encode: fix multiline comment (#775)
Fixes #768
2022-05-10 14:53:26 +02:00
dependabot[bot] ba95863cd3 build(deps): bump docker/login-action from 1 to 2 (#771)
Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2.
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](https://github.com/docker/login-action/compare/v1...v2)

---
updated-dependencies:
- dependency-name: docker/login-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-05-09 18:59:02 +02:00
Erik Hartwig db679df765 Typo in README.md fix (#770)
Change "document was not prevent in the target structure" to "document was not present in the target structure".
2022-05-09 18:46:46 +02:00
Thomas Pelletier c5ca2c682b Fix embedded struct with explicit field name (#773)
Fixes #772
2022-05-09 18:45:02 +02:00
Thomas Pelletier ed80712cb4 Copy version policy from v1 2022-04-28 11:55:39 -04:00
dependabot[bot] b24772942d build(deps): bump github/codeql-action from 1 to 2 (#764)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1 to 2.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v1...v2)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-04-28 08:24:14 -04:00
dependabot[bot] 9501a05ed7 build(deps): bump actions/checkout from 2 to 3 (#765)
Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-04-28 08:23:56 -04:00
Thomas Pelletier 171a592663 Prepare repository for v2.0.0 (#762) 2022-04-27 21:29:16 -04:00
Thomas Pelletier 5aaf5ef13b Fix internal entry size test (#761)
The test meant to assert that the size of entry does not grow beyond
what it is today. 48 is the current size in bytes on x64. On 32-bit
platform that value should be less. As written the test was doing the
opposite comparison.

Fixes #760
2022-04-24 18:50:54 -04:00
Thomas Pelletier adacebd8c7 Update benchmarks in README (#756) 2022-04-10 22:24:19 -04:00
Thomas Pelletier 8bbb673431 Fuzzing setup and fixes (#755)
* encode: fix localdate formatting
* encode: fix empty key marshaling
* encode: fix invalid quotation of time.Time
* encode: ensure control chars are escaped
* decode: always use UTC for zero tz
* encode: check for invalid characters in keys
* encode: always construct map for empty array tables
* fuzz: add go 1.18 fuzz test
* encode: handle NaNs
* encode: allow new lines in quoted keys
* encode: never emit table inside array
* encode: don't capitalize inf
2022-04-10 21:37:12 -04:00
Thomas Pelletier 2377ac4bc0 encode: fix embedded interfaces (#753)
Resolve marshaling regression when handling an embedded interface in a
struct.

Fixes #752
2022-04-08 09:25:54 -04:00
Thomas Pelletier f5cc8c49eb decoder: remove mention of UnmarshalText in errors (#751)
Fixes #737
2022-04-07 21:58:19 -04:00
Thomas Pelletier 89d7b412d8 decode: allow subtables to be defined later (#750)
Fixes #739
2022-04-07 21:49:16 -04:00
Thomas Pelletier 88a8aecdd4 tools: display error context when it exists (#749)
For example when failing to decode toml, display the context around the
error and the location of the problem.
2022-04-07 20:33:09 -04:00
Thomas Pelletier 9804fc57e0 decoder: support \e escape sequence (#748) 2022-04-07 20:18:30 -04:00
31 changed files with 564 additions and 131 deletions
+1
View File
@@ -1,3 +1,4 @@
* text=auto * text=auto
benchmark/benchmark.toml text eol=lf benchmark/benchmark.toml text eol=lf
testdata/** text eol=lf
+4 -4
View File
@@ -35,11 +35,11 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v2 uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v1 uses: github/codeql-action/init@v2
with: with:
languages: ${{ matrix.language }} languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file. # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below) # If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild - name: Autobuild
uses: github/codeql-action/autobuild@v1 uses: github/codeql-action/autobuild@v2
# ️ Command-line programs to run using the OS shell. # ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
# make release # make release
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1 uses: github/codeql-action/analyze@v2
+1 -1
View File
@@ -9,7 +9,7 @@ jobs:
runs-on: "ubuntu-latest" runs-on: "ubuntu-latest"
name: report name: report
steps: steps:
- uses: actions/checkout@master - uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Setup go - name: Setup go
+2 -2
View File
@@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Set up Go - name: Set up Go
@@ -24,7 +24,7 @@ jobs:
with: with:
go-version: 1.18 go-version: 1.18
- name: Login to GitHub Container Registry - name: Login to GitHub Container Registry
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
name: ${{ matrix.go }}/${{ matrix.os }} name: ${{ matrix.go }}/${{ matrix.os }}
steps: steps:
- uses: actions/checkout@master - uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Setup go ${{ matrix.go }} - name: Setup go ${{ matrix.go }}
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT) The MIT License (MIT)
Copyright (c) 2013 - 2021 Thomas Pelletier, Eric Anderton Copyright (c) 2013 - 2022 Thomas Pelletier, Eric Anderton
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
+24 -28
View File
@@ -4,17 +4,6 @@ Go library for the [TOML](https://toml.io/en/) format.
This library supports [TOML v1.0.0](https://toml.io/en/v1.0.0). This library supports [TOML v1.0.0](https://toml.io/en/v1.0.0).
## Development status
This is the upcoming major version of go-toml. It is currently in active
development. As of release v2.0.0-beta.1, the library has reached feature parity
with v1, and fixes a lot known bugs and performance issues along the way.
If you do not need the advanced document editing features of v1, you are
encouraged to try out this version.
[👉 Roadmap for v2](https://github.com/pelletier/go-toml/discussions/506)
[🐞 Bug Reports](https://github.com/pelletier/go-toml/issues) [🐞 Bug Reports](https://github.com/pelletier/go-toml/issues)
[💬 Anything else](https://github.com/pelletier/go-toml/discussions) [💬 Anything else](https://github.com/pelletier/go-toml/discussions)
@@ -49,7 +38,7 @@ operations should not be shockingly slow. See [benchmarks](#benchmarks).
### Strict mode ### Strict mode
`Decoder` can be set to "strict mode", which makes it error when some parts of `Decoder` can be set to "strict mode", which makes it error when some parts of
the TOML document was not prevent in the target structure. This is a great way the TOML document was not present in the target structure. This is a great way
to check for typos. [See example in the documentation][strict]. to check for typos. [See example in the documentation][strict].
[strict]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#example-Decoder.DisallowUnknownFields [strict]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#example-Decoder.DisallowUnknownFields
@@ -161,11 +150,11 @@ Execution time speedup compared to other Go TOML libraries:
</thead> </thead>
<tbody> <tbody>
<tr><td>Marshal/HugoFrontMatter-2</td><td>1.9x</td><td>1.9x</td></tr> <tr><td>Marshal/HugoFrontMatter-2</td><td>1.9x</td><td>1.9x</td></tr>
<tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>1.9x</td></tr> <tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>1.8x</td></tr>
<tr><td>Marshal/ReferenceFile/struct-2</td><td>2.4x</td><td>2.6x</td></tr> <tr><td>Marshal/ReferenceFile/struct-2</td><td>2.2x</td><td>2.5x</td></tr>
<tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.5x</td></tr> <tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.9x</td></tr>
<tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.7x</td><td>2.6x</td></tr> <tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.6x</td><td>2.9x</td></tr>
<tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.8x</td><td>5.1x</td></tr> <tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.4x</td><td>5.3x</td></tr>
</tbody> </tbody>
</table> </table>
<details><summary>See more</summary> <details><summary>See more</summary>
@@ -178,17 +167,17 @@ provided for completeness.</p>
<tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr> <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
</thead> </thead>
<tbody> <tbody>
<tr><td>Marshal/SimpleDocument/map-2</td><td>1.7x</td><td>2.1x</td></tr> <tr><td>Marshal/SimpleDocument/map-2</td><td>1.8x</td><td>2.9x</td></tr>
<tr><td>Marshal/SimpleDocument/struct-2</td><td>2.5x</td><td>2.8x</td></tr> <tr><td>Marshal/SimpleDocument/struct-2</td><td>2.7x</td><td>4.2x</td></tr>
<tr><td>Unmarshal/SimpleDocument/map-2</td><td>4.1x</td><td>3.1x</td></tr> <tr><td>Unmarshal/SimpleDocument/map-2</td><td>4.5x</td><td>3.1x</td></tr>
<tr><td>Unmarshal/SimpleDocument/struct-2</td><td>6.4x</td><td>4.3x</td></tr> <tr><td>Unmarshal/SimpleDocument/struct-2</td><td>6.2x</td><td>3.9x</td></tr>
<tr><td>UnmarshalDataset/example-2</td><td>3.4x</td><td>3.2x</td></tr> <tr><td>UnmarshalDataset/example-2</td><td>3.1x</td><td>3.5x</td></tr>
<tr><td>UnmarshalDataset/code-2</td><td>2.2x</td><td>2.5x</td></tr> <tr><td>UnmarshalDataset/code-2</td><td>2.3x</td><td>3.1x</td></tr>
<tr><td>UnmarshalDataset/twitter-2</td><td>2.8x</td><td>2.7x</td></tr> <tr><td>UnmarshalDataset/twitter-2</td><td>2.5x</td><td>2.6x</td></tr>
<tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.2x</td><td>2.0x</td></tr> <tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.1x</td><td>2.2x</td></tr>
<tr><td>UnmarshalDataset/canada-2</td><td>1.8x</td><td>1.4x</td></tr> <tr><td>UnmarshalDataset/canada-2</td><td>1.6x</td><td>1.3x</td></tr>
<tr><td>UnmarshalDataset/config-2</td><td>4.4x</td><td>2.9x</td></tr> <tr><td>UnmarshalDataset/config-2</td><td>4.3x</td><td>3.2x</td></tr>
<tr><td>[Geo mean]</td><td>2.8x</td><td>2.6x</td></tr> <tr><td>[Geo mean]</td><td>2.7x</td><td>2.8x</td></tr>
</tbody> </tbody>
</table> </table>
<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p> <p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>
@@ -551,6 +540,13 @@ complete solutions exist out there.
[query]: https://github.com/pelletier/go-toml/tree/f99d6bbca119636aeafcf351ee52b3d202782627/query [query]: https://github.com/pelletier/go-toml/tree/f99d6bbca119636aeafcf351ee52b3d202782627/query
[dasel]: https://github.com/TomWright/dasel [dasel]: https://github.com/TomWright/dasel
## Versioning
Go-toml follows [Semantic Versioning](http://semver.org/). The supported version
of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of
this document. The last two major versions of Go are supported
(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)).
## License ## License
The MIT License (MIT). Read [LICENSE](LICENSE). The MIT License (MIT). Read [LICENSE](LICENSE).
+13 -5
View File
@@ -76,7 +76,8 @@ cover() {
fi fi
pushd "$dir" pushd "$dir"
go test -covermode=atomic -coverprofile=coverage.out ./... go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./...
cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
go tool cover -func=coverage.out go tool cover -func=coverage.out
popd popd
@@ -103,16 +104,23 @@ coverage() {
echo "" echo ""
target_pct="$(cat ${target_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')" target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
head_pct="$(cat ${head_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')" head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
delta_pct=$(echo "$head_pct - $target_pct" | bc -l) delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
echo "Delta: ${delta_pct}" echo "Delta: ${delta_pct}"
if [[ $delta_pct = \-* ]]; then if [[ $delta_pct = \-* ]]; then
echo "Regression!"; echo "Regression!";
return 1
target_diff="${output_dir}/target.diff.txt"
head_diff="${output_dir}/head.diff.txt"
cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
return 1
fi fi
return 0 return 0
;; ;;
+5 -1
View File
@@ -130,7 +130,11 @@ func parseDateTime(b []byte) (time.Time, error) {
} }
seconds := direction * (hours*3600 + minutes*60) seconds := direction * (hours*3600 + minutes*60)
zone = time.FixedZone("", seconds) if seconds == 0 {
zone = time.UTC
} else {
zone = time.FixedZone("", seconds)
}
b = b[dateTimeByteLen:] b = b[dateTimeByteLen:]
} }
+56
View File
@@ -0,0 +1,56 @@
//go:build go1.18
// +build go1.18
package toml_test
import (
"io/ioutil"
"strings"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/require"
)
func FuzzUnmarshal(f *testing.F) {
file, err := ioutil.ReadFile("benchmark/benchmark.toml")
if err != nil {
panic(err)
}
f.Add(file)
f.Fuzz(func(t *testing.T, b []byte) {
if strings.Contains(string(b), "nan") {
// Current limitation of testify.
// https://github.com/stretchr/testify/issues/624
t.Skip("can't compare NaNs")
}
t.Log("INITIAL DOCUMENT ===========================")
t.Log(string(b))
var v interface{}
err := toml.Unmarshal(b, &v)
if err != nil {
return
}
t.Log("DECODED VALUE ===========================")
t.Logf("%#+v", v)
encoded, err := toml.Marshal(v)
if err != nil {
t.Fatalf("cannot marshal unmarshaled document: %s", err)
}
t.Log("ENCODED DOCUMENT ===========================")
t.Log(string(encoded))
var v2 interface{}
err = toml.Unmarshal(encoded, &v2)
if err != nil {
t.Fatalf("failed round trip: %s", err)
}
require.Equal(t, v, v2)
})
}
+13 -1
View File
@@ -2,11 +2,14 @@ package cli
import ( import (
"bytes" "bytes"
"errors"
"flag" "flag"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"github.com/pelletier/go-toml/v2"
) )
type ConvertFn func(r io.Reader, w io.Writer) error type ConvertFn func(r io.Reader, w io.Writer) error
@@ -28,7 +31,16 @@ func (p *Program) Execute() {
func (p *Program) main(files []string, input io.Reader, output, error io.Writer) int { func (p *Program) main(files []string, input io.Reader, output, error io.Writer) int {
err := p.run(files, input, output) err := p.run(files, input, output)
if err != nil { if err != nil {
fmt.Fprintln(error, err.Error())
var derr *toml.DecodeError
if errors.As(err, &derr) {
fmt.Fprintln(error, derr.String())
row, col := derr.Position()
fmt.Fprintln(error, "error occurred at row", row, "column", col)
} else {
fmt.Fprintln(error, err.Error())
}
return -1 return -1
} }
return 0 return 0
+16
View File
@@ -10,6 +10,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -47,6 +48,21 @@ func TestProcessMainStdinErr(t *testing.T) {
assert.NotEmpty(t, stderr.String()) assert.NotEmpty(t, stderr.String())
} }
func TestProcessMainStdinDecodeErr(t *testing.T) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
input := strings.NewReader("this is the input")
exit := processMain([]string{}, input, stdout, stderr, func(r io.Reader, w io.Writer) error {
var v interface{}
return toml.Unmarshal([]byte(`qwe = 001`), &v)
})
assert.Equal(t, -1, exit)
assert.Empty(t, stdout.String())
assert.Contains(t, stderr.String(), "error occurred at")
}
func TestProcessMainFileExists(t *testing.T) { func TestProcessMainFileExists(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "example") tmpfile, err := ioutil.TempFile("", "example")
require.NoError(t, err) require.NoError(t, err)
+12 -7
View File
@@ -79,6 +79,7 @@ type entry struct {
name []byte name []byte
kind keyKind kind keyKind
explicit bool explicit bool
kv bool
} }
// Find the index of the child of parentIdx with key k. Returns -1 if // Find the index of the child of parentIdx with key k. Returns -1 if
@@ -111,7 +112,7 @@ func (s *SeenTracker) clear(idx int) {
s.entries[idx].child = -1 s.entries[idx].child = -1
} }
func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool) int { func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool, kv bool) int {
e := entry{ e := entry{
child: -1, child: -1,
next: s.entries[parentIdx].child, next: s.entries[parentIdx].child,
@@ -119,6 +120,7 @@ func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit
name: name, name: name,
kind: kind, kind: kind,
explicit: explicit, explicit: explicit,
kv: kv,
} }
var idx int var idx int
if s.entries[0].next >= 0 { if s.entries[0].next >= 0 {
@@ -137,7 +139,10 @@ func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit
func (s *SeenTracker) setExplicitFlag(parentIdx int) { func (s *SeenTracker) setExplicitFlag(parentIdx int) {
for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next { for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next {
s.entries[i].explicit = true if s.entries[i].kv {
s.entries[i].explicit = true
s.entries[i].kv = false
}
s.setExplicitFlag(i) s.setExplicitFlag(i)
} }
} }
@@ -183,7 +188,7 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
idx := s.find(parentIdx, k) idx := s.find(parentIdx, k)
if idx < 0 { if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false) idx = s.create(parentIdx, k, tableKind, false, false)
} else { } else {
entry := s.entries[idx] entry := s.entries[idx]
if entry.kind == valueKind { if entry.kind == valueKind {
@@ -206,7 +211,7 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
} }
s.entries[idx].explicit = true s.entries[idx].explicit = true
} else { } else {
idx = s.create(parentIdx, k, tableKind, true) idx = s.create(parentIdx, k, tableKind, true, false)
} }
s.currentIdx = idx s.currentIdx = idx
@@ -233,7 +238,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
idx := s.find(parentIdx, k) idx := s.find(parentIdx, k)
if idx < 0 { if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false) idx = s.create(parentIdx, k, tableKind, false, false)
} else { } else {
entry := s.entries[idx] entry := s.entries[idx]
if entry.kind == valueKind { if entry.kind == valueKind {
@@ -254,7 +259,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
} }
s.clear(idx) s.clear(idx)
} else { } else {
idx = s.create(parentIdx, k, arrayTableKind, true) idx = s.create(parentIdx, k, arrayTableKind, true, false)
} }
s.currentIdx = idx s.currentIdx = idx
@@ -272,7 +277,7 @@ func (s *SeenTracker) checkKeyValue(node *ast.Node) error {
idx := s.find(parentIdx, k) idx := s.find(parentIdx, k)
if idx < 0 { if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false) idx = s.create(parentIdx, k, tableKind, false, true)
} else { } else {
entry := s.entries[idx] entry := s.entries[idx]
if it.IsLast() { if it.IsLast() {
+16
View File
@@ -0,0 +1,16 @@
package tracker
import (
"testing"
"unsafe"
"github.com/stretchr/testify/require"
)
func TestEntrySize(t *testing.T) {
// Validate no regression on the size of entry{}. This is a critical bit for
// performance of unmarshaling documents. Should only be increased with care
// and a very good reason.
maxExpectedEntrySize := 48
require.LessOrEqual(t, int(unsafe.Sizeof(entry{})), maxExpectedEntrySize)
}
+80 -50
View File
@@ -128,7 +128,8 @@ func (enc *Encoder) SetIndentTables(indent bool) *Encoder {
// //
// In addition to the "toml" tag struct tag, a "comment" tag can be used to emit // In addition to the "toml" tag struct tag, a "comment" tag can be used to emit
// a TOML comment before the value being annotated. Comments are ignored inside // a TOML comment before the value being annotated. Comments are ignored inside
// inline tables. // inline tables. For array tables, the comment is only present before the first
// element of the array.
func (enc *Encoder) Encode(v interface{}) error { func (enc *Encoder) Encode(v interface{}) error {
var ( var (
b []byte b []byte
@@ -208,11 +209,20 @@ func (ctx *encoderCtx) isRoot() bool {
} }
func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
if !v.IsZero() { i := v.Interface()
i, ok := v.Interface().(time.Time)
if ok { switch x := i.(type) {
return i.AppendFormat(b, time.RFC3339), nil case time.Time:
if x.Nanosecond() > 0 {
return x.AppendFormat(b, time.RFC3339Nano), nil
} }
return x.AppendFormat(b, time.RFC3339), nil
case LocalTime:
return append(b, x.String()...), nil
case LocalDate:
return append(b, x.String()...), nil
case LocalDateTime:
return append(b, x.String()...), nil
} }
hasTextMarshaler := v.Type().Implements(textMarshalerType) hasTextMarshaler := v.Type().Implements(textMarshalerType)
@@ -260,16 +270,31 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
case reflect.String: case reflect.String:
b = enc.encodeString(b, v.String(), ctx.options) b = enc.encodeString(b, v.String(), ctx.options)
case reflect.Float32: case reflect.Float32:
if math.Trunc(v.Float()) == v.Float() { f := v.Float()
b = strconv.AppendFloat(b, v.Float(), 'f', 1, 32)
if math.IsNaN(f) {
b = append(b, "nan"...)
} else if f > math.MaxFloat32 {
b = append(b, "inf"...)
} else if f < -math.MaxFloat32 {
b = append(b, "-inf"...)
} else if math.Trunc(f) == f {
b = strconv.AppendFloat(b, f, 'f', 1, 32)
} else { } else {
b = strconv.AppendFloat(b, v.Float(), 'f', -1, 32) b = strconv.AppendFloat(b, f, 'f', -1, 32)
} }
case reflect.Float64: case reflect.Float64:
if math.Trunc(v.Float()) == v.Float() { f := v.Float()
b = strconv.AppendFloat(b, v.Float(), 'f', 1, 64) if math.IsNaN(f) {
b = append(b, "nan"...)
} else if f > math.MaxFloat64 {
b = append(b, "inf"...)
} else if f < -math.MaxFloat64 {
b = append(b, "-inf"...)
} else if math.Trunc(f) == f {
b = strconv.AppendFloat(b, f, 'f', 1, 64)
} else { } else {
b = strconv.AppendFloat(b, v.Float(), 'f', -1, 64) b = strconv.AppendFloat(b, f, 'f', -1, 64)
} }
case reflect.Bool: case reflect.Bool:
if v.Bool() { if v.Bool() {
@@ -300,10 +325,6 @@ func isNil(v reflect.Value) bool {
func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v reflect.Value) ([]byte, error) { func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v reflect.Value) ([]byte, error) {
var err error var err error
if !ctx.hasKey {
panic("caller of encodeKv should have set the key in the context")
}
if (ctx.options.omitempty || options.omitempty) && isEmptyValue(v) { if (ctx.options.omitempty || options.omitempty) && isEmptyValue(v) {
return b, nil return b, nil
} }
@@ -313,12 +334,7 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
} }
b = enc.indent(ctx.indent, b) b = enc.indent(ctx.indent, b)
b = enc.encodeKey(b, ctx.key)
b, err = enc.encodeKey(b, ctx.key)
if err != nil {
return nil, err
}
b = append(b, " = "...) b = append(b, " = "...)
// create a copy of the context because the value of a KV shouldn't // create a copy of the context because the value of a KV shouldn't
@@ -365,7 +381,13 @@ func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byt
} }
func needsQuoting(v string) bool { func needsQuoting(v string) bool {
return strings.ContainsAny(v, "'\b\f\n\r\t") // TODO: vectorize
for _, b := range []byte(v) {
if b == '\'' || b == '\r' || b == '\n' || invalidAscii(b) {
return true
}
}
return false
} }
// caller should have checked that the string does not contain new lines or ' . // caller should have checked that the string does not contain new lines or ' .
@@ -437,7 +459,7 @@ func (enc *Encoder) encodeQuotedString(multiline bool, b []byte, v string) []byt
return b return b
} }
// called should have checked that the string is in A-Z / a-z / 0-9 / - / _ . // caller should have checked that the string is in A-Z / a-z / 0-9 / - / _ .
func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte { func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte {
return append(b, v...) return append(b, v...)
} }
@@ -453,20 +475,11 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
b = append(b, '[') b = append(b, '[')
var err error b = enc.encodeKey(b, ctx.parentKey[0])
b, err = enc.encodeKey(b, ctx.parentKey[0])
if err != nil {
return nil, err
}
for _, k := range ctx.parentKey[1:] { for _, k := range ctx.parentKey[1:] {
b = append(b, '.') b = append(b, '.')
b = enc.encodeKey(b, k)
b, err = enc.encodeKey(b, k)
if err != nil {
return nil, err
}
} }
b = append(b, "]\n"...) b = append(b, "]\n"...)
@@ -475,19 +488,19 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
} }
//nolint:cyclop //nolint:cyclop
func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) { func (enc *Encoder) encodeKey(b []byte, k string) []byte {
needsQuotation := false needsQuotation := false
cannotUseLiteral := false cannotUseLiteral := false
if len(k) == 0 {
return append(b, "''"...)
}
for _, c := range k { for _, c := range k {
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' { if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
continue continue
} }
if c == '\n' {
return nil, fmt.Errorf("toml: new line characters in keys are not supported")
}
if c == literalQuote { if c == literalQuote {
cannotUseLiteral = true cannotUseLiteral = true
} }
@@ -495,13 +508,17 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
needsQuotation = true needsQuotation = true
} }
if needsQuotation && needsQuoting(k) {
cannotUseLiteral = true
}
switch { switch {
case cannotUseLiteral: case cannotUseLiteral:
return enc.encodeQuotedString(false, b, k), nil return enc.encodeQuotedString(false, b, k)
case needsQuotation: case needsQuotation:
return enc.encodeLiteralString(b, k), nil return enc.encodeLiteralString(b, k)
default: default:
return enc.encodeUnquotedKey(b, k), nil return enc.encodeUnquotedKey(b, k)
} }
} }
@@ -600,7 +617,9 @@ func walkStruct(ctx encoderCtx, t *table, v reflect.Value) {
if k == "" { if k == "" {
if fieldType.Anonymous { if fieldType.Anonymous {
walkStruct(ctx, t, f) if fieldType.Type.Kind() == reflect.Struct {
walkStruct(ctx, t, f)
}
continue continue
} else { } else {
k = fieldType.Name k = fieldType.Name
@@ -634,10 +653,19 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
} }
func (enc *Encoder) encodeComment(indent int, comment string, b []byte) []byte { func (enc *Encoder) encodeComment(indent int, comment string, b []byte) []byte {
if comment != "" { for len(comment) > 0 {
var line string
idx := strings.IndexByte(comment, '\n')
if idx >= 0 {
line = comment[:idx]
comment = comment[idx+1:]
} else {
line = comment
comment = ""
}
b = enc.indent(indent, b) b = enc.indent(indent, b)
b = append(b, "# "...) b = append(b, "# "...)
b = append(b, comment...) b = append(b, line...)
b = append(b, '\n') b = append(b, '\n')
} }
return b return b
@@ -801,6 +829,9 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
} }
func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool { func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool {
if ctx.insideKv {
return false
}
t := v.Type() t := v.Type()
if t.Kind() == reflect.Interface { if t.Kind() == reflect.Interface {
@@ -846,7 +877,6 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by
func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
ctx.shiftKey() ctx.shiftKey()
var err error
scratch := make([]byte, 0, 64) scratch := make([]byte, 0, 64)
scratch = append(scratch, "[["...) scratch = append(scratch, "[["...)
@@ -855,18 +885,18 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
scratch = append(scratch, '.') scratch = append(scratch, '.')
} }
scratch, err = enc.encodeKey(scratch, k) scratch = enc.encodeKey(scratch, k)
if err != nil {
return nil, err
}
} }
scratch = append(scratch, "]]\n"...) scratch = append(scratch, "]]\n"...)
ctx.skipTableHeader = true ctx.skipTableHeader = true
b = enc.encodeComment(ctx.indent, ctx.options.comment, b)
for i := 0; i < v.Len(); i++ { for i := 0; i < v.Len(); i++ {
b = append(b, scratch...) b = append(b, scratch...)
var err error
b, err = enc.encode(b, ctx, v.Index(i)) b, err = enc.encode(b, ctx, v.Index(i))
if err != nil { if err != nil {
return nil, err return nil, err
+97 -6
View File
@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"math"
"math/big" "math/big"
"strings" "strings"
"testing" "testing"
@@ -45,7 +46,7 @@ func TestMarshal(t *testing.T) {
v: map[string]string{ v: map[string]string{
"hel\nlo": "world", "hel\nlo": "world",
}, },
err: true, expected: `"hel\nlo" = 'world'`,
}, },
{ {
desc: `map with " in key`, desc: `map with " in key`,
@@ -380,7 +381,8 @@ hello = 'world'`,
v: map[string][]map[string]string{ v: map[string][]map[string]string{
"a\n": {{"hello": "world"}}, "a\n": {{"hello": "world"}},
}, },
err: true, expected: `[["a\n"]]
hello = 'world'`,
}, },
{ {
desc: "newline in map in slice", desc: "newline in map in slice",
@@ -440,7 +442,7 @@ hello = 'world'`,
v: map[string]interface{}{ v: map[string]interface{}{
"hello\nworld": 42, "hello\nworld": 42,
}, },
err: true, expected: `"hello\nworld" = 42`,
}, },
{ {
desc: "new line in parent of nested table key", desc: "new line in parent of nested table key",
@@ -449,7 +451,8 @@ hello = 'world'`,
"inner": 42, "inner": 42,
}, },
}, },
err: true, expected: `["hello\nworld"]
inner = 42`,
}, },
{ {
desc: "new line in nested table key", desc: "new line in nested table key",
@@ -460,7 +463,9 @@ hello = 'world'`,
}, },
}, },
}, },
err: true, expected: `[parent]
[parent."in\ner"]
foo = 42`,
}, },
{ {
desc: "invalid map key", desc: "invalid map key",
@@ -483,7 +488,16 @@ hello = 'world'`,
}{ }{
T: time.Time{}, T: time.Time{},
}, },
expected: `T = '0001-01-01T00:00:00Z'`, expected: `T = 0001-01-01T00:00:00Z`,
},
{
desc: "time nano",
v: struct {
T time.Time
}{
T: time.Date(1979, time.May, 27, 0, 32, 0, 999999000, time.UTC),
},
expected: `T = 1979-05-27T00:32:00.999999Z`,
}, },
{ {
desc: "bool", desc: "bool",
@@ -656,6 +670,33 @@ func equalStringsIgnoreNewlines(t *testing.T, expected string, actual string) {
assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset)) assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset))
} }
func TestMarshalFloats(t *testing.T) {
v := map[string]float32{
"nan": float32(math.NaN()),
"+inf": float32(math.Inf(1)),
"-inf": float32(math.Inf(-1)),
}
expected := `'+inf' = inf
-inf = -inf
nan = nan
`
actual, err := toml.Marshal(v)
require.NoError(t, err)
require.Equal(t, expected, string(actual))
v64 := map[string]float64{
"nan": math.NaN(),
"+inf": math.Inf(1),
"-inf": math.Inf(-1),
}
actual, err = toml.Marshal(v64)
require.NoError(t, err)
require.Equal(t, expected, string(actual))
}
//nolint:funlen //nolint:funlen
func TestMarshalIndentTables(t *testing.T) { func TestMarshalIndentTables(t *testing.T) {
examples := []struct { examples := []struct {
@@ -947,6 +988,38 @@ func TestIssue678(t *testing.T) {
require.Equal(t, cfg, cfg2) require.Equal(t, cfg, cfg2)
} }
func TestIssue752(t *testing.T) {
type Fooer interface {
Foo() string
}
type Container struct {
Fooer
}
c := Container{}
out, err := toml.Marshal(c)
require.NoError(t, err)
require.Equal(t, "", string(out))
}
func TestIssue768(t *testing.T) {
type cfg struct {
Name string `comment:"This is a multiline comment.\nThis is line 2."`
}
out, err := toml.Marshal(&cfg{})
require.NoError(t, err)
expected := `# This is a multiline comment.
# This is line 2.
Name = ''
`
require.Equal(t, expected, string(out))
}
func TestMarshalNestedAnonymousStructs(t *testing.T) { func TestMarshalNestedAnonymousStructs(t *testing.T) {
type Embedded struct { type Embedded struct {
Value string `toml:"value" json:"value"` Value string `toml:"value" json:"value"`
@@ -1011,6 +1084,24 @@ value = ''
require.Equal(t, expected, string(result)) require.Equal(t, expected, string(result))
} }
func TestLocalTime(t *testing.T) {
v := map[string]toml.LocalTime{
"a": toml.LocalTime{
Hour: 1,
Minute: 2,
Second: 3,
Nanosecond: 4,
},
}
expected := `a = 01:02:03.000000004
`
out, err := toml.Marshal(v)
require.NoError(t, err)
require.Equal(t, expected, string(out))
}
func ExampleMarshal() { func ExampleMarshal() {
type MyConfig struct { type MyConfig struct {
Version int Version int
+4
View File
@@ -617,6 +617,8 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
builder.WriteByte('\r') builder.WriteByte('\r')
case 't': case 't':
builder.WriteByte('\t') builder.WriteByte('\t')
case 'e':
builder.WriteByte(0x1B)
case 'u': case 'u':
x, err := hexToRune(atmost(token[i+1:], 4), 4) x, err := hexToRune(atmost(token[i+1:], 4), 4)
if err != nil { if err != nil {
@@ -774,6 +776,8 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
builder.WriteByte('\r') builder.WriteByte('\r')
case 't': case 't':
builder.WriteByte('\t') builder.WriteByte('\t')
case 'e':
builder.WriteByte(0x1B)
case 'u': case 'u':
x, err := hexToRune(token[i+1:len(token)-1], 4) x, err := hexToRune(token[i+1:len(token)-1], 4)
if err != nil { if err != nil {
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01 00:00:00")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("\"\\n\"=\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("''=0")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=\"\"\"\\U00000000\"\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=[[{}]]")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("\"\\b\"=\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=inf")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01 00:00:00+00:00")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=[{}]")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=nan")
+90 -8
View File
@@ -1,4 +1,4 @@
// Generated by tomltestgen for toml-test ref master on 2021-12-31T17:10:15-05:00 // Generated by tomltestgen for toml-test ref master on 2022-04-07T20:09:42-04:00
package toml_test package toml_test
import ( import (
@@ -55,11 +55,51 @@ func TestTOMLTest_Invalid_Array_TextInArray(t *testing.T) {
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_Bool_AlmostFalseWithExtra(t *testing.T) {
input := "a = falsify\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostFalse(t *testing.T) {
input := "a = fals\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostTrueWithExtra(t *testing.T) {
input := "a = truthy\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostTrue(t *testing.T) {
input := "a = tru\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_JustF(t *testing.T) {
input := "a = f\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_JustT(t *testing.T) {
input := "a = t\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_MixedCase(t *testing.T) { func TestTOMLTest_Invalid_Bool_MixedCase(t *testing.T) {
input := "valid = False\n" input := "valid = False\n"
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_Bool_StartingSameFalse(t *testing.T) {
input := "a = falsey\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_StartingSameTrue(t *testing.T) {
input := "a = truer\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_WrongCaseFalse(t *testing.T) { func TestTOMLTest_Invalid_Bool_WrongCaseFalse(t *testing.T) {
input := "b = FALSE\n" input := "b = FALSE\n"
testgenInvalid(t, input) testgenInvalid(t, input)
@@ -70,11 +110,26 @@ func TestTOMLTest_Invalid_Bool_WrongCaseTrue(t *testing.T) {
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_Control_BareCr(t *testing.T) {
input := "# The following line contains a single carriage return control character\r\n\r"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareFormfeed(t *testing.T) {
input := "bare-formfeed = \f\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareNull(t *testing.T) { func TestTOMLTest_Invalid_Control_BareNull(t *testing.T) {
input := "bare-null = \"some value\" \x00\n" input := "bare-null = \"some value\" \x00\n"
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_Control_BareVerticalTab(t *testing.T) {
input := "bare-vertical-tab = \v\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_CommentCr(t *testing.T) { func TestTOMLTest_Invalid_Control_CommentCr(t *testing.T) {
input := "comment-cr = \"Carriage return in comment\" # \ra=1\n" input := "comment-cr = \"Carriage return in comment\" # \ra=1\n"
testgenInvalid(t, input) testgenInvalid(t, input)
@@ -445,6 +500,11 @@ func TestTOMLTest_Invalid_Float_UsBeforePoint(t *testing.T) {
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_InlineTable_Add(t *testing.T) {
input := "a={}\n# Inline tables are immutable and can't be extended\n[a.b]\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_DoubleComma(t *testing.T) { func TestTOMLTest_Invalid_InlineTable_DoubleComma(t *testing.T) {
input := "t = {x=3,,y=4}\n" input := "t = {x=3,,y=4}\n"
testgenInvalid(t, input) testgenInvalid(t, input)
@@ -525,6 +585,21 @@ func TestTOMLTest_Invalid_Integer_DoubleUs(t *testing.T) {
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_Integer_IncompleteBin(t *testing.T) {
input := "incomplete-bin = 0b\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_IncompleteHex(t *testing.T) {
input := "incomplete-hex = 0x\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_IncompleteOct(t *testing.T) {
input := "incomplete-oct = 0o\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_InvalidBin(t *testing.T) { func TestTOMLTest_Invalid_Integer_InvalidBin(t *testing.T) {
input := "invalid-bin = 0b0012\n" input := "invalid-bin = 0b0012\n"
testgenInvalid(t, input) testgenInvalid(t, input)
@@ -910,11 +985,6 @@ func TestTOMLTest_Invalid_String_MultilineQuotes1(t *testing.T) {
testgenInvalid(t, input) testgenInvalid(t, input)
} }
func TestTOMLTest_Invalid_String_MultilineQuotes2(t *testing.T) {
input := "a = \"\"\"6 quotes: \"\"\"\"\"\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_NoClose(t *testing.T) { func TestTOMLTest_Invalid_String_NoClose(t *testing.T) {
input := "no-ending-quote = \"One time, at band camp\n" input := "no-ending-quote = \"One time, at band camp\n"
testgenInvalid(t, input) testgenInvalid(t, input)
@@ -1471,6 +1541,12 @@ func TestTOMLTest_Valid_String_Empty(t *testing.T) {
testgenValid(t, input, jsonRef) testgenValid(t, input, jsonRef)
} }
func TestTOMLTest_Valid_String_EscapeEsc(t *testing.T) {
input := "esc = \"\\e There is no escape! \\e\"\n"
jsonRef := "{\n \"esc\": {\n \"type\": \"string\",\n \"value\": \"\\u001b There is no escape! \\u001b\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_EscapeTricky(t *testing.T) { func TestTOMLTest_Valid_String_EscapeTricky(t *testing.T) {
input := "end_esc = \"String does not end here\\\" but ends here\\\\\"\nlit_end_esc = 'String ends here\\'\n\nmultiline_unicode = \"\"\"\n\\u00a0\"\"\"\n\nmultiline_not_unicode = \"\"\"\n\\\\u0041\"\"\"\n\nmultiline_end_esc = \"\"\"When will it end? \\\"\"\"...\"\"\\\" should be here\\\"\"\"\"\n\nlit_multiline_not_unicode = '''\n\\u007f'''\n\nlit_multiline_end = '''There is no escape\\'''\n" input := "end_esc = \"String does not end here\\\" but ends here\\\\\"\nlit_end_esc = 'String ends here\\'\n\nmultiline_unicode = \"\"\"\n\\u00a0\"\"\"\n\nmultiline_not_unicode = \"\"\"\n\\\\u0041\"\"\"\n\nmultiline_end_esc = \"\"\"When will it end? \\\"\"\"...\"\"\\\" should be here\\\"\"\"\"\n\nlit_multiline_not_unicode = '''\n\\u007f'''\n\nlit_multiline_end = '''There is no escape\\'''\n"
jsonRef := "{\n \"end_esc\": {\n \"type\": \"string\",\n \"value\": \"String does not end here\\\" but ends here\\\\\"\n },\n \"lit_end_esc\": {\n \"type\": \"string\",\n \"value\": \"String ends here\\\\\"\n },\n \"lit_multiline_end\": {\n \"type\": \"string\",\n \"value\": \"There is no escape\\\\\"\n },\n \"lit_multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u007f\"\n },\n \"multiline_end_esc\": {\n \"type\": \"string\",\n \"value\": \"When will it end? \\\"\\\"\\\"...\\\"\\\"\\\" should be here\\\"\"\n },\n \"multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u0041\"\n },\n \"multiline_unicode\": {\n \"type\": \"string\",\n \"value\": \"\u00a0\"\n }\n}\n" jsonRef := "{\n \"end_esc\": {\n \"type\": \"string\",\n \"value\": \"String does not end here\\\" but ends here\\\\\"\n },\n \"lit_end_esc\": {\n \"type\": \"string\",\n \"value\": \"String ends here\\\\\"\n },\n \"lit_multiline_end\": {\n \"type\": \"string\",\n \"value\": \"There is no escape\\\\\"\n },\n \"lit_multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u007f\"\n },\n \"multiline_end_esc\": {\n \"type\": \"string\",\n \"value\": \"When will it end? \\\"\\\"\\\"...\\\"\\\"\\\" should be here\\\"\"\n },\n \"multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u0041\"\n },\n \"multiline_unicode\": {\n \"type\": \"string\",\n \"value\": \"\u00a0\"\n }\n}\n"
@@ -1489,9 +1565,15 @@ func TestTOMLTest_Valid_String_Escapes(t *testing.T) {
testgenValid(t, input, jsonRef) testgenValid(t, input, jsonRef)
} }
func TestTOMLTest_Valid_String_MultilineEscapedCrlf(t *testing.T) {
input := "# The following line should be an unescaped backslash followed by a Windows\r\n# newline sequence (\"\\r\\n\")\r\n0=\"\"\"\\\r\n\"\"\"\r\n"
jsonRef := "{\n \"0\": {\n \"type\": \"string\",\n \"value\": \"\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_MultilineQuotes(t *testing.T) { func TestTOMLTest_Valid_String_MultilineQuotes(t *testing.T) {
input := "# Make sure that quotes inside multiline strings are allowed, including right\n# after the opening '''/\"\"\" and before the closing '''/\"\"\"\n\nlit_one = ''''one quote''''\nlit_two = '''''two quotes'''''\nlit_one_space = ''' 'one quote' '''\nlit_two_space = ''' ''two quotes'' '''\n\none = \"\"\"\"one quote\"\"\"\"\ntwo = \"\"\"\"\"two quotes\"\"\"\"\"\none_space = \"\"\" \"one quote\" \"\"\"\ntwo_space = \"\"\" \"\"two quotes\"\" \"\"\"\n\nmismatch1 = \"\"\"aaa'''bbb\"\"\"\nmismatch2 = '''aaa\"\"\"bbb'''\n" input := "# Make sure that quotes inside multiline strings are allowed, including right\n# after the opening '''/\"\"\" and before the closing '''/\"\"\"\n\nlit_one = ''''one quote''''\nlit_two = '''''two quotes'''''\nlit_one_space = ''' 'one quote' '''\nlit_two_space = ''' ''two quotes'' '''\n\none = \"\"\"\"one quote\"\"\"\"\ntwo = \"\"\"\"\"two quotes\"\"\"\"\"\none_space = \"\"\" \"one quote\" \"\"\"\ntwo_space = \"\"\" \"\"two quotes\"\" \"\"\"\n\nmismatch1 = \"\"\"aaa'''bbb\"\"\"\nmismatch2 = '''aaa\"\"\"bbb'''\n\n# Three opening \"\"\", then one escaped \", then two \"\" (allowed), and then three\n# closing \"\"\"\nescaped = \"\"\"lol\\\"\"\"\"\"\"\n"
jsonRef := "{\n \"lit_one\": {\n \"type\": \"string\",\n \"value\": \"'one quote'\"\n },\n \"lit_one_space\": {\n \"type\": \"string\",\n \"value\": \" 'one quote' \"\n },\n \"lit_two\": {\n \"type\": \"string\",\n \"value\": \"''two quotes''\"\n },\n \"lit_two_space\": {\n \"type\": \"string\",\n \"value\": \" ''two quotes'' \"\n },\n \"mismatch1\": {\n \"type\": \"string\",\n \"value\": \"aaa'''bbb\"\n },\n \"mismatch2\": {\n \"type\": \"string\",\n \"value\": \"aaa\\\"\\\"\\\"bbb\"\n },\n \"one\": {\n \"type\": \"string\",\n \"value\": \"\\\"one quote\\\"\"\n },\n \"one_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"one quote\\\" \"\n },\n \"two\": {\n \"type\": \"string\",\n \"value\": \"\\\"\\\"two quotes\\\"\\\"\"\n },\n \"two_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"\\\"two quotes\\\"\\\" \"\n }\n}\n" jsonRef := "{\n \"escaped\": {\n \"type\": \"string\",\n \"value\": \"lol\\\"\\\"\\\"\"\n },\n \"lit_one\": {\n \"type\": \"string\",\n \"value\": \"'one quote'\"\n },\n \"lit_one_space\": {\n \"type\": \"string\",\n \"value\": \" 'one quote' \"\n },\n \"lit_two\": {\n \"type\": \"string\",\n \"value\": \"''two quotes''\"\n },\n \"lit_two_space\": {\n \"type\": \"string\",\n \"value\": \" ''two quotes'' \"\n },\n \"mismatch1\": {\n \"type\": \"string\",\n \"value\": \"aaa'''bbb\"\n },\n \"mismatch2\": {\n \"type\": \"string\",\n \"value\": \"aaa\\\"\\\"\\\"bbb\"\n },\n \"one\": {\n \"type\": \"string\",\n \"value\": \"\\\"one quote\\\"\"\n },\n \"one_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"one quote\\\" \"\n },\n \"two\": {\n \"type\": \"string\",\n \"value\": \"\\\"\\\"two quotes\\\"\\\"\"\n },\n \"two_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"\\\"two quotes\\\"\\\" \"\n }\n}\n"
testgenValid(t, input, jsonRef) testgenValid(t, input, jsonRef)
} }
+32 -14
View File
@@ -322,10 +322,12 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val
return v, nil return v, nil
case reflect.Slice: case reflect.Slice:
elemType := v.Type().Elem() elemType := v.Type().Elem()
var elem reflect.Value
if elemType.Kind() == reflect.Interface { if elemType.Kind() == reflect.Interface {
elemType = mapStringInterfaceType elem = makeMapStringInterface()
} else {
elem = reflect.New(elemType).Elem()
} }
elem := reflect.New(elemType).Elem()
elem2, err := d.handleArrayTable(key, elem) elem2, err := d.handleArrayTable(key, elem)
if err != nil { if err != nil {
return reflect.Value{}, err return reflect.Value{}, err
@@ -611,7 +613,7 @@ func (d *decoder) tryTextUnmarshaler(node *ast.Node, v reflect.Value) (bool, err
if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) { if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) {
err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data) err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
if err != nil { if err != nil {
return false, newDecodeError(d.p.Raw(node.Raw), "error calling UnmarshalText: %w", err) return false, newDecodeError(d.p.Raw(node.Raw), "%w", err)
} }
return true, nil return true, nil
@@ -864,12 +866,27 @@ func (d *decoder) unmarshalFloat(value *ast.Node, v reflect.Value) error {
return nil return nil
} }
func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error { const (
const ( maxInt = int64(^uint(0) >> 1)
maxInt = int64(^uint(0) >> 1) minInt = -maxInt - 1
minInt = -maxInt - 1 )
)
// Maximum value of uint for decoding. Currently the decoder parses the integer
// into an int64. As a result, on architectures where uint is 64 bits, the
// effective maximum uint we can decode is the maximum of int64. On
// architectures where uint is 32 bits, the maximum value we can decode is
// lower: the maximum of uint32. I didn't find a way to figure out this value at
// compile time, so it is computed during initialization.
var maxUint int64 = math.MaxInt64
func init() {
m := uint64(^uint(0))
if m < uint64(maxUint) {
maxUint = int64(m)
}
}
func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
i, err := parseInteger(value.Data) i, err := parseInteger(value.Data)
if err != nil { if err != nil {
return err return err
@@ -930,7 +947,7 @@ func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
r = reflect.ValueOf(uint8(i)) r = reflect.ValueOf(uint8(i))
case reflect.Uint: case reflect.Uint:
if i < 0 { if i < 0 || i > maxUint {
return fmt.Errorf("toml: negative number %d does not fit in an uint", i) return fmt.Errorf("toml: negative number %d does not fit in an uint", i)
} }
@@ -1165,11 +1182,6 @@ func forEachField(t reflect.Type, path []int, do func(name string, path []int))
fieldPath := append(path, i) fieldPath := append(path, i)
fieldPath = fieldPath[:len(fieldPath):len(fieldPath)] fieldPath = fieldPath[:len(fieldPath):len(fieldPath)]
if f.Anonymous {
forEachField(f.Type, fieldPath, do)
continue
}
name := f.Tag.Get("toml") name := f.Tag.Get("toml")
if name == "-" { if name == "-" {
continue continue
@@ -1178,6 +1190,12 @@ func forEachField(t reflect.Type, path []int, do func(name string, path []int))
if i := strings.IndexByte(name, ','); i >= 0 { if i := strings.IndexByte(name, ','); i >= 0 {
name = name[:i] name = name[:i]
} }
if f.Anonymous && name == "" {
forEachField(f.Type, fieldPath, do)
continue
}
if name == "" { if name == "" {
name = f.Name name = f.Name
} }
+74 -2
View File
@@ -545,6 +545,35 @@ func TestUnmarshal(t *testing.T) {
} }
}, },
}, },
{
desc: "issue 739 - table redefinition",
input: `
[foo.bar.baz]
wibble = 'wobble'
[foo]
[foo.bar]
huey = 'dewey'
`,
gen: func() test {
m := map[string]interface{}{}
return test{
target: &m,
expected: &map[string]interface{}{
`foo`: map[string]interface{}{
"bar": map[string]interface{}{
"huey": "dewey",
"baz": map[string]interface{}{
"wibble": "wobble",
},
},
},
},
}
},
},
{ {
desc: "multiline basic string", desc: "multiline basic string",
input: `A = """\ input: `A = """\
@@ -942,7 +971,7 @@ B = "data"`,
"Name": "Hammer", "Name": "Hammer",
"Sku": int64(738594937), "Sku": int64(738594937),
}, },
map[string]interface{}(nil), map[string]interface{}{},
map[string]interface{}{ map[string]interface{}{
"Name": "Nail", "Name": "Nail",
"Sku": int64(284758393), "Sku": int64(284758393),
@@ -1476,7 +1505,7 @@ B = "data"`,
target: &map[string]interface{}{}, target: &map[string]interface{}{},
expected: &map[string]interface{}{ expected: &map[string]interface{}{
"products": []interface{}{ "products": []interface{}{
map[string]interface{}(nil), map[string]interface{}{},
}, },
}, },
} }
@@ -2351,6 +2380,49 @@ func TestIssue714(t *testing.T) {
require.Error(t, err) require.Error(t, err)
} }
func TestIssue772(t *testing.T) {
type FileHandling struct {
FilePattern string `toml:"pattern"`
}
type Config struct {
FileHandling `toml:"filehandling"`
}
var defaultConfigFile = []byte(`
[filehandling]
pattern = "reach-masterdev-"`)
config := Config{}
err := toml.Unmarshal(defaultConfigFile, &config)
require.NoError(t, err)
require.Equal(t, "reach-masterdev-", config.FileHandling.FilePattern)
}
func TestIssue774(t *testing.T) {
type ScpData struct {
Host string `json:"host"`
}
type GenConfig struct {
SCP []ScpData `toml:"scp" comment:"Array of Secure Copy Configurations"`
}
c := &GenConfig{}
c.SCP = []ScpData{{Host: "main.domain.com"}}
b, err := toml.Marshal(c)
require.NoError(t, err)
expected := `# Array of Secure Copy Configurations
[[scp]]
Host = 'main.domain.com'
`
require.Equal(t, expected, string(b))
}
func TestUnmarshalDecodeErrors(t *testing.T) { func TestUnmarshalDecodeErrors(t *testing.T) {
examples := []struct { examples := []struct {
desc string desc string