Compare commits

...

18 Commits

Author SHA1 Message Date
Thomas Pelletier 95c701b253 Increase test coverage (#538)
Also fix array in map bug.
2021-05-10 20:17:05 -04:00
Thomas Pelletier 3db329a512 ci: basic github action for coverage (#537) 2021-05-09 17:37:03 -04:00
Thomas Pelletier 45ea20024b Readme (#535) 2021-05-08 17:03:51 -04:00
Thomas Pelletier ea225df3ed v2: errors (#534)
```
name                              old time/op    new time/op    delta
UnmarshalDataset/config-32          86.7ms ± 2%    87.5ms ± 2%     ~     (p=0.113 n=9+10)
UnmarshalDataset/canada-32           129ms ± 4%     106ms ± 3%  -17.94%  (p=0.000 n=10+10)
UnmarshalDataset/citm_catalog-32    59.4ms ± 5%    58.7ms ± 5%     ~     (p=0.393 n=10+10)
UnmarshalDataset/twitter-32         27.0ms ± 7%    26.9ms ± 6%     ~     (p=0.720 n=10+9)
UnmarshalDataset/code-32             326ms ± 4%     322ms ± 7%     ~     (p=0.661 n=9+10)
UnmarshalDataset/example-32          510µs ±11%     526µs ± 7%     ~     (p=0.182 n=10+9)
UnmarshalSimple-32                  1.41µs ± 6%    1.41µs ± 4%     ~     (p=0.736 n=10+9)
ReferenceFile-32                    45.6µs ± 3%    43.9µs ±10%     ~     (p=0.089 n=10+10)

name                              old speed      new speed      delta
UnmarshalDataset/config-32        12.1MB/s ± 2%  12.0MB/s ± 2%     ~     (p=0.108 n=9+10)
UnmarshalDataset/canada-32        17.1MB/s ± 4%  20.9MB/s ± 3%  +21.86%  (p=0.000 n=10+10)
UnmarshalDataset/citm_catalog-32  9.41MB/s ± 5%  9.51MB/s ± 5%     ~     (p=0.362 n=10+10)
UnmarshalDataset/twitter-32       16.4MB/s ± 8%  16.5MB/s ± 6%     ~     (p=0.704 n=10+9)
UnmarshalDataset/code-32          8.24MB/s ± 4%  8.34MB/s ± 7%     ~     (p=0.675 n=9+10)
UnmarshalDataset/example-32       15.9MB/s ±11%  15.4MB/s ± 7%     ~     (p=0.182 n=10+9)
ReferenceFile-32                   115MB/s ± 4%   120MB/s ±10%     ~     (p=0.085 n=10+10)

name                              old alloc/op   new alloc/op   delta
UnmarshalDataset/config-32          16.9MB ± 0%    16.9MB ± 0%   -0.02%  (p=0.000 n=10+10)
UnmarshalDataset/canada-32          76.8MB ± 0%    74.3MB ± 0%   -3.31%  (p=0.000 n=10+10)
UnmarshalDataset/citm_catalog-32    37.3MB ± 0%    37.1MB ± 0%   -0.60%  (p=0.000 n=9+10)
UnmarshalDataset/twitter-32         15.6MB ± 0%    15.6MB ± 0%   -0.09%  (p=0.000 n=10+10)
UnmarshalDataset/code-32            60.2MB ± 0%    59.3MB ± 0%   -1.51%  (p=0.000 n=10+9)
UnmarshalDataset/example-32          238kB ± 0%     238kB ± 0%   -0.18%  (p=0.000 n=10+10)
ReferenceFile-32                    11.8kB ± 0%    11.8kB ± 0%     ~     (all equal)

name                              old allocs/op  new allocs/op  delta
UnmarshalDataset/config-32            653k ± 0%      645k ± 0%   -1.20%  (p=0.000 n=10+6)
UnmarshalDataset/canada-32           1.01M ± 0%     0.90M ± 0%  -11.04%  (p=0.000 n=9+10)
UnmarshalDataset/citm_catalog-32      384k ± 0%      370k ± 0%   -3.75%  (p=0.000 n=10+10)
UnmarshalDataset/twitter-32           160k ± 0%      157k ± 0%   -1.32%  (p=0.000 n=10+10)
UnmarshalDataset/code-32             2.97M ± 0%     2.91M ± 0%   -2.15%  (p=0.000 n=10+7)
UnmarshalDataset/example-32          3.69k ± 0%     3.63k ± 0%   -1.52%  (p=0.000 n=10+10)
ReferenceFile-32                       253 ± 0%       253 ± 0%     ~     (all equal)
```
2021-05-08 16:04:25 -04:00
Thomas Pelletier 4545a3e94b ci: remove benchmarks
Both github actions and my own VPS have too much noise to be useful.
2021-05-07 23:34:17 -04:00
Vincent Serpoul 3f2bb0b363 golangci-lint (#530) 2021-05-06 22:29:21 -04:00
Vincent Serpoul 201d5dd422 golangci-lint: misc (#529) 2021-04-27 20:29:00 -04:00
Thomas Pelletier 1e80267558 parser: require \n after parsing integer in kv (#527)
Fixes #526
2021-04-24 09:57:21 -04:00
Thomas Pelletier 931f02a519 encoder: support indentation (#525) 2021-04-23 17:08:27 -04:00
Thomas Pelletier a533331aee v2: benchdiff (#524) 2021-04-23 15:21:41 -04:00
Vincent Serpoul 466faaab9f golangci-lint: marshaler, strict (#523) 2021-04-23 10:41:21 -04:00
Thomas Pelletier e443b4fdb8 encoder: support TextMarshaler (#522)
Fixes #521
2021-04-22 10:13:41 -04:00
Vincent Serpoul 2b1c52dddd golangci-lint: decoder/unmarshal (#518) 2021-04-22 09:29:23 -04:00
Thomas Pelletier 21445f5170 Add test for issue #424 2021-04-21 22:27:30 -04:00
Thomas Pelletier 9ba52996d8 Encoder multiline array (#520) 2021-04-21 22:13:45 -04:00
Thomas Pelletier 6fe332a869 Encoder inline tables (#519) 2021-04-21 19:11:15 -04:00
Thomas Pelletier 32c1a8d372 encoder: move nspow into the parseLocalTime 2021-04-20 23:19:40 -04:00
Thomas Pelletier ee102a3528 decoder: fix time fractional parsing 2021-04-20 23:16:08 -04:00
32 changed files with 3497 additions and 1318 deletions
+16 -2
View File
@@ -1,5 +1,19 @@
**Issue:** add link to pelletier/go-toml issue here
<!--
Thank you for your pull request!
Please read the Code changes section of the CONTRIBUTING.md file,
and make sure you have followed the instructions.
https://github.com/pelletier/go-toml/blob/v2/CONTRIBUTING.md#code-changes
-->
Explanation of what this pull request does.
More detailed description of the decisions being made and the reasons why (if the patch is non-trivial).
More detailed description of the decisions being made and the reasons why (if
the patch is non-trivial).
---
Paste `benchstat` results here
+23
View File
@@ -0,0 +1,23 @@
name: coverage
on:
push:
branches:
- v2
pull_request:
branches:
- v2
jobs:
report:
runs-on: 'ubuntu-latest'
name: report
steps:
- uses: actions/checkout@master
with:
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@master
with:
go-version: 1.16
- name: Run tests with coverage
run: ./ci.sh coverage -d "${GITHUB_BASE_REF-HEAD}"
-3
View File
@@ -23,6 +23,3 @@ jobs:
go-version: ${{ matrix.go }}
- name: Run unit tests
run: go test -race ./...
- name: Run benchmark tests
run: go test -race ./...
working-directory: benchmark
+7 -7
View File
@@ -21,13 +21,13 @@ enable = [
"errcheck",
"errorlint",
"exhaustive",
"exhaustivestruct",
# "exhaustivestruct",
"exportloopref",
"forbidigo",
"forcetypeassert",
# "forcetypeassert",
"funlen",
"gci",
"gochecknoglobals",
# "gochecknoglobals",
"gochecknoinits",
"gocognit",
"goconst",
@@ -35,7 +35,7 @@ enable = [
"gocyclo",
"godot",
"godox",
"goerr113",
# "goerr113",
"gofmt",
"gofumpt",
"goheader",
@@ -57,7 +57,7 @@ enable = [
"nakedret",
"nestif",
"nilerr",
"nlreturn",
# "nlreturn",
"noctx",
"nolintlint",
"paralleltest",
@@ -79,6 +79,6 @@ enable = [
"varcheck",
"wastedassign",
"whitespace",
"wrapcheck",
"wsl"
# "wrapcheck",
# "wsl"
]
+113 -63
View File
@@ -1,74 +1,74 @@
## Contributing
# Contributing
Thank you for your interest in go-toml! We appreciate you considering
contributing to go-toml!
The main goal is the project is to provide an easy-to-use TOML
implementation for Go that gets the job done and gets out of your way
dealing with TOML is probably not the central piece of your project.
The main goal is the project is to provide an easy-to-use and efficient TOML
implementation for Go that gets the job done and gets out of your way dealing
with TOML is probably not the central piece of your project.
As the single maintainer of go-toml, time is scarce. All help, big or
small, is more than welcomed!
As the single maintainer of go-toml, time is scarce. All help, big or small, is
more than welcomed!
### Ask questions
## Ask questions
Any question you may have, somebody else might have it too. Always feel
free to ask them on the [issues tracker][issues-tracker]. We will try to
answer them as clearly and quickly as possible, time permitting.
Any question you may have, somebody else might have it too. Always feel free to
ask them on the [discussion board][discussions]. We will try to answer them as
clearly and quickly as possible, time permitting.
Asking questions also helps us identify areas where the documentation needs
improvement, or new features that weren't envisioned before. Sometimes, a
seemingly innocent question leads to the fix of a bug. Don't hesitate and
ask away!
seemingly innocent question leads to the fix of a bug. Don't hesitate and ask
away!
### Improve the documentation
[discussions]: https://github.com/pelletier/go-toml/discussions
The best way to share your knowledge and experience with go-toml is to
improve the documentation. Fix a typo, clarify an interface, add an
example, anything goes!
## Improve the documentation
The documentation is present in the [README][readme] and thorough the
source code. On release, it gets updated on [pkg.go.dev][pkg.go.dev]. To make a
change to the documentation, create a pull request with your proposed
changes. For simple changes like that, the easiest way to go is probably
the "Fork this project and edit the file" button on Github, displayed at
the top right of the file. Unless it's a trivial change (for example a
typo), provide a little bit of context in your pull request description or
commit message.
The best way to share your knowledge and experience with go-toml is to improve
the documentation. Fix a typo, clarify an interface, add an example, anything
goes!
### Report a bug
The documentation is present in the [README][readme] and thorough the source
code. On release, it gets updated on [pkg.go.dev][pkg.go.dev]. To make a change
to the documentation, create a pull request with your proposed changes. For
simple changes like that, the easiest way to go is probably the "Fork this
project and edit the file" button on Github, displayed at the top right of the
file. Unless it's a trivial change (for example a typo), provide a little bit of
context in your pull request description or commit message.
Found a bug! Sorry to hear that :(. Help us and other track them down and
fix by reporting it. [File a new bug report][bug-report] on the [issues
tracker][issues-tracker]. The template should provide enough guidance on
what to include. When in doubt: add more details! By reducing ambiguity and
providing more information, it decreases back and forth and saves everyone
time.
## Report a bug
### Code changes
Found a bug! Sorry to hear that :(. Help us and other track them down and fix by
reporting it. [File a new bug report][bug-report] on the [issues
tracker][issues-tracker]. The template should provide enough guidance on what to
include. When in doubt: add more details! By reducing ambiguity and providing
more information, it decreases back and forth and saves everyone time.
## Code changes
Want to contribute a patch? Very happy to hear that!
First, some high-level rules:
* A short proposal with some POC code is better than a lengthy piece of
text with no code. Code speaks louder than words.
* No backward-incompatible patch will be accepted unless discussed.
Sometimes it's hard, and Go's lack of versioning by default does not
help, but we try not to break people's programs unless we absolutely have
- A short proposal with some POC code is better than a lengthy piece of text
with no code. Code speaks louder than words. That being said, bigger changes
should probably start with a [discussion][discussions].
- No backward-incompatible patch will be accepted unless discussed. Sometimes
it's hard, but we try not to break people's programs unless we absolutely have
to.
* If you are writing a new feature or extending an existing one, make sure
to write some documentation.
* Bug fixes need to be accompanied with regression tests.
* New code needs to be tested.
* Your commit messages need to explain why the change is needed, even if
already included in the PR description.
- If you are writing a new feature or extending an existing one, make sure to
write some documentation.
- Bug fixes need to be accompanied with regression tests.
- New code needs to be tested.
- Your commit messages need to explain why the change is needed, even if already
included in the PR description.
It does sound like a lot, but those best practices are here to save time
overall and continuously improve the quality of the project, which is
something everyone benefits from.
It does sound like a lot, but those best practices are here to save time overall
and continuously improve the quality of the project, which is something everyone
benefits from.
#### Get started
### Get started
The fairly standard code contribution process looks like that:
@@ -76,42 +76,92 @@ The fairly standard code contribution process looks like that:
2. Make your changes, commit on any branch you like.
3. [Open up a pull request][pull-request]
4. Review, potential ask for changes.
5. Merge. You're in!
5. Merge.
Feel free to ask for help! You can create draft pull requests to gather
some early feedback!
#### Run the tests
### Run the tests
You can run tests for go-toml using Go's test tool: `go test ./...`.
When creating a pull requests, all tests will be ran on Linux on a few Go
versions (Travis CI), and on Windows using the latest Go version
(AppVeyor).
You can run tests for go-toml using Go's test tool: `go test -race ./...`.
#### Style
During the pull request process, all tests will be ran on Linux, Windows, and
MacOS on the last two versions of Go.
Try to look around and follow the same format and structure as the rest of
the code. We enforce using `go fmt` on the whole code base.
However, given GitHub's new policy to _not_ run Actions on pull requests until a
maintainer clicks on button, it is highly recommended that you run them locally
as you make changes.
### Check coverage
We use `go tool cover` to compute test coverage. Most code editors have a way to
run and display code coverage, but at the end of the day, we do this:
```
go test -covermode=atomic -coverprofile=coverage.out
go tool cover -func=coverage.out
```
and verify that the overall percentage of tested code does not go down. This is
a requirement. As a rule of thumb, all lines of code touched by your changes
should be covered. On Unix you can use `./ci.sh coverage -d v2` to check if your
code lowers the coverage.
### Verify performance
Go-toml aims to stay efficient. We rely on a set of scenarios executed with Go's
builtin benchmark systems. Because of their noisy nature, containers provided by
Github Actions cannot be reliably used for benchmarking. As a result, you are
responsible for checking that your changes do not incur a performance penalty.
You can run their following to execute benchmarks:
```
go test ./... -bench=. -count=10
```
Benchmark results should be compared against each other with
[benchstat][benchstat]. Typical flow looks like this:
1. On the `v2` branch, run `go test ./... -bench=. -count 10` and save output to
a file (for example `old.txt`).
2. Make some code changes.
3. Run `go test ....` again, and save the output to an other file (for example
`new.txt`).
4. Run `benchstat old.txt new.txt` to check that time/op does not go up in any
test.
On Unix you can use `./ci.sh benchmark -d v2` to verify how your code impacts
performance.
It is highly encouraged to add the benchstat results to your pull request
description. Pull requests that lower performance will receive more scrutiny.
[benchstat]: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat
### Style
Try to look around and follow the same format and structure as the rest of the
code. We enforce using `go fmt` on the whole code base.
---
### Maintainers-only
## Maintainers-only
#### Merge pull request
### Merge pull request
Checklist:
* Passing CI.
* Does not introduce backward-incompatible changes (unless discussed).
* Has relevant doc changes.
* Has relevant unit tests.
- Passing CI.
- Does not introduce backward-incompatible changes (unless discussed).
- Has relevant doc changes.
- Benchstat does not show performance regression.
1. Merge using "squash and merge".
2. Make sure to edit the commit message to keep all the useful information
nice and clean.
3. Make sure the commit title is clear and contains the PR number (#123).
#### New release
### New release
1. Go to [releases][releases]. Click on "X commits to master since this
release".
+334 -40
View File
@@ -1,57 +1,351 @@
# go-toml V2
# go-toml v2
Development branch. Use at your own risk.
Go library for the [TOML](https://toml.io/en/) format.
[👉 Discussion on github](https://github.com/pelletier/go-toml/discussions/471).
This library supports [TOML v1.0.0](https://toml.io/en/v1.0.0).
* `toml.Unmarshal()` should work as well as v1.
## Must do
## Development status
### Unmarshal
This is the upcoming major version of go-toml. It is currently in active
development. As of release v2.0.0-beta.1, the library has reached feature parity
with v1, and fixes a lot known bugs and performance issues along the way.
- [x] Unmarshal into maps.
- [x] Support Array Tables.
- [x] Unmarshal into pointers.
- [x] Support Date / times.
- [x] Support struct tags annotations.
- [x] Support Arrays.
- [x] Support Unmarshaler interface.
- [x] Original go-toml unmarshal tests pass.
- [x] Benchmark!
- [x] Abstract AST.
- [x] Original go-toml testgen tests pass.
- [x] Track file position (line, column) for errors.
- [x] Strict mode.
- [ ] Document Unmarshal / Decode
If you do not need the advanced document editing features of v1, you are
encouraged to try out this version.
### Marshal
👉 [Roadmap for v2](https://github.com/pelletier/go-toml/discussions/506).
- [x] Minimal implementation
- [x] Multiline strings
- [ ] Multiline arrays
- [ ] `inline` tag for tables
- [ ] Optional indentation
- [ ] Option to pick default quotes
### Document
## Documentation
- [ ] Gather requirements and design API.
Full API, examples, and implementation notes are available in the Go documentation.
## Ideas
[![Go Reference](https://pkg.go.dev/badge/github.com/pelletier/go-toml/v2.svg)](https://pkg.go.dev/github.com/pelletier/go-toml/v2)
- [ ] Allow types to implement a `ASTUnmarshaler` interface to unmarshal
straight from the AST?
- [x] Rewrite AST to use a single array as storage instead of one allocation per
node.
- [ ] Provide "minimal allocations" option that uses `unsafe` to reuse the input
byte array as storage for strings.
- [x] Cache reflection operations per type.
- [ ] Optimize tracker pass.
## Differences with v1
## Import
* [unmarshal](https://github.com/pelletier/go-toml/discussions/488)
```go
import "github.com/pelletier/go-toml/v2"
```
## Features
### Stdlib behavior
As much as possible, this library is designed to behave similarly as the
standard library's `encoding/json`.
### Performance
While go-toml favors usability, it is written with performance in mind. Most
operations should not be shockingly slow.
### Strict mode
`Decoder` can be set to "strict mode", which makes it error when some parts of
the TOML document was not prevent in the target structure. This is a great way
to check for typos. [See example in the documentation][strict].
[strict]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#example-Decoder.SetStrict
### Contextualized errors
When decoding errors occur, go-toml returns [`DecodeError`][decode-err]), which
contains a human readable contextualized version of the error. For example:
```
2| key1 = "value1"
3| key2 = "missing2"
| ~~~~ missing field
4| key3 = "missing3"
5| key4 = "value4"
```
[decode-err]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#DecodeError
### Local date and time support
TOML supports native [local date/times][ldt]. It allows to represent a given
date, time, or date-time without relation to a timezone or offset. To support
this use-case, go-toml provides [`LocalDate`][tld], [`LocalTime`][tlt], and
[`LocalDateTime`][tldt]. Those types can be transformed to and from `time.Time`,
making them convenient yet unambiguous structures for their respective TOML
representation.
[ldt]: https://toml.io/en/v1.0.0#local-date-time
[tld]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDate
[tlt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalTime
[tldt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDateTime
## Getting started
Given the following struct, let's see how to read it and write it as TOML:
```go
type MyConfig struct {
Version int
Name string
Tags []string
}
```
### Unmarshaling
[`Unmarshal`][unmarshal] reads a TOML document and fills a Go structure with its
content. For example:
```go
doc := `
version = 2
name = "go-toml"
tags = ["go", "toml"]
`
var cfg MyConfig
err := toml.Unmarshal([]byte(doc), &cfg)
if err != nil {
panic(err)
}
fmt.Println("version:", cfg.Version)
fmt.Println("name:", cfg.Name)
fmt.Println("tags:", cfg.Tags)
// Output:
// version: 2
// name: go-toml
// tags: [go toml]
```
[unmarshal]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#Unmarshal
### Marshaling
[`Marshal`][marshal] is the opposite of Unmarshal: it represents a Go structure
as a TOML document:
```go
cfg := MyConfig{
Version: 2,
Name: "go-toml",
Tags: []string{"go", "toml"},
}
b, err := toml.Marshal(cfg)
if err != nil {
panic(err)
}
fmt.Println(string(b))
// Output:
// Version = 2
// Name = 'go-toml'
// Tags = ['go', 'toml']
```
[marshal]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#Marshal
## Migrating from v1
This section describes the differences between v1 and v2, with some pointers on
how to get the original behavior when possible.
### Decoding / Unmarshal
#### Automatic field name guessing
When unmarshaling to a struct, if a key in the TOML document does not exactly
match the name of a struct field or any of the `toml`-tagged field, v1 tries
multiple variations of the key ([code][v1-keys]).
V2 instead does a case-insensitive matching, like `encoding/json`.
This could impact you if you are relying on casing to differentiate two fields,
and one of them is a not using the `toml` struct tag. The recommended solution
is to be specific about tag names for those fields using the `toml` struct tag.
[v1-keys]: https://github.com/pelletier/go-toml/blob/a2e52561804c6cd9392ebf0048ca64fe4af67a43/marshal.go#L775-L781
#### Ignore preexisting value in interface
When decoding into a non-nil `interface{}`, go-toml v1 uses the type of the
element in the interface to decode the object. For example:
```go
type inner struct {
B interface{}
}
type doc struct {
A interface{}
}
d := doc{
A: inner{
B: "Before",
},
}
data := `
[A]
B = "After"
`
toml.Unmarshal([]byte(data), &d)
fmt.Printf("toml v1: %#v\n", d)
// toml v1: main.doc{A:main.inner{B:"After"}}
```
In this case, field `A` is of type `interface{}`, containing a `inner` struct.
V1 sees that type and uses it when decoding the object.
When decoding an object into an `interface{}`, V2 instead disregards whatever
value the `interface{}` may contain and replaces it with a
`map[string]interface{}`. With the same data structure as above, here is what
the result looks like:
```go
toml.Unmarshal([]byte(data), &d)
fmt.Printf("toml v2: %#v\n", d)
// toml v2: main.doc{A:map[string]interface {}{"B":"After"}}
```
This is to match `encoding/json`'s behavior. There is no way to make the v2
decoder behave like v1.
#### Values out of array bounds ignored
When decoding into an array, v1 returns an error when the number of elements
contained in the doc is superior to the capacity of the array. For example:
```go
type doc struct {
A [2]string
}
d := doc{}
err := toml.Unmarshal([]byte(`A = ["one", "two", "many"]`), &d)
fmt.Println(err)
// (1, 1): unmarshal: TOML array length (3) exceeds destination array length (2)
```
In the same situation, v2 ignores the last value:
```go
err := toml.Unmarshal([]byte(`A = ["one", "two", "many"]`), &d)
fmt.Println("err:", err, "d:", d)
// err: <nil> d: {[one two]}
```
This is to match `encoding/json`'s behavior. There is no way to make the v2
decoder behave like v1.
#### Support for `toml.Unmarshaler` has been dropped
This method was not widely used, poorly defined, and added a lot of complexity.
A similar effect can be achieved by implementing the `encoding.TextUnmarshaler`
interface and use strings.
### Encoding / Marshal
#### Default struct fields order
V1 emits struct fields order alphabetically by default. V2 struct fields are
emitted in order they are defined. For example:
```go
type S struct {
B string
A string
}
data := S{
B: "B",
A: "A",
}
b, _ := tomlv1.Marshal(data)
fmt.Println("v1:\n" + string(b))
b, _ = tomlv2.Marshal(data)
fmt.Println("v2:\n" + string(b))
// Output:
// v1:
// A = "A"
// B = "B"
// v2:
// B = 'B'
// A = 'A'
```
There is no way to make v2 encoder behave like v1. A workaround could be to
manually sort the fields alphabetically in the struct definition.
#### No indentation by default
V1 automatically indents content of tables by default. V2 does not. However the
same behavior can be obtained using [`Encoder.SetIndentTables`][sit]. For example:
```go
data := map[string]interface{}{
"table": map[string]string{
"key": "value",
},
}
b, _ := tomlv1.Marshal(data)
fmt.Println("v1:\n" + string(b))
b, _ = tomlv2.Marshal(data)
fmt.Println("v2:\n" + string(b))
buf := bytes.Buffer{}
enc := tomlv2.NewEncoder(&buf)
enc.SetIndentTables(true)
enc.Encode(data)
fmt.Println("v2 Encoder:\n" + string(buf.Bytes()))
// Output:
// v1:
//
// [table]
// key = "value"
//
// v2:
// [table]
// key = 'value'
//
//
// v2 Encoder:
// [table]
// key = 'value'
```
[sit]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#Encoder.SetIndentTables
#### Keys and strings are single quoted
V1 always uses double quotes (`"`) around strings and keys that cannot be
represented bare (unquoted). V2 uses single quotes instead by default (`'`),
unless a character cannot be represented, then falls back to double quotes.
There is no way to make v2 encoder behave like v1.
#### `TextMarshaler` emits as a string, not TOML
Types that implement [`encoding.TextMarshaler`][tm] can emit arbitrary TOML in
v1. The encoder would append the result to the output directly. In v2 the result
is wrapped in a string. As a result, this interface cannot be implemented by the
root object.
There is no way to make v2 encoder behave like v1.
[tm]: https://golang.org/pkg/encoding/#TextMarshaler
## License
+13 -26
View File
@@ -8,6 +8,7 @@ import (
"path/filepath"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/require"
)
@@ -32,20 +33,12 @@ func TestUnmarshalDatasetCode(t *testing.T) {
for _, tc := range bench_inputs {
buf := fixture(t, tc.name)
t.Run(tc.name, func(t *testing.T) {
for _, r := range runners {
if r.name == "bs" && tc.name == "canada" {
t.Skip("skipping: burntsushi can't handle mixed arrays")
}
var v interface{}
check(t, toml.Unmarshal(buf, &v))
t.Run(r.name, func(t *testing.T) {
var v interface{}
check(t, r.unmarshal(buf, &v))
b, err := json.Marshal(v)
check(t, err)
require.Equal(t, len(b), tc.jsonLen)
})
}
b, err := json.Marshal(v)
check(t, err)
require.Equal(t, len(b), tc.jsonLen)
})
}
}
@@ -54,19 +47,13 @@ func BenchmarkUnmarshalDataset(b *testing.B) {
for _, tc := range bench_inputs {
buf := fixture(b, tc.name)
b.Run(tc.name, func(b *testing.B) {
bench(b, func(r runner, b *testing.B) {
if r.name == "bs" && tc.name == "canada" {
b.Skip("skipping: burntsushi can't handle mixed arrays")
}
b.SetBytes(int64(len(buf)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
check(b, r.unmarshal(buf, &v))
}
})
b.SetBytes(int64(len(buf)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
check(b, toml.Unmarshal(buf, &v))
}
})
}
}
+21 -46
View File
@@ -5,44 +5,21 @@ import (
"testing"
"time"
tomlbs "github.com/BurntSushi/toml"
tomlv1 "github.com/pelletier/go-toml-v1"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/require"
)
type runner struct {
name string
unmarshal func([]byte, interface{}) error
}
var runners = []runner{
{"v2", toml.Unmarshal},
{"v1", tomlv1.Unmarshal},
{"bs", tomlbs.Unmarshal},
}
func bench(b *testing.B, f func(r runner, b *testing.B)) {
for _, r := range runners {
b.Run(r.name, func(b *testing.B) {
f(r, b)
})
}
}
func BenchmarkUnmarshalSimple(b *testing.B) {
bench(b, func(r runner, b *testing.B) {
d := struct {
A string
}{}
doc := []byte(`A = "hello"`)
for i := 0; i < b.N; i++ {
err := r.unmarshal(doc, &d)
if err != nil {
panic(err)
}
d := struct {
A string
}{}
doc := []byte(`A = "hello"`)
for i := 0; i < b.N; i++ {
err := toml.Unmarshal(doc, &d)
if err != nil {
panic(err)
}
})
}
}
type benchmarkDoc struct {
@@ -152,22 +129,20 @@ type benchmarkDoc struct {
}
func BenchmarkReferenceFile(b *testing.B) {
bench(b, func(r runner, b *testing.B) {
bytes, err := ioutil.ReadFile("benchmark.toml")
bytes, err := ioutil.ReadFile("benchmark.toml")
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(bytes)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
d := benchmarkDoc{}
err := toml.Unmarshal(bytes, &d)
if err != nil {
b.Fatal(err)
panic(err)
}
b.SetBytes(int64(len(bytes)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
d := benchmarkDoc{}
err := r.unmarshal(bytes, &d)
if err != nil {
panic(err)
}
}
})
}
}
func TestReferenceFile(t *testing.T) {
-14
View File
@@ -1,14 +0,0 @@
module github.com/pelletier/go-toml/v2/benchmark
go 1.16
replace github.com/pelletier/go-toml/v2 => ../
replace github.com/pelletier/go-toml-v1 => github.com/pelletier/go-toml v1.8.1
require (
github.com/BurntSushi/toml v0.3.1
github.com/pelletier/go-toml-v1 v0.0.0-00010101000000-000000000000
github.com/pelletier/go-toml/v2 v2.0.0-00010101000000-000000000000
github.com/stretchr/testify v1.7.0
)
-16
View File
@@ -1,16 +0,0 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Executable
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env bash
stderr() {
echo "$@" 1>&2
}
usage() {
b=$(basename "$0")
echo $b: ERROR: "$@" 1>&2
cat 1>&2 <<EOF
DESCRIPTION
$(basename "$0") is the script to run continuous integration commands for
go-toml on unix.
Requires Go and Git to be available in the PATH. Expects to be ran from the
root of go-toml's Git repository.
USAGE
$b COMMAND [OPTIONS...]
COMMANDS
benchmark [OPTIONS...] [BRANCH]
Run benchmarks.
ARGUMENTS
BRANCH Optional. Defines which Git branch to use when running
benchmarks.
OPTIONS
-d Compare benchmarks of HEAD with BRANCH using benchstats. In
this form the BRANCH argument is required.
coverage [OPTIONS...] [BRANCH]
Generates code coverage.
ARGUMENTS
BRANCH Optional. Defines which Git branch to use when reporting
coverage. Defaults to HEAD.
OPTIONS
-d Compare coverage of HEAD with the one of BRANCH. In this form,
the BRANCH argument is required. Exit code is non-zero when
coverage percentage decreased.
EOF
exit 1
}
cover() {
branch="${1}"
dir="$(mktemp -d)"
stderr "Executing coverage for ${branch} at ${dir}"
if [ "${branch}" = "HEAD" ]; then
cp -r . "${dir}/"
else
git worktree add "$dir" "$branch"
fi
pushd "$dir"
go test -covermode=atomic -coverprofile=coverage.out ./...
go tool cover -func=coverage.out
popd
if [ "${branch}" != "HEAD" ]; then
git worktree remove --force "$dir"
fi
}
coverage() {
case "$1" in
-d)
shift
target="${1?Need to provide a target branch argument}"
output_dir="$(mktemp -d)"
target_out="${output_dir}/target.txt"
head_out="${output_dir}/head.txt"
cover "${target}" > "${target_out}"
cover "HEAD" > "${head_out}"
cat "${target_out}"
cat "${head_out}"
echo ""
target_pct="$(cat ${target_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
head_pct="$(cat ${head_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
echo "Delta: ${delta_pct}"
if [[ $delta_pct = \-* ]]; then
echo "Regression!";
return 1
fi
return 0
;;
esac
cover "${1-HEAD}"
}
bench() {
branch="${1}"
out="${2}"
dir="$(mktemp -d)"
stderr "Executing benchmark for ${branch} at ${dir}"
if [ "${branch}" = "HEAD" ]; then
cp -r . "${dir}/"
else
git worktree add "$dir" "$branch"
fi
pushd "$dir"
go test -bench=. -count=10 ./... | tee "${out}"
popd
if [ "${branch}" != "HEAD" ]; then
git worktree remove --force "$dir"
fi
}
benchmark() {
case "$1" in
-d)
shift
target="${1?Need to provide a target branch argument}"
old=`mktemp`
bench "${target}" "${old}"
new=`mktemp`
bench HEAD "${new}"
benchstat "${old}" "${new}"
return 0
;;
esac
bench "${1-HEAD}" `mktemp`
}
case "$1" in
coverage) shift; coverage $@;;
benchmark) shift; benchmark $@;;
*) usage "bad argument $1";;
esac
+93 -160
View File
@@ -1,11 +1,9 @@
package toml
import (
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
)
@@ -19,7 +17,7 @@ func parseInteger(b []byte) (int64, error) {
case 'o':
return parseIntOct(b)
default:
return 0, newDecodeError(b[1:2], "invalid base: '%c'", b[1])
panic(fmt.Errorf("invalid base '%c', should have been checked by scanIntOrFloat", b[1]))
}
}
@@ -37,52 +35,34 @@ func parseLocalDate(b []byte) (LocalDate, error) {
return date, newDecodeError(b, "dates are expected to have the format YYYY-MM-DD")
}
var err error
date.Year = parseDecimalDigits(b[0:4])
date.Year, err = parseDecimalDigits(b[0:4])
if err != nil {
return date, err
}
v, err := parseDecimalDigits(b[5:7])
if err != nil {
return date, err
}
v := parseDecimalDigits(b[5:7])
date.Month = time.Month(v)
date.Day, err = parseDecimalDigits(b[8:10])
if err != nil {
return date, err
}
date.Day = parseDecimalDigits(b[8:10])
return date, nil
}
var errNotDigit = errors.New("not a digit")
func parseDecimalDigits(b []byte) (int, error) {
func parseDecimalDigits(b []byte) int {
v := 0
for _, c := range b {
if !isDigit(c) {
return 0, fmt.Errorf("%s: %w", b, errNotDigit)
}
v *= 10
v += int(c - '0')
}
return v, nil
return v
}
var errParseDateTimeMissingInfo = errors.New("date-time missing timezone information")
func parseDateTime(b []byte) (time.Time, error) {
// offset-date-time = full-date time-delim full-time
// full-time = partial-time time-offset
// time-offset = "Z" / time-numoffset
// time-numoffset = ( "+" / "-" ) time-hour ":" time-minute
dt, b, err := parseLocalDateTime(b)
if err != nil {
return time.Time{}, err
@@ -91,7 +71,8 @@ func parseDateTime(b []byte) (time.Time, error) {
var zone *time.Location
if len(b) == 0 {
return time.Time{}, errParseDateTimeMissingInfo
// parser should have checked that when assigning the date time node
panic("date time should have a timezone")
}
if b[0] == 'Z' {
@@ -103,18 +84,15 @@ func parseDateTime(b []byte) (time.Time, error) {
return time.Time{}, newDecodeError(b, "invalid date-time timezone")
}
direction := 1
switch b[0] {
case '+':
case '-':
if b[0] == '-' {
direction = -1
default:
return time.Time{}, newDecodeError(b[0:1], "invalid timezone offset character")
}
hours := digitsToInt(b[1:3])
minutes := digitsToInt(b[4:6])
seconds := direction * (hours*3600 + minutes*60)
zone = time.FixedZone("", seconds)
b = b[dateTimeByteLen:]
}
if len(b) > 0 {
@@ -134,19 +112,12 @@ func parseDateTime(b []byte) (time.Time, error) {
return t, nil
}
var (
errParseLocalDateTimeWrongLength = errors.New(
"local datetimes are expected to have the format YYYY-MM-DDTHH:MM:SS[.NNNNNN]",
)
errParseLocalDateTimeWrongSeparator = errors.New("datetime separator is expected to be T or a space")
)
func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) {
var dt LocalDateTime
const localDateTimeByteLen = 11
if len(b) < localDateTimeByteLen {
return dt, nil, errParseLocalDateTimeWrongLength
const localDateTimeByteMinLen = 11
if len(b) < localDateTimeByteMinLen {
return dt, nil, newDecodeError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM:SS[.NNNNNNNNN]")
}
date, err := parseLocalDate(b[:10])
@@ -157,7 +128,7 @@ func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) {
sep := b[10]
if sep != 'T' && sep != ' ' {
return dt, nil, errParseLocalDateTimeWrongSeparator
return dt, nil, newDecodeError(b[10:11], "datetime separator is expected to be T or a space")
}
t, rest, err := parseLocalTime(b[11:])
@@ -169,213 +140,175 @@ func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) {
return dt, rest, nil
}
var errParseLocalTimeWrongLength = errors.New("times are expected to have the format HH:MM:SS[.NNNNNN]")
// parseLocalTime is a bit different because it also returns the remaining
// []byte that is didn't need. This is to allow parseDateTime to parse those
// remaining bytes as a timezone.
func parseLocalTime(b []byte) (LocalTime, []byte, error) {
var t LocalTime
var (
nspow = [10]int{0, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 1e2, 1e1, 1e0}
t LocalTime
)
const localTimeByteLen = 8
if len(b) < localTimeByteLen {
return t, nil, errParseLocalTimeWrongLength
}
var err error
t.Hour, err = parseDecimalDigits(b[0:2])
if err != nil {
return t, nil, err
return t, nil, newDecodeError(b, "times are expected to have the format HH:MM:SS[.NNNNNN]")
}
t.Hour = parseDecimalDigits(b[0:2])
if b[2] != ':' {
return t, nil, newDecodeError(b[2:3], "expecting colon between hours and minutes")
}
t.Minute, err = parseDecimalDigits(b[3:5])
if err != nil {
return t, nil, err
}
t.Minute = parseDecimalDigits(b[3:5])
if b[5] != ':' {
return t, nil, newDecodeError(b[5:6], "expecting colon between minutes and seconds")
}
t.Second, err = parseDecimalDigits(b[6:8])
if err != nil {
return t, nil, err
}
t.Second = parseDecimalDigits(b[6:8])
if len(b) >= 15 && b[8] == '.' {
t.Nanosecond, err = parseDecimalDigits(b[9:15])
if err != nil {
return t, nil, err
const minLengthWithFrac = 9
if len(b) >= minLengthWithFrac && b[minLengthWithFrac-1] == '.' {
frac := 0
digits := 0
for i, c := range b[minLengthWithFrac:] {
const maxFracPrecision = 9
if i >= maxFracPrecision {
return t, nil, newDecodeError(b[i:i+1], "maximum precision for date time is nanosecond")
}
frac *= 10
frac += int(c - '0')
digits++
}
return t, b[15:], nil
t.Nanosecond = frac * nspow[digits]
return t, b[9+digits:], nil
}
return t, b[8:], nil
}
var (
errParseFloatStartDot = errors.New("float cannot start with a dot")
errParseFloatEndDot = errors.New("float cannot end with a dot")
)
//nolint:cyclop
func parseFloat(b []byte) (float64, error) {
//nolint:godox
// TODO: inefficient
if len(b) == 4 && (b[0] == '+' || b[0] == '-') && b[1] == 'n' && b[2] == 'a' && b[3] == 'n' {
return math.NaN(), nil
}
tok := string(b)
err := numberContainsInvalidUnderscore(tok)
cleaned, err := checkAndRemoveUnderscores(b)
if err != nil {
return 0, err
}
cleanedVal := cleanupNumberToken(tok)
if cleanedVal[0] == '.' {
return 0, errParseFloatStartDot
if cleaned[0] == '.' {
return 0, newDecodeError(b, "float cannot start with a dot")
}
if cleanedVal[len(cleanedVal)-1] == '.' {
return 0, errParseFloatEndDot
if cleaned[len(cleaned)-1] == '.' {
return 0, newDecodeError(b, "float cannot end with a dot")
}
f, err := strconv.ParseFloat(cleanedVal, 64)
f, err := strconv.ParseFloat(string(cleaned), 64)
if err != nil {
return 0, fmt.Errorf("coudn't ParseFloat %w", err)
return 0, newDecodeError(b, "unable to parse float: %w", err)
}
return f, nil
}
func parseIntHex(b []byte) (int64, error) {
tok := string(b)
cleanedVal := cleanupNumberToken(tok)
err := hexNumberContainsInvalidUnderscore(cleanedVal)
cleaned, err := checkAndRemoveUnderscores(b[2:])
if err != nil {
return 0, err
}
i, err := strconv.ParseInt(cleanedVal[2:], 16, 64)
i, err := strconv.ParseInt(string(cleaned), 16, 64)
if err != nil {
return 0, fmt.Errorf("coudn't ParseIntHex %w", err)
return 0, newDecodeError(b, "couldn't parse hexadecimal number: %w", err)
}
return i, nil
}
func parseIntOct(b []byte) (int64, error) {
tok := string(b)
cleanedVal := cleanupNumberToken(tok)
err := numberContainsInvalidUnderscore(cleanedVal)
cleaned, err := checkAndRemoveUnderscores(b[2:])
if err != nil {
return 0, err
}
i, err := strconv.ParseInt(cleanedVal[2:], 8, 64)
i, err := strconv.ParseInt(string(cleaned), 8, 64)
if err != nil {
return 0, fmt.Errorf("coudn't ParseIntOct %w", err)
return 0, newDecodeError(b, "couldn't parse octal number: %w", err)
}
return i, nil
}
func parseIntBin(b []byte) (int64, error) {
tok := string(b)
cleanedVal := cleanupNumberToken(tok)
err := numberContainsInvalidUnderscore(cleanedVal)
cleaned, err := checkAndRemoveUnderscores(b[2:])
if err != nil {
return 0, err
}
i, err := strconv.ParseInt(cleanedVal[2:], 2, 64)
i, err := strconv.ParseInt(string(cleaned), 2, 64)
if err != nil {
return 0, fmt.Errorf("coudn't ParseIntBin %w", err)
return 0, newDecodeError(b, "couldn't parse binary number: %w", err)
}
return i, nil
}
func parseIntDec(b []byte) (int64, error) {
tok := string(b)
cleanedVal := cleanupNumberToken(tok)
err := numberContainsInvalidUnderscore(cleanedVal)
cleaned, err := checkAndRemoveUnderscores(b)
if err != nil {
return 0, err
}
i, err := strconv.ParseInt(cleanedVal, 10, 64)
i, err := strconv.ParseInt(string(cleaned), 10, 64)
if err != nil {
return 0, fmt.Errorf("coudn't parseIntDec %w", err)
return 0, newDecodeError(b, "couldn't parse decimal number: %w", err)
}
return i, nil
}
func numberContainsInvalidUnderscore(value string) error {
// For large numbers, you may use underscores between digits to enhance
// readability. Each underscore must be surrounded by at least one digit on
// each side.
hasBefore := false
for idx, r := range value {
if r == '_' {
if !hasBefore || idx+1 >= len(value) {
// can't end with an underscore
return errInvalidUnderscore
}
}
hasBefore = isDigitRune(r)
func checkAndRemoveUnderscores(b []byte) ([]byte, error) {
if b[0] == '_' {
return nil, newDecodeError(b[0:1], "number cannot start with underscore")
}
return nil
}
func hexNumberContainsInvalidUnderscore(value string) error {
hasBefore := false
for idx, r := range value {
if r == '_' {
if !hasBefore || idx+1 >= len(value) {
// can't end with an underscore
return errInvalidUnderscoreHex
}
}
hasBefore = isHexDigit(r)
if b[len(b)-1] == '_' {
return nil, newDecodeError(b[len(b)-1:], "number cannot end with underscore")
}
return nil
// fast path
i := 0
for ; i < len(b); i++ {
if b[i] == '_' {
break
}
}
if i == len(b) {
return b, nil
}
before := false
cleaned := make([]byte, i, len(b))
copy(cleaned, b)
for i++; i < len(b); i++ {
c := b[i]
if c == '_' {
if !before {
return nil, newDecodeError(b[i-1:i+1], "number must have at least one digit between underscores")
}
before = false
} else {
before = true
cleaned = append(cleaned, c)
}
}
return cleaned, nil
}
func cleanupNumberToken(value string) string {
cleanedVal := strings.ReplaceAll(value, "_", "")
return cleanedVal
}
func isHexDigit(r rune) bool {
return isDigitRune(r) ||
(r >= 'a' && r <= 'f') ||
(r >= 'A' && r <= 'F')
}
func isDigitRune(r rune) bool {
return r >= '0' && r <= '9'
}
var (
errInvalidUnderscore = errors.New("invalid use of _ in number")
errInvalidUnderscoreHex = errors.New("invalid use of _ in hex number")
)
+2
View File
@@ -0,0 +1,2 @@
// Package toml is a library to read and write TOML documents.
package toml
+10 -10
View File
@@ -33,7 +33,7 @@ type StrictMissingError struct {
Errors []DecodeError
}
// Error returns the cannonical string for this error.
// Error returns the canonical string for this error.
func (s *StrictMissingError) Error() string {
return "strict mode: fields in the document are missing in the target struct"
}
@@ -41,12 +41,15 @@ func (s *StrictMissingError) Error() string {
// String returns a human readable description of all errors.
func (s *StrictMissingError) String() string {
var buf strings.Builder
for i, e := range s.Errors {
if i > 0 {
buf.WriteString("\n---\n")
}
buf.WriteString(e.String())
}
return buf.String()
}
@@ -67,13 +70,13 @@ func (de *decodeError) Error() string {
func newDecodeError(highlight []byte, format string, args ...interface{}) error {
return &decodeError{
highlight: highlight,
message: fmt.Sprintf(format, args...),
message: fmt.Errorf(format, args...).Error(),
}
}
// Error returns the error message contained in the DecodeError.
func (e *DecodeError) Error() string {
return e.message
return "toml: " + e.message
}
// String returns the human-readable contextualized error. This string is multi-line.
@@ -87,12 +90,13 @@ func (e *DecodeError) Position() (row int, column int) {
return e.line, e.column
}
// Key that was being processed when the error occured.
// Key that was being processed when the error occurred. The key is present only
// if this DecodeError is part of a StrictMissingError.
func (e *DecodeError) Key() Key {
return e.key
}
// decodeErrorFromHighlight creates a DecodeError referencing to a highlighted
// decodeErrorFromHighlight creates a DecodeError referencing a highlighted
// range of bytes from document.
//
// highlight needs to be a sub-slice of document, or this function panics.
@@ -101,13 +105,9 @@ func (e *DecodeError) Key() Key {
// highlight can be freely deallocated.
//nolint:funlen
func wrapDecodeError(document []byte, de *decodeError) *DecodeError {
if de == nil {
return nil
}
offset := unsafe.SubsliceOffset(document, de.highlight)
errMessage := de.message
errMessage := de.Error()
errLine, errColumn := positionAtEnd(document[:offset])
before, after := linesOfContext(document, de.highlight, offset, 3)
+40
View File
@@ -3,6 +3,7 @@ package toml
import (
"bytes"
"errors"
"fmt"
"strings"
"testing"
@@ -179,3 +180,42 @@ line 5`,
})
}
}
func TestDecodeError_Accessors(t *testing.T) {
t.Parallel()
e := DecodeError{
message: "foo",
line: 1,
column: 2,
key: []string{"one", "two"},
human: "bar",
}
assert.Equal(t, "toml: foo", e.Error())
r, c := e.Position()
assert.Equal(t, 1, r)
assert.Equal(t, 2, c)
assert.Equal(t, Key{"one", "two"}, e.Key())
assert.Equal(t, "bar", e.String())
}
func ExampleDecodeError() {
doc := `name = 123__456`
s := map[string]interface{}{}
err := Unmarshal([]byte(doc), &s)
fmt.Println(err)
//nolint:errorlint
de := err.(*DecodeError)
fmt.Println(de.String())
row, col := de.Position()
fmt.Println("error occurred at row", row, "column", col)
// Output:
// toml: number must have at least one digit between underscores
// 1| name = 123__456
// | ~~ number must have at least one digit between underscores
// error occurred at row 1 column 11
}
@@ -4,6 +4,7 @@ package imported_tests
// defaults of v2.
import (
"fmt"
"testing"
"time"
@@ -164,3 +165,34 @@ stringlist = []
require.Equal(t, string(expected), string(result))
}
type textMarshaler struct {
FirstName string
LastName string
}
func (m textMarshaler) MarshalText() ([]byte, error) {
fullName := fmt.Sprintf("%s %s", m.FirstName, m.LastName)
return []byte(fullName), nil
}
func TestTextMarshaler(t *testing.T) {
type wrap struct {
TM textMarshaler
}
m := textMarshaler{FirstName: "Sally", LastName: "Fields"}
t.Run("at root", func(t *testing.T) {
_, err := toml.Marshal(m)
// in v2 we do not allow TextMarshaler at root
require.Error(t, err)
})
t.Run("leaf", func(t *testing.T) {
res, err := toml.Marshal(wrap{m})
require.NoError(t, err)
require.Equal(t, "TM = 'Sally Fields'\n", string(res))
})
}
+204 -215
View File
@@ -612,16 +612,6 @@ func (x *IntOrString) MarshalTOML() ([]byte, error) {
return []byte(s), nil
}
type textMarshaler struct {
FirstName string
LastName string
}
func (m textMarshaler) MarshalText() ([]byte, error) {
fullName := fmt.Sprintf("%s %s", m.FirstName, m.LastName)
return []byte(fullName), nil
}
func TestUnmarshalTextMarshaler(t *testing.T) {
var nested = struct {
Friends textMarshaler `toml:"friends"`
@@ -1430,211 +1420,210 @@ func TestUnmarshalPreservesUnexportedFields(t *testing.T) {
})
}
//
//func TestUnmarshalLocalDate(t *testing.T) {
// t.Run("ToLocalDate", func(t *testing.T) {
// type dateStruct struct {
// Date toml.LocalDate
// }
//
// doc := `date = 1979-05-27`
//
// var obj dateStruct
//
// err := toml.Unmarshal([]byte(doc), &obj)
//
// if err != nil {
// t.Fatal(err)
// }
//
// if obj.Date.Year != 1979 {
// t.Errorf("expected year 1979, got %d", obj.Date.Year)
// }
// if obj.Date.Month != 5 {
// t.Errorf("expected month 5, got %d", obj.Date.Month)
// }
// if obj.Date.Day != 27 {
// t.Errorf("expected day 27, got %d", obj.Date.Day)
// }
// })
//
// t.Run("ToLocalDate", func(t *testing.T) {
// type dateStruct struct {
// Date time.Time
// }
//
// doc := `date = 1979-05-27`
//
// var obj dateStruct
//
// err := toml.Unmarshal([]byte(doc), &obj)
//
// if err != nil {
// t.Fatal(err)
// }
//
// if obj.Date.Year() != 1979 {
// t.Errorf("expected year 1979, got %d", obj.Date.Year())
// }
// if obj.Date.Month() != 5 {
// t.Errorf("expected month 5, got %d", obj.Date.Month())
// }
// if obj.Date.Day() != 27 {
// t.Errorf("expected day 27, got %d", obj.Date.Day())
// }
// })
//}
//
//func TestUnmarshalLocalDateTime(t *testing.T) {
// examples := []struct {
// name string
// in string
// out toml.LocalDateTime
// }{
// {
// name: "normal",
// in: "1979-05-27T07:32:00",
// out: toml.LocalDateTime{
// Date: toml.LocalDate{
// Year: 1979,
// Month: 5,
// Day: 27,
// },
// Time: toml.LocalTime{
// Hour: 7,
// Minute: 32,
// Second: 0,
// Nanosecond: 0,
// },
// }},
// {
// name: "with nanoseconds",
// in: "1979-05-27T00:32:00.999999",
// out: toml.LocalDateTime{
// Date: toml.LocalDate{
// Year: 1979,
// Month: 5,
// Day: 27,
// },
// Time: toml.LocalTime{
// Hour: 0,
// Minute: 32,
// Second: 0,
// Nanosecond: 999999000,
// },
// },
// },
// }
//
// for i, example := range examples {
// doc := fmt.Sprintf(`date = %s`, example.in)
//
// t.Run(fmt.Sprintf("ToLocalDateTime_%d_%s", i, example.name), func(t *testing.T) {
// type dateStruct struct {
// Date toml.LocalDateTime
// }
//
// var obj dateStruct
//
// err := toml.Unmarshal([]byte(doc), &obj)
//
// if err != nil {
// t.Fatal(err)
// }
//
// if obj.Date != example.out {
// t.Errorf("expected '%s', got '%s'", example.out, obj.Date)
// }
// })
//
// t.Run(fmt.Sprintf("ToTime_%d_%s", i, example.name), func(t *testing.T) {
// type dateStruct struct {
// Date time.Time
// }
//
// var obj dateStruct
//
// err := toml.Unmarshal([]byte(doc), &obj)
//
// if err != nil {
// t.Fatal(err)
// }
//
// if obj.Date.Year() != example.out.Date.Year {
// t.Errorf("expected year %d, got %d", example.out.Date.Year, obj.Date.Year())
// }
// if obj.Date.Month() != example.out.Date.Month {
// t.Errorf("expected month %d, got %d", example.out.Date.Month, obj.Date.Month())
// }
// if obj.Date.Day() != example.out.Date.Day {
// t.Errorf("expected day %d, got %d", example.out.Date.Day, obj.Date.Day())
// }
// if obj.Date.Hour() != example.out.Time.Hour {
// t.Errorf("expected hour %d, got %d", example.out.Time.Hour, obj.Date.Hour())
// }
// if obj.Date.Minute() != example.out.Time.Minute {
// t.Errorf("expected minute %d, got %d", example.out.Time.Minute, obj.Date.Minute())
// }
// if obj.Date.Second() != example.out.Time.Second {
// t.Errorf("expected second %d, got %d", example.out.Time.Second, obj.Date.Second())
// }
// if obj.Date.Nanosecond() != example.out.Time.Nanosecond {
// t.Errorf("expected nanoseconds %d, got %d", example.out.Time.Nanosecond, obj.Date.Nanosecond())
// }
// })
// }
//}
//
//func TestUnmarshalLocalTime(t *testing.T) {
// examples := []struct {
// name string
// in string
// out toml.LocalTime
// }{
// {
// name: "normal",
// in: "07:32:00",
// out: toml.LocalTime{
// Hour: 7,
// Minute: 32,
// Second: 0,
// Nanosecond: 0,
// },
// },
// {
// name: "with nanoseconds",
// in: "00:32:00.999999",
// out: toml.LocalTime{
// Hour: 0,
// Minute: 32,
// Second: 0,
// Nanosecond: 999999000,
// },
// },
// }
//
// for i, example := range examples {
// doc := fmt.Sprintf(`Time = %s`, example.in)
//
// t.Run(fmt.Sprintf("ToLocalTime_%d_%s", i, example.name), func(t *testing.T) {
// type dateStruct struct {
// Time toml.LocalTime
// }
//
// var obj dateStruct
//
// err := toml.Unmarshal([]byte(doc), &obj)
//
// if err != nil {
// t.Fatal(err)
// }
//
// if obj.Time != example.out {
// t.Errorf("expected '%s', got '%s'", example.out, obj.Time)
// }
// })
// }
//}
func TestUnmarshalLocalDate(t *testing.T) {
t.Run("ToLocalDate", func(t *testing.T) {
type dateStruct struct {
Date toml.LocalDate
}
doc := `date = 1979-05-27`
var obj dateStruct
err := toml.Unmarshal([]byte(doc), &obj)
if err != nil {
t.Fatal(err)
}
if obj.Date.Year != 1979 {
t.Errorf("expected year 1979, got %d", obj.Date.Year)
}
if obj.Date.Month != 5 {
t.Errorf("expected month 5, got %d", obj.Date.Month)
}
if obj.Date.Day != 27 {
t.Errorf("expected day 27, got %d", obj.Date.Day)
}
})
t.Run("ToLocalDate", func(t *testing.T) {
type dateStruct struct {
Date time.Time
}
doc := `date = 1979-05-27`
var obj dateStruct
err := toml.Unmarshal([]byte(doc), &obj)
if err != nil {
t.Fatal(err)
}
if obj.Date.Year() != 1979 {
t.Errorf("expected year 1979, got %d", obj.Date.Year())
}
if obj.Date.Month() != 5 {
t.Errorf("expected month 5, got %d", obj.Date.Month())
}
if obj.Date.Day() != 27 {
t.Errorf("expected day 27, got %d", obj.Date.Day())
}
})
}
func TestUnmarshalLocalDateTime(t *testing.T) {
examples := []struct {
name string
in string
out toml.LocalDateTime
}{
{
name: "normal",
in: "1979-05-27T07:32:00",
out: toml.LocalDateTime{
Date: toml.LocalDate{
Year: 1979,
Month: 5,
Day: 27,
},
Time: toml.LocalTime{
Hour: 7,
Minute: 32,
Second: 0,
Nanosecond: 0,
},
}},
{
name: "with nanoseconds",
in: "1979-05-27T00:32:00.999999",
out: toml.LocalDateTime{
Date: toml.LocalDate{
Year: 1979,
Month: 5,
Day: 27,
},
Time: toml.LocalTime{
Hour: 0,
Minute: 32,
Second: 0,
Nanosecond: 999999000,
},
},
},
}
for i, example := range examples {
doc := fmt.Sprintf(`date = %s`, example.in)
t.Run(fmt.Sprintf("ToLocalDateTime_%d_%s", i, example.name), func(t *testing.T) {
type dateStruct struct {
Date toml.LocalDateTime
}
var obj dateStruct
err := toml.Unmarshal([]byte(doc), &obj)
if err != nil {
t.Fatal(err)
}
if obj.Date != example.out {
t.Errorf("expected '%s', got '%s'", example.out, obj.Date)
}
})
t.Run(fmt.Sprintf("ToTime_%d_%s", i, example.name), func(t *testing.T) {
type dateStruct struct {
Date time.Time
}
var obj dateStruct
err := toml.Unmarshal([]byte(doc), &obj)
if err != nil {
t.Fatal(err)
}
if obj.Date.Year() != example.out.Date.Year {
t.Errorf("expected year %d, got %d", example.out.Date.Year, obj.Date.Year())
}
if obj.Date.Month() != example.out.Date.Month {
t.Errorf("expected month %d, got %d", example.out.Date.Month, obj.Date.Month())
}
if obj.Date.Day() != example.out.Date.Day {
t.Errorf("expected day %d, got %d", example.out.Date.Day, obj.Date.Day())
}
if obj.Date.Hour() != example.out.Time.Hour {
t.Errorf("expected hour %d, got %d", example.out.Time.Hour, obj.Date.Hour())
}
if obj.Date.Minute() != example.out.Time.Minute {
t.Errorf("expected minute %d, got %d", example.out.Time.Minute, obj.Date.Minute())
}
if obj.Date.Second() != example.out.Time.Second {
t.Errorf("expected second %d, got %d", example.out.Time.Second, obj.Date.Second())
}
if obj.Date.Nanosecond() != example.out.Time.Nanosecond {
t.Errorf("expected nanoseconds %d, got %d", example.out.Time.Nanosecond, obj.Date.Nanosecond())
}
})
}
}
func TestUnmarshalLocalTime(t *testing.T) {
examples := []struct {
name string
in string
out toml.LocalTime
}{
{
name: "normal",
in: "07:32:00",
out: toml.LocalTime{
Hour: 7,
Minute: 32,
Second: 0,
Nanosecond: 0,
},
},
{
name: "with nanoseconds",
in: "00:32:00.999999",
out: toml.LocalTime{
Hour: 0,
Minute: 32,
Second: 0,
Nanosecond: 999999000,
},
},
}
for i, example := range examples {
doc := fmt.Sprintf(`Time = %s`, example.in)
t.Run(fmt.Sprintf("ToLocalTime_%d_%s", i, example.name), func(t *testing.T) {
type dateStruct struct {
Time toml.LocalTime
}
var obj dateStruct
err := toml.Unmarshal([]byte(doc), &obj)
if err != nil {
t.Fatal(err)
}
if obj.Time != example.out {
t.Errorf("expected '%s', got '%s'", example.out, obj.Time)
}
})
}
}
// test case for issue #339
func TestUnmarshalSameInnerField(t *testing.T) {
+4 -4
View File
@@ -123,10 +123,10 @@ func (s *SeenTracker) checkTable(node ast.Node) error {
i, found := s.current.Has(k)
if found {
if i.kind != tableKind {
return fmt.Errorf("key %s should be a table", k)
return fmt.Errorf("toml: key %s should be a table, not a %s", k, i.kind)
}
if i.explicit {
return fmt.Errorf("table %s already exists", k)
return fmt.Errorf("toml: table %s already exists", k)
}
i.explicit = true
s.current = i
@@ -162,7 +162,7 @@ func (s *SeenTracker) checkArrayTable(node ast.Node) error {
info, found := s.current.Has(k)
if found {
if info.kind != arrayTableKind {
return fmt.Errorf("key %s already exists but is not an array table", k)
return fmt.Errorf("toml: key %s already exists as a %s, but should be an array table", info.kind, k)
}
info.Clear()
} else {
@@ -182,7 +182,7 @@ func (s *SeenTracker) checkKeyValue(context *info, node ast.Node) error {
child, found := context.Has(k)
if found {
if child.kind != tableKind {
return fmt.Errorf("expected %s to be a table, not a %s", k, child.kind)
return fmt.Errorf("toml: expected %s to be a table, not a %s", k, child.kind)
}
} else {
child = context.CreateTable(k, false)
+3 -3
View File
@@ -53,7 +53,7 @@ func LocalDateOf(t time.Time) LocalDate {
func ParseLocalDate(s string) (LocalDate, error) {
t, err := time.Parse("2006-01-02", s)
if err != nil {
return LocalDate{}, fmt.Errorf("ParseLocalDate: %w", err)
return LocalDate{}, err
}
return LocalDateOf(t), nil
@@ -166,7 +166,7 @@ func LocalTimeOf(t time.Time) LocalTime {
func ParseLocalTime(s string) (LocalTime, error) {
t, err := time.Parse("15:04:05.999999999", s)
if err != nil {
return LocalTime{}, fmt.Errorf("ParseLocalTime: %w", err)
return LocalTime{}, err
}
return LocalTimeOf(t), nil
@@ -237,7 +237,7 @@ func ParseLocalDateTime(s string) (LocalDateTime, error) {
if err != nil {
t, err = time.Parse("2006-01-02t15:04:05.999999999", s)
if err != nil {
return LocalDateTime{}, fmt.Errorf("ParseLocalDateTime: %w", err)
return LocalDateTime{}, err
}
}
+29 -11
View File
@@ -56,6 +56,7 @@ func TestDates(t *testing.T) {
if got := test.date.String(); got != test.wantStr {
t.Errorf("%#v.String() = %q, want %q", test.date, got, test.wantStr)
}
if got := test.date.In(test.loc); !got.Equal(test.wantTime) {
t.Errorf("%#v.In(%v) = %v, want %v", test.date, test.loc, got, test.wantTime)
}
@@ -109,6 +110,7 @@ func TestParseDate(t *testing.T) {
if got != test.want {
t.Errorf("ParseLocalDate(%q) = %+v, want %+v", test.str, got, test.want)
}
if err != nil && test.want != (emptyDate) {
t.Errorf("Unexpected error %v from ParseLocalDate(%q)", err, test.str)
}
@@ -170,6 +172,7 @@ func TestDateArithmetic(t *testing.T) {
if got := test.start.AddDays(test.days); got != test.end {
t.Errorf("[%s] %#v.AddDays(%v) = %#v, want %#v", test.desc, test.start, test.days, got, test.end)
}
if got := test.end.DaysSince(test.start); got != test.days {
t.Errorf("[%s] %#v.Sub(%#v) = %v, want %v", test.desc, test.end, test.start, got, test.days)
}
@@ -231,9 +234,11 @@ func TestTimeToString(t *testing.T) {
continue
}
if gotTime != test.time {
t.Errorf("ParseLocalTime(%q) = %+v, want %+v", test.str, gotTime, test.time)
}
if test.roundTrip {
gotStr := test.time.String()
if gotStr != test.str {
@@ -293,9 +298,9 @@ func TestDateTimeToString(t *testing.T) {
dateTime LocalDateTime
roundTrip bool // ParseLocalDateTime(str).String() == str?
}{
{"2016-03-22T13:26:33", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 0}}, true},
{"2016-03-22T13:26:33.000000600", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 600}}, true},
{"2016-03-22t13:26:33", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 0}}, false},
{"2016-03-22T13:26:33", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 0}}, true},
{"2016-03-22T13:26:33.000000600", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 600}}, true},
{"2016-03-22t13:26:33", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 0}}, false},
} {
gotDateTime, err := ParseLocalDateTime(test.str)
if err != nil {
@@ -303,9 +308,11 @@ func TestDateTimeToString(t *testing.T) {
continue
}
if gotDateTime != test.dateTime {
t.Errorf("ParseLocalDateTime(%q) = %+v, want %+v", test.str, gotDateTime, test.dateTime)
}
if test.roundTrip {
gotStr := test.dateTime.String()
if gotStr != test.str {
@@ -338,10 +345,14 @@ func TestDateTimeOf(t *testing.T) {
time time.Time
want LocalDateTime
}{
{time.Date(2014, 8, 20, 15, 8, 43, 1, time.Local),
LocalDateTime{LocalDate{2014, 8, 20}, LocalTime{15, 8, 43, 1}}},
{time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC),
LocalDateTime{LocalDate{1, 1, 1}, LocalTime{0, 0, 0, 0}}},
{
time.Date(2014, 8, 20, 15, 8, 43, 1, time.Local),
LocalDateTime{LocalDate{2014, 8, 20}, LocalTime{15, 8, 43, 1}},
},
{
time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC),
LocalDateTime{LocalDate{1, 1, 1}, LocalTime{0, 0, 0, 0}},
},
} {
if got := LocalDateTimeOf(test.time); got != test.want {
t.Errorf("LocalDateTimeOf(%v) = %+v, want %+v", test.time, got, test.want)
@@ -440,6 +451,7 @@ func TestMarshalJSON(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if got := string(bgot); got != test.want {
t.Errorf("%#v: got %s, want %s", test.value, got, test.want)
}
@@ -449,9 +461,11 @@ func TestMarshalJSON(t *testing.T) {
func TestUnmarshalJSON(t *testing.T) {
t.Parallel()
var d LocalDate
var tm LocalTime
var dt LocalDateTime
var (
d LocalDate
tm LocalTime
dt LocalDateTime
)
for _, test := range []struct {
data string
@@ -466,12 +480,14 @@ func TestUnmarshalJSON(t *testing.T) {
if err := json.Unmarshal([]byte(test.data), test.ptr); err != nil {
t.Fatalf("%s: %v", test.data, err)
}
if !cmpEqual(test.ptr, test.want) {
t.Errorf("%s: got %#v, want %#v", test.data, test.ptr, test.want)
}
}
for _, bad := range []string{"", `""`, `"bad"`, `"1987-04-15x"`,
for _, bad := range []string{
"", `""`, `"bad"`, `"1987-04-15x"`,
`19870415`, // a JSON number
`11987-04-15x`, // not a JSON string
@@ -479,9 +495,11 @@ func TestUnmarshalJSON(t *testing.T) {
if json.Unmarshal([]byte(bad), &d) == nil {
t.Errorf("%q, LocalDate: got nil, want error", bad)
}
if json.Unmarshal([]byte(bad), &tm) == nil {
t.Errorf("%q, LocalTime: got nil, want error", bad)
}
if json.Unmarshal([]byte(bad), &dt) == nil {
t.Errorf("%q, LocalDateTime: got nil, want error", bad)
}
+235 -162
View File
@@ -2,7 +2,7 @@ package toml
import (
"bytes"
"errors"
"encoding"
"fmt"
"io"
"reflect"
@@ -29,7 +29,123 @@ func Marshal(v interface{}) ([]byte, error) {
// Encoder writes a TOML document to an output stream.
type Encoder struct {
// output
w io.Writer
// global settings
tablesInline bool
arraysMultiline bool
indentSymbol string
indentTables bool
}
// NewEncoder returns a new Encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: w,
indentSymbol: " ",
}
}
// SetTablesInline forces the encoder to emit all tables inline.
//
// This behavior can be controlled on an individual struct field basis with the
// inline tag:
//
// MyField `inline:"true"`
func (enc *Encoder) SetTablesInline(inline bool) {
enc.tablesInline = inline
}
// SetArraysMultiline forces the encoder to emit all arrays with one element per
// line.
//
// This behavior can be controlled on an individual struct field basis with the multiline tag:
//
// MyField `multiline:"true"`
func (enc *Encoder) SetArraysMultiline(multiline bool) {
enc.arraysMultiline = multiline
}
// SetIndentSymbol defines the string that should be used for indentation. The
// provided string is repeated for each indentation level. Defaults to two
// spaces.
func (enc *Encoder) SetIndentSymbol(s string) {
enc.indentSymbol = s
}
// SetIndentTables forces the encoder to intent tables and array tables.
func (enc *Encoder) SetIndentTables(indent bool) {
enc.indentTables = indent
}
// Encode writes a TOML representation of v to the stream.
//
// If v cannot be represented to TOML it returns an error.
//
// Encoding rules
//
// A top level slice containing only maps or structs is encoded as [[table
// array]].
//
// All slices not matching rule 1 are encoded as [array]. As a result, any map
// or struct they contain is encoded as an {inline table}.
//
// Nil interfaces and nil pointers are not supported.
//
// Keys in key-values always have one part.
//
// Intermediate tables are always printed.
//
// By default, strings are encoded as literal string, unless they contain either
// a newline character or a single quote. In that case they are emitted as quoted
// strings.
//
// When encoding structs, fields are encoded in order of definition, with their
// exact name.
//
// Struct tags
//
// The following struct tags are available to tweak encoding on a per-field
// basis:
//
// toml:"foo"
// Changes the name of the key to use for the field to foo.
//
// multiline:"true"
// When the field contains a string, it will be emitted as a quoted
// multi-line TOML string.
//
// inline:"true"
// When the field would normally be encoded as a table, it is instead
// encoded as an inline table.
func (enc *Encoder) Encode(v interface{}) error {
var (
b []byte
ctx encoderCtx
)
ctx.inline = enc.tablesInline
if v == nil {
return fmt.Errorf("toml: cannot encode a nil interface")
}
b, err := enc.encode(b, ctx, reflect.ValueOf(v))
if err != nil {
return err
}
_, err = enc.w.Write(b)
if err != nil {
return fmt.Errorf("toml: cannot write: %w", err)
}
return nil
}
type valueOptions struct {
multiline bool
}
type encoderCtx struct {
@@ -48,11 +164,14 @@ type encoderCtx struct {
// Set to true to skip the first table header in an array table.
skipTableHeader bool
options valueOptions
}
// Should the next table be encoded as inline
inline bool
type valueOptions struct {
multiline bool
// Indentation level
indent int
// Options coming from struct tags
options valueOptions
}
func (ctx *encoderCtx) shiftKey() {
@@ -72,69 +191,30 @@ func (ctx *encoderCtx) clearKey() {
ctx.hasKey = false
}
// NewEncoder returns a new Encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: w,
}
func (ctx *encoderCtx) isRoot() bool {
return len(ctx.parentKey) == 0 && !ctx.hasKey
}
// Encode writes a TOML representation of v to the stream.
//
// If v cannot be represented to TOML it returns an error.
//
// Encoding rules:
//
// 1. A top level slice containing only maps or structs is encoded as [[table
// array]].
//
// 2. All slices not matching rule 1 are encoded as [array]. As a result, any
// map or struct they contain is encoded as an {inline table}.
//
// 3. Nil interfaces and nil pointers are not supported.
//
// 4. Keys in key-values always have one part.
//
// 5. Intermediate tables are always printed.
//
// By default, strings are encoded as literal string, unless they contain either
// a newline character or a single quote. In that case they are emitted as quoted
// strings.
//
// When encoding structs, fields are encoded in order of definition, with their
// exact name. The following struct tags are available:
//
// `toml:"foo"`: changes the name of the key to use for the field to foo.
//
// `multiline:"true"`: when the field contains a string, it will be emitted as
// a quoted multi-line TOML string.
func (enc *Encoder) Encode(v interface{}) error {
var (
b []byte
ctx encoderCtx
)
b, err := enc.encode(b, ctx, reflect.ValueOf(v))
if err != nil {
return fmt.Errorf("Encode: %w", err)
}
_, err = enc.w.Write(b)
if err != nil {
return fmt.Errorf("Encode: %w", err)
}
return nil
}
var errUnsupportedValue = errors.New("unsupported encode value kind")
//nolint:cyclop
//nolint:cyclop,funlen
func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
//nolint:gocritic,godox
switch i := v.Interface().(type) {
case time.Time: // TODO: add TextMarshaler
b = i.AppendFormat(b, time.RFC3339)
if !v.IsZero() {
i, ok := v.Interface().(time.Time)
if ok {
return i.AppendFormat(b, time.RFC3339), nil
}
}
if v.Type().Implements(textMarshalerType) {
if ctx.isRoot() {
return nil, fmt.Errorf("toml: type %s implementing the TextMarshaler interface cannot be a root element", v.Type())
}
text, err := v.Interface().(encoding.TextMarshaler).MarshalText()
if err != nil {
return nil, err
}
b = enc.encodeString(b, string(text), ctx.options)
return b, nil
}
@@ -149,7 +229,7 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
return enc.encodeSlice(b, ctx, v)
case reflect.Interface:
if v.IsNil() {
return nil, errNilInterface
return nil, fmt.Errorf("toml: encoding a nil interface is not supported")
}
return enc.encode(b, ctx, v.Elem())
@@ -178,7 +258,7 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int:
b = strconv.AppendInt(b, v.Int(), 10)
default:
return nil, fmt.Errorf("encode(type %s): %w", v.Kind(), errUnsupportedValue)
return nil, fmt.Errorf("toml: cannot encode value of type %s", v.Kind())
}
return b, nil
@@ -199,10 +279,7 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
if !ctx.hasKey {
panic("caller of encodeKv should have set the key in the context")
}
if isNil(v) {
return b, nil
}
b = enc.indent(ctx.indent, b)
b, err = enc.encodeKey(b, ctx.key)
if err != nil {
@@ -314,21 +391,23 @@ func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte {
return append(b, v...)
}
func (enc *Encoder) encodeTableHeader(b []byte, key []string) ([]byte, error) {
if len(key) == 0 {
func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error) {
if len(ctx.parentKey) == 0 {
return b, nil
}
b = enc.indent(ctx.indent, b)
b = append(b, '[')
var err error
b, err = enc.encodeKey(b, key[0])
b, err = enc.encodeKey(b, ctx.parentKey[0])
if err != nil {
return nil, err
}
for _, k := range key[1:] {
for _, k := range ctx.parentKey[1:] {
b = append(b, '.')
b, err = enc.encodeKey(b, k)
@@ -342,8 +421,6 @@ func (enc *Encoder) encodeTableHeader(b []byte, key []string) ([]byte, error) {
return b, nil
}
var errTomlNoMultiline = errors.New("TOML does not support multiline keys")
//nolint:cyclop
func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
needsQuotation := false
@@ -355,7 +432,7 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
}
if c == '\n' {
return nil, errTomlNoMultiline
return nil, fmt.Errorf("toml: new line characters in keys are not supported")
}
if c == literalQuote {
@@ -375,11 +452,9 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
}
}
var errNotSupportedAsMapKey = errors.New("type not supported as map key")
func (enc *Encoder) encodeMap(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
if v.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("encodeMap '%s': %w", v.Type().Key().Kind(), errNotSupportedAsMapKey)
return nil, fmt.Errorf("toml: type %s is not supported as a map key", v.Type().Key().Kind())
}
var (
@@ -396,12 +471,7 @@ func (enc *Encoder) encodeMap(b []byte, ctx encoderCtx, v reflect.Value) ([]byte
continue
}
table, err := willConvertToTableOrArrayTable(v)
if err != nil {
return nil, err
}
if table {
if willConvertToTableOrArrayTable(ctx, v) {
t.pushTable(k, v, emptyValueOptions)
} else {
t.pushKV(k, v, emptyValueOptions)
@@ -469,42 +539,47 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
continue
}
willConvert, err := willConvertToTableOrArrayTable(f)
if err != nil {
return nil, err
options := valueOptions{
multiline: fieldBoolTag(fieldType, "multiline"),
}
var options valueOptions
inline := fieldBoolTag(fieldType, "inline")
ml, ok := fieldType.Tag.Lookup("multiline")
if ok {
options.multiline = ml == "true"
}
if willConvert {
t.pushTable(k, f, options)
} else {
if inline || !willConvertToTableOrArrayTable(ctx, f) {
t.pushKV(k, f, options)
} else {
t.pushTable(k, f, options)
}
}
return enc.encodeTable(b, ctx, t)
}
func fieldBoolTag(field reflect.StructField, tag string) bool {
x, ok := field.Tag.Lookup(tag)
return ok && x == "true"
}
//nolint:cyclop
func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, error) {
var err error
ctx.shiftKey()
if ctx.insideKv {
return enc.encodeTableInsideKV(b, ctx, t)
if ctx.insideKv || (ctx.inline && !ctx.isRoot()) {
return enc.encodeTableInline(b, ctx, t)
}
if !ctx.skipTableHeader {
b, err = enc.encodeTableHeader(b, ctx.parentKey)
b, err = enc.encodeTableHeader(ctx, b)
if err != nil {
return nil, err
}
if enc.indentTables && len(ctx.parentKey) > 0 {
ctx.indent++
}
}
ctx.skipTableHeader = false
@@ -522,6 +597,8 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
for _, table := range t.tables {
ctx.setKey(table.Key)
ctx.options = table.Options
b, err = enc.encode(b, ctx, table.Value)
if err != nil {
return nil, err
@@ -533,7 +610,7 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
return b, nil
}
func (enc *Encoder) encodeTableInsideKV(b []byte, ctx encoderCtx, t table) ([]byte, error) {
func (enc *Encoder) encodeTableInline(b []byte, ctx encoderCtx, t table) ([]byte, error) {
var err error
b = append(b, '{')
@@ -554,90 +631,64 @@ func (enc *Encoder) encodeTableInsideKV(b []byte, ctx encoderCtx, t table) ([]by
}
}
for _, table := range t.tables {
if first {
first = false
} else {
b = append(b, `, `...)
}
ctx.setKey(table.Key)
b, err = enc.encode(b, ctx, table.Value)
if err != nil {
return nil, err
}
b = append(b, '\n')
if len(t.tables) > 0 {
panic("inline table cannot contain nested tables, online key-values")
}
b = append(b, "}\n"...)
b = append(b, "}"...)
return b, nil
}
var errNilInterface = errors.New("nil interface not supported")
var textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem()
func willConvertToTable(v reflect.Value) (bool, error) {
//nolint:gocritic,godox
switch v.Interface().(type) {
case time.Time: // TODO: add TextMarshaler
return false, nil
func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
if v.Type() == timeType || v.Type().Implements(textMarshalerType) {
return false
}
t := v.Type()
switch t.Kind() {
case reflect.Map, reflect.Struct:
return true, nil
return !ctx.inline
case reflect.Interface:
if v.IsNil() {
return false, errNilInterface
}
return willConvertToTable(v.Elem())
return willConvertToTable(ctx, v.Elem())
case reflect.Ptr:
if v.IsNil() {
return false, nil
return false
}
return willConvertToTable(v.Elem())
return willConvertToTable(ctx, v.Elem())
default:
return false, nil
return false
}
}
func willConvertToTableOrArrayTable(v reflect.Value) (bool, error) {
func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool {
t := v.Type()
if t.Kind() == reflect.Interface {
if v.IsNil() {
return false, errNilInterface
}
return willConvertToTableOrArrayTable(v.Elem())
return willConvertToTableOrArrayTable(ctx, v.Elem())
}
if t.Kind() == reflect.Slice {
if v.Len() == 0 {
// An empty slice should be a kv = [].
return false, nil
return false
}
for i := 0; i < v.Len(); i++ {
t, err := willConvertToTable(v.Index(i))
if err != nil {
return false, err
}
t := willConvertToTable(ctx, v.Index(i))
if !t {
return false, nil
return false
}
}
return true, nil
return true
}
return willConvertToTable(v)
return willConvertToTable(ctx, v)
}
func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
@@ -647,12 +698,7 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by
return b, nil
}
allTables, err := willConvertToTableOrArrayTable(v)
if err != nil {
return nil, err
}
if allTables {
if willConvertToTableOrArrayTable(ctx, v) {
return enc.encodeSliceAsArrayTable(b, ctx, v)
}
@@ -662,10 +708,6 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by
// caller should have checked that v is a slice that only contains values that
// encode into tables.
func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
if v.Len() == 0 {
return b, nil
}
ctx.shiftKey()
var err error
@@ -699,25 +741,56 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
}
func (enc *Encoder) encodeSliceAsArray(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
multiline := ctx.options.multiline || enc.arraysMultiline
separator := ", "
b = append(b, '[')
subCtx := ctx
subCtx.options = valueOptions{}
if multiline {
separator = ",\n"
b = append(b, '\n')
subCtx.indent++
}
var err error
first := true
for i := 0; i < v.Len(); i++ {
if !first {
b = append(b, ", "...)
if first {
first = false
} else {
b = append(b, separator...)
}
first = false
if multiline {
b = enc.indent(subCtx.indent, b)
}
b, err = enc.encode(b, ctx, v.Index(i))
b, err = enc.encode(b, subCtx, v.Index(i))
if err != nil {
return nil, err
}
}
if multiline {
b = append(b, '\n')
b = enc.indent(ctx.indent, b)
}
b = append(b, ']')
return b, nil
}
func (enc *Encoder) indent(level int, b []byte) []byte {
for i := 0; i < level; i++ {
b = append(b, enc.indentSymbol...)
}
return b
}
+542 -6
View File
@@ -3,6 +3,7 @@ package toml_test
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"testing"
@@ -15,6 +16,12 @@ import (
func TestMarshal(t *testing.T) {
t.Parallel()
someInt := 42
type structInline struct {
A interface{} `inline:"true"`
}
examples := []struct {
desc string
v interface{}
@@ -68,9 +75,6 @@ hello = 'world'`,
a = 'test'`,
},
{
//nolint:godox
// TODO: this test is flaky because output changes depending on
// the map iteration order.
desc: "map in map in map and string with values",
v: map[string]interface{}{
"this": map[string]interface{}{
@@ -248,6 +252,265 @@ name = 'Alice'
hello
world"""`,
},
{
desc: "inline field",
v: struct {
A map[string]string `inline:"true"`
B map[string]string
}{
A: map[string]string{
"isinline": "yes",
},
B: map[string]string{
"isinline": "no",
},
},
expected: `
A = {isinline = 'yes'}
[B]
isinline = 'no'
`,
},
{
desc: "mutiline array int",
v: struct {
A []int `multiline:"true"`
B []int
}{
A: []int{1, 2, 3, 4},
B: []int{1, 2, 3, 4},
},
expected: `
A = [
1,
2,
3,
4
]
B = [1, 2, 3, 4]
`,
},
{
desc: "mutiline array in array",
v: struct {
A [][]int `multiline:"true"`
}{
A: [][]int{{1, 2}, {3, 4}},
},
expected: `
A = [
[1, 2],
[3, 4]
]
`,
},
{
desc: "nil interface not supported at root",
v: nil,
err: true,
},
{
desc: "nil interface not supported in slice",
v: map[string]interface{}{
"a": []interface{}{"a", nil, 2},
},
err: true,
},
{
desc: "nil pointer in slice uses zero value",
v: struct {
A []*int
}{
A: []*int{nil},
},
expected: `A = [0]`,
},
{
desc: "nil pointer in slice uses zero value",
v: struct {
A []*int
}{
A: []*int{nil},
},
expected: `A = [0]`,
},
{
desc: "pointer in slice",
v: struct {
A []*int
}{
A: []*int{&someInt},
},
expected: `A = [42]`,
},
{
desc: "inline table in inline table",
v: structInline{
A: structInline{
A: structInline{
A: "hello",
},
},
},
expected: `A = {A = {A = 'hello'}}`,
},
{
desc: "empty slice in map",
v: map[string][]string{
"a": {},
},
expected: `a = []`,
},
{
desc: "map in slice",
v: map[string][]map[string]string{
"a": {{"hello": "world"}},
},
expected: `
[[a]]
hello = 'world'`,
},
{
desc: "newline in map in slice",
v: map[string][]map[string]string{
"a\n": {{"hello": "world"}},
},
err: true,
},
{
desc: "newline in map in slice",
v: map[string][]map[string]*customTextMarshaler{
"a": {{"hello": &customTextMarshaler{1}}},
},
err: true,
},
{
desc: "empty slice of empty struct",
v: struct {
A []struct{}
}{
A: []struct{}{},
},
expected: `A = []`,
},
{
desc: "nil field is ignored",
v: struct {
A interface{}
}{
A: nil,
},
expected: ``,
},
{
desc: "private fields are ignored",
v: struct {
Public string
private string
}{
Public: "shown",
private: "hidden",
},
expected: `Public = 'shown'`,
},
{
desc: "fields tagged - are ignored",
v: struct {
Public string `toml:"-"`
private string
}{
Public: "hidden",
},
expected: ``,
},
{
desc: "nil value in map is ignored",
v: map[string]interface{}{
"A": nil,
},
expected: ``,
},
{
desc: "new line in table key",
v: map[string]interface{}{
"hello\nworld": 42,
},
err: true,
},
{
desc: "new line in parent of nested table key",
v: map[string]interface{}{
"hello\nworld": map[string]interface{}{
"inner": 42,
},
},
err: true,
},
{
desc: "new line in nested table key",
v: map[string]interface{}{
"parent": map[string]interface{}{
"in\ner": map[string]interface{}{
"foo": 42,
},
},
},
err: true,
},
{
desc: "invalid map key",
v: map[int]interface{}{},
err: true,
},
{
desc: "unhandled type",
v: struct {
A chan int
}{
A: make(chan int),
},
err: true,
},
{
desc: "numbers",
v: struct {
A float32
B uint64
C uint32
D uint16
E uint8
F uint
G int64
H int32
I int16
J int8
K int
}{
A: 1.1,
B: 42,
C: 42,
D: 42,
E: 42,
F: 42,
G: 42,
H: 42,
I: 42,
J: 42,
K: 42,
},
expected: `
A = 1.1
B = 42
C = 42
D = 42
E = 42
F = 42
G = 42
H = 42
I = 42
J = 42
K = 42`,
},
}
for _, e := range examples {
@@ -258,10 +521,82 @@ world"""`,
b, err := toml.Marshal(e.v)
if e.err {
require.Error(t, err)
} else {
require.NoError(t, err)
equalStringsIgnoreNewlines(t, e.expected, string(b))
return
}
require.NoError(t, err)
equalStringsIgnoreNewlines(t, e.expected, string(b))
// make sure the output is always valid TOML
defaultMap := map[string]interface{}{}
err = toml.Unmarshal(b, &defaultMap)
require.NoError(t, err)
testWithAllFlags(t, func(t *testing.T, flags int) {
t.Helper()
var buf bytes.Buffer
enc := toml.NewEncoder(&buf)
setFlags(enc, flags)
err := enc.Encode(e.v)
require.NoError(t, err)
inlineMap := map[string]interface{}{}
err = toml.Unmarshal(buf.Bytes(), &inlineMap)
require.NoError(t, err)
require.Equal(t, defaultMap, inlineMap)
})
})
}
}
type flagsSetters []struct {
name string
f func(enc *toml.Encoder, flag bool)
}
var allFlags = flagsSetters{
{"arrays-multiline", (*toml.Encoder).SetArraysMultiline},
{"tables-inline", (*toml.Encoder).SetTablesInline},
{"indent-tables", (*toml.Encoder).SetIndentTables},
}
func setFlags(enc *toml.Encoder, flags int) {
for i := 0; i < len(allFlags); i++ {
enabled := flags&1 > 0
allFlags[i].f(enc, enabled)
}
}
func testWithAllFlags(t *testing.T, testfn func(t *testing.T, flags int)) {
t.Helper()
testWithFlags(t, 0, allFlags, testfn)
}
func testWithFlags(t *testing.T, flags int, setters flagsSetters, testfn func(t *testing.T, flags int)) {
t.Helper()
if len(setters) == 0 {
testfn(t, flags)
return
}
s := setters[0]
for _, enabled := range []bool{false, true} {
name := fmt.Sprintf("%s=%t", s.name, enabled)
newFlags := flags << 1
if enabled {
newFlags++
}
t.Run(name, func(t *testing.T) {
testWithFlags(t, newFlags, setters[1:], testfn)
})
}
}
@@ -272,6 +607,151 @@ func equalStringsIgnoreNewlines(t *testing.T, expected string, actual string) {
assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset))
}
//nolint:funlen
func TestMarshalIndentTables(t *testing.T) {
t.Parallel()
examples := []struct {
desc string
v interface{}
expected string
}{
{
desc: "one kv",
v: map[string]interface{}{
"foo": "bar",
},
expected: `foo = 'bar'`,
},
{
desc: "one level table",
v: map[string]map[string]string{
"foo": {
"one": "value1",
"two": "value2",
},
},
expected: `
[foo]
one = 'value1'
two = 'value2'
`,
},
{
desc: "two levels table",
v: map[string]interface{}{
"root": "value0",
"level1": map[string]interface{}{
"one": "value1",
"level2": map[string]interface{}{
"two": "value2",
},
},
},
expected: `
root = 'value0'
[level1]
one = 'value1'
[level1.level2]
two = 'value2'
`,
},
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
var buf strings.Builder
enc := toml.NewEncoder(&buf)
enc.SetIndentTables(true)
err := enc.Encode(e.v)
require.NoError(t, err)
equalStringsIgnoreNewlines(t, e.expected, buf.String())
})
}
}
type customTextMarshaler struct {
value int64
}
func (c *customTextMarshaler) MarshalText() ([]byte, error) {
if c.value == 1 {
return nil, fmt.Errorf("cannot represent 1 because this is a silly test")
}
return []byte(fmt.Sprintf("::%d", c.value)), nil
}
func TestMarshalTextMarshaler_NoRoot(t *testing.T) {
t.Parallel()
c := customTextMarshaler{}
_, err := toml.Marshal(&c)
require.Error(t, err)
}
func TestMarshalTextMarshaler_Error(t *testing.T) {
t.Parallel()
m := map[string]interface{}{"a": &customTextMarshaler{value: 1}}
_, err := toml.Marshal(m)
require.Error(t, err)
}
func TestMarshalTextMarshaler_ErrorInline(t *testing.T) {
t.Parallel()
type s struct {
A map[string]interface{} `inline:"true"`
}
d := s{
A: map[string]interface{}{"a": &customTextMarshaler{value: 1}},
}
_, err := toml.Marshal(d)
require.Error(t, err)
}
func TestMarshalTextMarshaler(t *testing.T) {
t.Parallel()
m := map[string]interface{}{"a": &customTextMarshaler{value: 2}}
r, err := toml.Marshal(m)
require.NoError(t, err)
equalStringsIgnoreNewlines(t, "a = '::2'", string(r))
}
type brokenWriter struct{}
func (b *brokenWriter) Write([]byte) (int, error) {
return 0, fmt.Errorf("dead")
}
func TestEncodeToBrokenWriter(t *testing.T) {
t.Parallel()
w := brokenWriter{}
enc := toml.NewEncoder(&w)
err := enc.Encode(map[string]string{"hello": "world"})
require.Error(t, err)
}
func TestEncoderSetIndentSymbol(t *testing.T) {
t.Parallel()
var w strings.Builder
enc := toml.NewEncoder(&w)
enc.SetIndentTables(true)
enc.SetIndentSymbol(">>>")
err := enc.Encode(map[string]map[string]string{"parent": {"hello": "world"}})
require.NoError(t, err)
expected := `
[parent]
>>>hello = 'world'`
equalStringsIgnoreNewlines(t, expected, w.String())
}
func TestIssue436(t *testing.T) {
t.Parallel()
@@ -292,3 +772,59 @@ c = 'd'
`
equalStringsIgnoreNewlines(t, expected, buf.String())
}
func TestIssue424(t *testing.T) {
t.Parallel()
type Message1 struct {
Text string
}
type Message2 struct {
Text string `multiline:"true"`
}
msg1 := Message1{"Hello\\World"}
msg2 := Message2{"Hello\\World"}
toml1, err := toml.Marshal(msg1)
require.NoError(t, err)
toml2, err := toml.Marshal(msg2)
require.NoError(t, err)
msg1parsed := Message1{}
err = toml.Unmarshal(toml1, &msg1parsed)
require.NoError(t, err)
require.Equal(t, msg1, msg1parsed)
msg2parsed := Message2{}
err = toml.Unmarshal(toml2, &msg2parsed)
require.NoError(t, err)
require.Equal(t, msg2, msg2parsed)
}
func ExampleMarshal() {
type MyConfig struct {
Version int
Name string
Tags []string
}
cfg := MyConfig{
Version: 2,
Name: "go-toml",
Tags: []string{"go", "toml"},
}
b, err := toml.Marshal(cfg)
if err != nil {
panic(err)
}
fmt.Println(string(b))
// Output:
// Version = 2
// Name = 'go-toml'
// Tags = ['go', 'toml']
}
+205 -144
View File
@@ -2,7 +2,6 @@ package toml
import (
"bytes"
"fmt"
"strconv"
"github.com/pelletier/go-toml/v2/internal/ast"
@@ -26,6 +25,7 @@ func (p *parser) Reset(b []byte) {
p.first = true
}
//nolint:cyclop
func (p *parser) NextExpression() bool {
if len(p.left) == 0 || p.err != nil {
return false
@@ -53,11 +53,11 @@ func (p *parser) NextExpression() bool {
return false
}
p.first = false
if p.ref.Valid() {
return true
}
p.first = false
}
}
@@ -73,18 +73,19 @@ func (p *parser) parseNewline(b []byte) ([]byte, error) {
if b[0] == '\n' {
return b[1:], nil
}
if b[0] == '\r' {
_, rest, err := scanWindowsNewline(b)
return rest, err
}
return nil, fmt.Errorf("expected newline but got %#U", b[0])
return nil, newDecodeError(b[0:1], "expected newline but got %#U", b[0])
}
func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
//expression = ws [ comment ]
//expression =/ ws keyval ws [ comment ]
//expression =/ ws table ws [ comment ]
// expression = ws [ comment ]
// expression =/ ws keyval ws [ comment ]
// expression =/ ws table ws [ comment ]
var ref ast.Reference
b = p.parseWhitespace(b)
@@ -94,9 +95,11 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
}
if b[0] == '#' {
_, rest, err := scanComment(b)
return ref, rest, err
_, rest := scanComment(b)
return ref, rest, nil
}
if b[0] == '\n' || b[0] == '\r' {
return ref, b, nil
}
@@ -107,6 +110,7 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
} else {
ref, b, err = p.parseKeyval(b)
}
if err != nil {
return ref, nil, err
}
@@ -114,57 +118,63 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
b = p.parseWhitespace(b)
if len(b) > 0 && b[0] == '#' {
_, rest, err := scanComment(b)
return ref, rest, err
_, rest := scanComment(b)
return ref, rest, nil
}
return ref, b, nil
}
func (p *parser) parseTable(b []byte) (ast.Reference, []byte, error) {
//table = std-table / array-table
// table = std-table / array-table
if len(b) > 1 && b[1] == '[' {
return p.parseArrayTable(b)
}
return p.parseStdTable(b)
}
func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) {
//array-table = array-table-open key array-table-close
//array-table-open = %x5B.5B ws ; [[ Double left square bracket
//array-table-close = ws %x5D.5D ; ]] Double right square bracket
// array-table = array-table-open key array-table-close
// array-table-open = %x5B.5B ws ; [[ Double left square bracket
// array-table-close = ws %x5D.5D ; ]] Double right square bracket
ref := p.builder.Push(ast.Node{
Kind: ast.ArrayTable,
})
b = b[2:]
b = p.parseWhitespace(b)
k, b, err := p.parseKey(b)
if err != nil {
return ref, nil, err
}
p.builder.AttachChild(ref, k)
b = p.parseWhitespace(b)
b, err = expect(']', b)
if err != nil {
return ref, nil, err
}
b, err = expect(']', b)
return ref, b, err
}
func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
//std-table = std-table-open key std-table-close
//std-table-open = %x5B ws ; [ Left square bracket
//std-table-close = ws %x5D ; ] Right square bracket
// std-table = std-table-open key std-table-close
// std-table-open = %x5B ws ; [ Left square bracket
// std-table-close = ws %x5D ; ] Right square bracket
ref := p.builder.Push(ast.Node{
Kind: ast.Table,
})
b = b[1:]
b = p.parseWhitespace(b)
key, b, err := p.parseKey(b)
if err != nil {
return ref, nil, err
@@ -180,8 +190,7 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
}
func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
//keyval = key keyval-sep val
// keyval = key keyval-sep val
ref := p.builder.Push(ast.Node{
Kind: ast.KeyValue,
})
@@ -191,31 +200,39 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
return ast.Reference{}, nil, err
}
//keyval-sep = ws %x3D ws ; =
// keyval-sep = ws %x3D ws ; =
b = p.parseWhitespace(b)
if len(b) == 0 {
return ast.Reference{}, nil, newDecodeError(b, "expected = after a key, but the document ends there")
}
b, err = expect('=', b)
if err != nil {
return ast.Reference{}, nil, err
}
b = p.parseWhitespace(b)
valRef, b, err := p.parseVal(b)
if err != nil {
return ref, b, err
}
p.builder.Chain(valRef, key)
p.builder.AttachChild(ref, valRef)
return ref, b, err
}
//nolint:cyclop,funlen
func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
// val = string / boolean / array / inline-table / date-time / float / integer
var ref ast.Reference
if len(b) == 0 {
return ref, nil, fmt.Errorf("expected value, not eof")
return ref, nil, newDecodeError(b, "expected value, not eof")
}
var err error
@@ -229,12 +246,14 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
} else {
v, b, err = p.parseBasicString(b)
}
if err == nil {
ref = p.builder.Push(ast.Node{
Kind: ast.String,
Data: v,
})
}
return ref, b, err
case '\'':
var v []byte
@@ -243,30 +262,36 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
} else {
v, b, err = p.parseLiteralString(b)
}
if err == nil {
ref = p.builder.Push(ast.Node{
Kind: ast.String,
Data: v,
})
}
return ref, b, err
case 't':
if !scanFollowsTrue(b) {
return ref, nil, fmt.Errorf("expected 'true'")
return ref, nil, newDecodeError(atmost(b, 4), "expected 'true'")
}
ref = p.builder.Push(ast.Node{
Kind: ast.Bool,
Data: b[:4],
})
return ref, b[4:], nil
case 'f':
if !scanFollowsFalse(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'false'")
return ref, nil, newDecodeError(atmost(b, 5), "expected 'false'")
}
ref = p.builder.Push(ast.Node{
Kind: ast.Bool,
Data: b[:5],
})
return ref, b[5:], nil
case '[':
return p.parseValArray(b)
@@ -277,31 +302,41 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
}
}
func atmost(b []byte, n int) []byte {
if n >= len(b) {
return b
}
return b[:n]
}
func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, error) {
v, rest, err := scanLiteralString(b)
if err != nil {
return nil, nil, err
}
return v[1 : len(v)-1], rest, nil
}
func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
//inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close
//inline-table-open = %x7B ws ; {
//inline-table-close = ws %x7D ; }
//inline-table-sep = ws %x2C ws ; , Comma
//inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
// inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close
// inline-table-open = %x7B ws ; {
// inline-table-close = ws %x7D ; }
// inline-table-sep = ws %x2C ws ; , Comma
// inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
parent := p.builder.Push(ast.Node{
Kind: ast.InlineTable,
})
first := true
var child ast.Reference
b = b[1:]
var err error
for len(b) > 0 {
b = p.parseWhitespace(b)
if b[0] == '}' {
@@ -315,7 +350,9 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
}
b = p.parseWhitespace(b)
}
var kv ast.Reference
kv, b, err = p.parseKeyval(b)
if err != nil {
return parent, nil, err
@@ -323,7 +360,6 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
if first {
p.builder.AttachChild(parent, kv)
first = false
} else {
p.builder.Chain(child, kv)
}
@@ -333,18 +369,19 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
}
rest, err := expect('}', b)
return parent, rest, err
}
//nolint:funlen,cyclop
func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
//array = array-open [ array-values ] ws-comment-newline array-close
//array-open = %x5B ; [
//array-close = %x5D ; ]
//array-values = ws-comment-newline val ws-comment-newline array-sep array-values
//array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ]
//array-sep = %x2C ; , Comma
//ws-comment-newline = *( wschar / [ comment ] newline )
// array = array-open [ array-values ] ws-comment-newline array-close
// array-open = %x5B ; [
// array-close = %x5D ; ]
// array-values = ws-comment-newline val ws-comment-newline array-sep array-values
// array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ]
// array-sep = %x2C ; , Comma
// ws-comment-newline = *( wschar / [ comment ] newline )
b = b[1:]
parent := p.builder.Push(ast.Node{
@@ -352,6 +389,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
})
first := true
var lastChild ast.Reference
var err error
@@ -362,17 +400,19 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
}
if len(b) == 0 {
return parent, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF
return parent, nil, newDecodeError(b, "array is incomplete")
}
if b[0] == ']' {
break
}
if b[0] == ',' {
if first {
return parent, nil, fmt.Errorf("array cannot start with comma")
return parent, nil, newDecodeError(b[0:1], "array cannot start with comma")
}
b = b[1:]
b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil {
return parent, nil, err
@@ -385,6 +425,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
}
var valueRef ast.Reference
valueRef, b, err = p.parseVal(b)
if err != nil {
return parent, nil, err
@@ -392,7 +433,6 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
if first {
p.builder.AttachChild(parent, valueRef)
first = false
} else {
p.builder.Chain(lastChild, valueRef)
}
@@ -406,6 +446,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
}
rest, err := expect(']', b)
return parent, rest, err
}
@@ -413,15 +454,15 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
for len(b) > 0 {
var err error
b = p.parseWhitespace(b)
if len(b) > 0 && b[0] == '#' {
_, b, err = scanComment(b)
if err != nil {
return nil, err
}
_, b = scanComment(b)
}
if len(b) == 0 {
break
}
if b[0] == '\n' || b[0] == '\r' {
b, err = p.parseNewline(b)
if err != nil {
@@ -431,6 +472,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
break
}
}
return b, nil
}
@@ -448,25 +490,27 @@ func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, error) {
} else if token[i] == '\r' && token[i+1] == '\n' {
i += 2
}
return token[i : len(token)-3], rest, err
}
//nolint:funlen,gocognit,cyclop
func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
//ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
//ml-basic-string-delim
//ml-basic-string-delim = 3quotation-mark
//ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
// ml-basic-string-delim
// ml-basic-string-delim = 3quotation-mark
// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
//
//mlb-content = mlb-char / newline / mlb-escaped-nl
//mlb-char = mlb-unescaped / escaped
//mlb-quotes = 1*2quotation-mark
//mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
//mlb-escaped-nl = escape ws newline *( wschar / newline )
// mlb-content = mlb-char / newline / mlb-escaped-nl
// mlb-char = mlb-unescaped / escaped
// mlb-quotes = 1*2quotation-mark
// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// mlb-escaped-nl = escape ws newline *( wschar / newline )
token, rest, err := scanMultilineBasicString(b)
if err != nil {
return nil, nil, err
}
var builder bytes.Buffer
i := 3
@@ -482,6 +526,8 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
// escapes are balanced.
for ; i < len(token)-3; i++ {
c := token[i]
//nolint:nestif
if c == '\\' {
// When the last non-whitespace character on a line is an unescaped \,
// it will be trimmed along with all whitespace (including newlines) up
@@ -492,15 +538,18 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
c := token[i]
if !(c == '\n' || c == '\r' || c == ' ' || c == '\t') {
i--
break
}
}
continue
}
// handle escaping
i++
c = token[i]
switch c {
case '"', '\\':
builder.WriteByte(c)
@@ -515,21 +564,23 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
case 't':
builder.WriteByte('\t')
case 'u':
x, err := hexToString(token[i+3:len(token)-3], 4)
x, err := hexToString(atmost(token[i+1:], 4), 4)
if err != nil {
return nil, nil, err
}
builder.WriteString(x)
i += 4
case 'U':
x, err := hexToString(token[i+3:len(token)-3], 8)
x, err := hexToString(atmost(token[i+1:], 8), 8)
if err != nil {
return nil, nil, err
}
builder.WriteString(x)
i += 8
default:
return nil, nil, fmt.Errorf("invalid escaped character: %#U", c)
return nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
}
} else {
builder.WriteByte(c)
@@ -540,15 +591,14 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
}
func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
//key = simple-key / dotted-key
//simple-key = quoted-key / unquoted-key
// key = simple-key / dotted-key
// simple-key = quoted-key / unquoted-key
//
//unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
//quoted-key = basic-string / literal-string
//dotted-key = simple-key 1*( dot-sep simple-key )
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
// quoted-key = basic-string / literal-string
// dotted-key = simple-key 1*( dot-sep simple-key )
//
//dot-sep = ws %x2E ws ; . Period
// dot-sep = ws %x2E ws ; . Period
key, b, err := p.parseSimpleKey(b)
if err != nil {
return ast.Reference{}, nil, err
@@ -562,15 +612,13 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
for {
b = p.parseWhitespace(b)
if len(b) > 0 && b[0] == '.' {
b, err = expect('.', b)
if err != nil {
return ref, nil, err
}
b = p.parseWhitespace(b)
b = p.parseWhitespace(b[1:])
key, b, err = p.parseSimpleKey(b)
if err != nil {
return ref, nil, err
}
p.builder.PushAndChain(ast.Node{
Kind: ast.Key,
Data: key,
@@ -584,46 +632,47 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
}
func (p *parser) parseSimpleKey(b []byte) (key, rest []byte, err error) {
//simple-key = quoted-key / unquoted-key
//unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
//quoted-key = basic-string / literal-string
// simple-key = quoted-key / unquoted-key
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
// quoted-key = basic-string / literal-string
if len(b) == 0 {
return nil, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF
return nil, nil, newDecodeError(b, "key is incomplete")
}
if b[0] == '\'' {
key, rest, err = p.parseLiteralString(b)
} else if b[0] == '"' {
key, rest, err = p.parseBasicString(b)
} else if isUnquotedKeyChar(b[0]) {
key, rest, err = scanUnquotedKey(b)
} else {
err = unexpectedCharacter{b: b} // TODO: should contain expected characters
switch {
case b[0] == '\'':
return p.parseLiteralString(b)
case b[0] == '"':
return p.parseBasicString(b)
case isUnquotedKeyChar(b[0]):
key, rest = scanUnquotedKey(b)
return key, rest, nil
default:
return nil, nil, newDecodeError(b[0:1], "invalid character at start of key: %c", b[0])
}
return
}
//nolint:funlen,cyclop
func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
//basic-string = quotation-mark *basic-char quotation-mark
//quotation-mark = %x22 ; "
//basic-char = basic-unescaped / escaped
//basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
//escaped = escape escape-seq-char
//escape-seq-char = %x22 ; " quotation mark U+0022
//escape-seq-char =/ %x5C ; \ reverse solidus U+005C
//escape-seq-char =/ %x62 ; b backspace U+0008
//escape-seq-char =/ %x66 ; f form feed U+000C
//escape-seq-char =/ %x6E ; n line feed U+000A
//escape-seq-char =/ %x72 ; r carriage return U+000D
//escape-seq-char =/ %x74 ; t tab U+0009
//escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX
//escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
// basic-string = quotation-mark *basic-char quotation-mark
// quotation-mark = %x22 ; "
// basic-char = basic-unescaped / escaped
// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// escaped = escape escape-seq-char
// escape-seq-char = %x22 ; " quotation mark U+0022
// escape-seq-char =/ %x5C ; \ reverse solidus U+005C
// escape-seq-char =/ %x62 ; b backspace U+0008
// escape-seq-char =/ %x66 ; f form feed U+000C
// escape-seq-char =/ %x6E ; n line feed U+000A
// escape-seq-char =/ %x72 ; r carriage return U+000D
// escape-seq-char =/ %x74 ; t tab U+0009
// escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
token, rest, err := scanBasicString(b)
if err != nil {
return nil, nil, err
}
var builder bytes.Buffer
// The scanner ensures that the token starts and ends with quotes and that
@@ -633,6 +682,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if c == '\\' {
i++
c = token[i]
switch c {
case '"', '\\':
builder.WriteByte(c)
@@ -651,6 +701,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if err != nil {
return nil, nil, err
}
builder.WriteString(x)
i += 4
case 'U':
@@ -658,10 +709,11 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if err != nil {
return nil, nil, err
}
builder.WriteString(x)
i += 8
default:
return nil, nil, fmt.Errorf("invalid escaped character: %#U", c)
return nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
}
} else {
builder.WriteByte(c)
@@ -673,39 +725,46 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
func hexToString(b []byte, length int) (string, error) {
if len(b) < length {
return "", fmt.Errorf("unicode point needs %d hex characters", length)
return "", newDecodeError(b, "unicode point needs %d character, not %d", length, len(b))
}
b = b[:length]
//nolint:godox
// TODO: slow
intcode, err := strconv.ParseInt(string(b[:length]), 16, 32)
intcode, err := strconv.ParseInt(string(b), 16, 32)
if err != nil {
return "", err
return "", newDecodeError(b, "couldn't parse hexadecimal number: %w", err)
}
return string(rune(intcode)), nil
}
func (p *parser) parseWhitespace(b []byte) []byte {
//ws = *wschar
//wschar = %x20 ; Space
//wschar =/ %x09 ; Horizontal tab
// ws = *wschar
// wschar = %x20 ; Space
// wschar =/ %x09 ; Horizontal tab
_, rest := scanWhitespace(b)
return rest
}
//nolint:cyclop
func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, error) {
switch b[0] {
case 'i':
if !scanFollowsInf(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'inf'")
return ast.Reference{}, nil, newDecodeError(atmost(b, 3), "expected 'inf'")
}
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:3],
}), b[3:], nil
case 'n':
if !scanFollowsNan(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'nan'")
return ast.Reference{}, nil, newDecodeError(atmost(b, 3), "expected 'nan'")
}
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:3],
@@ -714,60 +773,74 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.scanIntOrFloat(b)
}
//nolint:gomnd
if len(b) < 3 {
return p.scanIntOrFloat(b)
}
s := 5
if len(b) < s {
s = len(b)
}
for idx, c := range b[:s] {
if isDigit(c) {
continue
}
if idx == 2 && c == ':' || (idx == 4 && c == '-') {
return p.scanDateTime(b)
}
}
return p.scanIntOrFloat(b)
}
func digitsToInt(b []byte) int {
x := 0
for _, d := range b {
x *= 10
x += int(d - '0')
}
return x
}
//nolint:gocognit,cyclop
func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
// scans for contiguous characters in [0-9T:Z.+-], and up to one space if
// followed by a digit.
hasTime := false
hasTz := false
seenSpace := false
i := 0
byteLoop:
for ; i < len(b); i++ {
c := b[i]
if isDigit(c) || c == '-' {
} else if c == 'T' || c == ':' || c == '.' {
switch {
case isDigit(c):
case c == '-':
const offsetOfTz = 19
if i == offsetOfTz {
hasTz = true
}
case c == 'T' || c == ':' || c == '.':
hasTime = true
continue
} else if c == '+' || c == '-' || c == 'Z' {
case c == '+' || c == '-' || c == 'Z':
hasTz = true
} else if c == ' ' {
case c == ' ':
if !seenSpace && i+1 < len(b) && isDigit(b[i+1]) {
i += 2
seenSpace = true
hasTime = true
} else {
break
break byteLoop
}
} else {
break
default:
break byteLoop
}
}
@@ -780,9 +853,6 @@ func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
kind = ast.LocalDateTime
}
} else {
if hasTz {
return ast.Reference{}, nil, fmt.Errorf("possible DateTime cannot have a timezone but no time component")
}
kind = ast.LocalDate
}
@@ -792,11 +862,13 @@ func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
}), b[i:], nil
}
//nolint:funlen,gocognit,cyclop
func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
i := 0
if len(b) > 2 && b[0] == '0' && b[1] != '.' {
var isValidRune validRuneFn
switch b[1] {
case 'x':
isValidRune = isValidHexRune
@@ -834,6 +906,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
if c == '.' || c == 'e' || c == 'E' {
isFloat = true
continue
}
@@ -844,8 +917,10 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
Data: b[:i+3],
}), b[i+3:], nil
}
return ast.Reference{}, nil, fmt.Errorf("unexpected character i while scanning for a number")
return ast.Reference{}, nil, newDecodeError(b[i:i+1], "unexpected character 'i' while scanning for a number")
}
if c == 'n' {
if scanFollowsNan(b[i:]) {
return p.builder.Push(ast.Node{
@@ -853,14 +928,15 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
Data: b[:i+3],
}), b[i+3:], nil
}
return ast.Reference{}, nil, fmt.Errorf("unexpected character n while scanning for a number")
return ast.Reference{}, nil, newDecodeError(b[i:i+1], "unexpected character 'n' while scanning for a number")
}
break
}
if i == 0 {
return ast.Reference{}, b, fmt.Errorf("expected integer or float")
return ast.Reference{}, b, newDecodeError(b, "incomplete number")
}
kind := ast.Integer
@@ -897,24 +973,9 @@ func isValidBinaryRune(r byte) bool {
}
func expect(x byte, b []byte) ([]byte, error) {
if len(b) == 0 {
return nil, newDecodeError(b[:0], "expecting %#U", x)
}
if b[0] != x {
return nil, newDecodeError(b[0:1], "expected character %U", x)
}
return b[1:], nil
}
type unexpectedCharacter struct {
r byte
b []byte
}
func (u unexpectedCharacter) Error() string {
if len(u.b) == 0 {
return fmt.Sprintf("expected %#U, not EOF", u.r)
}
return fmt.Sprintf("expected %#U, not %#U", u.r, u.b[0])
}
+10 -54
View File
@@ -7,7 +7,10 @@ import (
"github.com/stretchr/testify/require"
)
//nolint:funlen
func TestParser_AST_Numbers(t *testing.T) {
t.Parallel()
examples := []struct {
desc string
input string
@@ -132,7 +135,9 @@ func TestParser_AST_Numbers(t *testing.T) {
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
p := parser{}
p.Reset([]byte(`A = ` + e.input))
p.NextExpression()
@@ -156,7 +161,6 @@ func TestParser_AST_Numbers(t *testing.T) {
}
type (
astRoot []astNode
astNode struct {
Kind ast.Kind
Data []byte
@@ -164,11 +168,6 @@ type (
}
)
func compareAST(t *testing.T, expected astRoot, actual *ast.Root) {
it := actual.Iterator()
compareIterator(t, expected, it)
}
func compareNode(t *testing.T, e astNode, n ast.Node) {
t.Helper()
require.Equal(t, e.Kind, n.Kind)
@@ -199,55 +198,10 @@ func compareIterator(t *testing.T, expected []astNode, actual ast.Iterator) {
}
}
func (r astRoot) toOrig() *ast.Root {
builder := &ast.Builder{}
var last ast.Reference
for i, n := range r {
ref := builder.Push(ast.Node{
Kind: n.Kind,
Data: n.Data,
})
if i > 0 {
builder.Chain(last, ref)
}
last = ref
if len(n.Children) > 0 {
c := childrenToOrig(builder, n.Children)
builder.AttachChild(ref, c)
}
}
return builder.Tree()
}
func childrenToOrig(b *ast.Builder, nodes []astNode) ast.Reference {
var first ast.Reference
var last ast.Reference
for i, n := range nodes {
ref := b.Push(ast.Node{
Kind: n.Kind,
Data: n.Data,
})
if i == 0 {
first = ref
} else {
b.Chain(last, ref)
}
last = ref
if len(n.Children) > 0 {
c := childrenToOrig(b, n.Children)
b.AttachChild(ref, c)
}
}
return first
}
//nolint:funlen
func TestParser_AST(t *testing.T) {
t.Parallel()
examples := []struct {
desc string
input string
@@ -384,7 +338,9 @@ func TestParser_AST(t *testing.T) {
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
p := parser{}
p.Reset([]byte(e.input))
p.NextExpression()
+43 -46
View File
@@ -1,9 +1,8 @@
package toml
import "fmt"
func scanFollows(b []byte, pattern string) bool {
n := len(pattern)
return len(b) >= n && string(b[:n]) == pattern
}
@@ -31,14 +30,15 @@ func scanFollowsNan(b []byte) bool {
return scanFollows(b, `nan`)
}
func scanUnquotedKey(b []byte) ([]byte, []byte, error) {
func scanUnquotedKey(b []byte) ([]byte, []byte) {
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
for i := 0; i < len(b); i++ {
if !isUnquotedKeyChar(b[i]) {
return b[:i], b[i:], nil
return b[:i], b[i:]
}
}
return b, b[len(b):], nil
return b, b[len(b):]
}
func isUnquotedKeyChar(r byte) bool {
@@ -57,24 +57,22 @@ func scanLiteralString(b []byte) ([]byte, []byte, error) {
return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines")
}
}
return nil, nil, newDecodeError(b[len(b):], "unterminated literal string")
}
func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
//ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
//ml-literal-string-delim
//ml-literal-string-delim = 3apostrophe
//ml-literal-body = *mll-content *( mll-quotes 1*mll-content ) [ mll-quotes ]
// ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
// ml-literal-string-delim
// ml-literal-string-delim = 3apostrophe
// ml-literal-body = *mll-content *( mll-quotes 1*mll-content ) [ mll-quotes ]
//
//mll-content = mll-char / newline
//mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
//mll-quotes = 1*2apostrophe
// mll-content = mll-char / newline
// mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
// mll-quotes = 1*2apostrophe
for i := 3; i < len(b); i++ {
switch b[i] {
case '\'':
if scanFollowsMultilineLiteralStringDelimiter(b[i:]) {
return b[:i+3], b[i+3:], nil
}
if b[i] == '\'' && scanFollowsMultilineLiteralStringDelimiter(b[i:]) {
return b[:i+3], b[i+3:], nil
}
}
@@ -82,13 +80,16 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
}
func scanWindowsNewline(b []byte) ([]byte, []byte, error) {
if len(b) < 2 {
return nil, nil, fmt.Errorf(`windows new line missing \n`)
const lenCRLF = 2
if len(b) < lenCRLF {
return nil, nil, newDecodeError(b, "windows new line expected")
}
if b[1] != '\n' {
return nil, nil, fmt.Errorf(`windows new line should be \r\n`)
return nil, nil, newDecodeError(b, `windows new line should be \r\n`)
}
return b[:2], b[2:], nil
return b[:lenCRLF], b[lenCRLF:], nil
}
func scanWhitespace(b []byte) ([]byte, []byte) {
@@ -100,28 +101,26 @@ func scanWhitespace(b []byte) ([]byte, []byte) {
return b[:i], b[i:]
}
}
return b, b[len(b):]
}
func scanComment(b []byte) ([]byte, []byte, error) {
//;; Comment
//nolint:unparam
func scanComment(b []byte) ([]byte, []byte) {
// comment-start-symbol = %x23 ; #
// non-ascii = %x80-D7FF / %xE000-10FFFF
// non-eol = %x09 / %x20-7F / non-ascii
//
//comment-start-symbol = %x23 ; #
//non-ascii = %x80-D7FF / %xE000-10FFFF
//non-eol = %x09 / %x20-7F / non-ascii
//
//comment = comment-start-symbol *non-eol
// comment = comment-start-symbol *non-eol
for i := 1; i < len(b); i++ {
switch b[i] {
case '\n':
return b[:i], b[i:], nil
if b[i] == '\n' {
return b[:i], b[i:]
}
}
return b, nil, nil
return b, nil
}
// TODO perform validation on the string?
func scanBasicString(b []byte) ([]byte, []byte, error) {
// basic-string = quotation-mark *basic-char quotation-mark
// quotation-mark = %x22 ; "
@@ -142,22 +141,20 @@ func scanBasicString(b []byte) ([]byte, []byte, error) {
}
}
return nil, nil, fmt.Errorf(`basic string not terminated by "`)
return nil, nil, newDecodeError(b[len(b):], `basic string not terminated by "`)
}
// TODO perform validation on the string?
func scanMultilineBasicString(b []byte) ([]byte, []byte, error) {
//ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
//ml-basic-string-delim
//ml-basic-string-delim = 3quotation-mark
//ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
// ml-basic-string-delim
// ml-basic-string-delim = 3quotation-mark
// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
//
//mlb-content = mlb-char / newline / mlb-escaped-nl
//mlb-char = mlb-unescaped / escaped
//mlb-quotes = 1*2quotation-mark
//mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
//mlb-escaped-nl = escape ws newline *( wschar / newline )
// mlb-content = mlb-char / newline / mlb-escaped-nl
// mlb-char = mlb-unescaped / escaped
// mlb-quotes = 1*2quotation-mark
// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// mlb-escaped-nl = escape ws newline *( wschar / newline )
for i := 3; i < len(b); i++ {
switch b[i] {
case '"':
+9
View File
@@ -18,6 +18,7 @@ func (s *strict) EnterTable(node ast.Node) {
if !s.Enabled {
return
}
s.key.UpdateTable(node)
}
@@ -25,6 +26,7 @@ func (s *strict) EnterArrayTable(node ast.Node) {
if !s.Enabled {
return
}
s.key.UpdateArrayTable(node)
}
@@ -32,6 +34,7 @@ func (s *strict) EnterKeyValue(node ast.Node) {
if !s.Enabled {
return
}
s.key.Push(node)
}
@@ -39,6 +42,7 @@ func (s *strict) ExitKeyValue(node ast.Node) {
if !s.Enabled {
return
}
s.key.Pop(node)
}
@@ -46,6 +50,7 @@ func (s *strict) MissingTable(node ast.Node) {
if !s.Enabled {
return
}
s.missing = append(s.missing, decodeError{
highlight: keyLocation(node),
message: "missing table",
@@ -57,6 +62,7 @@ func (s *strict) MissingField(node ast.Node) {
if !s.Enabled {
return
}
s.missing = append(s.missing, decodeError{
highlight: keyLocation(node),
message: "missing field",
@@ -72,8 +78,11 @@ func (s *strict) Error(doc []byte) error {
err := &StrictMissingError{
Errors: make([]DecodeError, 0, len(s.missing)),
}
for _, derr := range s.missing {
derr := derr
err.Errors = append(err.Errors, *wrapDecodeError(doc, &derr))
}
return err
}
+161 -178
View File
@@ -13,19 +13,19 @@ type target interface {
get() reflect.Value
// Store a string at the target.
setString(v string) error
setString(v string)
// Store a boolean at the target
setBool(v bool) error
setBool(v bool)
// Store an int64 at the target
setInt64(v int64) error
setInt64(v int64)
// Store a float64 at the target
setFloat64(v float64) error
setFloat64(v float64)
// Stores any value at the target
set(v reflect.Value) error
set(v reflect.Value)
}
// valueTarget just contains a reflect.Value that can be set.
@@ -36,29 +36,24 @@ func (t valueTarget) get() reflect.Value {
return reflect.Value(t)
}
func (t valueTarget) set(v reflect.Value) error {
func (t valueTarget) set(v reflect.Value) {
reflect.Value(t).Set(v)
return nil
}
func (t valueTarget) setString(v string) error {
func (t valueTarget) setString(v string) {
t.get().SetString(v)
return nil
}
func (t valueTarget) setBool(v bool) error {
func (t valueTarget) setBool(v bool) {
t.get().SetBool(v)
return nil
}
func (t valueTarget) setInt64(v int64) error {
func (t valueTarget) setInt64(v int64) {
t.get().SetInt(v)
return nil
}
func (t valueTarget) setFloat64(v float64) error {
func (t valueTarget) setFloat64(v float64) {
t.get().SetFloat(v)
return nil
}
// interfaceTarget wraps an other target to dereference on get.
@@ -70,24 +65,24 @@ func (t interfaceTarget) get() reflect.Value {
return t.x.get().Elem()
}
func (t interfaceTarget) set(v reflect.Value) error {
return t.x.set(v)
func (t interfaceTarget) set(v reflect.Value) {
t.x.set(v)
}
func (t interfaceTarget) setString(v string) error {
return t.x.setString(v)
func (t interfaceTarget) setString(v string) {
panic("interface targets should always go through set")
}
func (t interfaceTarget) setBool(v bool) error {
return t.x.setBool(v)
func (t interfaceTarget) setBool(v bool) {
panic("interface targets should always go through set")
}
func (t interfaceTarget) setInt64(v int64) error {
return t.x.setInt64(v)
func (t interfaceTarget) setInt64(v int64) {
panic("interface targets should always go through set")
}
func (t interfaceTarget) setFloat64(v float64) error {
return t.x.setFloat64(v)
func (t interfaceTarget) setFloat64(v float64) {
panic("interface targets should always go through set")
}
// mapTarget targets a specific key of a map.
@@ -100,25 +95,24 @@ func (t mapTarget) get() reflect.Value {
return t.v.MapIndex(t.k)
}
func (t mapTarget) set(v reflect.Value) error {
func (t mapTarget) set(v reflect.Value) {
t.v.SetMapIndex(t.k, v)
return nil
}
func (t mapTarget) setString(v string) error {
return t.set(reflect.ValueOf(v))
func (t mapTarget) setString(v string) {
t.set(reflect.ValueOf(v))
}
func (t mapTarget) setBool(v bool) error {
return t.set(reflect.ValueOf(v))
func (t mapTarget) setBool(v bool) {
t.set(reflect.ValueOf(v))
}
func (t mapTarget) setInt64(v int64) error {
return t.set(reflect.ValueOf(v))
func (t mapTarget) setInt64(v int64) {
t.set(reflect.ValueOf(v))
}
func (t mapTarget) setFloat64(v float64) error {
return t.set(reflect.ValueOf(v))
func (t mapTarget) setFloat64(v float64) {
t.set(reflect.ValueOf(v))
}
// makes sure that the value pointed at by t is indexable (Slice, Array), or
@@ -129,40 +123,36 @@ func ensureValueIndexable(t target) error {
switch f.Type().Kind() {
case reflect.Slice:
if f.IsNil() {
return t.set(reflect.MakeSlice(f.Type(), 0, 0))
t.set(reflect.MakeSlice(f.Type(), 0, 0))
return nil
}
case reflect.Interface:
if f.IsNil() || f.Elem().Type() != sliceInterfaceType {
return t.set(reflect.MakeSlice(sliceInterfaceType, 0, 0))
}
if f.Elem().Type().Kind() != reflect.Slice {
return fmt.Errorf("interface is pointing to a %s, not a slice", f.Kind())
t.set(reflect.MakeSlice(sliceInterfaceType, 0, 0))
return nil
}
case reflect.Ptr:
if f.IsNil() {
ptr := reflect.New(f.Type().Elem())
err := t.set(ptr)
if err != nil {
return err
}
f = t.get()
}
return ensureValueIndexable(valueTarget(f.Elem()))
panic("pointer should have already been dereferenced")
case reflect.Array:
// arrays are always initialized.
default:
return fmt.Errorf("cannot initialize a slice in %s", f.Kind())
return fmt.Errorf("toml: cannot store array in a %s", f.Kind())
}
return nil
}
var sliceInterfaceType = reflect.TypeOf([]interface{}{})
var mapStringInterfaceType = reflect.TypeOf(map[string]interface{}{})
var (
sliceInterfaceType = reflect.TypeOf([]interface{}{})
mapStringInterfaceType = reflect.TypeOf(map[string]interface{}{})
)
func ensureMapIfInterface(x target) {
v := x.get()
if v.Kind() == reflect.Interface && v.IsNil() {
newElement := reflect.MakeMap(mapStringInterfaceType)
x.set(newElement)
}
}
@@ -172,12 +162,14 @@ func setString(t target, v string) error {
switch f.Kind() {
case reflect.String:
return t.setString(v)
t.setString(v)
case reflect.Interface:
return t.set(reflect.ValueOf(v))
t.set(reflect.ValueOf(v))
default:
return fmt.Errorf("cannot assign string to a %s", f.Kind())
return fmt.Errorf("toml: cannot assign string to a %s", f.Kind())
}
return nil
}
func setBool(t target, v bool) error {
@@ -185,83 +177,90 @@ func setBool(t target, v bool) error {
switch f.Kind() {
case reflect.Bool:
return t.setBool(v)
t.setBool(v)
case reflect.Interface:
return t.set(reflect.ValueOf(v))
t.set(reflect.ValueOf(v))
default:
return fmt.Errorf("cannot assign bool to a %s", f.String())
return fmt.Errorf("toml: cannot assign boolean to a %s", f.Kind())
}
return nil
}
const maxInt = int64(^uint(0) >> 1)
const minInt = -maxInt - 1
const (
maxInt = int64(^uint(0) >> 1)
minInt = -maxInt - 1
)
//nolint:funlen,gocognit,cyclop
func setInt64(t target, v int64) error {
f := t.get()
switch f.Kind() {
case reflect.Int64:
return t.setInt64(v)
t.setInt64(v)
case reflect.Int32:
if v < math.MinInt32 || v > math.MaxInt32 {
return fmt.Errorf("integer %d does not fit in an int32", v)
return fmt.Errorf("toml: number %d does not fit in an int32", v)
}
return t.set(reflect.ValueOf(int32(v)))
t.set(reflect.ValueOf(int32(v)))
return nil
case reflect.Int16:
if v < math.MinInt16 || v > math.MaxInt16 {
return fmt.Errorf("integer %d does not fit in an int16", v)
return fmt.Errorf("toml: number %d does not fit in an int16", v)
}
return t.set(reflect.ValueOf(int16(v)))
t.set(reflect.ValueOf(int16(v)))
case reflect.Int8:
if v < math.MinInt8 || v > math.MaxInt8 {
return fmt.Errorf("integer %d does not fit in an int8", v)
return fmt.Errorf("toml: number %d does not fit in an int8", v)
}
return t.set(reflect.ValueOf(int8(v)))
t.set(reflect.ValueOf(int8(v)))
case reflect.Int:
if v < minInt || v > maxInt {
return fmt.Errorf("integer %d does not fit in an int", v)
return fmt.Errorf("toml: number %d does not fit in an int", v)
}
return t.set(reflect.ValueOf(int(v)))
t.set(reflect.ValueOf(int(v)))
case reflect.Uint64:
if v < 0 {
return fmt.Errorf("negative integer %d cannot be stored in an uint64", v)
return fmt.Errorf("toml: negative number %d does not fit in an uint64", v)
}
return t.set(reflect.ValueOf(uint64(v)))
t.set(reflect.ValueOf(uint64(v)))
case reflect.Uint32:
if v < 0 {
return fmt.Errorf("negative integer %d cannot be stored in an uint32", v)
if v < 0 || v > math.MaxUint32 {
return fmt.Errorf("toml: negative number %d does not fit in an uint32", v)
}
if v > math.MaxUint32 {
return fmt.Errorf("integer %d cannot be stored in an uint32", v)
}
return t.set(reflect.ValueOf(uint32(v)))
t.set(reflect.ValueOf(uint32(v)))
case reflect.Uint16:
if v < 0 {
return fmt.Errorf("negative integer %d cannot be stored in an uint16", v)
if v < 0 || v > math.MaxUint16 {
return fmt.Errorf("toml: negative number %d does not fit in an uint16", v)
}
if v > math.MaxUint16 {
return fmt.Errorf("integer %d cannot be stored in an uint16", v)
}
return t.set(reflect.ValueOf(uint16(v)))
t.set(reflect.ValueOf(uint16(v)))
case reflect.Uint8:
if v < 0 {
return fmt.Errorf("negative integer %d cannot be stored in an uint8", v)
if v < 0 || v > math.MaxUint8 {
return fmt.Errorf("toml: negative number %d does not fit in an uint8", v)
}
if v > math.MaxUint8 {
return fmt.Errorf("integer %d cannot be stored in an uint8", v)
}
return t.set(reflect.ValueOf(uint8(v)))
t.set(reflect.ValueOf(uint8(v)))
case reflect.Uint:
if v < 0 {
return fmt.Errorf("negative integer %d cannot be stored in an uint", v)
return fmt.Errorf("toml: negative number %d does not fit in an uint", v)
}
return t.set(reflect.ValueOf(uint(v)))
t.set(reflect.ValueOf(uint(v)))
case reflect.Interface:
return t.set(reflect.ValueOf(v))
t.set(reflect.ValueOf(v))
default:
return fmt.Errorf("cannot assign int64 to a %s", f.String())
return fmt.Errorf("toml: integer cannot be assigned to %s", f.Kind())
}
return nil
}
func setFloat64(t target, v float64) error {
@@ -269,98 +268,86 @@ func setFloat64(t target, v float64) error {
switch f.Kind() {
case reflect.Float64:
return t.setFloat64(v)
t.setFloat64(v)
case reflect.Float32:
if v > math.MaxFloat32 {
return fmt.Errorf("float %f cannot be stored in a float32", v)
return fmt.Errorf("toml: number %f does not fit in a float32", v)
}
return t.set(reflect.ValueOf(float32(v)))
t.set(reflect.ValueOf(float32(v)))
case reflect.Interface:
return t.set(reflect.ValueOf(v))
t.set(reflect.ValueOf(v))
default:
return fmt.Errorf("cannot assign float64 to a %s", f.String())
return fmt.Errorf("toml: float cannot be assigned to %s", f.Kind())
}
return nil
}
// Returns the element at idx of the value pointed at by target, or an error if
// t does not point to an indexable.
// If the target points to an Array and idx is out of bounds, it returns
// (nil, nil) as this is not a fatal error (the unmarshaler will skip).
func elementAt(t target, idx int) (target, error) {
func elementAt(t target, idx int) target {
f := t.get()
switch f.Kind() {
case reflect.Slice:
//nolint:godox
// TODO: use the idx function argument and avoid alloc if possible.
idx := f.Len()
err := t.set(reflect.Append(f, reflect.New(f.Type().Elem()).Elem()))
if err != nil {
return nil, err
}
return valueTarget(t.get().Index(idx)), nil
t.set(reflect.Append(f, reflect.New(f.Type().Elem()).Elem()))
return valueTarget(t.get().Index(idx))
case reflect.Array:
if idx >= f.Len() {
return nil, nil
return nil
}
return valueTarget(f.Index(idx)), nil
return valueTarget(f.Index(idx))
case reflect.Interface:
if f.IsNil() {
panic("interface should have been initialized")
}
// This function is called after ensureValueIndexable, so it's
// guaranteed that f contains an initialized slice.
ifaceElem := f.Elem()
if ifaceElem.Kind() != reflect.Slice {
return nil, fmt.Errorf("cannot elementAt on a %s", f.Kind())
}
idx := ifaceElem.Len()
newElem := reflect.New(ifaceElem.Type().Elem()).Elem()
newSlice := reflect.Append(ifaceElem, newElem)
err := t.set(newSlice)
if err != nil {
return nil, err
}
return valueTarget(t.get().Elem().Index(idx)), nil
case reflect.Ptr:
return elementAt(valueTarget(f.Elem()), idx)
t.set(newSlice)
return valueTarget(t.get().Elem().Index(idx))
default:
return nil, fmt.Errorf("cannot elementAt on a %s", f.Kind())
// Why ensureValueIndexable let it go through?
panic(fmt.Errorf("elementAt received unhandled value type: %s", f.Kind()))
}
}
func (d *decoder) scopeTableTarget(append bool, t target, name string) (target, bool, error) {
func (d *decoder) scopeTableTarget(shouldAppend bool, t target, name string) (target, bool, error) {
x := t.get()
switch x.Kind() {
// Kinds that need to recurse
case reflect.Interface:
t, err := scopeInterface(append, t)
if err != nil {
return t, false, err
}
return d.scopeTableTarget(append, t, name)
t := scopeInterface(shouldAppend, t)
return d.scopeTableTarget(shouldAppend, t, name)
case reflect.Ptr:
t, err := scopePtr(t)
if err != nil {
return t, false, err
}
return d.scopeTableTarget(append, t, name)
t := scopePtr(t)
return d.scopeTableTarget(shouldAppend, t, name)
case reflect.Slice:
t, err := scopeSlice(append, t)
if err != nil {
return t, false, err
}
append = false
return d.scopeTableTarget(append, t, name)
t := scopeSlice(shouldAppend, t)
shouldAppend = false
return d.scopeTableTarget(shouldAppend, t, name)
case reflect.Array:
t, err := d.scopeArray(append, t)
t, err := d.scopeArray(shouldAppend, t)
if err != nil {
return t, false, err
}
append = false
return d.scopeTableTarget(append, t, name)
shouldAppend = false
return d.scopeTableTarget(shouldAppend, t, name)
// Terminal kinds
case reflect.Struct:
return scopeStruct(x, name)
case reflect.Map:
@@ -371,38 +358,33 @@ func (d *decoder) scopeTableTarget(append bool, t target, name string) (target,
return scopeMap(x, name)
default:
panic(fmt.Errorf("can't scope on a %s", x.Kind()))
panic(fmt.Sprintf("can't scope on a %s", x.Kind()))
}
}
func scopeInterface(append bool, t target) (target, error) {
err := initInterface(append, t)
if err != nil {
return t, err
}
return interfaceTarget{t}, nil
func scopeInterface(shouldAppend bool, t target) target {
initInterface(shouldAppend, t)
return interfaceTarget{t}
}
func scopePtr(t target) (target, error) {
err := initPtr(t)
if err != nil {
return t, err
}
return valueTarget(t.get().Elem()), nil
func scopePtr(t target) target {
initPtr(t)
return valueTarget(t.get().Elem())
}
func initPtr(t target) error {
func initPtr(t target) {
x := t.get()
if !x.IsNil() {
return nil
return
}
return t.set(reflect.New(x.Type().Elem()))
t.set(reflect.New(x.Type().Elem()))
}
// initInterface makes sure that the interface pointed at by the target is not
// nil.
// Returns the target to the initialized value of the target.
func initInterface(append bool, t target) error {
func initInterface(shouldAppend bool, t target) {
x := t.get()
if x.Kind() != reflect.Interface {
@@ -410,45 +392,41 @@ func initInterface(append bool, t target) error {
}
if !x.IsNil() && (x.Elem().Type() == sliceInterfaceType || x.Elem().Type() == mapStringInterfaceType) {
return nil
return
}
var newElement reflect.Value
if append {
if shouldAppend {
newElement = reflect.MakeSlice(sliceInterfaceType, 0, 0)
} else {
newElement = reflect.MakeMap(mapStringInterfaceType)
}
err := t.set(newElement)
if err != nil {
return err
}
return nil
t.set(newElement)
}
func scopeSlice(append bool, t target) (target, error) {
func scopeSlice(shouldAppend bool, t target) target {
v := t.get()
if append {
if shouldAppend {
newElem := reflect.New(v.Type().Elem())
newSlice := reflect.Append(v, newElem.Elem())
err := t.set(newSlice)
if err != nil {
return t, err
}
t.set(newSlice)
v = t.get()
}
return valueTarget(v.Index(v.Len() - 1)), nil
return valueTarget(v.Index(v.Len() - 1))
}
func (d *decoder) scopeArray(append bool, t target) (target, error) {
func (d *decoder) scopeArray(shouldAppend bool, t target) (target, error) {
v := t.get()
idx := d.arrayIndex(append, v)
idx := d.arrayIndex(shouldAppend, v)
if idx >= v.Len() {
return nil, fmt.Errorf("not enough space in the array")
return nil, fmt.Errorf("toml: impossible to insert element beyond array's size: %d", v.Len())
}
return valueTarget(v.Index(idx)), nil
@@ -460,8 +438,9 @@ func scopeMap(v reflect.Value, name string) (target, bool, error) {
keyType := v.Type().Key()
if !k.Type().AssignableTo(keyType) {
if !k.Type().ConvertibleTo(keyType) {
return nil, false, fmt.Errorf("cannot convert string into map key type %s", keyType)
return nil, false, fmt.Errorf("toml: cannot convert map key of type %s to expected type %s", k.Type(), keyType)
}
k = k.Convert(keyType)
}
@@ -487,6 +466,7 @@ func (c *fieldPathsCache) get(t reflect.Type) (fieldPathsMap, bool) {
c.l.RLock()
paths, ok := c.m[t]
c.l.RUnlock()
return paths, ok
}
@@ -502,13 +482,14 @@ var globalFieldPathsCache = fieldPathsCache{
}
func scopeStruct(v reflect.Value, name string) (target, bool, error) {
//nolint:godox
// TODO: cache this, and reduce allocations
fieldPaths, ok := globalFieldPathsCache.get(v.Type())
if !ok {
fieldPaths = map[string][]int{}
path := make([]int, 0, 16)
var walk func(reflect.Value)
walk = func(v reflect.Value) {
t := v.Type()
@@ -516,6 +497,7 @@ func scopeStruct(v reflect.Value, name string) (target, bool, error) {
l := len(path)
path = append(path, i)
f := t.Field(i)
if f.Anonymous {
walk(v.Field(i))
} else if f.PkgPath == "" {
@@ -545,6 +527,7 @@ func scopeStruct(v reflect.Value, name string) (target, bool, error) {
if !ok {
path, ok = fieldPaths[strings.ToLower(name)]
}
if !ok {
return nil, false, nil
}
+34 -11
View File
@@ -9,6 +9,8 @@ import (
)
func TestStructTarget_Ensure(t *testing.T) {
t.Parallel()
examples := []struct {
desc string
input reflect.Value
@@ -31,14 +33,23 @@ func TestStructTarget_Ensure(t *testing.T) {
test: func(v reflect.Value, err error) {
assert.NoError(t, err)
require.False(t, v.IsNil())
s := v.Interface().([]string)
s, ok := v.Interface().([]string)
if !ok {
t.Errorf("interface %v should be castable into []string", s)
return
}
assert.Equal(t, []string{"foo"}, s)
},
},
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
d := decoder{}
target, _, err := d.scopeTableTarget(false, valueTarget(e.input), e.name)
require.NoError(t, err)
@@ -50,6 +61,8 @@ func TestStructTarget_Ensure(t *testing.T) {
}
func TestStructTarget_SetString(t *testing.T) {
t.Parallel()
str := "value"
examples := []struct {
@@ -86,7 +99,10 @@ func TestStructTarget_SetString(t *testing.T) {
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
d := decoder{}
target, _, err := d.scopeTableTarget(false, valueTarget(e.input), e.name)
require.NoError(t, err)
@@ -98,7 +114,11 @@ func TestStructTarget_SetString(t *testing.T) {
}
func TestPushNew(t *testing.T) {
t.Parallel()
t.Run("slice of strings", func(t *testing.T) {
t.Parallel()
type Doc struct {
A []string
}
@@ -108,18 +128,18 @@ func TestPushNew(t *testing.T) {
x, _, err := dec.scopeTableTarget(false, valueTarget(reflect.ValueOf(&d).Elem()), "A")
require.NoError(t, err)
n, err := elementAt(x, 0)
require.NoError(t, err)
require.NoError(t, n.setString("hello"))
n := elementAt(x, 0)
n.setString("hello")
require.Equal(t, []string{"hello"}, d.A)
n, err = elementAt(x, 1)
require.NoError(t, err)
require.NoError(t, n.setString("world"))
n = elementAt(x, 1)
n.setString("world")
require.Equal(t, []string{"hello", "world"}, d.A)
})
t.Run("slice of interfaces", func(t *testing.T) {
t.Parallel()
type Doc struct {
A []interface{}
}
@@ -129,19 +149,19 @@ func TestPushNew(t *testing.T) {
x, _, err := dec.scopeTableTarget(false, valueTarget(reflect.ValueOf(&d).Elem()), "A")
require.NoError(t, err)
n, err := elementAt(x, 0)
require.NoError(t, err)
n := elementAt(x, 0)
require.NoError(t, setString(n, "hello"))
require.Equal(t, []interface{}{"hello"}, d.A)
n, err = elementAt(x, 1)
require.NoError(t, err)
n = elementAt(x, 1)
require.NoError(t, setString(n, "world"))
require.Equal(t, []interface{}{"hello", "world"}, d.A)
})
}
func TestScope_Struct(t *testing.T) {
t.Parallel()
examples := []struct {
desc string
input reflect.Value
@@ -167,7 +187,10 @@ func TestScope_Struct(t *testing.T) {
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
t.Parallel()
dec := decoder{}
x, found, err := dec.scopeTableTarget(false, valueTarget(e.input), e.name)
assert.Equal(t, e.found, found)
+27 -13
View File
@@ -30,6 +30,7 @@ func testgenValid(t *testing.T, input string, jsonRef string) {
t.Logf("Input TOML:\n%s", input)
doc := map[string]interface{}{}
err := toml.Unmarshal([]byte(input), &doc)
if err != nil {
t.Fatalf("failed parsing toml: %s", err)
@@ -49,25 +50,23 @@ func testgenValid(t *testing.T, input string, jsonRef string) {
require.Equal(t, refDoc, doc2)
}
type testGenDescNode struct {
Type string
Value interface{}
}
func testgenBuildRefDoc(jsonRef string) map[string]interface{} {
descTree := map[string]interface{}{}
err := json.Unmarshal([]byte(jsonRef), &descTree)
if err != nil {
panic(fmt.Errorf("reference doc should be valid JSON: %s", err))
panic(fmt.Sprintf("reference doc should be valid JSON: %s", err))
}
doc := testGenTranslateDesc(descTree)
if doc == nil {
return map[string]interface{}{}
}
return doc.(map[string]interface{})
}
//nolint:funlen,gocognit,cyclop
func testGenTranslateDesc(input interface{}) interface{} {
a, ok := input.([]interface{})
if ok {
@@ -75,48 +74,61 @@ func testGenTranslateDesc(input interface{}) interface{} {
for i, v := range a {
xs[i] = testGenTranslateDesc(v)
}
return xs
}
d := input.(map[string]interface{})
d, ok := input.(map[string]interface{})
if !ok {
panic(fmt.Sprintf("input should be valid map[string]: %v", input))
}
var dtype string
var dvalue interface{}
var (
dtype string
dvalue interface{}
)
//nolint:nestif
if len(d) == 2 {
dtypeiface, ok := d["type"]
if ok {
dvalue, ok = d["value"]
if ok {
dtype = dtypeiface.(string)
switch dtype {
case "string":
return dvalue.(string)
case "float":
v, err := strconv.ParseFloat(dvalue.(string), 64)
if err != nil {
panic(fmt.Errorf("invalid float '%s': %s", dvalue, err))
panic(fmt.Sprintf("invalid float '%s': %s", dvalue, err))
}
return v
case "integer":
v, err := strconv.ParseInt(dvalue.(string), 10, 64)
if err != nil {
panic(fmt.Errorf("invalid int '%s': %s", dvalue, err))
panic(fmt.Sprintf("invalid int '%s': %s", dvalue, err))
}
return v
case "bool":
return dvalue.(string) == "true"
case "datetime":
dt, err := time.Parse("2006-01-02T15:04:05Z", dvalue.(string))
if err != nil {
panic(fmt.Errorf("invalid datetime '%s': %s", dvalue, err))
panic(fmt.Sprintf("invalid datetime '%s': %s", dvalue, err))
}
return dt
case "array":
if dvalue == nil {
return nil
}
a := dvalue.([]interface{})
xs := make([]interface{}, len(a))
for i, v := range a {
@@ -125,7 +137,8 @@ func testGenTranslateDesc(input interface{}) interface{} {
return xs
}
panic(fmt.Errorf("unknown type: %s", dtype))
panic(fmt.Sprintf("unknown type: %s", dtype))
}
}
}
@@ -134,5 +147,6 @@ func testGenTranslateDesc(input interface{}) interface{} {
for k, v := range d {
dest[k] = testGenTranslateDesc(v)
}
return dest
}
+150 -2
View File
@@ -6,26 +6,36 @@ import (
)
func TestInvalidDatetimeMalformedNoLeads(t *testing.T) {
t.Parallel()
input := `no-leads = 1987-7-05T17:45:00Z`
testgenInvalid(t, input)
}
func TestInvalidDatetimeMalformedNoSecs(t *testing.T) {
t.Parallel()
input := `no-secs = 1987-07-05T17:45Z`
testgenInvalid(t, input)
}
func TestInvalidDatetimeMalformedNoT(t *testing.T) {
t.Parallel()
input := `no-t = 1987-07-0517:45:00Z`
testgenInvalid(t, input)
}
func TestInvalidDatetimeMalformedWithMilli(t *testing.T) {
t.Parallel()
input := `with-milli = 1987-07-5T17:45:00.12Z`
testgenInvalid(t, input)
}
func TestInvalidDuplicateKeyTable(t *testing.T) {
t.Parallel()
input := `[fruit]
type = "apple"
@@ -35,71 +45,97 @@ apple = "yes"`
}
func TestInvalidDuplicateKeys(t *testing.T) {
t.Parallel()
input := `dupe = false
dupe = true`
testgenInvalid(t, input)
}
func TestInvalidDuplicateTables(t *testing.T) {
t.Parallel()
input := `[a]
[a]`
testgenInvalid(t, input)
}
func TestInvalidEmptyImplicitTable(t *testing.T) {
t.Parallel()
input := `[naughty..naughty]`
testgenInvalid(t, input)
}
func TestInvalidEmptyTable(t *testing.T) {
t.Parallel()
input := `[]`
testgenInvalid(t, input)
}
func TestInvalidFloatNoLeadingZero(t *testing.T) {
t.Parallel()
input := `answer = .12345
neganswer = -.12345`
testgenInvalid(t, input)
}
func TestInvalidFloatNoTrailingDigits(t *testing.T) {
t.Parallel()
input := `answer = 1.
neganswer = -1.`
testgenInvalid(t, input)
}
func TestInvalidKeyEmpty(t *testing.T) {
t.Parallel()
input := ` = 1`
testgenInvalid(t, input)
}
func TestInvalidKeyHash(t *testing.T) {
t.Parallel()
input := `a# = 1`
testgenInvalid(t, input)
}
func TestInvalidKeyNewline(t *testing.T) {
t.Parallel()
input := `a
= 1`
testgenInvalid(t, input)
}
func TestInvalidKeyOpenBracket(t *testing.T) {
t.Parallel()
input := `[abc = 1`
testgenInvalid(t, input)
}
func TestInvalidKeySingleOpenBracket(t *testing.T) {
t.Parallel()
input := `[`
testgenInvalid(t, input)
}
func TestInvalidKeySpace(t *testing.T) {
t.Parallel()
input := `a b = 1`
testgenInvalid(t, input)
}
func TestInvalidKeyStartBracket(t *testing.T) {
t.Parallel()
input := `[a]
[xyz = 5
[b]`
@@ -107,31 +143,43 @@ func TestInvalidKeyStartBracket(t *testing.T) {
}
func TestInvalidKeyTwoEquals(t *testing.T) {
t.Parallel()
input := `key= = 1`
testgenInvalid(t, input)
}
func TestInvalidStringBadByteEscape(t *testing.T) {
t.Parallel()
input := `naughty = "\xAg"`
testgenInvalid(t, input)
}
func TestInvalidStringBadEscape(t *testing.T) {
t.Parallel()
input := `invalid-escape = "This string has a bad \a escape character."`
testgenInvalid(t, input)
}
func TestInvalidStringByteEscapes(t *testing.T) {
t.Parallel()
input := `answer = "\x33"`
testgenInvalid(t, input)
}
func TestInvalidStringNoClose(t *testing.T) {
t.Parallel()
input := `no-ending-quote = "One time, at band camp`
testgenInvalid(t, input)
}
func TestInvalidTableArrayImplicit(t *testing.T) {
t.Parallel()
input := "# This test is a bit tricky. It should fail because the first use of\n" +
"# `[[albums.songs]]` without first declaring `albums` implies that `albums`\n" +
"# must be a table. The alternative would be quite weird. Namely, it wouldn't\n" +
@@ -150,46 +198,62 @@ func TestInvalidTableArrayImplicit(t *testing.T) {
}
func TestInvalidTableArrayMalformedBracket(t *testing.T) {
t.Parallel()
input := `[[albums]
name = "Born to Run"`
testgenInvalid(t, input)
}
func TestInvalidTableArrayMalformedEmpty(t *testing.T) {
t.Parallel()
input := `[[]]
name = "Born to Run"`
testgenInvalid(t, input)
}
func TestInvalidTableEmpty(t *testing.T) {
t.Parallel()
input := `[]`
testgenInvalid(t, input)
}
func TestInvalidTableNestedBracketsClose(t *testing.T) {
t.Parallel()
input := `[a]b]
zyx = 42`
testgenInvalid(t, input)
}
func TestInvalidTableNestedBracketsOpen(t *testing.T) {
t.Parallel()
input := `[a[b]
zyx = 42`
testgenInvalid(t, input)
}
func TestInvalidTableWhitespace(t *testing.T) {
t.Parallel()
input := `[invalid key]`
testgenInvalid(t, input)
}
func TestInvalidTableWithPound(t *testing.T) {
t.Parallel()
input := `[key#group]
answer = 42`
testgenInvalid(t, input)
}
func TestInvalidTextAfterArrayEntries(t *testing.T) {
t.Parallel()
input := `array = [
"Is there life after an array separator?", No
"Entry"
@@ -198,21 +262,29 @@ func TestInvalidTextAfterArrayEntries(t *testing.T) {
}
func TestInvalidTextAfterInteger(t *testing.T) {
t.Parallel()
input := `answer = 42 the ultimate answer?`
testgenInvalid(t, input)
}
func TestInvalidTextAfterString(t *testing.T) {
t.Parallel()
input := `string = "Is there life after strings?" No.`
testgenInvalid(t, input)
}
func TestInvalidTextAfterTable(t *testing.T) {
t.Parallel()
input := `[error] this shouldn't be here`
testgenInvalid(t, input)
}
func TestInvalidTextBeforeArraySeparator(t *testing.T) {
t.Parallel()
input := `array = [
"Is there life before an array separator?" No,
"Entry"
@@ -221,6 +293,8 @@ func TestInvalidTextBeforeArraySeparator(t *testing.T) {
}
func TestInvalidTextInArray(t *testing.T) {
t.Parallel()
input := `array = [
"Entry 1",
I don't belong,
@@ -230,6 +304,8 @@ func TestInvalidTextInArray(t *testing.T) {
}
func TestValidArrayEmpty(t *testing.T) {
t.Parallel()
input := `thevoid = [[[[[]]]]]`
jsonRef := `{
"thevoid": { "type": "array", "value": [
@@ -246,6 +322,8 @@ func TestValidArrayEmpty(t *testing.T) {
}
func TestValidArrayNospaces(t *testing.T) {
t.Parallel()
input := `ints = [1,2,3]`
jsonRef := `{
"ints": {
@@ -261,6 +339,8 @@ func TestValidArrayNospaces(t *testing.T) {
}
func TestValidArraysHetergeneous(t *testing.T) {
t.Parallel()
input := `mixed = [[1, 2], ["a", "b"], [1.1, 2.1]]`
jsonRef := `{
"mixed": {
@@ -285,6 +365,8 @@ func TestValidArraysHetergeneous(t *testing.T) {
}
func TestValidArraysNested(t *testing.T) {
t.Parallel()
input := `nest = [["a"], ["b"]]`
jsonRef := `{
"nest": {
@@ -303,6 +385,8 @@ func TestValidArraysNested(t *testing.T) {
}
func TestValidArrays(t *testing.T) {
t.Parallel()
input := `ints = [1, 2, 3]
floats = [1.1, 2.1, 3.1]
strings = ["a", "b", "c"]
@@ -349,6 +433,8 @@ dates = [
}
func TestValidBool(t *testing.T) {
t.Parallel()
input := `t = true
f = false`
jsonRef := `{
@@ -359,6 +445,8 @@ f = false`
}
func TestValidCommentsEverywhere(t *testing.T) {
t.Parallel()
input := `# Top comment.
# Top comment.
# Top comment.
@@ -368,7 +456,7 @@ func TestValidCommentsEverywhere(t *testing.T) {
[group] # Comment
answer = 42 # Comment
# no-extraneous-keys-please = 999
# Inbetween comment.
# In between comment.
more = [ # Comment
# What about multiple # comments?
# Can you handle it?
@@ -399,6 +487,8 @@ more = [ # Comment
}
func TestValidDatetime(t *testing.T) {
t.Parallel()
input := `bestdayever = 1987-07-05T17:45:00Z`
jsonRef := `{
"bestdayever": {"type": "datetime", "value": "1987-07-05T17:45:00Z"}
@@ -407,12 +497,16 @@ func TestValidDatetime(t *testing.T) {
}
func TestValidEmpty(t *testing.T) {
t.Parallel()
input := ``
jsonRef := `{}`
testgenValid(t, input, jsonRef)
}
func TestValidExample(t *testing.T) {
t.Parallel()
input := `best-day-ever = 1987-07-05T17:45:00Z
[numtheory]
@@ -436,6 +530,8 @@ perfection = [6, 28, 496]`
}
func TestValidFloat(t *testing.T) {
t.Parallel()
input := `pi = 3.14
negpi = -3.14`
jsonRef := `{
@@ -446,6 +542,8 @@ negpi = -3.14`
}
func TestValidImplicitAndExplicitAfter(t *testing.T) {
t.Parallel()
input := `[a.b.c]
answer = 42
@@ -465,6 +563,8 @@ better = 43`
}
func TestValidImplicitAndExplicitBefore(t *testing.T) {
t.Parallel()
input := `[a]
better = 43
@@ -484,6 +584,8 @@ answer = 42`
}
func TestValidImplicitGroups(t *testing.T) {
t.Parallel()
input := `[a.b.c]
answer = 42`
jsonRef := `{
@@ -499,6 +601,8 @@ answer = 42`
}
func TestValidInteger(t *testing.T) {
t.Parallel()
input := `answer = 42
neganswer = -42`
jsonRef := `{
@@ -509,6 +613,8 @@ neganswer = -42`
}
func TestValidKeyEqualsNospace(t *testing.T) {
t.Parallel()
input := `answer=42`
jsonRef := `{
"answer": {"type": "integer", "value": "42"}
@@ -517,6 +623,8 @@ func TestValidKeyEqualsNospace(t *testing.T) {
}
func TestValidKeySpace(t *testing.T) {
t.Parallel()
input := `"a b" = 1`
jsonRef := `{
"a b": {"type": "integer", "value": "1"}
@@ -525,6 +633,8 @@ func TestValidKeySpace(t *testing.T) {
}
func TestValidKeySpecialChars(t *testing.T) {
t.Parallel()
input := "\"~!@$^&*()_+-`1234567890[]|/?><.,;:'\" = 1\n"
jsonRef := "{\n" +
" \"~!@$^&*()_+-`1234567890[]|/?><.,;:'\": {\n" +
@@ -535,6 +645,8 @@ func TestValidKeySpecialChars(t *testing.T) {
}
func TestValidLongFloat(t *testing.T) {
t.Parallel()
input := `longpi = 3.141592653589793
neglongpi = -3.141592653589793`
jsonRef := `{
@@ -545,6 +657,8 @@ neglongpi = -3.141592653589793`
}
func TestValidLongInteger(t *testing.T) {
t.Parallel()
input := `answer = 9223372036854775807
neganswer = -9223372036854775808`
jsonRef := `{
@@ -555,6 +669,8 @@ neganswer = -9223372036854775808`
}
func TestValidMultilineString(t *testing.T) {
t.Parallel()
input := `multiline_empty_one = """"""
multiline_empty_two = """
"""
@@ -612,6 +728,8 @@ equivalent_three = """\
}
func TestValidRawMultilineString(t *testing.T) {
t.Parallel()
input := `oneline = '''This string has a ' quote character.'''
firstnl = '''
This string has a ' quote character.'''
@@ -639,6 +757,8 @@ in it.'''`
}
func TestValidRawString(t *testing.T) {
t.Parallel()
input := `backspace = 'This string has a \b backspace character.'
tab = 'This string has a \t tab character.'
newline = 'This string has a \n new line character.'
@@ -680,6 +800,8 @@ backslash = 'This string has a \\ backslash character.'`
}
func TestValidStringEmpty(t *testing.T) {
t.Parallel()
input := `answer = ""`
jsonRef := `{
"answer": {
@@ -691,6 +813,8 @@ func TestValidStringEmpty(t *testing.T) {
}
func TestValidStringEscapes(t *testing.T) {
t.Parallel()
input := `backspace = "This string has a \b backspace character."
tab = "This string has a \t tab character."
newline = "This string has a \n new line character."
@@ -752,6 +876,8 @@ notunicode4 = "This string does not have a unicode \\\u0075 escape."`
}
func TestValidStringSimple(t *testing.T) {
t.Parallel()
input := `answer = "You are not drinking enough whisky."`
jsonRef := `{
"answer": {
@@ -763,6 +889,8 @@ func TestValidStringSimple(t *testing.T) {
}
func TestValidStringWithPound(t *testing.T) {
t.Parallel()
input := `pound = "We see no # comments here."
poundcomment = "But there are # some comments here." # Did I # mess you up?`
jsonRef := `{
@@ -776,6 +904,8 @@ poundcomment = "But there are # some comments here." # Did I # mess you up?`
}
func TestValidTableArrayImplicit(t *testing.T) {
t.Parallel()
input := `[[albums.songs]]
name = "Glory Days"`
jsonRef := `{
@@ -789,6 +919,8 @@ name = "Glory Days"`
}
func TestValidTableArrayMany(t *testing.T) {
t.Parallel()
input := `[[people]]
first_name = "Bruce"
last_name = "Springsteen"
@@ -820,6 +952,8 @@ last_name = "Seger"`
}
func TestValidTableArrayNest(t *testing.T) {
t.Parallel()
input := `[[albums]]
name = "Born to Run"
@@ -831,7 +965,7 @@ name = "Born to Run"
[[albums]]
name = "Born in the USA"
[[albums.songs]]
name = "Glory Days"
@@ -859,6 +993,8 @@ name = "Born in the USA"
}
func TestValidTableArrayOne(t *testing.T) {
t.Parallel()
input := `[[people]]
first_name = "Bruce"
last_name = "Springsteen"`
@@ -874,6 +1010,8 @@ last_name = "Springsteen"`
}
func TestValidTableEmpty(t *testing.T) {
t.Parallel()
input := `[a]`
jsonRef := `{
"a": {}
@@ -882,6 +1020,8 @@ func TestValidTableEmpty(t *testing.T) {
}
func TestValidTableSubEmpty(t *testing.T) {
t.Parallel()
input := `[a]
[a.b]`
jsonRef := `{
@@ -891,6 +1031,8 @@ func TestValidTableSubEmpty(t *testing.T) {
}
func TestValidTableWhitespace(t *testing.T) {
t.Parallel()
input := `["valid key"]`
jsonRef := `{
"valid key": {}
@@ -899,6 +1041,8 @@ func TestValidTableWhitespace(t *testing.T) {
}
func TestValidTableWithPound(t *testing.T) {
t.Parallel()
input := `["key#group"]
answer = 42`
jsonRef := `{
@@ -910,6 +1054,8 @@ answer = 42`
}
func TestValidUnicodeEscape(t *testing.T) {
t.Parallel()
input := `answer4 = "\u03B4"
answer8 = "\U000003B4"`
jsonRef := `{
@@ -920,6 +1066,8 @@ answer8 = "\U000003B4"`
}
func TestValidUnicodeLiteral(t *testing.T) {
t.Parallel()
input := `answer = "δ"`
jsonRef := `{
"answer": {"type": "string", "value": "δ"}
+176 -57
View File
@@ -2,6 +2,7 @@ package toml
import (
"encoding"
"errors"
"fmt"
"io"
"io/ioutil"
@@ -13,10 +14,14 @@ import (
"github.com/pelletier/go-toml/v2/internal/unsafe"
)
// Unmarshal deserializes a TOML document into a Go value.
//
// It is a shortcut for Decoder.Decode() with the default options.
func Unmarshal(data []byte, v interface{}) error {
p := parser{}
p.Reset(data)
d := decoder{}
return d.FromParser(&p, v)
}
@@ -46,16 +51,45 @@ func (d *Decoder) SetStrict(strict bool) {
// Decode the whole content of r into v.
//
// When a TOML local date is decoded into a time.Time, its value is represented
// in time.Local timezone.
// By default, values in the document that don't exist in the target Go value
// are ignored. See Decoder.SetStrict() to change this behavior.
//
// When a TOML local date, time, or date-time is decoded into a time.Time, its
// value is represented in time.Local timezone. Otherwise the approriate Local*
// structure is used.
//
// Empty tables decoded in an interface{} create an empty initialized
// map[string]interface{}.
//
// Types implementing the encoding.TextUnmarshaler interface are decoded from a
// TOML string.
//
// When decoding a number, go-toml will return an error if the number is out of
// bounds for the target type (which includes negative numbers when decoding
// into an unsigned int).
//
// Type mapping
//
// List of supported TOML types and their associated accepted Go types:
//
// String -> string
// Integer -> uint*, int*, depending on size
// Float -> float*, depending on size
// Boolean -> bool
// Offset Date-Time -> time.Time
// Local Date-time -> LocalDateTime, time.Time
// Local Date -> LocalDate, time.Time
// Local Time -> LocalTime, time.Time
// Array -> slice and array, depending on elements types
// Table -> map and struct
// Inline Table -> same as Table
// Array of Tables -> same as Array and Table
func (d *Decoder) Decode(v interface{}) error {
b, err := ioutil.ReadAll(d.r)
if err != nil {
return err
return fmt.Errorf("toml: %w", err)
}
p := parser{}
p.Reset(b)
dec := decoder{
@@ -63,6 +97,7 @@ func (d *Decoder) Decode(v interface{}) error {
Enabled: d.strict,
},
}
return dec.FromParser(&p, v)
}
@@ -77,7 +112,7 @@ type decoder struct {
strict strict
}
func (d *decoder) arrayIndex(append bool, v reflect.Value) int {
func (d *decoder) arrayIndex(shouldAppend bool, v reflect.Value) int {
if d.arrayIndexes == nil {
d.arrayIndexes = make(map[reflect.Value]int, 1)
}
@@ -86,23 +121,23 @@ func (d *decoder) arrayIndex(append bool, v reflect.Value) int {
if !ok {
d.arrayIndexes[v] = 0
} else if append {
} else if shouldAppend {
idx++
d.arrayIndexes[v] = idx
}
return idx
}
func (d *decoder) FromParser(p *parser, v interface{}) error {
err := d.fromParser(p, v)
if err != nil {
de, ok := err.(*decodeError)
if ok {
err = wrapDecodeError(p.data, de)
}
}
if err == nil {
err = d.strict.Error(p.data)
return d.strict.Error(p.data)
}
var e *decodeError
if errors.As(err, &e) {
return wrapDecodeError(p.data, e)
}
return err
@@ -110,29 +145,38 @@ func (d *decoder) FromParser(p *parser, v interface{}) error {
func keyLocation(node ast.Node) []byte {
k := node.Key()
hasOne := k.Next()
if !hasOne {
panic("should not be called with empty key")
}
start := k.Node().Data
end := k.Node().Data
for k.Next() {
end = k.Node().Data
}
return unsafe.BytesRange(start, end)
}
//nolint:funlen,cyclop
func (d *decoder) fromParser(p *parser, v interface{}) error {
r := reflect.ValueOf(v)
if r.Kind() != reflect.Ptr {
return fmt.Errorf("need to target a pointer, not %s", r.Kind())
}
if r.IsNil() {
return fmt.Errorf("target pointer must be non-nil")
return fmt.Errorf("toml: decoding can only be performed into a pointer, not %s", r.Kind())
}
var skipUntilTable bool
var root target = valueTarget(r.Elem())
if r.IsNil() {
return fmt.Errorf("toml: decoding pointer target cannot be nil")
}
var (
skipUntilTable bool
root target = valueTarget(r.Elem())
)
current := root
for p.NextExpression() {
@@ -148,12 +192,14 @@ func (d *decoder) fromParser(p *parser, v interface{}) error {
}
var found bool
switch node.Kind {
case ast.KeyValue:
err = d.unmarshalKeyValue(current, node)
found = true
case ast.Table:
d.strict.EnterTable(node)
current, found, err = d.scopeWithKey(root, node.Key())
if err == nil && found {
// In case this table points to an interface,
@@ -167,7 +213,7 @@ func (d *decoder) fromParser(p *parser, v interface{}) error {
d.strict.EnterArrayTable(node)
current, found, err = d.scopeWithArrayTable(root, node.Key())
default:
panic(fmt.Errorf("this should not be a top level node type: %s", node.Kind))
panic(fmt.Sprintf("this should not be a top level node type: %s", node.Kind))
}
if err != nil {
@@ -176,6 +222,7 @@ func (d *decoder) fromParser(p *parser, v interface{}) error {
if !found {
skipUntilTable = true
d.strict.MissingTable(node)
}
}
@@ -192,38 +239,49 @@ func (d *decoder) fromParser(p *parser, v interface{}) error {
// When encountering slices, it should always use its last element, and error
// if the slice does not have any.
func (d *decoder) scopeWithKey(x target, key ast.Iterator) (target, bool, error) {
var err error
found := true
var (
err error
found bool
)
for key.Next() {
n := key.Node()
x, found, err = d.scopeTableTarget(false, x, string(n.Data))
if err != nil || !found {
return nil, found, err
}
}
return x, true, nil
}
//nolint:cyclop
// scopeWithArrayTable performs target scoping when unmarshaling an
// ast.ArrayTable node.
//
// It is the same as scopeWithKey, but when scoping the last part of the key
// it creates a new element in the array instead of using the last one.
func (d *decoder) scopeWithArrayTable(x target, key ast.Iterator) (target, bool, error) {
var err error
found := true
var (
err error
found bool
)
for key.Next() {
n := key.Node()
if !n.Next().Valid() { // want to stop at one before last
break
}
x, found, err = d.scopeTableTarget(false, x, string(n.Data))
if err != nil || !found {
return nil, found, err
}
}
n := key.Node()
x, found, err = d.scopeTableTarget(false, x, string(n.Data))
if err != nil || !found {
return x, found, err
@@ -232,26 +290,21 @@ func (d *decoder) scopeWithArrayTable(x target, key ast.Iterator) (target, bool,
v := x.get()
if v.Kind() == reflect.Ptr {
x, err = scopePtr(x)
if err != nil {
return x, false, err
}
x = scopePtr(x)
v = x.get()
}
if v.Kind() == reflect.Interface {
x, err = scopeInterface(true, x)
if err != nil {
return x, found, err
}
x = scopeInterface(true, x)
v = x.get()
}
switch v.Kind() {
case reflect.Slice:
x, err = scopeSlice(true, x)
x = scopeSlice(true, x)
case reflect.Array:
x, err = d.scopeArray(true, x)
default:
}
return x, found, err
@@ -271,6 +324,7 @@ func (d *decoder) unmarshalKeyValue(x target, node ast.Node) error {
// A struct in the path was not found. Skip this value.
if !found {
d.strict.MissingField(node)
return nil
}
@@ -286,31 +340,43 @@ func tryTextUnmarshaler(x target, node ast.Node) (bool, error) {
return false, nil
}
// Special case for time, becase we allow to unmarshal to it from
// Special case for time, because we allow to unmarshal to it from
// different kind of AST nodes.
if v.Type() == timeType {
return false, nil
}
if v.Type().Implements(textUnmarshalerType) {
return true, v.Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
err := v.Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
if err != nil {
return false, newDecodeError(node.Data, "error calling UnmarshalText: %w", err)
}
return true, nil
}
if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) {
return true, v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
if err != nil {
return false, newDecodeError(node.Data, "error calling UnmarshalText: %w", err)
}
return true, nil
}
return false, nil
}
//nolint:cyclop
func (d *decoder) unmarshalValue(x target, node ast.Node) error {
v := x.get()
if v.Kind() == reflect.Ptr {
if !v.Elem().IsValid() {
err := x.set(reflect.New(v.Type().Elem()))
if err != nil {
return err
}
x.set(reflect.New(v.Type().Elem()))
v = x.get()
}
return d.unmarshalValue(valueTarget(v.Elem()), node)
}
@@ -339,85 +405,113 @@ func (d *decoder) unmarshalValue(x target, node ast.Node) error {
case ast.LocalDate:
return unmarshalLocalDate(x, node)
default:
panic(fmt.Errorf("unhandled unmarshalValue kind %s", node.Kind))
panic(fmt.Sprintf("unhandled node kind %s", node.Kind))
}
}
func unmarshalLocalDate(x target, node ast.Node) error {
assertNode(ast.LocalDate, node)
v, err := parseLocalDate(node.Data)
if err != nil {
return err
}
return setDate(x, v)
setDate(x, v)
return nil
}
func unmarshalLocalDateTime(x target, node ast.Node) error {
assertNode(ast.LocalDateTime, node)
v, rest, err := parseLocalDateTime(node.Data)
if err != nil {
return err
}
if len(rest) > 0 {
return newDecodeError(rest, "extra characters at the end of a local date time")
}
return setLocalDateTime(x, v)
setLocalDateTime(x, v)
return nil
}
func unmarshalDateTime(x target, node ast.Node) error {
assertNode(ast.DateTime, node)
v, err := parseDateTime(node.Data)
if err != nil {
return err
}
return setDateTime(x, v)
setDateTime(x, v)
return nil
}
func setLocalDateTime(x target, v LocalDateTime) error {
return x.set(reflect.ValueOf(v))
func setLocalDateTime(x target, v LocalDateTime) {
if x.get().Type() == timeType {
cast := v.In(time.Local)
setDateTime(x, cast)
return
}
x.set(reflect.ValueOf(v))
}
func setDateTime(x target, v time.Time) error {
return x.set(reflect.ValueOf(v))
func setDateTime(x target, v time.Time) {
x.set(reflect.ValueOf(v))
}
var timeType = reflect.TypeOf(time.Time{})
func setDate(x target, v LocalDate) error {
func setDate(x target, v LocalDate) {
if x.get().Type() == timeType {
cast := v.In(time.Local)
return setDateTime(x, cast)
setDateTime(x, cast)
return
}
return x.set(reflect.ValueOf(v))
x.set(reflect.ValueOf(v))
}
func unmarshalString(x target, node ast.Node) error {
assertNode(ast.String, node)
return setString(x, string(node.Data))
}
func unmarshalBool(x target, node ast.Node) error {
assertNode(ast.Bool, node)
v := node.Data[0] == 't'
return setBool(x, v)
}
func unmarshalInteger(x target, node ast.Node) error {
assertNode(ast.Integer, node)
v, err := parseInteger(node.Data)
if err != nil {
return err
}
return setInt64(x, v)
}
func unmarshalFloat(x target, node ast.Node) error {
assertNode(ast.Float, node)
v, err := parseFloat(node.Data)
if err != nil {
return err
}
return setFloat64(x, v)
}
@@ -429,11 +523,13 @@ func (d *decoder) unmarshalInlineTable(x target, node ast.Node) error {
it := node.Children()
for it.Next() {
n := it.Node()
err := d.unmarshalKeyValue(x, n)
if err != nil {
return err
}
}
return nil
}
@@ -445,23 +541,46 @@ func (d *decoder) unmarshalArray(x target, node ast.Node) error {
return err
}
it := node.Children()
// Special work around when unmarshaling into an array.
// If the array is not addressable, for example when stored as a value in a
// map, calling elementAt in the inner function would fail.
// Instead, we allocate a new array that will be filled then inserted into
// the container.
// This problem does not exist with slices because they are addressable.
// There may be a better way of doing this, but it is not obvious to me
// with the target system.
if x.get().Kind() == reflect.Array {
container := x
newArrayPtr := reflect.New(x.get().Type())
x = valueTarget(newArrayPtr.Elem())
defer func() {
container.set(newArrayPtr.Elem())
}()
}
return d.unmarshalArrayInner(x, node)
}
func (d *decoder) unmarshalArrayInner(x target, node ast.Node) error {
idx := 0
it := node.Children()
for it.Next() {
n := it.Node()
v, err := elementAt(x, idx)
if err != nil {
return err
}
v := elementAt(x, idx)
if v == nil {
// when we go out of bound for an array just stop processing it to
// mimic encoding/json
break
}
err = d.unmarshalValue(v, n)
err := d.unmarshalValue(v, n)
if err != nil {
return err
}
idx++
}
return nil
@@ -469,6 +588,6 @@ func (d *decoder) unmarshalArray(x target, node ast.Node) error {
func assertNode(expected ast.Kind, node ast.Node) {
if node.Kind != expected {
panic(fmt.Errorf("expected node of kind %s, not %s", expected, node.Kind))
panic(fmt.Sprintf("expected node of kind %s, not %s", expected, node.Kind))
}
}
+799 -25
View File
File diff suppressed because it is too large Load Diff