Compare commits

...

102 Commits

Author SHA1 Message Date
Thomas Pelletier b371733c67 Make all nodes contain Raw 2022-08-22 21:05:41 -04:00
Thomas Pelletier 64dcce07ea WIP 2022-08-22 23:04:44 +00:00
Thomas Pelletier 28f1efc7d3 Decode: don't break on non-struct embed field (#810) 2022-08-22 18:39:11 -04:00
Piotr Buliński 7d69e4a728 Add missing '+build' comment to fuzz_test.go (#809) 2022-08-22 14:05:37 -04:00
Thomas Pelletier e46d245c09 Decode: don't crash on embedded nil pointers (#808)
Also has the perks of reducing the overhead of FindByIndex:

```
name                                old time/op    new time/op    delta
UnmarshalDataset/config-32            17.0ms ± 1%    17.0ms ± 1%    ~     (p=1.000 n=5+5)
UnmarshalDataset/canada-32            71.6ms ± 1%    71.4ms ± 1%    ~     (p=1.000 n=5+5)
UnmarshalDataset/citm_catalog-32      24.2ms ± 3%    23.5ms ± 2%  -3.03%  (p=0.032 n=5+5)
UnmarshalDataset/twitter-32           9.37ms ± 1%    9.09ms ± 2%  -2.97%  (p=0.032 n=5+5)
UnmarshalDataset/code-32              75.4ms ± 2%    74.9ms ± 0%    ~     (p=0.222 n=5+5)
UnmarshalDataset/example-32            147µs ±10%     136µs ± 1%  -7.14%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-32     512ns ± 2%     500ns ± 0%  -2.35%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/map-32        721ns ± 2%     702ns ± 1%  -2.68%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/struct-32     40.1µs ± 0%    39.6µs ± 0%  -1.30%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/map-32        62.3µs ± 1%    60.6µs ± 0%  -2.83%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-32          10.8µs ± 1%    10.5µs ± 1%  -2.86%  (p=0.008 n=5+5)

name                                old speed      new speed      delta
UnmarshalDataset/config-32          61.8MB/s ± 1%  61.8MB/s ± 1%    ~     (p=1.000 n=5+5)
UnmarshalDataset/canada-32          30.8MB/s ± 1%  30.8MB/s ± 1%    ~     (p=1.000 n=5+5)
UnmarshalDataset/citm_catalog-32    23.0MB/s ± 3%  23.8MB/s ± 2%  +3.09%  (p=0.032 n=5+5)
UnmarshalDataset/twitter-32         47.2MB/s ± 1%  48.6MB/s ± 2%  +3.09%  (p=0.032 n=5+5)
UnmarshalDataset/code-32            35.6MB/s ± 2%  35.9MB/s ± 0%    ~     (p=0.222 n=5+5)
UnmarshalDataset/example-32         55.3MB/s ±10%  59.4MB/s ± 1%  +7.36%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-32  21.5MB/s ± 2%  22.0MB/s ± 0%  +2.41%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/map-32     15.2MB/s ± 2%  15.7MB/s ± 1%  +2.74%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/struct-32    131MB/s ± 0%   132MB/s ± 0%  +1.31%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/map-32      84.1MB/s ± 1%  86.6MB/s ± 0%  +2.91%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-32        50.6MB/s ± 1%  52.1MB/s ± 1%  +2.93%  (p=0.008 n=5+5)

name                                old alloc/op   new alloc/op   delta
UnmarshalDataset/config-32            5.86MB ± 0%    5.86MB ± 0%    ~     (p=0.579 n=5+5)
UnmarshalDataset/canada-32            83.0MB ± 0%    83.0MB ± 0%    ~     (p=0.651 n=5+5)
UnmarshalDataset/citm_catalog-32      34.7MB ± 0%    34.7MB ± 0%    ~     (p=0.548 n=5+5)
UnmarshalDataset/twitter-32           12.7MB ± 0%    12.7MB ± 0%    ~     (p=1.000 n=5+5)
UnmarshalDataset/code-32              22.2MB ± 0%    22.2MB ± 0%    ~     (p=0.841 n=5+5)
UnmarshalDataset/example-32            186kB ± 0%     186kB ± 0%    ~     (p=0.111 n=5+5)
Unmarshal/SimpleDocument/struct-32      805B ± 0%      805B ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/map-32       1.13kB ± 0%    1.13kB ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/struct-32     20.9kB ± 0%    20.9kB ± 0%    ~     (p=0.643 n=5+5)
Unmarshal/ReferenceFile/map-32        38.3kB ± 0%    38.3kB ± 0%    ~     (p=0.397 n=5+5)
Unmarshal/HugoFrontMatter-32          7.44kB ± 0%    7.44kB ± 0%    ~     (all equal)

name                                old allocs/op  new allocs/op  delta
UnmarshalDataset/config-32              227k ± 0%      227k ± 0%    ~     (p=1.000 n=5+5)
UnmarshalDataset/canada-32              782k ± 0%      782k ± 0%    ~     (all equal)
UnmarshalDataset/citm_catalog-32        192k ± 0%      192k ± 0%    ~     (p=0.968 n=4+5)
UnmarshalDataset/twitter-32            56.9k ± 0%     56.9k ± 0%    ~     (p=0.429 n=4+5)
UnmarshalDataset/code-32               1.05M ± 0%     1.05M ± 0%    ~     (p=0.556 n=4+5)
UnmarshalDataset/example-32            1.36k ± 0%     1.36k ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/struct-32      9.00 ± 0%      9.00 ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/map-32         13.0 ± 0%      13.0 ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/struct-32        183 ± 0%       183 ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/map-32           642 ± 0%       642 ± 0%    ~     (all equal)
Unmarshal/HugoFrontMatter-32             141 ± 0%       141 ± 0%    ~     (all equal)
```

Fixes #807
2022-08-20 21:24:03 -04:00
Thomas Pelletier 7baa23f493 Decode: error on array table mismatched type (#804)
Prevent the decoder from continuing if it encounters a type it cannot decode an
array table into.

Fixes #799
2022-08-15 16:38:07 -04:00
Thomas Pelletier 2d8433b69e Encode: don't inherit omitempty (#803)
Fixes #786.
2022-08-15 11:29:46 -04:00
Thomas Pelletier 67bc5422f3 Go 1.19 (#802) 2022-08-15 10:56:33 -04:00
Thomas Pelletier fb6d1d6c2b Marshal: define and fix newlines behavior when using omitempty (#798)
Ref #786
2022-07-24 15:40:20 -04:00
dependabot[bot] d017a6dc89 build(deps): bump github.com/stretchr/testify from 1.7.5 to 1.8.0 (#795) 2022-06-29 09:51:28 -04:00
dependabot[bot] d6d3196163 build(deps): bump github.com/stretchr/testify from 1.7.4 to 1.7.5 (#794) 2022-06-24 12:49:56 -04:00
dependabot[bot] 41718a6db3 build(deps): bump github.com/stretchr/testify from 1.7.2 to 1.7.4 (#793)
Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.2 to 1.7.4.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.7.2...v1.7.4)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-06-21 08:32:13 -04:00
Thomas Pelletier 216628222f Build arm + arm64 binaries for linux and windows (#790)
* Build arm + arm64 binaries for linux and windows

* Type MaxInt64 to avoid overflow on 32 bits arch

On a 32 bits arch, math.MaxIn64 is interpreted as an int, and therefore
overflows. This causes compilation to build on those platforms.
2022-06-08 18:05:42 -04:00
dependabot[bot] 322e0b15d2 build(deps): bump github.com/stretchr/testify from 1.7.1 to 1.7.2 (#788)
Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.1 to 1.7.2.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.7.1...v1.7.2)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-06-08 17:44:59 -04:00
Thomas Pelletier 85bfc0ed51 Encode: add bound check for uint64 > math.Int64 (#785)
As brought up on #782, there is an asymetry between numbers go-toml can encode
and decode. Specifically, unsigned numbers larger than `math.Int64` could be
encoded but not decoded.

We considered two options: allow decoding of those numbers, or prevent those
numbers to be decoded.

Ultimately we decided to narrow the range of numbers to be encoded. The TOML
specification only requires parsers to support at least the 64 bits integer
range. Allowing larger numbers would create non-standard TOML documents, which
may not be readable (at best) by other implementations. It is a safer default to
generate documents valid by default. People who wish to deal with larger numbers
can provide a custom type implementing `encoding.TextMarshaler`.

Refs #781
2022-05-31 22:10:41 -04:00
dependabot[bot] 295a720dfb build(deps): bump goreleaser/goreleaser-action from 2 to 3 (#783)
Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 2 to 3.
- [Release notes](https://github.com/goreleaser/goreleaser-action/releases)
- [Commits](https://github.com/goreleaser/goreleaser-action/compare/v2...v3)

---
updated-dependencies:
- dependency-name: goreleaser/goreleaser-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-05-23 15:05:00 +02:00
Thomas Pelletier 0a422e3dbd Decoder: check max uint on 32 bit platforms (#778)
Fixes #777
2022-05-10 15:43:26 +02:00
Thomas Pelletier 627dade0c7 Encode: support comment on array tables (#776)
Fixes #774
2022-05-10 15:17:36 +02:00
Thomas Pelletier b2e0231cc9 Encode: fix multiline comment (#775)
Fixes #768
2022-05-10 14:53:26 +02:00
dependabot[bot] ba95863cd3 build(deps): bump docker/login-action from 1 to 2 (#771)
Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2.
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](https://github.com/docker/login-action/compare/v1...v2)

---
updated-dependencies:
- dependency-name: docker/login-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-05-09 18:59:02 +02:00
Erik Hartwig db679df765 Typo in README.md fix (#770)
Change "document was not prevent in the target structure" to "document was not present in the target structure".
2022-05-09 18:46:46 +02:00
Thomas Pelletier c5ca2c682b Fix embedded struct with explicit field name (#773)
Fixes #772
2022-05-09 18:45:02 +02:00
Thomas Pelletier ed80712cb4 Copy version policy from v1 2022-04-28 11:55:39 -04:00
dependabot[bot] b24772942d build(deps): bump github/codeql-action from 1 to 2 (#764)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1 to 2.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v1...v2)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-04-28 08:24:14 -04:00
dependabot[bot] 9501a05ed7 build(deps): bump actions/checkout from 2 to 3 (#765)
Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-04-28 08:23:56 -04:00
Thomas Pelletier 171a592663 Prepare repository for v2.0.0 (#762) 2022-04-27 21:29:16 -04:00
Thomas Pelletier 5aaf5ef13b Fix internal entry size test (#761)
The test meant to assert that the size of entry does not grow beyond
what it is today. 48 is the current size in bytes on x64. On 32-bit
platform that value should be less. As written the test was doing the
opposite comparison.

Fixes #760
2022-04-24 18:50:54 -04:00
Thomas Pelletier adacebd8c7 Update benchmarks in README (#756) 2022-04-10 22:24:19 -04:00
Thomas Pelletier 8bbb673431 Fuzzing setup and fixes (#755)
* encode: fix localdate formatting
* encode: fix empty key marshaling
* encode: fix invalid quotation of time.Time
* encode: ensure control chars are escaped
* decode: always use UTC for zero tz
* encode: check for invalid characters in keys
* encode: always construct map for empty array tables
* fuzz: add go 1.18 fuzz test
* encode: handle NaNs
* encode: allow new lines in quoted keys
* encode: never emit table inside array
* encode: don't capitalize inf
2022-04-10 21:37:12 -04:00
Thomas Pelletier 2377ac4bc0 encode: fix embedded interfaces (#753)
Resolve marshaling regression when handling an embedded interface in a
struct.

Fixes #752
2022-04-08 09:25:54 -04:00
Thomas Pelletier f5cc8c49eb decoder: remove mention of UnmarshalText in errors (#751)
Fixes #737
2022-04-07 21:58:19 -04:00
Thomas Pelletier 89d7b412d8 decode: allow subtables to be defined later (#750)
Fixes #739
2022-04-07 21:49:16 -04:00
Thomas Pelletier 88a8aecdd4 tools: display error context when it exists (#749)
For example when failing to decode toml, display the context around the
error and the location of the problem.
2022-04-07 20:33:09 -04:00
Thomas Pelletier 9804fc57e0 decoder: support \e escape sequence (#748) 2022-04-07 20:18:30 -04:00
Thomas Pelletier 068279f13b encode: respect stdlib rules for embedded structs (#747) 2022-04-07 19:51:09 -04:00
Thomas Pelletier b9edbeb611 Update testify dependency (#746) 2022-03-30 15:10:57 -04:00
Thomas Pelletier a97c9317d4 Go 1.18 (#745) 2022-03-23 10:26:12 -04:00
Gregory Oschwald 3229a0abfb Decode: convert table key to correct type (#741)
Fixes #740.
2022-03-02 09:24:01 -05:00
Thomas Pelletier 3f5d8a6b06 Mention removal of go-toml/query (#736)
Fixes #722
2022-01-07 18:58:33 -05:00
Cameron Moore 146f70ea8a Decode: use cleaned byte slice throughout parseFloat (#735)
Fixes #734
2022-01-06 14:34:27 -05:00
Thomas Pelletier e83cf535f5 Decoder: rename SetStrict to DisallowUnknownFields (#731) 2022-01-02 14:32:34 -05:00
Thomas Pelletier c3ba3ef97a readme: add docker image 2022-01-01 09:53:14 -05:00
Thomas Pelletier 7ee3c8ff25 build: login to github repository 2022-01-01 09:24:52 -05:00
Thomas Pelletier 1e85aa6d78 build: add contents permissions 2021-12-31 20:47:50 -05:00
Thomas Pelletier 46fa3225e2 build: allows the token to write packages 2021-12-31 20:25:11 -05:00
Thomas Pelletier 4d51831dab build: replace grc.io with ghcr.io 2021-12-31 20:10:36 -05:00
Thomas Pelletier 5a1a96cb2d build: pass the github token to goreleaser 2021-12-31 20:03:18 -05:00
Thomas Pelletier ea9040ae83 build: change workflow reference to v2 2021-12-31 19:57:41 -05:00
Thomas Pelletier 2373685f1e Docker + goreleaser (#729) 2021-12-31 19:55:31 -05:00
Thomas Pelletier f1391952d4 Update and move testsuite to internal package (#730)
* Regenerate test suite

* Move test suite to /internal
2021-12-31 18:52:26 -05:00
Thomas Pelletier 4a73a200ed Clean up tools godoc 2021-12-31 15:56:40 -05:00
Thomas Pelletier 4807229e94 tomll: port to v2 (#727)
Fixes #721
2021-12-31 15:40:18 -05:00
Thomas Pelletier d8ddc00c61 jsontoml: port to v2 (#726)
Fixes #719
2021-12-31 14:40:20 -05:00
Thomas Pelletier 82f8dad811 tomljson: port to v2 (#725) 2021-12-31 13:25:53 -05:00
Thomas Pelletier 75db1016e8 Remove extra words from CONTRIBUTING 2021-12-29 10:26:22 -05:00
Thomas Pelletier de6d715bd2 Update CONTRIBUTING.md 2021-12-29 10:25:28 -05:00
Thomas Pelletier 3ab2fc2b87 Update release proces documentation 2021-12-29 10:24:01 -05:00
Thomas Pelletier 1b1dd3d6d5 Exclude testing PRs from release notes 2021-12-29 09:53:43 -05:00
Cameron Moore 128b7a8bfb Decode: check buffer length before parsing simple key (#717)
Fixes #714
2021-12-29 08:58:42 -05:00
Cameron Moore 892df5c28e Decode: fix index out of range bug (#716)
Fixes #715
2021-12-29 08:49:33 -05:00
Thomas Pelletier d58eb50ebf Doc: clarify errors returned by Decode (#713)
Fixes #625
2021-12-26 20:04:09 +01:00
Thomas Pelletier 535fc65c5f Fix link in README 2021-12-26 19:49:35 +01:00
Thomas Pelletier f158d7d278 Readme: document more differences with v1 (#712)
* Readme: document more changes with v1

Closes #552
2021-12-26 19:47:03 +01:00
Thomas Pelletier 5fd6e9cce0 Encode: add comment struct tag (#711)
Similar to v1, add a `comment` struct that that makes the encoder emit a comment
before the annotated element, if permitted. Unlike v1, comments are compact by
default (and cannot be changed).

Fixes #595.
2021-12-26 18:29:46 +01:00
Thomas Pelletier 8ce5c3d78f Decoder: time allows extra precision (#710)
As discussed[1], this change allows times to provide precision beyond the
nanosecond (nine digits fractional part). Extra precision is truncated according
to the TOML specificiation.

[1]: https://github.com/pelletier/go-toml/discussions/707
2021-12-26 17:05:10 +01:00
Thomas Pelletier 177b4a5e53 Decode: allow \r\n as line whitespace before \ (#709)
Fixes #708
2021-12-26 16:38:15 +01:00
Cameron Moore 5cbdea6192 decode: fix maximum time offset values (#706)
According to RFC3339 section 5.6, the maximum time offset values for
hours and minutes is 23 and 59, respectively.
2021-12-22 10:29:52 +01:00
Thomas Pelletier 696dd25c17 Decoder: disallow modification of existing table (#704)
Fixes #703
2021-12-15 11:05:27 -05:00
Thomas Pelletier facb2b13e8 Decoder: prevent modification of inline tables (#702)
Fixes #701
2021-12-12 09:43:42 -05:00
Cameron Moore 8bbb519477 Decode: ensure signed exponents don't start with an underscore (#699) 2021-12-05 20:02:19 -05:00
Cameron Moore b37e11d74d Decode: allow maximum seconds value of 60 (#700)
RFC3339 allows seconds to be 60 when adding leap seconds
2021-12-05 20:00:42 -05:00
Cameron Moore 6cd86876b8 Decode: ensure signed numbers don't start with an underscore (#698) 2021-12-04 16:56:48 -05:00
Cameron Moore f53bc740c1 Decode: restrict timezone offset values (#696)
Don't allow hours greater than 24 and minutes greater than 60 per RFC
3339.
2021-12-02 18:59:32 -05:00
Thomas Pelletier 9bf9be681e Decoder: check for invalid chars in timezone (#695)
Fixes #694
2021-12-02 09:00:20 -05:00
Thomas Pelletier c862c344b3 Decoder: allow commas in tags (#693) 2021-11-30 21:59:22 -05:00
Thomas Pelletier 0d20a84523 Encoder: omitempty flag (#692)
Fixes #597
2021-11-30 21:32:28 -05:00
Thomas Pelletier 3990899d7e Decoder: check tz has : between hours and minutes (#691)
Fixes #690
2021-11-30 20:22:11 -05:00
Cameron Moore 4c7a337083 Decoder: fix typo in test description (#689) 2021-11-30 15:28:01 -05:00
Thomas Pelletier bbaae540ce Decoder: check timezones start with +,-,z,Z (#688)
Also simplifies local time seconds scanning.

Fixes #686
2021-11-30 13:01:15 -05:00
Thomas Pelletier ede6445608 Decoder: flag bad \r in literal multiline strings (#687)
Fixes #685
2021-11-30 10:44:48 -05:00
Thomas Pelletier b226db6a29 Decoder: show struct field in type mismatch errors (#684)
The goal is to provide some context as to why the type were mismatched. This
change only works for that case, on structs. This is the same a encoding/json. A
more general solution would be great, but this would require a broader change in
the decoder, which I don't think is necessary at the moment.

Fixes #628
2021-11-24 20:43:56 -05:00
Thomas Pelletier d8997efb5a Mention "-" to prevent encoding field in doc (#683) 2021-11-24 19:52:23 -05:00
Thomas Pelletier 79e78b234c Decoder: fix panic on table array behind a pointer (#682)
Fixes #677
2021-11-24 18:50:04 -05:00
Thomas Pelletier 1b5a25c0ef Decoder: fail on unescaped \r not followed by \n (#681)
Fixes #674
2021-11-24 18:11:36 -05:00
Thomas Pelletier 8eae15b2ee Decoder: validate bounds of day and month in dates (#680)
Fixes #676
2021-11-24 17:42:01 -05:00
Thomas Pelletier 2b3de620e8 Encoder: try to use pointer type TextMarshaler (#679)
If a type does not implement the encoding.TextMarshaler interface but
its pointer type does, use it if possible.

Fixes #678
2021-11-24 14:43:49 -05:00
Cameron Moore 8645d6376b Decoder: flag invalid carriage returns in literal strings (#673) 2021-11-23 22:41:59 -05:00
Thomas Pelletier 64fe47161f API: Encoder and Decoder options are chainable (#670)
Fixes #583
2021-11-13 19:04:53 -05:00
Thomas Pelletier 4dff8eaa4d Decoder: prevent duplicates of inline tables (#667)
* seen: prevent duplicates of inline tables

* Provide clearer error message for redefined keys

For example:

``
toml: key b is already defined
```
2021-11-10 10:04:43 -05:00
Cameron Moore 2dbd29a565 parser: Fix missing check for upper exponent (#665) 2021-11-09 21:15:23 -05:00
Thomas Pelletier f27a07d31a seen: verify arrays (#663)
Fixes #662
2021-11-09 20:26:30 -05:00
Thomas Pelletier 644515958c Update TOML test suite (#661)
Ref #658
2021-11-08 22:35:35 -05:00
Thomas Pelletier 8683be35f6 seen: check inline tables (#660)
Fixes #658
2021-11-08 21:53:02 -05:00
Thomas Pelletier dc1740d473 Decode: code cleanup for struct cache (#659) 2021-11-07 18:35:30 -05:00
Thomas Pelletier 11f789ef11 Decode: prevent comments that look like dates to be accepted (#657)
* parser: fix date detection

When the parser has to decide between parsing and integer or a date, it should
check that all characters are actually acceptable (digits, or date/time
elements).

Fixes #655
2021-11-04 22:06:12 -04:00
Thomas Pelletier 74d21b367f scanner: handle carriage return in comments (#656)
Fixes #653
2021-11-04 21:40:16 -04:00
Thomas Pelletier 6617e7e73d utf8: use lookup table to validate ASCII (#654) 2021-11-04 16:05:36 -04:00
Thomas Pelletier 3dbca20bc9 Decoder: flag invalid carriage returns in strings (#652)
Fixes #651
2021-11-02 10:02:25 -04:00
Thomas Pelletier 85c0658984 Decode: add missing checks for LocalTime (#650) 2021-10-29 22:13:08 -04:00
Thomas Pelletier 772d169b52 testsuite: return error when can't encode tag (#648) 2021-10-29 21:51:50 -04:00
Cameron Moore b4ec220f7e Update tomltestgen and regenerate tests (#645)
Remove testsuite build tag from generated tests file

Co-authored-by: Thomas Pelletier <thomas@pelletier.codes>
2021-10-28 20:46:08 -04:00
Thomas Pelletier 3694ae88f6 decode: error on _ before exponent in floats (#647)
Fixes #646
2021-10-28 20:41:10 -04:00
62 changed files with 3804 additions and 1074 deletions
+1
View File
@@ -1,3 +1,4 @@
* text=auto
benchmark/benchmark.toml text eol=lf
testdata/** text eol=lf
+1
View File
@@ -2,6 +2,7 @@ changelog:
exclude:
labels:
- build
- testing
categories:
- title: What's new
labels:
+4 -4
View File
@@ -35,11 +35,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
uses: github/codeql-action/autobuild@v2
# ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
uses: github/codeql-action/analyze@v2
+2 -2
View File
@@ -9,12 +9,12 @@ jobs:
runs-on: "ubuntu-latest"
name: report
steps:
- uses: actions/checkout@master
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@master
with:
go-version: 1.16
go-version: 1.19
- name: Run tests with coverage
run: ./ci.sh coverage -d "${GITHUB_BASE_REF-HEAD}"
+39
View File
@@ -0,0 +1,39 @@
name: release
on:
push:
tags:
- "v2.*"
workflow_call:
inputs:
args:
description: "Extra arguments to pass goreleaser"
default: ""
required: false
type: string
jobs:
release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.19
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v3
with:
distribution: goreleaser
version: latest
args: release ${{ inputs.args }} --rm-dist
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+9 -2
View File
@@ -12,14 +12,21 @@ jobs:
strategy:
matrix:
os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest']
go: [ '1.16', '1.17' ]
go: [ '1.18', '1.19' ]
runs-on: ${{ matrix.os }}
name: ${{ matrix.go }}/${{ matrix.os }}
steps:
- uses: actions/checkout@master
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup go ${{ matrix.go }}
uses: actions/setup-go@master
with:
go-version: ${{ matrix.go }}
- name: Run unit tests
run: go test -race ./...
release-check:
if: ${{ github.ref != 'refs/heads/v2' }}
uses: pelletier/go-toml/.github/workflows/release.yml@v2
with:
args: --snapshot
+1
View File
@@ -3,3 +3,4 @@ fuzz/
cmd/tomll/tomll
cmd/tomljson/tomljson
cmd/tomltestgen/tomltestgen
dist
+123
View File
@@ -0,0 +1,123 @@
before:
hooks:
- go mod tidy
- go fmt ./...
- go test ./...
builds:
- id: tomll
main: ./cmd/tomll
binary: tomll
env:
- CGO_ENABLED=0
flags:
- -trimpath
ldflags:
- -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}}
mod_timestamp: '{{ .CommitTimestamp }}'
targets:
- linux_amd64
- linux_arm64
- linux_arm
- windows_amd64
- windows_arm64
- windows_arm
- darwin_amd64
- darwin_arm64
- id: tomljson
main: ./cmd/tomljson
binary: tomljson
env:
- CGO_ENABLED=0
flags:
- -trimpath
ldflags:
- -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}}
mod_timestamp: '{{ .CommitTimestamp }}'
targets:
- linux_amd64
- linux_arm64
- linux_arm
- windows_amd64
- windows_arm64
- windows_arm
- darwin_amd64
- darwin_arm64
- id: jsontoml
main: ./cmd/jsontoml
binary: jsontoml
env:
- CGO_ENABLED=0
flags:
- -trimpath
ldflags:
- -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}}
mod_timestamp: '{{ .CommitTimestamp }}'
targets:
- linux_amd64
- linux_arm64
- linux_arm
- windows_amd64
- windows_arm64
- windows_arm
- darwin_amd64
- darwin_arm64
universal_binaries:
- id: tomll
replace: true
name_template: tomll
- id: tomljson
replace: true
name_template: tomljson
- id: jsontoml
replace: true
name_template: jsontoml
archives:
- id: jsontoml
format: tar.xz
builds:
- jsontoml
files:
- none*
name_template: "{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}"
- id: tomljson
format: tar.xz
builds:
- tomljson
files:
- none*
name_template: "{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}"
- id: tomll
format: tar.xz
builds:
- tomll
files:
- none*
name_template: "{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}"
dockers:
- id: tools
goos: linux
goarch: amd64
ids:
- jsontoml
- tomljson
- tomll
image_templates:
- "ghcr.io/pelletier/go-toml:latest"
- "ghcr.io/pelletier/go-toml:{{ .Tag }}"
- "ghcr.io/pelletier/go-toml:v{{ .Major }}"
skip_push: false
checksum:
name_template: 'sha256sums.txt'
snapshot:
name_template: "{{ incpatch .Version }}-next"
release:
github:
owner: pelletier
name: go-toml
draft: true
prerelease: auto
mode: replace
changelog:
use: github-native
announce:
skip: true
+23 -9
View File
@@ -155,6 +155,8 @@ Checklist:
- Does not introduce backward-incompatible changes (unless discussed).
- Has relevant doc changes.
- Benchstat does not show performance regression.
- Pull request is [labeled appropriately][pr-labels].
- Title will be understandable in the changelog.
1. Merge using "squash and merge".
2. Make sure to edit the commit message to keep all the useful information
@@ -163,13 +165,25 @@ Checklist:
### New release
1. Go to [releases][releases]. Click on "X commits to master since this
release".
2. Make note of all the changes. Look for backward incompatible changes,
new features, and bug fixes.
3. Pick the new version using the above and semver.
4. Create a [new release][new-release].
5. Follow the same format as [1.1.0][release-110].
1. Decide on the next version number. Use semver.
2. Generate release notes using [`gh`][gh]. Example:
```
$ gh api -X POST \
-F tag_name='v2.0.0-beta.5' \
-F target_commitish='v2' \
-F previous_tag_name='v2.0.0-beta.4' \
--jq '.body' \
repos/pelletier/go-toml/releases/generate-notes
```
3. Look for "Other changes". That would indicate a pull request not labeled
properly. Tweak labels and pull request titles until changelog looks good for
users.
4. [Draft new release][new-release].
5. Fill tag and target with the same value used to generate the changelog.
6. Set title to the new tag value.
7. Paste the generated changelog.
8. Check "create discussion", in the "Releases" category.
9. Check pre-release if new version is an alpha or beta.
[issues-tracker]: https://github.com/pelletier/go-toml/issues
[bug-report]: https://github.com/pelletier/go-toml/issues/new?template=bug_report.md
@@ -177,6 +191,6 @@ Checklist:
[readme]: ./README.md
[fork]: https://help.github.com/articles/fork-a-repo
[pull-request]: https://help.github.com/en/articles/creating-a-pull-request
[releases]: https://github.com/pelletier/go-toml/releases
[new-release]: https://github.com/pelletier/go-toml/releases/new
[release-110]: https://github.com/pelletier/go-toml/releases/tag/v1.1.0
[gh]: https://github.com/cli/cli
[pr-labels]: https://github.com/pelletier/go-toml/blob/v2/.github/release.yml
+5
View File
@@ -0,0 +1,5 @@
FROM scratch
ENV PATH "$PATH:/bin"
COPY tomll /bin/tomll
COPY tomljson /bin/tomljson
COPY jsontoml /bin/jsontoml
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2013 - 2021 Thomas Pelletier, Eric Anderton
Copyright (c) 2013 - 2022 Thomas Pelletier, Eric Anderton
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+159 -34
View File
@@ -4,24 +4,14 @@ Go library for the [TOML](https://toml.io/en/) format.
This library supports [TOML v1.0.0](https://toml.io/en/v1.0.0).
## Development status
This is the upcoming major version of go-toml. It is currently in active
development. As of release v2.0.0-beta.1, the library has reached feature parity
with v1, and fixes a lot known bugs and performance issues along the way.
If you do not need the advanced document editing features of v1, you are
encouraged to try out this version.
[👉 Roadmap for v2](https://github.com/pelletier/go-toml/discussions/506)
[🐞 Bug Reports](https://github.com/pelletier/go-toml/issues)
[💬 Anything else](https://github.com/pelletier/go-toml/discussions)
## Documentation
Full API, examples, and implementation notes are available in the Go documentation.
Full API, examples, and implementation notes are available in the Go
documentation.
[![Go Reference](https://pkg.go.dev/badge/github.com/pelletier/go-toml/v2.svg)](https://pkg.go.dev/github.com/pelletier/go-toml/v2)
@@ -48,15 +38,16 @@ operations should not be shockingly slow. See [benchmarks](#benchmarks).
### Strict mode
`Decoder` can be set to "strict mode", which makes it error when some parts of
the TOML document was not prevent in the target structure. This is a great way
the TOML document was not present in the target structure. This is a great way
to check for typos. [See example in the documentation][strict].
[strict]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#example-Decoder.SetStrict
[strict]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#example-Decoder.DisallowUnknownFields
### Contextualized errors
When decoding errors occur, go-toml returns [`DecodeError`][decode-err]), which
contains a human readable contextualized version of the error. For example:
When most decoding errors occur, go-toml returns [`DecodeError`][decode-err]),
which contains a human readable contextualized version of the error. For
example:
```
2| key1 = "value1"
@@ -159,11 +150,11 @@ Execution time speedup compared to other Go TOML libraries:
</thead>
<tbody>
<tr><td>Marshal/HugoFrontMatter-2</td><td>1.9x</td><td>1.9x</td></tr>
<tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>1.9x</td></tr>
<tr><td>Marshal/ReferenceFile/struct-2</td><td>2.4x</td><td>2.6x</td></tr>
<tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.5x</td></tr>
<tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.7x</td><td>2.6x</td></tr>
<tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.8x</td><td>5.1x</td></tr>
<tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>1.8x</td></tr>
<tr><td>Marshal/ReferenceFile/struct-2</td><td>2.2x</td><td>2.5x</td></tr>
<tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.9x</td></tr>
<tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.6x</td><td>2.9x</td></tr>
<tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.4x</td><td>5.3x</td></tr>
</tbody>
</table>
<details><summary>See more</summary>
@@ -176,17 +167,17 @@ provided for completeness.</p>
<tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
</thead>
<tbody>
<tr><td>Marshal/SimpleDocument/map-2</td><td>1.7x</td><td>2.1x</td></tr>
<tr><td>Marshal/SimpleDocument/struct-2</td><td>2.5x</td><td>2.8x</td></tr>
<tr><td>Unmarshal/SimpleDocument/map-2</td><td>4.1x</td><td>3.1x</td></tr>
<tr><td>Unmarshal/SimpleDocument/struct-2</td><td>6.4x</td><td>4.3x</td></tr>
<tr><td>UnmarshalDataset/example-2</td><td>3.4x</td><td>3.2x</td></tr>
<tr><td>UnmarshalDataset/code-2</td><td>2.2x</td><td>2.5x</td></tr>
<tr><td>UnmarshalDataset/twitter-2</td><td>2.8x</td><td>2.7x</td></tr>
<tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.2x</td><td>2.0x</td></tr>
<tr><td>UnmarshalDataset/canada-2</td><td>1.8x</td><td>1.4x</td></tr>
<tr><td>UnmarshalDataset/config-2</td><td>4.4x</td><td>2.9x</td></tr>
<tr><td>[Geo mean]</td><td>2.8x</td><td>2.6x</td></tr>
<tr><td>Marshal/SimpleDocument/map-2</td><td>1.8x</td><td>2.9x</td></tr>
<tr><td>Marshal/SimpleDocument/struct-2</td><td>2.7x</td><td>4.2x</td></tr>
<tr><td>Unmarshal/SimpleDocument/map-2</td><td>4.5x</td><td>3.1x</td></tr>
<tr><td>Unmarshal/SimpleDocument/struct-2</td><td>6.2x</td><td>3.9x</td></tr>
<tr><td>UnmarshalDataset/example-2</td><td>3.1x</td><td>3.5x</td></tr>
<tr><td>UnmarshalDataset/code-2</td><td>2.3x</td><td>3.1x</td></tr>
<tr><td>UnmarshalDataset/twitter-2</td><td>2.5x</td><td>2.6x</td></tr>
<tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.1x</td><td>2.2x</td></tr>
<tr><td>UnmarshalDataset/canada-2</td><td>1.6x</td><td>1.3x</td></tr>
<tr><td>UnmarshalDataset/config-2</td><td>4.3x</td><td>3.2x</td></tr>
<tr><td>[Geo mean]</td><td>2.7x</td><td>2.8x</td></tr>
</tbody>
</table>
<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>
@@ -206,6 +197,44 @@ In case of trouble: [Go Modules FAQ][mod-faq].
[mod-faq]: https://github.com/golang/go/wiki/Modules#why-does-installing-a-tool-via-go-get-fail-with-error-cannot-find-main-module
## Tools
Go-toml provides three handy command line tools:
* `tomljson`: Reads a TOML file and outputs its JSON representation.
```
$ go install github.com/pelletier/go-toml/v2/cmd/tomljson@latest
$ tomljson --help
```
* `jsontoml`: Reads a JSON file and outputs a TOML representation.
```
$ go install github.com/pelletier/go-toml/v2/cmd/jsontoml@latest
$ jsontoml --help
```
* `tomll`: Lints and reformats a TOML file.
```
$ go install github.com/pelletier/go-toml/v2/cmd/tomll@latest
$ tomll --help
```
### Docker image
Those tools are also available as a [Docker image][docker]. For example, to use
`tomljson`:
```
docker run -i ghcr.io/pelletier/go-toml:v2 tomljson < example.toml
```
Multiple versions are availble on [ghcr.io][docker].
[docker]: https://github.com/pelletier/go-toml/pkgs/container/go-toml
## Migrating from v1
This section describes the differences between v1 and v2, with some pointers on
@@ -324,6 +353,29 @@ The recommended replacement is pre-filling the struct before unmarshaling.
[go-defaults]: https://github.com/mcuadros/go-defaults
#### `toml.Tree` replacement
This structure was the initial attempt at providing a document model for
go-toml. It allows manipulating the structure of any document, encoding and
decoding from their TOML representation. While a more robust feature was
initially planned in go-toml v2, this has been ultimately [removed from
scope][nodoc] of this library, with no plan to add it back at the moment. The
closest equivalent at the moment would be to unmarshal into an `interface{}` and
use type assertions and/or reflection to manipulate the arbitrary
structure. However this would fall short of providing all of the TOML features
such as adding comments and be specific about whitespace.
#### `toml.Position` are not retrievable anymore
The API for retrieving the position (line, column) of a specific TOML element do
not exist anymore. This was done to minimize the amount of concepts introduced
by the library (query path), and avoid the performance hit related to storing
positions in the absence of a document model, for a feature that seemed to have
little use. Errors however have gained more detailed position
information. Position retrieval seems better fitted for a document model, which
has been [removed from the scope][nodoc] of go-toml v2 at the moment.
### Encoding / Marshal
#### Default struct fields order
@@ -359,7 +411,8 @@ fmt.Println("v2:\n" + string(b))
```
There is no way to make v2 encoder behave like v1. A workaround could be to
manually sort the fields alphabetically in the struct definition.
manually sort the fields alphabetically in the struct definition, or generate
struct types using `reflect.StructOf`.
#### No indentation by default
@@ -407,7 +460,9 @@ fmt.Println("v2 Encoder:\n" + string(buf.Bytes()))
V1 always uses double quotes (`"`) around strings and keys that cannot be
represented bare (unquoted). V2 uses single quotes instead by default (`'`),
unless a character cannot be represented, then falls back to double quotes.
unless a character cannot be represented, then falls back to double quotes. As a
result of this change, `Encoder.QuoteMapKeys` has been removed, as it is not
useful anymore.
There is no way to make v2 encoder behave like v1.
@@ -422,6 +477,76 @@ There is no way to make v2 encoder behave like v1.
[tm]: https://golang.org/pkg/encoding/#TextMarshaler
#### `Encoder.CompactComments` has been removed
Emitting compact comments is now the default behavior of go-toml. This option
is not necessary anymore.
#### Struct tags have been merged
V1 used to provide multiple struct tags: `comment`, `commented`, `multiline`,
`toml`, and `omitempty`. To behave more like the standard library, v2 has merged
`toml`, `multiline`, and `omitempty`. For example:
```go
type doc struct {
// v1
F string `toml:"field" multiline:"true" omitempty:"true"`
// v2
F string `toml:"field,multiline,omitempty"`
}
```
Has a result, the `Encoder.SetTag*` methods have been removed, as there is just
one tag now.
#### `commented` tag has been removed
There is no replacement for the `commented` tag. This feature would be better
suited in a proper document model for go-toml v2, which has been [cut from
scope][nodoc] at the moment.
#### `Encoder.ArraysWithOneElementPerLine` has been renamed
The new name is `Encoder.SetArraysMultiline`. The behavior should be the same.
#### `Encoder.Indentation` has been renamed
The new name is `Encoder.SetIndentSymbol`. The behavior should be the same.
#### Embedded structs behave like stdlib
V1 defaults to merging embedded struct fields into the embedding struct. This
behavior was unexpected because it does not follow the standard library. To
avoid breaking backward compatibility, the `Encoder.PromoteAnonymous` method was
added to make the encoder behave correctly. Given backward compatibility is not
a problem anymore, v2 does the right thing by default: it follows the behavior
of `encoding/json`. `Encoder.PromoteAnonymous` has been removed.
[nodoc]: https://github.com/pelletier/go-toml/discussions/506#discussioncomment-1526038
### `query`
go-toml v1 provided the [`go-toml/query`][query] package. It allowed to run
JSONPath-style queries on TOML files. This feature is not available in v2. For a
replacement, check out [dasel][dasel].
This package has been removed because it was essentially not supported anymore
(last commit May 2020), increased the complexity of the code base, and more
complete solutions exist out there.
[query]: https://github.com/pelletier/go-toml/tree/f99d6bbca119636aeafcf351ee52b3d202782627/query
[dasel]: https://github.com/TomWright/dasel
## Versioning
Go-toml follows [Semantic Versioning](http://semver.org/). The supported version
of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of
this document. The last two major versions of Go are supported
(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)).
## License
The MIT License (MIT). Read [LICENSE](LICENSE).
+11 -3
View File
@@ -76,7 +76,8 @@ cover() {
fi
pushd "$dir"
go test -covermode=atomic -coverprofile=coverage.out ./...
go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./...
cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
go tool cover -func=coverage.out
popd
@@ -103,8 +104,8 @@ coverage() {
echo ""
target_pct="$(cat ${target_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
head_pct="$(cat ${head_out} |sed -E 's/.*total.*\t([0-9.]+)%/\1/;t;d')"
target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
@@ -112,6 +113,13 @@ coverage() {
if [[ $delta_pct = \-* ]]; then
echo "Regression!";
target_diff="${output_dir}/target.diff.txt"
head_diff="${output_dir}/head.diff.txt"
cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
return 1
fi
return 0
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"os"
"path"
"github.com/pelletier/go-toml/v2/testsuite"
"github.com/pelletier/go-toml/v2/internal/testsuite"
)
func main() {
+55
View File
@@ -0,0 +1,55 @@
// Package jsontoml is a program that converts JSON to TOML.
//
// # Usage
//
// Reading from stdin:
//
// cat file.json | jsontoml > file.toml
//
// Reading from a file:
//
// jsontoml file.json > file.toml
//
// # Installation
//
// Using Go:
//
// go install github.com/pelletier/go-toml/v2/cmd/jsontoml@latest
package main
import (
"encoding/json"
"io"
"github.com/pelletier/go-toml/v2"
"github.com/pelletier/go-toml/v2/internal/cli"
)
const usage = `jsontoml can be used in two ways:
Reading from stdin:
cat file.json | jsontoml > file.toml
Reading from a file:
jsontoml file.json > file.toml
`
func main() {
p := cli.Program{
Usage: usage,
Fn: convert,
}
p.Execute()
}
func convert(r io.Reader, w io.Writer) error {
var v interface{}
d := json.NewDecoder(r)
err := d.Decode(&v)
if err != nil {
return err
}
e := toml.NewEncoder(w)
return e.Encode(v)
}
+48
View File
@@ -0,0 +1,48 @@
package main
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvert(t *testing.T) {
examples := []struct {
name string
input string
expected string
errors bool
}{
{
name: "valid json",
input: `
{
"mytoml": {
"a": 42
}
}`,
expected: `[mytoml]
a = 42.0
`,
},
{
name: "invalid json",
input: `{ foo`,
errors: true,
},
}
for _, e := range examples {
b := new(bytes.Buffer)
err := convert(strings.NewReader(e.input), b)
if e.errors {
require.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, e.expected, b.String())
}
}
}
+63
View File
@@ -0,0 +1,63 @@
// Package tomljson is a program that converts TOML to JSON.
//
// # Usage
//
// Reading from stdin:
//
// cat file.toml | tomljson > file.json
//
// Reading from a file:
//
// tomljson file.toml > file.json
//
// # Installation
//
// Using Go:
//
// go install github.com/pelletier/go-toml/v2/cmd/tomljson@latest
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"github.com/pelletier/go-toml/v2"
"github.com/pelletier/go-toml/v2/internal/cli"
)
const usage = `tomljson can be used in two ways:
Reading from stdin:
cat file.toml | tomljson > file.json
Reading from a file:
tomljson file.toml > file.json
`
func main() {
p := cli.Program{
Usage: usage,
Fn: convert,
}
p.Execute()
}
func convert(r io.Reader, w io.Writer) error {
var v interface{}
d := toml.NewDecoder(r)
err := d.Decode(&v)
if err != nil {
var derr *toml.DecodeError
if errors.As(err, &derr) {
row, col := derr.Position()
return fmt.Errorf("%s\nerror occurred at row %d column %d", derr.String(), row, col)
}
return err
}
e := json.NewEncoder(w)
e.SetIndent("", " ")
return e.Encode(v)
}
+61
View File
@@ -0,0 +1,61 @@
package main
import (
"bytes"
"fmt"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvert(t *testing.T) {
examples := []struct {
name string
input io.Reader
expected string
errors bool
}{
{
name: "valid toml",
input: strings.NewReader(`
[mytoml]
a = 42`),
expected: `{
"mytoml": {
"a": 42
}
}
`,
},
{
name: "invalid toml",
input: strings.NewReader(`bad = []]`),
errors: true,
},
{
name: "bad reader",
input: &badReader{},
errors: true,
},
}
for _, e := range examples {
b := new(bytes.Buffer)
err := convert(e.input, b)
if e.errors {
require.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, e.expected, b.String())
}
}
}
type badReader struct{}
func (r *badReader) Read([]byte) (int, error) {
return 0, fmt.Errorf("reader failed on purpose")
}
+58
View File
@@ -0,0 +1,58 @@
// Package tomll is a linter program for TOML.
//
// # Usage
//
// Reading from stdin, writing to stdout:
//
// cat file.toml | tomll
//
// Reading and updating a list of files in place:
//
// tomll a.toml b.toml c.toml
//
// # Installation
//
// Using Go:
//
// go install github.com/pelletier/go-toml/v2/cmd/tomll@latest
package main
import (
"io"
"github.com/pelletier/go-toml/v2"
"github.com/pelletier/go-toml/v2/internal/cli"
)
const usage = `tomll can be used in two ways:
Reading from stdin, writing to stdout:
cat file.toml | tomll > file.toml
Reading and updating a list of files in place:
tomll a.toml b.toml c.toml
When given a list of files, tomll will modify all files in place without asking.
`
func main() {
p := cli.Program{
Usage: usage,
Fn: convert,
Inplace: true,
}
p.Execute()
}
func convert(r io.Reader, w io.Writer) error {
var v interface{}
d := toml.NewDecoder(r)
err := d.Decode(&v)
if err != nil {
return err
}
e := toml.NewEncoder(w)
return e.Encode(v)
}
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvert(t *testing.T) {
examples := []struct {
name string
input string
expected string
errors bool
}{
{
name: "valid toml",
input: `
mytoml.a = 42.0
`,
expected: `[mytoml]
a = 42.0
`,
},
{
name: "invalid toml",
input: `[what`,
errors: true,
},
}
for _, e := range examples {
b := new(bytes.Buffer)
err := convert(strings.NewReader(e.input), b)
if e.errors {
require.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, e.expected, b.String())
}
}
}
+1 -2
View File
@@ -43,8 +43,7 @@ type testsCollection struct {
Count int
}
const srcTemplate = "// +build testsuite\n\n" +
"// Generated by tomltestgen for toml-test ref {{.Ref}} on {{.Timestamp}}\n" +
const srcTemplate = "// Generated by tomltestgen for toml-test ref {{.Ref}} on {{.Timestamp}}\n" +
"package toml_test\n" +
" import (\n" +
" \"testing\"\n" +
+113 -33
View File
@@ -35,13 +35,22 @@ func parseLocalDate(b []byte) (LocalDate, error) {
return date, newDecodeError(b, "dates are expected to have the format YYYY-MM-DD")
}
date.Year = parseDecimalDigits(b[0:4])
var err error
v := parseDecimalDigits(b[5:7])
date.Year, err = parseDecimalDigits(b[0:4])
if err != nil {
return LocalDate{}, err
}
date.Month = v
date.Month, err = parseDecimalDigits(b[5:7])
if err != nil {
return LocalDate{}, err
}
date.Day = parseDecimalDigits(b[8:10])
date.Day, err = parseDecimalDigits(b[8:10])
if err != nil {
return LocalDate{}, err
}
if !isValidDate(date.Year, date.Month, date.Day) {
return LocalDate{}, newDecodeError(b, "impossible date")
@@ -50,15 +59,18 @@ func parseLocalDate(b []byte) (LocalDate, error) {
return date, nil
}
func parseDecimalDigits(b []byte) int {
func parseDecimalDigits(b []byte) (int, error) {
v := 0
for _, c := range b {
for i, c := range b {
if c < '0' || c > '9' {
return 0, newDecodeError(b[i:i+1], "expected digit (0-9)")
}
v *= 10
v += int(c - '0')
}
return v
return v, nil
}
func parseDateTime(b []byte) (time.Time, error) {
@@ -87,15 +99,42 @@ func parseDateTime(b []byte) (time.Time, error) {
if len(b) != dateTimeByteLen {
return time.Time{}, newDecodeError(b, "invalid date-time timezone")
}
direction := 1
if b[0] == '-' {
var direction int
switch b[0] {
case '-':
direction = -1
case '+':
direction = +1
default:
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset character")
}
if b[3] != ':' {
return time.Time{}, newDecodeError(b[3:4], "expected a : separator")
}
hours, err := parseDecimalDigits(b[1:3])
if err != nil {
return time.Time{}, err
}
if hours > 23 {
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset hours")
}
minutes, err := parseDecimalDigits(b[4:6])
if err != nil {
return time.Time{}, err
}
if minutes > 59 {
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset minutes")
}
hours := digitsToInt(b[1:3])
minutes := digitsToInt(b[4:6])
seconds := direction * (hours*3600 + minutes*60)
if seconds == 0 {
zone = time.UTC
} else {
zone = time.FixedZone("", seconds)
}
b = b[dateTimeByteLen:]
}
@@ -159,7 +198,13 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
return t, nil, newDecodeError(b, "times are expected to have the format HH:MM:SS[.NNNNNN]")
}
t.Hour = parseDecimalDigits(b[0:2])
var err error
t.Hour, err = parseDecimalDigits(b[0:2])
if err != nil {
return t, nil, err
}
if t.Hour > 23 {
return t, nil, newDecodeError(b[0:2], "hour cannot be greater 23")
}
@@ -167,7 +212,10 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
return t, nil, newDecodeError(b[2:3], "expecting colon between hours and minutes")
}
t.Minute = parseDecimalDigits(b[3:5])
t.Minute, err = parseDecimalDigits(b[3:5])
if err != nil {
return t, nil, err
}
if t.Minute > 59 {
return t, nil, newDecodeError(b[3:5], "minutes cannot be greater 59")
}
@@ -175,42 +223,58 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
return t, nil, newDecodeError(b[5:6], "expecting colon between minutes and seconds")
}
t.Second = parseDecimalDigits(b[6:8])
if t.Second > 59 {
return t, nil, newDecodeError(b[3:5], "seconds cannot be greater 59")
t.Second, err = parseDecimalDigits(b[6:8])
if err != nil {
return t, nil, err
}
const minLengthWithFrac = 9
if len(b) >= minLengthWithFrac && b[minLengthWithFrac-1] == '.' {
if t.Second > 60 {
return t, nil, newDecodeError(b[6:8], "seconds cannot be greater 60")
}
b = b[8:]
if len(b) >= 1 && b[0] == '.' {
frac := 0
precision := 0
digits := 0
for i, c := range b[minLengthWithFrac:] {
for i, c := range b[1:] {
if !isDigit(c) {
if i == 0 {
return t, nil, newDecodeError(b[i:i+1], "need at least one digit after fraction point")
return t, nil, newDecodeError(b[0:1], "need at least one digit after fraction point")
}
break
}
digits++
const maxFracPrecision = 9
if i >= maxFracPrecision {
return t, nil, newDecodeError(b[i:i+1], "maximum precision for date time is nanosecond")
// go-toml allows decoding fractional seconds
// beyond the supported precision of 9
// digits. It truncates the fractional component
// to the supported precision and ignores the
// remaining digits.
//
// https://github.com/pelletier/go-toml/discussions/707
continue
}
frac *= 10
frac += int(c - '0')
digits++
precision++
}
t.Nanosecond = frac * nspow[digits]
t.Precision = digits
return t, b[9+digits:], nil
if precision == 0 {
return t, nil, newDecodeError(b[:1], "nanoseconds need at least one digit")
}
return t, b[8:], nil
t.Nanosecond = frac * nspow[precision]
t.Precision = precision
return t, b[1+digits:], nil
}
return t, b, nil
}
//nolint:cyclop
@@ -249,10 +313,10 @@ func parseFloat(b []byte) (float64, error) {
}
start := 0
if b[0] == '+' || b[0] == '-' {
if cleaned[0] == '+' || cleaned[0] == '-' {
start = 1
}
if b[start] == '0' && isDigit(b[start+1]) {
if cleaned[start] == '0' && isDigit(cleaned[start+1]) {
return 0, newDecodeError(b, "float integer part cannot have leading zeroes")
}
@@ -335,8 +399,17 @@ func parseIntDec(b []byte) (int64, error) {
}
func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) {
if b[0] == '_' {
return nil, newDecodeError(b[0:1], "number cannot start with underscore")
start := 0
if b[start] == '+' || b[start] == '-' {
start++
}
if len(b) == start {
return b, nil
}
if b[start] == '_' {
return nil, newDecodeError(b[start:start+1], "number cannot start with underscore")
}
if b[len(b)-1] == '_' {
@@ -405,6 +478,13 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) {
if !before {
return nil, newDecodeError(b[i-1:i+1], "number must have at least one digit between underscores")
}
if i < len(b)-1 && (b[i+1] == 'e' || b[i+1] == 'E') {
return nil, newDecodeError(b[i+1:i+2], "cannot have underscore before exponent")
}
before = false
case '+', '-':
// signed exponents
cleaned = append(cleaned, c)
before = false
case 'e', 'E':
if i < len(b)-1 && b[i+1] == '_' {
@@ -430,7 +510,7 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) {
// isValidDate checks if a provided date is a date that exists.
func isValidDate(year int, month int, day int) bool {
return day <= daysIn(month, year)
return month > 0 && month < 13 && day > 0 && day <= daysIn(month, year)
}
// daysBefore[m] counts the number of days in a non-leap year
+2 -1
View File
@@ -27,7 +27,7 @@ type DecodeError struct {
// corresponding field in the target value. It contains all the missing fields
// in Errors.
//
// Emitted by Decoder when SetStrict(true) was called.
// Emitted by Decoder when DisallowUnknownFields() was called.
type StrictMissingError struct {
// One error per field that could not be found.
Errors []DecodeError
@@ -103,6 +103,7 @@ func (e *DecodeError) Key() Key {
//
// The function copies all bytes used in DecodeError, so that document and
// highlight can be freely deallocated.
//
//nolint:funlen
func wrapDecodeError(document []byte, de *decodeError) *DecodeError {
offset := danger.SubsliceOffset(document, de.highlight)
+5 -5
View File
@@ -212,12 +212,12 @@ func ExampleDecodeError() {
fmt.Println(err)
//nolint:errorlint
de := err.(*DecodeError)
fmt.Println(de.String())
row, col := de.Position()
var derr *DecodeError
if errors.As(err, &derr) {
fmt.Println(derr.String())
row, col := derr.Position()
fmt.Println("error occurred at row", row, "column", col)
}
// Output:
// toml: number must have at least one digit between underscores
// 1| name = 123__456
+56
View File
@@ -0,0 +1,56 @@
//go:build go1.18 || go1.19
// +build go1.18 go1.19
package toml_test
import (
"io/ioutil"
"strings"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/require"
)
func FuzzUnmarshal(f *testing.F) {
file, err := ioutil.ReadFile("benchmark/benchmark.toml")
if err != nil {
panic(err)
}
f.Add(file)
f.Fuzz(func(t *testing.T, b []byte) {
if strings.Contains(string(b), "nan") {
// Current limitation of testify.
// https://github.com/stretchr/testify/issues/624
t.Skip("can't compare NaNs")
}
t.Log("INITIAL DOCUMENT ===========================")
t.Log(string(b))
var v interface{}
err := toml.Unmarshal(b, &v)
if err != nil {
return
}
t.Log("DECODED VALUE ===========================")
t.Logf("%#+v", v)
encoded, err := toml.Marshal(v)
if err != nil {
t.Fatalf("cannot marshal unmarshaled document: %s", err)
}
t.Log("ENCODED DOCUMENT ===========================")
t.Log(string(encoded))
var v2 interface{}
err = toml.Unmarshal(encoded, &v2)
if err != nil {
t.Fatalf("failed round trip: %s", err)
}
require.Equal(t, v, v2)
})
}
+1 -2
View File
@@ -2,5 +2,4 @@ module github.com/pelletier/go-toml/v2
go 1.16
// latest (v1.7.0) doesn't have the fix for time.Time
require github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942
require github.com/stretchr/testify v1.8.0
+8 -4
View File
@@ -1,11 +1,15 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU=
github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+24 -32
View File
@@ -20,8 +20,8 @@ type Iterator struct {
node *Node
}
// Next moves the iterator forward and returns true if points to a node, false
// otherwise.
// Next moves the iterator forward and returns true if points to a
// node, false otherwise.
func (c *Iterator) Next() bool {
if !c.started {
c.started = true
@@ -31,8 +31,8 @@ func (c *Iterator) Next() bool {
return c.node.Valid()
}
// IsLast returns true if the current node of the iterator is the last one.
// Subsequent call to Next() will return false.
// IsLast returns true if the current node of the iterator is the last
// one. Subsequent call to Next() will return false.
func (c *Iterator) IsLast() bool {
return c.node.next == 0
}
@@ -62,20 +62,20 @@ func (r *Root) at(idx Reference) *Node {
return &r.nodes[idx]
}
// Arrays have one child per element in the array.
// InlineTables have one child per key-value pair in the table.
// KeyValues have at least two children. The first one is the value. The
// rest make a potentially dotted key.
// Table and Array table have one child per element of the key they
// represent (same as KeyValue, but without the last node being the value).
// children []Node
// Arrays have one child per element in the array. InlineTables have
// one child per key-value pair in the table. KeyValues have at least
// two children. The first one is the value. The rest make a
// potentially dotted key. Table and Array table have one child per
// element of the key they represent (same as KeyValue, but without
// the last node being the value).
type Node struct {
Kind Kind
Raw Range // Raw bytes from the input.
Data []byte // Node value (could be either allocated or referencing the input).
Data []byte // Node value (either allocated or referencing the input).
// References to other nodes, as offsets in the backing array from this
// node. References can go backward, so those can be negative.
// References to other nodes, as offsets in the backing array
// from this node. References can go backward, so those can be
// negative.
next int // 0 if last element
child int // 0 if no child
}
@@ -85,8 +85,8 @@ type Range struct {
Length uint32
}
// Next returns a copy of the next node, or an invalid Node if there is no
// next node.
// Next returns a copy of the next node, or an invalid Node if there
// is no next node.
func (n *Node) Next() *Node {
if n.next == 0 {
return nil
@@ -96,9 +96,9 @@ func (n *Node) Next() *Node {
return (*Node)(danger.Stride(ptr, size, n.next))
}
// Child returns a copy of the first child node of this node. Other children
// can be accessed calling Next on the first child.
// Returns an invalid Node if there is none.
// Child returns a copy of the first child node of this node. Other
// children can be accessed calling Next on the first child. Returns
// an invalid Node if there is none.
func (n *Node) Child() *Node {
if n.child == 0 {
return nil
@@ -113,10 +113,9 @@ func (n *Node) Valid() bool {
return n != nil
}
// Key returns the child nodes making the Key on a supported node. Panics
// otherwise.
// They are guaranteed to be all be of the Kind Key. A simple key would return
// just one element.
// Key returns the child nodes making the Key on a supported
// node. Panics otherwise. They are guaranteed to be all be of the
// Kind Key. A simple key would return just one element.
func (n *Node) Key() Iterator {
switch n.Kind {
case KeyValue:
@@ -133,10 +132,9 @@ func (n *Node) Key() Iterator {
}
// Value returns a pointer to the value node of a KeyValue.
// Guaranteed to be non-nil.
// Panics if not called on a KeyValue node, or if the Children are malformed.
// Guaranteed to be non-nil. Panics if not called on a KeyValue node,
// or if the Children are malformed.
func (n *Node) Value() *Node {
assertKind(KeyValue, *n)
return n.Child()
}
@@ -144,9 +142,3 @@ func (n *Node) Value() *Node {
func (n *Node) Children() Iterator {
return Iterator{node: n.Child()}
}
func assertKind(k Kind, n Node) {
if n.Kind != k {
panic(fmt.Errorf("method was expecting a %s, not a %s", k, n.Kind))
}
}
+88
View File
@@ -0,0 +1,88 @@
package cli
import (
"bytes"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"github.com/pelletier/go-toml/v2"
)
type ConvertFn func(r io.Reader, w io.Writer) error
type Program struct {
Usage string
Fn ConvertFn
// Inplace allows the command to take more than one file as argument and
// perform convertion in place on each provided file.
Inplace bool
}
func (p *Program) Execute() {
flag.Usage = func() { fmt.Fprintf(os.Stderr, p.Usage) }
flag.Parse()
os.Exit(p.main(flag.Args(), os.Stdin, os.Stdout, os.Stderr))
}
func (p *Program) main(files []string, input io.Reader, output, error io.Writer) int {
err := p.run(files, input, output)
if err != nil {
var derr *toml.DecodeError
if errors.As(err, &derr) {
fmt.Fprintln(error, derr.String())
row, col := derr.Position()
fmt.Fprintln(error, "error occurred at row", row, "column", col)
} else {
fmt.Fprintln(error, err.Error())
}
return -1
}
return 0
}
func (p *Program) run(files []string, input io.Reader, output io.Writer) error {
if len(files) > 0 {
if p.Inplace {
return p.runAllFilesInPlace(files)
}
f, err := os.Open(files[0])
if err != nil {
return err
}
defer f.Close()
input = f
}
return p.Fn(input, output)
}
func (p *Program) runAllFilesInPlace(files []string) error {
for _, path := range files {
err := p.runFileInPlace(path)
if err != nil {
return err
}
}
return nil
}
func (p *Program) runFileInPlace(path string) error {
in, err := ioutil.ReadFile(path)
if err != nil {
return err
}
out := new(bytes.Buffer)
err = p.Fn(bytes.NewReader(in), out)
if err != nil {
return err
}
return ioutil.WriteFile(path, out.Bytes(), 0600)
}
+172
View File
@@ -0,0 +1,172 @@
package cli
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strings"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func processMain(args []string, input io.Reader, stdout, stderr io.Writer, f ConvertFn) int {
p := Program{Fn: f}
return p.main(args, input, stdout, stderr)
}
func TestProcessMainStdin(t *testing.T) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
input := strings.NewReader("this is the input")
exit := processMain([]string{}, input, stdout, stderr, func(r io.Reader, w io.Writer) error {
return nil
})
assert.Equal(t, 0, exit)
assert.Empty(t, stdout.String())
assert.Empty(t, stderr.String())
}
func TestProcessMainStdinErr(t *testing.T) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
input := strings.NewReader("this is the input")
exit := processMain([]string{}, input, stdout, stderr, func(r io.Reader, w io.Writer) error {
return fmt.Errorf("something bad")
})
assert.Equal(t, -1, exit)
assert.Empty(t, stdout.String())
assert.NotEmpty(t, stderr.String())
}
func TestProcessMainStdinDecodeErr(t *testing.T) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
input := strings.NewReader("this is the input")
exit := processMain([]string{}, input, stdout, stderr, func(r io.Reader, w io.Writer) error {
var v interface{}
return toml.Unmarshal([]byte(`qwe = 001`), &v)
})
assert.Equal(t, -1, exit)
assert.Empty(t, stdout.String())
assert.Contains(t, stderr.String(), "error occurred at")
}
func TestProcessMainFileExists(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "example")
require.NoError(t, err)
defer os.Remove(tmpfile.Name())
_, err = tmpfile.Write([]byte(`some data`))
require.NoError(t, err)
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
exit := processMain([]string{tmpfile.Name()}, nil, stdout, stderr, func(r io.Reader, w io.Writer) error {
return nil
})
assert.Equal(t, 0, exit)
assert.Empty(t, stdout.String())
assert.Empty(t, stderr.String())
}
func TestProcessMainFileDoesNotExist(t *testing.T) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
exit := processMain([]string{"/lets/hope/this/does/not/exist"}, nil, stdout, stderr, func(r io.Reader, w io.Writer) error {
return nil
})
assert.Equal(t, -1, exit)
assert.Empty(t, stdout.String())
assert.NotEmpty(t, stderr.String())
}
func TestProcessMainFilesInPlace(t *testing.T) {
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
defer os.RemoveAll(dir)
path1 := path.Join(dir, "file1")
path2 := path.Join(dir, "file2")
err = ioutil.WriteFile(path1, []byte("content 1"), 0600)
require.NoError(t, err)
err = ioutil.WriteFile(path2, []byte("content 2"), 0600)
require.NoError(t, err)
p := Program{
Fn: dummyFileFn,
Inplace: true,
}
exit := p.main([]string{path1, path2}, os.Stdin, os.Stdout, os.Stderr)
require.Equal(t, 0, exit)
v1, err := ioutil.ReadFile(path1)
require.NoError(t, err)
require.Equal(t, "1", string(v1))
v2, err := ioutil.ReadFile(path2)
require.NoError(t, err)
require.Equal(t, "2", string(v2))
}
func TestProcessMainFilesInPlaceErrRead(t *testing.T) {
p := Program{
Fn: dummyFileFn,
Inplace: true,
}
exit := p.main([]string{"/this/path/is/invalid"}, os.Stdin, os.Stdout, os.Stderr)
require.Equal(t, -1, exit)
}
func TestProcessMainFilesInPlaceFailFn(t *testing.T) {
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
defer os.RemoveAll(dir)
path1 := path.Join(dir, "file1")
err = ioutil.WriteFile(path1, []byte("content 1"), 0600)
require.NoError(t, err)
p := Program{
Fn: func(io.Reader, io.Writer) error { return fmt.Errorf("oh no") },
Inplace: true,
}
exit := p.main([]string{path1}, os.Stdin, os.Stdout, os.Stderr)
require.Equal(t, -1, exit)
v1, err := ioutil.ReadFile(path1)
require.NoError(t, err)
require.Equal(t, "content 1", string(v1))
}
func dummyFileFn(r io.Reader, w io.Writer) error {
b, err := ioutil.ReadAll(r)
if err != nil {
return err
}
v := strings.SplitN(string(b), " ", 2)[1]
_, err = w.Write([]byte(v))
return err
}
+23
View File
@@ -0,0 +1,23 @@
package danger
import (
"reflect"
"unsafe"
)
// typeID is used as key in encoder and decoder caches to enable using
// the optimize runtime.mapaccess2_fast64 function instead of the more
// expensive lookup if we were to use reflect.Type as map key.
//
// typeID holds the pointer to the reflect.Type value, which is unique
// in the program.
//
// https://github.com/segmentio/encoding/blob/master/json/codec.go#L59-L61
type TypeID unsafe.Pointer
func MakeTypeID(t reflect.Type) TypeID {
// reflect.Type has the fields:
// typ unsafe.Pointer
// ptr unsafe.Pointer
return TypeID((*[2]unsafe.Pointer)(unsafe.Pointer(&t))[1])
}
@@ -67,6 +67,7 @@ func TestDocMarshal(t *testing.T) {
}
marshalTestToml := `title = 'TOML Marshal Testing'
[basic_lists]
floats = [12.3, 45.6, 78.9]
bools = [true, false, true]
@@ -89,7 +90,6 @@ name = 'Second'
[subdoc.first]
name = 'First'
[basic]
uint = 5001
bool = true
@@ -101,9 +101,9 @@ date = 1979-05-27T07:32:00Z
[[subdoclist]]
name = 'List.First'
[[subdoclist]]
name = 'List.Second'
`
result, err := toml.Marshal(docData)
@@ -117,14 +117,15 @@ func TestBasicMarshalQuotedKey(t *testing.T) {
expected := `'Z.string-àéù' = 'Hello'
'Yfloat-𝟘' = 3.5
['Xsubdoc-àéù']
String2 = 'One'
[['W.sublist-𝟘']]
String2 = 'Two'
[['W.sublist-𝟘']]
String2 = 'Three'
`
require.Equal(t, string(expected), string(result))
@@ -159,8 +160,8 @@ bool = false
int = 0
string = ''
stringlist = []
[map]
[map]
`
require.Equal(t, string(expected), string(result))
@@ -151,6 +151,7 @@ type quotedKeyMarshalTestStruct struct {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var quotedKeyMarshalTestData = quotedKeyMarshalTestStruct{
String: "Hello",
@@ -160,6 +161,7 @@ var quotedKeyMarshalTestData = quotedKeyMarshalTestStruct{
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var quotedKeyMarshalTestToml = []byte(`"Yfloat-𝟘" = 3.5
"Z.string-àéù" = "Hello"
@@ -272,6 +274,7 @@ var docData = testDoc{
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var mapTestDoc = testMapDoc{
Title: "TOML Marshal Testing",
@@ -457,35 +460,6 @@ func TestEmptytomlUnmarshal(t *testing.T) {
assert.Equal(t, emptyTestData, result)
}
func TestEmptyUnmarshalOmit(t *testing.T) {
t.Skipf("Have not figured yet if omitempty is a good idea")
type emptyMarshalTestStruct2 struct {
Title string `toml:"title"`
Bool bool `toml:"bool,omitempty"`
Int int `toml:"int, omitempty"`
String string `toml:"string,omitempty "`
StringList []string `toml:"stringlist,omitempty"`
Ptr *basicMarshalTestStruct `toml:"ptr,omitempty"`
Map map[string]string `toml:"map,omitempty"`
}
emptyTestData2 := emptyMarshalTestStruct2{
Title: "Placeholder",
Bool: false,
Int: 0,
String: "",
StringList: []string{},
Ptr: nil,
Map: map[string]string{},
}
result := emptyMarshalTestStruct2{}
err := toml.Unmarshal(emptyTestToml, &result)
require.NoError(t, err)
assert.Equal(t, emptyTestData2, result)
}
type pointerMarshalTestStruct struct {
Str *string
List *[]string
@@ -588,10 +562,12 @@ func (c customMarshaler) MarshalTOML() ([]byte, error) {
var customMarshalerData = customMarshaler{FirstName: "Sally", LastName: "Fields"}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var customMarshalerToml = []byte(`Sally Fields`)
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var nestedCustomMarshalerData = customMarshalerParent{
Self: customMarshaler{FirstName: "Maiku", LastName: "Suteda"},
@@ -599,6 +575,7 @@ var nestedCustomMarshalerData = customMarshalerParent{
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var nestedCustomMarshalerToml = []byte(`friends = ["Sally Fields"]
me = "Maiku Suteda"
@@ -640,6 +617,7 @@ func TestUnmarshalTextMarshaler(t *testing.T) {
}
// TODO: Remove nolint once type and methods are used by a test
//
//nolint:unused
type precedentMarshaler struct {
FirstName string
@@ -658,6 +636,7 @@ func (m precedentMarshaler) MarshalTOML() ([]byte, error) {
}
// TODO: Remove nolint once type and method are used by a test
//
//nolint:unused
type customPointerMarshaler struct {
FirstName string
@@ -670,6 +649,7 @@ func (m *customPointerMarshaler) MarshalTOML() ([]byte, error) {
}
// TODO: Remove nolint once type and method are used by a test
//
//nolint:unused
type textPointerMarshaler struct {
FirstName string
@@ -682,6 +662,7 @@ func (m *textPointerMarshaler) MarshalText() ([]byte, error) {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var commentTestToml = []byte(`
# it's a comment on type
@@ -719,6 +700,7 @@ type mapsTestStruct struct {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var mapsTestData = mapsTestStruct{
Simple: map[string]string{
@@ -742,6 +724,7 @@ var mapsTestData = mapsTestStruct{
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var mapsTestToml = []byte(`
[Other]
@@ -764,6 +747,7 @@ var mapsTestToml = []byte(`
`)
// TODO: Remove nolint once type is used by a test
//
//nolint:deadcode,unused
type structArrayNoTag struct {
A struct {
@@ -773,6 +757,7 @@ type structArrayNoTag struct {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var customTagTestToml = []byte(`
[postgres]
@@ -787,6 +772,7 @@ var customTagTestToml = []byte(`
`)
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var customCommentTagTestToml = []byte(`
# db connection
@@ -800,6 +786,7 @@ var customCommentTagTestToml = []byte(`
`)
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var customCommentedTagTestToml = []byte(`
[postgres]
@@ -854,6 +841,7 @@ func TestUnmarshalTabInStringAndQuotedKey(t *testing.T) {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var customMultilineTagTestToml = []byte(`int_slice = [
1,
@@ -863,6 +851,7 @@ var customMultilineTagTestToml = []byte(`int_slice = [
`)
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var testDocBasicToml = []byte(`
[document]
@@ -875,12 +864,14 @@ var testDocBasicToml = []byte(`
`)
// TODO: Remove nolint once type is used by a test
//
//nolint:deadcode
type testDocCustomTag struct {
Doc testDocBasicsCustomTag `file:"document"`
}
// TODO: Remove nolint once type is used by a test
//
//nolint:deadcode
type testDocBasicsCustomTag struct {
Bool bool `file:"bool_val"`
@@ -893,6 +884,7 @@ type testDocBasicsCustomTag struct {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,varcheck
var testDocCustomTagData = testDocCustomTag{
Doc: testDocBasicsCustomTag{
@@ -956,6 +948,29 @@ func TestUnmarshalMapWithTypedKey(t *testing.T) {
}
}
func TestUnmarshalTypeTableHeader(t *testing.T) {
testToml := []byte(`
[test]
a = 1
`)
type header string
var result map[header]map[string]int
err := toml.Unmarshal(testToml, &result)
if err != nil {
t.Errorf("Received unexpected error: %s", err)
return
}
expected := map[header]map[string]int{
"test": map[string]int{"a": 1},
}
if !reflect.DeepEqual(result, expected) {
t.Errorf("Bad unmarshal: expected %v, got %v", expected, result)
}
}
func TestUnmarshalNonPointer(t *testing.T) {
a := 1
err := toml.Unmarshal([]byte{}, a)
@@ -972,6 +987,7 @@ func TestUnmarshalInvalidPointerKind(t *testing.T) {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused
type testDuration struct {
Nanosec time.Duration `toml:"nanosec"`
@@ -986,6 +1002,7 @@ type testDuration struct {
}
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var testDurationToml = []byte(`
nanosec = "1ns"
@@ -1000,6 +1017,7 @@ a_string = "15s"
`)
// TODO: Remove nolint once var is used by a test
//
//nolint:deadcode,unused,varcheck
var testDurationToml2 = []byte(`a_string = "15s"
hour = "1h0m0s"
@@ -1013,6 +1031,7 @@ sec = "1s"
`)
// TODO: Remove nolint once type is used by a test
//
//nolint:deadcode,unused
type testBadDuration struct {
Val time.Duration `toml:"val"`
@@ -1954,7 +1973,7 @@ func decoder(doc string) *toml.Decoder {
func strictDecoder(doc string) *toml.Decoder {
d := decoder(doc)
d.SetStrict(true)
d.DisallowUnknownFields()
return d
}
@@ -43,7 +43,7 @@ func DecodeStdin() error {
j := json.NewEncoder(os.Stdout)
j.SetIndent("", " ")
if err := j.Encode(addTag("", decoded)); err != nil {
fmt.Errorf("Error encoding JSON: %s", err)
return fmt.Errorf("Error encoding JSON: %s", err)
}
return nil
+164 -67
View File
@@ -3,6 +3,7 @@ package tracker
import (
"bytes"
"fmt"
"sync"
"github.com/pelletier/go-toml/v2/internal/ast"
)
@@ -54,63 +55,104 @@ func (k keyKind) String() string {
type SeenTracker struct {
entries []entry
currentIdx int
nextID int
}
var pool sync.Pool
func (s *SeenTracker) reset() {
// Always contains a root element at index 0.
s.currentIdx = 0
if len(s.entries) == 0 {
s.entries = make([]entry, 1, 2)
} else {
s.entries = s.entries[:1]
}
s.entries[0].child = -1
s.entries[0].next = -1
}
type entry struct {
id int
parent int
// Use -1 to indicate no child or no sibling.
child int
next int
name []byte
kind keyKind
explicit bool
kv bool
}
// Remove all descendent of node at position idx.
// Find the index of the child of parentIdx with key k. Returns -1 if
// it does not exist.
func (s *SeenTracker) find(parentIdx int, k []byte) int {
for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next {
if bytes.Equal(s.entries[i].name, k) {
return i
}
}
return -1
}
// Remove all descendants of node at position idx.
func (s *SeenTracker) clear(idx int) {
p := s.entries[idx].id
rest := clear(p, s.entries[idx+1:])
s.entries = s.entries[:idx+1+len(rest)]
if idx >= len(s.entries) {
return
}
for i := s.entries[idx].child; i >= 0; {
next := s.entries[i].next
n := s.entries[0].next
s.entries[0].next = i
s.entries[i].next = n
s.entries[i].name = nil
s.clear(i)
i = next
}
s.entries[idx].child = -1
}
func clear(parentID int, entries []entry) []entry {
for i := 0; i < len(entries); {
if entries[i].parent == parentID {
id := entries[i].id
copy(entries[i:], entries[i+1:])
entries = entries[:len(entries)-1]
rest := clear(id, entries[i:])
entries = entries[:i+len(rest)]
} else {
i++
}
}
return entries
}
func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool, kv bool) int {
e := entry{
child: -1,
next: s.entries[parentIdx].child,
func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool) int {
parentID := s.id(parentIdx)
idx := len(s.entries)
s.entries = append(s.entries, entry{
id: s.nextID,
parent: parentID,
name: name,
kind: kind,
explicit: explicit,
})
s.nextID++
kv: kv,
}
var idx int
if s.entries[0].next >= 0 {
idx = s.entries[0].next
s.entries[0].next = s.entries[idx].next
s.entries[idx] = e
} else {
idx = len(s.entries)
s.entries = append(s.entries, e)
}
s.entries[parentIdx].child = idx
return idx
}
// CheckExpression takes a top-level node and checks that it does not contain keys
// that have been seen in previous calls, and validates that types are consistent.
func (s *SeenTracker) setExplicitFlag(parentIdx int) {
for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next {
if s.entries[i].kv {
s.entries[i].explicit = true
s.entries[i].kv = false
}
s.setExplicitFlag(i)
}
}
// CheckExpression takes a top-level node and checks that it does not contain
// keys that have been seen in previous calls, and validates that types are
// consistent.
func (s *SeenTracker) CheckExpression(node *ast.Node) error {
if s.entries == nil {
// Skip ID = 0 to remove the confusion between nodes whose parent has
// id 0 and root nodes (parent id is 0 because it's the zero value).
s.nextID = 1
// Start unscoped, so idx is negative.
s.currentIdx = -1
s.reset()
}
switch node.Kind {
case ast.KeyValue:
@@ -125,9 +167,13 @@ func (s *SeenTracker) CheckExpression(node *ast.Node) error {
}
func (s *SeenTracker) checkTable(node *ast.Node) error {
if s.currentIdx >= 0 {
s.setExplicitFlag(s.currentIdx)
}
it := node.Key()
parentIdx := -1
parentIdx := 0
// This code is duplicated in checkArrayTable. This is because factoring
// it in a function requires to copy the iterator, or allocate it to the
@@ -142,7 +188,12 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
idx := s.find(parentIdx, k)
if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false)
idx = s.create(parentIdx, k, tableKind, false, false)
} else {
entry := s.entries[idx]
if entry.kind == valueKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), entry.kind)
}
}
parentIdx = idx
}
@@ -160,7 +211,7 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
}
s.entries[idx].explicit = true
} else {
idx = s.create(parentIdx, k, tableKind, true)
idx = s.create(parentIdx, k, tableKind, true, false)
}
s.currentIdx = idx
@@ -169,9 +220,13 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
}
func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
if s.currentIdx >= 0 {
s.setExplicitFlag(s.currentIdx)
}
it := node.Key()
parentIdx := -1
parentIdx := 0
for it.Next() {
if it.IsLast() {
@@ -183,8 +238,14 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
idx := s.find(parentIdx, k)
if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false)
idx = s.create(parentIdx, k, tableKind, false, false)
} else {
entry := s.entries[idx]
if entry.kind == valueKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), entry.kind)
}
}
parentIdx = idx
}
@@ -198,7 +259,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
}
s.clear(idx)
} else {
idx = s.create(parentIdx, k, arrayTableKind, true)
idx = s.create(parentIdx, k, arrayTableKind, true, false)
}
s.currentIdx = idx
@@ -207,53 +268,89 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
}
func (s *SeenTracker) checkKeyValue(node *ast.Node) error {
it := node.Key()
parentIdx := s.currentIdx
it := node.Key()
for it.Next() {
k := it.Node().Data
idx := s.find(parentIdx, k)
if idx >= 0 {
if s.entries[idx].kind != tableKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), s.entries[idx].kind)
}
if s.entries[idx].explicit {
if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false, true)
} else {
entry := s.entries[idx]
if it.IsLast() {
return fmt.Errorf("toml: key %s is already defined", string(k))
} else if entry.kind != tableKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), entry.kind)
} else if entry.explicit {
return fmt.Errorf("toml: cannot redefine table %s that has already been explicitly defined", string(k))
}
} else {
idx = s.create(parentIdx, k, tableKind, false)
}
parentIdx = idx
}
kind := valueKind
s.entries[parentIdx].kind = valueKind
if node.Value().Kind == ast.InlineTable {
kind = tableKind
value := node.Value()
switch value.Kind {
case ast.InlineTable:
return s.checkInlineTable(value)
case ast.Array:
return s.checkArray(value)
}
s.entries[parentIdx].kind = kind
return nil
}
func (s *SeenTracker) id(idx int) int {
if idx >= 0 {
return s.entries[idx].id
func (s *SeenTracker) checkArray(node *ast.Node) error {
it := node.Children()
for it.Next() {
n := it.Node()
switch n.Kind {
case ast.InlineTable:
err := s.checkInlineTable(n)
if err != nil {
return err
}
return 0
case ast.Array:
err := s.checkArray(n)
if err != nil {
return err
}
}
}
return nil
}
func (s *SeenTracker) find(parentIdx int, k []byte) int {
parentID := s.id(parentIdx)
for i := parentIdx + 1; i < len(s.entries); i++ {
if s.entries[i].parent == parentID && bytes.Equal(s.entries[i].name, k) {
return i
func (s *SeenTracker) checkInlineTable(node *ast.Node) error {
if pool.New == nil {
pool.New = func() interface{} {
return &SeenTracker{}
}
}
return -1
s = pool.Get().(*SeenTracker)
s.reset()
it := node.Children()
for it.Next() {
n := it.Node()
err := s.checkKeyValue(n)
if err != nil {
return err
}
}
// As inline tables are self-contained, the tracker does not
// need to retain the details of what they contain. The
// keyValue element that creates the inline table is kept to
// mark the presence of the inline table and prevent
// redefinition of its keys: check* functions cannot walk into
// a value.
pool.Put(s)
return nil
}
+16
View File
@@ -0,0 +1,16 @@
package tracker
import (
"testing"
"unsafe"
"github.com/stretchr/testify/require"
)
func TestEntrySize(t *testing.T) {
// Validate no regression on the size of entry{}. This is a critical bit for
// performance of unmarshaling documents. Should only be increased with care
// and a very good reason.
maxExpectedEntrySize := 48
require.LessOrEqual(t, int(unsafe.Sizeof(entry{})), maxExpectedEntrySize)
}
+324 -90
View File
@@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"time"
"unicode"
)
// Marshal serializes a Go value as a TOML document.
@@ -53,9 +54,10 @@ func NewEncoder(w io.Writer) *Encoder {
// This behavior can be controlled on an individual struct field basis with the
// inline tag:
//
// MyField `inline:"true"`
func (enc *Encoder) SetTablesInline(inline bool) {
// MyField `toml:",inline"`
func (enc *Encoder) SetTablesInline(inline bool) *Encoder {
enc.tablesInline = inline
return enc
}
// SetArraysMultiline forces the encoder to emit all arrays with one element per
@@ -64,27 +66,30 @@ func (enc *Encoder) SetTablesInline(inline bool) {
// This behavior can be controlled on an individual struct field basis with the multiline tag:
//
// MyField `multiline:"true"`
func (enc *Encoder) SetArraysMultiline(multiline bool) {
func (enc *Encoder) SetArraysMultiline(multiline bool) *Encoder {
enc.arraysMultiline = multiline
return enc
}
// SetIndentSymbol defines the string that should be used for indentation. The
// provided string is repeated for each indentation level. Defaults to two
// spaces.
func (enc *Encoder) SetIndentSymbol(s string) {
func (enc *Encoder) SetIndentSymbol(s string) *Encoder {
enc.indentSymbol = s
return enc
}
// SetIndentTables forces the encoder to intent tables and array tables.
func (enc *Encoder) SetIndentTables(indent bool) {
func (enc *Encoder) SetIndentTables(indent bool) *Encoder {
enc.indentTables = indent
return enc
}
// Encode writes a TOML representation of v to the stream.
//
// If v cannot be represented to TOML it returns an error.
//
// Encoding rules
// # Encoding rules
//
// A top level slice containing only maps or structs is encoded as [[table
// array]].
@@ -99,27 +104,52 @@ func (enc *Encoder) SetIndentTables(indent bool) {
// Intermediate tables are always printed.
//
// By default, strings are encoded as literal string, unless they contain either
// a newline character or a single quote. In that case they are emitted as quoted
// strings.
// a newline character or a single quote. In that case they are emitted as
// quoted strings.
//
// Unsigned integers larger than math.MaxInt64 cannot be encoded. Doing so
// results in an error. This rule exists because the TOML specification only
// requires parsers to support at least the 64 bits integer range. Allowing
// larger numbers would create non-standard TOML documents, which may not be
// readable (at best) by other implementations. To encode such numbers, a
// solution is a custom type that implements encoding.TextMarshaler.
//
// When encoding structs, fields are encoded in order of definition, with their
// exact name.
//
// Struct tags
// Tables and array tables are separated by empty lines. However, consecutive
// subtables definitions are not. For example:
//
// The following struct tags are available to tweak encoding on a per-field
// basis:
// [top1]
//
// toml:"foo"
// Changes the name of the key to use for the field to foo.
// [top2]
// [top2.child1]
//
// multiline:"true"
// When the field contains a string, it will be emitted as a quoted
// multi-line TOML string.
// [[array]]
//
// inline:"true"
// When the field would normally be encoded as a table, it is instead
// encoded as an inline table.
// [[array]]
// [array.child2]
//
// # Struct tags
//
// The encoding of each public struct field can be customized by the format
// string in the "toml" key of the struct field's tag. This follows
// encoding/json's convention. The format string starts with the name of the
// field, optionally followed by a comma-separated list of options. The name may
// be empty in order to provide options without overriding the default name.
//
// The "multiline" option emits strings as quoted multi-line TOML strings. It
// has no effect on fields that would not be encoded as strings.
//
// The "inline" option turns fields that would be emitted as tables into inline
// tables instead. It has no effect on other fields.
//
// The "omitempty" option prevents empty values or groups from being emitted.
//
// In addition to the "toml" tag struct tag, a "comment" tag can be used to emit
// a TOML comment before the value being annotated. Comments are ignored inside
// inline tables. For array tables, the comment is only present before the first
// element of the array.
func (enc *Encoder) Encode(v interface{}) error {
var (
b []byte
@@ -147,6 +177,8 @@ func (enc *Encoder) Encode(v interface{}) error {
type valueOptions struct {
multiline bool
omitempty bool
comment string
}
type encoderCtx struct {
@@ -196,16 +228,29 @@ func (ctx *encoderCtx) isRoot() bool {
return len(ctx.parentKey) == 0 && !ctx.hasKey
}
//nolint:cyclop,funlen
func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
if !v.IsZero() {
i, ok := v.Interface().(time.Time)
if ok {
return i.AppendFormat(b, time.RFC3339), nil
i := v.Interface()
switch x := i.(type) {
case time.Time:
if x.Nanosecond() > 0 {
return x.AppendFormat(b, time.RFC3339Nano), nil
}
return x.AppendFormat(b, time.RFC3339), nil
case LocalTime:
return append(b, x.String()...), nil
case LocalDate:
return append(b, x.String()...), nil
case LocalDateTime:
return append(b, x.String()...), nil
}
hasTextMarshaler := v.Type().Implements(textMarshalerType)
if hasTextMarshaler || (v.CanAddr() && reflect.PtrTo(v.Type()).Implements(textMarshalerType)) {
if !hasTextMarshaler {
v = v.Addr()
}
if v.Type().Implements(textMarshalerType) {
if ctx.isRoot() {
return nil, fmt.Errorf("toml: type %s implementing the TextMarshaler interface cannot be a root element", v.Type())
}
@@ -245,16 +290,31 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
case reflect.String:
b = enc.encodeString(b, v.String(), ctx.options)
case reflect.Float32:
if math.Trunc(v.Float()) == v.Float() {
b = strconv.AppendFloat(b, v.Float(), 'f', 1, 32)
f := v.Float()
if math.IsNaN(f) {
b = append(b, "nan"...)
} else if f > math.MaxFloat32 {
b = append(b, "inf"...)
} else if f < -math.MaxFloat32 {
b = append(b, "-inf"...)
} else if math.Trunc(f) == f {
b = strconv.AppendFloat(b, f, 'f', 1, 32)
} else {
b = strconv.AppendFloat(b, v.Float(), 'f', -1, 32)
b = strconv.AppendFloat(b, f, 'f', -1, 32)
}
case reflect.Float64:
if math.Trunc(v.Float()) == v.Float() {
b = strconv.AppendFloat(b, v.Float(), 'f', 1, 64)
f := v.Float()
if math.IsNaN(f) {
b = append(b, "nan"...)
} else if f > math.MaxFloat64 {
b = append(b, "inf"...)
} else if f < -math.MaxFloat64 {
b = append(b, "-inf"...)
} else if math.Trunc(f) == f {
b = strconv.AppendFloat(b, f, 'f', 1, 64)
} else {
b = strconv.AppendFloat(b, v.Float(), 'f', -1, 64)
b = strconv.AppendFloat(b, f, 'f', -1, 64)
}
case reflect.Bool:
if v.Bool() {
@@ -263,7 +323,11 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
b = append(b, "false"...)
}
case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint:
b = strconv.AppendUint(b, v.Uint(), 10)
x := v.Uint()
if x > uint64(math.MaxInt64) {
return nil, fmt.Errorf("toml: not encoding uint (%d) greater than max int64 (%d)", x, int64(math.MaxInt64))
}
b = strconv.AppendUint(b, x, 10)
case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int:
b = strconv.AppendInt(b, v.Int(), 10)
default:
@@ -282,19 +346,19 @@ func isNil(v reflect.Value) bool {
}
}
func shouldOmitEmpty(options valueOptions, v reflect.Value) bool {
return options.omitempty && isEmptyValue(v)
}
func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v reflect.Value) ([]byte, error) {
var err error
if !ctx.hasKey {
panic("caller of encodeKv should have set the key in the context")
if !ctx.inline {
b = enc.encodeComment(ctx.indent, options.comment, b)
}
b = enc.indent(ctx.indent, b)
b, err = enc.encodeKey(b, ctx.key)
if err != nil {
return nil, err
}
b = enc.encodeKey(b, ctx.key)
b = append(b, " = "...)
// create a copy of the context because the value of a KV shouldn't
@@ -312,6 +376,54 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
return b, nil
}
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Struct:
return isEmptyStruct(v)
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
return false
}
func isEmptyStruct(v reflect.Value) bool {
// TODO: merge with walkStruct and cache.
typ := v.Type()
for i := 0; i < typ.NumField(); i++ {
fieldType := typ.Field(i)
// only consider exported fields
if fieldType.PkgPath != "" {
continue
}
tag := fieldType.Tag.Get("toml")
// special field name to skip field
if tag == "-" {
continue
}
f := v.Field(i)
if !isEmptyValue(f) {
return false
}
}
return true
}
const literalQuote = '\''
func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byte {
@@ -323,7 +435,13 @@ func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byt
}
func needsQuoting(v string) bool {
return strings.ContainsAny(v, "'\b\f\n\r\t")
// TODO: vectorize
for _, b := range []byte(v) {
if b == '\'' || b == '\r' || b == '\n' || invalidAscii(b) {
return true
}
}
return false
}
// caller should have checked that the string does not contain new lines or ' .
@@ -335,7 +453,6 @@ func (enc *Encoder) encodeLiteralString(b []byte, v string) []byte {
return b
}
//nolint:cyclop
func (enc *Encoder) encodeQuotedString(multiline bool, b []byte, v string) []byte {
stringQuote := `"`
@@ -395,7 +512,7 @@ func (enc *Encoder) encodeQuotedString(multiline bool, b []byte, v string) []byt
return b
}
// called should have checked that the string is in A-Z / a-z / 0-9 / - / _ .
// caller should have checked that the string is in A-Z / a-z / 0-9 / - / _ .
func (enc *Encoder) encodeUnquotedKey(b []byte, v string) []byte {
return append(b, v...)
}
@@ -405,24 +522,17 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
return b, nil
}
b = enc.encodeComment(ctx.indent, ctx.options.comment, b)
b = enc.indent(ctx.indent, b)
b = append(b, '[')
var err error
b, err = enc.encodeKey(b, ctx.parentKey[0])
if err != nil {
return nil, err
}
b = enc.encodeKey(b, ctx.parentKey[0])
for _, k := range ctx.parentKey[1:] {
b = append(b, '.')
b, err = enc.encodeKey(b, k)
if err != nil {
return nil, err
}
b = enc.encodeKey(b, k)
}
b = append(b, "]\n"...)
@@ -431,19 +541,19 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
}
//nolint:cyclop
func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
func (enc *Encoder) encodeKey(b []byte, k string) []byte {
needsQuotation := false
cannotUseLiteral := false
if len(k) == 0 {
return append(b, "''"...)
}
for _, c := range k {
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
continue
}
if c == '\n' {
return nil, fmt.Errorf("toml: new line characters in keys are not supported")
}
if c == literalQuote {
cannotUseLiteral = true
}
@@ -451,13 +561,17 @@ func (enc *Encoder) encodeKey(b []byte, k string) ([]byte, error) {
needsQuotation = true
}
if needsQuotation && needsQuoting(k) {
cannotUseLiteral = true
}
switch {
case cannotUseLiteral:
return enc.encodeQuotedString(false, b, k), nil
return enc.encodeQuotedString(false, b, k)
case needsQuotation:
return enc.encodeLiteralString(b, k), nil
return enc.encodeLiteralString(b, k)
default:
return enc.encodeUnquotedKey(b, k), nil
return enc.encodeUnquotedKey(b, k)
}
}
@@ -511,18 +625,26 @@ type table struct {
}
func (t *table) pushKV(k string, v reflect.Value, options valueOptions) {
for _, e := range t.kvs {
if e.Key == k {
return
}
}
t.kvs = append(t.kvs, entry{Key: k, Value: v, Options: options})
}
func (t *table) pushTable(k string, v reflect.Value, options valueOptions) {
for _, e := range t.tables {
if e.Key == k {
return
}
}
t.tables = append(t.tables, entry{Key: k, Value: v, Options: options})
}
func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
var t table
//nolint:godox
// TODO: cache this?
func walkStruct(ctx encoderCtx, t *table, v reflect.Value) {
// TODO: cache this
typ := v.Type()
for i := 0; i < typ.NumField(); i++ {
fieldType := typ.Field(i)
@@ -532,45 +654,130 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
continue
}
k, ok := fieldType.Tag.Lookup("toml")
if !ok {
k = fieldType.Name
}
tag := fieldType.Tag.Get("toml")
// special field name to skip field
if k == "-" {
if tag == "-" {
continue
}
k, opts := parseTag(tag)
if !isValidName(k) {
k = ""
}
f := v.Field(i)
if k == "" {
if fieldType.Anonymous {
if fieldType.Type.Kind() == reflect.Struct {
walkStruct(ctx, t, f)
}
continue
} else {
k = fieldType.Name
}
}
if isNil(f) {
continue
}
options := valueOptions{
multiline: fieldBoolTag(fieldType, "multiline"),
multiline: opts.multiline,
omitempty: opts.omitempty,
comment: fieldType.Tag.Get("comment"),
}
inline := fieldBoolTag(fieldType, "inline")
if inline || !willConvertToTableOrArrayTable(ctx, f) {
if opts.inline || !willConvertToTableOrArrayTable(ctx, f) {
t.pushKV(k, f, options)
} else {
t.pushTable(k, f, options)
}
}
}
func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
var t table
walkStruct(ctx, &t, v)
return enc.encodeTable(b, ctx, t)
}
func fieldBoolTag(field reflect.StructField, tag string) bool {
x, ok := field.Tag.Lookup(tag)
return ok && x == "true"
func (enc *Encoder) encodeComment(indent int, comment string, b []byte) []byte {
for len(comment) > 0 {
var line string
idx := strings.IndexByte(comment, '\n')
if idx >= 0 {
line = comment[:idx]
comment = comment[idx+1:]
} else {
line = comment
comment = ""
}
b = enc.indent(indent, b)
b = append(b, "# "...)
b = append(b, line...)
b = append(b, '\n')
}
return b
}
func isValidName(s string) bool {
if s == "" {
return false
}
for _, c := range s {
switch {
case strings.ContainsRune("!#$%&()*+-./:;<=>?@[]^_{|}~ ", c):
// Backslash and quote chars are reserved, but
// otherwise any punctuation chars are allowed
// in a tag name.
case !unicode.IsLetter(c) && !unicode.IsDigit(c):
return false
}
}
return true
}
type tagOptions struct {
multiline bool
inline bool
omitempty bool
}
func parseTag(tag string) (string, tagOptions) {
opts := tagOptions{}
idx := strings.Index(tag, ",")
if idx == -1 {
return tag, opts
}
raw := tag[idx+1:]
tag = string(tag[:idx])
for raw != "" {
var o string
i := strings.Index(raw, ",")
if i >= 0 {
o, raw = raw[:i], raw[i+1:]
} else {
o, raw = raw, ""
}
switch o {
case "multiline":
opts.multiline = true
case "inline":
opts.inline = true
case "omitempty":
opts.omitempty = true
}
}
return tag, opts
}
//nolint:cyclop
func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, error) {
var err error
@@ -592,7 +799,13 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
}
ctx.skipTableHeader = false
hasNonEmptyKV := false
for _, kv := range t.kvs {
if shouldOmitEmpty(kv.Options, kv.Value) {
continue
}
hasNonEmptyKV = true
ctx.setKey(kv.Key)
b, err = enc.encodeKv(b, ctx, kv.Options, kv.Value)
@@ -603,7 +816,20 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
b = append(b, '\n')
}
first := true
for _, table := range t.tables {
if shouldOmitEmpty(table.Options, table.Value) {
continue
}
if first {
first = false
if hasNonEmptyKV {
b = append(b, '\n')
}
} else {
b = append(b, "\n"...)
}
ctx.setKey(table.Key)
ctx.options = table.Options
@@ -612,8 +838,6 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
if err != nil {
return nil, err
}
b = append(b, '\n')
}
return b, nil
@@ -626,6 +850,10 @@ func (enc *Encoder) encodeTableInline(b []byte, ctx encoderCtx, t table) ([]byte
first := true
for _, kv := range t.kvs {
if shouldOmitEmpty(kv.Options, kv.Value) {
continue
}
if first {
first = false
} else {
@@ -641,7 +869,7 @@ func (enc *Encoder) encodeTableInline(b []byte, ctx encoderCtx, t table) ([]byte
}
if len(t.tables) > 0 {
panic("inline table cannot contain nested tables, online key-values")
panic("inline table cannot contain nested tables, only key-values")
}
b = append(b, "}"...)
@@ -653,7 +881,7 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
if !v.IsValid() {
return false
}
if v.Type() == timeType || v.Type().Implements(textMarshalerType) {
if v.Type() == timeType || v.Type().Implements(textMarshalerType) || (v.Kind() != reflect.Ptr && v.CanAddr() && reflect.PtrTo(v.Type()).Implements(textMarshalerType)) {
return false
}
@@ -675,6 +903,9 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
}
func willConvertToTableOrArrayTable(ctx encoderCtx, v reflect.Value) bool {
if ctx.insideKv {
return false
}
t := v.Type()
if t.Kind() == reflect.Interface {
@@ -720,7 +951,6 @@ func (enc *Encoder) encodeSlice(b []byte, ctx encoderCtx, v reflect.Value) ([]by
func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
ctx.shiftKey()
var err error
scratch := make([]byte, 0, 64)
scratch = append(scratch, "[["...)
@@ -729,18 +959,22 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
scratch = append(scratch, '.')
}
scratch, err = enc.encodeKey(scratch, k)
if err != nil {
return nil, err
}
scratch = enc.encodeKey(scratch, k)
}
scratch = append(scratch, "]]\n"...)
ctx.skipTableHeader = true
b = enc.encodeComment(ctx.indent, ctx.options.comment, b)
for i := 0; i < v.Len(); i++ {
if i != 0 {
b = append(b, "\n"...)
}
b = append(b, scratch...)
var err error
b, err = enc.encode(b, ctx, v.Index(i))
if err != nil {
return nil, err
+475 -106
View File
@@ -4,20 +4,28 @@ import (
"bytes"
"encoding/json"
"fmt"
"math"
"math/big"
"strings"
"testing"
"time"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
//nolint:funlen
func TestMarshal(t *testing.T) {
someInt := 42
type structInline struct {
A interface{} `inline:"true"`
A interface{} `toml:",inline"`
}
type comments struct {
One int
Two int `comment:"Before kv"`
Three []int `comment:"Before array"`
}
examples := []struct {
@@ -31,21 +39,21 @@ func TestMarshal(t *testing.T) {
v: map[string]string{
"hello": "world",
},
expected: "hello = 'world'",
expected: "hello = 'world'\n",
},
{
desc: "map with new line in key",
v: map[string]string{
"hel\nlo": "world",
},
err: true,
expected: "\"hel\\nlo\" = 'world'\n",
},
{
desc: `map with " in key`,
v: map[string]string{
`hel"lo`: "world",
},
expected: `'hel"lo' = 'world'`,
expected: "'hel\"lo' = 'world'\n",
},
{
desc: "map in map and string",
@@ -54,9 +62,9 @@ func TestMarshal(t *testing.T) {
"hello": "world",
},
},
expected: `
[table]
hello = 'world'`,
expected: `[table]
hello = 'world'
`,
},
{
desc: "map in map in map and string",
@@ -67,10 +75,10 @@ hello = 'world'`,
},
},
},
expected: `
[this]
expected: `[this]
[this.is]
a = 'test'`,
a = 'test'
`,
},
{
desc: "map in map in map and string with values",
@@ -82,18 +90,20 @@ a = 'test'`,
"also": "that",
},
},
expected: `
[this]
expected: `[this]
also = 'that'
[this.is]
a = 'test'`,
a = 'test'
`,
},
{
desc: "simple string array",
v: map[string][]string{
"array": {"one", "two", "three"},
},
expected: `array = ['one', 'two', 'three']`,
expected: `array = ['one', 'two', 'three']
`,
},
{
desc: "empty string array",
@@ -110,14 +120,16 @@ a = 'test'`,
v: map[string][][]string{
"array": {{"one", "two"}, {"three"}},
},
expected: `array = [['one', 'two'], ['three']]`,
expected: `array = [['one', 'two'], ['three']]
`,
},
{
desc: "mixed strings and nested string arrays",
v: map[string][]interface{}{
"array": {"a string", []string{"one", "two"}, "last"},
},
expected: `array = ['a string', ['one', 'two'], 'last']`,
expected: `array = ['a string', ['one', 'two'], 'last']
`,
},
{
desc: "array of maps",
@@ -127,9 +139,9 @@ a = 'test'`,
{"map2.1": "v2.1"},
},
},
expected: `
[[top]]
expected: `[[top]]
'map1.1' = 'v1.1'
[[top]]
'map2.1' = 'v2.1'
`,
@@ -140,9 +152,9 @@ a = 'test'`,
"key1": "value1",
"key2": "value2",
},
expected: `
key1 = 'value1'
key2 = 'value2'`,
expected: `key1 = 'value1'
key2 = 'value2'
`,
},
{
desc: "simple struct",
@@ -151,7 +163,8 @@ key2 = 'value2'`,
}{
A: "foo",
},
expected: `A = 'foo'`,
expected: `A = 'foo'
`,
},
{
desc: "one level of structs within structs",
@@ -166,8 +179,7 @@ key2 = 'value2'`,
K2: "v2",
},
},
expected: `
[A]
expected: `[A]
K1 = 'v1'
K2 = 'v2'
`,
@@ -182,10 +194,10 @@ K2 = 'v2'
},
},
},
expected: `
[root]
expected: `[root]
[[root.nested]]
name = 'Bob'
[[root.nested]]
name = 'Alice'
`,
@@ -193,67 +205,72 @@ name = 'Alice'
{
desc: "string escapes",
v: map[string]interface{}{
"a": `'"\`,
"a": "'\b\f\r\t\"\\",
},
expected: `a = "'\"\\"`,
expected: `a = "'\b\f\r\t\"\\"
`,
},
{
desc: "string utf8 low",
v: map[string]interface{}{
"a": "'Ę",
},
expected: `a = "'Ę"`,
expected: `a = "'Ę"
`,
},
{
desc: "string utf8 low 2",
v: map[string]interface{}{
"a": "'\u10A85",
},
expected: "a = \"'\u10A85\"",
expected: "a = \"'\u10A85\"\n",
},
{
desc: "string utf8 low 2",
v: map[string]interface{}{
"a": "'\u10A85",
},
expected: "a = \"'\u10A85\"",
expected: "a = \"'\u10A85\"\n",
},
{
desc: "emoji",
v: map[string]interface{}{
"a": "'😀",
},
expected: "a = \"'😀\"",
expected: "a = \"'😀\"\n",
},
{
desc: "control char",
v: map[string]interface{}{
"a": "'\u001A",
},
expected: `a = "'\u001A"`,
expected: `a = "'\u001A"
`,
},
{
desc: "multi-line string",
v: map[string]interface{}{
"a": "hello\nworld",
},
expected: `a = "hello\nworld"`,
expected: `a = "hello\nworld"
`,
},
{
desc: "multi-line forced",
v: struct {
A string `multiline:"true"`
A string `toml:",multiline"`
}{
A: "hello\nworld",
},
expected: `A = """
hello
world"""`,
world"""
`,
},
{
desc: "inline field",
v: struct {
A map[string]string `inline:"true"`
A map[string]string `toml:",inline"`
B map[string]string
}{
A: map[string]string{
@@ -263,8 +280,8 @@ world"""`,
"isinline": "no",
},
},
expected: `
A = {isinline = 'yes'}
expected: `A = {isinline = 'yes'}
[B]
isinline = 'no'
`,
@@ -272,14 +289,13 @@ isinline = 'no'
{
desc: "mutiline array int",
v: struct {
A []int `multiline:"true"`
A []int `toml:",multiline"`
B []int
}{
A: []int{1, 2, 3, 4},
B: []int{1, 2, 3, 4},
},
expected: `
A = [
expected: `A = [
1,
2,
3,
@@ -291,12 +307,11 @@ B = [1, 2, 3, 4]
{
desc: "mutiline array in array",
v: struct {
A [][]int `multiline:"true"`
A [][]int `toml:",multiline"`
}{
A: [][]int{{1, 2}, {3, 4}},
},
expected: `
A = [
expected: `A = [
[1, 2],
[3, 4]
]
@@ -321,7 +336,8 @@ A = [
}{
A: []*int{nil},
},
expected: `A = [0]`,
expected: `A = [0]
`,
},
{
desc: "nil pointer in slice uses zero value",
@@ -330,7 +346,8 @@ A = [
}{
A: []*int{nil},
},
expected: `A = [0]`,
expected: `A = [0]
`,
},
{
desc: "pointer in slice",
@@ -339,7 +356,8 @@ A = [
}{
A: []*int{&someInt},
},
expected: `A = [42]`,
expected: `A = [42]
`,
},
{
desc: "inline table in inline table",
@@ -350,30 +368,34 @@ A = [
},
},
},
expected: `A = {A = {A = 'hello'}}`,
expected: `A = {A = {A = 'hello'}}
`,
},
{
desc: "empty slice in map",
v: map[string][]string{
"a": {},
},
expected: `a = []`,
expected: `a = []
`,
},
{
desc: "map in slice",
v: map[string][]map[string]string{
"a": {{"hello": "world"}},
},
expected: `
[[a]]
hello = 'world'`,
expected: `[[a]]
hello = 'world'
`,
},
{
desc: "newline in map in slice",
v: map[string][]map[string]string{
"a\n": {{"hello": "world"}},
},
err: true,
expected: `[["a\n"]]
hello = 'world'
`,
},
{
desc: "newline in map in slice",
@@ -389,7 +411,8 @@ hello = 'world'`,
}{
A: []struct{}{},
},
expected: `A = []`,
expected: `A = []
`,
},
{
desc: "nil field is ignored",
@@ -409,7 +432,8 @@ hello = 'world'`,
Public: "shown",
private: "hidden",
},
expected: `Public = 'shown'`,
expected: `Public = 'shown'
`,
},
{
desc: "fields tagged - are ignored",
@@ -433,7 +457,8 @@ hello = 'world'`,
v: map[string]interface{}{
"hello\nworld": 42,
},
err: true,
expected: `"hello\nworld" = 42
`,
},
{
desc: "new line in parent of nested table key",
@@ -442,7 +467,9 @@ hello = 'world'`,
"inner": 42,
},
},
err: true,
expected: `["hello\nworld"]
inner = 42
`,
},
{
desc: "new line in nested table key",
@@ -453,7 +480,10 @@ hello = 'world'`,
},
},
},
err: true,
expected: `[parent]
[parent."in\ner"]
foo = 42
`,
},
{
desc: "invalid map key",
@@ -469,6 +499,39 @@ hello = 'world'`,
},
err: true,
},
{
desc: "time",
v: struct {
T time.Time
}{
T: time.Time{},
},
expected: `T = 0001-01-01T00:00:00Z
`,
},
{
desc: "time nano",
v: struct {
T time.Time
}{
T: time.Date(1979, time.May, 27, 0, 32, 0, 999999000, time.UTC),
},
expected: `T = 1979-05-27T00:32:00.999999Z
`,
},
{
desc: "bool",
v: struct {
A bool
B bool
}{
A: false,
B: true,
},
expected: `A = false
B = true
`,
},
{
desc: "numbers",
v: struct {
@@ -483,6 +546,7 @@ hello = 'world'`,
I int16
J int8
K int
L float64
}{
A: 1.1,
B: 42,
@@ -495,9 +559,9 @@ hello = 'world'`,
I: 42,
J: 42,
K: 42,
L: 2.2,
},
expected: `
A = 1.1
expected: `A = 1.1
B = 42
C = 42
D = 42
@@ -507,7 +571,29 @@ G = 42
H = 42
I = 42
J = 42
K = 42`,
K = 42
L = 2.2
`,
},
{
desc: "comments",
v: struct {
Table comments `comment:"Before table"`
}{
Table: comments{
One: 1,
Two: 2,
Three: []int{1, 2, 3},
},
},
expected: `# Before table
[Table]
One = 1
# Before kv
Two = 2
# Before array
Three = [1, 2, 3]
`,
},
}
@@ -522,7 +608,7 @@ K = 42`,
}
require.NoError(t, err)
equalStringsIgnoreNewlines(t, e.expected, string(b))
assert.Equal(t, e.expected, string(b))
// make sure the output is always valid TOML
defaultMap := map[string]interface{}{}
@@ -551,7 +637,7 @@ K = 42`,
type flagsSetters []struct {
name string
f func(enc *toml.Encoder, flag bool)
f func(enc *toml.Encoder, flag bool) *toml.Encoder
}
var allFlags = flagsSetters{
@@ -597,10 +683,31 @@ func testWithFlags(t *testing.T, flags int, setters flagsSetters, testfn func(t
}
}
func equalStringsIgnoreNewlines(t *testing.T, expected string, actual string) {
t.Helper()
cutset := "\n"
assert.Equal(t, strings.Trim(expected, cutset), strings.Trim(actual, cutset))
func TestMarshalFloats(t *testing.T) {
v := map[string]float32{
"nan": float32(math.NaN()),
"+inf": float32(math.Inf(1)),
"-inf": float32(math.Inf(-1)),
}
expected := `'+inf' = inf
-inf = -inf
nan = nan
`
actual, err := toml.Marshal(v)
require.NoError(t, err)
require.Equal(t, expected, string(actual))
v64 := map[string]float64{
"nan": math.NaN(),
"+inf": math.Inf(1),
"-inf": math.Inf(-1),
}
actual, err = toml.Marshal(v64)
require.NoError(t, err)
require.Equal(t, expected, string(actual))
}
//nolint:funlen
@@ -615,7 +722,8 @@ func TestMarshalIndentTables(t *testing.T) {
v: map[string]interface{}{
"foo": "bar",
},
expected: `foo = 'bar'`,
expected: `foo = 'bar'
`,
},
{
desc: "one level table",
@@ -625,8 +733,7 @@ func TestMarshalIndentTables(t *testing.T) {
"two": "value2",
},
},
expected: `
[foo]
expected: `[foo]
one = 'value1'
two = 'value2'
`,
@@ -642,10 +749,11 @@ func TestMarshalIndentTables(t *testing.T) {
},
},
},
expected: `
root = 'value0'
expected: `root = 'value0'
[level1]
one = 'value1'
[level1.level2]
two = 'value2'
`,
@@ -660,7 +768,7 @@ root = 'value0'
enc.SetIndentTables(true)
err := enc.Encode(e.v)
require.NoError(t, err)
equalStringsIgnoreNewlines(t, e.expected, buf.String())
assert.Equal(t, e.expected, buf.String())
})
}
}
@@ -705,7 +813,7 @@ func TestMarshalTextMarshaler(t *testing.T) {
m := map[string]interface{}{"a": &customTextMarshaler{value: 2}}
r, err := toml.Marshal(m)
require.NoError(t, err)
equalStringsIgnoreNewlines(t, "a = '::2'", string(r))
assert.Equal(t, "a = '::2'\n", string(r))
}
type brokenWriter struct{}
@@ -728,10 +836,63 @@ func TestEncoderSetIndentSymbol(t *testing.T) {
enc.SetIndentSymbol(">>>")
err := enc.Encode(map[string]map[string]string{"parent": {"hello": "world"}})
require.NoError(t, err)
expected := `
[parent]
>>>hello = 'world'`
equalStringsIgnoreNewlines(t, expected, w.String())
expected := `[parent]
>>>hello = 'world'
`
assert.Equal(t, expected, w.String())
}
func TestEncoderOmitempty(t *testing.T) {
type doc struct {
String string `toml:",omitempty,multiline"`
Bool bool `toml:",omitempty,multiline"`
Int int `toml:",omitempty,multiline"`
Int8 int8 `toml:",omitempty,multiline"`
Int16 int16 `toml:",omitempty,multiline"`
Int32 int32 `toml:",omitempty,multiline"`
Int64 int64 `toml:",omitempty,multiline"`
Uint uint `toml:",omitempty,multiline"`
Uint8 uint8 `toml:",omitempty,multiline"`
Uint16 uint16 `toml:",omitempty,multiline"`
Uint32 uint32 `toml:",omitempty,multiline"`
Uint64 uint64 `toml:",omitempty,multiline"`
Float32 float32 `toml:",omitempty,multiline"`
Float64 float64 `toml:",omitempty,multiline"`
MapNil map[string]string `toml:",omitempty,multiline"`
Slice []string `toml:",omitempty,multiline"`
Ptr *string `toml:",omitempty,multiline"`
Iface interface{} `toml:",omitempty,multiline"`
Struct struct{} `toml:",omitempty,multiline"`
}
d := doc{}
b, err := toml.Marshal(d)
require.NoError(t, err)
expected := ``
assert.Equal(t, expected, string(b))
}
func TestEncoderTagFieldName(t *testing.T) {
type doc struct {
String string `toml:"hello"`
OkSym string `toml:"#"`
Bad string `toml:"\"`
}
d := doc{String: "world"}
b, err := toml.Marshal(d)
require.NoError(t, err)
expected := `hello = 'world'
'#' = ''
Bad = ''
`
assert.Equal(t, expected, string(b))
}
func TestIssue436(t *testing.T) {
@@ -745,12 +906,11 @@ func TestIssue436(t *testing.T) {
err = toml.NewEncoder(&buf).Encode(v)
require.NoError(t, err)
expected := `
[[a]]
expected := `[[a]]
[a.b]
c = 'd'
`
equalStringsIgnoreNewlines(t, expected, buf.String())
assert.Equal(t, expected, buf.String())
}
func TestIssue424(t *testing.T) {
@@ -798,6 +958,238 @@ func TestIssue590(t *testing.T) {
require.NoError(t, err)
}
func TestIssue571(t *testing.T) {
type Foo struct {
Float32 float32
Float64 float64
}
const closeEnough = 1e-9
foo := Foo{
Float32: 42,
Float64: 43,
}
b, err := toml.Marshal(foo)
require.NoError(t, err)
var foo2 Foo
err = toml.Unmarshal(b, &foo2)
require.NoError(t, err)
assert.InDelta(t, 42, foo2.Float32, closeEnough)
assert.InDelta(t, 43, foo2.Float64, closeEnough)
}
func TestIssue678(t *testing.T) {
type Config struct {
BigInt big.Int
}
cfg := &Config{
BigInt: *big.NewInt(123),
}
out, err := toml.Marshal(cfg)
require.NoError(t, err)
assert.Equal(t, "BigInt = '123'\n", string(out))
cfg2 := &Config{}
err = toml.Unmarshal(out, cfg2)
require.NoError(t, err)
require.Equal(t, cfg, cfg2)
}
func TestIssue752(t *testing.T) {
type Fooer interface {
Foo() string
}
type Container struct {
Fooer
}
c := Container{}
out, err := toml.Marshal(c)
require.NoError(t, err)
require.Equal(t, "", string(out))
}
func TestIssue768(t *testing.T) {
type cfg struct {
Name string `comment:"This is a multiline comment.\nThis is line 2."`
}
out, err := toml.Marshal(&cfg{})
require.NoError(t, err)
expected := `# This is a multiline comment.
# This is line 2.
Name = ''
`
require.Equal(t, expected, string(out))
}
func TestIssue786(t *testing.T) {
type Dependencies struct {
Dependencies []string `toml:"dependencies,multiline,omitempty"`
BuildDependencies []string `toml:"buildDependencies,multiline,omitempty"`
OptionalDependencies []string `toml:"optionalDependencies,multiline,omitempty"`
}
type Test struct {
Dependencies Dependencies `toml:"dependencies,omitempty"`
}
x := Test{}
b, err := toml.Marshal(x)
require.NoError(t, err)
require.Equal(t, "", string(b))
type General struct {
From string `toml:"from,omitempty" json:"from,omitempty" comment:"from in graphite-web format, the local TZ is used"`
Randomize bool `toml:"randomize" json:"randomize" comment:"randomize starting time with [0,step)"`
}
type Custom struct {
Name string `toml:"name" json:"name,omitempty" comment:"names for generator, braces are expanded like in shell"`
Type string `toml:"type,omitempty" json:"type,omitempty" comment:"type of generator"`
General
}
type Config struct {
General
Custom []Custom `toml:"custom,omitempty" json:"custom,omitempty" comment:"generators with custom parameters can be specified separately"`
}
buf := new(bytes.Buffer)
config := &Config{General: General{From: "-2d", Randomize: true}}
config.Custom = []Custom{{Name: "omit", General: General{Randomize: false}}}
config.Custom = append(config.Custom, Custom{Name: "present", General: General{From: "-2d", Randomize: true}})
encoder := toml.NewEncoder(buf)
encoder.Encode(config)
expected := `# from in graphite-web format, the local TZ is used
from = '-2d'
# randomize starting time with [0,step)
randomize = true
# generators with custom parameters can be specified separately
[[custom]]
# names for generator, braces are expanded like in shell
name = 'omit'
# randomize starting time with [0,step)
randomize = false
[[custom]]
# names for generator, braces are expanded like in shell
name = 'present'
# from in graphite-web format, the local TZ is used
from = '-2d'
# randomize starting time with [0,step)
randomize = true
`
require.Equal(t, expected, buf.String())
}
func TestMarshalNestedAnonymousStructs(t *testing.T) {
type Embedded struct {
Value string `toml:"value" json:"value"`
Top struct {
Value string `toml:"value" json:"value"`
} `toml:"top" json:"top"`
}
type Named struct {
Value string `toml:"value" json:"value"`
}
var doc struct {
Embedded
Named `toml:"named" json:"named"`
Anonymous struct {
Value string `toml:"value" json:"value"`
} `toml:"anonymous" json:"anonymous"`
}
expected := `value = ''
[top]
value = ''
[named]
value = ''
[anonymous]
value = ''
`
result, err := toml.Marshal(doc)
require.NoError(t, err)
require.Equal(t, expected, string(result))
}
func TestMarshalNestedAnonymousStructs_DuplicateField(t *testing.T) {
type Embedded struct {
Value string `toml:"value" json:"value"`
Top struct {
Value string `toml:"value" json:"value"`
} `toml:"top" json:"top"`
}
var doc struct {
Value string `toml:"value" json:"value"`
Embedded
}
doc.Embedded.Value = "shadowed"
doc.Value = "shadows"
expected := `value = 'shadows'
[top]
value = ''
`
result, err := toml.Marshal(doc)
require.NoError(t, err)
require.NoError(t, err)
require.Equal(t, expected, string(result))
}
func TestLocalTime(t *testing.T) {
v := map[string]toml.LocalTime{
"a": {
Hour: 1,
Minute: 2,
Second: 3,
Nanosecond: 4,
},
}
expected := `a = 01:02:03.000000004
`
out, err := toml.Marshal(v)
require.NoError(t, err)
require.Equal(t, expected, string(out))
}
func TestMarshalUint64Overflow(t *testing.T) {
// The TOML spec only requires implementation to provide support for the
// int64 range. To avoid generating TOML documents that would not be
// supported by standard-compliant parsers, uint64 > max int64 cannot be
// marshaled.
x := map[string]interface{}{
"foo": uint64(math.MaxInt64) + 1,
}
_, err := toml.Marshal(x)
require.Error(t, err)
}
func ExampleMarshal() {
type MyConfig struct {
Version int
@@ -822,26 +1214,3 @@ func ExampleMarshal() {
// Name = 'go-toml'
// Tags = ['go', 'toml']
}
func TestIssue571(t *testing.T) {
type Foo struct {
Float32 float32
Float64 float64
}
const closeEnough = 1e-9
foo := Foo{
Float32: 42,
Float64: 43,
}
b, err := toml.Marshal(foo)
require.NoError(t, err)
var foo2 Foo
err = toml.Unmarshal(b, &foo2)
require.NoError(t, err)
assert.InDelta(t, 42, foo2.Float32, closeEnough)
assert.InDelta(t, 43, foo2.Float64, closeEnough)
}
+100 -62
View File
@@ -28,6 +28,11 @@ func (p *parser) Raw(raw ast.Range) []byte {
return p.data[raw.Offset : raw.Offset+raw.Length]
}
func (p *parser) SetRaw(ref ast.Reference, from []byte, to []byte) {
b := danger.BytesRange(from, to)
p.builder.NodeAt(ref).Raw = p.Range(b)
}
func (p *parser) Reset(b []byte) {
p.builder.Reset()
p.ref = ast.InvalidReference
@@ -152,12 +157,14 @@ func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) {
ref := p.builder.Push(ast.Node{
Kind: ast.ArrayTable,
})
start := b
b = b[2:]
b = p.parseWhitespace(b)
k, b, err := p.parseKey(b)
if err != nil {
p.SetRaw(ref, start, b)
return ref, nil, err
}
@@ -166,11 +173,12 @@ func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) {
b, err = expect(']', b)
if err != nil {
p.SetRaw(ref, start, b)
return ref, nil, err
}
b, err = expect(']', b)
p.SetRaw(ref, start, b)
return ref, b, err
}
@@ -181,12 +189,14 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
ref := p.builder.Push(ast.Node{
Kind: ast.Table,
})
start := b
b = b[1:]
b = p.parseWhitespace(b)
key, b, err := p.parseKey(b)
if err != nil {
p.SetRaw(ref, start, b)
return ref, nil, err
}
@@ -196,6 +206,7 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
b, err = expect(']', b)
p.SetRaw(ref, start, b)
return ref, b, err
}
@@ -204,10 +215,12 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
ref := p.builder.Push(ast.Node{
Kind: ast.KeyValue,
})
start := b
key, b, err := p.parseKey(b)
if err != nil {
return ast.InvalidReference, nil, err
p.SetRaw(ref, start, b)
return ast.InvalidReference, b, err
}
// keyval-sep = ws %x3D ws ; =
@@ -215,24 +228,28 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
b = p.parseWhitespace(b)
if len(b) == 0 {
return ast.InvalidReference, nil, newDecodeError(b, "expected = after a key, but the document ends there")
p.SetRaw(ref, start, b)
return ast.InvalidReference, b, newDecodeError(b, "expected = after a key, but the document ends there")
}
b, err = expect('=', b)
if err != nil {
return ast.InvalidReference, nil, err
p.SetRaw(ref, start, b)
return ast.InvalidReference, b, err
}
b = p.parseWhitespace(b)
valRef, b, err := p.parseVal(b)
if err != nil {
p.SetRaw(ref, start, b)
return ref, b, err
}
p.builder.Chain(valRef, key)
p.builder.AttachChild(ref, valRef)
p.SetRaw(ref, start, b)
return ref, b, err
}
@@ -242,7 +259,7 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
ref := ast.InvalidReference
if len(b) == 0 {
return ref, nil, newDecodeError(b, "expected value, not eof")
return ref, b, newDecodeError(b, "expected value, not eof")
}
var err error
@@ -287,23 +304,25 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
return ref, b, err
case 't':
if !scanFollowsTrue(b) {
return ref, nil, newDecodeError(atmost(b, 4), "expected 'true'")
return ref, b, newDecodeError(atmost(b, 4), "expected 'true'")
}
ref = p.builder.Push(ast.Node{
Kind: ast.Bool,
Data: b[:4],
Raw: p.Range(b[:4]),
})
return ref, b[4:], nil
case 'f':
if !scanFollowsFalse(b) {
return ref, nil, newDecodeError(atmost(b, 5), "expected 'false'")
return ref, b, newDecodeError(atmost(b, 5), "expected 'false'")
}
ref = p.builder.Push(ast.Node{
Kind: ast.Bool,
Data: b[:5],
Raw: p.Range(b[:5]),
})
return ref, b[5:], nil
@@ -327,7 +346,7 @@ func atmost(b []byte, n int) []byte {
func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) {
v, rest, err := scanLiteralString(b)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
return v, v[1 : len(v)-1], rest, nil
@@ -342,6 +361,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
parent := p.builder.Push(ast.Node{
Kind: ast.InlineTable,
})
start := b
first := true
@@ -356,7 +376,8 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
b = p.parseWhitespace(b)
if len(b) == 0 {
return parent, nil, newDecodeError(previousB[:1], "inline table is incomplete")
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(previousB[:1], "inline table is incomplete")
}
if b[0] == '}' {
@@ -366,7 +387,8 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
if !first {
b, err = expect(',', b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
b = p.parseWhitespace(b)
}
@@ -375,7 +397,8 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
kv, b, err = p.parseKeyval(b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
if first {
@@ -390,6 +413,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
rest, err := expect('}', b)
p.SetRaw(parent, start, b)
return parent, rest, err
}
@@ -403,6 +427,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
// array-sep = %x2C ; , Comma
// ws-comment-newline = *( wschar / [ comment ] newline )
arrayStart := b
start := b
b = b[1:]
parent := p.builder.Push(ast.Node{
@@ -417,11 +442,13 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
for len(b) > 0 {
b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
if len(b) == 0 {
return parent, nil, newDecodeError(arrayStart[:1], "array is incomplete")
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(arrayStart[:1], "array is incomplete")
}
if b[0] == ']' {
@@ -430,16 +457,19 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
if b[0] == ',' {
if first {
return parent, nil, newDecodeError(b[0:1], "array cannot start with comma")
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(b[0:1], "array cannot start with comma")
}
b = b[1:]
b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
} else if !first {
return parent, nil, newDecodeError(b[0:1], "array elements must be separated by commas")
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(b[0:1], "array elements must be separated by commas")
}
// TOML allows trailing commas in arrays.
@@ -450,7 +480,8 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
var valueRef ast.Reference
valueRef, b, err = p.parseVal(b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
if first {
@@ -462,13 +493,16 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil {
return parent, nil, err
p.SetRaw(parent, start, b)
return parent, b, err
}
first = false
}
rest, err := expect(']', b)
p.SetRaw(parent, start, rest)
return parent, rest, err
}
@@ -480,7 +514,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
if len(b) > 0 && b[0] == '#' {
_, b, err = scanComment(b)
if err != nil {
return nil, err
return b, err
}
}
@@ -491,7 +525,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
if b[0] == '\n' || b[0] == '\r' {
b, err = p.parseNewline(b)
if err != nil {
return nil, err
return b, err
}
} else {
break
@@ -504,7 +538,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) {
token, rest, err := scanMultilineLiteralString(b)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
i := 3
@@ -533,7 +567,7 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
// mlb-escaped-nl = escape ws newline *( wschar / newline )
token, escaped, rest, err := scanMultilineBasicString(b)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
i := 3
@@ -549,13 +583,13 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
startIdx := i
endIdx := len(token) - len(`"""`)
if escaped < 0 {
if !escaped {
str := token[startIdx:endIdx]
verr := utf8TomlValidAlreadyEscaped(str)
if verr.Zero() {
return token, str, rest, nil
}
return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
return nil, nil, rest, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
}
var builder bytes.Buffer
@@ -578,6 +612,10 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
switch token[i+j] {
case ' ', '\t':
continue
case '\r':
if token[i+j+1] == '\n' {
continue
}
case '\n':
isLastNonWhitespaceOnLine = true
}
@@ -613,29 +651,31 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
builder.WriteByte('\r')
case 't':
builder.WriteByte('\t')
case 'e':
builder.WriteByte(0x1B)
case 'u':
x, err := hexToRune(atmost(token[i+1:], 4), 4)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
builder.WriteRune(x)
i += 4
case 'U':
x, err := hexToRune(atmost(token[i+1:], 8), 8)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
builder.WriteRune(x)
i += 8
default:
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
return nil, nil, rest, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
}
i++
} else {
size := utf8ValidNext(token[i:])
if size == 0 {
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c)
return nil, nil, rest, newDecodeError(token[i:i+1], "invalid character %#U", c)
}
builder.Write(token[i : i+size])
i += size
@@ -656,7 +696,7 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
// dot-sep = ws %x2E ws ; . Period
raw, key, b, err := p.parseSimpleKey(b)
if err != nil {
return ast.InvalidReference, nil, err
return ast.InvalidReference, b, err
}
ref := p.builder.Push(ast.Node{
@@ -672,7 +712,7 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
raw, key, b, err = p.parseSimpleKey(b)
if err != nil {
return ref, nil, err
return ref, b, err
}
p.builder.PushAndChain(ast.Node{
@@ -689,13 +729,13 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
}
func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
if len(b) == 0 {
return nil, nil, b, newDecodeError(b, "expected key but found none")
}
// simple-key = quoted-key / unquoted-key
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
// quoted-key = basic-string / literal-string
if len(b) == 0 {
return nil, nil, nil, newDecodeError(b, "key is incomplete")
}
switch {
case b[0] == '\'':
return p.parseLiteralString(b)
@@ -705,7 +745,7 @@ func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
key, rest = scanUnquotedKey(b)
return key, key, rest, nil
default:
return nil, nil, nil, newDecodeError(b[0:1], "invalid character at start of key: %c", b[0])
return nil, nil, b[1:], newDecodeError(b[0:1], "invalid character at start of key: %c", b[0])
}
}
@@ -727,7 +767,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
token, escaped, rest, err := scanBasicString(b)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
startIdx := len(`"`)
@@ -736,13 +776,13 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
// Fast path. If there is no escape sequence, the string should just be
// an UTF-8 encoded string, which is the same as Go. In that case,
// validate the string and return a direct reference to the buffer.
if escaped < 0 {
if !escaped {
str := token[startIdx:endIdx]
verr := utf8TomlValidAlreadyEscaped(str)
if verr.Zero() {
return token, str, rest, nil
}
return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
return nil, nil, rest, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
}
i := startIdx
@@ -770,10 +810,12 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
builder.WriteByte('\r')
case 't':
builder.WriteByte('\t')
case 'e':
builder.WriteByte(0x1B)
case 'u':
x, err := hexToRune(token[i+1:len(token)-1], 4)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
builder.WriteRune(x)
@@ -781,19 +823,19 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
case 'U':
x, err := hexToRune(token[i+1:len(token)-1], 8)
if err != nil {
return nil, nil, nil, err
return nil, nil, rest, err
}
builder.WriteRune(x)
i += 8
default:
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
return nil, nil, rest, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
}
i++
} else {
size := utf8ValidNext(token[i:])
if size == 0 {
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c)
return nil, nil, rest, newDecodeError(token[i:i+1], "invalid character %#U", c)
}
builder.Write(token[i : i+size])
i += size
@@ -846,27 +888,28 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
switch b[0] {
case 'i':
if !scanFollowsInf(b) {
return ast.InvalidReference, nil, newDecodeError(atmost(b, 3), "expected 'inf'")
return ast.InvalidReference, b, newDecodeError(atmost(b, 3), "expected 'inf'")
}
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:3],
Raw: p.Range(b[:3]),
}), b[3:], nil
case 'n':
if !scanFollowsNan(b) {
return ast.InvalidReference, nil, newDecodeError(atmost(b, 3), "expected 'nan'")
return ast.InvalidReference, b, newDecodeError(atmost(b, 3), "expected 'nan'")
}
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:3],
Raw: p.Range(b[:3]),
}), b[3:], nil
case '+', '-':
return p.scanIntOrFloat(b)
}
//nolint:gomnd
if len(b) < 3 {
return p.scanIntOrFloat(b)
}
@@ -884,23 +927,13 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
if idx == 2 && c == ':' || (idx == 4 && c == '-') {
return p.scanDateTime(b)
}
break
}
return p.scanIntOrFloat(b)
}
func digitsToInt(b []byte) int {
x := 0
for _, d := range b {
x *= 10
x += int(d - '0')
}
return x
}
//nolint:gocognit,cyclop
func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
// scans for contiguous characters in [0-9T:Z.+-], and up to one space if
// followed by a digit.
@@ -963,6 +996,7 @@ byteLoop:
return p.builder.Push(ast.Node{
Kind: kind,
Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil
}
@@ -970,7 +1004,7 @@ byteLoop:
func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
i := 0
if len(b) > 2 && b[0] == '0' && b[1] != '.' && b[1] != 'e' {
if len(b) > 2 && b[0] == '0' && b[1] != '.' && b[1] != 'e' && b[1] != 'E' {
var isValidRune validRuneFn
switch b[1] {
@@ -996,6 +1030,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{
Kind: ast.Integer,
Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil
}
@@ -1019,10 +1054,11 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:i+3],
Raw: p.Range(b[:i+3]),
}), b[i+3:], nil
}
return ast.InvalidReference, nil, newDecodeError(b[i:i+1], "unexpected character 'i' while scanning for a number")
return ast.InvalidReference, b[i:], newDecodeError(b[i:i+1], "unexpected character 'i' while scanning for a number")
}
if c == 'n' {
@@ -1030,10 +1066,11 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{
Kind: ast.Float,
Data: b[:i+3],
Raw: p.Range(b[:i+3]),
}), b[i+3:], nil
}
return ast.InvalidReference, nil, newDecodeError(b[i:i+1], "unexpected character 'n' while scanning for a number")
return ast.InvalidReference, b[i:], newDecodeError(b[i:i+1], "unexpected character 'n' while scanning for a number")
}
break
@@ -1052,6 +1089,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{
Kind: kind,
Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil
}
@@ -1078,11 +1116,11 @@ func isValidBinaryRune(r byte) bool {
func expect(x byte, b []byte) ([]byte, error) {
if len(b) == 0 {
return nil, newDecodeError(b, "expected character %c but the document ended here", x)
return b, newDecodeError(b, "expected character %c but the document ended here", x)
}
if b[0] != x {
return nil, newDecodeError(b[0:1], "expected character %c", x)
return b, newDecodeError(b[0:1], "expected character %c", x)
}
return b[1:], nil
+19
View File
@@ -1,6 +1,8 @@
package toml
import (
"strconv"
"strings"
"testing"
"github.com/pelletier/go-toml/v2/internal/ast"
@@ -371,6 +373,23 @@ func BenchmarkParseBasicStringWithUnicode(b *testing.B) {
})
}
func BenchmarkParseBasicStringsEasy(b *testing.B) {
p := &parser{}
for _, size := range []int{1, 4, 8, 16, 21} {
b.Run(strconv.Itoa(size), func(b *testing.B) {
input := []byte(`"` + strings.Repeat("A", size) + `"`)
b.ReportAllocs()
b.SetBytes(int64(len(input)))
for i := 0; i < b.N; i++ {
p.parseBasicString(input)
}
})
}
}
func TestParser_AST_DateTimes(t *testing.T) {
examples := []struct {
desc string
+49 -55
View File
@@ -53,17 +53,17 @@ func scanLiteralString(b []byte) ([]byte, []byte, error) {
switch b[i] {
case '\'':
return b[:i+1], b[i+1:], nil
case '\n':
return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines")
case '\n', '\r':
return nil, b[i+1:], newDecodeError(b[i:i+1], "literal strings cannot have new lines")
}
size := utf8ValidNext(b[i:])
if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character")
return nil, b[i+1:], newDecodeError(b[i:i+1], "invalid character")
}
i += size
}
return nil, nil, newDecodeError(b[len(b):], "unterminated literal string")
return nil, b[len(b):], newDecodeError(b[len(b):], "unterminated literal string")
}
func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
@@ -76,6 +76,8 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
// mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
// mll-quotes = 1*2apostrophe
for i := 3; i < len(b); {
switch b[i] {
case '\'':
if scanFollowsMultilineLiteralStringDelimiter(b[i:]) {
i += 3
@@ -96,29 +98,39 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
i++
if i < len(b) && b[i] == '\'' {
return nil, nil, newDecodeError(b[i-3:i+1], "''' not allowed in multiline literal string")
return nil, b[i:], newDecodeError(b[i-3:i+1], "''' not allowed in multiline literal string")
}
return b[:i], b[i:], nil
}
case '\r':
if len(b) < i+2 {
return nil, b[i:], newDecodeError(b[len(b):], `need a \n after \r`)
}
if b[i+1] != '\n' {
return nil, b[i+2:], newDecodeError(b[i:i+2], `need a \n after \r`)
}
i += 2 // skip the \n
continue
}
size := utf8ValidNext(b[i:])
if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character")
return nil, b[i:], newDecodeError(b[i:i+1], "invalid character")
}
i += size
}
return nil, nil, newDecodeError(b[len(b):], `multiline literal string not terminated by '''`)
return nil, b[len(b):], newDecodeError(b[len(b):], `multiline literal string not terminated by '''`)
}
func scanWindowsNewline(b []byte) ([]byte, []byte, error) {
const lenCRLF = 2
if len(b) < lenCRLF {
return nil, nil, newDecodeError(b, "windows new line expected")
return nil, b, newDecodeError(b, "windows new line expected")
}
if b[1] != '\n' {
return nil, nil, newDecodeError(b, `windows new line should be \r\n`)
return nil, b[2:], newDecodeError(b, `windows new line should be \r\n`)
}
return b[:lenCRLF], b[lenCRLF:], nil
@@ -149,9 +161,15 @@ func scanComment(b []byte) ([]byte, []byte, error) {
if b[i] == '\n' {
return b[:i], b[i:], nil
}
if b[i] == '\r' {
if i+1 < len(b) && b[i+1] == '\n' {
return b[:i+1], b[i+1:], nil
}
return nil, nil, newDecodeError(b[i:i+1], "invalid character in comment")
}
size := utf8ValidNext(b[i:])
if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character in comment")
return nil, b[i+1:], newDecodeError(b[i:i+1], "invalid character in comment")
}
i += size
@@ -160,50 +178,34 @@ func scanComment(b []byte) ([]byte, []byte, error) {
return b, b[len(b):], nil
}
func scanBasicString(b []byte) ([]byte, int, []byte, error) {
func scanBasicString(b []byte) ([]byte, bool, []byte, error) {
// basic-string = quotation-mark *basic-char quotation-mark
// quotation-mark = %x22 ; "
// basic-char = basic-unescaped / escaped
// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// escaped = escape escape-seq-char
escaped := -1 // index of the first \. -1 means no escape character in there.
escaped := false
i := 1
loop:
for ; i < len(b); i++ {
switch b[i] {
case '"':
return b[:i+1], escaped, b[i+1:], nil
case '\n':
return nil, escaped, nil, newDecodeError(b[i:i+1], "basic strings cannot have new lines")
case '\n', '\r':
return nil, escaped, b[i+1:], newDecodeError(b[i:i+1], "basic strings cannot have new lines")
case '\\':
if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[i:i+1], "need a character after \\")
}
escaped = i
i += 2 // skip the next character
break loop
}
}
for ; i < len(b); i++ {
switch b[i] {
case '"':
return b[:i+1], escaped, b[i+1:], nil
case '\n':
return nil, escaped, nil, newDecodeError(b[i:i+1], "basic strings cannot have new lines")
case '\\':
if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[i:i+1], "need a character after \\")
return nil, escaped, b[i+1:], newDecodeError(b[i:i+1], "need a character after \\")
}
escaped = true
i++ // skip the next character
}
}
return nil, escaped, nil, newDecodeError(b[len(b):], `basic string not terminated by "`)
return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `basic string not terminated by "`)
}
func scanMultilineBasicString(b []byte) ([]byte, int, []byte, error) {
func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) {
// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
// ml-basic-string-delim
// ml-basic-string-delim = 3quotation-mark
@@ -215,10 +217,9 @@ func scanMultilineBasicString(b []byte) ([]byte, int, []byte, error) {
// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// mlb-escaped-nl = escape ws newline *( wschar / newline )
escaped := -1
escaped := false
i := 3
loop:
for ; i < len(b); i++ {
switch b[i] {
case '"':
@@ -242,34 +243,27 @@ loop:
i++
if i < len(b) && b[i] == '"' {
return nil, escaped, nil, newDecodeError(b[i-3:i+1], `""" not allowed in multiline basic string`)
return nil, escaped, b[i+1:], newDecodeError(b[i-3:i+1], `""" not allowed in multiline basic string`)
}
return b[:i], escaped, b[i:], nil
}
case '\\':
if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[len(b):], "need a character after \\")
}
escaped = i
i += 2 // skip the next character
break loop
}
}
for ; i < len(b); i++ {
switch b[i] {
case '"':
if scanFollowsMultilineBasicStringDelimiter(b[i:]) {
return b[:i+3], escaped, b[i+3:], nil
}
case '\\':
if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[len(b):], "need a character after \\")
return nil, escaped, b[len(b):], newDecodeError(b[len(b):], "need a character after \\")
}
escaped = true
i++ // skip the next character
case '\r':
if len(b) < i+2 {
return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `need a \n after \r`)
}
if b[i+1] != '\n' {
return nil, escaped, b[i+2:], newDecodeError(b[i:i+2], `need a \n after \r`)
}
i++ // skip the \n
}
}
return nil, escaped, nil, newDecodeError(b[len(b):], `multiline basic string not terminated by """`)
return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `multiline basic string not terminated by """`)
}
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01 00:00:00")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("\"\\n\"=\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("''=0")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=\"\"\"\\U00000000\"\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=[[{}]]")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("\"\\b\"=\"\"")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=inf")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=0000-01-01 00:00:00+00:00")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=[{}]")
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("0=nan")
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/pelletier/go-toml/v2/testsuite"
"github.com/pelletier/go-toml/v2/internal/testsuite"
"github.com/stretchr/testify/require"
)
+225 -32
View File
@@ -1,4 +1,4 @@
// Generated by tomltestgen for toml-test ref master on 2021-09-30T20:29:36-05:00
// Generated by tomltestgen for toml-test ref master on 2022-04-07T20:09:42-04:00
package toml_test
import (
@@ -55,11 +55,51 @@ func TestTOMLTest_Invalid_Array_TextInArray(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostFalseWithExtra(t *testing.T) {
input := "a = falsify\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostFalse(t *testing.T) {
input := "a = fals\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostTrueWithExtra(t *testing.T) {
input := "a = truthy\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_AlmostTrue(t *testing.T) {
input := "a = tru\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_JustF(t *testing.T) {
input := "a = f\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_JustT(t *testing.T) {
input := "a = t\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_MixedCase(t *testing.T) {
input := "valid = False\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_StartingSameFalse(t *testing.T) {
input := "a = falsey\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_StartingSameTrue(t *testing.T) {
input := "a = truer\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Bool_WrongCaseFalse(t *testing.T) {
input := "b = FALSE\n"
testgenInvalid(t, input)
@@ -70,6 +110,31 @@ func TestTOMLTest_Invalid_Bool_WrongCaseTrue(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareCr(t *testing.T) {
input := "# The following line contains a single carriage return control character\r\n\r"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareFormfeed(t *testing.T) {
input := "bare-formfeed = \f\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareNull(t *testing.T) {
input := "bare-null = \"some value\" \x00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_BareVerticalTab(t *testing.T) {
input := "bare-vertical-tab = \v\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_CommentCr(t *testing.T) {
input := "comment-cr = \"Carriage return in comment\" # \ra=1\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Control_CommentDel(t *testing.T) {
input := "comment-del = \"0x7f\" # \u007f\n"
testgenInvalid(t, input)
@@ -175,33 +240,73 @@ func TestTOMLTest_Invalid_Control_StringUs(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_ImpossibleDate(t *testing.T) {
input := "d = 2006-01-50T00:00:00Z\n"
func TestTOMLTest_Invalid_Datetime_HourOver(t *testing.T) {
input := "# time-hour = 2DIGIT ; 00-23\nd = 2006-01-01T24:00:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_MdayOver(t *testing.T) {
input := "# date-mday = 2DIGIT ; 01-28, 01-29, 01-30, 01-31 based on\n# ; month/year\nd = 2006-01-32T00:00:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_MdayUnder(t *testing.T) {
input := "# date-mday = 2DIGIT ; 01-28, 01-29, 01-30, 01-31 based on\n# ; month/year\nd = 2006-01-00T00:00:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_MinuteOver(t *testing.T) {
input := "# time-minute = 2DIGIT ; 00-59\nd = 2006-01-01T00:60:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_MonthOver(t *testing.T) {
input := "# date-month = 2DIGIT ; 01-12\nd = 2006-13-01T00:00:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_MonthUnder(t *testing.T) {
input := "# date-month = 2DIGIT ; 01-12\nd = 2007-00-01T00:00:00-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_NoLeadsWithMilli(t *testing.T) {
input := "with-milli = 1987-07-5T17:45:00.12Z\n"
input := "# Day \"5\" instead of \"05\"; the leading zero is required.\nwith-milli = 1987-07-5T17:45:00.12Z\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_NoLeads(t *testing.T) {
input := "no-leads = 1987-7-05T17:45:00Z\n"
input := "# Month \"7\" instead of \"07\"; the leading zero is required.\nno-leads = 1987-7-05T17:45:00Z\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_NoSecs(t *testing.T) {
input := "no-secs = 1987-07-05T17:45Z\n"
input := "# No seconds in time.\nno-secs = 1987-07-05T17:45Z\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_NoT(t *testing.T) {
input := "no-t = 1987-07-0517:45:00Z\n"
input := "# No \"t\" or \"T\" between the date and time.\nno-t = 1987-07-0517:45:00Z\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_SecondOver(t *testing.T) {
input := "# time-second = 2DIGIT ; 00-58, 00-59, 00-60 based on leap second\n# ; rules\nd = 2006-01-01T00:00:61-00:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_TimeNoLeads2(t *testing.T) {
input := "# Leading 0 is always required.\nd = 01:32:0\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_TimeNoLeads(t *testing.T) {
input := "# Leading 0 is always required.\nd = 1:32:00\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Datetime_TrailingT(t *testing.T) {
input := "d = 2006-01-30T\n"
input := "# Date cannot end with trailing T\nd = 2006-01-30T\n"
testgenInvalid(t, input)
}
@@ -375,6 +480,11 @@ func TestTOMLTest_Invalid_Float_TrailingPoint(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Float_TrailingUsExp(t *testing.T) {
input := "# trailing underscore in integer part is not allowed\ntrailing-us-exp = 1_e2\n# trailing underscore in float part is not allowed\ntrailing-us-exp2 = 1.2_e2\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Float_TrailingUs(t *testing.T) {
input := "trailing-us = 1.2_\n"
testgenInvalid(t, input)
@@ -390,11 +500,21 @@ func TestTOMLTest_Invalid_Float_UsBeforePoint(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_Add(t *testing.T) {
input := "a={}\n# Inline tables are immutable and can't be extended\n[a.b]\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_DoubleComma(t *testing.T) {
input := "t = {x=3,,y=4}\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_DuplicateKey(t *testing.T) {
input := "# Duplicate keys within an inline table are invalid\na={b=1, b=2}\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_Empty(t *testing.T) {
input := "t = {,}\n"
testgenInvalid(t, input)
@@ -425,6 +545,11 @@ func TestTOMLTest_Invalid_InlineTable_NoComma(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_Overwrite(t *testing.T) {
input := "a.b=0\n# Since table \"a\" is already defined, it can't be replaced by an inline table.\na={}\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_InlineTable_TrailingComma(t *testing.T) {
input := "# A terminating comma (also called trailing comma) is not permitted after the\n# last key/value pair in an inline table\nabc = { abc = 123, }\n"
testgenInvalid(t, input)
@@ -460,6 +585,21 @@ func TestTOMLTest_Invalid_Integer_DoubleUs(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_IncompleteBin(t *testing.T) {
input := "incomplete-bin = 0b\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_IncompleteHex(t *testing.T) {
input := "incomplete-hex = 0x\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_IncompleteOct(t *testing.T) {
input := "incomplete-oct = 0o\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_InvalidBin(t *testing.T) {
input := "invalid-bin = 0b0012\n"
testgenInvalid(t, input)
@@ -505,6 +645,11 @@ func TestTOMLTest_Invalid_Integer_LeadingZero2(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_LeadingZero3(t *testing.T) {
input := "leading-zero-3 = 0_0\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_LeadingZeroSign1(t *testing.T) {
input := "leading-zero-sign-1 = -01\n"
testgenInvalid(t, input)
@@ -515,6 +660,11 @@ func TestTOMLTest_Invalid_Integer_LeadingZeroSign2(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_LeadingZeroSign3(t *testing.T) {
input := "leading-zero-sign-3 = +0_1\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Integer_NegativeBin(t *testing.T) {
input := "negative-bin = -0b11010110\n"
testgenInvalid(t, input)
@@ -720,11 +870,16 @@ func TestTOMLTest_Invalid_String_BadConcat(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_BadEscape(t *testing.T) {
func TestTOMLTest_Invalid_String_BadEscape1(t *testing.T) {
input := "invalid-escape = \"This string has a bad \\a escape character.\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_BadEscape2(t *testing.T) {
input := "invalid-escape = \"This string has a bad \\ escape character.\"\n\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_BadMultiline(t *testing.T) {
input := "multi = \"first line\nsecond line\"\n"
testgenInvalid(t, input)
@@ -795,6 +950,21 @@ func TestTOMLTest_Invalid_String_MissingQuotes(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_MultilineBadEscape1(t *testing.T) {
input := "k = \"\"\"t\\a\"\"\"\n\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_MultilineBadEscape2(t *testing.T) {
input := "# \\<Space> is not a valid escape.\nk = \"\"\"t\\ t\"\"\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_MultilineBadEscape3(t *testing.T) {
input := "# \\<Space> is not a valid escape.\nk = \"\"\"t\\ \"\"\"\n\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_MultilineEscapeSpace(t *testing.T) {
input := "a = \"\"\"\n foo \\ \\n\n bar\"\"\"\n"
testgenInvalid(t, input)
@@ -815,11 +985,6 @@ func TestTOMLTest_Invalid_String_MultilineQuotes1(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_MultilineQuotes2(t *testing.T) {
input := "a = \"\"\"6 quotes: \"\"\"\"\"\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_String_NoClose(t *testing.T) {
input := "no-ending-quote = \"One time, at band camp\n"
testgenInvalid(t, input)
@@ -835,6 +1000,16 @@ func TestTOMLTest_Invalid_String_WrongClose(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_AppendWithDottedKeys1(t *testing.T) {
input := "# First a.b.c defines a table: a.b.c = {z=9}\n#\n# Then we define a.b.c.t = \"str\" to add a str to the above table, making it:\n#\n# a.b.c = {z=9, t=\"...\"}\n#\n# While this makes sense, logically, it was decided this is not valid TOML as\n# it's too confusing/convoluted.\n# \n# See: https://github.com/toml-lang/toml/issues/846\n# https://github.com/toml-lang/toml/pull/859\n\n[a.b.c]\n z = 9\n\n[a]\n b.c.t = \"Using dotted keys to add to [a.b.c] after explicitly defining it above is not allowed\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_AppendWithDottedKeys2(t *testing.T) {
input := "# This is the same issue as in injection-1.toml, except that nests one level\n# deeper. See that file for a more complete description.\n\n[a.b.c.d]\n z = 9\n\n[a]\n b.c.d.k.t = \"Using dotted keys to add to [a.b.c.d] after explicitly defining it above is not allowed\"\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_ArrayEmpty(t *testing.T) {
input := "[[]]\nname = \"Born to Run\"\n"
testgenInvalid(t, input)
@@ -850,6 +1025,16 @@ func TestTOMLTest_Invalid_Table_ArrayMissingBracket(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_DuplicateKeyDottedTable(t *testing.T) {
input := "[fruit]\napple.color = \"red\"\n\n[fruit.apple] # INVALID\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_DuplicateKeyDottedTable2(t *testing.T) {
input := "[fruit]\napple.taste.sweet = true\n\n[fruit.apple.taste] # INVALID\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_DuplicateKeyTable(t *testing.T) {
input := "[fruit]\ntype = \"apple\"\n\n[fruit.type]\napple = \"yes\"\n"
testgenInvalid(t, input)
@@ -885,16 +1070,6 @@ func TestTOMLTest_Invalid_Table_EqualsSign(t *testing.T) {
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_Injection1(t *testing.T) {
input := "[a.b.c]\n z = 9\n[a]\n b.c.t = \"Using dotted keys to add to [a.b.c] after explicitly defining it above is not allowed\"\n \n# see https://github.com/toml-lang/toml/issues/846\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_Injection2(t *testing.T) {
input := "[a.b.c.d]\n z = 9\n[a]\n b.c.d.k.t = \"Using dotted keys to add to [a.b.c.d] after explicitly defining it above is not allowed\"\n \n# see https://github.com/toml-lang/toml/issues/846\n"
testgenInvalid(t, input)
}
func TestTOMLTest_Invalid_Table_Llbrace(t *testing.T) {
input := "[ [table]]\n"
testgenInvalid(t, input)
@@ -1061,8 +1236,14 @@ func TestTOMLTest_Valid_Comment_AtEof2(t *testing.T) {
}
func TestTOMLTest_Valid_Comment_Everywhere(t *testing.T) {
input := "# Top comment.\n # Top comment.\n# Top comment.\n\n# [no-extraneous-groups-please]\n\n[group] # Comment\nanswer = 42 # Comment\n# no-extraneous-keys-please = 999\n# Inbetween comment.\nmore = [ # Comment\n # What about multiple # comments?\n # Can you handle it?\n #\n # Evil.\n# Evil.\n 42, 42, # Comments within arrays are fun.\n # What about multiple # comments?\n # Can you handle it?\n #\n # Evil.\n# Evil.\n# ] Did I fool you?\n] # Hopefully not.\n\n# Make sure the space between the datetime and \"#\" isn't lexed.\nd = 1979-05-27T07:32:12-07:00 # c\n"
jsonRef := "{\n \"group\": {\n \"answer\": {\n \"type\": \"integer\",\n \"value\": \"42\"\n },\n \"d\": {\n \"type\": \"datetime\",\n \"value\": \"1979-05-27T07:32:12-07:00\"\n },\n \"more\": [\n {\n \"type\": \"integer\",\n \"value\": \"42\"\n },\n {\n \"type\": \"integer\",\n \"value\": \"42\"\n }\n ]\n }\n}\n"
input := "# Top comment.\n # Top comment.\n# Top comment.\n\n# [no-extraneous-groups-please]\n\n[group] # Comment\nanswer = 42 # Comment\n# no-extraneous-keys-please = 999\n# Inbetween comment.\nmore = [ # Comment\n # What about multiple # comments?\n # Can you handle it?\n #\n # Evil.\n# Evil.\n 42, 42, # Comments within arrays are fun.\n # What about multiple # comments?\n # Can you handle it?\n #\n # Evil.\n# Evil.\n# ] Did I fool you?\n] # Hopefully not.\n\n# Make sure the space between the datetime and \"#\" isn't lexed.\ndt = 1979-05-27T07:32:12-07:00 # c\nd = 1979-05-27 # Comment\n"
jsonRef := "{\n \"group\": {\n \"answer\": {\n \"type\": \"integer\",\n \"value\": \"42\"\n },\n \"dt\": {\n \"type\": \"datetime\",\n \"value\": \"1979-05-27T07:32:12-07:00\"\n },\n \"d\": {\n \"type\": \"date-local\",\n \"value\": \"1979-05-27\"\n },\n \"more\": [\n {\n \"type\": \"integer\",\n \"value\": \"42\"\n },\n {\n \"type\": \"integer\",\n \"value\": \"42\"\n }\n ]\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_Comment_Noeol(t *testing.T) {
input := "# single comment without any eol characters"
jsonRef := "{}\n"
testgenValid(t, input, jsonRef)
}
@@ -1097,8 +1278,8 @@ func TestTOMLTest_Valid_Datetime_Local(t *testing.T) {
}
func TestTOMLTest_Valid_Datetime_Milliseconds(t *testing.T) {
input := "utc1 = 1987-07-05T17:45:56.123456Z\nutc2 = 1987-07-05T17:45:56.6Z\nwita1 = 1987-07-05T17:45:56.123456+08:00\nwita2 = 1987-07-05T17:45:56.6+08:00\n"
jsonRef := "{\n \"utc1\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.123456Z\"\n },\n \"utc2\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.600000Z\"\n },\n \"wita1\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.123456+08:00\"\n },\n \"wita2\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.600000+08:00\"\n }\n}\n"
input := "utc1 = 1987-07-05T17:45:56.1234Z\nutc2 = 1987-07-05T17:45:56.6Z\nwita1 = 1987-07-05T17:45:56.1234+08:00\nwita2 = 1987-07-05T17:45:56.6+08:00\n"
jsonRef := "{\n \"utc1\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.1234Z\"\n },\n \"utc2\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.6000Z\"\n },\n \"wita1\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.1234+08:00\"\n },\n \"wita2\": {\n \"type\": \"datetime\",\n \"value\": \"1987-07-05T17:45:56.6000+08:00\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
@@ -1360,6 +1541,12 @@ func TestTOMLTest_Valid_String_Empty(t *testing.T) {
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_EscapeEsc(t *testing.T) {
input := "esc = \"\\e There is no escape! \\e\"\n"
jsonRef := "{\n \"esc\": {\n \"type\": \"string\",\n \"value\": \"\\u001b There is no escape! \\u001b\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_EscapeTricky(t *testing.T) {
input := "end_esc = \"String does not end here\\\" but ends here\\\\\"\nlit_end_esc = 'String ends here\\'\n\nmultiline_unicode = \"\"\"\n\\u00a0\"\"\"\n\nmultiline_not_unicode = \"\"\"\n\\\\u0041\"\"\"\n\nmultiline_end_esc = \"\"\"When will it end? \\\"\"\"...\"\"\\\" should be here\\\"\"\"\"\n\nlit_multiline_not_unicode = '''\n\\u007f'''\n\nlit_multiline_end = '''There is no escape\\'''\n"
jsonRef := "{\n \"end_esc\": {\n \"type\": \"string\",\n \"value\": \"String does not end here\\\" but ends here\\\\\"\n },\n \"lit_end_esc\": {\n \"type\": \"string\",\n \"value\": \"String ends here\\\\\"\n },\n \"lit_multiline_end\": {\n \"type\": \"string\",\n \"value\": \"There is no escape\\\\\"\n },\n \"lit_multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u007f\"\n },\n \"multiline_end_esc\": {\n \"type\": \"string\",\n \"value\": \"When will it end? \\\"\\\"\\\"...\\\"\\\"\\\" should be here\\\"\"\n },\n \"multiline_not_unicode\": {\n \"type\": \"string\",\n \"value\": \"\\\\u0041\"\n },\n \"multiline_unicode\": {\n \"type\": \"string\",\n \"value\": \"\u00a0\"\n }\n}\n"
@@ -1378,14 +1565,20 @@ func TestTOMLTest_Valid_String_Escapes(t *testing.T) {
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_MultilineEscapedCrlf(t *testing.T) {
input := "# The following line should be an unescaped backslash followed by a Windows\r\n# newline sequence (\"\\r\\n\")\r\n0=\"\"\"\\\r\n\"\"\"\r\n"
jsonRef := "{\n \"0\": {\n \"type\": \"string\",\n \"value\": \"\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_MultilineQuotes(t *testing.T) {
input := "# Make sure that quotes inside multiline strings are allowed, including right\n# after the opening '''/\"\"\" and before the closing '''/\"\"\"\n\nlit_one = ''''one quote''''\nlit_two = '''''two quotes'''''\nlit_one_space = ''' 'one quote' '''\nlit_two_space = ''' ''two quotes'' '''\n\none = \"\"\"\"one quote\"\"\"\"\ntwo = \"\"\"\"\"two quotes\"\"\"\"\"\none_space = \"\"\" \"one quote\" \"\"\"\ntwo_space = \"\"\" \"\"two quotes\"\" \"\"\"\n\nmismatch1 = \"\"\"aaa'''bbb\"\"\"\nmismatch2 = '''aaa\"\"\"bbb'''\n"
jsonRef := "{\n \"lit_one\": {\n \"type\": \"string\",\n \"value\": \"'one quote'\"\n },\n \"lit_one_space\": {\n \"type\": \"string\",\n \"value\": \" 'one quote' \"\n },\n \"lit_two\": {\n \"type\": \"string\",\n \"value\": \"''two quotes''\"\n },\n \"lit_two_space\": {\n \"type\": \"string\",\n \"value\": \" ''two quotes'' \"\n },\n \"mismatch1\": {\n \"type\": \"string\",\n \"value\": \"aaa'''bbb\"\n },\n \"mismatch2\": {\n \"type\": \"string\",\n \"value\": \"aaa\\\"\\\"\\\"bbb\"\n },\n \"one\": {\n \"type\": \"string\",\n \"value\": \"\\\"one quote\\\"\"\n },\n \"one_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"one quote\\\" \"\n },\n \"two\": {\n \"type\": \"string\",\n \"value\": \"\\\"\\\"two quotes\\\"\\\"\"\n },\n \"two_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"\\\"two quotes\\\"\\\" \"\n }\n}\n"
input := "# Make sure that quotes inside multiline strings are allowed, including right\n# after the opening '''/\"\"\" and before the closing '''/\"\"\"\n\nlit_one = ''''one quote''''\nlit_two = '''''two quotes'''''\nlit_one_space = ''' 'one quote' '''\nlit_two_space = ''' ''two quotes'' '''\n\none = \"\"\"\"one quote\"\"\"\"\ntwo = \"\"\"\"\"two quotes\"\"\"\"\"\none_space = \"\"\" \"one quote\" \"\"\"\ntwo_space = \"\"\" \"\"two quotes\"\" \"\"\"\n\nmismatch1 = \"\"\"aaa'''bbb\"\"\"\nmismatch2 = '''aaa\"\"\"bbb'''\n\n# Three opening \"\"\", then one escaped \", then two \"\" (allowed), and then three\n# closing \"\"\"\nescaped = \"\"\"lol\\\"\"\"\"\"\"\n"
jsonRef := "{\n \"escaped\": {\n \"type\": \"string\",\n \"value\": \"lol\\\"\\\"\\\"\"\n },\n \"lit_one\": {\n \"type\": \"string\",\n \"value\": \"'one quote'\"\n },\n \"lit_one_space\": {\n \"type\": \"string\",\n \"value\": \" 'one quote' \"\n },\n \"lit_two\": {\n \"type\": \"string\",\n \"value\": \"''two quotes''\"\n },\n \"lit_two_space\": {\n \"type\": \"string\",\n \"value\": \" ''two quotes'' \"\n },\n \"mismatch1\": {\n \"type\": \"string\",\n \"value\": \"aaa'''bbb\"\n },\n \"mismatch2\": {\n \"type\": \"string\",\n \"value\": \"aaa\\\"\\\"\\\"bbb\"\n },\n \"one\": {\n \"type\": \"string\",\n \"value\": \"\\\"one quote\\\"\"\n },\n \"one_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"one quote\\\" \"\n },\n \"two\": {\n \"type\": \"string\",\n \"value\": \"\\\"\\\"two quotes\\\"\\\"\"\n },\n \"two_space\": {\n \"type\": \"string\",\n \"value\": \" \\\"\\\"two quotes\\\"\\\" \"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
func TestTOMLTest_Valid_String_Multiline(t *testing.T) {
input := "# NOTE: this file includes some literal tab characters.\n\nmultiline_empty_one = \"\"\"\"\"\"\nmultiline_empty_two = \"\"\"\n\"\"\"\nmultiline_empty_three = \"\"\"\\\n \"\"\"\nmultiline_empty_four = \"\"\"\\\n \\\n \\ \n \"\"\"\n\nequivalent_one = \"The quick brown fox jumps over the lazy dog.\"\nequivalent_two = \"\"\"\nThe quick brown \\\n\n\n fox jumps over \\\n the lazy dog.\"\"\"\n\nequivalent_three = \"\"\"\\\n The quick brown \\\n fox jumps over \\\n the lazy dog.\\\n \"\"\"\n\nwhitespace-after-bs = \"\"\"\\\n The quick brown \\\n fox jumps over \\ \n the lazy dog.\\\t\n \"\"\"\n\nno-space = \"\"\"a\\\n b\"\"\"\n\nkeep-ws-before = \"\"\"a \t\\\n b\"\"\"\n\nescape-bs-1 = \"\"\"a \\\\\nb\"\"\"\n\nescape-bs-2 = \"\"\"a \\\\\\\nb\"\"\"\n\nescape-bs-3 = \"\"\"a \\\\\\\\\n b\"\"\"\n"
input := "# NOTE: this file includes some literal tab characters.\n\nmultiline_empty_one = \"\"\"\"\"\"\n\n# A newline immediately following the opening delimiter will be trimmed.\nmultiline_empty_two = \"\"\"\n\"\"\"\n\n# \\ at the end of line trims newlines as well; note that last \\ is followed by\n# two spaces, which are ignored.\nmultiline_empty_three = \"\"\"\\\n \"\"\"\nmultiline_empty_four = \"\"\"\\\n \\\n \\ \n \"\"\"\n\nequivalent_one = \"The quick brown fox jumps over the lazy dog.\"\nequivalent_two = \"\"\"\nThe quick brown \\\n\n\n fox jumps over \\\n the lazy dog.\"\"\"\n\nequivalent_three = \"\"\"\\\n The quick brown \\\n fox jumps over \\\n the lazy dog.\\\n \"\"\"\n\nwhitespace-after-bs = \"\"\"\\\n The quick brown \\\n fox jumps over \\ \n the lazy dog.\\\t\n \"\"\"\n\nno-space = \"\"\"a\\\n b\"\"\"\n\n# Has tab character.\nkeep-ws-before = \"\"\"a \t\\\n b\"\"\"\n\nescape-bs-1 = \"\"\"a \\\\\nb\"\"\"\n\nescape-bs-2 = \"\"\"a \\\\\\\nb\"\"\"\n\nescape-bs-3 = \"\"\"a \\\\\\\\\n b\"\"\"\n"
jsonRef := "{\n \"equivalent_one\": {\n \"type\": \"string\",\n \"value\": \"The quick brown fox jumps over the lazy dog.\"\n },\n \"equivalent_three\": {\n \"type\": \"string\",\n \"value\": \"The quick brown fox jumps over the lazy dog.\"\n },\n \"equivalent_two\": {\n \"type\": \"string\",\n \"value\": \"The quick brown fox jumps over the lazy dog.\"\n },\n \"escape-bs-1\": {\n \"type\": \"string\",\n \"value\": \"a \\\\\\nb\"\n },\n \"escape-bs-2\": {\n \"type\": \"string\",\n \"value\": \"a \\\\b\"\n },\n \"escape-bs-3\": {\n \"type\": \"string\",\n \"value\": \"a \\\\\\\\\\n b\"\n },\n \"keep-ws-before\": {\n \"type\": \"string\",\n \"value\": \"a \\tb\"\n },\n \"multiline_empty_four\": {\n \"type\": \"string\",\n \"value\": \"\"\n },\n \"multiline_empty_one\": {\n \"type\": \"string\",\n \"value\": \"\"\n },\n \"multiline_empty_three\": {\n \"type\": \"string\",\n \"value\": \"\"\n },\n \"multiline_empty_two\": {\n \"type\": \"string\",\n \"value\": \"\"\n },\n \"no-space\": {\n \"type\": \"string\",\n \"value\": \"ab\"\n },\n \"whitespace-after-bs\": {\n \"type\": \"string\",\n \"value\": \"The quick brown fox jumps over the lazy dog.\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
@@ -1397,7 +1590,7 @@ func TestTOMLTest_Valid_String_Nl(t *testing.T) {
}
func TestTOMLTest_Valid_String_RawMultiline(t *testing.T) {
input := "oneline = '''This string has a ' quote character.'''\nfirstnl = '''\nThis string has a ' quote character.'''\nmultiline = '''\nThis string\nhas ' a quote character\nand more than\none newline\nin it.'''\n"
input := "# Single ' should be allowed.\noneline = '''This string has a ' quote character.'''\n\n# A newline immediately following the opening delimiter will be trimmed.\nfirstnl = '''\nThis string has a ' quote character.'''\n\n# All other whitespace and newline characters remain intact.\nmultiline = '''\nThis string\nhas ' a quote character\nand more than\none newline\nin it.'''\n"
jsonRef := "{\n \"firstnl\": {\n \"type\": \"string\",\n \"value\": \"This string has a ' quote character.\"\n },\n \"multiline\": {\n \"type\": \"string\",\n \"value\": \"This string\\nhas ' a quote character\\nand more than\\none newline\\nin it.\"\n },\n \"oneline\": {\n \"type\": \"string\",\n \"value\": \"This string has a ' quote character.\"\n }\n}\n"
testgenValid(t, input, jsonRef)
}
+1
View File
@@ -11,3 +11,4 @@ var textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem()
var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem()
var mapStringInterfaceType = reflect.TypeOf(map[string]interface{}{})
var sliceInterfaceType = reflect.TypeOf([]interface{}{})
var stringType = reflect.TypeOf("")
+185 -101
View File
@@ -9,10 +9,11 @@ import (
"math"
"reflect"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pelletier/go-toml/v2/internal/ast"
"github.com/pelletier/go-toml/v2/internal/danger"
"github.com/pelletier/go-toml/v2/internal/tracker"
)
@@ -41,24 +42,27 @@ func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}
// SetStrict toggles decoding in stict mode.
// DisallowUnknownFields causes the Decoder to return an error when the
// destination is a struct and the input contains a key that does not match a
// non-ignored field.
//
// When the decoder is in strict mode, it will record fields from the document
// that could not be set on the target value. In that case, the decoder returns
// a StrictMissingError that can be used to retrieve the individual errors as
// well as generate a human readable description of the missing fields.
func (d *Decoder) SetStrict(strict bool) {
d.strict = strict
// In that case, the Decoder returns a StrictMissingError that can be used to
// retrieve the individual errors as well as generate a human readable
// description of the missing fields.
func (d *Decoder) DisallowUnknownFields() *Decoder {
d.strict = true
return d
}
// Decode the whole content of r into v.
//
// By default, values in the document that don't exist in the target Go value
// are ignored. See Decoder.SetStrict() to change this behavior.
// are ignored. See Decoder.DisallowUnknownFields() to change this behavior.
//
// When a TOML local date, time, or date-time is decoded into a time.Time, its
// value is represented in time.Local timezone. Otherwise the approriate Local*
// structure is used.
// structure is used. For time values, precision up to the nanosecond is
// supported by truncating extra digits.
//
// Empty tables decoded in an interface{} create an empty initialized
// map[string]interface{}.
@@ -70,7 +74,12 @@ func (d *Decoder) SetStrict(strict bool) {
// bounds for the target type (which includes negative numbers when decoding
// into an unsigned int).
//
// Type mapping
// If an error occurs while decoding the content of the document, this function
// returns a toml.DecodeError, providing context about the issue. When using
// strict mode and a field is missing, a `toml.StrictMissingError` is
// returned. In any other case, this function returns a standard Go error.
//
// # Type mapping
//
// List of supported TOML types and their associated accepted Go types:
//
@@ -114,7 +123,7 @@ type decoder struct {
stashedExpr bool
// Skip expressions until a table is found. This is set to true when a
// table could not be create (missing field in map), so all KV expressions
// table could not be created (missing field in map), so all KV expressions
// need to be skipped.
skipUntilTable bool
@@ -129,6 +138,23 @@ type decoder struct {
// Strict mode
strict strict
// Current context for the error.
errorContext *errorContext
}
type errorContext struct {
Struct reflect.Type
Field []int
}
func (d *decoder) typeMismatchError(toml string, target reflect.Type) error {
if d.errorContext != nil && d.errorContext.Struct != nil {
ctx := d.errorContext
f := ctx.Struct.FieldByIndex(ctx.Field)
return fmt.Errorf("toml: cannot decode TOML %s into struct field %s.%s of type %s", toml, ctx.Struct, f.Name, f.Type)
}
return fmt.Errorf("toml: cannot decode TOML %s into a Go value of type %s", toml, target)
}
func (d *decoder) expr() *ast.Node {
@@ -296,10 +322,12 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val
return v, nil
case reflect.Slice:
elemType := v.Type().Elem()
var elem reflect.Value
if elemType.Kind() == reflect.Interface {
elemType = mapStringInterfaceType
elem = makeMapStringInterface()
} else {
elem = reflect.New(elemType).Elem()
}
elem := reflect.New(elemType).Elem()
elem2, err := d.handleArrayTable(key, elem)
if err != nil {
return reflect.Value{}, err
@@ -316,9 +344,9 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val
elem := v.Index(idx)
_, err := d.handleArrayTable(key, elem)
return v, err
default:
return reflect.Value{}, fmt.Errorf("toml: cannot decode array table into a %s", v.Type())
}
return d.handleArrayTable(key, v)
}
// When parsing an array table expression, each part of the key needs to be
@@ -343,7 +371,9 @@ func (d *decoder) handleArrayTableCollection(key ast.Iterator, v reflect.Value)
if err != nil {
return reflect.Value{}, err
}
if elem.IsValid() {
v.Elem().Set(elem)
}
return v, nil
case reflect.Slice:
@@ -384,12 +414,15 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
elem = v.Elem()
return d.handleKeyPart(key, elem, nextFn, makeFn)
case reflect.Map:
// Create the key for the map element. For now assume it's a string.
mk := reflect.ValueOf(string(key.Node().Data))
vt := v.Type()
// Create the key for the map element. Convert to key type.
mk := reflect.ValueOf(string(key.Node().Data)).Convert(vt.Key())
// If the map does not exist, create it.
if v.IsNil() {
v = reflect.MakeMap(v.Type())
vt := v.Type()
v = reflect.MakeMap(vt)
rv = v
}
@@ -401,7 +434,7 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
// map[string]interface{} or a []interface{} depending on whether
// this is the last part of the array table key.
t := v.Type().Elem()
t := vt.Elem()
if t.Kind() == reflect.Interface {
mv = makeFn()
} else {
@@ -415,7 +448,8 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
}
set = true
} else if !mv.CanAddr() {
t := v.Type().Elem()
vt := v.Type()
t := vt.Elem()
oldmv := mv
mv = reflect.New(t).Elem()
mv.Set(oldmv)
@@ -436,12 +470,20 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
v.SetMapIndex(mk, mv)
}
case reflect.Struct:
f, found := structField(v, string(key.Node().Data))
path, found := structFieldPath(v, string(key.Node().Data))
if !found {
d.skipUntilTable = true
return reflect.Value{}, nil
}
if d.errorContext == nil {
d.errorContext = new(errorContext)
}
t := v.Type()
d.errorContext.Struct = t
d.errorContext.Field = path
f := fieldByIndex(v, path)
x, err := nextFn(key, f)
if err != nil || d.skipUntilTable {
return reflect.Value{}, err
@@ -449,11 +491,13 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
if x.IsValid() {
f.Set(x)
}
d.errorContext.Field = nil
d.errorContext.Struct = nil
case reflect.Interface:
if v.Elem().IsValid() {
v = v.Elem()
} else {
v = reflect.MakeMap(mapStringInterfaceType)
v = makeMapStringInterface()
}
x, err := d.handleKeyPart(key, v, nextFn, makeFn)
@@ -569,7 +613,7 @@ func (d *decoder) tryTextUnmarshaler(node *ast.Node, v reflect.Value) (bool, err
if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) {
err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data)
if err != nil {
return false, newDecodeError(d.p.Raw(node.Raw), "error calling UnmarshalText: %w", err)
return false, newDecodeError(d.p.Raw(node.Raw), "%w", err)
}
return true, nil
@@ -649,7 +693,7 @@ func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error {
default:
// TODO: use newDecodeError, but first the parser needs to fill
// array.Data.
return fmt.Errorf("toml: cannot store array in Go type %s", v.Kind())
return d.typeMismatchError("array", v.Type())
}
elemType := v.Type().Elem()
@@ -697,7 +741,7 @@ func (d *decoder) unmarshalInlineTable(itable *ast.Node, v reflect.Value) error
case reflect.Interface:
elem := v.Elem()
if !elem.IsValid() {
elem = reflect.MakeMap(mapStringInterfaceType)
elem = makeMapStringInterface()
v.Set(elem)
}
return d.unmarshalInlineTable(itable, elem)
@@ -822,12 +866,27 @@ func (d *decoder) unmarshalFloat(value *ast.Node, v reflect.Value) error {
return nil
}
func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
const (
const (
maxInt = int64(^uint(0) >> 1)
minInt = -maxInt - 1
)
)
// Maximum value of uint for decoding. Currently the decoder parses the integer
// into an int64. As a result, on architectures where uint is 64 bits, the
// effective maximum uint we can decode is the maximum of int64. On
// architectures where uint is 32 bits, the maximum value we can decode is
// lower: the maximum of uint32. I didn't find a way to figure out this value at
// compile time, so it is computed during initialization.
var maxUint int64 = math.MaxInt64
func init() {
m := uint64(^uint(0))
if m < uint64(maxUint) {
maxUint = int64(m)
}
}
func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
i, err := parseInteger(value.Data)
if err != nil {
return err
@@ -888,7 +947,7 @@ func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
r = reflect.ValueOf(uint8(i))
case reflect.Uint:
if i < 0 {
if i < 0 || i > maxUint {
return fmt.Errorf("toml: negative number %d does not fit in an uint", i)
}
@@ -896,7 +955,7 @@ func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
case reflect.Interface:
r = reflect.ValueOf(i)
default:
return fmt.Errorf("toml: cannot store TOML integer into a Go %s", v.Kind())
return d.typeMismatchError("integer", v.Type())
}
if !r.Type().AssignableTo(v.Type()) {
@@ -953,12 +1012,15 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
// There is no guarantee over what it could be.
switch v.Kind() {
case reflect.Map:
mk := reflect.ValueOf(string(key.Node().Data))
vt := v.Type()
keyType := v.Type().Key()
if !mk.Type().AssignableTo(keyType) {
if !mk.Type().ConvertibleTo(keyType) {
return reflect.Value{}, fmt.Errorf("toml: cannot convert map key of type %s to expected type %s", mk.Type(), keyType)
mk := reflect.ValueOf(string(key.Node().Data))
mkt := stringType
keyType := vt.Key()
if !mkt.AssignableTo(keyType) {
if !mkt.ConvertibleTo(keyType) {
return reflect.Value{}, fmt.Errorf("toml: cannot convert map key of type %s to expected type %s", mkt, keyType)
}
mk = mk.Convert(keyType)
@@ -966,7 +1028,7 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
// If the map does not exist, create it.
if v.IsNil() {
v = reflect.MakeMap(v.Type())
v = reflect.MakeMap(vt)
rv = v
}
@@ -996,12 +1058,20 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
v.SetMapIndex(mk, mv)
}
case reflect.Struct:
f, found := structField(v, string(key.Node().Data))
path, found := structFieldPath(v, string(key.Node().Data))
if !found {
d.skipUntilTable = true
break
}
if d.errorContext == nil {
d.errorContext = new(errorContext)
}
t := v.Type()
d.errorContext.Struct = t
d.errorContext.Field = path
f := fieldByIndex(v, path)
x, err := d.handleKeyValueInner(key, value, f)
if err != nil {
return reflect.Value{}, err
@@ -1010,14 +1080,17 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
if x.IsValid() {
f.Set(x)
}
d.errorContext.Struct = nil
d.errorContext.Field = nil
case reflect.Interface:
v = v.Elem()
// Following encoding/toml: decoding an object into an interface{}, it
// needs to always hold a map[string]interface{}. This is for the types
// to be consistent whether a previous value was set or not.
// Following encoding/json: decoding an object into an
// interface{}, it needs to always hold a
// map[string]interface{}. This is for the types to be
// consistent whether a previous value was set or not.
if !v.IsValid() || v.Type() != mapStringInterfaceType {
v = reflect.MakeMap(mapStringInterfaceType)
v = makeMapStringInterface()
}
x, err := d.handleKeyValuePart(key, value, v)
@@ -1062,82 +1135,93 @@ func initAndDereferencePointer(v reflect.Value) reflect.Value {
return elem
}
// Same as reflect.Value.FieldByIndex, but creates pointers if needed.
func fieldByIndex(v reflect.Value, path []int) reflect.Value {
for i, x := range path {
v = v.Field(x)
if i < len(path)-1 && v.Kind() == reflect.Pointer {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
}
}
return v
}
type fieldPathsMap = map[string][]int
type fieldPathsCache struct {
m map[reflect.Type]fieldPathsMap
l sync.RWMutex
}
var globalFieldPathsCache atomic.Value // map[danger.TypeID]fieldPathsMap
func (c *fieldPathsCache) get(t reflect.Type) (fieldPathsMap, bool) {
c.l.RLock()
paths, ok := c.m[t]
c.l.RUnlock()
func structFieldPath(v reflect.Value, name string) ([]int, bool) {
t := v.Type()
return paths, ok
}
cache, _ := globalFieldPathsCache.Load().(map[danger.TypeID]fieldPathsMap)
fieldPaths, ok := cache[danger.MakeTypeID(t)]
func (c *fieldPathsCache) set(t reflect.Type, m fieldPathsMap) {
c.l.Lock()
c.m[t] = m
c.l.Unlock()
}
var globalFieldPathsCache = fieldPathsCache{
m: map[reflect.Type]fieldPathsMap{},
l: sync.RWMutex{},
}
func structField(v reflect.Value, name string) (reflect.Value, bool) {
//nolint:godox
// TODO: cache this, and reduce allocations
fieldPaths, ok := globalFieldPathsCache.get(v.Type())
if !ok {
fieldPaths = map[string][]int{}
path := make([]int, 0, 16)
var walk func(reflect.Value)
walk = func(v reflect.Value) {
t := v.Type()
for i := 0; i < t.NumField(); i++ {
l := len(path)
path = append(path, i)
f := t.Field(i)
if f.Anonymous {
walk(v.Field(i))
} else if f.PkgPath == "" {
// only consider exported fields
fieldName, ok := f.Tag.Lookup("toml")
if !ok {
fieldName = f.Name
}
pathCopy := make([]int, len(path))
copy(pathCopy, path)
fieldPaths[fieldName] = pathCopy
forEachField(t, nil, func(name string, path []int) {
fieldPaths[name] = path
// extra copy for the case-insensitive match
fieldPaths[strings.ToLower(fieldName)] = pathCopy
}
path = path[:l]
}
}
fieldPaths[strings.ToLower(name)] = path
})
walk(v)
globalFieldPathsCache.set(v.Type(), fieldPaths)
newCache := make(map[danger.TypeID]fieldPathsMap, len(cache)+1)
newCache[danger.MakeTypeID(t)] = fieldPaths
for k, v := range cache {
newCache[k] = v
}
globalFieldPathsCache.Store(newCache)
}
path, ok := fieldPaths[name]
if !ok {
path, ok = fieldPaths[strings.ToLower(name)]
}
return path, ok
}
if !ok {
return reflect.Value{}, false
func forEachField(t reflect.Type, path []int, do func(name string, path []int)) {
n := t.NumField()
for i := 0; i < n; i++ {
f := t.Field(i)
if !f.Anonymous && f.PkgPath != "" {
// only consider exported fields.
continue
}
return v.FieldByIndex(path), true
fieldPath := append(path, i)
fieldPath = fieldPath[:len(fieldPath):len(fieldPath)]
name := f.Tag.Get("toml")
if name == "-" {
continue
}
if i := strings.IndexByte(name, ','); i >= 0 {
name = name[:i]
}
if f.Anonymous && name == "" {
t2 := f.Type
if t2.Kind() == reflect.Pointer {
t2 = t2.Elem()
}
if t2.Kind() == reflect.Struct {
forEachField(t2, fieldPath, do)
}
continue
}
if name == "" {
name = f.Name
}
do(name, fieldPath)
}
}
+879 -341
View File
File diff suppressed because it is too large Load Diff
+38 -1
View File
@@ -140,8 +140,45 @@ func utf8ValidNext(p []byte) int {
return size
}
var invalidAsciiTable = [256]bool{
0x00: true,
0x01: true,
0x02: true,
0x03: true,
0x04: true,
0x05: true,
0x06: true,
0x07: true,
0x08: true,
// 0x09 TAB
// 0x0A LF
0x0B: true,
0x0C: true,
// 0x0D CR
0x0E: true,
0x0F: true,
0x10: true,
0x11: true,
0x12: true,
0x13: true,
0x14: true,
0x15: true,
0x16: true,
0x17: true,
0x18: true,
0x19: true,
0x1A: true,
0x1B: true,
0x1C: true,
0x1D: true,
0x1E: true,
0x1F: true,
// 0x20 - 0x7E Printable ASCII characters
0x7F: true,
}
func invalidAscii(b byte) bool {
return b <= 0x08 || (b > 0x0A && b < 0x0D) || (b > 0x0D && b <= 0x1F) || b == 0x7F
return invalidAsciiTable[b]
}
// acceptRange gives the range of valid values for the second byte in a UTF-8