Compare commits

..

6 Commits

Author SHA1 Message Date
Thomas Pelletier dc72d75f3e Keep separate fn for []interface{} unmarshal 2021-11-13 19:20:20 -05:00
Thomas Pelletier f77775b59e Use less reflection when making slices
```
name                               old time/op    new time/op    delta
UnmarshalDataset/config-2            24.9ms ± 0%    24.6ms ± 0%  -1.09%  (p=0.029 n=4+4)
UnmarshalDataset/canada-2            61.7ms ± 1%    62.1ms ± 3%    ~     (p=1.000 n=5+5)
UnmarshalDataset/citm_catalog-2      24.7ms ± 1%    24.2ms ± 0%  -2.30%  (p=0.008 n=5+5)
UnmarshalDataset/twitter-2           10.9ms ± 2%    10.7ms ± 1%  -1.46%  (p=0.008 n=5+5)
UnmarshalDataset/code-2               108ms ± 0%     106ms ± 0%  -1.91%  (p=0.008 n=5+5)
UnmarshalDataset/example-2            176µs ± 0%     173µs ± 0%  -1.83%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-2     586ns ± 1%     587ns ± 0%    ~     (p=0.690 n=5+5)
Unmarshal/SimpleDocument/map-2        876ns ± 0%     872ns ± 0%    ~     (p=0.095 n=5+5)
Unmarshal/ReferenceFile/struct-2     49.5µs ± 0%    49.5µs ± 0%    ~     (p=0.222 n=5+5)
Unmarshal/ReferenceFile/map-2        79.6µs ± 0%    79.1µs ± 0%  -0.62%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-2          13.7µs ± 0%    13.5µs ± 0%  -0.91%  (p=0.008 n=5+5)

name                               old speed      new speed      delta
UnmarshalDataset/config-2          42.2MB/s ± 0%  42.7MB/s ± 0%  +1.10%  (p=0.029 n=4+4)
UnmarshalDataset/canada-2          35.7MB/s ± 1%  35.5MB/s ± 3%    ~     (p=1.000 n=5+5)
UnmarshalDataset/citm_catalog-2    22.6MB/s ± 1%  23.1MB/s ± 0%  +2.36%  (p=0.008 n=5+5)
UnmarshalDataset/twitter-2         40.6MB/s ± 2%  41.2MB/s ± 1%  +1.47%  (p=0.008 n=5+5)
UnmarshalDataset/code-2            24.9MB/s ± 0%  25.4MB/s ± 0%  +1.95%  (p=0.008 n=5+5)
UnmarshalDataset/example-2         46.0MB/s ± 0%  46.9MB/s ± 0%  +1.86%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-2  18.8MB/s ± 1%  18.7MB/s ± 0%    ~     (p=0.651 n=5+5)
Unmarshal/SimpleDocument/map-2     12.6MB/s ± 0%  12.6MB/s ± 0%    ~     (p=0.087 n=5+5)
Unmarshal/ReferenceFile/struct-2    106MB/s ± 0%   106MB/s ± 0%    ~     (p=0.222 n=5+5)
Unmarshal/ReferenceFile/map-2      65.8MB/s ± 0%  66.2MB/s ± 0%  +0.63%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-2        40.0MB/s ± 0%  40.3MB/s ± 0%  +0.92%  (p=0.008 n=5+5)

name                               old alloc/op   new alloc/op   delta
UnmarshalDataset/config-2            5.85MB ± 0%    5.85MB ± 0%    ~     (p=1.000 n=5+5)
UnmarshalDataset/canada-2            75.2MB ± 0%    75.2MB ± 0%    ~     (p=1.000 n=5+5)
UnmarshalDataset/citm_catalog-2      35.0MB ± 0%    35.0MB ± 0%    ~     (p=0.841 n=5+5)
UnmarshalDataset/twitter-2           13.5MB ± 0%    13.5MB ± 0%    ~     (p=0.548 n=5+5)
UnmarshalDataset/code-2              22.0MB ± 0%    22.0MB ± 0%    ~     (p=0.738 n=5+5)
UnmarshalDataset/example-2            203kB ± 0%     203kB ± 0%    ~     (p=0.714 n=5+5)
Unmarshal/SimpleDocument/struct-2      709B ± 0%      709B ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/map-2       1.08kB ± 0%    1.08kB ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/struct-2     19.7kB ± 0%    19.7kB ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/map-2        37.0kB ± 0%    37.0kB ± 0%    ~     (p=0.333 n=4+5)
Unmarshal/HugoFrontMatter-2          7.22kB ± 0%    7.22kB ± 0%    ~     (all equal)

name                               old allocs/op  new allocs/op  delta
UnmarshalDataset/config-2              230k ± 0%      230k ± 0%    ~     (p=0.556 n=4+5)
UnmarshalDataset/canada-2              391k ± 0%      391k ± 0%    ~     (all equal)
UnmarshalDataset/citm_catalog-2        158k ± 0%      158k ± 0%    ~     (p=1.000 n=4+5)
UnmarshalDataset/twitter-2            54.7k ± 0%     54.7k ± 0%    ~     (p=1.000 n=4+5)
UnmarshalDataset/code-2               1.05M ± 0%     1.05M ± 0%    ~     (all equal)
UnmarshalDataset/example-2            1.28k ± 0%     1.28k ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/struct-2      8.00 ± 0%      8.00 ± 0%    ~     (all equal)
Unmarshal/SimpleDocument/map-2         13.0 ± 0%      13.0 ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/struct-2        123 ± 0%       123 ± 0%    ~     (all equal)
Unmarshal/ReferenceFile/map-2           590 ± 0%       590 ± 0%    ~     (all equal)
Unmarshal/HugoFrontMatter-2             130 ± 0%       130 ± 0%    ~     (all equal)
```
2021-11-13 19:20:20 -05:00
Thomas Pelletier b52f6c9823 Remove some allocs for slices in interfaces
```
name                               old time/op    new time/op    delta
UnmarshalDataset/config-2            24.9ms ± 1%    24.9ms ± 0%     ~     (p=0.413 n=5+4)
UnmarshalDataset/canada-2            66.1ms ± 0%    61.7ms ± 1%   -6.63%  (p=0.008 n=5+5)
UnmarshalDataset/citm_catalog-2      25.3ms ± 5%    24.7ms ± 1%   -2.09%  (p=0.032 n=5+5)
UnmarshalDataset/twitter-2           10.9ms ± 2%    10.9ms ± 2%     ~     (p=1.000 n=5+5)
UnmarshalDataset/code-2               108ms ± 0%     108ms ± 0%     ~     (p=0.095 n=5+5)
UnmarshalDataset/example-2            177µs ± 2%     176µs ± 0%     ~     (p=0.841 n=5+5)
Unmarshal/SimpleDocument/struct-2     579ns ± 0%     586ns ± 1%   +1.30%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/map-2        875ns ± 1%     876ns ± 0%     ~     (p=0.548 n=5+5)
Unmarshal/ReferenceFile/struct-2     49.7µs ± 1%    49.5µs ± 0%     ~     (p=0.095 n=5+5)
Unmarshal/ReferenceFile/map-2        80.4µs ± 0%    79.6µs ± 0%   -0.99%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-2          13.9µs ± 0%    13.7µs ± 0%   -1.70%  (p=0.008 n=5+5)

name                               old speed      new speed      delta
UnmarshalDataset/config-2          42.1MB/s ± 1%  42.2MB/s ± 0%     ~     (p=0.381 n=5+4)
UnmarshalDataset/canada-2          33.3MB/s ± 0%  35.7MB/s ± 1%   +7.11%  (p=0.008 n=5+5)
UnmarshalDataset/citm_catalog-2    22.1MB/s ± 5%  22.6MB/s ± 1%   +2.08%  (p=0.032 n=5+5)
UnmarshalDataset/twitter-2         40.7MB/s ± 2%  40.6MB/s ± 2%     ~     (p=1.000 n=5+5)
UnmarshalDataset/code-2            24.8MB/s ± 0%  24.9MB/s ± 0%     ~     (p=0.103 n=5+5)
UnmarshalDataset/example-2         45.8MB/s ± 2%  46.0MB/s ± 0%     ~     (p=0.841 n=5+5)
Unmarshal/SimpleDocument/struct-2  19.0MB/s ± 0%  18.8MB/s ± 1%   -1.26%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/map-2     12.6MB/s ± 1%  12.6MB/s ± 0%     ~     (p=0.508 n=5+5)
Unmarshal/ReferenceFile/struct-2    105MB/s ± 1%   106MB/s ± 0%     ~     (p=0.095 n=5+5)
Unmarshal/ReferenceFile/map-2      65.2MB/s ± 0%  65.8MB/s ± 0%   +1.00%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-2        39.3MB/s ± 0%  40.0MB/s ± 0%   +1.73%  (p=0.008 n=5+5)

name                               old alloc/op   new alloc/op   delta
UnmarshalDataset/config-2            5.85MB ± 0%    5.85MB ± 0%   -0.00%  (p=0.008 n=5+5)
UnmarshalDataset/canada-2            76.6MB ± 0%    75.2MB ± 0%   -1.76%  (p=0.016 n=4+5)
UnmarshalDataset/citm_catalog-2      35.3MB ± 0%    35.0MB ± 0%   -0.71%  (p=0.008 n=5+5)
UnmarshalDataset/twitter-2           13.5MB ± 0%    13.5MB ± 0%   -0.19%  (p=0.016 n=4+5)
UnmarshalDataset/code-2              22.3MB ± 0%    22.0MB ± 0%   -1.31%  (p=0.008 n=5+5)
UnmarshalDataset/example-2            204kB ± 0%     203kB ± 0%   -0.34%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-2      709B ± 0%      709B ± 0%     ~     (all equal)
Unmarshal/SimpleDocument/map-2       1.08kB ± 0%    1.08kB ± 0%     ~     (all equal)
Unmarshal/ReferenceFile/struct-2     19.8kB ± 0%    19.7kB ± 0%   -0.24%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/map-2        37.3kB ± 0%    37.0kB ± 0%   -0.64%  (p=0.029 n=4+4)
Unmarshal/HugoFrontMatter-2          7.26kB ± 0%    7.22kB ± 0%   -0.66%  (p=0.008 n=5+5)

name                               old allocs/op  new allocs/op  delta
UnmarshalDataset/config-2              230k ± 0%      230k ± 0%   -0.00%  (p=0.000 n=5+4)
UnmarshalDataset/canada-2              447k ± 0%      391k ± 0%  -12.53%  (p=0.008 n=5+5)
UnmarshalDataset/citm_catalog-2        169k ± 0%      158k ± 0%   -6.20%  (p=0.029 n=4+4)
UnmarshalDataset/twitter-2            55.8k ± 0%     54.7k ± 0%   -1.88%  (p=0.029 n=4+4)
UnmarshalDataset/code-2               1.06M ± 0%     1.05M ± 0%   -1.14%  (p=0.008 n=5+5)
UnmarshalDataset/example-2            1.31k ± 0%     1.28k ± 0%   -2.21%  (p=0.008 n=5+5)
Unmarshal/SimpleDocument/struct-2      8.00 ± 0%      8.00 ± 0%     ~     (all equal)
Unmarshal/SimpleDocument/map-2         13.0 ± 0%      13.0 ± 0%     ~     (all equal)
Unmarshal/ReferenceFile/struct-2        125 ± 0%       123 ± 0%   -1.60%  (p=0.008 n=5+5)
Unmarshal/ReferenceFile/map-2           600 ± 0%       590 ± 0%   -1.67%  (p=0.008 n=5+5)
Unmarshal/HugoFrontMatter-2             132 ± 0%       130 ± 0%   -1.52%  (p=0.008 n=5+5)
```
2021-11-13 19:20:20 -05:00
Thomas Pelletier 12244064bb Use global cache to unmarshal all slice types 2021-11-13 19:20:20 -05:00
Thomas Pelletier 6430ee0bfa Generic slice unmarshal fn 2021-11-13 19:20:20 -05:00
Thomas Pelletier cf530eba46 Specialize array unmarshal into []interface{} 2021-11-13 19:20:19 -05:00
17 changed files with 489 additions and 1031 deletions
-1
View File
@@ -2,7 +2,6 @@ changelog:
exclude: exclude:
labels: labels:
- build - build
- testing
categories: categories:
- title: What's new - title: What's new
labels: labels:
+4 -109
View File
@@ -55,9 +55,8 @@ to check for typos. [See example in the documentation][strict].
### Contextualized errors ### Contextualized errors
When most decoding errors occur, go-toml returns [`DecodeError`][decode-err]), When decoding errors occur, go-toml returns [`DecodeError`][decode-err]), which
which contains a human readable contextualized version of the error. For contains a human readable contextualized version of the error. For example:
example:
``` ```
2| key1 = "value1" 2| key1 = "value1"
@@ -325,29 +324,6 @@ The recommended replacement is pre-filling the struct before unmarshaling.
[go-defaults]: https://github.com/mcuadros/go-defaults [go-defaults]: https://github.com/mcuadros/go-defaults
#### `toml.Tree` replacement
This structure was the initial attempt at providing a document model for
go-toml. It allows manipulating the structure of any document, encoding and
decoding from their TOML representation. While a more robust feature was
initially planned in go-toml v2, this has been ultimately [removed from
scope][nodoc] of this library, with no plan to add it back at the moment. The
closest equivalent at the moment would be to unmarshal into an `interface{}` and
use type assertions and/or reflection to manipulate the arbitrary
structure. However this would fall short of providing all of the TOML features
such as adding comments and be specific about whitespace.
#### `toml.Position` are not retrievable anymore
The API for retrieving the position (line, column) of a specific TOML element do
not exist anymore. This was done to minimize the amount of concepts introduced
by the library (query path), and avoid the performance hit related to storing
positions in the absence of a document model, for a feature that seemed to have
little use. Errors however have gained more detailed position
information. Position retrieval seems better fitted for a document model, which
has been [removed from the scope][nodoc] of go-toml v2 at the moment.
### Encoding / Marshal ### Encoding / Marshal
#### Default struct fields order #### Default struct fields order
@@ -383,8 +359,7 @@ fmt.Println("v2:\n" + string(b))
``` ```
There is no way to make v2 encoder behave like v1. A workaround could be to There is no way to make v2 encoder behave like v1. A workaround could be to
manually sort the fields alphabetically in the struct definition, or generate manually sort the fields alphabetically in the struct definition.
struct types using `reflect.StructOf`.
#### No indentation by default #### No indentation by default
@@ -432,9 +407,7 @@ fmt.Println("v2 Encoder:\n" + string(buf.Bytes()))
V1 always uses double quotes (`"`) around strings and keys that cannot be V1 always uses double quotes (`"`) around strings and keys that cannot be
represented bare (unquoted). V2 uses single quotes instead by default (`'`), represented bare (unquoted). V2 uses single quotes instead by default (`'`),
unless a character cannot be represented, then falls back to double quotes. As a unless a character cannot be represented, then falls back to double quotes.
result of this change, `Encoder.QuoteMapKeys` has been removed, as it is not
useful anymore.
There is no way to make v2 encoder behave like v1. There is no way to make v2 encoder behave like v1.
@@ -449,84 +422,6 @@ There is no way to make v2 encoder behave like v1.
[tm]: https://golang.org/pkg/encoding/#TextMarshaler [tm]: https://golang.org/pkg/encoding/#TextMarshaler
#### `Encoder.CompactComments` has been removed
Emitting compact comments is now the default behavior of go-toml. This option
is not necessary anymore.
#### Struct tags have been merged
V1 used to provide multiple struct tags: `comment`, `commented`, `multiline`,
`toml`, and `omitempty`. To behave more like the standard library, v2 has merged
`toml`, `multiline`, and `omitempty`. For example:
```go
type doc struct {
// v1
F string `toml:"field" multiline:"true" omitempty:"true"`
// v2
F string `toml:"field,multiline,omitempty"`
}
```
Has a result, the `Encoder.SetTag*` methods have been removed, as there is just
one tag now.
#### `commented` tag has been removed
There is no replacement for the `commented` tag. This feature would be better
suited in a proper document model for go-toml v2, which has been [cut from
scope][nodoc] at the moment.
#### `Encoder.ArraysWithOneElementPerLine` has been renamed
The new name is `Encoder.SetArraysMultiline`. The behavior should be the same.
#### `Encoder.Indentation` has been renamed
The new name is `Encoder.SetIndentSymbol`. The behavior should be the same.
#### Embedded structs are tables
V1 defaults to merging embedded struct fields into the embedding struct. This
behavior was unexpected because it does not follow the standard library. To
avoid breaking backward compatibility, the `Encoder.PromoteAnonymous` method was
added to make the encoder behave correctly. Given backward compatibility is not
a problem anymore, v2 does the right thing by default. There is no way to revert
to the old behavior, and `Encoder.PromoteAnonymous` has been removed.
```go
type Embedded struct {
Value string `toml:"value"`
}
type Doc struct {
Embedded
}
d := Doc{}
fmt.Println("v1:")
b, err := v1.Marshal(d)
fmt.Println(string(b))
fmt.Println("v2:")
b, err = v2.Marshal(d)
fmt.Println(string(b))
// Output:
// v1:
// value = ""
//
// v2:
// [Embedded]
// value = ''
```
[nodoc]: https://github.com/pelletier/go-toml/discussions/506#discussioncomment-1526038
## License ## License
The MIT License (MIT). Read [LICENSE](LICENSE). The MIT License (MIT). Read [LICENSE](LICENSE).
-1
View File
@@ -321,7 +321,6 @@ type benchmarkDoc struct {
Key1 []int64 Key1 []int64
Key2 []string Key2 []string
Key3 [][]int64 Key3 [][]int64
// TODO: Key4 not supported by go-toml's Unmarshal
Key4 []interface{} Key4 []interface{}
Key5 []int64 Key5 []int64
Key6 []int64 Key6 []int64
+23 -67
View File
@@ -99,36 +99,13 @@ func parseDateTime(b []byte) (time.Time, error) {
if len(b) != dateTimeByteLen { if len(b) != dateTimeByteLen {
return time.Time{}, newDecodeError(b, "invalid date-time timezone") return time.Time{}, newDecodeError(b, "invalid date-time timezone")
} }
var direction int direction := 1
switch b[0] { if b[0] == '-' {
case '-':
direction = -1 direction = -1
case '+':
direction = +1
default:
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset character")
}
if b[3] != ':' {
return time.Time{}, newDecodeError(b[3:4], "expected a : separator")
}
hours, err := parseDecimalDigits(b[1:3])
if err != nil {
return time.Time{}, err
}
if hours > 23 {
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset hours")
}
minutes, err := parseDecimalDigits(b[4:6])
if err != nil {
return time.Time{}, err
}
if minutes > 59 {
return time.Time{}, newDecodeError(b[:1], "invalid timezone offset minutes")
} }
hours := digitsToInt(b[1:3])
minutes := digitsToInt(b[4:6])
seconds := direction * (hours*3600 + minutes*60) seconds := direction * (hours*3600 + minutes*60)
zone = time.FixedZone("", seconds) zone = time.FixedZone("", seconds)
b = b[dateTimeByteLen:] b = b[dateTimeByteLen:]
@@ -224,53 +201,45 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
return t, nil, err return t, nil, err
} }
if t.Second > 60 { if t.Second > 59 {
return t, nil, newDecodeError(b[6:8], "seconds cannot be greater 60") return t, nil, newDecodeError(b[3:5], "seconds cannot be greater 59")
} }
b = b[8:] const minLengthWithFrac = 9
if len(b) >= minLengthWithFrac && b[minLengthWithFrac-1] == '.' {
if len(b) >= 1 && b[0] == '.' {
frac := 0 frac := 0
precision := 0
digits := 0 digits := 0
for i, c := range b[1:] { for i, c := range b[minLengthWithFrac:] {
if !isDigit(c) { if !isDigit(c) {
if i == 0 { if i == 0 {
return t, nil, newDecodeError(b[0:1], "need at least one digit after fraction point") return t, nil, newDecodeError(b[i:i+1], "need at least one digit after fraction point")
} }
break break
} }
digits++
const maxFracPrecision = 9 const maxFracPrecision = 9
if i >= maxFracPrecision { if i >= maxFracPrecision {
// go-toml allows decoding fractional seconds return t, nil, newDecodeError(b[i:i+1], "maximum precision for date time is nanosecond")
// beyond the supported precision of 9
// digits. It truncates the fractional component
// to the supported precision and ignores the
// remaining digits.
//
// https://github.com/pelletier/go-toml/discussions/707
continue
} }
frac *= 10 frac *= 10
frac += int(c - '0') frac += int(c - '0')
precision++ digits++
} }
if precision == 0 { if digits == 0 {
return t, nil, newDecodeError(b[:1], "nanoseconds need at least one digit") return t, nil, newDecodeError(b[minLengthWithFrac-1:minLengthWithFrac], "nanoseconds need at least one digit")
} }
t.Nanosecond = frac * nspow[precision] t.Nanosecond = frac * nspow[digits]
t.Precision = precision t.Precision = digits
return t, b[1+digits:], nil return t, b[9+digits:], nil
} }
return t, b, nil
return t, b[8:], nil
} }
//nolint:cyclop //nolint:cyclop
@@ -395,17 +364,8 @@ func parseIntDec(b []byte) (int64, error) {
} }
func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) { func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) {
start := 0 if b[0] == '_' {
if b[start] == '+' || b[start] == '-' { return nil, newDecodeError(b[0:1], "number cannot start with underscore")
start++
}
if len(b) == start {
return b, nil
}
if b[start] == '_' {
return nil, newDecodeError(b[start:start+1], "number cannot start with underscore")
} }
if b[len(b)-1] == '_' { if b[len(b)-1] == '_' {
@@ -478,10 +438,6 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) {
return nil, newDecodeError(b[i+1:i+2], "cannot have underscore before exponent") return nil, newDecodeError(b[i+1:i+2], "cannot have underscore before exponent")
} }
before = false before = false
case '+', '-':
// signed exponents
cleaned = append(cleaned, c)
before = false
case 'e', 'E': case 'e', 'E':
if i < len(b)-1 && b[i+1] == '_' { if i < len(b)-1 && b[i+1] == '_' {
return nil, newDecodeError(b[i+1:i+2], "cannot have underscore after exponent") return nil, newDecodeError(b[i+1:i+2], "cannot have underscore after exponent")
@@ -506,7 +462,7 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) {
// isValidDate checks if a provided date is a date that exists. // isValidDate checks if a provided date is a date that exists.
func isValidDate(year int, month int, day int) bool { func isValidDate(year int, month int, day int) bool {
return month > 0 && month < 13 && day > 0 && day <= daysIn(month, year) return day <= daysIn(month, year)
} }
// daysBefore[m] counts the number of days in a non-leap year // daysBefore[m] counts the number of days in a non-leap year
+5 -5
View File
@@ -212,12 +212,12 @@ func ExampleDecodeError() {
fmt.Println(err) fmt.Println(err)
var derr *DecodeError //nolint:errorlint
if errors.As(err, &derr) { de := err.(*DecodeError)
fmt.Println(derr.String()) fmt.Println(de.String())
row, col := derr.Position()
row, col := de.Position()
fmt.Println("error occurred at row", row, "column", col) fmt.Println("error occurred at row", row, "column", col)
}
// Output: // Output:
// toml: number must have at least one digit between underscores // toml: number must have at least one digit between underscores
// 1| name = 123__456 // 1| name = 123__456
+25 -24
View File
@@ -20,8 +20,8 @@ type Iterator struct {
node *Node node *Node
} }
// Next moves the iterator forward and returns true if points to a // Next moves the iterator forward and returns true if points to a node, false
// node, false otherwise. // otherwise.
func (c *Iterator) Next() bool { func (c *Iterator) Next() bool {
if !c.started { if !c.started {
c.started = true c.started = true
@@ -31,8 +31,8 @@ func (c *Iterator) Next() bool {
return c.node.Valid() return c.node.Valid()
} }
// IsLast returns true if the current node of the iterator is the last // IsLast returns true if the current node of the iterator is the last one.
// one. Subsequent call to Next() will return false. // Subsequent call to Next() will return false.
func (c *Iterator) IsLast() bool { func (c *Iterator) IsLast() bool {
return c.node.next == 0 return c.node.next == 0
} }
@@ -62,20 +62,20 @@ func (r *Root) at(idx Reference) *Node {
return &r.nodes[idx] return &r.nodes[idx]
} }
// Arrays have one child per element in the array. InlineTables have // Arrays have one child per element in the array.
// one child per key-value pair in the table. KeyValues have at least // InlineTables have one child per key-value pair in the table.
// two children. The first one is the value. The rest make a // KeyValues have at least two children. The first one is the value. The
// potentially dotted key. Table and Array table have one child per // rest make a potentially dotted key.
// element of the key they represent (same as KeyValue, but without // Table and Array table have one child per element of the key they
// the last node being the value). // represent (same as KeyValue, but without the last node being the value).
// children []Node
type Node struct { type Node struct {
Kind Kind Kind Kind
Raw Range // Raw bytes from the input. Raw Range // Raw bytes from the input.
Data []byte // Node value (either allocated or referencing the input). Data []byte // Node value (could be either allocated or referencing the input).
// References to other nodes, as offsets in the backing array // References to other nodes, as offsets in the backing array from this
// from this node. References can go backward, so those can be // node. References can go backward, so those can be negative.
// negative.
next int // 0 if last element next int // 0 if last element
child int // 0 if no child child int // 0 if no child
} }
@@ -85,8 +85,8 @@ type Range struct {
Length uint32 Length uint32
} }
// Next returns a copy of the next node, or an invalid Node if there // Next returns a copy of the next node, or an invalid Node if there is no
// is no next node. // next node.
func (n *Node) Next() *Node { func (n *Node) Next() *Node {
if n.next == 0 { if n.next == 0 {
return nil return nil
@@ -96,9 +96,9 @@ func (n *Node) Next() *Node {
return (*Node)(danger.Stride(ptr, size, n.next)) return (*Node)(danger.Stride(ptr, size, n.next))
} }
// Child returns a copy of the first child node of this node. Other // Child returns a copy of the first child node of this node. Other children
// children can be accessed calling Next on the first child. Returns // can be accessed calling Next on the first child.
// an invalid Node if there is none. // Returns an invalid Node if there is none.
func (n *Node) Child() *Node { func (n *Node) Child() *Node {
if n.child == 0 { if n.child == 0 {
return nil return nil
@@ -113,9 +113,10 @@ func (n *Node) Valid() bool {
return n != nil return n != nil
} }
// Key returns the child nodes making the Key on a supported // Key returns the child nodes making the Key on a supported node. Panics
// node. Panics otherwise. They are guaranteed to be all be of the // otherwise.
// Kind Key. A simple key would return just one element. // They are guaranteed to be all be of the Kind Key. A simple key would return
// just one element.
func (n *Node) Key() Iterator { func (n *Node) Key() Iterator {
switch n.Kind { switch n.Kind {
case KeyValue: case KeyValue:
@@ -132,8 +133,8 @@ func (n *Node) Key() Iterator {
} }
// Value returns a pointer to the value node of a KeyValue. // Value returns a pointer to the value node of a KeyValue.
// Guaranteed to be non-nil. Panics if not called on a KeyValue node, // Guaranteed to be non-nil.
// or if the Children are malformed. // Panics if not called on a KeyValue node, or if the Children are malformed.
func (n *Node) Value() *Node { func (n *Node) Value() *Node {
return n.Child() return n.Child()
} }
+11
View File
@@ -63,3 +63,14 @@ func Stride(ptr unsafe.Pointer, size uintptr, offset int) unsafe.Pointer {
// https://github.com/golang/go/issues/40481 // https://github.com/golang/go/issues/40481
return unsafe.Pointer(uintptr(ptr) + uintptr(int(size)*offset)) return unsafe.Pointer(uintptr(ptr) + uintptr(int(size)*offset))
} }
type Slice struct {
Data unsafe.Pointer
Len int
Cap int
}
type iface struct {
typ unsafe.Pointer
ptr unsafe.Pointer
}
+20
View File
@@ -0,0 +1,20 @@
//go:build go1.18
// +build go1.18
package danger
import (
"reflect"
"unsafe"
)
func ExtendSlice(t reflect.Type, s *Slice, n int) Slice {
arrayType := reflect.ArrayOf(n, t.Elem())
arrayData := reflect.New(arrayType)
reflect.Copy(arrayData.Elem(), reflect.NewAt(t, unsafe.Pointer(s)).Elem())
return Slice{
Data: unsafe.Pointer(arrayData.Pointer()),
Len: s.Len,
Cap: n,
}
}
+30
View File
@@ -0,0 +1,30 @@
//go:build !go1.18
// +build !go1.18
package danger
import (
"reflect"
"unsafe"
)
//go:linkname unsafe_NewArray reflect.unsafe_NewArray
func unsafe_NewArray(rtype unsafe.Pointer, length int) unsafe.Pointer
//go:linkname typedslicecopy reflect.typedslicecopy
//go:noescape
func typedslicecopy(elemType unsafe.Pointer, dst, src Slice) int
func ExtendSlice(t reflect.Type, s *Slice, n int) Slice {
elemTypeRef := t.Elem()
elemTypePtr := ((*iface)(unsafe.Pointer(&elemTypeRef))).ptr
d := Slice{
Data: unsafe_NewArray(elemTypePtr, n),
Len: s.Len,
Cap: n,
}
typedslicecopy(elemTypePtr, d, *s)
return d
}
@@ -457,6 +457,35 @@ func TestEmptytomlUnmarshal(t *testing.T) {
assert.Equal(t, emptyTestData, result) assert.Equal(t, emptyTestData, result)
} }
func TestEmptyUnmarshalOmit(t *testing.T) {
t.Skipf("Have not figured yet if omitempty is a good idea")
type emptyMarshalTestStruct2 struct {
Title string `toml:"title"`
Bool bool `toml:"bool,omitempty"`
Int int `toml:"int, omitempty"`
String string `toml:"string,omitempty "`
StringList []string `toml:"stringlist,omitempty"`
Ptr *basicMarshalTestStruct `toml:"ptr,omitempty"`
Map map[string]string `toml:"map,omitempty"`
}
emptyTestData2 := emptyMarshalTestStruct2{
Title: "Placeholder",
Bool: false,
Int: 0,
String: "",
StringList: []string{},
Ptr: nil,
Map: map[string]string{},
}
result := emptyMarshalTestStruct2{}
err := toml.Unmarshal(emptyTestToml, &result)
require.NoError(t, err)
assert.Equal(t, emptyTestData2, result)
}
type pointerMarshalTestStruct struct { type pointerMarshalTestStruct struct {
Str *string Str *string
List *[]string List *[]string
+67 -114
View File
@@ -3,7 +3,6 @@ package tracker
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"sync"
"github.com/pelletier/go-toml/v2/internal/ast" "github.com/pelletier/go-toml/v2/internal/ast"
) )
@@ -55,103 +54,69 @@ func (k keyKind) String() string {
type SeenTracker struct { type SeenTracker struct {
entries []entry entries []entry
currentIdx int currentIdx int
} nextID int
var pool sync.Pool
func (s *SeenTracker) reset() {
// Always contains a root element at index 0.
s.currentIdx = 0
if len(s.entries) == 0 {
s.entries = make([]entry, 1, 2)
} else {
s.entries = s.entries[:1]
}
s.entries[0].child = -1
s.entries[0].next = -1
} }
type entry struct { type entry struct {
// Use -1 to indicate no child or no sibling. id int
child int parent int
next int
name []byte name []byte
kind keyKind kind keyKind
explicit bool explicit bool
} }
// Find the index of the child of parentIdx with key k. Returns -1 if
// it does not exist.
func (s *SeenTracker) find(parentIdx int, k []byte) int {
for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next {
if bytes.Equal(s.entries[i].name, k) {
return i
}
}
return -1
}
// Remove all descendants of node at position idx. // Remove all descendants of node at position idx.
func (s *SeenTracker) clear(idx int) { func (s *SeenTracker) clear(idx int) {
if idx >= len(s.entries) { p := s.entries[idx].id
return rest := clear(p, s.entries[idx+1:])
s.entries = s.entries[:idx+1+len(rest)]
} }
for i := s.entries[idx].child; i >= 0; { func clear(parentID int, entries []entry) []entry {
next := s.entries[i].next for i := 0; i < len(entries); {
n := s.entries[0].next if entries[i].parent == parentID {
s.entries[0].next = i id := entries[i].id
s.entries[i].next = n copy(entries[i:], entries[i+1:])
s.entries[i].name = nil entries = entries[:len(entries)-1]
s.clear(i) rest := clear(id, entries[i:])
i = next entries = entries[:i+len(rest)]
} else {
i++
} }
}
s.entries[idx].child = -1 return entries
} }
func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool) int { func (s *SeenTracker) create(parentIdx int, name []byte, kind keyKind, explicit bool) int {
e := entry{ parentID := s.id(parentIdx)
child: -1,
next: s.entries[parentIdx].child,
idx := len(s.entries)
s.entries = append(s.entries, entry{
id: s.nextID,
parent: parentID,
name: name, name: name,
kind: kind, kind: kind,
explicit: explicit, explicit: explicit,
} })
var idx int s.nextID++
if s.entries[0].next >= 0 {
idx = s.entries[0].next
s.entries[0].next = s.entries[idx].next
s.entries[idx] = e
} else {
idx = len(s.entries)
s.entries = append(s.entries, e)
}
s.entries[parentIdx].child = idx
return idx return idx
} }
func (s *SeenTracker) setExplicitFlag(parentIdx int) {
for i := s.entries[parentIdx].child; i >= 0; i = s.entries[i].next {
s.entries[i].explicit = true
s.setExplicitFlag(i)
}
}
// CheckExpression takes a top-level node and checks that it does not contain // CheckExpression takes a top-level node and checks that it does not contain
// keys that have been seen in previous calls, and validates that types are // keys that have been seen in previous calls, and validates that types are
// consistent. // consistent.
func (s *SeenTracker) CheckExpression(node *ast.Node) error { func (s *SeenTracker) CheckExpression(node *ast.Node) error {
if s.entries == nil { if s.entries == nil {
s.reset() // Skip ID = 0 to remove the confusion between nodes whose
// parent has id 0 and root nodes (parent id is 0 because it's
// the zero value).
s.nextID = 1
// Start unscoped, so idx is negative.
s.currentIdx = -1
} }
switch node.Kind { switch node.Kind {
case ast.KeyValue: case ast.KeyValue:
return s.checkKeyValue(node) return s.checkKeyValue(s.currentIdx, node)
case ast.Table: case ast.Table:
return s.checkTable(node) return s.checkTable(node)
case ast.ArrayTable: case ast.ArrayTable:
@@ -162,13 +127,9 @@ func (s *SeenTracker) CheckExpression(node *ast.Node) error {
} }
func (s *SeenTracker) checkTable(node *ast.Node) error { func (s *SeenTracker) checkTable(node *ast.Node) error {
if s.currentIdx >= 0 {
s.setExplicitFlag(s.currentIdx)
}
it := node.Key() it := node.Key()
parentIdx := 0 parentIdx := -1
// This code is duplicated in checkArrayTable. This is because factoring // This code is duplicated in checkArrayTable. This is because factoring
// it in a function requires to copy the iterator, or allocate it to the // it in a function requires to copy the iterator, or allocate it to the
@@ -184,11 +145,6 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
if idx < 0 { if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false) idx = s.create(parentIdx, k, tableKind, false)
} else {
entry := s.entries[idx]
if entry.kind == valueKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), entry.kind)
}
} }
parentIdx = idx parentIdx = idx
} }
@@ -215,13 +171,9 @@ func (s *SeenTracker) checkTable(node *ast.Node) error {
} }
func (s *SeenTracker) checkArrayTable(node *ast.Node) error { func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
if s.currentIdx >= 0 {
s.setExplicitFlag(s.currentIdx)
}
it := node.Key() it := node.Key()
parentIdx := 0 parentIdx := -1
for it.Next() { for it.Next() {
if it.IsLast() { if it.IsLast() {
@@ -234,13 +186,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
if idx < 0 { if idx < 0 {
idx = s.create(parentIdx, k, tableKind, false) idx = s.create(parentIdx, k, tableKind, false)
} else {
entry := s.entries[idx]
if entry.kind == valueKind {
return fmt.Errorf("toml: expected %s to be a table, not a %s", string(k), entry.kind)
} }
}
parentIdx = idx parentIdx = idx
} }
@@ -262,8 +208,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error {
return nil return nil
} }
func (s *SeenTracker) checkKeyValue(node *ast.Node) error { func (s *SeenTracker) checkKeyValue(parentIdx int, node *ast.Node) error {
parentIdx := s.currentIdx
it := node.Key() it := node.Key()
for it.Next() { for it.Next() {
@@ -293,59 +238,67 @@ func (s *SeenTracker) checkKeyValue(node *ast.Node) error {
switch value.Kind { switch value.Kind {
case ast.InlineTable: case ast.InlineTable:
return s.checkInlineTable(value) return s.checkInlineTable(parentIdx, value)
case ast.Array: case ast.Array:
return s.checkArray(value) return s.checkArray(parentIdx, value)
} }
return nil return nil
} }
func (s *SeenTracker) checkArray(node *ast.Node) error { func (s *SeenTracker) checkArray(parentIdx int, node *ast.Node) error {
set := false
it := node.Children() it := node.Children()
for it.Next() { for it.Next() {
if set {
s.clear(parentIdx)
}
n := it.Node() n := it.Node()
switch n.Kind { switch n.Kind {
case ast.InlineTable: case ast.InlineTable:
err := s.checkInlineTable(n) err := s.checkInlineTable(parentIdx, n)
if err != nil { if err != nil {
return err return err
} }
set = true
case ast.Array: case ast.Array:
err := s.checkArray(n) err := s.checkArray(parentIdx, n)
if err != nil { if err != nil {
return err return err
} }
set = true
} }
} }
return nil return nil
} }
func (s *SeenTracker) checkInlineTable(node *ast.Node) error { func (s *SeenTracker) checkInlineTable(parentIdx int, node *ast.Node) error {
if pool.New == nil {
pool.New = func() interface{} {
return &SeenTracker{}
}
}
s = pool.Get().(*SeenTracker)
s.reset()
it := node.Children() it := node.Children()
for it.Next() { for it.Next() {
n := it.Node() n := it.Node()
err := s.checkKeyValue(n) err := s.checkKeyValue(parentIdx, n)
if err != nil { if err != nil {
return err return err
} }
} }
// As inline tables are self-contained, the tracker does not
// need to retain the details of what they contain. The
// keyValue element that creates the inline table is kept to
// mark the presence of the inline table and prevent
// redefinition of its keys: check* functions cannot walk into
// a value.
pool.Put(s)
return nil return nil
} }
func (s *SeenTracker) id(idx int) int {
if idx >= 0 {
return s.entries[idx].id
}
return 0
}
func (s *SeenTracker) find(parentIdx int, k []byte) int {
parentID := s.id(parentIdx)
for i := parentIdx + 1; i < len(s.entries); i++ {
if s.entries[i].parent == parentID && bytes.Equal(s.entries[i].name, k) {
return i
}
}
return -1
}
+32 -132
View File
@@ -11,7 +11,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"unicode"
) )
// Marshal serializes a Go value as a TOML document. // Marshal serializes a Go value as a TOML document.
@@ -104,31 +103,27 @@ func (enc *Encoder) SetIndentTables(indent bool) *Encoder {
// Intermediate tables are always printed. // Intermediate tables are always printed.
// //
// By default, strings are encoded as literal string, unless they contain either // By default, strings are encoded as literal string, unless they contain either
// a newline character or a single quote. In that case they are emitted as // a newline character or a single quote. In that case they are emitted as quoted
// quoted strings. // strings.
// //
// When encoding structs, fields are encoded in order of definition, with their // When encoding structs, fields are encoded in order of definition, with their
// exact name. // exact name.
// //
// Struct tags // Struct tags
// //
// The encoding of each public struct field can be customized by the format // The following struct tags are available to tweak encoding on a per-field
// string in the "toml" key of the struct field's tag. This follows // basis:
// encoding/json's convention. The format string starts with the name of the
// field, optionally followed by a comma-separated list of options. The name may
// be empty in order to provide options without overriding the default name.
// //
// The "multiline" option emits strings as quoted multi-line TOML strings. It // toml:"foo"
// has no effect on fields that would not be encoded as strings. // Changes the name of the key to use for the field to foo.
// //
// The "inline" option turns fields that would be emitted as tables into inline // multiline:"true"
// tables instead. It has no effect on other fields. // When the field contains a string, it will be emitted as a quoted
// multi-line TOML string.
// //
// The "omitempty" option prevents empty values or groups from being emitted. // inline:"true"
// // When the field would normally be encoded as a table, it is instead
// In addition to the "toml" tag struct tag, a "comment" tag can be used to emit // encoded as an inline table.
// a TOML comment before the value being annotated. Comments are ignored inside
// inline tables.
func (enc *Encoder) Encode(v interface{}) error { func (enc *Encoder) Encode(v interface{}) error {
var ( var (
b []byte b []byte
@@ -156,8 +151,6 @@ func (enc *Encoder) Encode(v interface{}) error {
type valueOptions struct { type valueOptions struct {
multiline bool multiline bool
omitempty bool
comment string
} }
type encoderCtx struct { type encoderCtx struct {
@@ -207,6 +200,7 @@ func (ctx *encoderCtx) isRoot() bool {
return len(ctx.parentKey) == 0 && !ctx.hasKey return len(ctx.parentKey) == 0 && !ctx.hasKey
} }
//nolint:cyclop,funlen
func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
if !v.IsZero() { if !v.IsZero() {
i, ok := v.Interface().(time.Time) i, ok := v.Interface().(time.Time)
@@ -215,12 +209,7 @@ func (enc *Encoder) encode(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, e
} }
} }
hasTextMarshaler := v.Type().Implements(textMarshalerType) if v.Type().Implements(textMarshalerType) {
if hasTextMarshaler || (v.CanAddr() && reflect.PtrTo(v.Type()).Implements(textMarshalerType)) {
if !hasTextMarshaler {
v = v.Addr()
}
if ctx.isRoot() { if ctx.isRoot() {
return nil, fmt.Errorf("toml: type %s implementing the TextMarshaler interface cannot be a root element", v.Type()) return nil, fmt.Errorf("toml: type %s implementing the TextMarshaler interface cannot be a root element", v.Type())
} }
@@ -303,15 +292,6 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
if !ctx.hasKey { if !ctx.hasKey {
panic("caller of encodeKv should have set the key in the context") panic("caller of encodeKv should have set the key in the context")
} }
if (ctx.options.omitempty || options.omitempty) && isEmptyValue(v) {
return b, nil
}
if !ctx.inline {
b = enc.encodeComment(ctx.indent, options.comment, b)
}
b = enc.indent(ctx.indent, b) b = enc.indent(ctx.indent, b)
b, err = enc.encodeKey(b, ctx.key) b, err = enc.encodeKey(b, ctx.key)
@@ -336,24 +316,6 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
return b, nil return b, nil
} }
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
return false
}
const literalQuote = '\'' const literalQuote = '\''
func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byte { func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byte {
@@ -447,8 +409,6 @@ func (enc *Encoder) encodeTableHeader(ctx encoderCtx, b []byte) ([]byte, error)
return b, nil return b, nil
} }
b = enc.encodeComment(ctx.indent, ctx.options.comment, b)
b = enc.indent(ctx.indent, b) b = enc.indent(ctx.indent, b)
b = append(b, '[') b = append(b, '[')
@@ -565,7 +525,8 @@ func (t *table) pushTable(k string, v reflect.Value, options valueOptions) {
func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) { func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]byte, error) {
var t table var t table
// TODO: cache this //nolint:godox
// TODO: cache this?
typ := v.Type() typ := v.Type()
for i := 0; i < typ.NumField(); i++ { for i := 0; i < typ.NumField(); i++ {
fieldType := typ.Field(i) fieldType := typ.Field(i)
@@ -575,18 +536,14 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
continue continue
} }
k := fieldType.Name k, ok := fieldType.Tag.Lookup("toml")
if !ok {
tag := fieldType.Tag.Get("toml") k = fieldType.Name
// special field name to skip field
if tag == "-" {
continue
} }
name, opts := parseTag(tag) // special field name to skip field
if isValidName(name) { if k == "-" {
k = name continue
} }
f := v.Field(i) f := v.Field(i)
@@ -596,12 +553,12 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
} }
options := valueOptions{ options := valueOptions{
multiline: opts.multiline, multiline: fieldBoolTag(fieldType, "multiline"),
omitempty: opts.omitempty,
comment: fieldType.Tag.Get("comment"),
} }
if opts.inline || !willConvertToTableOrArrayTable(ctx, f) { inline := fieldBoolTag(fieldType, "inline")
if inline || !willConvertToTableOrArrayTable(ctx, f) {
t.pushKV(k, f, options) t.pushKV(k, f, options)
} else { } else {
t.pushTable(k, f, options) t.pushTable(k, f, options)
@@ -611,70 +568,13 @@ func (enc *Encoder) encodeStruct(b []byte, ctx encoderCtx, v reflect.Value) ([]b
return enc.encodeTable(b, ctx, t) return enc.encodeTable(b, ctx, t)
} }
func (enc *Encoder) encodeComment(indent int, comment string, b []byte) []byte { func fieldBoolTag(field reflect.StructField, tag string) bool {
if comment != "" { x, ok := field.Tag.Lookup(tag)
b = enc.indent(indent, b)
b = append(b, "# "...) return ok && x == "true"
b = append(b, comment...)
b = append(b, '\n')
}
return b
}
func isValidName(s string) bool {
if s == "" {
return false
}
for _, c := range s {
switch {
case strings.ContainsRune("!#$%&()*+-./:;<=>?@[]^_{|}~ ", c):
// Backslash and quote chars are reserved, but
// otherwise any punctuation chars are allowed
// in a tag name.
case !unicode.IsLetter(c) && !unicode.IsDigit(c):
return false
}
}
return true
}
type tagOptions struct {
multiline bool
inline bool
omitempty bool
}
func parseTag(tag string) (string, tagOptions) {
opts := tagOptions{}
idx := strings.Index(tag, ",")
if idx == -1 {
return tag, opts
}
raw := tag[idx+1:]
tag = string(tag[:idx])
for raw != "" {
var o string
i := strings.Index(raw, ",")
if i >= 0 {
o, raw = raw[:i], raw[i+1:]
} else {
o, raw = raw, ""
}
switch o {
case "multiline":
opts.multiline = true
case "inline":
opts.inline = true
case "omitempty":
opts.omitempty = true
}
}
return tag, opts
} }
//nolint:cyclop
func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, error) { func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, error) {
var err error var err error
@@ -757,7 +657,7 @@ func willConvertToTable(ctx encoderCtx, v reflect.Value) bool {
if !v.IsValid() { if !v.IsValid() {
return false return false
} }
if v.Type() == timeType || v.Type().Implements(textMarshalerType) || (v.Kind() != reflect.Ptr && v.CanAddr() && reflect.PtrTo(v.Type()).Implements(textMarshalerType)) { if v.Type() == timeType || v.Type().Implements(textMarshalerType) {
return false return false
} }
+32 -158
View File
@@ -4,27 +4,20 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"math/big"
"strings" "strings"
"testing" "testing"
"time"
"github.com/pelletier/go-toml/v2" "github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
//nolint:funlen
func TestMarshal(t *testing.T) { func TestMarshal(t *testing.T) {
someInt := 42 someInt := 42
type structInline struct { type structInline struct {
A interface{} `toml:",inline"` A interface{} `inline:"true"`
}
type comments struct {
One int
Two int `comment:"Before kv"`
Three []int `comment:"Before array"`
} }
examples := []struct { examples := []struct {
@@ -200,9 +193,9 @@ name = 'Alice'
{ {
desc: "string escapes", desc: "string escapes",
v: map[string]interface{}{ v: map[string]interface{}{
"a": "'\b\f\r\t\"\\", "a": `'"\`,
}, },
expected: `a = "'\b\f\r\t\"\\"`, expected: `a = "'\"\\"`,
}, },
{ {
desc: "string utf8 low", desc: "string utf8 low",
@@ -249,7 +242,7 @@ name = 'Alice'
{ {
desc: "multi-line forced", desc: "multi-line forced",
v: struct { v: struct {
A string `toml:",multiline"` A string `multiline:"true"`
}{ }{
A: "hello\nworld", A: "hello\nworld",
}, },
@@ -260,7 +253,7 @@ world"""`,
{ {
desc: "inline field", desc: "inline field",
v: struct { v: struct {
A map[string]string `toml:",inline"` A map[string]string `inline:"true"`
B map[string]string B map[string]string
}{ }{
A: map[string]string{ A: map[string]string{
@@ -279,7 +272,7 @@ isinline = 'no'
{ {
desc: "mutiline array int", desc: "mutiline array int",
v: struct { v: struct {
A []int `toml:",multiline"` A []int `multiline:"true"`
B []int B []int
}{ }{
A: []int{1, 2, 3, 4}, A: []int{1, 2, 3, 4},
@@ -298,7 +291,7 @@ B = [1, 2, 3, 4]
{ {
desc: "mutiline array in array", desc: "mutiline array in array",
v: struct { v: struct {
A [][]int `toml:",multiline"` A [][]int `multiline:"true"`
}{ }{
A: [][]int{{1, 2}, {3, 4}}, A: [][]int{{1, 2}, {3, 4}},
}, },
@@ -476,28 +469,6 @@ hello = 'world'`,
}, },
err: true, err: true,
}, },
{
desc: "time",
v: struct {
T time.Time
}{
T: time.Time{},
},
expected: `T = '0001-01-01T00:00:00Z'`,
},
{
desc: "bool",
v: struct {
A bool
B bool
}{
A: false,
B: true,
},
expected: `
A = false
B = true`,
},
{ {
desc: "numbers", desc: "numbers",
v: struct { v: struct {
@@ -512,7 +483,6 @@ B = true`,
I int16 I int16
J int8 J int8
K int K int
L float64
}{ }{
A: 1.1, A: 1.1,
B: 42, B: 42,
@@ -525,7 +495,6 @@ B = true`,
I: 42, I: 42,
J: 42, J: 42,
K: 42, K: 42,
L: 2.2,
}, },
expected: ` expected: `
A = 1.1 A = 1.1
@@ -538,29 +507,7 @@ G = 42
H = 42 H = 42
I = 42 I = 42
J = 42 J = 42
K = 42 K = 42`,
L = 2.2`,
},
{
desc: "comments",
v: struct {
Table comments `comment:"Before table"`
}{
Table: comments{
One: 1,
Two: 2,
Three: []int{1, 2, 3},
},
},
expected: `
# Before table
[Table]
One = 1
# Before kv
Two = 2
# Before array
Three = [1, 2, 3]
`,
}, },
} }
@@ -787,60 +734,6 @@ func TestEncoderSetIndentSymbol(t *testing.T) {
equalStringsIgnoreNewlines(t, expected, w.String()) equalStringsIgnoreNewlines(t, expected, w.String())
} }
func TestEncoderOmitempty(t *testing.T) {
type doc struct {
String string `toml:",omitempty,multiline"`
Bool bool `toml:",omitempty,multiline"`
Int int `toml:",omitempty,multiline"`
Int8 int8 `toml:",omitempty,multiline"`
Int16 int16 `toml:",omitempty,multiline"`
Int32 int32 `toml:",omitempty,multiline"`
Int64 int64 `toml:",omitempty,multiline"`
Uint uint `toml:",omitempty,multiline"`
Uint8 uint8 `toml:",omitempty,multiline"`
Uint16 uint16 `toml:",omitempty,multiline"`
Uint32 uint32 `toml:",omitempty,multiline"`
Uint64 uint64 `toml:",omitempty,multiline"`
Float32 float32 `toml:",omitempty,multiline"`
Float64 float64 `toml:",omitempty,multiline"`
MapNil map[string]string `toml:",omitempty,multiline"`
Slice []string `toml:",omitempty,multiline"`
Ptr *string `toml:",omitempty,multiline"`
Iface interface{} `toml:",omitempty,multiline"`
Struct struct{} `toml:",omitempty,multiline"`
}
d := doc{}
b, err := toml.Marshal(d)
require.NoError(t, err)
expected := `[Struct]`
equalStringsIgnoreNewlines(t, expected, string(b))
}
func TestEncoderTagFieldName(t *testing.T) {
type doc struct {
String string `toml:"hello"`
OkSym string `toml:"#"`
Bad string `toml:"\"`
}
d := doc{String: "world"}
b, err := toml.Marshal(d)
require.NoError(t, err)
expected := `
hello = 'world'
'#' = ''
Bad = ''
`
equalStringsIgnoreNewlines(t, expected, string(b))
}
func TestIssue436(t *testing.T) { func TestIssue436(t *testing.T) {
data := []byte(`{"a": [ { "b": { "c": "d" } } ]}`) data := []byte(`{"a": [ { "b": { "c": "d" } } ]}`)
@@ -905,48 +798,6 @@ func TestIssue590(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestIssue571(t *testing.T) {
type Foo struct {
Float32 float32
Float64 float64
}
const closeEnough = 1e-9
foo := Foo{
Float32: 42,
Float64: 43,
}
b, err := toml.Marshal(foo)
require.NoError(t, err)
var foo2 Foo
err = toml.Unmarshal(b, &foo2)
require.NoError(t, err)
assert.InDelta(t, 42, foo2.Float32, closeEnough)
assert.InDelta(t, 43, foo2.Float64, closeEnough)
}
func TestIssue678(t *testing.T) {
type Config struct {
BigInt big.Int
}
cfg := &Config{
BigInt: *big.NewInt(123),
}
out, err := toml.Marshal(cfg)
require.NoError(t, err)
equalStringsIgnoreNewlines(t, "BigInt = '123'", string(out))
cfg2 := &Config{}
err = toml.Unmarshal(out, cfg2)
require.NoError(t, err)
require.Equal(t, cfg, cfg2)
}
func ExampleMarshal() { func ExampleMarshal() {
type MyConfig struct { type MyConfig struct {
Version int Version int
@@ -971,3 +822,26 @@ func ExampleMarshal() {
// Name = 'go-toml' // Name = 'go-toml'
// Tags = ['go', 'toml'] // Tags = ['go', 'toml']
} }
func TestIssue571(t *testing.T) {
type Foo struct {
Float32 float32
Float64 float64
}
const closeEnough = 1e-9
foo := Foo{
Float32: 42,
Float64: 43,
}
b, err := toml.Marshal(foo)
require.NoError(t, err)
var foo2 Foo
err = toml.Unmarshal(b, &foo2)
require.NoError(t, err)
assert.InDelta(t, 42, foo2.Float32, closeEnough)
assert.InDelta(t, 43, foo2.Float64, closeEnough)
}
+13 -8
View File
@@ -578,10 +578,6 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
switch token[i+j] { switch token[i+j] {
case ' ', '\t': case ' ', '\t':
continue continue
case '\r':
if token[i+j+1] == '\n' {
continue
}
case '\n': case '\n':
isLastNonWhitespaceOnLine = true isLastNonWhitespaceOnLine = true
} }
@@ -693,10 +689,6 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
} }
func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) { func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
if len(b) == 0 {
return nil, nil, nil, newDecodeError(b, "expected key but found none")
}
// simple-key = quoted-key / unquoted-key // simple-key = quoted-key / unquoted-key
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _ // unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
// quoted-key = basic-string / literal-string // quoted-key = basic-string / literal-string
@@ -870,6 +862,7 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
//nolint:gomnd
if len(b) < 3 { if len(b) < 3 {
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
@@ -894,6 +887,18 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
func digitsToInt(b []byte) int {
x := 0
for _, d := range b {
x *= 10
x += int(d - '0')
}
return x
}
//nolint:gocognit,cyclop
func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) { func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
// scans for contiguous characters in [0-9T:Z.+-], and up to one space if // scans for contiguous characters in [0-9T:Z.+-], and up to one space if
// followed by a digit. // followed by a digit.
+1 -21
View File
@@ -53,7 +53,7 @@ func scanLiteralString(b []byte) ([]byte, []byte, error) {
switch b[i] { switch b[i] {
case '\'': case '\'':
return b[:i+1], b[i+1:], nil return b[:i+1], b[i+1:], nil
case '\n', '\r': case '\n':
return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines") return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines")
} }
size := utf8ValidNext(b[i:]) size := utf8ValidNext(b[i:])
@@ -76,8 +76,6 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
// mll-char = %x09 / %x20-26 / %x28-7E / non-ascii // mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
// mll-quotes = 1*2apostrophe // mll-quotes = 1*2apostrophe
for i := 3; i < len(b); { for i := 3; i < len(b); {
switch b[i] {
case '\'':
if scanFollowsMultilineLiteralStringDelimiter(b[i:]) { if scanFollowsMultilineLiteralStringDelimiter(b[i:]) {
i += 3 i += 3
@@ -103,16 +101,6 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
return b[:i], b[i:], nil return b[:i], b[i:], nil
} }
case '\r':
if len(b) < i+2 {
return nil, nil, newDecodeError(b[len(b):], `need a \n after \r`)
}
if b[i+1] != '\n' {
return nil, nil, newDecodeError(b[i:i+2], `need a \n after \r`)
}
i += 2 // skip the \n
continue
}
size := utf8ValidNext(b[i:]) size := utf8ValidNext(b[i:])
if size == 0 { if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character") return nil, nil, newDecodeError(b[i:i+1], "invalid character")
@@ -254,14 +242,6 @@ func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) {
} }
escaped = true escaped = true
i++ // skip the next character i++ // skip the next character
case '\r':
if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[len(b):], `need a \n after \r`)
}
if b[i+1] != '\n' {
return nil, escaped, nil, newDecodeError(b[i:i+2], `need a \n after \r`)
}
i++ // skip the \n
} }
} }
+159 -105
View File
@@ -11,6 +11,7 @@ import (
"strings" "strings"
"sync/atomic" "sync/atomic"
"time" "time"
"unsafe"
"github.com/pelletier/go-toml/v2/internal/ast" "github.com/pelletier/go-toml/v2/internal/ast"
"github.com/pelletier/go-toml/v2/internal/danger" "github.com/pelletier/go-toml/v2/internal/danger"
@@ -60,8 +61,7 @@ func (d *Decoder) SetStrict(strict bool) *Decoder {
// //
// When a TOML local date, time, or date-time is decoded into a time.Time, its // When a TOML local date, time, or date-time is decoded into a time.Time, its
// value is represented in time.Local timezone. Otherwise the approriate Local* // value is represented in time.Local timezone. Otherwise the approriate Local*
// structure is used. For time values, precision up to the nanosecond is // structure is used.
// supported by truncating extra digits.
// //
// Empty tables decoded in an interface{} create an empty initialized // Empty tables decoded in an interface{} create an empty initialized
// map[string]interface{}. // map[string]interface{}.
@@ -73,11 +73,6 @@ func (d *Decoder) SetStrict(strict bool) *Decoder {
// bounds for the target type (which includes negative numbers when decoding // bounds for the target type (which includes negative numbers when decoding
// into an unsigned int). // into an unsigned int).
// //
// If an error occurs while decoding the content of the document, this function
// returns a toml.DecodeError, providing context about the issue. When using
// strict mode and a field is missing, a `toml.StrictMissingError` is
// returned. In any other case, this function returns a standard Go error.
//
// Type mapping // Type mapping
// //
// List of supported TOML types and their associated accepted Go types: // List of supported TOML types and their associated accepted Go types:
@@ -137,23 +132,6 @@ type decoder struct {
// Strict mode // Strict mode
strict strict strict strict
// Current context for the error.
errorContext *errorContext
}
type errorContext struct {
Struct reflect.Type
Field []int
}
func (d *decoder) typeMismatchError(toml string, target reflect.Type) error {
if d.errorContext != nil && d.errorContext.Struct != nil {
ctx := d.errorContext
f := ctx.Struct.FieldByIndex(ctx.Field)
return fmt.Errorf("toml: cannot decode TOML %s into struct field %s.%s of type %s", toml, ctx.Struct, f.Name, f.Type)
}
return fmt.Errorf("toml: cannot decode TOML %s into a Go value of type %s", toml, target)
} }
func (d *decoder) expr() *ast.Node { func (d *decoder) expr() *ast.Node {
@@ -368,9 +346,7 @@ func (d *decoder) handleArrayTableCollection(key ast.Iterator, v reflect.Value)
if err != nil { if err != nil {
return reflect.Value{}, err return reflect.Value{}, err
} }
if elem.IsValid() {
v.Elem().Set(elem) v.Elem().Set(elem)
}
return v, nil return v, nil
case reflect.Slice: case reflect.Slice:
@@ -467,20 +443,12 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
v.SetMapIndex(mk, mv) v.SetMapIndex(mk, mv)
} }
case reflect.Struct: case reflect.Struct:
path, found := structFieldPath(v, string(key.Node().Data)) f, found := structField(v, string(key.Node().Data))
if !found { if !found {
d.skipUntilTable = true d.skipUntilTable = true
return reflect.Value{}, nil return reflect.Value{}, nil
} }
if d.errorContext == nil {
d.errorContext = new(errorContext)
}
t := v.Type()
d.errorContext.Struct = t
d.errorContext.Field = path
f := v.FieldByIndex(path)
x, err := nextFn(key, f) x, err := nextFn(key, f)
if err != nil || d.skipUntilTable { if err != nil || d.skipUntilTable {
return reflect.Value{}, err return reflect.Value{}, err
@@ -488,8 +456,6 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle
if x.IsValid() { if x.IsValid() {
f.Set(x) f.Set(x)
} }
d.errorContext.Field = nil
d.errorContext.Struct = nil
case reflect.Interface: case reflect.Interface:
if v.Elem().IsValid() { if v.Elem().IsValid() {
v = v.Elem() v = v.Elem()
@@ -655,62 +621,128 @@ func (d *decoder) handleValue(value *ast.Node, v reflect.Value) error {
} }
} }
func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error { type unmarshalArrayFn func(d *decoder, array *ast.Node, v reflect.Value) error
switch v.Kind() {
case reflect.Slice: var globalUnmarshalArrayFnCache atomic.Value // map[danger.TypeID]unmarshalArrayFn
if v.IsNil() {
v.Set(reflect.MakeSlice(v.Type(), 0, 16)) func unmarshalArrayFnForSlice(vt reflect.Type) unmarshalArrayFn {
} else { tid := danger.MakeTypeID(vt)
v.SetLen(0)
} cache, _ := globalUnmarshalArrayFnCache.Load().(map[danger.TypeID]unmarshalArrayFn)
case reflect.Array: fn, ok := cache[tid]
// arrays are always initialized
case reflect.Interface: if ok {
elem := v.Elem() return fn
if !elem.IsValid() {
elem = reflect.New(sliceInterfaceType).Elem()
elem.Set(reflect.MakeSlice(sliceInterfaceType, 0, 16))
} else if elem.Kind() == reflect.Slice {
if elem.Type() != sliceInterfaceType {
elem = reflect.New(sliceInterfaceType).Elem()
elem.Set(reflect.MakeSlice(sliceInterfaceType, 0, 16))
} else if !elem.CanSet() {
nelem := reflect.New(sliceInterfaceType).Elem()
nelem.Set(reflect.MakeSlice(sliceInterfaceType, elem.Len(), elem.Cap()))
reflect.Copy(nelem, elem)
elem = nelem
}
}
err := d.unmarshalArray(array, elem)
if err != nil {
return err
}
v.Set(elem)
return nil
default:
// TODO: use newDecodeError, but first the parser needs to fill
// array.Data.
return d.typeMismatchError("array", v.Type())
} }
elemType := v.Type().Elem() elemType := vt.Elem()
elemSize := elemType.Size()
fn = func(d *decoder, array *ast.Node, v reflect.Value) error {
sp := (*danger.Slice)(unsafe.Pointer(v.UnsafeAddr()))
sp.Len = 0
it := array.Children() it := array.Children()
idx := 0
for it.Next() { for it.Next() {
n := it.Node() n := it.Node()
// TODO: optimize idx := sp.Len
if v.Kind() == reflect.Slice {
elem := reflect.New(elemType).Elem() if sp.Len == sp.Cap {
c := sp.Cap
if c == 0 {
c = 16
} else {
c *= 2
}
*sp = danger.ExtendSlice(vt, sp, c)
}
datap := unsafe.Pointer(sp.Data)
elemp := danger.Stride(datap, elemSize, idx)
elem := reflect.NewAt(elemType, elemp).Elem()
err := d.handleValue(n, elem) err := d.handleValue(n, elem)
if err != nil { if err != nil {
return err return err
} }
v.Set(reflect.Append(v, elem)) sp.Len++
} else { // array }
if sp.Data == nil {
*sp = danger.ExtendSlice(vt, sp, 0)
}
return nil
}
newCache := make(map[danger.TypeID]unmarshalArrayFn, len(cache)+1)
newCache[tid] = fn
for k, v := range cache {
newCache[k] = v
}
globalUnmarshalArrayFnCache.Store(newCache)
return fn
}
func unmarshalArraySliceInterface(d *decoder, array *ast.Node, v reflect.Value) error {
sp := (*danger.Slice)(unsafe.Pointer(v.UnsafeAddr()))
sp.Len = 0
var x interface{}
it := array.Children()
for it.Next() {
n := it.Node()
idx := sp.Len
if sp.Len == sp.Cap {
c := sp.Cap
if c == 0 {
c = 16
} else {
c *= 2
}
*sp = danger.ExtendSlice(sliceInterfaceType, sp, c)
}
datap := unsafe.Pointer(sp.Data)
elemp := danger.Stride(datap, unsafe.Sizeof(x), idx)
elem := reflect.NewAt(sliceInterfaceType.Elem(), elemp).Elem()
err := d.handleValue(n, elem)
if err != nil {
return err
}
sp.Len++
}
if sp.Data == nil {
*sp = danger.ExtendSlice(sliceInterfaceType, sp, 0)
}
return nil
}
func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error {
switch v.Kind() {
case reflect.Slice:
fn := unmarshalArrayFnForSlice(v.Type())
return fn(d, array, v)
case reflect.Array:
// arrays are always initialized
it := array.Children()
idx := 0
for it.Next() {
n := it.Node()
if idx >= v.Len() { if idx >= v.Len() {
return nil return nil
} }
@@ -721,6 +753,39 @@ func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error {
} }
idx++ idx++
} }
case reflect.Interface:
elemIsSliceInterface := false
elem := v.Elem()
if !elem.IsValid() {
s := make([]interface{}, 0, 16)
elem = reflect.ValueOf(&s).Elem()
elemIsSliceInterface = true
} else if elem.Kind() == reflect.Slice {
if elem.Type() != sliceInterfaceType {
s := make([]interface{}, 0, 16)
elem = reflect.ValueOf(&s).Elem()
} else if !elem.CanSet() {
s := make([]interface{}, elem.Len(), elem.Cap())
nelem := reflect.ValueOf(&s).Elem()
reflect.Copy(nelem, elem)
elem = nelem
}
elemIsSliceInterface = true
}
var err error
if elemIsSliceInterface {
err = unmarshalArraySliceInterface(d, array, elem)
} else {
err = d.unmarshalArray(array, elem)
}
v.Set(elem)
return err
default:
// TODO: use newDecodeError, but first the parser needs to fill
// array.Data.
return fmt.Errorf("toml: cannot store array in Go type %s", v.Kind())
} }
return nil return nil
@@ -937,7 +1002,7 @@ func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error {
case reflect.Interface: case reflect.Interface:
r = reflect.ValueOf(i) r = reflect.ValueOf(i)
default: default:
return d.typeMismatchError("integer", v.Type()) return fmt.Errorf("toml: cannot store TOML integer into a Go %s", v.Kind())
} }
if !r.Type().AssignableTo(v.Type()) { if !r.Type().AssignableTo(v.Type()) {
@@ -1040,20 +1105,12 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
v.SetMapIndex(mk, mv) v.SetMapIndex(mk, mv)
} }
case reflect.Struct: case reflect.Struct:
path, found := structFieldPath(v, string(key.Node().Data)) f, found := structField(v, string(key.Node().Data))
if !found { if !found {
d.skipUntilTable = true d.skipUntilTable = true
break break
} }
if d.errorContext == nil {
d.errorContext = new(errorContext)
}
t := v.Type()
d.errorContext.Struct = t
d.errorContext.Field = path
f := v.FieldByIndex(path)
x, err := d.handleKeyValueInner(key, value, f) x, err := d.handleKeyValueInner(key, value, f)
if err != nil { if err != nil {
return reflect.Value{}, err return reflect.Value{}, err
@@ -1062,8 +1119,6 @@ func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflec
if x.IsValid() { if x.IsValid() {
f.Set(x) f.Set(x)
} }
d.errorContext.Struct = nil
d.errorContext.Field = nil
case reflect.Interface: case reflect.Interface:
v = v.Elem() v = v.Elem()
@@ -1121,11 +1176,12 @@ type fieldPathsMap = map[string][]int
var globalFieldPathsCache atomic.Value // map[danger.TypeID]fieldPathsMap var globalFieldPathsCache atomic.Value // map[danger.TypeID]fieldPathsMap
func structFieldPath(v reflect.Value, name string) ([]int, bool) { func structField(v reflect.Value, name string) (reflect.Value, bool) {
t := v.Type() t := v.Type()
tid := danger.MakeTypeID(t)
cache, _ := globalFieldPathsCache.Load().(map[danger.TypeID]fieldPathsMap) cache, _ := globalFieldPathsCache.Load().(map[danger.TypeID]fieldPathsMap)
fieldPaths, ok := cache[danger.MakeTypeID(t)] fieldPaths, ok := cache[tid]
if !ok { if !ok {
fieldPaths = map[string][]int{} fieldPaths = map[string][]int{}
@@ -1137,7 +1193,7 @@ func structFieldPath(v reflect.Value, name string) ([]int, bool) {
}) })
newCache := make(map[danger.TypeID]fieldPathsMap, len(cache)+1) newCache := make(map[danger.TypeID]fieldPathsMap, len(cache)+1)
newCache[danger.MakeTypeID(t)] = fieldPaths newCache[tid] = fieldPaths
for k, v := range cache { for k, v := range cache {
newCache[k] = v newCache[k] = v
} }
@@ -1148,7 +1204,12 @@ func structFieldPath(v reflect.Value, name string) ([]int, bool) {
if !ok { if !ok {
path, ok = fieldPaths[strings.ToLower(name)] path, ok = fieldPaths[strings.ToLower(name)]
} }
return path, ok
if !ok {
return reflect.Value{}, false
}
return v.FieldByIndex(path), true
} }
func forEachField(t reflect.Type, path []int, do func(name string, path []int)) { func forEachField(t reflect.Type, path []int, do func(name string, path []int)) {
@@ -1169,15 +1230,8 @@ func forEachField(t reflect.Type, path []int, do func(name string, path []int))
continue continue
} }
name := f.Tag.Get("toml") name, ok := f.Tag.Lookup("toml")
if name == "-" { if !ok {
continue
}
if i := strings.IndexByte(name, ','); i >= 0 {
name = name[:i]
}
if name == "" {
name = f.Name name = f.Name
} }
+14 -262
View File
@@ -571,7 +571,7 @@ func TestUnmarshal(t *testing.T) {
}, },
{ {
desc: "multiline basic string with windows newline", desc: "multiline basic string with windows newline",
input: "A = \"\"\"\r\nTe\r\nst\"\"\"", input: "A = \"\"\"\r\nTest\"\"\"",
gen: func() test { gen: func() test {
type doc struct { type doc struct {
A string A string
@@ -579,7 +579,7 @@ func TestUnmarshal(t *testing.T) {
return test{ return test{
target: &doc{}, target: &doc{},
expected: &doc{A: "Te\r\nst"}, expected: &doc{A: "Test"},
} }
}, },
}, },
@@ -664,36 +664,6 @@ func TestUnmarshal(t *testing.T) {
} }
}, },
}, },
{
desc: "long string array into []string",
input: `A = ["0","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17"]`,
gen: func() test {
type doc struct {
A []string
}
return test{
target: &doc{},
expected: &doc{A: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17"}},
}
},
},
{
desc: "long string array into []interface{}",
input: `A = ["0","1","2","3","4","5","6","7","8","9","10","11","12","13","14",
"15","16","17"]`,
gen: func() test {
type doc struct {
A []interface{}
}
return test{
target: &doc{},
expected: &doc{A: []interface{}{"0", "1", "2", "3", "4", "5", "6",
"7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17"}},
}
},
},
{ {
desc: "standard table", desc: "standard table",
input: `[A] input: `[A]
@@ -1790,20 +1760,6 @@ func TestUnmarshalOverflows(t *testing.T) {
} }
} }
func TestUnmarshalErrors(t *testing.T) {
type mystruct struct {
Bar string
}
data := `bar = 42`
s := mystruct{}
err := toml.Unmarshal([]byte(data), &s)
require.Error(t, err)
require.Equal(t, "toml: cannot decode TOML integer into struct field toml_test.mystruct.Bar of type string", err.Error())
}
func TestUnmarshalInvalidTarget(t *testing.T) { func TestUnmarshalInvalidTarget(t *testing.T) {
x := "foo" x := "foo"
err := toml.Unmarshal([]byte{}, x) err := toml.Unmarshal([]byte{}, x)
@@ -2129,7 +2085,7 @@ xz_hash = "1a48f723fea1f17d786ce6eadd9d00914d38062d28fd9c455ed3c3801905b388"
expected := doc{ expected := doc{
Pkg: map[string]pkg{ Pkg: map[string]pkg{
"cargo": { "cargo": pkg{
Target: map[string]target{ Target: map[string]target{
"aarch64-apple-darwin": { "aarch64-apple-darwin": {
XZ_URL: "https://static.rust-lang.org/dist/2021-07-29/cargo-1.54.0-aarch64-apple-darwin.tar.xz", XZ_URL: "https://static.rust-lang.org/dist/2021-07-29/cargo-1.54.0-aarch64-apple-darwin.tar.xz",
@@ -2234,119 +2190,7 @@ func TestIssue666(t *testing.T) {
require.Error(t, err) require.Error(t, err)
} }
func TestIssue677(t *testing.T) { //nolint:funlen
doc := `
[Build]
Name = "publication build"
[[Build.Dependencies]]
Name = "command"
Program = "hugo"
`
type _tomlJob struct {
Dependencies []map[string]interface{}
}
type tomlParser struct {
Build *_tomlJob
}
p := tomlParser{}
err := toml.Unmarshal([]byte(doc), &p)
require.NoError(t, err)
expected := tomlParser{
Build: &_tomlJob{
Dependencies: []map[string]interface{}{
{
"Name": "command",
"Program": "hugo",
},
},
},
}
require.Equal(t, expected, p)
}
func TestIssue701(t *testing.T) {
// Expected behavior:
// Return an error since a cannot be modified. From the TOML spec:
//
// > Inline tables are fully self-contained and define all
// keys and sub-tables within them. Keys and sub-tables cannot
// be added outside the braces.
docs := []string{
`
a={}
[a.b]
z=0
`,
`
a={}
[[a.b]]
z=0
`,
}
for _, doc := range docs {
var v interface{}
err := toml.Unmarshal([]byte(doc), &v)
assert.Error(t, err)
}
}
func TestIssue703(t *testing.T) {
var v interface{}
err := toml.Unmarshal([]byte("[a]\nx.y=0\n[a.x]"), &v)
require.Error(t, err)
}
func TestIssue708(t *testing.T) {
v := map[string]string{}
err := toml.Unmarshal([]byte("0=\"\"\"\\\r\n\"\"\""), &v)
require.NoError(t, err)
require.Equal(t, map[string]string{"0": ""}, v)
}
func TestIssue710(t *testing.T) {
v := map[string]toml.LocalTime{}
err := toml.Unmarshal([]byte(`0=00:00:00.0000000000`), &v)
require.NoError(t, err)
require.Equal(t, map[string]toml.LocalTime{"0": {Precision: 9}}, v)
v1 := map[string]toml.LocalTime{}
err = toml.Unmarshal([]byte(`0=00:00:00.0000000001`), &v1)
require.NoError(t, err)
require.Equal(t, map[string]toml.LocalTime{"0": {Precision: 9}}, v1)
v2 := map[string]toml.LocalTime{}
err = toml.Unmarshal([]byte(`0=00:00:00.1111111119`), &v2)
require.NoError(t, err)
require.Equal(t, map[string]toml.LocalTime{"0": {Nanosecond: 111111111, Precision: 9}}, v2)
}
func TestIssue715(t *testing.T) {
var v interface{}
err := toml.Unmarshal([]byte("0=+"), &v)
require.Error(t, err)
err = toml.Unmarshal([]byte("0=-"), &v)
require.Error(t, err)
err = toml.Unmarshal([]byte("0=+A"), &v)
require.Error(t, err)
}
func TestIssue714(t *testing.T) {
var v interface{}
err := toml.Unmarshal([]byte("0."), &v)
require.Error(t, err)
err = toml.Unmarshal([]byte("0={0=0,"), &v)
require.Error(t, err)
}
func TestUnmarshalDecodeErrors(t *testing.T) { func TestUnmarshalDecodeErrors(t *testing.T) {
examples := []struct { examples := []struct {
desc string desc string
@@ -2361,10 +2205,18 @@ func TestUnmarshalDecodeErrors(t *testing.T) {
desc: "local time with fractional", desc: "local time with fractional",
data: `a = 11:22:33.x`, data: `a = 11:22:33.x`,
}, },
{
desc: "local time frac precision too large",
data: `a = 2021-05-09T11:22:33.99999999999`,
},
{ {
desc: "wrong time offset separator", desc: "wrong time offset separator",
data: `a = 1979-05-27T00:32:00.-07:00`, data: `a = 1979-05-27T00:32:00.-07:00`,
}, },
{
desc: "missing fractional with tz",
data: `a = 2021-05-09T11:22:33.99999999999`,
},
{ {
desc: "wrong time offset separator", desc: "wrong time offset separator",
data: `a = 1979-05-27T00:32:00Z07:00`, data: `a = 1979-05-27T00:32:00Z07:00`,
@@ -2374,25 +2226,9 @@ func TestUnmarshalDecodeErrors(t *testing.T) {
data: `flt8 = 224_617.445_991__228`, data: `flt8 = 224_617.445_991__228`,
}, },
{ {
desc: "float with double .", desc: "float with double _",
data: `flt8 = 1..2`, data: `flt8 = 1..2`,
}, },
{
desc: "number with plus sign and leading underscore",
data: `a = +_0`,
},
{
desc: "number with negative sign and leading underscore",
data: `a = -_0`,
},
{
desc: "exponent with plus sign and leading underscore",
data: `a = 0e+_0`,
},
{
desc: "exponent with negative sign and leading underscore",
data: `a = 0e-_0`,
},
{ {
desc: "int with wrong base", desc: "int with wrong base",
data: `a = 0f2`, data: `a = 0f2`,
@@ -2500,7 +2336,7 @@ world'`,
{ {
desc: "invalid seconds value", desc: "invalid seconds value",
data: `a=1979-05-27T12:45:99`, data: `a=1979-05-27T12:45:99`,
msg: `seconds cannot be greater 60`, msg: `seconds cannot be greater 59`,
}, },
{ {
desc: `binary with invalid digit`, desc: `binary with invalid digit`,
@@ -2704,70 +2540,14 @@ world'`,
desc: `invalid month`, desc: `invalid month`,
data: `a=2021-0--29`, data: `a=2021-0--29`,
}, },
{
desc: `zero is an invalid day`,
data: `a=2021-11-00`,
},
{
desc: `zero is an invalid month`,
data: `a=2021-00-11`,
},
{
desc: `invalid number of seconds digits with trailing digit`,
data: `a=0000-01-01 00:00:000000Z3`,
},
{
desc: `invalid zone offset hours`,
data: `a=0000-01-01 00:00:00+24:00`,
},
{
desc: `invalid zone offset minutes`,
data: `a=0000-01-01 00:00:00+00:60`,
},
{
desc: `invalid character in zone offset hours`,
data: `a=0000-01-01 00:00:00+0Z:00`,
},
{
desc: `invalid character in zone offset minutes`,
data: `a=0000-01-01 00:00:00+00:0Z`,
},
{
desc: `invalid number of seconds`,
data: `a=0000-01-01 00:00:00+27000`,
},
{ {
desc: `carriage return inside basic key`, desc: `carriage return inside basic key`,
data: "\"\r\"=42", data: "\"\r\"=42",
}, },
{
desc: `carriage return inside literal key`,
data: "'\r'=42",
},
{ {
desc: `carriage return inside basic string`, desc: `carriage return inside basic string`,
data: "A = \"\r\"", data: "A = \"\r\"",
}, },
{
desc: `carriage return inside basic multiline string`,
data: "a=\"\"\"\r\"\"\"",
},
{
desc: `carriage return at the trail of basic multiline string`,
data: "a=\"\"\"\r",
},
{
desc: `carriage return inside literal string`,
data: "A = '\r'",
},
{
desc: `carriage return inside multiline literal string`,
data: "a='''\r'''",
},
{
desc: `carriage return at trail of multiline literal string`,
data: "a='''\r",
},
{ {
desc: `carriage return in comment`, desc: `carriage return in comment`,
data: "# this is a test\ra=1", data: "# this is a test\ra=1",
@@ -2798,34 +2578,6 @@ world'`,
} }
} }
func TestUnmarshalTags(t *testing.T) {
type doc struct {
Dash string `toml:"-,"`
Ignore string `toml:"-"`
A string `toml:"hello"`
B string `toml:"comma,omitempty"`
}
data := `
'-' = "dash"
Ignore = 'me'
hello = 'content'
comma = 'ok'
`
d := doc{}
expected := doc{
Dash: "dash",
Ignore: "",
A: "content",
B: "ok",
}
err := toml.Unmarshal([]byte(data), &d)
require.NoError(t, err)
require.Equal(t, expected, d)
}
func TestASCIIControlCharacters(t *testing.T) { func TestASCIIControlCharacters(t *testing.T) {
invalidCharacters := []byte{0x7F} invalidCharacters := []byte{0x7F}
for c := byte(0x0); c <= 0x08; c++ { for c := byte(0x0); c <= 0x08; c++ {