Compare commits

..

8 Commits

Author SHA1 Message Date
Claude fa98989475 fix: resolve lint issues and improve test coverage for TOML v1.1.0
- Fix dupl lint: add nolint:dupl to parseInlineTable and parseValArray
  which have intentionally similar loop structures
- Fix gci lint: correct alignment in multiline basic string test entries
- Add unmarshaler tests for new TOML v1.1.0 features exercising public
  APIs: multiline inline tables with comments, trailing commas, leading
  commas, error cases (comma at start, missing separator, double comma,
  incomplete table), escape sequences, and type mismatch errors
- Add parser test for inline table comment handling with KeepComments
- Coverage increases from 97.37% to 97.44% vs v2 base

https://claude.ai/code/session_01RdiWykFQdmwkQ2nbLwJwwP
2026-03-29 17:37:29 +00:00
João Fernandes 189bf9820b test: parsing inline table 2026-03-03 14:03:57 +00:00
João Fernandes 8455b10edd test: regen toml-test tests targetting TOML v1.1.0 2026-02-11 11:49:40 +00:00
João Fernandes 6de639d0ae docs: update README 2026-02-11 11:15:44 +00:00
João Fernandes 517ceb4eb8 feat: allow newlines and trailing commas in inline tables
TOML v1.1.0 relaxes inline table syntax to allow newlines, comments,
and trailing commas, matching the existing behavior of arrays.
2026-02-11 11:15:33 +00:00
João Fernandes dd7970eb93 feat: make seconds optional in datetime and time values
TOML v1.1.0 allows times to be specified as HH:MM without the seconds
component (previously HH:MM:SS was required). This applies to local
times, local datetimes, and offset datetimes.
2026-02-11 11:15:16 +00:00
João Fernandes 3405e8a1d9 test: \e escape character 2026-02-11 11:14:59 +00:00
João Fernandes 5794be6251 feat: add \xHH escape sequence support in basic strings
TOML v1.1.0 introduces the \xHH escape notation for basic strings,
allowing two-digit hex escapes for Unicode code points U+0000 to
U+00FF.

We keep emitting \u00XX for backwards compatibility.
2026-02-11 11:14:23 +00:00
24 changed files with 1928 additions and 1236 deletions
-25
View File
@@ -1,25 +0,0 @@
name: capabilities
on:
push:
branches:
- v2
pull_request:
branches:
- v2
jobs:
check:
name: check capabilities
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@v6
with:
go-version: "1.26"
- name: Install capslock
run: go install github.com/google/capslock/cmd/capslock@latest
- name: Check for new capabilities
run: ./caps.sh check
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
dry-run: false dry-run: false
language: go language: go
- name: Upload Crash - name: Upload Crash
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v6
if: failure() && steps.build.outcome == 'success' if: failure() && steps.build.outcome == 'success'
with: with:
name: artifacts name: artifacts
+1 -1
View File
@@ -15,6 +15,6 @@ jobs:
- name: Setup go - name: Setup go
uses: actions/setup-go@v6 uses: actions/setup-go@v6
with: with:
go-version: "1.26" go-version: "1.25"
- name: Run tests with coverage - name: Run tests with coverage
run: ./ci.sh coverage -d "${GITHUB_BASE_REF-HEAD}" run: ./ci.sh coverage -d "${GITHUB_BASE_REF-HEAD}"
+2 -2
View File
@@ -20,7 +20,7 @@ jobs:
fetch-depth: 0 fetch-depth: 0
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4 uses: docker/setup-buildx-action@v3
- name: Run Go versions compatibility test - name: Run Go versions compatibility test
run: | run: |
@@ -28,7 +28,7 @@ jobs:
./test-go-versions.sh --output ./test-results $VERSIONS ./test-go-versions.sh --output ./test-results $VERSIONS
- name: Upload test results - name: Upload test results
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v6
with: with:
name: go-versions-test-results name: go-versions-test-results
path: | path: |
+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
- name: Setup go - name: Setup go
uses: actions/setup-go@v6 uses: actions/setup-go@v6
with: with:
go-version: "1.26" go-version: "1.24"
- name: Run golangci-lint - name: Run golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
+3 -3
View File
@@ -22,15 +22,15 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v6 uses: actions/setup-go@v6
with: with:
go-version: "1.26" go-version: "1.25"
- name: Login to GitHub Container Registry - name: Login to GitHub Container Registry
uses: docker/login-action@v4 uses: docker/login-action@v3
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Run GoReleaser - name: Run GoReleaser
uses: goreleaser/goreleaser-action@v7 uses: goreleaser/goreleaser-action@v6
with: with:
distribution: goreleaser distribution: goreleaser
version: '~> v2' version: '~> v2'
+1 -2
View File
@@ -10,10 +10,9 @@ on:
jobs: jobs:
build: build:
strategy: strategy:
fail-fast: false
matrix: matrix:
os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest', 'macos-14' ] os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest', 'macos-14' ]
go: [ '1.25', '1.26' ] go: [ '1.24', '1.25' ]
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
name: ${{ matrix.go }}/${{ matrix.os }} name: ${{ matrix.go }}/${{ matrix.os }}
steps: steps:
+3
View File
@@ -22,6 +22,7 @@ builds:
- linux_riscv64 - linux_riscv64
- windows_amd64 - windows_amd64
- windows_arm64 - windows_arm64
- windows_arm
- darwin_amd64 - darwin_amd64
- darwin_arm64 - darwin_arm64
- id: tomljson - id: tomljson
@@ -41,6 +42,7 @@ builds:
- linux_riscv64 - linux_riscv64
- windows_amd64 - windows_amd64
- windows_arm64 - windows_arm64
- windows_arm
- darwin_amd64 - darwin_amd64
- darwin_arm64 - darwin_arm64
- id: jsontoml - id: jsontoml
@@ -60,6 +62,7 @@ builds:
- linux_arm - linux_arm
- windows_amd64 - windows_amd64
- windows_arm64 - windows_arm64
- windows_arm
- darwin_amd64 - darwin_amd64
- darwin_arm64 - darwin_arm64
universal_binaries: universal_binaries:
+1 -10
View File
@@ -53,14 +53,6 @@ go-toml is a TOML library for Go. The goal is to provide an easy-to-use and effi
- Commit messages must explain **why** the change is needed - Commit messages must explain **why** the change is needed
- Keep messages clear and informative even if details are in the PR description - Keep messages clear and informative even if details are in the PR description
### Capabilities
go-toml tracks system-level capabilities using [capslock](https://github.com/google/capslock). The baseline is in `capability_baseline.txt` and CI enforces that it does not grow.
- **Do not introduce new capabilities.** PRs that increase the capability set (e.g., adding network access, subprocess execution, syscalls) are unlikely to be accepted.
- If a change causes the capabilities check to fail, do not update the baseline to make it pass. Instead, rethink the approach to avoid requiring new capabilities.
- To check locally: `./caps.sh check` (requires `capslock` installed via `go install github.com/google/capslock/cmd/capslock@latest`)
## Pull Request Checklist ## Pull Request Checklist
Before submitting: Before submitting:
@@ -69,5 +61,4 @@ Before submitting:
2. No backward-incompatible changes (unless discussed) 2. No backward-incompatible changes (unless discussed)
3. Relevant documentation added/updated 3. Relevant documentation added/updated
4. No performance regression (verify with benchmarks) 4. No performance regression (verify with benchmarks)
5. Capabilities are not increasing (`./caps.sh check`) 5. Title is clear and understandable for changelog
6. Title is clear and understandable for changelog
-19
View File
@@ -180,25 +180,6 @@ description. Pull requests that lower performance will receive more scrutiny.
[benchstat]: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat [benchstat]: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat
### Capabilities
We use [capslock](https://github.com/google/capslock) to track what
system-level capabilities (file access, network, syscalls, etc.) each package
requires. The current baseline is in `capability_baseline.txt`. CI will fail if
a change introduces a new capability.
**Pull requests that increase the set of capabilities are unlikely to be
accepted.** go-toml is a parsing library and should not need network access,
subprocess execution, or other capabilities beyond what it already uses.
If you believe a new capability is genuinely needed, discuss it in an issue
first. To update the baseline after approval:
```bash
go install github.com/google/capslock/cmd/capslock@latest
./caps.sh generate
```
### Style ### Style
Try to look around and follow the same format and structure as the rest of the Try to look around and follow the same format and structure as the rest of the
+2 -2
View File
@@ -2,7 +2,7 @@
Go library for the [TOML](https://toml.io/en/) format. Go library for the [TOML](https://toml.io/en/) format.
This library supports [TOML v1.0.0](https://toml.io/en/v1.0.0). This library supports [TOML v1.1.0](https://toml.io/en/v1.1.0).
[🐞 Bug Reports](https://github.com/pelletier/go-toml/issues) [🐞 Bug Reports](https://github.com/pelletier/go-toml/issues)
@@ -67,7 +67,7 @@ this use-case, go-toml provides [`LocalDate`][tld], [`LocalTime`][tlt], and
making them convenient yet unambiguous structures for their respective TOML making them convenient yet unambiguous structures for their respective TOML
representation. representation.
[ldt]: https://toml.io/en/v1.0.0#local-date-time [ldt]: https://toml.io/en/v1.1.0#local-date-time
[tld]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDate [tld]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDate
[tlt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalTime [tlt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalTime
[tldt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDateTime [tldt]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#LocalDateTime
-1
View File
@@ -1 +0,0 @@
github.com/pelletier/go-toml/v2: CAPABILITY_REFLECT, CAPABILITY_UNANALYZED, CAPABILITY_UNSAFE_POINTER
-101
View File
@@ -1,101 +0,0 @@
#!/usr/bin/env bash
#
# Generates or checks the capability baseline for go-toml.
#
# Usage:
# ./caps.sh generate # regenerate capability_baseline.txt
# ./caps.sh check # check that capabilities haven't grown
#
# Requires: go, capslock (go install github.com/google/capslock/cmd/capslock@latest)
set -euo pipefail
BASELINE="capability_baseline.txt"
CAPSLOCK="${CAPSLOCK:-capslock}"
# Capabilities that must never appear in any package.
FORBIDDEN_CAPS=(
CAPABILITY_NETWORK
CAPABILITY_CGO
CAPABILITY_EXEC
)
capslock_to_baseline() {
"$CAPSLOCK" -packages=. -output=package -granularity=package \
| jq -r 'to_entries | sort_by(.key) | .[] | .key + ": " + (.value | sort | join(", "))'
}
generate() {
capslock_to_baseline > "$BASELINE"
echo "Wrote $BASELINE"
}
check() {
if [ ! -f "$BASELINE" ]; then
echo "ERROR: $BASELINE not found. Run '$0 generate' first."
exit 1
fi
current=$(mktemp)
trap 'rm -f "$current"' EXIT
capslock_to_baseline > "$current"
failed=0
# Check for forbidden capabilities in current output.
for cap in "${FORBIDDEN_CAPS[@]}"; do
if grep -q "$cap" "$current"; then
echo "FORBIDDEN capability found: $cap"
grep "$cap" "$current"
failed=1
fi
done
# Extract all unique capability names from baseline and current.
baseline_caps=$(grep -oE 'CAPABILITY_[A-Z_]+' "$BASELINE" | sort -u)
current_caps=$(grep -oE 'CAPABILITY_[A-Z_]+' "$current" | sort -u)
# Check for new capability names not in the baseline.
new_caps=$(comm -13 <(echo "$baseline_caps") <(echo "$current_caps"))
if [ -n "$new_caps" ]; then
echo "NEW capabilities detected (not in baseline):"
echo "$new_caps"
failed=1
fi
# Check for new per-package capabilities (a package gained a capability it didn't have before).
while IFS=': ' read -r pkg caps; do
baseline_pkg_caps=$(grep "^${pkg}:" "$BASELINE" 2>/dev/null | sed 's/^[^:]*: //' || true)
if [ -z "$baseline_pkg_caps" ]; then
echo "NEW package with capabilities: $pkg: $caps"
failed=1
continue
fi
# Check each capability in current for this package
for cap in $(echo "$caps" | tr ', ' '\n' | grep -v '^$'); do
if ! echo "$baseline_pkg_caps" | grep -q "$cap"; then
echo "NEW capability for $pkg: $cap"
failed=1
fi
done
done < "$current"
if [ "$failed" -eq 1 ]; then
echo ""
echo "FAILED: capabilities have grown."
echo "If this is intentional, run '$0 generate' and commit the updated $BASELINE."
exit 1
fi
echo "OK: no new capabilities detected."
}
case "${1:-}" in
generate) generate ;;
check) check ;;
*)
echo "Usage: $0 {generate|check}"
exit 1
;;
esac
+15 -10
View File
@@ -162,7 +162,7 @@ func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) {
const localDateTimeByteMinLen = 11 const localDateTimeByteMinLen = 11
if len(b) < localDateTimeByteMinLen { if len(b) < localDateTimeByteMinLen {
return dt, nil, unstable.NewParserError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM:SS[.NNNNNNNNN]") return dt, nil, unstable.NewParserError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM[:SS[.NNNNNNNNN]]")
} }
date, err := parseLocalDate(b[:10]) date, err := parseLocalDate(b[:10])
@@ -194,10 +194,10 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
t LocalTime t LocalTime
) )
// check if b matches to have expected format HH:MM:SS[.NNNNNN] // check if b matches to have expected format HH:MM[:SS[.NNNNNN]]
const localTimeByteLen = 8 const localTimeByteMinLen = 5
if len(b) < localTimeByteLen { if len(b) < localTimeByteMinLen {
return t, nil, unstable.NewParserError(b, "times are expected to have the format HH:MM:SS[.NNNNNN]") return t, nil, unstable.NewParserError(b, "times are expected to have the format HH:MM[:SS[.NNNNNN]]")
} }
var err error var err error
@@ -221,20 +221,25 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
if t.Minute > 59 { if t.Minute > 59 {
return t, nil, unstable.NewParserError(b[3:5], "minutes cannot be greater 59") return t, nil, unstable.NewParserError(b[3:5], "minutes cannot be greater 59")
} }
if b[5] != ':' {
return t, nil, unstable.NewParserError(b[5:6], "expecting colon between minutes and seconds") b = b[5:]
if len(b) >= 1 && b[0] == ':' {
if len(b) < 3 {
return t, nil, unstable.NewParserError(b, "incomplete seconds")
} }
t.Second, err = parseDecimalDigits(b[6:8]) t.Second, err = parseDecimalDigits(b[1:3])
if err != nil { if err != nil {
return t, nil, err return t, nil, err
} }
if t.Second > 59 { if t.Second > 59 {
return t, nil, unstable.NewParserError(b[6:8], "seconds cannot be greater than 59") return t, nil, unstable.NewParserError(b[1:3], "seconds cannot be greater than 59")
} }
b = b[8:] b = b[3:]
}
if len(b) >= 1 && b[0] == '.' { if len(b) >= 1 && b[0] == '.' {
frac := 0 frac := 0
-6
View File
@@ -259,12 +259,6 @@ func TestDecodeError_Position(t *testing.T) {
expectedRow: 3, expectedRow: 3,
minCol: 5, minCol: 5,
}, },
{
name: "missing equals on last line without trailing newline",
doc: "a = 1\nb = 2\nc",
expectedRow: 3,
minCol: 1,
},
} }
for _, e := range examples { for _, e := range examples {
+7
View File
@@ -67,6 +67,13 @@ func TestLocalTime_UnmarshalMarshalText(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestLocalTime_UnmarshalText_WithoutSeconds(t *testing.T) {
d := toml.LocalTime{}
err := d.UnmarshalText([]byte("14:15"))
assert.NoError(t, err)
assert.Equal(t, toml.LocalTime{14, 15, 0, 0, 0}, d)
}
func TestLocalTime_RoundTrip(t *testing.T) { func TestLocalTime_RoundTrip(t *testing.T) {
var d struct{ A toml.LocalTime } var d struct{ A toml.LocalTime }
err := toml.Unmarshal([]byte("a=20:12:01.500"), &d) err := toml.Unmarshal([]byte("a=20:12:01.500"), &d)
+4 -5
View File
@@ -9,7 +9,7 @@ YELLOW='\033[1;33m'
BLUE='\033[0;34m' BLUE='\033[0;34m'
NC='\033[0m' # No Color NC='\033[0m' # No Color
# Go versions to test (1.11 through 1.26) # Go versions to test (1.11 through 1.25)
GO_VERSIONS=( GO_VERSIONS=(
"1.11" "1.11"
"1.12" "1.12"
@@ -26,7 +26,6 @@ GO_VERSIONS=(
"1.23" "1.23"
"1.24" "1.24"
"1.25" "1.25"
"1.26"
) )
# Default values # Default values
@@ -65,7 +64,7 @@ EXAMPLES:
$0 # Test all Go versions in parallel $0 # Test all Go versions in parallel
$0 --sequential # Test all Go versions sequentially $0 --sequential # Test all Go versions sequentially
$0 1.21 1.22 1.23 # Test specific versions $0 1.21 1.22 1.23 # Test specific versions
$0 --verbose --output ./results 1.25 1.26 # Verbose output to custom directory $0 --verbose --output ./results 1.24 1.25 # Verbose output to custom directory
EXIT CODES: EXIT CODES:
0 Recent Go versions pass (good compatibility) 0 Recent Go versions pass (good compatibility)
@@ -137,8 +136,8 @@ fi
# Validate Go versions # Validate Go versions
for version in "${GO_VERSIONS[@]}"; do for version in "${GO_VERSIONS[@]}"; do
if ! [[ "$version" =~ ^1\.(1[1-9]|2[0-6])$ ]]; then if ! [[ "$version" =~ ^1\.(1[1-9]|2[0-5])$ ]]; then
log_error "Invalid Go version: $version. Supported versions: 1.11-1.26" log_error "Invalid Go version: $version. Supported versions: 1.11-1.25"
exit 1 exit 1
fi fi
done done
+2 -2
View File
@@ -1,5 +1,5 @@
//go:generate go run github.com/toml-lang/toml-test/cmd/toml-test@v1.6.0 -copy ./tests //go:generate go run github.com/toml-lang/toml-test/v2/cmd/toml-test@v2.1.0 copy -toml 1.1 ./tests
//go:generate go run ./cmd/tomltestgen/main.go -r v1.6.0 -o toml_testgen_test.go //go:generate go run ./cmd/tomltestgen/main.go -r v2.1.0 -o toml_testgen_test.go
package toml_test package toml_test
+1151 -532
View File
File diff suppressed because it is too large Load Diff
+8 -78
View File
@@ -56,18 +56,13 @@ func (d *Decoder) DisallowUnknownFields() *Decoder {
// EnableUnmarshalerInterface allows to enable unmarshaler interface. // EnableUnmarshalerInterface allows to enable unmarshaler interface.
// //
// With this feature enabled, types implementing the unstable.Unmarshaler // With this feature enabled, types implementing the unstable/Unmarshaler
// interface can be decoded from any structure of the document. It allows types // interface can be decoded from any structure of the document. It allows types
// that don't have a straightforward TOML representation to provide their own // that don't have a straightforward TOML representation to provide their own
// decoding logic. // decoding logic.
// //
// The UnmarshalTOML method receives raw TOML bytes: // Currently, types can only decode from a single value. Tables and array tables
// - For single values: the raw value bytes (e.g., `"hello"` for a string) // are not supported.
// - For tables: all key-value lines belonging to that table
// - For inline tables/arrays: the raw bytes of the inline structure
//
// The unstable.RawMessage type can be used to capture raw TOML bytes for
// later processing, similar to json.RawMessage.
// //
// *Unstable:* This method does not follow the compatibility guarantees of // *Unstable:* This method does not follow the compatibility guarantees of
// semver. It can be changed or removed without a new major version being // semver. It can be changed or removed without a new major version being
@@ -604,8 +599,9 @@ func (d *decoder) handleArrayTablePart(key unstable.Iterator, v reflect.Value) (
// cannot handle it. // cannot handle it.
func (d *decoder) handleTable(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { func (d *decoder) handleTable(key unstable.Iterator, v reflect.Value) (reflect.Value, error) {
if v.Kind() == reflect.Slice { if v.Kind() == reflect.Slice {
// For non-empty slices, work with the last element if v.Len() == 0 {
if v.Len() > 0 { return reflect.Value{}, unstable.NewParserError(key.Node().Data, "cannot store a table in a slice")
}
elem := v.Index(v.Len() - 1) elem := v.Index(v.Len() - 1)
x, err := d.handleTable(key, elem) x, err := d.handleTable(key, elem)
if err != nil { if err != nil {
@@ -616,17 +612,6 @@ func (d *decoder) handleTable(key unstable.Iterator, v reflect.Value) (reflect.V
} }
return reflect.Value{}, nil return reflect.Value{}, nil
} }
// Empty slice - check if it implements Unmarshaler (e.g., RawMessage)
// and we're at the end of the key path
if d.unmarshalerInterface && !key.Next() {
if v.CanAddr() && v.Addr().CanInterface() {
if outi, ok := v.Addr().Interface().(unstable.Unmarshaler); ok {
return d.handleKeyValuesUnmarshaler(outi)
}
}
}
return reflect.Value{}, unstable.NewParserError(key.Node().Data, "cannot store a table in a slice")
}
if key.Next() { if key.Next() {
// Still scoping the key // Still scoping the key
return d.handleTablePart(key, v) return d.handleTablePart(key, v)
@@ -639,24 +624,6 @@ func (d *decoder) handleTable(key unstable.Iterator, v reflect.Value) (reflect.V
// Handle root expressions until the end of the document or the next // Handle root expressions until the end of the document or the next
// non-key-value. // non-key-value.
func (d *decoder) handleKeyValues(v reflect.Value) (reflect.Value, error) { func (d *decoder) handleKeyValues(v reflect.Value) (reflect.Value, error) {
// Check if target implements Unmarshaler before processing key-values.
// This allows types to handle entire tables themselves.
if d.unmarshalerInterface {
vv := v
for vv.Kind() == reflect.Ptr {
if vv.IsNil() {
vv.Set(reflect.New(vv.Type().Elem()))
}
vv = vv.Elem()
}
if vv.CanAddr() && vv.Addr().CanInterface() {
if outi, ok := vv.Addr().Interface().(unstable.Unmarshaler); ok {
// Collect all key-value expressions for this table
return d.handleKeyValuesUnmarshaler(outi)
}
}
}
var rv reflect.Value var rv reflect.Value
for d.nextExpr() { for d.nextExpr() {
expr := d.expr() expr := d.expr()
@@ -686,41 +653,6 @@ func (d *decoder) handleKeyValues(v reflect.Value) (reflect.Value, error) {
return rv, nil return rv, nil
} }
// handleKeyValuesUnmarshaler collects all key-value expressions for a table
// and passes them to the Unmarshaler as raw TOML bytes.
func (d *decoder) handleKeyValuesUnmarshaler(u unstable.Unmarshaler) (reflect.Value, error) {
// Collect raw bytes from all key-value expressions for this table.
// We use the Raw field on each KeyValue expression to preserve the
// original formatting (whitespace, quoting style, etc.) from the document.
var buf []byte
for d.nextExpr() {
expr := d.expr()
if expr.Kind != unstable.KeyValue {
d.stashExpr()
break
}
_, err := d.seen.CheckExpression(expr)
if err != nil {
return reflect.Value{}, err
}
// Use the raw bytes from the original document to preserve formatting
if expr.Raw.Length > 0 {
raw := d.p.Raw(expr.Raw)
buf = append(buf, raw...)
}
buf = append(buf, '\n')
}
if err := u.UnmarshalTOML(buf); err != nil {
return reflect.Value{}, err
}
return reflect.Value{}, nil
}
type ( type (
handlerFn func(key unstable.Iterator, v reflect.Value) (reflect.Value, error) handlerFn func(key unstable.Iterator, v reflect.Value) (reflect.Value, error)
valueMakerFn func() reflect.Value valueMakerFn func() reflect.Value
@@ -765,8 +697,7 @@ func (d *decoder) handleValue(value *unstable.Node, v reflect.Value) error {
if d.unmarshalerInterface { if d.unmarshalerInterface {
if v.CanAddr() && v.Addr().CanInterface() { if v.CanAddr() && v.Addr().CanInterface() {
if outi, ok := v.Addr().Interface().(unstable.Unmarshaler); ok { if outi, ok := v.Addr().Interface().(unstable.Unmarshaler); ok {
// Pass raw bytes from the original document return outi.UnmarshalTOML(value)
return outi.UnmarshalTOML(d.p.Raw(value.Raw))
} }
} }
} }
@@ -1270,8 +1201,7 @@ func (d *decoder) handleKeyValuePart(key unstable.Iterator, value *unstable.Node
if d.unmarshalerInterface { if d.unmarshalerInterface {
if v.CanAddr() && v.Addr().CanInterface() { if v.CanAddr() && v.Addr().CanInterface() {
if outi, ok := v.Addr().Interface().(unstable.Unmarshaler); ok { if outi, ok := v.Addr().Interface().(unstable.Unmarshaler); ok {
// Pass raw bytes from the original document return reflect.Value{}, outi.UnmarshalTOML(value)
return reflect.Value{}, outi.UnmarshalTOML(d.p.Raw(value.Raw))
} }
} }
} }
+445 -356
View File
@@ -96,132 +96,6 @@ func ExampleUnmarshal() {
// tags: [go toml] // tags: [go toml]
} }
// pluginConfig demonstrates how to implement dynamic unmarshaling
// based on a "type" field. This pattern is useful for plugin systems
// or polymorphic configuration.
type pluginConfig struct {
Type string
Config any
}
func (p *pluginConfig) UnmarshalTOML(data []byte) error {
// First, decode just the type field
var typeOnly struct {
Type string `toml:"type"`
}
if err := toml.Unmarshal(data, &typeOnly); err != nil {
return err
}
p.Type = typeOnly.Type
// Now decode the config based on the type
switch typeOnly.Type {
case "database":
var cfg struct {
Type string `toml:"type"`
Host string `toml:"host"`
Port int `toml:"port"`
}
if err := toml.Unmarshal(data, &cfg); err != nil {
return err
}
p.Config = map[string]any{"host": cfg.Host, "port": cfg.Port}
case "cache":
var cfg struct {
Type string `toml:"type"`
TTL int `toml:"ttl"`
}
if err := toml.Unmarshal(data, &cfg); err != nil {
return err
}
p.Config = map[string]any{"ttl": cfg.TTL}
}
return nil
}
// This example demonstrates dynamic unmarshaling based on a discriminator
// field. The pluginConfig type uses UnmarshalTOML to first read the "type"
// field, then decode the rest of the configuration based on that type.
// This pattern is useful for plugin systems or configuration that varies
// by type.
func ExampleDecoder_EnableUnmarshalerInterface_dynamicConfig() {
doc := `
[[plugins]]
type = "database"
host = "localhost"
port = 5432
[[plugins]]
type = "cache"
ttl = 300
`
type Config struct {
Plugins []pluginConfig `toml:"plugins"`
}
var cfg Config
err := toml.NewDecoder(strings.NewReader(doc)).
EnableUnmarshalerInterface().
Decode(&cfg)
if err != nil {
panic(err)
}
for _, p := range cfg.Plugins {
fmt.Printf("type=%s config=%v\n", p.Type, p.Config)
}
// Output:
// type=database config=map[host:localhost port:5432]
// type=cache config=map[ttl:300]
}
// This example demonstrates using RawMessage to capture raw TOML bytes
// for later processing. RawMessage is similar to json.RawMessage - it
// delays decoding so you can inspect the raw content or decode it
// differently based on context.
func ExampleDecoder_EnableUnmarshalerInterface_rawMessage() {
doc := `
[plugin]
name = "example"
version = "1.0"
enabled = true
`
type Config struct {
Plugin unstable.RawMessage `toml:"plugin"`
}
var cfg Config
err := toml.NewDecoder(strings.NewReader(doc)).
EnableUnmarshalerInterface().
Decode(&cfg)
if err != nil {
panic(err)
}
// cfg.Plugin contains the raw TOML bytes
fmt.Printf("Raw TOML captured:\n%s", cfg.Plugin)
// You can later decode it into a specific type
var plugin struct {
Name string `toml:"name"`
Version string `toml:"version"`
Enabled bool `toml:"enabled"`
}
if err := toml.Unmarshal(cfg.Plugin, &plugin); err != nil {
panic(err)
}
fmt.Printf("Decoded: name=%s version=%s enabled=%v\n",
plugin.Name, plugin.Version, plugin.Enabled)
// Output:
// Raw TOML captured:
// name = "example"
// version = "1.0"
// enabled = true
// Decoded: name=example version=1.0 enabled=true
}
type badReader struct{} type badReader struct{}
func (r *badReader) Read([]byte) (int, error) { func (r *badReader) Read([]byte) (int, error) {
@@ -726,6 +600,96 @@ foo = "bar"`,
} }
}, },
}, },
{
desc: "local-time without seconds",
input: `a = 14:15`,
gen: func() test {
var v map[string]interface{}
return test{
target: &v,
expected: &map[string]interface{}{
"a": toml.LocalTime{Hour: 14, Minute: 15},
},
}
},
},
{
desc: "local-datetime without seconds using T",
input: `a = 2010-02-03T14:15`,
gen: func() test {
var v map[string]interface{}
return test{
target: &v,
expected: &map[string]interface{}{
"a": toml.LocalDateTime{
LocalDate: toml.LocalDate{2010, 2, 3},
LocalTime: toml.LocalTime{Hour: 14, Minute: 15},
},
},
}
},
},
{
desc: "local-datetime without seconds using space",
input: `a = 2010-02-03 14:15`,
gen: func() test {
var v map[string]interface{}
return test{
target: &v,
expected: &map[string]interface{}{
"a": toml.LocalDateTime{
LocalDate: toml.LocalDate{2010, 2, 3},
LocalTime: toml.LocalTime{Hour: 14, Minute: 15},
},
},
}
},
},
{
desc: "datetime without seconds with Z",
input: `a = 2010-02-03T14:15Z`,
gen: func() test {
var v map[string]time.Time
return test{
target: &v,
expected: &map[string]time.Time{
"a": time.Date(2010, 2, 3, 14, 15, 0, 0, time.UTC),
},
}
},
},
{
desc: "datetime without seconds with offset",
input: `a = 2010-02-03T14:15+05:00`,
gen: func() test {
var v map[string]time.Time
return test{
target: &v,
expected: &map[string]time.Time{
"a": time.Date(2010, 2, 3, 14, 15, 0, 0, time.FixedZone("", 5*3600)),
},
}
},
},
{
desc: "local-time with seconds and fractional regression",
input: `a = 14:15:30.123`,
gen: func() test {
var v map[string]interface{}
return test{
target: &v,
expected: &map[string]interface{}{
"a": toml.LocalTime{Hour: 14, Minute: 15, Second: 30, Nanosecond: 123000000, Precision: 3},
},
}
},
},
{ {
desc: "local-time missing digit", desc: "local-time missing digit",
input: `a = 12:08:0`, input: `a = 12:08:0`,
@@ -885,6 +849,104 @@ huey = 'dewey'
} }
}, },
}, },
{
desc: "basic string escape character",
input: `A = "\e"`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "\x1B"},
}
},
},
{
desc: "multiline basic string escape character",
input: `A = """\e"""`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "\x1B"},
}
},
},
{
desc: "escape character combined with bracket",
input: `A = "\e["`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "\x1B["},
}
},
},
{
desc: "basic string hex escape lowercase letter",
input: `A = "\x61"`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "a"},
}
},
},
{
desc: "basic string hex escape null byte",
input: `A = "\x00"`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "\x00"},
}
},
},
{
desc: "basic string hex escape max value",
input: `A = "\xFF"`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "\u00FF"},
}
},
},
{
desc: "multiline basic string hex escape",
input: `A = """\x61"""`,
gen: func() test {
type doc struct {
A string
}
return test{
target: &doc{},
expected: &doc{A: "a"},
}
},
},
{ {
desc: "spaces around dotted keys", desc: "spaces around dotted keys",
input: "a . b = 1", input: "a . b = 1",
@@ -1032,6 +1094,87 @@ B = "data"`,
} }
}, },
}, },
{
desc: "multiline inline table",
input: "Name = {\n First = \"hello\",\n Last = \"world\"\n}",
gen: func() test {
type name struct {
First string
Last string
}
type doc struct {
Name name
}
return test{
target: &doc{},
expected: &doc{Name: name{
First: "hello",
Last: "world",
}},
}
},
},
{
desc: "inline table with trailing comma",
input: `Name = {First = "hello", Last = "world",}`,
gen: func() test {
type name struct {
First string
Last string
}
type doc struct {
Name name
}
return test{
target: &doc{},
expected: &doc{Name: name{
First: "hello",
Last: "world",
}},
}
},
},
{
desc: "multiline inline table with trailing comma and comments",
input: "Name = {\n # first name\n First = \"hello\",\n # last name\n Last = \"world\",\n}",
gen: func() test {
type name struct {
First string
Last string
}
type doc struct {
Name name
}
return test{
target: &doc{},
expected: &doc{Name: name{
First: "hello",
Last: "world",
}},
}
},
},
{
desc: "nested multiline inline tables",
input: "A = {\n B = {\n C = 1,\n },\n}",
gen: func() test {
var v map[string]interface{}
return test{
target: &v,
expected: &map[string]interface{}{
"A": map[string]interface{}{
"B": map[string]interface{}{
"C": int64(1),
},
},
},
}
},
},
{ {
desc: "inline table inside array", desc: "inline table inside array",
input: `Names = [{First = "hello", Last = "world"}, {First = "ab", Last = "cd"}]`, input: `Names = [{First = "hello", Last = "world"}, {First = "ab", Last = "cd"}]`,
@@ -3271,7 +3414,7 @@ world'`,
{ {
desc: "bad char between minutes and seconds", desc: "bad char between minutes and seconds",
data: `a = 2021-03-30 21:312:0`, data: `a = 2021-03-30 21:312:0`,
msg: `expecting colon between minutes and seconds`, msg: `extra characters at the end of a local date time`,
}, },
{ {
desc: "invalid hour value", desc: "invalid hour value",
@@ -3386,6 +3529,18 @@ world'`,
desc: `invalid escape char basic multiline string`, desc: `invalid escape char basic multiline string`,
data: `A = """\z"""`, data: `A = """\z"""`,
}, },
{
desc: `invalid hex escape non-hex character in basic string`,
data: `A = "\xGG"`,
},
{
desc: `incomplete hex escape in basic string`,
data: `A = "\x6"`,
},
{
desc: `invalid hex escape non-hex character in multiline basic string`,
data: `A = """\xGG"""`,
},
{ {
desc: `invalid inf`, desc: `invalid inf`,
data: `A = ick`, data: `A = ick`,
@@ -3572,6 +3727,30 @@ world'`,
desc: `backspace in comment`, desc: `backspace in comment`,
data: "# this is a test\ba=1", data: "# this is a test\ba=1",
}, },
{
desc: `inline table comma at start`,
data: `a = { , b = 1 }`,
},
{
desc: `inline table missing separator`,
data: `a = { b = 1 c = 2 }`,
},
{
desc: `inline table double comma across newline`,
data: "a = { b = 1,\n, c = 2 }",
},
{
desc: `incomplete inline table`,
data: "a = { b = 1,\n",
},
{
desc: `incomplete hex escape in multiline basic string`,
data: `A = """\x6"""`,
},
{
desc: `invalid escape char in basic string`,
data: `A = "\z"`,
},
} }
for _, e := range examples { for _, e := range examples {
@@ -4026,8 +4205,8 @@ type CustomUnmarshalerKey struct {
A int64 A int64
} }
func (k *CustomUnmarshalerKey) UnmarshalTOML(data []byte) error { func (k *CustomUnmarshalerKey) UnmarshalTOML(value *unstable.Node) error {
item, err := strconv.ParseInt(string(data), 10, 64) item, err := strconv.ParseInt(string(value.Data), 10, 64)
if err != nil { if err != nil {
return fmt.Errorf("error converting to int64, %w", err) return fmt.Errorf("error converting to int64, %w", err)
} }
@@ -4115,7 +4294,7 @@ foo = "bar"`,
type doc994 struct{} type doc994 struct{}
func (d *doc994) UnmarshalTOML([]byte) error { func (d *doc994) UnmarshalTOML(*unstable.Node) error {
return errors.New("expected-error") return errors.New("expected-error")
} }
@@ -4138,8 +4317,8 @@ type doc994ok struct {
S string S string
} }
func (d *doc994ok) UnmarshalTOML(data []byte) error { func (d *doc994ok) UnmarshalTOML(value *unstable.Node) error {
d.S = string(data) + " from unmarshaler" d.S = string(value.Data) + " from unmarshaler"
return nil return nil
} }
@@ -4152,8 +4331,7 @@ func TestIssue994_OK(t *testing.T) {
Decode(&d) Decode(&d)
assert.NoError(t, err) assert.NoError(t, err)
// With bytes-based interface, raw TOML bytes are passed including quotes assert.Equal(t, "bar from unmarshaler", d.S)
assert.Equal(t, "\"bar\" from unmarshaler", d.S)
} }
func TestIssue995(t *testing.T) { func TestIssue995(t *testing.T) {
@@ -4513,264 +4691,175 @@ func TestIssue1028(t *testing.T) {
}) })
} }
// Tests for issue #873 - Bring back toml.Unmarshaler for tables and arrays // customFieldUnmarshaler implements unstable.Unmarshaler and captures all
// key-value pairs directed to it, including unknown fields.
type customTable873 struct { type customFieldUnmarshaler struct {
Keys []string
Values map[string]string Values map[string]string
} }
func (c *customTable873) UnmarshalTOML(data []byte) error { func (c *customFieldUnmarshaler) UnmarshalTOML(value *unstable.Node) error {
c.Keys = []string{} c.Values = map[string]string{
c.Values = make(map[string]string) "kind": value.Kind.String(),
"data": string(value.Data),
// Parse the raw TOML bytes into a map to extract keys in order
// For this test, we use a simple line-by-line parser to preserve order
lines := bytes.Split(data, []byte{'\n'})
for _, line := range lines {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
} }
// Skip table headers
if line[0] == '[' {
continue
}
// Parse key = value
eqIdx := bytes.Index(line, []byte{'='})
if eqIdx < 0 {
continue
}
key := string(bytes.TrimSpace(line[:eqIdx]))
// Remove quotes from quoted keys
if len(key) >= 2 && key[0] == '"' && key[len(key)-1] == '"' {
key = key[1 : len(key)-1]
}
valueBytes := bytes.TrimSpace(line[eqIdx+1:])
// Remove quotes from string values
if len(valueBytes) >= 2 && valueBytes[0] == '"' && valueBytes[len(valueBytes)-1] == '"' {
valueBytes = valueBytes[1 : len(valueBytes)-1]
}
c.Keys = append(c.Keys, key)
c.Values[key] = string(valueBytes)
}
return nil return nil
} }
// Test for split tables - when the same parent table is defined in multiple places func TestUnmarshalerInterface_StructFieldFallback(t *testing.T) {
// This is a key requirement for issue #873: if type A implements Unmarshaler, // When EnableUnmarshalerInterface is active and a struct field is not found,
// and [a.b] and [a.d] are defined with another table [x] in between, // the decoder should fall back to the Unmarshaler interface on the struct.
// A should receive content for both b and d, but not x.
func TestIssue873_SplitTables(t *testing.T) {
// For this test, we expect each sub-table to be handled separately
// The parent doesn't receive the sub-tables directly - each sub-table
// (b and d) gets its own call to handleKeyValues
type Config struct { type Config struct {
A struct { Name string `toml:"name"`
B customTable873 `toml:"b"`
D customTable873 `toml:"d"`
} `toml:"a"`
X customTable873 `toml:"x"`
} }
doc := ` t.Run("unknown field with unmarshaler", func(t *testing.T) {
[a.b] doc := `name = "hello"
C = "1" unknown = "world"`
[x]
Y = "100"
[a.d]
E = "2"
`
var cfg Config var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))). decoder := toml.NewDecoder(bytes.NewReader([]byte(doc)))
EnableUnmarshalerInterface(). decoder.EnableUnmarshalerInterface()
Decode(&cfg) err := decoder.Decode(&cfg)
assert.NoError(t, err) assert.NoError(t, err)
// Each sub-table should have received its own key-values assert.Equal(t, "hello", cfg.Name)
assert.Equal(t, []string{"C"}, cfg.A.B.Keys) })
assert.Equal(t, "1", cfg.A.B.Values["C"])
assert.Equal(t, []string{"E"}, cfg.A.D.Keys)
assert.Equal(t, "2", cfg.A.D.Values["E"])
assert.Equal(t, []string{"Y"}, cfg.X.Keys)
assert.Equal(t, "100", cfg.X.Values["Y"])
} }
// Test using RawMessage to capture raw TOML bytes func TestUnmarshalerInterface_Value(t *testing.T) {
func TestIssue873_RawMessage(t *testing.T) { // Test that EnableUnmarshalerInterface delegates value decoding
// to the UnmarshalTOML method.
type Config struct { type Config struct {
Plugin unstable.RawMessage `toml:"plugin"` Field customFieldUnmarshaler `toml:"field"`
} }
doc := ` doc := `field = "test-value"`
[plugin]
name = "example"
version = "1.0"
`
var cfg Config var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))). decoder := toml.NewDecoder(bytes.NewReader([]byte(doc)))
EnableUnmarshalerInterface(). decoder.EnableUnmarshalerInterface()
Decode(&cfg) err := decoder.Decode(&cfg)
assert.NoError(t, err) assert.NoError(t, err)
// RawMessage should contain the raw key-value bytes assert.Equal(t, "test-value", cfg.Field.Values["data"])
expected := "name = \"example\"\nversion = \"1.0\"\n"
assert.Equal(t, expected, string(cfg.Plugin))
} }
// Test keys that need quoting (contain special characters) func TestTypeMismatchString_StructFieldContext(t *testing.T) {
func TestIssue873_QuotedKeys(t *testing.T) { // Exercise the typeMismatchString code path that includes struct field info
// in the error message.
type Inner struct {
Value int `toml:"value"`
}
type Config struct { type Config struct {
Section customTable873 `toml:"section"` Inner Inner `toml:"inner"`
} }
doc := ` doc := `inner = "not-a-table"`
[section]
"key with spaces" = "value1"
"key.with.dots" = "value2"
`
var cfg Config var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))). err := toml.Unmarshal([]byte(doc), &cfg)
EnableUnmarshalerInterface().
Decode(&cfg)
assert.NoError(t, err)
assert.Equal(t, 2, len(cfg.Section.Keys))
assert.Equal(t, "value1", cfg.Section.Values["key with spaces"])
assert.Equal(t, "value2", cfg.Section.Values["key.with.dots"])
}
// errorUnmarshaler873 is used to test error propagation from UnmarshalTOML
type errorUnmarshaler873 struct{}
func (e *errorUnmarshaler873) UnmarshalTOML([]byte) error {
return errors.New("intentional error")
}
// Test error propagation from UnmarshalTOML
func TestIssue873_UnmarshalerError(t *testing.T) {
doc := `
[section]
key = "value"
`
type Config struct {
Section errorUnmarshaler873 `toml:"section"`
}
var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))).
EnableUnmarshalerInterface().
Decode(&cfg)
assert.Error(t, err) assert.Error(t, err)
assert.True(t, strings.Contains(err.Error(), "intentional error"))
} }
// Test dotted keys in a table (e.g., a.b = value) func TestUnmarshalInlineTable_IncompatibleType(t *testing.T) {
func TestIssue873_DottedKeys(t *testing.T) { // Exercise the default branch of unmarshalInlineTable when the target
type Config struct { // is not a map, struct, or interface.
Section customTable873 `toml:"section"` type doc struct {
A int `toml:"a"`
}
var v doc
err := toml.Unmarshal([]byte(`a = {b = 1}`), &v)
assert.Error(t, err)
} }
doc := ` func TestTypeMismatchString_NoStructContext(t *testing.T) {
[section] // Exercise the typeMismatchString code path without struct field context (line 186).
sub.key = "value1" // Decoding a string into a bare int triggers this path.
another.nested.key = "value2" var v map[string]int
` err := toml.Unmarshal([]byte(`a = "hello"`), &v)
assert.Error(t, err)
var cfg Config }
err := toml.NewDecoder(bytes.NewReader([]byte(doc))).
EnableUnmarshalerInterface().
Decode(&cfg)
func TestMultilineInlineTable_EmptyWithNewlines(t *testing.T) {
doc := "a = {\n\n}"
var v map[string]interface{}
err := toml.Unmarshal([]byte(doc), &v)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 2, len(cfg.Section.Keys)) inner := v["a"]
// The dotted keys should be preserved in the raw output if inner == nil {
assert.Equal(t, "value1", cfg.Section.Values["sub.key"]) t.Fatal("expected key 'a' to be present")
assert.Equal(t, "value2", cfg.Section.Values["another.nested.key"]) }
m, ok := inner.(map[string]interface{})
if !ok {
t.Fatalf("expected map, got %T", inner)
}
if len(m) != 0 {
t.Fatalf("expected empty map, got %v", m)
}
} }
// Test pointer to pointer to Unmarshaler (covers pointer dereferencing loop) func TestMultilineInlineTable_CommentsOnly(t *testing.T) {
func TestIssue873_DoublePointerUnmarshaler(t *testing.T) { doc := "a = {\n # just a comment\n}"
type Config struct { var v map[string]interface{}
Section **customTable873 `toml:"section"` err := toml.Unmarshal([]byte(doc), &v)
}
doc := `
[section]
key = "value"
`
var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))).
EnableUnmarshalerInterface().
Decode(&cfg)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, cfg.Section != nil) inner := v["a"]
assert.True(t, *cfg.Section != nil) if inner == nil {
assert.Equal(t, []string{"key"}, (*cfg.Section).Keys) t.Fatal("expected key 'a' to be present")
assert.Equal(t, "value", (*cfg.Section).Values["key"]) }
m, ok := inner.(map[string]interface{})
if !ok {
t.Fatalf("expected map, got %T", inner)
}
if len(m) != 0 {
t.Fatalf("expected empty map, got %v", m)
}
} }
// formattingCapture captures the raw TOML bytes to verify formatting preservation func TestMultilineInlineTable_CommentAfterComma(t *testing.T) {
type formattingCapture struct { // Exercises comment handling after comma in inline table (parser lines 518-524).
RawBytes string doc := "a = { b = 1, # comment\nc = 2 }"
} var v map[string]interface{}
err := toml.Unmarshal([]byte(doc), &v)
func (f *formattingCapture) UnmarshalTOML(data []byte) error {
f.RawBytes = string(data)
return nil
}
func TestIssue873_FormattingPreservation(t *testing.T) {
type Config struct {
Section *formattingCapture `toml:"section"`
}
// Test that various formatting styles are preserved:
// - Extra spaces around '='
// - Literal strings (single quotes)
// - Hex numbers
// - Inline tables
doc := `[section]
key1 = "value with spaces"
key2 = 'literal string'
hex_val = 0xDEADBEEF
inline = { a = 1, b = 2 }
`
var cfg Config
err := toml.NewDecoder(bytes.NewReader([]byte(doc))).
EnableUnmarshalerInterface().
Decode(&cfg)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, cfg.Section != nil) m, ok := v["a"].(map[string]interface{})
if !ok {
// The raw bytes should preserve original formatting t.Fatal("expected a map")
raw := cfg.Section.RawBytes }
if m["b"] != int64(1) {
// Check that extra spaces around '=' are preserved t.Fatalf("expected b=1, got %v", m["b"])
assert.True(t, strings.Contains(raw, "key1 = \"value with spaces\""), }
"Expected spacing to be preserved, got: %s", raw) if m["c"] != int64(2) {
t.Fatalf("expected c=2, got %v", m["c"])
// Check that literal string style is preserved }
assert.True(t, strings.Contains(raw, "key2 = 'literal string'"), }
"Expected literal string to be preserved, got: %s", raw)
func TestMultilineInlineTable_CommentAfterValue(t *testing.T) {
// Check that hex format is preserved // Exercises comment handling after keyval in inline table (parser lines 542-548).
assert.True(t, strings.Contains(raw, "hex_val = 0xDEADBEEF"), doc := "a = { b = 1 # comment\n, c = 2 }"
"Expected hex format to be preserved, got: %s", raw) var v map[string]interface{}
err := toml.Unmarshal([]byte(doc), &v)
// Check that inline table is preserved assert.NoError(t, err)
assert.True(t, strings.Contains(raw, "inline = { a = 1, b = 2 }"), m, ok := v["a"].(map[string]interface{})
"Expected inline table to be preserved, got: %s", raw) if !ok {
t.Fatal("expected a map")
}
if m["b"] != int64(1) {
t.Fatalf("expected b=1, got %v", m["b"])
}
if m["c"] != int64(2) {
t.Fatalf("expected c=2, got %v", m["c"])
}
}
func TestMultilineInlineTable_LeadingComma(t *testing.T) {
doc := "a = { b = 1\n, c = 2 }"
var v map[string]interface{}
err := toml.Unmarshal([]byte(doc), &v)
assert.NoError(t, err)
m, ok := v["a"].(map[string]interface{})
if !ok {
t.Fatal("expected a map")
}
if m["b"] != int64(1) {
t.Fatalf("expected b=1, got %v", m["b"])
}
if m["c"] != int64(2) {
t.Fatalf("expected c=2, got %v", m["c"])
}
} }
+63 -22
View File
@@ -328,9 +328,6 @@ func (p *Parser) parseStdTable(b []byte) (reference, []byte, error) {
func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) { func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) {
// keyval = key keyval-sep val // keyval = key keyval-sep val
// Track the start position for Raw range
startB := b
ref := p.builder.Push(Node{ ref := p.builder.Push(Node{
Kind: KeyValue, Kind: KeyValue,
}) })
@@ -345,7 +342,7 @@ func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) {
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
if len(b) == 0 { if len(b) == 0 {
return invalidReference, nil, NewParserError(startB[:len(startB)-len(b)], "expected = after a key, but the document ends there") return invalidReference, nil, NewParserError(b, "expected = after a key, but the document ends there")
} }
b, err = expect('=', b) b, err = expect('=', b)
@@ -363,10 +360,6 @@ func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) {
p.builder.Chain(valRef, key) p.builder.Chain(valRef, key)
p.builder.AttachChild(ref, valRef) p.builder.AttachChild(ref, valRef)
// Set Raw to span the entire key-value expression
node := p.builder.NodeAt(ref)
node.Raw = p.rangeOfToken(startB[:len(startB)-len(b)], b)
return ref, b, err return ref, b, err
} }
@@ -467,12 +460,14 @@ func (p *Parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) {
return v, v[1 : len(v)-1], rest, nil return v, v[1 : len(v)-1], rest, nil
} }
//nolint:funlen,cyclop,dupl
func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) { func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) {
// inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close // inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close
// inline-table-open = %x7B ws ; { // inline-table-open = %x7B ws ; {
// inline-table-close = ws %x7D ; } // inline-table-close = ws %x7D ; }
// inline-table-sep = ws %x2C ws ; , Comma // inline-table-sep = ws %x2C ws ; , Comma
// inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ] // inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
tableStart := b
parent := p.builder.Push(Node{ parent := p.builder.Push(Node{
Kind: InlineTable, Kind: InlineTable,
Raw: p.rangeOfToken(b[:1], b[1:]), Raw: p.rangeOfToken(b[:1], b[1:]),
@@ -480,45 +475,77 @@ func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) {
first := true first := true
var child reference lastChild := invalidReference
addChild := func(ref reference) {
if lastChild == invalidReference {
p.builder.AttachChild(parent, ref)
} else {
p.builder.Chain(lastChild, ref)
}
lastChild = ref
}
b = b[1:] b = b[1:]
var err error var err error
for len(b) > 0 { for len(b) > 0 {
previousB := b var cref reference
b = p.parseWhitespace(b) cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil {
return parent, nil, err
}
if cref != invalidReference {
addChild(cref)
}
if len(b) == 0 { if len(b) == 0 {
return parent, nil, NewParserError(previousB[:1], "inline table is incomplete") return parent, nil, NewParserError(tableStart[:1], "inline table is incomplete")
} }
if b[0] == '}' { if b[0] == '}' {
break break
} }
if !first { if b[0] == ',' {
b, err = expect(',', b) if first {
return parent, nil, NewParserError(b[0:1], "inline table cannot start with comma")
}
b = b[1:]
cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil { if err != nil {
return parent, nil, err return parent, nil, err
} }
b = p.parseWhitespace(b) if cref != invalidReference {
addChild(cref)
}
} else if !first {
return parent, nil, NewParserError(b[0:1], "inline table entries must be separated by commas")
}
// trailing comma: if '}' follows, stop
if len(b) > 0 && b[0] == '}' {
break
} }
var kv reference var kv reference
kv, b, err = p.parseKeyval(b) kv, b, err = p.parseKeyval(b)
if err != nil { if err != nil {
return parent, nil, err return parent, nil, err
} }
if first { addChild(kv)
p.builder.AttachChild(parent, kv)
} else { cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
p.builder.Chain(child, kv) if err != nil {
return parent, nil, err
}
if cref != invalidReference {
addChild(cref)
} }
child = kv
first = false first = false
} }
@@ -528,7 +555,7 @@ func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) {
return parent, rest, err return parent, rest, err
} }
//nolint:funlen,cyclop //nolint:funlen,cyclop,dupl
func (p *Parser) parseValArray(b []byte) (reference, []byte, error) { func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
// array = array-open [ array-values ] ws-comment-newline array-close // array = array-open [ array-values ] ws-comment-newline array-close
// array-open = %x5B ; [ // array-open = %x5B ; [
@@ -792,6 +819,13 @@ func (p *Parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
builder.WriteByte('\t') builder.WriteByte('\t')
case 'e': case 'e':
builder.WriteByte(0x1B) builder.WriteByte(0x1B)
case 'x':
x, err := hexToRune(atmost(token[i+1:], 2), 2)
if err != nil {
return nil, nil, nil, err
}
builder.WriteRune(x)
i += 2
case 'u': case 'u':
x, err := hexToRune(atmost(token[i+1:], 4), 4) x, err := hexToRune(atmost(token[i+1:], 4), 4)
if err != nil { if err != nil {
@@ -951,6 +985,13 @@ func (p *Parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
builder.WriteByte('\t') builder.WriteByte('\t')
case 'e': case 'e':
builder.WriteByte(0x1B) builder.WriteByte(0x1B)
case 'x':
x, err := hexToRune(token[i+1:len(token)-1], 2)
if err != nil {
return nil, nil, nil, err
}
builder.WriteRune(x)
i += 2
case 'u': case 'u':
x, err := hexToRune(token[i+1:len(token)-1], 4) x, err := hexToRune(token[i+1:len(token)-1], 4)
if err != nil { if err != nil {
+192 -6
View File
@@ -331,6 +331,154 @@ func TestParser_AST(t *testing.T) {
}, },
}, },
}, },
{
desc: "multiline inline table",
input: "name = {\n first = \"Tom\",\n last = \"Preston-Werner\"\n}",
ast: astNode{
Kind: KeyValue,
Children: []astNode{
{
Kind: InlineTable,
Children: []astNode{
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Tom`)},
{Kind: Key, Data: []byte(`first`)},
},
},
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Preston-Werner`)},
{Kind: Key, Data: []byte(`last`)},
},
},
},
},
{
Kind: Key,
Data: []byte(`name`),
},
},
},
},
{
desc: "inline table with trailing comma",
input: `name = { first = "Tom", last = "Preston-Werner", }`,
ast: astNode{
Kind: KeyValue,
Children: []astNode{
{
Kind: InlineTable,
Children: []astNode{
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Tom`)},
{Kind: Key, Data: []byte(`first`)},
},
},
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Preston-Werner`)},
{Kind: Key, Data: []byte(`last`)},
},
},
},
},
{
Kind: Key,
Data: []byte(`name`),
},
},
},
},
{
desc: "empty inline table with newline",
input: "name = {\n}",
ast: astNode{
Kind: KeyValue,
Children: []astNode{
{
Kind: InlineTable,
Children: nil,
},
{
Kind: Key,
Data: []byte(`name`),
},
},
},
},
{
desc: "inline table with leading comma",
input: "name = { first = \"Tom\"\n, last = \"Werner\" }",
ast: astNode{
Kind: KeyValue,
Children: []astNode{
{
Kind: InlineTable,
Children: []astNode{
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Tom`)},
{Kind: Key, Data: []byte(`first`)},
},
},
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Werner`)},
{Kind: Key, Data: []byte(`last`)},
},
},
},
},
{
Kind: Key,
Data: []byte(`name`),
},
},
},
},
{
desc: "inline table with leading trailing comma",
input: "name = { first = \"Tom\"\n, }",
ast: astNode{
Kind: KeyValue,
Children: []astNode{
{
Kind: InlineTable,
Children: []astNode{
{
Kind: KeyValue,
Children: []astNode{
{Kind: String, Data: []byte(`Tom`)},
{Kind: Key, Data: []byte(`first`)},
},
},
},
},
{
Kind: Key,
Data: []byte(`name`),
},
},
},
},
{
desc: "inline table comma at start is error",
input: "name = { , first = \"Tom\" }",
err: true,
},
{
desc: "inline table double comma across newline is error",
input: "name = { first = \"Tom\",\n, last = \"Werner\" }",
err: true,
},
} }
for _, e := range examples { for _, e := range examples {
@@ -350,6 +498,44 @@ func TestParser_AST(t *testing.T) {
} }
} }
func TestParseInlineTable_CommentsWithKeepComments(t *testing.T) {
// Exercise comment reference handling inside parseInlineTable when
// KeepComments is true. This covers the addChild(cref) branches
// at the start of the loop, after comma, and after keyval.
examples := []struct {
desc string
input string
}{
{
desc: "comment at start of inline table",
input: "a = {\n# comment\nb = 1\n}",
},
{
desc: "comment after comma",
input: "a = {b = 1,\n# comment\nc = 2\n}",
},
{
desc: "comment after keyval",
input: "a = {b = 1 # comment\n, c = 2}",
},
{
desc: "comment only in inline table",
input: "a = {\n# just a comment\n}",
},
}
for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) {
p := Parser{KeepComments: true}
p.Reset([]byte(e.input))
p.NextExpression()
err := p.Error()
assert.NoError(t, err)
})
}
}
func BenchmarkParseBasicStringWithUnicode(b *testing.B) { func BenchmarkParseBasicStringWithUnicode(b *testing.B) {
p := &Parser{} p := &Parser{}
b.Run("4", func(b *testing.B) { b.Run("4", func(b *testing.B) {
@@ -539,7 +725,7 @@ key5 = [ # Next to start of inline array.
// --- // ---
// 6:1->6:22 (105->126) | Comment [# Above simple value.] // 6:1->6:22 (105->126) | Comment [# Above simple value.]
// --- // ---
// 7:1->7:14 (127->140) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 7:7->7:14 (133->140) | String [value] // 7:7->7:14 (133->140) | String [value]
// 7:1->7:4 (127->130) | Key [key] // 7:1->7:4 (127->130) | Key [key]
// 7:15->7:38 (141->164) | Comment [# Next to simple value.] // 7:15->7:38 (141->164) | Comment [# Next to simple value.]
@@ -552,12 +738,12 @@ key5 = [ # Next to start of inline array.
// --- // ---
// 14:1->14:22 (252->273) | Comment [# Above inline table.] // 14:1->14:22 (252->273) | Comment [# Above inline table.]
// --- // ---
// 15:1->15:50 (274->323) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 15:8->15:9 (281->282) | InlineTable [] // 15:8->15:9 (281->282) | InlineTable []
// 15:10->15:23 (283->296) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 15:18->15:23 (291->296) | String [Tom] // 15:18->15:23 (291->296) | String [Tom]
// 15:10->15:15 (283->288) | Key [first] // 15:10->15:15 (283->288) | Key [first]
// 15:25->15:48 (298->321) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 15:32->15:48 (305->321) | String [Preston-Werner] // 15:32->15:48 (305->321) | String [Preston-Werner]
// 15:25->15:29 (298->302) | Key [last] // 15:25->15:29 (298->302) | Key [last]
// 15:1->15:5 (274->278) | Key [name] // 15:1->15:5 (274->278) | Key [name]
@@ -567,7 +753,7 @@ key5 = [ # Next to start of inline array.
// --- // ---
// 18:1->18:15 (371->385) | Comment [# Above array.] // 18:1->18:15 (371->385) | Comment [# Above array.]
// --- // ---
// 19:1->19:20 (386->405) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 1:1->1:1 (0->0) | Array [] // 1:1->1:1 (0->0) | Array []
// 19:11->19:12 (396->397) | Integer [1] // 19:11->19:12 (396->397) | Integer [1]
// 19:14->19:15 (399->400) | Integer [2] // 19:14->19:15 (399->400) | Integer [2]
@@ -579,7 +765,7 @@ key5 = [ # Next to start of inline array.
// --- // ---
// 22:1->22:26 (448->473) | Comment [# Above multi-line array.] // 22:1->22:26 (448->473) | Comment [# Above multi-line array.]
// --- // ---
// 23:1->31:2 (474->694) | KeyValue [] // 1:1->1:1 (0->0) | KeyValue []
// 1:1->1:1 (0->0) | Array [] // 1:1->1:1 (0->0) | Array []
// 23:10->23:42 (483->515) | Comment [# Next to start of inline array.] // 23:10->23:42 (483->515) | Comment [# Next to start of inline array.]
// 24:3->24:38 (518->553) | Comment [# Second line before array content.] // 24:3->24:38 (518->553) | Comment [# Second line before array content.]
+3 -28
View File
@@ -1,32 +1,7 @@
package unstable package unstable
// Unmarshaler is implemented by types that can unmarshal a TOML // The Unmarshaler interface may be implemented by types to customize their
// description of themselves. The input is a valid TOML document // behavior when being unmarshaled from a TOML document.
// containing the relevant portion of the parsed document.
//
// For tables (including split tables defined in multiple places),
// the data contains the raw key-value bytes from the original document
// with adjusted table headers to be relative to the unmarshaling target.
type Unmarshaler interface { type Unmarshaler interface {
UnmarshalTOML(data []byte) error UnmarshalTOML(value *Node) error
}
// RawMessage is a raw encoded TOML value. It implements Unmarshaler
// and can be used to delay TOML decoding or capture raw content.
//
// Example usage:
//
// type Config struct {
// Plugin RawMessage `toml:"plugin"`
// }
//
// var cfg Config
// toml.NewDecoder(r).EnableUnmarshalerInterface().Decode(&cfg)
// // cfg.Plugin now contains the raw TOML bytes for [plugin]
type RawMessage []byte
// UnmarshalTOML implements Unmarshaler.
func (m *RawMessage) UnmarshalTOML(data []byte) error {
*m = append((*m)[0:0], data...)
return nil
} }