ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make all nodes contain Raw

Browse Source
This commit is contained in:
Thomas Pelletier
2022-08-22 21:05:41 -04:00
parent 64dcce07ea
commit b371733c67
2 changed files with 63 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parser.go
+44 -20
View File
@@ -310,6 +310,7 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.Bool, Kind: ast.Bool,
Data: b[:4], Data: b[:4],
Raw: p.Range(b[:4]),
}) })
return ref, b[4:], nil return ref, b[4:], nil
@@ -321,6 +322,7 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.Bool, Kind: ast.Bool,
Data: b[:5], Data: b[:5],
Raw: p.Range(b[:5]),
}) })
return ref, b[5:], nil return ref, b[5:], nil
@@ -344,7 +346,7 @@ func atmost(b []byte, n int) []byte {
func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) { func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) {
v, rest, err := scanLiteralString(b) v, rest, err := scanLiteralString(b)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
return v, v[1 : len(v)-1], rest, nil return v, v[1 : len(v)-1], rest, nil
@@ -359,6 +361,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
parent := p.builder.Push(ast.Node{ parent := p.builder.Push(ast.Node{
Kind: ast.InlineTable, Kind: ast.InlineTable,
}) })
start := b
first := true first := true
@@ -373,6 +376,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
if len(b) == 0 { if len(b) == 0 {
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(previousB[:1], "inline table is incomplete") return parent, b, newDecodeError(previousB[:1], "inline table is incomplete")
} }
@@ -383,6 +387,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
if !first { if !first {
b, err = expect(',', b) b, err = expect(',', b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
@@ -392,6 +397,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
kv, b, err = p.parseKeyval(b) kv, b, err = p.parseKeyval(b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
@@ -407,6 +413,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
rest, err := expect('}', b) rest, err := expect('}', b)
p.SetRaw(parent, start, b)
return parent, rest, err return parent, rest, err
} }
@@ -420,6 +427,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
// array-sep = %x2C ; , Comma // array-sep = %x2C ; , Comma
// ws-comment-newline = *( wschar / [ comment ] newline ) // ws-comment-newline = *( wschar / [ comment ] newline )
arrayStart := b arrayStart := b
start := b
b = b[1:] b = b[1:]
parent := p.builder.Push(ast.Node{ parent := p.builder.Push(ast.Node{
@@ -434,10 +442,12 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
for len(b) > 0 { for len(b) > 0 {
b, err = p.parseOptionalWhitespaceCommentNewline(b) b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
if len(b) == 0 { if len(b) == 0 {
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(arrayStart[:1], "array is incomplete") return parent, b, newDecodeError(arrayStart[:1], "array is incomplete")
} }
@@ -447,15 +457,18 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
if b[0] == ',' { if b[0] == ',' {
if first { if first {
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(b[0:1], "array cannot start with comma") return parent, b, newDecodeError(b[0:1], "array cannot start with comma")
} }
b = b[1:] b = b[1:]
b, err = p.parseOptionalWhitespaceCommentNewline(b) b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
} else if !first { } else if !first {
p.SetRaw(parent, start, b)
return parent, b, newDecodeError(b[0:1], "array elements must be separated by commas") return parent, b, newDecodeError(b[0:1], "array elements must be separated by commas")
} }
@@ -467,6 +480,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
var valueRef ast.Reference var valueRef ast.Reference
valueRef, b, err = p.parseVal(b) valueRef, b, err = p.parseVal(b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
@@ -479,6 +493,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
b, err = p.parseOptionalWhitespaceCommentNewline(b) b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil { if err != nil {
p.SetRaw(parent, start, b)
return parent, b, err return parent, b, err
} }
first = false first = false
@@ -486,6 +501,8 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
rest, err := expect(']', b) rest, err := expect(']', b)
p.SetRaw(parent, start, rest)
return parent, rest, err return parent, rest, err
} }
@@ -497,7 +514,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
if len(b) > 0 && b[0] == '#' { if len(b) > 0 && b[0] == '#' {
_, b, err = scanComment(b) _, b, err = scanComment(b)
if err != nil { if err != nil {
return nil, err return b, err
} }
} }
@@ -508,7 +525,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
if b[0] == '\n' || b[0] == '\r' { if b[0] == '\n' || b[0] == '\r' {
b, err = p.parseNewline(b) b, err = p.parseNewline(b)
if err != nil { if err != nil {
return nil, err return b, err
} }
} else { } else {
break break
@@ -521,7 +538,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) { func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) {
token, rest, err := scanMultilineLiteralString(b) token, rest, err := scanMultilineLiteralString(b)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
i := 3 i := 3
@@ -550,7 +567,7 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
// mlb-escaped-nl = escape ws newline *( wschar / newline ) // mlb-escaped-nl = escape ws newline *( wschar / newline )
token, escaped, rest, err := scanMultilineBasicString(b) token, escaped, rest, err := scanMultilineBasicString(b)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
i := 3 i := 3
@@ -572,7 +589,7 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
if verr.Zero() { if verr.Zero() {
return token, str, rest, nil return token, str, rest, nil
} }
return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") return nil, nil, rest, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
} }
var builder bytes.Buffer var builder bytes.Buffer
@@ -639,26 +656,26 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er
case 'u': case 'u':
x, err := hexToRune(atmost(token[i+1:], 4), 4) x, err := hexToRune(atmost(token[i+1:], 4), 4)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
builder.WriteRune(x) builder.WriteRune(x)
i += 4 i += 4
case 'U': case 'U':
x, err := hexToRune(atmost(token[i+1:], 8), 8) x, err := hexToRune(atmost(token[i+1:], 8), 8)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
builder.WriteRune(x) builder.WriteRune(x)
i += 8 i += 8
default: default:
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c) return nil, nil, rest, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
} }
i++ i++
} else { } else {
size := utf8ValidNext(token[i:]) size := utf8ValidNext(token[i:])
if size == 0 { if size == 0 {
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c) return nil, nil, rest, newDecodeError(token[i:i+1], "invalid character %#U", c)
} }
builder.Write(token[i : i+size]) builder.Write(token[i : i+size])
i += size i += size
@@ -679,7 +696,7 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
// dot-sep = ws %x2E ws ; . Period // dot-sep = ws %x2E ws ; . Period
raw, key, b, err := p.parseSimpleKey(b) raw, key, b, err := p.parseSimpleKey(b)
if err != nil { if err != nil {
return ast.InvalidReference, nil, err return ast.InvalidReference, b, err
} }
ref := p.builder.Push(ast.Node{ ref := p.builder.Push(ast.Node{
@@ -695,7 +712,7 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
raw, key, b, err = p.parseSimpleKey(b) raw, key, b, err = p.parseSimpleKey(b)
if err != nil { if err != nil {
return ref, nil, err return ref, b, err
} }
p.builder.PushAndChain(ast.Node{ p.builder.PushAndChain(ast.Node{
@@ -713,7 +730,7 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) { func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
if len(b) == 0 { if len(b) == 0 {
return nil, nil, nil, newDecodeError(b, "expected key but found none") return nil, nil, b, newDecodeError(b, "expected key but found none")
} }
// simple-key = quoted-key / unquoted-key // simple-key = quoted-key / unquoted-key
@@ -728,7 +745,7 @@ func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
key, rest = scanUnquotedKey(b) key, rest = scanUnquotedKey(b)
return key, key, rest, nil return key, key, rest, nil
default: default:
return nil, nil, nil, newDecodeError(b[0:1], "invalid character at start of key: %c", b[0]) return nil, nil, b[1:], newDecodeError(b[0:1], "invalid character at start of key: %c", b[0])
} }
} }
@@ -750,7 +767,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX // escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
token, escaped, rest, err := scanBasicString(b) token, escaped, rest, err := scanBasicString(b)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
startIdx := len(`"`) startIdx := len(`"`)
@@ -765,7 +782,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
if verr.Zero() { if verr.Zero() {
return token, str, rest, nil return token, str, rest, nil
} }
return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") return nil, nil, rest, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
} }
i := startIdx i := startIdx
@@ -798,7 +815,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
case 'u': case 'u':
x, err := hexToRune(token[i+1:len(token)-1], 4) x, err := hexToRune(token[i+1:len(token)-1], 4)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
builder.WriteRune(x) builder.WriteRune(x)
@@ -806,19 +823,19 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
case 'U': case 'U':
x, err := hexToRune(token[i+1:len(token)-1], 8) x, err := hexToRune(token[i+1:len(token)-1], 8)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, rest, err
} }
builder.WriteRune(x) builder.WriteRune(x)
i += 8 i += 8
default: default:
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c) return nil, nil, rest, newDecodeError(token[i:i+1], "invalid escaped character %#U", c)
} }
i++ i++
} else { } else {
size := utf8ValidNext(token[i:]) size := utf8ValidNext(token[i:])
if size == 0 { if size == 0 {
return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c) return nil, nil, rest, newDecodeError(token[i:i+1], "invalid character %#U", c)
} }
builder.Write(token[i : i+size]) builder.Write(token[i : i+size])
i += size i += size
@@ -877,6 +894,7 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:3], Data: b[:3],
Raw: p.Range(b[:3]),
}), b[3:], nil }), b[3:], nil
case 'n': case 'n':
if !scanFollowsNan(b) { if !scanFollowsNan(b) {
@@ -886,6 +904,7 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:3], Data: b[:3],
Raw: p.Range(b[:3]),
}), b[3:], nil }), b[3:], nil
case '+', '-': case '+', '-':
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
@@ -977,6 +996,7 @@ byteLoop:
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: kind, Kind: kind,
Data: b[:i], Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil }), b[i:], nil
} }
@@ -1010,6 +1030,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Integer, Kind: ast.Integer,
Data: b[:i], Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil }), b[i:], nil
} }
@@ -1033,6 +1054,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:i+3], Data: b[:i+3],
Raw: p.Range(b[:i+3]),
}), b[i+3:], nil }), b[i+3:], nil
} }
@@ -1044,6 +1066,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:i+3], Data: b[:i+3],
Raw: p.Range(b[:i+3]),
}), b[i+3:], nil }), b[i+3:], nil
} }
@@ -1066,6 +1089,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: kind, Kind: kind,
Data: b[:i], Data: b[:i],
Raw: p.Range(b[:i]),
}), b[i:], nil }), b[i:], nil
} }
scanner.go
+19 -19
View File
@@ -54,16 +54,16 @@ func scanLiteralString(b []byte) ([]byte, []byte, error) {
case '\'': case '\'':
return b[:i+1], b[i+1:], nil return b[:i+1], b[i+1:], nil
case '\n', '\r': case '\n', '\r':
return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines") return nil, b[i+1:], newDecodeError(b[i:i+1], "literal strings cannot have new lines")
} }
size := utf8ValidNext(b[i:]) size := utf8ValidNext(b[i:])
if size == 0 { if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character") return nil, b[i+1:], newDecodeError(b[i:i+1], "invalid character")
} }
i += size i += size
} }
return nil, nil, newDecodeError(b[len(b):], "unterminated literal string") return nil, b[len(b):], newDecodeError(b[len(b):], "unterminated literal string")
} }
func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) { func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
@@ -98,39 +98,39 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) {
i++ i++
if i < len(b) && b[i] == '\'' { if i < len(b) && b[i] == '\'' {
return nil, nil, newDecodeError(b[i-3:i+1], "''' not allowed in multiline literal string") return nil, b[i:], newDecodeError(b[i-3:i+1], "''' not allowed in multiline literal string")
} }
return b[:i], b[i:], nil return b[:i], b[i:], nil
} }
case '\r': case '\r':
if len(b) < i+2 { if len(b) < i+2 {
return nil, nil, newDecodeError(b[len(b):], `need a \n after \r`) return nil, b[i:], newDecodeError(b[len(b):], `need a \n after \r`)
} }
if b[i+1] != '\n' { if b[i+1] != '\n' {
return nil, nil, newDecodeError(b[i:i+2], `need a \n after \r`) return nil, b[i+2:], newDecodeError(b[i:i+2], `need a \n after \r`)
} }
i += 2 // skip the \n i += 2 // skip the \n
continue continue
} }
size := utf8ValidNext(b[i:]) size := utf8ValidNext(b[i:])
if size == 0 { if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character") return nil, b[i:], newDecodeError(b[i:i+1], "invalid character")
} }
i += size i += size
} }
return nil, nil, newDecodeError(b[len(b):], `multiline literal string not terminated by '''`) return nil, b[len(b):], newDecodeError(b[len(b):], `multiline literal string not terminated by '''`)
} }
func scanWindowsNewline(b []byte) ([]byte, []byte, error) { func scanWindowsNewline(b []byte) ([]byte, []byte, error) {
const lenCRLF = 2 const lenCRLF = 2
if len(b) < lenCRLF { if len(b) < lenCRLF {
return nil, nil, newDecodeError(b, "windows new line expected") return nil, b, newDecodeError(b, "windows new line expected")
} }
if b[1] != '\n' { if b[1] != '\n' {
return nil, nil, newDecodeError(b, `windows new line should be \r\n`) return nil, b[2:], newDecodeError(b, `windows new line should be \r\n`)
} }
return b[:lenCRLF], b[lenCRLF:], nil return b[:lenCRLF], b[lenCRLF:], nil
@@ -169,7 +169,7 @@ func scanComment(b []byte) ([]byte, []byte, error) {
} }
size := utf8ValidNext(b[i:]) size := utf8ValidNext(b[i:])
if size == 0 { if size == 0 {
return nil, nil, newDecodeError(b[i:i+1], "invalid character in comment") return nil, b[i+1:], newDecodeError(b[i:i+1], "invalid character in comment")
} }
i += size i += size
@@ -192,17 +192,17 @@ func scanBasicString(b []byte) ([]byte, bool, []byte, error) {
case '"': case '"':
return b[:i+1], escaped, b[i+1:], nil return b[:i+1], escaped, b[i+1:], nil
case '\n', '\r': case '\n', '\r':
return nil, escaped, nil, newDecodeError(b[i:i+1], "basic strings cannot have new lines") return nil, escaped, b[i+1:], newDecodeError(b[i:i+1], "basic strings cannot have new lines")
case '\\': case '\\':
if len(b) < i+2 { if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[i:i+1], "need a character after \\") return nil, escaped, b[i+1:], newDecodeError(b[i:i+1], "need a character after \\")
} }
escaped = true escaped = true
i++ // skip the next character i++ // skip the next character
} }
} }
return nil, escaped, nil, newDecodeError(b[len(b):], `basic string not terminated by "`) return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `basic string not terminated by "`)
} }
func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) { func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) {
@@ -243,27 +243,27 @@ func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) {
i++ i++
if i < len(b) && b[i] == '"' { if i < len(b) && b[i] == '"' {
return nil, escaped, nil, newDecodeError(b[i-3:i+1], `""" not allowed in multiline basic string`) return nil, escaped, b[i+1:], newDecodeError(b[i-3:i+1], `""" not allowed in multiline basic string`)
} }
return b[:i], escaped, b[i:], nil return b[:i], escaped, b[i:], nil
} }
case '\\': case '\\':
if len(b) < i+2 { if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[len(b):], "need a character after \\") return nil, escaped, b[len(b):], newDecodeError(b[len(b):], "need a character after \\")
} }
escaped = true escaped = true
i++ // skip the next character i++ // skip the next character
case '\r': case '\r':
if len(b) < i+2 { if len(b) < i+2 {
return nil, escaped, nil, newDecodeError(b[len(b):], `need a \n after \r`) return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `need a \n after \r`)
} }
if b[i+1] != '\n' { if b[i+1] != '\n' {
return nil, escaped, nil, newDecodeError(b[i:i+2], `need a \n after \r`) return nil, escaped, b[i+2:], newDecodeError(b[i:i+2], `need a \n after \r`)
} }
i++ // skip the \n i++ // skip the \n
} }
} }
return nil, escaped, nil, newDecodeError(b[len(b):], `multiline basic string not terminated by """`) return nil, escaped, b[len(b):], newDecodeError(b[len(b):], `multiline basic string not terminated by """`)
} }