golangci-lint: decoder/unmarshal (#518)

This commit is contained in:
Vincent Serpoul
2021-04-22 21:29:23 +08:00
committed by GitHub
parent 21445f5170
commit 2b1c52dddd
6 changed files with 253 additions and 177 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ enable = [
"errcheck", "errcheck",
"errorlint", "errorlint",
"exhaustive", "exhaustive",
"exhaustivestruct", # "exhaustivestruct",
"exportloopref", "exportloopref",
"forbidigo", "forbidigo",
"forcetypeassert", "forcetypeassert",
+7 -2
View File
@@ -174,9 +174,12 @@ var errParseLocalTimeWrongLength = errors.New("times are expected to have the fo
// parseLocalTime is a bit different because it also returns the remaining // parseLocalTime is a bit different because it also returns the remaining
// []byte that is didn't need. This is to allow parseDateTime to parse those // []byte that is didn't need. This is to allow parseDateTime to parse those
// remaining bytes as a timezone. // remaining bytes as a timezone.
//nolint:cyclop,funlen
func parseLocalTime(b []byte) (LocalTime, []byte, error) { func parseLocalTime(b []byte) (LocalTime, []byte, error) {
var nspow = [10]int{0, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 1e2, 1e1, 1e0} var (
var t LocalTime nspow = [10]int{0, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 1e2, 1e1, 1e0}
t LocalTime
)
const localTimeByteLen = 8 const localTimeByteLen = 8
if len(b) < localTimeByteLen { if len(b) < localTimeByteLen {
@@ -220,6 +223,8 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) {
break break
} }
//nolint:gomnd
if i >= 9 { if i >= 9 {
return t, nil, newDecodeError(b[i:i+1], "maximum precision for date time is nanosecond") return t, nil, newDecodeError(b[i:i+1], "maximum precision for date time is nanosecond")
} }
+5 -2
View File
@@ -33,7 +33,7 @@ type StrictMissingError struct {
Errors []DecodeError Errors []DecodeError
} }
// Error returns the cannonical string for this error. // Error returns the canonical string for this error.
func (s *StrictMissingError) Error() string { func (s *StrictMissingError) Error() string {
return "strict mode: fields in the document are missing in the target struct" return "strict mode: fields in the document are missing in the target struct"
} }
@@ -41,12 +41,15 @@ func (s *StrictMissingError) Error() string {
// String returns a human readable description of all errors. // String returns a human readable description of all errors.
func (s *StrictMissingError) String() string { func (s *StrictMissingError) String() string {
var buf strings.Builder var buf strings.Builder
for i, e := range s.Errors { for i, e := range s.Errors {
if i > 0 { if i > 0 {
buf.WriteString("\n---\n") buf.WriteString("\n---\n")
} }
buf.WriteString(e.String()) buf.WriteString(e.String())
} }
return buf.String() return buf.String()
} }
@@ -87,7 +90,7 @@ func (e *DecodeError) Position() (row int, column int) {
return e.line, e.column return e.line, e.column
} }
// Key that was being processed when the error occured. // Key that was being processed when the error occurred.
func (e *DecodeError) Key() Key { func (e *DecodeError) Key() Key {
return e.key return e.key
} }
+18 -11
View File
@@ -293,9 +293,9 @@ func TestDateTimeToString(t *testing.T) {
dateTime LocalDateTime dateTime LocalDateTime
roundTrip bool // ParseLocalDateTime(str).String() == str? roundTrip bool // ParseLocalDateTime(str).String() == str?
}{ }{
{"2016-03-22T13:26:33", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 0}}, true}, {"2016-03-22T13:26:33", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 0}}, true},
{"2016-03-22T13:26:33.000000600", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 600}}, true}, {"2016-03-22T13:26:33.000000600", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 600}}, true},
{"2016-03-22t13:26:33", LocalDateTime{LocalDate{2016, 03, 22}, LocalTime{13, 26, 33, 0}}, false}, {"2016-03-22t13:26:33", LocalDateTime{LocalDate{2016, 3, 22}, LocalTime{13, 26, 33, 0}}, false},
} { } {
gotDateTime, err := ParseLocalDateTime(test.str) gotDateTime, err := ParseLocalDateTime(test.str)
if err != nil { if err != nil {
@@ -338,10 +338,14 @@ func TestDateTimeOf(t *testing.T) {
time time.Time time time.Time
want LocalDateTime want LocalDateTime
}{ }{
{time.Date(2014, 8, 20, 15, 8, 43, 1, time.Local), {
LocalDateTime{LocalDate{2014, 8, 20}, LocalTime{15, 8, 43, 1}}}, time.Date(2014, 8, 20, 15, 8, 43, 1, time.Local),
{time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), LocalDateTime{LocalDate{2014, 8, 20}, LocalTime{15, 8, 43, 1}},
LocalDateTime{LocalDate{1, 1, 1}, LocalTime{0, 0, 0, 0}}}, },
{
time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC),
LocalDateTime{LocalDate{1, 1, 1}, LocalTime{0, 0, 0, 0}},
},
} { } {
if got := LocalDateTimeOf(test.time); got != test.want { if got := LocalDateTimeOf(test.time); got != test.want {
t.Errorf("LocalDateTimeOf(%v) = %+v, want %+v", test.time, got, test.want) t.Errorf("LocalDateTimeOf(%v) = %+v, want %+v", test.time, got, test.want)
@@ -449,9 +453,11 @@ func TestMarshalJSON(t *testing.T) {
func TestUnmarshalJSON(t *testing.T) { func TestUnmarshalJSON(t *testing.T) {
t.Parallel() t.Parallel()
var d LocalDate var (
var tm LocalTime d LocalDate
var dt LocalDateTime tm LocalTime
dt LocalDateTime
)
for _, test := range []struct { for _, test := range []struct {
data string data string
@@ -471,7 +477,8 @@ func TestUnmarshalJSON(t *testing.T) {
} }
} }
for _, bad := range []string{"", `""`, `"bad"`, `"1987-04-15x"`, for _, bad := range []string{
"", `""`, `"bad"`, `"1987-04-15x"`,
`19870415`, // a JSON number `19870415`, // a JSON number
`11987-04-15x`, // not a JSON string `11987-04-15x`, // not a JSON string
+152 -47
View File
@@ -2,6 +2,7 @@ package toml
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"strconv" "strconv"
@@ -26,6 +27,7 @@ func (p *parser) Reset(b []byte) {
p.first = true p.first = true
} }
//nolint:cyclop
func (p *parser) NextExpression() bool { func (p *parser) NextExpression() bool {
if len(p.left) == 0 || p.err != nil { if len(p.left) == 0 || p.err != nil {
return false return false
@@ -69,22 +71,26 @@ func (p *parser) Error() error {
return p.err return p.err
} }
var errUnexpectedByte = errors.New("expected newline but got something else")
func (p *parser) parseNewline(b []byte) ([]byte, error) { func (p *parser) parseNewline(b []byte) ([]byte, error) {
if b[0] == '\n' { if b[0] == '\n' {
return b[1:], nil return b[1:], nil
} }
if b[0] == '\r' { if b[0] == '\r' {
_, rest, err := scanWindowsNewline(b) _, rest, err := scanWindowsNewline(b)
return rest, err return rest, err
} }
return nil, fmt.Errorf("expected newline but got %#U", b[0])
return nil, fmt.Errorf("parseNewline: %w - %#U", errUnexpectedByte, b[0])
} }
func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) { func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
// expression = ws [ comment ] // expression = ws [ comment ]
// expression =/ ws keyval ws [ comment ] // expression =/ ws keyval ws [ comment ]
// expression =/ ws table ws [ comment ] // expression =/ ws table ws [ comment ]
var ref ast.Reference var ref ast.Reference
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
@@ -95,8 +101,10 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
if b[0] == '#' { if b[0] == '#' {
_, rest, err := scanComment(b) _, rest, err := scanComment(b)
return ref, rest, err return ref, rest, err
} }
if b[0] == '\n' || b[0] == '\r' { if b[0] == '\n' || b[0] == '\r' {
return ref, b, nil return ref, b, nil
} }
@@ -107,6 +115,7 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
} else { } else {
ref, b, err = p.parseKeyval(b) ref, b, err = p.parseKeyval(b)
} }
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
} }
@@ -115,6 +124,7 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) {
if len(b) > 0 && b[0] == '#' { if len(b) > 0 && b[0] == '#' {
_, rest, err := scanComment(b) _, rest, err := scanComment(b)
return ref, rest, err return ref, rest, err
} }
@@ -126,6 +136,7 @@ func (p *parser) parseTable(b []byte) (ast.Reference, []byte, error) {
if len(b) > 1 && b[1] == '[' { if len(b) > 1 && b[1] == '[' {
return p.parseArrayTable(b) return p.parseArrayTable(b)
} }
return p.parseStdTable(b) return p.parseStdTable(b)
} }
@@ -133,24 +144,28 @@ func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) {
// array-table = array-table-open key array-table-close // array-table = array-table-open key array-table-close
// array-table-open = %x5B.5B ws ; [[ Double left square bracket // array-table-open = %x5B.5B ws ; [[ Double left square bracket
// array-table-close = ws %x5D.5D ; ]] Double right square bracket // array-table-close = ws %x5D.5D ; ]] Double right square bracket
ref := p.builder.Push(ast.Node{ ref := p.builder.Push(ast.Node{
Kind: ast.ArrayTable, Kind: ast.ArrayTable,
}) })
b = b[2:] b = b[2:]
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
k, b, err := p.parseKey(b) k, b, err := p.parseKey(b)
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
} }
p.builder.AttachChild(ref, k) p.builder.AttachChild(ref, k)
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
b, err = expect(']', b) b, err = expect(']', b)
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
} }
b, err = expect(']', b) b, err = expect(']', b)
return ref, b, err return ref, b, err
} }
@@ -158,13 +173,13 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
// std-table = std-table-open key std-table-close // std-table = std-table-open key std-table-close
// std-table-open = %x5B ws ; [ Left square bracket // std-table-open = %x5B ws ; [ Left square bracket
// std-table-close = ws %x5D ; ] Right square bracket // std-table-close = ws %x5D ; ] Right square bracket
ref := p.builder.Push(ast.Node{ ref := p.builder.Push(ast.Node{
Kind: ast.Table, Kind: ast.Table,
}) })
b = b[1:] b = b[1:]
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
key, b, err := p.parseKey(b) key, b, err := p.parseKey(b)
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
@@ -181,7 +196,6 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) {
func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) { func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
// keyval = key keyval-sep val // keyval = key keyval-sep val
ref := p.builder.Push(ast.Node{ ref := p.builder.Push(ast.Node{
Kind: ast.KeyValue, Kind: ast.KeyValue,
}) })
@@ -194,28 +208,38 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) {
// keyval-sep = ws %x3D ws ; = // keyval-sep = ws %x3D ws ; =
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
b, err = expect('=', b) b, err = expect('=', b)
if err != nil { if err != nil {
return ast.Reference{}, nil, err return ast.Reference{}, nil, err
} }
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
valRef, b, err := p.parseVal(b) valRef, b, err := p.parseVal(b)
if err != nil { if err != nil {
return ref, b, err return ref, b, err
} }
p.builder.Chain(valRef, key) p.builder.Chain(valRef, key)
p.builder.AttachChild(ref, valRef) p.builder.AttachChild(ref, valRef)
return ref, b, err return ref, b, err
} }
var (
errExpectedValNotEOF = errors.New("expected value, not eof")
errExpectedTrue = errors.New("expected 'true'")
errExpectedFalse = errors.New("expected 'false'")
)
//nolint:cyclop,funlen
func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) { func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
// val = string / boolean / array / inline-table / date-time / float / integer // val = string / boolean / array / inline-table / date-time / float / integer
var ref ast.Reference var ref ast.Reference
if len(b) == 0 { if len(b) == 0 {
return ref, nil, fmt.Errorf("expected value, not eof") return ref, nil, errExpectedValNotEOF
} }
var err error var err error
@@ -229,12 +253,14 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
} else { } else {
v, b, err = p.parseBasicString(b) v, b, err = p.parseBasicString(b)
} }
if err == nil { if err == nil {
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.String, Kind: ast.String,
Data: v, Data: v,
}) })
} }
return ref, b, err return ref, b, err
case '\'': case '\'':
var v []byte var v []byte
@@ -243,30 +269,36 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) {
} else { } else {
v, b, err = p.parseLiteralString(b) v, b, err = p.parseLiteralString(b)
} }
if err == nil { if err == nil {
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.String, Kind: ast.String,
Data: v, Data: v,
}) })
} }
return ref, b, err return ref, b, err
case 't': case 't':
if !scanFollowsTrue(b) { if !scanFollowsTrue(b) {
return ref, nil, fmt.Errorf("expected 'true'") return ref, nil, errExpectedTrue
} }
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.Bool, Kind: ast.Bool,
Data: b[:4], Data: b[:4],
}) })
return ref, b[4:], nil return ref, b[4:], nil
case 'f': case 'f':
if !scanFollowsFalse(b) { if !scanFollowsFalse(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'false'") return ast.Reference{}, nil, errExpectedFalse
} }
ref = p.builder.Push(ast.Node{ ref = p.builder.Push(ast.Node{
Kind: ast.Bool, Kind: ast.Bool,
Data: b[:5], Data: b[:5],
}) })
return ref, b[5:], nil return ref, b[5:], nil
case '[': case '[':
return p.parseValArray(b) return p.parseValArray(b)
@@ -282,6 +314,7 @@ func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
return v[1 : len(v)-1], rest, nil return v[1 : len(v)-1], rest, nil
} }
@@ -291,17 +324,18 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
// inline-table-close = ws %x7D ; } // inline-table-close = ws %x7D ; }
// inline-table-sep = ws %x2C ws ; , Comma // inline-table-sep = ws %x2C ws ; , Comma
// inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ] // inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
parent := p.builder.Push(ast.Node{ parent := p.builder.Push(ast.Node{
Kind: ast.InlineTable, Kind: ast.InlineTable,
}) })
first := true first := true
var child ast.Reference var child ast.Reference
b = b[1:] b = b[1:]
var err error var err error
for len(b) > 0 { for len(b) > 0 {
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
if b[0] == '}' { if b[0] == '}' {
@@ -315,7 +349,9 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
} }
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
} }
var kv ast.Reference var kv ast.Reference
kv, b, err = p.parseKeyval(b) kv, b, err = p.parseKeyval(b)
if err != nil { if err != nil {
return parent, nil, err return parent, nil, err
@@ -323,7 +359,6 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
if first { if first {
p.builder.AttachChild(parent, kv) p.builder.AttachChild(parent, kv)
first = false
} else { } else {
p.builder.Chain(child, kv) p.builder.Chain(child, kv)
} }
@@ -333,9 +368,13 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) {
} }
rest, err := expect('}', b) rest, err := expect('}', b)
return parent, rest, err return parent, rest, err
} }
var errArrayCanNotStartWithComma = errors.New("array cannot start with comma")
//nolint:funlen,cyclop
func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
// array = array-open [ array-values ] ws-comment-newline array-close // array = array-open [ array-values ] ws-comment-newline array-close
// array-open = %x5B ; [ // array-open = %x5B ; [
@@ -344,7 +383,6 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
// array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ] // array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ]
// array-sep = %x2C ; , Comma // array-sep = %x2C ; , Comma
// ws-comment-newline = *( wschar / [ comment ] newline ) // ws-comment-newline = *( wschar / [ comment ] newline )
b = b[1:] b = b[1:]
parent := p.builder.Push(ast.Node{ parent := p.builder.Push(ast.Node{
@@ -352,6 +390,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
}) })
first := true first := true
var lastChild ast.Reference var lastChild ast.Reference
var err error var err error
@@ -362,17 +401,20 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
} }
if len(b) == 0 { if len(b) == 0 {
//nolint:godox
return parent, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF return parent, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF
} }
if b[0] == ']' { if b[0] == ']' {
break break
} }
if b[0] == ',' { if b[0] == ',' {
if first { if first {
return parent, nil, fmt.Errorf("array cannot start with comma") return parent, nil, errArrayCanNotStartWithComma
} }
b = b[1:] b = b[1:]
b, err = p.parseOptionalWhitespaceCommentNewline(b) b, err = p.parseOptionalWhitespaceCommentNewline(b)
if err != nil { if err != nil {
return parent, nil, err return parent, nil, err
@@ -385,6 +427,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
} }
var valueRef ast.Reference var valueRef ast.Reference
valueRef, b, err = p.parseVal(b) valueRef, b, err = p.parseVal(b)
if err != nil { if err != nil {
return parent, nil, err return parent, nil, err
@@ -392,7 +435,6 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
if first { if first {
p.builder.AttachChild(parent, valueRef) p.builder.AttachChild(parent, valueRef)
first = false
} else { } else {
p.builder.Chain(lastChild, valueRef) p.builder.Chain(lastChild, valueRef)
} }
@@ -406,6 +448,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) {
} }
rest, err := expect(']', b) rest, err := expect(']', b)
return parent, rest, err return parent, rest, err
} }
@@ -413,15 +456,18 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
for len(b) > 0 { for len(b) > 0 {
var err error var err error
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
if len(b) > 0 && b[0] == '#' { if len(b) > 0 && b[0] == '#' {
_, b, err = scanComment(b) _, b, err = scanComment(b)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
if len(b) == 0 { if len(b) == 0 {
break break
} }
if b[0] == '\n' || b[0] == '\r' { if b[0] == '\n' || b[0] == '\r' {
b, err = p.parseNewline(b) b, err = p.parseNewline(b)
if err != nil { if err != nil {
@@ -431,6 +477,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error)
break break
} }
} }
return b, nil return b, nil
} }
@@ -448,9 +495,13 @@ func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, error) {
} else if token[i] == '\r' && token[i+1] == '\n' { } else if token[i] == '\r' && token[i+1] == '\n' {
i += 2 i += 2
} }
return token[i : len(token)-3], rest, err return token[i : len(token)-3], rest, err
} }
var errInvalidEscapeChar = errors.New("invalid escaped character")
//nolint:funlen,gocognit,cyclop
func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) { func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body // ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
// ml-basic-string-delim // ml-basic-string-delim
@@ -462,11 +513,11 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
// mlb-quotes = 1*2quotation-mark // mlb-quotes = 1*2quotation-mark
// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii // mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
// mlb-escaped-nl = escape ws newline *( wschar / newline ) // mlb-escaped-nl = escape ws newline *( wschar / newline )
token, rest, err := scanMultilineBasicString(b) token, rest, err := scanMultilineBasicString(b)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
var builder bytes.Buffer var builder bytes.Buffer
i := 3 i := 3
@@ -482,6 +533,8 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
// escapes are balanced. // escapes are balanced.
for ; i < len(token)-3; i++ { for ; i < len(token)-3; i++ {
c := token[i] c := token[i]
//nolint:nestif
if c == '\\' { if c == '\\' {
// When the last non-whitespace character on a line is an unescaped \, // When the last non-whitespace character on a line is an unescaped \,
// it will be trimmed along with all whitespace (including newlines) up // it will be trimmed along with all whitespace (including newlines) up
@@ -492,15 +545,18 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
c := token[i] c := token[i]
if !(c == '\n' || c == '\r' || c == ' ' || c == '\t') { if !(c == '\n' || c == '\r' || c == ' ' || c == '\t') {
i-- i--
break break
} }
} }
continue continue
} }
// handle escaping // handle escaping
i++ i++
c = token[i] c = token[i]
switch c { switch c {
case '"', '\\': case '"', '\\':
builder.WriteByte(c) builder.WriteByte(c)
@@ -519,6 +575,7 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
builder.WriteString(x) builder.WriteString(x)
i += 4 i += 4
case 'U': case 'U':
@@ -526,10 +583,11 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
builder.WriteString(x) builder.WriteString(x)
i += 8 i += 8
default: default:
return nil, nil, fmt.Errorf("invalid escaped character: %#U", c) return nil, nil, fmt.Errorf("parseMultilineBasicString: %w - %#U", errInvalidEscapeChar, c)
} }
} else { } else {
builder.WriteByte(c) builder.WriteByte(c)
@@ -548,7 +606,6 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
// dotted-key = simple-key 1*( dot-sep simple-key ) // dotted-key = simple-key 1*( dot-sep simple-key )
// //
// dot-sep = ws %x2E ws ; . Period // dot-sep = ws %x2E ws ; . Period
key, b, err := p.parseSimpleKey(b) key, b, err := p.parseSimpleKey(b)
if err != nil { if err != nil {
return ast.Reference{}, nil, err return ast.Reference{}, nil, err
@@ -566,11 +623,14 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) {
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
} }
b = p.parseWhitespace(b) b = p.parseWhitespace(b)
key, b, err = p.parseSimpleKey(b) key, b, err = p.parseSimpleKey(b)
if err != nil { if err != nil {
return ref, nil, err return ref, nil, err
} }
p.builder.PushAndChain(ast.Node{ p.builder.PushAndChain(ast.Node{
Kind: ast.Key, Kind: ast.Key,
Data: key, Data: key,
@@ -587,23 +647,25 @@ func (p *parser) parseSimpleKey(b []byte) (key, rest []byte, err error) {
// simple-key = quoted-key / unquoted-key // simple-key = quoted-key / unquoted-key
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _ // unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
// quoted-key = basic-string / literal-string // quoted-key = basic-string / literal-string
if len(b) == 0 { if len(b) == 0 {
//nolint:godox
return nil, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF return nil, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF
} }
if b[0] == '\'' { switch {
key, rest, err = p.parseLiteralString(b) case b[0] == '\'':
} else if b[0] == '"' { return p.parseLiteralString(b)
key, rest, err = p.parseBasicString(b) case b[0] == '"':
} else if isUnquotedKeyChar(b[0]) { return p.parseBasicString(b)
key, rest, err = scanUnquotedKey(b) case isUnquotedKeyChar(b[0]):
} else { return scanUnquotedKey(b)
err = unexpectedCharacter{b: b} // TODO: should contain expected characters default:
//nolint:godox
return nil, nil, unexpectedCharacter{b: b} // TODO: should be unexpected EOF
} }
return
} }
//nolint:funlen,cyclop
func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) { func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
// basic-string = quotation-mark *basic-char quotation-mark // basic-string = quotation-mark *basic-char quotation-mark
// quotation-mark = %x22 ; " // quotation-mark = %x22 ; "
@@ -619,11 +681,11 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
// escape-seq-char =/ %x74 ; t tab U+0009 // escape-seq-char =/ %x74 ; t tab U+0009
// escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX // escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX // escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
token, rest, err := scanBasicString(b) token, rest, err := scanBasicString(b)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
var builder bytes.Buffer var builder bytes.Buffer
// The scanner ensures that the token starts and ends with quotes and that // The scanner ensures that the token starts and ends with quotes and that
@@ -633,6 +695,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if c == '\\' { if c == '\\' {
i++ i++
c = token[i] c = token[i]
switch c { switch c {
case '"', '\\': case '"', '\\':
builder.WriteByte(c) builder.WriteByte(c)
@@ -651,6 +714,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
builder.WriteString(x) builder.WriteString(x)
i += 4 i += 4
case 'U': case 'U':
@@ -658,10 +722,11 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
builder.WriteString(x) builder.WriteString(x)
i += 8 i += 8
default: default:
return nil, nil, fmt.Errorf("invalid escaped character: %#U", c) return nil, nil, fmt.Errorf("parseBasicString: %w - %#U", errInvalidEscapeChar, c)
} }
} else { } else {
builder.WriteByte(c) builder.WriteByte(c)
@@ -671,15 +736,20 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, error) {
return builder.Bytes(), rest, nil return builder.Bytes(), rest, nil
} }
var errUnicodePointNeedsRightCountChar = errors.New("unicode point needs right number of hex characters")
func hexToString(b []byte, length int) (string, error) { func hexToString(b []byte, length int) (string, error) {
if len(b) < length { if len(b) < length {
return "", fmt.Errorf("unicode point needs %d hex characters", length) return "", fmt.Errorf("hexToString: %w - %d", errUnicodePointNeedsRightCountChar, length)
} }
//nolint:godox
// TODO: slow // TODO: slow
intcode, err := strconv.ParseInt(string(b[:length]), 16, 32) intcode, err := strconv.ParseInt(string(b[:length]), 16, 32)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("hexToString: %w", err)
} }
return string(rune(intcode)), nil return string(rune(intcode)), nil
} }
@@ -687,25 +757,33 @@ func (p *parser) parseWhitespace(b []byte) []byte {
// ws = *wschar // ws = *wschar
// wschar = %x20 ; Space // wschar = %x20 ; Space
// wschar =/ %x09 ; Horizontal tab // wschar =/ %x09 ; Horizontal tab
_, rest := scanWhitespace(b) _, rest := scanWhitespace(b)
return rest return rest
} }
var (
errExpectedInf = errors.New("expected 'inf'")
errExpectedNan = errors.New("expected 'nan'")
)
//nolint:cyclop
func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, error) { func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, error) {
switch b[0] { switch b[0] {
case 'i': case 'i':
if !scanFollowsInf(b) { if !scanFollowsInf(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'inf'") return ast.Reference{}, nil, errExpectedInf
} }
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:3], Data: b[:3],
}), b[3:], nil }), b[3:], nil
case 'n': case 'n':
if !scanFollowsNan(b) { if !scanFollowsNan(b) {
return ast.Reference{}, nil, fmt.Errorf("expected 'nan'") return ast.Reference{}, nil, errExpectedNan
} }
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
Kind: ast.Float, Kind: ast.Float,
Data: b[:3], Data: b[:3],
@@ -714,60 +792,73 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
//nolint:gomnd
if len(b) < 3 { if len(b) < 3 {
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
s := 5 s := 5
if len(b) < s { if len(b) < s {
s = len(b) s = len(b)
} }
for idx, c := range b[:s] { for idx, c := range b[:s] {
if isDigit(c) { if isDigit(c) {
continue continue
} }
if idx == 2 && c == ':' || (idx == 4 && c == '-') { if idx == 2 && c == ':' || (idx == 4 && c == '-') {
return p.scanDateTime(b) return p.scanDateTime(b)
} }
} }
return p.scanIntOrFloat(b) return p.scanIntOrFloat(b)
} }
func digitsToInt(b []byte) int { func digitsToInt(b []byte) int {
x := 0 x := 0
for _, d := range b { for _, d := range b {
x *= 10 x *= 10
x += int(d - '0') x += int(d - '0')
} }
return x return x
} }
var errTimezoneButNoTimeComponent = errors.New("possible DateTime cannot have a timezone but no time component")
//nolint:gocognit,cyclop
func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) { func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
// scans for contiguous characters in [0-9T:Z.+-], and up to one space if // scans for contiguous characters in [0-9T:Z.+-], and up to one space if
// followed by a digit. // followed by a digit.
hasTime := false hasTime := false
hasTz := false hasTz := false
seenSpace := false seenSpace := false
i := 0 i := 0
byteLoop:
for ; i < len(b); i++ { for ; i < len(b); i++ {
c := b[i] c := b[i]
if isDigit(c) || c == '-' {
} else if c == 'T' || c == ':' || c == '.' { switch {
case isDigit(c) || c == '-':
case c == 'T' || c == ':' || c == '.':
hasTime = true hasTime = true
continue
} else if c == '+' || c == '-' || c == 'Z' { continue byteLoop
case c == '+' || c == '-' || c == 'Z':
hasTz = true hasTz = true
} else if c == ' ' { case c == ' ':
if !seenSpace && i+1 < len(b) && isDigit(b[i+1]) { if !seenSpace && i+1 < len(b) && isDigit(b[i+1]) {
i += 2 i += 2
seenSpace = true seenSpace = true
hasTime = true hasTime = true
} else { } else {
break break byteLoop
} }
} else { default:
break break byteLoop
} }
} }
@@ -781,7 +872,7 @@ func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
} }
} else { } else {
if hasTz { if hasTz {
return ast.Reference{}, nil, fmt.Errorf("possible DateTime cannot have a timezone but no time component") return ast.Reference{}, nil, errTimezoneButNoTimeComponent
} }
kind = ast.LocalDate kind = ast.LocalDate
} }
@@ -792,11 +883,19 @@ func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) {
}), b[i:], nil }), b[i:], nil
} }
var (
errUnexpectedCharI = fmt.Errorf("unexpected character i while scanning for a number")
errUnexpectedCharN = fmt.Errorf("unexpected character n while scanning for a number")
errExpectedIntOrFloat = fmt.Errorf("expected integer or float")
)
//nolint:funlen,gocognit,cyclop
func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) { func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
i := 0 i := 0
if len(b) > 2 && b[0] == '0' && b[1] != '.' { if len(b) > 2 && b[0] == '0' && b[1] != '.' {
var isValidRune validRuneFn var isValidRune validRuneFn
switch b[1] { switch b[1] {
case 'x': case 'x':
isValidRune = isValidHexRune isValidRune = isValidHexRune
@@ -834,6 +933,7 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
if c == '.' || c == 'e' || c == 'E' { if c == '.' || c == 'e' || c == 'E' {
isFloat = true isFloat = true
continue continue
} }
@@ -844,8 +944,10 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
Data: b[:i+3], Data: b[:i+3],
}), b[i+3:], nil }), b[i+3:], nil
} }
return ast.Reference{}, nil, fmt.Errorf("unexpected character i while scanning for a number")
return ast.Reference{}, nil, errUnexpectedCharI
} }
if c == 'n' { if c == 'n' {
if scanFollowsNan(b[i:]) { if scanFollowsNan(b[i:]) {
return p.builder.Push(ast.Node{ return p.builder.Push(ast.Node{
@@ -853,14 +955,15 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) {
Data: b[:i+3], Data: b[:i+3],
}), b[i+3:], nil }), b[i+3:], nil
} }
return ast.Reference{}, nil, fmt.Errorf("unexpected character n while scanning for a number")
return ast.Reference{}, nil, errUnexpectedCharN
} }
break break
} }
if i == 0 { if i == 0 {
return ast.Reference{}, b, fmt.Errorf("expected integer or float") return ast.Reference{}, b, errExpectedIntOrFloat
} }
kind := ast.Integer kind := ast.Integer
@@ -900,9 +1003,11 @@ func expect(x byte, b []byte) ([]byte, error) {
if len(b) == 0 { if len(b) == 0 {
return nil, newDecodeError(b[:0], "expecting %#U", x) return nil, newDecodeError(b[:0], "expecting %#U", x)
} }
if b[0] != x { if b[0] != x {
return nil, newDecodeError(b[0:1], "expected character %U", x) return nil, newDecodeError(b[0:1], "expected character %U", x)
} }
return b[1:], nil return b[1:], nil
} }
@@ -914,7 +1019,7 @@ type unexpectedCharacter struct {
func (u unexpectedCharacter) Error() string { func (u unexpectedCharacter) Error() string {
if len(u.b) == 0 { if len(u.b) == 0 {
return fmt.Sprintf("expected %#U, not EOF", u.r) return fmt.Sprintf("expected %#U, not EOF", u.r)
} }
return fmt.Sprintf("expected %#U, not %#U", u.r, u.b[0]) return fmt.Sprintf("expected %#U, not %#U", u.r, u.b[0])
} }
+10 -54
View File
@@ -7,7 +7,10 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
//nolint:funlen
func TestParser_AST_Numbers(t *testing.T) { func TestParser_AST_Numbers(t *testing.T) {
t.Parallel()
examples := []struct { examples := []struct {
desc string desc string
input string input string
@@ -132,7 +135,9 @@ func TestParser_AST_Numbers(t *testing.T) {
} }
for _, e := range examples { for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) { t.Run(e.desc, func(t *testing.T) {
t.Parallel()
p := parser{} p := parser{}
p.Reset([]byte(`A = ` + e.input)) p.Reset([]byte(`A = ` + e.input))
p.NextExpression() p.NextExpression()
@@ -156,7 +161,6 @@ func TestParser_AST_Numbers(t *testing.T) {
} }
type ( type (
astRoot []astNode
astNode struct { astNode struct {
Kind ast.Kind Kind ast.Kind
Data []byte Data []byte
@@ -164,11 +168,6 @@ type (
} }
) )
func compareAST(t *testing.T, expected astRoot, actual *ast.Root) {
it := actual.Iterator()
compareIterator(t, expected, it)
}
func compareNode(t *testing.T, e astNode, n ast.Node) { func compareNode(t *testing.T, e astNode, n ast.Node) {
t.Helper() t.Helper()
require.Equal(t, e.Kind, n.Kind) require.Equal(t, e.Kind, n.Kind)
@@ -199,55 +198,10 @@ func compareIterator(t *testing.T, expected []astNode, actual ast.Iterator) {
} }
} }
func (r astRoot) toOrig() *ast.Root { //nolint:funlen
builder := &ast.Builder{}
var last ast.Reference
for i, n := range r {
ref := builder.Push(ast.Node{
Kind: n.Kind,
Data: n.Data,
})
if i > 0 {
builder.Chain(last, ref)
}
last = ref
if len(n.Children) > 0 {
c := childrenToOrig(builder, n.Children)
builder.AttachChild(ref, c)
}
}
return builder.Tree()
}
func childrenToOrig(b *ast.Builder, nodes []astNode) ast.Reference {
var first ast.Reference
var last ast.Reference
for i, n := range nodes {
ref := b.Push(ast.Node{
Kind: n.Kind,
Data: n.Data,
})
if i == 0 {
first = ref
} else {
b.Chain(last, ref)
}
last = ref
if len(n.Children) > 0 {
c := childrenToOrig(b, n.Children)
b.AttachChild(ref, c)
}
}
return first
}
func TestParser_AST(t *testing.T) { func TestParser_AST(t *testing.T) {
t.Parallel()
examples := []struct { examples := []struct {
desc string desc string
input string input string
@@ -384,7 +338,9 @@ func TestParser_AST(t *testing.T) {
} }
for _, e := range examples { for _, e := range examples {
e := e
t.Run(e.desc, func(t *testing.T) { t.Run(e.desc, func(t *testing.T) {
t.Parallel()
p := parser{} p := parser{}
p.Reset([]byte(e.input)) p.Reset([]byte(e.input))
p.NextExpression() p.NextExpression()