ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'toml-0.3.1'

Browse Source
This commit is contained in:
Thomas Pelletier
2014-12-06 15:27:39 +01:00
parent b814e1a94f a56707c85f
commit 36d65b681a
5 changed files with 175 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer.go
+42 -7
View File
@@ -138,7 +138,7 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
return l.lexRvalue
}
if isKeyChar(next) {
if isKeyStartChar(next) {
return l.lexKey
}
@@ -169,6 +169,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
return l.lexComment
case '"':
return l.lexString
case '\'':
return l.lexLiteralString
case ',':
return l.lexComma
case '\n':
@@ -192,7 +194,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
return l.lexKey
}
if dateRegexp.FindString(l.input[l.pos:]) != "" {
dateMatch := dateRegexp.FindString(l.input[l.pos:])
if dateMatch != "" {
l.ignore()
l.pos += len(dateMatch)
return l.lexDate
}
@@ -214,8 +219,6 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
}
func (l *tomlLexer) lexDate() tomlLexStateFn {
l.ignore()
l.pos += 20 // Fixed size of a date in TOML
l.emit(tokenDate)
return l.lexRvalue
}
@@ -250,7 +253,10 @@ func (l *tomlLexer) lexComma() tomlLexStateFn {
func (l *tomlLexer) lexKey() tomlLexStateFn {
l.ignore()
for isKeyChar(l.next()) {
for r := l.next(); isKeyChar(r); r = l.next() {
if (r == '#') {
return l.errorf("keys cannot contain # character")
}
}
l.backup()
l.emit(tokenKey)
@@ -275,6 +281,29 @@ func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
return l.lexRvalue
}
func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
l.pos++
l.ignore()
growingString := ""
for {
if l.peek() == '\'' {
l.emitWithValue(tokenString, growingString)
l.pos++
l.ignore()
return l.lexRvalue
}
growingString += string(l.peek())
if l.next() == eof {
break
}
}
return l.errorf("unclosed string")
}
func (l *tomlLexer) lexString() tomlLexStateFn {
l.pos++
l.ignore()
@@ -418,6 +447,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
l.accept("-")
}
pointSeen := false
expSeen := false
digitSeen := false
for {
next := l.next()
@@ -429,6 +459,11 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
return l.errorf("float cannot end with a dot")
}
pointSeen = true
} else if next == 'e' || next == 'E' {
expSeen = true
if !l.accept("+") {
l.accept("-")
}
} else if isDigit(next) {
digitSeen = true
} else {
@@ -443,7 +478,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
if !digitSeen {
return l.errorf("no digit in that number")
}
if pointSeen {
if pointSeen || expSeen {
l.emit(tokenFloat)
} else {
l.emit(tokenInteger)
@@ -452,7 +487,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
}
func init() {
dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z")
dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})")
}
// Entry point
lexer_test.go
+97 -10
View File
@@ -118,19 +118,16 @@ func TestBasicKeyAndEqual(t *testing.T) {
func TestKeyWithSharpAndEqual(t *testing.T) {
testFlow(t, "key#name = 5", []token{
token{Position{1, 1}, tokenKey, "key#name"},
token{Position{1, 10}, tokenEqual, "="},
token{Position{1, 12}, tokenInteger, "5"},
token{Position{1, 13}, tokenEOF, ""},
token{Position{1, 1}, tokenError, "keys cannot contain # character"},
})
}
func TestKeyWithSymbolsAndEqual(t *testing.T) {
testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
token{Position{1, 1}, tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
token{Position{1, 39}, tokenEqual, "="},
token{Position{1, 41}, tokenInteger, "5"},
token{Position{1, 42}, tokenEOF, ""},
testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"},
token{Position{1, 38}, tokenEqual, "="},
token{Position{1, 40}, tokenInteger, "5"},
token{Position{1, 41}, tokenEOF, ""},
})
}
@@ -276,7 +273,13 @@ func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
func TestDateRegexp(t *testing.T) {
if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
t.Fail()
t.Error("basic lexing")
}
if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
t.Error("offset lexing")
}
if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
t.Error("nano precision lexing")
}
}
@@ -287,6 +290,18 @@ func TestKeyEqualDate(t *testing.T) {
token{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
token{Position{1, 27}, tokenEOF, ""},
})
testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
token{Position{1, 32}, tokenEOF, ""},
})
testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
token{Position{1, 39}, tokenEOF, ""},
})
}
func TestFloatEndingWithDot(t *testing.T) {
@@ -305,6 +320,51 @@ func TestFloatWithTwoDots(t *testing.T) {
})
}
func TestFloatWithExponent1(t *testing.T) {
testFlow(t, "a = 5e+22", []token{
token{Position{1, 1}, tokenKey, "a"},
token{Position{1, 3}, tokenEqual, "="},
token{Position{1, 5}, tokenFloat, "5e+22"},
token{Position{1, 10}, tokenEOF, ""},
})
}
func TestFloatWithExponent2(t *testing.T) {
testFlow(t, "a = 5E+22", []token{
token{Position{1, 1}, tokenKey, "a"},
token{Position{1, 3}, tokenEqual, "="},
token{Position{1, 5}, tokenFloat, "5E+22"},
token{Position{1, 10}, tokenEOF, ""},
})
}
func TestFloatWithExponent3(t *testing.T) {
testFlow(t, "a = -5e+22", []token{
token{Position{1, 1}, tokenKey, "a"},
token{Position{1, 3}, tokenEqual, "="},
token{Position{1, 5}, tokenFloat, "-5e+22"},
token{Position{1, 11}, tokenEOF, ""},
})
}
func TestFloatWithExponent4(t *testing.T) {
testFlow(t, "a = -5e-22", []token{
token{Position{1, 1}, tokenKey, "a"},
token{Position{1, 3}, tokenEqual, "="},
token{Position{1, 5}, tokenFloat, "-5e-22"},
token{Position{1, 11}, tokenEOF, ""},
})
}
func TestFloatWithExponent5(t *testing.T) {
testFlow(t, "a = 6.626e-34", []token{
token{Position{1, 1}, tokenKey, "a"},
token{Position{1, 3}, tokenEqual, "="},
token{Position{1, 5}, tokenFloat, "6.626e-34"},
token{Position{1, 14}, tokenEOF, ""},
})
}
func TestDoubleEqualKey(t *testing.T) {
testFlow(t, "foo= = 2", []token{
token{Position{1, 1}, tokenKey, "foo"},
@@ -400,6 +460,33 @@ func TestKeyEqualStringUnicodeEscape(t *testing.T) {
})
}
func TestLiteralString(t *testing.T) {
testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
token{Position{1, 34}, tokenEOF, ""},
})
testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
token{Position{1, 35}, tokenEOF, ""},
})
testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
token{Position{1, 34}, tokenEOF, ""},
})
testFlow(t, `foo = '<\i\c*\s*>'`, []token{
token{Position{1, 1}, tokenKey, "foo"},
token{Position{1, 5}, tokenEqual, "="},
token{Position{1, 8}, tokenString, `<\i\c*\s*>`},
token{Position{1, 19}, tokenEOF, ""},
})
}
func TestUnicodeString(t *testing.T) {
testFlow(t, `foo = "hello ♥ world"`, []token{
token{Position{1, 1}, tokenKey, "foo"},
parser.go
+1 -1
View File
@@ -222,7 +222,7 @@ func (p *tomlParser) parseRvalue() interface{} {
}
return val
case tokenDate:
val, err := time.Parse(time.RFC3339, tok.val)
val, err := time.Parse(time.RFC3339Nano, tok.val)
if err != nil {
p.raiseError(tok, "%s", err)
}
parser_test.go
+28 -2
View File
@@ -54,9 +54,9 @@ func TestSimpleKV(t *testing.T) {
// NOTE: from the BurntSushi test suite
// NOTE: this test is pure evil due to the embedded '.'
func TestSpecialKV(t *testing.T) {
tree, err := Load("~!@#$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
assertTree(t, tree, err, map[string]interface{}{
"~!@#$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
"~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
})
}
@@ -70,6 +70,17 @@ func TestSimpleNumbers(t *testing.T) {
})
}
func TestFloatsWithExponents(t *testing.T) {
tree, err := Load("a = 5e+22\nb = 5E+22\nc = -5e+22\nd = -5e-22\ne = 6.626e-34")
assertTree(t, tree, err, map[string]interface{}{
"a": float64(5e+22),
"b": float64(5E+22),
"c": float64(-5e+22),
"d": float64(-5e-22),
"e": float64(6.626e-34),
})
}
func TestSimpleDate(t *testing.T) {
tree, err := Load("a = 1979-05-27T07:32:00Z")
assertTree(t, tree, err, map[string]interface{}{
@@ -77,6 +88,21 @@ func TestSimpleDate(t *testing.T) {
})
}
func TestDateOffset(t *testing.T) {
tree, err := Load("a = 1979-05-27T00:32:00-07:00")
assertTree(t, tree, err, map[string]interface{}{
"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7 * 60 * 60)),
})
}
func TestDateNano(t *testing.T) {
tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
assertTree(t, tree, err, map[string]interface{}{
"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7 * 60 * 60)),
})
}
func TestSimpleString(t *testing.T) {
tree, err := Load("a = \"hello world\"")
assertTree(t, tree, err, map[string]interface{}{
token.go
+7 -2
View File
@@ -117,11 +117,16 @@ func isAlphanumeric(r rune) bool {
}
func isKeyChar(r rune) bool {
// "Keys start with the first non-whitespace character and end with the last
// non-whitespace character before the equals sign."
// Keys start with the first character that isn't whitespace or [ and end
// with the last non-whitespace character before the equals sign. Keys
// cannot contain a # character."
return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
}
func isKeyStartChar(r rune) bool {
return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
}
func isDigit(r rune) bool {
return unicode.IsNumber(r)
}