ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'eanderton-master'

Browse Source
This commit is contained in:
Thomas Pelletier
2014-08-07 12:53:56 +02:00
parent e493544dfd bf549a2194
commit 68d2a60b37
5 changed files with 279 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer.go
+33 -5
View File
@@ -46,6 +46,8 @@ const (
type token struct { type token struct {
typ tokenType typ tokenType
val string val string
line int
col int
} }
func (i token) String() string { func (i token) String() string {
@@ -62,6 +64,10 @@ func (i token) String() string {
return fmt.Sprintf("%q", i.val) return fmt.Sprintf("%q", i.val)
} }
func (i token) Pos() string {
return fmt.Sprintf("(%d, %d)", i.line+1, i.col+1)
}
func isSpace(r rune) bool { func isSpace(r rune) bool {
return r == ' ' || r == '\t' return r == ' ' || r == '\t'
} }
@@ -93,6 +99,8 @@ type lexer struct {
width int width int
tokens chan token tokens chan token
depth int depth int
line int
col int
} }
func (l *lexer) run() { func (l *lexer) run() {
@@ -102,14 +110,32 @@ func (l *lexer) run() {
close(l.tokens) close(l.tokens)
} }
func (l *lexer) emit(t tokenType) { func (l *lexer) nextStart() {
l.tokens <- token{t, l.input[l.start:l.pos]} // iterate by runes (utf8 characters)
// search for newlines and advance line/col counts
for i := l.start; i < l.pos; {
r, width := utf8.DecodeRuneInString(l.input[i:])
if r == '\n' {
l.line += 1
l.col = 0
} else {
l.col += 1
}
i += width
// fmt.Printf("'%c'\n", r)
}
// advance start position to next token
l.start = l.pos l.start = l.pos
} }
func (l *lexer) emit(t tokenType) {
l.tokens <- token{t, l.input[l.start:l.pos], l.line, l.col}
l.nextStart()
}
func (l *lexer) emitWithValue(t tokenType, value string) { func (l *lexer) emitWithValue(t tokenType, value string) {
l.tokens <- token{t, value} l.tokens <- token{t, value, l.line, l.col}
l.start = l.pos l.nextStart()
} }
func (l *lexer) next() rune { func (l *lexer) next() rune {
@@ -124,7 +150,7 @@ func (l *lexer) next() rune {
} }
func (l *lexer) ignore() { func (l *lexer) ignore() {
l.start = l.pos l.nextStart()
} }
func (l *lexer) backup() { func (l *lexer) backup() {
@@ -135,6 +161,8 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn {
l.tokens <- token{ l.tokens <- token{
tokenError, tokenError,
fmt.Sprintf(format, args...), fmt.Sprintf(format, args...),
l.line,
l.col,
} }
return nil return nil
} }
lexer_test.go
+203 -199
View File
@@ -7,9 +7,12 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
for _, expected := range expectedFlow { for _, expected := range expectedFlow {
token := <-ch token := <-ch
if token != expected { if token != expected {
t.Log("While testing: ", input)
t.Log("compared", token, "to", expected) t.Log("compared", token, "to", expected)
t.Log(token.val, "<->", expected.val) t.Log(token.val, "<->", expected.val)
t.Log(token.typ, "<->", expected.typ) t.Log(token.typ, "<->", expected.typ)
t.Log(token.line, "<->", expected.line)
t.Log(token.col, "<->", expected.col)
t.FailNow() t.FailNow()
} }
} }
@@ -29,244 +32,245 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
func TestValidKeyGroup(t *testing.T) { func TestValidKeyGroup(t *testing.T) {
testFlow(t, "[hello world]", []token{ testFlow(t, "[hello world]", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 13},
}) })
} }
func TestUnclosedKeyGroup(t *testing.T) { func TestUnclosedKeyGroup(t *testing.T) {
testFlow(t, "[hello world", []token{ testFlow(t, "[hello world", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenError, "unclosed key group"}, token{tokenError, "unclosed key group", 0, 1},
}) })
} }
func TestComment(t *testing.T) { func TestComment(t *testing.T) {
testFlow(t, "# blahblah", []token{ testFlow(t, "# blahblah", []token{
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestKeyGroupComment(t *testing.T) { func TestKeyGroupComment(t *testing.T) {
testFlow(t, "[hello world] # blahblah", []token{ testFlow(t, "[hello world] # blahblah", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 24},
}) })
} }
func TestMultipleKeyGroupsComment(t *testing.T) { func TestMultipleKeyGroupsComment(t *testing.T) {
testFlow(t, "[hello world] # blahblah\n[test]", []token{ testFlow(t, "[hello world] # blahblah\n[test]", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 1, 0},
token{tokenKeyGroup, "test"}, token{tokenKeyGroup, "test", 1, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 1, 5},
token{tokenEOF, ""}, token{tokenEOF, "", 1, 6},
}) })
} }
func TestBasicKey(t *testing.T) { func TestBasicKey(t *testing.T) {
testFlow(t, "hello", []token{ testFlow(t, "hello", []token{
token{tokenKey, "hello"}, token{tokenKey, "hello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 5},
}) })
} }
func TestBasicKeyWithUnderscore(t *testing.T) { func TestBasicKeyWithUnderscore(t *testing.T) {
testFlow(t, "hello_hello", []token{ testFlow(t, "hello_hello", []token{
token{tokenKey, "hello_hello"}, token{tokenKey, "hello_hello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestBasicKeyWithDash(t *testing.T) { func TestBasicKeyWithDash(t *testing.T) {
testFlow(t, "hello-world", []token{ testFlow(t, "hello-world", []token{
token{tokenKey, "hello-world"}, token{tokenKey, "hello-world", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestBasicKeyWithUppercaseMix(t *testing.T) { func TestBasicKeyWithUppercaseMix(t *testing.T) {
testFlow(t, "helloHELLOHello", []token{ testFlow(t, "helloHELLOHello", []token{
token{tokenKey, "helloHELLOHello"}, token{tokenKey, "helloHELLOHello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 15},
}) })
} }
func TestBasicKeyWithInternationalCharacters(t *testing.T) { func TestBasicKeyWithInternationalCharacters(t *testing.T) {
testFlow(t, "héllÖ", []token{ testFlow(t, "héllÖ", []token{
token{tokenKey, "héllÖ"}, token{tokenKey, "héllÖ", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 5},
}) })
} }
func TestBasicKeyAndEqual(t *testing.T) { func TestBasicKeyAndEqual(t *testing.T) {
testFlow(t, "hello =", []token{ testFlow(t, "hello =", []token{
token{tokenKey, "hello"}, token{tokenKey, "hello", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 7},
}) })
} }
func TestKeyWithSharpAndEqual(t *testing.T) { func TestKeyWithSharpAndEqual(t *testing.T) {
testFlow(t, "key#name = 5", []token{ testFlow(t, "key#name = 5", []token{
token{tokenKey, "key#name"}, token{tokenKey, "key#name", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 9},
token{tokenInteger, "5"}, token{tokenInteger, "5", 0, 11},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 12},
}) })
} }
func TestKeyWithSymbolsAndEqual(t *testing.T) { func TestKeyWithSymbolsAndEqual(t *testing.T) {
testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"}, token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 38},
token{tokenInteger, "5"}, token{tokenInteger, "5", 0, 40},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 41},
}) })
} }
func TestKeyEqualStringEscape(t *testing.T) { func TestKeyEqualStringEscape(t *testing.T) {
testFlow(t, "foo = \"hello\\\"\"", []token{ testFlow(t, `foo = "hello\""`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello\""}, token{tokenString, "hello\"", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 15},
}) })
} }
func TestKeyEqualStringUnfinished(t *testing.T) { func TestKeyEqualStringUnfinished(t *testing.T) {
testFlow(t, "foo = \"bar", []token{ testFlow(t, `foo = "bar`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "unclosed string"}, token{tokenError, "unclosed string", 0, 7},
}) })
} }
func TestKeyEqualString(t *testing.T) { func TestKeyEqualString(t *testing.T) {
testFlow(t, "foo = \"bar\"", []token{ testFlow(t, `foo = "bar"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "bar"}, token{tokenString, "bar", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestKeyEqualTrue(t *testing.T) { func TestKeyEqualTrue(t *testing.T) {
testFlow(t, "foo = true", []token{ testFlow(t, "foo = true", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestKeyEqualFalse(t *testing.T) { func TestKeyEqualFalse(t *testing.T) {
testFlow(t, "foo = false", []token{ testFlow(t, "foo = false", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestArrayNestedString(t *testing.T) { func TestArrayNestedString(t *testing.T) {
testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{ testFlow(t, `a = [ ["hello", "world"] ]`, []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenString, "hello"}, token{tokenString, "hello", 0, 8},
token{tokenComma, ","}, token{tokenComma, ",", 0, 14},
token{tokenString, "world"}, token{tokenString, "world", 0, 17},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 23},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 25},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 26},
}) })
} }
func TestArrayNestedInts(t *testing.T) { func TestArrayNestedInts(t *testing.T) {
testFlow(t, "a = [ [42, 21], [10] ]", []token{ testFlow(t, "a = [ [42, 21], [10] ]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 9},
token{tokenInteger, "21"}, token{tokenInteger, "21", 0, 11},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 14},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 16},
token{tokenInteger, "10"}, token{tokenInteger, "10", 0, 17},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 19},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 21},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 22},
}) })
} }
func TestArrayInts(t *testing.T) { func TestArrayInts(t *testing.T) {
testFlow(t, "a = [ 42, 21, 10, ]", []token{ testFlow(t, "a = [ 42, 21, 10, ]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenComma, ","}, token{tokenComma, ",", 0, 8},
token{tokenInteger, "21"}, token{tokenInteger, "21", 0, 10},
token{tokenComma, ","}, token{tokenComma, ",", 0, 12},
token{tokenInteger, "10"}, token{tokenInteger, "10", 0, 14},
token{tokenComma, ","}, token{tokenComma, ",", 0, 16},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 18},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 19},
}) })
} }
func TestMultilineArrayComments(t *testing.T) { func TestMultilineArrayComments(t *testing.T) {
testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{ testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenInteger, "1"}, token{tokenInteger, "1", 0, 5},
token{tokenComma, ","}, token{tokenComma, ",", 0, 6},
token{tokenInteger, "2"}, token{tokenInteger, "2", 1, 0},
token{tokenComma, ","}, token{tokenComma, ",", 1, 1},
token{tokenInteger, "3"}, token{tokenInteger, "3", 2, 0},
token{tokenComma, ","}, token{tokenComma, ",", 2, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 3, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 3, 1},
}) })
} }
func TestKeyEqualArrayBools(t *testing.T) { func TestKeyEqualArrayBools(t *testing.T) {
testFlow(t, "foo = [true, false, true]", []token{ testFlow(t, "foo = [true, false, true]", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 11},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 18},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 20},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 24},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 25},
}) })
} }
func TestKeyEqualArrayBoolsWithComments(t *testing.T) { func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
testFlow(t, "foo = [true, false, true] # YEAH", []token{ testFlow(t, "foo = [true, false, true] # YEAH", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 11},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 18},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 20},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 24},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 32},
}) })
} }
@@ -278,138 +282,138 @@ func TestDateRegexp(t *testing.T) {
func TestKeyEqualDate(t *testing.T) { func TestKeyEqualDate(t *testing.T) {
testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{ testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenDate, "1979-05-27T07:32:00Z"}, token{tokenDate, "1979-05-27T07:32:00Z", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 26},
}) })
} }
func TestFloatEndingWithDot(t *testing.T) { func TestFloatEndingWithDot(t *testing.T) {
testFlow(t, "foo = 42.", []token{ testFlow(t, "foo = 42.", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "float cannot end with a dot"}, token{tokenError, "float cannot end with a dot", 0, 6},
}) })
} }
func TestFloatWithTwoDots(t *testing.T) { func TestFloatWithTwoDots(t *testing.T) {
testFlow(t, "foo = 4.2.", []token{ testFlow(t, "foo = 4.2.", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "cannot have two dots in one float"}, token{tokenError, "cannot have two dots in one float", 0, 6},
}) })
} }
func TestDoubleEqualKey(t *testing.T) { func TestDoubleEqualKey(t *testing.T) {
testFlow(t, "foo= = 2", []token{ testFlow(t, "foo= = 2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 3},
token{tokenError, "cannot have multiple equals for the same key"}, token{tokenError, "cannot have multiple equals for the same key", 0, 4},
}) })
} }
func TestInvalidEsquapeSequence(t *testing.T) { func TestInvalidEsquapeSequence(t *testing.T) {
testFlow(t, "foo = \"\\x\"", []token{ testFlow(t, `foo = "\x"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "invalid escape sequence: \\x"}, token{tokenError, "invalid escape sequence: \\x", 0, 7},
}) })
} }
func TestNestedArrays(t *testing.T) { func TestNestedArrays(t *testing.T) {
testFlow(t, "foo = [[[]]]", []token{ testFlow(t, "foo = [[[]]]", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 7},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 8},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 9},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 10},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 11},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 12},
}) })
} }
func TestKeyEqualNumber(t *testing.T) { func TestKeyEqualNumber(t *testing.T) {
testFlow(t, "foo = 42", []token{ testFlow(t, "foo = 42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 8},
}) })
testFlow(t, "foo = +42", []token{ testFlow(t, "foo = +42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "+42"}, token{tokenInteger, "+42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = -42", []token{ testFlow(t, "foo = -42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "-42"}, token{tokenInteger, "-42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = 4.2", []token{ testFlow(t, "foo = 4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "4.2"}, token{tokenFloat, "4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = +4.2", []token{ testFlow(t, "foo = +4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "+4.2"}, token{tokenFloat, "+4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
testFlow(t, "foo = -4.2", []token{ testFlow(t, "foo = -4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "-4.2"}, token{tokenFloat, "-4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestMultiline(t *testing.T) { func TestMultiline(t *testing.T) {
testFlow(t, "foo = 42\nbar=21", []token{ testFlow(t, "foo = 42\nbar=21", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenKey, "bar"}, token{tokenKey, "bar", 1, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 1, 3},
token{tokenInteger, "21"}, token{tokenInteger, "21", 1, 4},
token{tokenEOF, ""}, token{tokenEOF, "", 1, 6},
}) })
} }
func TestKeyEqualStringUnicodeEscape(t *testing.T) { func TestKeyEqualStringUnicodeEscape(t *testing.T) {
testFlow(t, "foo = \"hello \\u2665\"", []token{ testFlow(t, `foo = "hello \u2665"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello ♥"}, token{tokenString, "hello ♥", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 20},
}) })
} }
func TestUnicodeString(t *testing.T) { func TestUnicodeString(t *testing.T) {
testFlow(t, "foo = \"hello ♥ world\"", []token{ testFlow(t, `foo = "hello ♥ world"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello ♥ world"}, token{tokenString, "hello ♥ world", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 21},
}) })
} }
func TestKeyGroupArray(t *testing.T) { func TestKeyGroupArray(t *testing.T) {
testFlow(t, "[[foo]]", []token{ testFlow(t, "[[foo]]", []token{
token{tokenDoubleLeftBracket, "[["}, token{tokenDoubleLeftBracket, "[[", 0, 0},
token{tokenKeyGroupArray, "foo"}, token{tokenKeyGroupArray, "foo", 0, 2},
token{tokenDoubleRightBracket, "]]"}, token{tokenDoubleRightBracket, "]]", 0, 5},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 7},
}) })
} }
parser.go
+27 -20
View File
@@ -20,6 +20,11 @@ type parser struct {
type parserStateFn func(*parser) parserStateFn type parserStateFn func(*parser) parserStateFn
// Formats and panics an error message based on a token
func (p *parser) raiseError(tok *token, msg string, args ...interface{}) {
panic(tok.Pos() + ": " + fmt.Sprintf(msg, args...))
}
func (p *parser) run() { func (p *parser) run() {
for state := parseStart; state != nil; { for state := parseStart; state != nil; {
state = state(p) state = state(p)
@@ -42,10 +47,10 @@ func (p *parser) peek() *token {
func (p *parser) assume(typ tokenType) { func (p *parser) assume(typ tokenType) {
tok := p.getToken() tok := p.getToken()
if tok == nil { if tok == nil {
panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ)) p.raiseError(tok, "was expecting token %s, but token stream is empty", tok.typ)
} }
if tok.typ != typ { if tok.typ != typ {
panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ)) p.raiseError(tok, "was expecting token %s, but got %s", typ, tok.typ)
} }
} }
@@ -80,7 +85,7 @@ func parseStart(p *parser) parserStateFn {
case tokenEOF: case tokenEOF:
return nil return nil
default: default:
panic("unexpected token") p.raiseError(tok, "unexpected token")
} }
return nil return nil
} }
@@ -89,7 +94,7 @@ func parseGroupArray(p *parser) parserStateFn {
p.getToken() // discard the [[ p.getToken() // discard the [[
key := p.getToken() key := p.getToken()
if key.typ != tokenKeyGroupArray { if key.typ != tokenKeyGroupArray {
panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key)) p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
} }
// get or create group array element at the indicated part in the path // get or create group array element at the indicated part in the path
@@ -101,7 +106,7 @@ func parseGroupArray(p *parser) parserStateFn {
} else if dest_tree.([]*TomlTree) != nil { } else if dest_tree.([]*TomlTree) != nil {
array = dest_tree.([]*TomlTree) array = dest_tree.([]*TomlTree)
} else { } else {
panic(fmt.Sprintf("key %s is already assigned and not of type group array", key)) p.raiseError(key, "key %s is already assigned and not of type group array", key)
} }
// add a new tree to the end of the group array // add a new tree to the end of the group array
@@ -121,15 +126,17 @@ func parseGroup(p *parser) parserStateFn {
p.getToken() // discard the [ p.getToken() // discard the [
key := p.getToken() key := p.getToken()
if key.typ != tokenKeyGroup { if key.typ != tokenKeyGroup {
panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key)) p.raiseError(key, "unexpected token %s, was expecting a key group", key)
} }
for _, item := range p.seenGroupKeys { for _, item := range p.seenGroupKeys {
if item == key.val { if item == key.val {
panic("duplicated tables") p.raiseError(key, "duplicated tables")
} }
} }
p.seenGroupKeys = append(p.seenGroupKeys, key.val) p.seenGroupKeys = append(p.seenGroupKeys, key.val)
p.tree.createSubTree(key.val) if err := p.tree.createSubTree(key.val); err != nil {
p.raiseError(key, "%s", err)
}
p.assume(tokenRightBracket) p.assume(tokenRightBracket)
p.currentGroup = strings.Split(key.val, ".") p.currentGroup = strings.Split(key.val, ".")
return parseStart(p) return parseStart(p)
@@ -154,14 +161,14 @@ func parseAssign(p *parser) parserStateFn {
case *TomlTree: case *TomlTree:
target_node = node target_node = node
default: default:
panic(fmt.Sprintf("Unknown group type for path %v", group_key)) p.raiseError(key, "Unknown group type for path %s", group_key)
} }
// assign value to the found group // assign value to the found group
local_key := []string{key.val} local_key := []string{key.val}
final_key := append(group_key, key.val) final_key := append(group_key, key.val)
if target_node.GetPath(local_key) != nil { if target_node.GetPath(local_key) != nil {
panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, "."))) p.raiseError(key, "the following key was defined twice: %s", strings.Join(final_key, "."))
} }
target_node.SetPath(local_key, value) target_node.SetPath(local_key, value)
return parseStart(p) return parseStart(p)
@@ -170,7 +177,7 @@ func parseAssign(p *parser) parserStateFn {
func parseRvalue(p *parser) interface{} { func parseRvalue(p *parser) interface{} {
tok := p.getToken() tok := p.getToken()
if tok == nil || tok.typ == tokenEOF { if tok == nil || tok.typ == tokenEOF {
panic("expecting a value") p.raiseError(tok, "expecting a value")
} }
switch tok.typ { switch tok.typ {
@@ -183,28 +190,28 @@ func parseRvalue(p *parser) interface{} {
case tokenInteger: case tokenInteger:
val, err := strconv.ParseInt(tok.val, 10, 64) val, err := strconv.ParseInt(tok.val, 10, 64)
if err != nil { if err != nil {
panic(err) p.raiseError(tok, "%s", err)
} }
return val return val
case tokenFloat: case tokenFloat:
val, err := strconv.ParseFloat(tok.val, 64) val, err := strconv.ParseFloat(tok.val, 64)
if err != nil { if err != nil {
panic(err) p.raiseError(tok, "%s", err)
} }
return val return val
case tokenDate: case tokenDate:
val, err := time.Parse(time.RFC3339, tok.val) val, err := time.Parse(time.RFC3339, tok.val)
if err != nil { if err != nil {
panic(err) p.raiseError(tok, "%s", err)
} }
return val return val
case tokenLeftBracket: case tokenLeftBracket:
return parseArray(p) return parseArray(p)
case tokenError: case tokenError:
panic(tok.val) p.raiseError(tok, "%s", tok)
} }
panic("never reached") p.raiseError(tok, "never reached")
return nil return nil
} }
@@ -215,7 +222,7 @@ func parseArray(p *parser) []interface{} {
for { for {
follow := p.peek() follow := p.peek()
if follow == nil || follow.typ == tokenEOF { if follow == nil || follow.typ == tokenEOF {
panic("unterminated array") p.raiseError(follow, "unterminated array")
} }
if follow.typ == tokenRightBracket { if follow.typ == tokenRightBracket {
p.getToken() p.getToken()
@@ -226,15 +233,15 @@ func parseArray(p *parser) []interface{} {
arrayType = reflect.TypeOf(val) arrayType = reflect.TypeOf(val)
} }
if reflect.TypeOf(val) != arrayType { if reflect.TypeOf(val) != arrayType {
panic("mixed types in array") p.raiseError(follow, "mixed types in array")
} }
array = append(array, val) array = append(array, val)
follow = p.peek() follow = p.peek()
if follow == nil { if follow == nil {
panic("unterminated array") p.raiseError(follow, "unterminated array")
} }
if follow.typ != tokenRightBracket && follow.typ != tokenComma { if follow.typ != tokenRightBracket && follow.typ != tokenComma {
panic("missing comma") p.raiseError(follow, "missing comma")
} }
if follow.typ == tokenComma { if follow.typ == tokenComma {
p.getToken() p.getToken()
parser_test.go
+9 -9
View File
@@ -160,12 +160,12 @@ func TestNestedEmptyArrays(t *testing.T) {
func TestArrayMixedTypes(t *testing.T) { func TestArrayMixedTypes(t *testing.T) {
_, err := Load("a = [42, 16.0]") _, err := Load("a = [42, 16.0]")
if err.Error() != "mixed types in array" { if err.Error() != "(1, 10): mixed types in array" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
_, err = Load("a = [42, \"hello\"]") _, err = Load("a = [42, \"hello\"]")
if err.Error() != "mixed types in array" { if err.Error() != "(1, 11): mixed types in array" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
@@ -179,14 +179,14 @@ func TestArrayNestedStrings(t *testing.T) {
func TestMissingValue(t *testing.T) { func TestMissingValue(t *testing.T) {
_, err := Load("a = ") _, err := Load("a = ")
if err.Error() != "expecting a value" { if err.Error() != "(1, 4): expecting a value" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
func TestUnterminatedArray(t *testing.T) { func TestUnterminatedArray(t *testing.T) {
_, err := Load("a = [1,") _, err := Load("a = [1,")
if err.Error() != "unterminated array" { if err.Error() != "(1, 8): unterminated array" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
@@ -214,21 +214,21 @@ func TestArrayWithExtraCommaComment(t *testing.T) {
func TestDuplicateGroups(t *testing.T) { func TestDuplicateGroups(t *testing.T) {
_, err := Load("[foo]\na=2\n[foo]b=3") _, err := Load("[foo]\na=2\n[foo]b=3")
if err.Error() != "duplicated tables" { if err.Error() != "(3, 2): duplicated tables" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
func TestDuplicateKeys(t *testing.T) { func TestDuplicateKeys(t *testing.T) {
_, err := Load("foo = 2\nfoo = 3") _, err := Load("foo = 2\nfoo = 3")
if err.Error() != "the following key was defined twice: foo" { if err.Error() != "(2, 1): the following key was defined twice: foo" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
func TestEmptyIntermediateTable(t *testing.T) { func TestEmptyIntermediateTable(t *testing.T) {
_, err := Load("[foo..bar]") _, err := Load("[foo..bar]")
if err.Error() != "empty intermediate table" { if err.Error() != "(1, 2): empty intermediate table" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
@@ -249,12 +249,12 @@ func TestImplicitDeclarationBefore(t *testing.T) {
func TestFloatsWithoutLeadingZeros(t *testing.T) { func TestFloatsWithoutLeadingZeros(t *testing.T) {
_, err := Load("a = .42") _, err := Load("a = .42")
if err.Error() != "cannot start float with a dot" { if err.Error() != "(1, 4): cannot start float with a dot" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
_, err = Load("a = -.42") _, err = Load("a = -.42")
if err.Error() != "cannot start float with a dot" { if err.Error() != "(1, 5): cannot start float with a dot" {
t.Error("Bad error message:", err.Error()) t.Error("Bad error message:", err.Error())
} }
} }
toml.go
+5 -2
View File
@@ -142,11 +142,13 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
// //
// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b] // e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
// and tree[a][b][c] // and tree[a][b][c]
func (t *TomlTree) createSubTree(key string) { //
// Returns nil on success, error object on failure
func (t *TomlTree) createSubTree(key string) error {
subtree := t subtree := t
for _, intermediate_key := range strings.Split(key, ".") { for _, intermediate_key := range strings.Split(key, ".") {
if intermediate_key == "" { if intermediate_key == "" {
panic("empty intermediate table") return fmt.Errorf("empty intermediate table")
} }
_, exists := (*subtree)[intermediate_key] _, exists := (*subtree)[intermediate_key]
if !exists { if !exists {
@@ -155,6 +157,7 @@ func (t *TomlTree) createSubTree(key string) {
} }
subtree = ((*subtree)[intermediate_key]).(*TomlTree) subtree = ((*subtree)[intermediate_key]).(*TomlTree)
} }
return nil
} }
// encodes a string to a TOML-compliant string value // encodes a string to a TOML-compliant string value