ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added line/col support to lexer

Browse Source
This commit is contained in:
eanderton
2014-08-06 07:13:15 -04:00
parent e493544dfd
commit dd04a2f3cd
2 changed files with 235 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer.go
+29 -5
View File
@@ -46,6 +46,8 @@ const (
type token struct { type token struct {
typ tokenType typ tokenType
val string val string
line int
col int
} }
func (i token) String() string { func (i token) String() string {
@@ -93,6 +95,8 @@ type lexer struct {
width int width int
tokens chan token tokens chan token
depth int depth int
line int
col int
} }
func (l *lexer) run() { func (l *lexer) run() {
@@ -102,14 +106,32 @@ func (l *lexer) run() {
close(l.tokens) close(l.tokens)
} }
func (l *lexer) nextStart() {
// iterate by runes (utf8 characters)
// search for newlines and advance line/col counts
for i:=l.start; i<l.pos; {
r, width := utf8.DecodeRuneInString(l.input[i:])
if r == '\n' {
l.line += 1
l.col = 0
} else {
l.col += 1
}
i += width
// fmt.Printf("'%c'\n", r)
}
// advance start position to next token
l.start = l.pos
}
func (l *lexer) emit(t tokenType) { func (l *lexer) emit(t tokenType) {
l.tokens <- token{t, l.input[l.start:l.pos]} l.tokens <- token{t, l.input[l.start:l.pos], l.line, l.col}
l.start = l.pos l.nextStart()
} }
func (l *lexer) emitWithValue(t tokenType, value string) { func (l *lexer) emitWithValue(t tokenType, value string) {
l.tokens <- token{t, value} l.tokens <- token{t, value, l.line, l.col}
l.start = l.pos l.nextStart()
} }
func (l *lexer) next() rune { func (l *lexer) next() rune {
@@ -124,7 +146,7 @@ func (l *lexer) next() rune {
} }
func (l *lexer) ignore() { func (l *lexer) ignore() {
l.start = l.pos l.nextStart()
} }
func (l *lexer) backup() { func (l *lexer) backup() {
@@ -135,6 +157,8 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn {
l.tokens <- token{ l.tokens <- token{
tokenError, tokenError,
fmt.Sprintf(format, args...), fmt.Sprintf(format, args...),
l.line,
l.col,
} }
return nil return nil
} }
lexer_test.go
+206 -201
View File
@@ -7,9 +7,12 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
for _, expected := range expectedFlow { for _, expected := range expectedFlow {
token := <-ch token := <-ch
if token != expected { if token != expected {
t.Log("While testing: ", input)
t.Log("compared", token, "to", expected) t.Log("compared", token, "to", expected)
t.Log(token.val, "<->", expected.val) t.Log(token.val, "<->", expected.val)
t.Log(token.typ, "<->", expected.typ) t.Log(token.typ, "<->", expected.typ)
t.Log(token.line, "<->", expected.line)
t.Log(token.col, "<->", expected.col)
t.FailNow() t.FailNow()
} }
} }
@@ -29,244 +32,246 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
func TestValidKeyGroup(t *testing.T) { func TestValidKeyGroup(t *testing.T) {
testFlow(t, "[hello world]", []token{ testFlow(t, "[hello world]", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 13},
}) })
} }
func TestUnclosedKeyGroup(t *testing.T) { func TestUnclosedKeyGroup(t *testing.T) {
testFlow(t, "[hello world", []token{ testFlow(t, "[hello world", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenError, "unclosed key group"}, token{tokenError, "unclosed key group", 0, 1},
}) })
} }
func TestComment(t *testing.T) { func TestComment(t *testing.T) {
testFlow(t, "# blahblah", []token{ testFlow(t, "# blahblah", []token{
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestKeyGroupComment(t *testing.T) { func TestKeyGroupComment(t *testing.T) {
testFlow(t, "[hello world] # blahblah", []token{ testFlow(t, "[hello world] # blahblah", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 24},
}) })
} }
func TestMultipleKeyGroupsComment(t *testing.T) { func TestMultipleKeyGroupsComment(t *testing.T) {
testFlow(t, "[hello world] # blahblah\n[test]", []token{ testFlow(t, "[hello world] # blahblah\n[test]", []token{
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 0},
token{tokenKeyGroup, "hello world"}, token{tokenKeyGroup, "hello world", 0, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 12},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 1, 0},
token{tokenKeyGroup, "test"}, token{tokenKeyGroup, "test", 1, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 1, 5},
token{tokenEOF, ""}, token{tokenEOF, "", 1, 6},
}) })
} }
func TestBasicKey(t *testing.T) { func TestBasicKey(t *testing.T) {
testFlow(t, "hello", []token{ testFlow(t, "hello", []token{
token{tokenKey, "hello"}, token{tokenKey, "hello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 5},
}) })
} }
func TestBasicKeyWithUnderscore(t *testing.T) { func TestBasicKeyWithUnderscore(t *testing.T) {
testFlow(t, "hello_hello", []token{ testFlow(t, "hello_hello", []token{
token{tokenKey, "hello_hello"}, token{tokenKey, "hello_hello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestBasicKeyWithDash(t *testing.T) { func TestBasicKeyWithDash(t *testing.T) {
testFlow(t, "hello-world", []token{ testFlow(t, "hello-world", []token{
token{tokenKey, "hello-world"}, token{tokenKey, "hello-world", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestBasicKeyWithUppercaseMix(t *testing.T) { func TestBasicKeyWithUppercaseMix(t *testing.T) {
testFlow(t, "helloHELLOHello", []token{ testFlow(t, "helloHELLOHello", []token{
token{tokenKey, "helloHELLOHello"}, token{tokenKey, "helloHELLOHello", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 15},
}) })
} }
func TestBasicKeyWithInternationalCharacters(t *testing.T) { func TestBasicKeyWithInternationalCharacters(t *testing.T) {
testFlow(t, "héllÖ", []token{ testFlow(t, "héllÖ", []token{
token{tokenKey, "héllÖ"}, token{tokenKey, "héllÖ", 0, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 5},
}) })
} }
func TestBasicKeyAndEqual(t *testing.T) { func TestBasicKeyAndEqual(t *testing.T) {
testFlow(t, "hello =", []token{ testFlow(t, "hello =", []token{
token{tokenKey, "hello"}, token{tokenKey, "hello", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 7},
}) })
} }
func TestKeyWithSharpAndEqual(t *testing.T) { func TestKeyWithSharpAndEqual(t *testing.T) {
testFlow(t, "key#name = 5", []token{ testFlow(t, "key#name = 5", []token{
token{tokenKey, "key#name"}, token{tokenKey, "key#name", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 9},
token{tokenInteger, "5"}, token{tokenInteger, "5", 0, 11},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 12},
}) })
} }
func TestKeyWithSymbolsAndEqual(t *testing.T) { func TestKeyWithSymbolsAndEqual(t *testing.T) {
testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"}, token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 38},
token{tokenInteger, "5"}, token{tokenInteger, "5", 0, 40},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 41},
}) })
} }
func TestKeyEqualStringEscape(t *testing.T) { func TestKeyEqualStringEscape(t *testing.T) {
testFlow(t, "foo = \"hello\\\"\"", []token{ testFlow(t, `foo = "hello\""`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello\""}, token{tokenString, "hello\"" ,0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 15},
}) })
} }
func TestKeyEqualStringUnfinished(t *testing.T) { func TestKeyEqualStringUnfinished(t *testing.T) {
testFlow(t, "foo = \"bar", []token{ testFlow(t, `foo = "bar`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "unclosed string"}, token{tokenError, "unclosed string", 0, 7},
}) })
} }
func TestKeyEqualString(t *testing.T) { func TestKeyEqualString(t *testing.T) {
testFlow(t, "foo = \"bar\"", []token{ testFlow(t, `foo = "bar"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "bar"}, token{tokenString, "bar", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestKeyEqualTrue(t *testing.T) { func TestKeyEqualTrue(t *testing.T) {
testFlow(t, "foo = true", []token{ testFlow(t, "foo = true", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestKeyEqualFalse(t *testing.T) { func TestKeyEqualFalse(t *testing.T) {
testFlow(t, "foo = false", []token{ testFlow(t, "foo = false", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 11},
}) })
} }
func TestArrayNestedString(t *testing.T) { func TestArrayNestedString(t *testing.T) {
testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{ testFlow(t, `a = [ ["hello", "world"] ]`, []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenString, "hello"}, token{tokenString, "hello", 0, 8},
token{tokenComma, ","}, token{tokenComma, ",", 0, 14},
token{tokenString, "world"}, token{tokenString, "world", 0, 17},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 23},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 25},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 26},
}) })
} }
func TestArrayNestedInts(t *testing.T) { func TestArrayNestedInts(t *testing.T) {
testFlow(t, "a = [ [42, 21], [10] ]", []token{ testFlow(t, "a = [ [42, 21], [10] ]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 9},
token{tokenInteger, "21"}, token{tokenInteger, "21", 0, 11},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 14},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 16},
token{tokenInteger, "10"}, token{tokenInteger, "10", 0, 17},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 19},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 21},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 22},
}) })
} }
func TestArrayInts(t *testing.T) { func TestArrayInts(t *testing.T) {
testFlow(t, "a = [ 42, 21, 10, ]", []token{ testFlow(t, "a = [ 42, 21, 10, ]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenComma, ","}, token{tokenComma, ",", 0, 8},
token{tokenInteger, "21"}, token{tokenInteger, "21", 0, 10},
token{tokenComma, ","}, token{tokenComma, ",", 0, 12},
token{tokenInteger, "10"}, token{tokenInteger, "10", 0, 14},
token{tokenComma, ","}, token{tokenComma, ",", 0, 16},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 18},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 19},
}) })
} }
func TestMultilineArrayComments(t *testing.T) { func TestMultilineArrayComments(t *testing.T) {
testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{ testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
token{tokenKey, "a"}, token{tokenKey, "a", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 2},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 4},
token{tokenInteger, "1"}, token{tokenInteger, "1", 0, 5},
token{tokenComma, ","}, token{tokenComma, ",", 0, 6},
token{tokenInteger, "2"}, token{tokenInteger, "2", 1, 0},
token{tokenComma, ","}, token{tokenComma, ",", 1, 1},
token{tokenInteger, "3"}, token{tokenInteger, "3", 2, 0},
token{tokenComma, ","}, token{tokenComma, ",", 2, 1},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 3, 0},
token{tokenEOF, ""}, token{tokenEOF, "", 3, 1},
}) })
} }
func TestKeyEqualArrayBools(t *testing.T) { func TestKeyEqualArrayBools(t *testing.T) {
testFlow(t, "foo = [true, false, true]", []token{ testFlow(t, "foo = [true, false, true]", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 11},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 18},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 20},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 24},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 25},
}) })
} }
func TestKeyEqualArrayBoolsWithComments(t *testing.T) { func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
testFlow(t, "foo = [true, false, true] # YEAH", []token{ testFlow(t, "foo = [true, false, true] # YEAH", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 7},
token{tokenComma, ","}, token{tokenComma, ",", 0, 11},
token{tokenFalse, "false"}, token{tokenFalse, "false", 0, 13},
token{tokenComma, ","}, token{tokenComma, ",", 0, 18},
token{tokenTrue, "true"}, token{tokenTrue, "true", 0, 20},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 24},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 32},
}) })
} }
@@ -278,138 +283,138 @@ func TestDateRegexp(t *testing.T) {
func TestKeyEqualDate(t *testing.T) { func TestKeyEqualDate(t *testing.T) {
testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{ testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenDate, "1979-05-27T07:32:00Z"}, token{tokenDate, "1979-05-27T07:32:00Z", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 26},
}) })
} }
func TestFloatEndingWithDot(t *testing.T) { func TestFloatEndingWithDot(t *testing.T) {
testFlow(t, "foo = 42.", []token{ testFlow(t, "foo = 42.", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "float cannot end with a dot"}, token{tokenError, "float cannot end with a dot", 0, 6},
}) })
} }
func TestFloatWithTwoDots(t *testing.T) { func TestFloatWithTwoDots(t *testing.T) {
testFlow(t, "foo = 4.2.", []token{ testFlow(t, "foo = 4.2.", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "cannot have two dots in one float"}, token{tokenError, "cannot have two dots in one float", 0, 6},
}) })
} }
func TestDoubleEqualKey(t *testing.T) { func TestDoubleEqualKey(t *testing.T) {
testFlow(t, "foo= = 2", []token{ testFlow(t, "foo= = 2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 3},
token{tokenError, "cannot have multiple equals for the same key"}, token{tokenError, "cannot have multiple equals for the same key", 0, 4},
}) })
} }
func TestInvalidEsquapeSequence(t *testing.T) { func TestInvalidEsquapeSequence(t *testing.T) {
testFlow(t, "foo = \"\\x\"", []token{ testFlow(t, `foo = "\x"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenError, "invalid escape sequence: \\x"}, token{tokenError, "invalid escape sequence: \\x", 0, 7},
}) })
} }
func TestNestedArrays(t *testing.T) { func TestNestedArrays(t *testing.T) {
testFlow(t, "foo = [[[]]]", []token{ testFlow(t, "foo = [[[]]]", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 6},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 7},
token{tokenLeftBracket, "["}, token{tokenLeftBracket, "[", 0, 8},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 9},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 10},
token{tokenRightBracket, "]"}, token{tokenRightBracket, "]", 0, 11},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 12},
}) })
} }
func TestKeyEqualNumber(t *testing.T) { func TestKeyEqualNumber(t *testing.T) {
testFlow(t, "foo = 42", []token{ testFlow(t, "foo = 42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 8},
}) })
testFlow(t, "foo = +42", []token{ testFlow(t, "foo = +42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "+42"}, token{tokenInteger, "+42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = -42", []token{ testFlow(t, "foo = -42", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "-42"}, token{tokenInteger, "-42", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = 4.2", []token{ testFlow(t, "foo = 4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "4.2"}, token{tokenFloat, "4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 9},
}) })
testFlow(t, "foo = +4.2", []token{ testFlow(t, "foo = +4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "+4.2"}, token{tokenFloat, "+4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
testFlow(t, "foo = -4.2", []token{ testFlow(t, "foo = -4.2", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenFloat, "-4.2"}, token{tokenFloat, "-4.2", 0, 6},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 10},
}) })
} }
func TestMultiline(t *testing.T) { func TestMultiline(t *testing.T) {
testFlow(t, "foo = 42\nbar=21", []token{ testFlow(t, "foo = 42\nbar=21", []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenInteger, "42"}, token{tokenInteger, "42", 0, 6},
token{tokenKey, "bar"}, token{tokenKey, "bar", 1, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 1, 3},
token{tokenInteger, "21"}, token{tokenInteger, "21", 1, 4},
token{tokenEOF, ""}, token{tokenEOF, "", 1, 6},
}) })
} }
func TestKeyEqualStringUnicodeEscape(t *testing.T) { func TestKeyEqualStringUnicodeEscape(t *testing.T) {
testFlow(t, "foo = \"hello \\u2665\"", []token{ testFlow(t, `foo = "hello \u2665"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello ♥"}, token{tokenString, "hello ♥", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 20},
}) })
} }
func TestUnicodeString(t *testing.T) { func TestUnicodeString(t *testing.T) {
testFlow(t, "foo = \"hello ♥ world\"", []token{ testFlow(t, `foo = "hello ♥ world"`, []token{
token{tokenKey, "foo"}, token{tokenKey, "foo", 0, 0},
token{tokenEqual, "="}, token{tokenEqual, "=", 0, 4},
token{tokenString, "hello ♥ world"}, token{tokenString, "hello ♥ world", 0, 7},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 21},
}) })
} }
func TestKeyGroupArray(t *testing.T) { func TestKeyGroupArray(t *testing.T) {
testFlow(t, "[[foo]]", []token{ testFlow(t, "[[foo]]", []token{
token{tokenDoubleLeftBracket, "[["}, token{tokenDoubleLeftBracket, "[[", 0, 0},
token{tokenKeyGroupArray, "foo"}, token{tokenKeyGroupArray, "foo", 0, 2},
token{tokenDoubleRightBracket, "]]"}, token{tokenDoubleRightBracket, "]]", 0, 5},
token{tokenEOF, ""}, token{tokenEOF, "", 0, 7},
}) })
} }