Merge branch 'eanderton-master'

lexer.go

@@ -46,6 +46,8 @@ const (
 type token struct {
 	typ  tokenType
 	val  string
+	line int
+	col  int
 }
 
 func (i token) String() string {
@@ -62,6 +64,10 @@ func (i token) String() string {
 	return fmt.Sprintf("%q", i.val)
 }
 
+func (i token) Pos() string {
+	return fmt.Sprintf("(%d, %d)", i.line+1, i.col+1)
+}
+
 func isSpace(r rune) bool {
 	return r == ' ' || r == '\t'
 }
@@ -93,6 +99,8 @@ type lexer struct {
 	width  int
 	tokens chan token
 	depth  int
+	line   int
+	col    int
 }
 
 func (l *lexer) run() {
@@ -102,14 +110,32 @@ func (l *lexer) run() {
 	close(l.tokens)
 }
 
-func (l *lexer) emit(t tokenType) {
-	l.tokens <- token{t, l.input[l.start:l.pos]}
+func (l *lexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line += 1
+			l.col = 0
+		} else {
+			l.col += 1
+		}
+		i += width
+		//    fmt.Printf("'%c'\n", r)
+	}
+	// advance start position to next token
 	l.start = l.pos
 }
 
+func (l *lexer) emit(t tokenType) {
+	l.tokens <- token{t, l.input[l.start:l.pos], l.line, l.col}
+	l.nextStart()
+}
+
 func (l *lexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{t, value}
-	l.start = l.pos
+	l.tokens <- token{t, value, l.line, l.col}
+	l.nextStart()
 }
 
 func (l *lexer) next() rune {
@@ -124,7 +150,7 @@ func (l *lexer) next() rune {
 }
 
 func (l *lexer) ignore() {
-	l.start = l.pos
+	l.nextStart()
 }
 
 func (l *lexer) backup() {
@@ -135,6 +161,8 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn {
 	l.tokens <- token{
 		tokenError,
 		fmt.Sprintf(format, args...),
+		l.line,
+		l.col,
 	}
 	return nil
 }

lexer_test.go

@@ -7,9 +7,12 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {
+			t.Log("While testing: ", input)
 			t.Log("compared", token, "to", expected)
 			t.Log(token.val, "<->", expected.val)
 			t.Log(token.typ, "<->", expected.typ)
+			t.Log(token.line, "<->", expected.line)
+			t.Log(token.col, "<->", expected.col)
 			t.FailNow()
 		}
 	}
@@ -29,244 +32,245 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
 
 func TestValidKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenLeftBracket, "[", 0, 0},
+		token{tokenKeyGroup, "hello world", 0, 1},
+		token{tokenRightBracket, "]", 0, 12},
+		token{tokenEOF, "", 0, 13},
 	})
 }
 
 func TestUnclosedKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenError, "unclosed key group"},
+		token{tokenLeftBracket, "[", 0, 0},
+		token{tokenError, "unclosed key group", 0, 1},
 	})
 }
 
 func TestComment(t *testing.T) {
 	testFlow(t, "# blahblah", []token{
-		token{tokenEOF, ""},
+		token{tokenEOF, "", 0, 10},
 	})
 }
 
 func TestKeyGroupComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenLeftBracket, "[", 0, 0},
+		token{tokenKeyGroup, "hello world", 0, 1},
+		token{tokenRightBracket, "]", 0, 12},
+		token{tokenEOF, "", 0, 24},
 	})
 }
 
 func TestMultipleKeyGroupsComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah\n[test]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "test"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenLeftBracket, "[", 0, 0},
+		token{tokenKeyGroup, "hello world", 0, 1},
+		token{tokenRightBracket, "]", 0, 12},
+		token{tokenLeftBracket, "[", 1, 0},
+		token{tokenKeyGroup, "test", 1, 1},
+		token{tokenRightBracket, "]", 1, 5},
+		token{tokenEOF, "", 1, 6},
 	})
 }
 
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
-		token{tokenKey, "hello"},
-		token{tokenEOF, ""},
+		token{tokenKey, "hello", 0, 0},
+		token{tokenEOF, "", 0, 5},
 	})
 }
 
 func TestBasicKeyWithUnderscore(t *testing.T) {
 	testFlow(t, "hello_hello", []token{
-		token{tokenKey, "hello_hello"},
-		token{tokenEOF, ""},
+		token{tokenKey, "hello_hello", 0, 0},
+		token{tokenEOF, "", 0, 11},
 	})
 }
 
 func TestBasicKeyWithDash(t *testing.T) {
 	testFlow(t, "hello-world", []token{
-		token{tokenKey, "hello-world"},
-		token{tokenEOF, ""},
+		token{tokenKey, "hello-world", 0, 0},
+		token{tokenEOF, "", 0, 11},
 	})
 }
 
 func TestBasicKeyWithUppercaseMix(t *testing.T) {
 	testFlow(t, "helloHELLOHello", []token{
-		token{tokenKey, "helloHELLOHello"},
-		token{tokenEOF, ""},
+		token{tokenKey, "helloHELLOHello", 0, 0},
+		token{tokenEOF, "", 0, 15},
 	})
 }
 
 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
 	testFlow(t, "héllÖ", []token{
-		token{tokenKey, "héllÖ"},
-		token{tokenEOF, ""},
+		token{tokenKey, "héllÖ", 0, 0},
+		token{tokenEOF, "", 0, 5},
 	})
 }
 
 func TestBasicKeyAndEqual(t *testing.T) {
 	testFlow(t, "hello =", []token{
-		token{tokenKey, "hello"},
-		token{tokenEqual, "="},
-		token{tokenEOF, ""},
+		token{tokenKey, "hello", 0, 0},
+		token{tokenEqual, "=", 0, 6},
+		token{tokenEOF, "", 0, 7},
 	})
 }
 
 func TestKeyWithSharpAndEqual(t *testing.T) {
 	testFlow(t, "key#name = 5", []token{
-		token{tokenKey, "key#name"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
+		token{tokenKey, "key#name", 0, 0},
+		token{tokenEqual, "=", 0, 9},
+		token{tokenInteger, "5", 0, 11},
+		token{tokenEOF, "", 0, 12},
 	})
 }
+
 func TestKeyWithSymbolsAndEqual(t *testing.T) {
 	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
-		token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
+		token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'", 0, 0},
+		token{tokenEqual, "=", 0, 38},
+		token{tokenInteger, "5", 0, 40},
+		token{tokenEOF, "", 0, 41},
 	})
 }
 
 func TestKeyEqualStringEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello\\\"\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello\""},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello\""`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenString, "hello\"", 0, 7},
+		token{tokenEOF, "", 0, 15},
 	})
 }
 
 func TestKeyEqualStringUnfinished(t *testing.T) {
-	testFlow(t, "foo = \"bar", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "unclosed string"},
+	testFlow(t, `foo = "bar`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenError, "unclosed string", 0, 7},
 	})
 }
 
 func TestKeyEqualString(t *testing.T) {
-	testFlow(t, "foo = \"bar\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "bar"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "bar"`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenString, "bar", 0, 7},
+		token{tokenEOF, "", 0, 11},
 	})
 }
 
 func TestKeyEqualTrue(t *testing.T) {
 	testFlow(t, "foo = true", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenTrue, "true"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenTrue, "true", 0, 6},
+		token{tokenEOF, "", 0, 10},
 	})
 }
 
 func TestKeyEqualFalse(t *testing.T) {
 	testFlow(t, "foo = false", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFalse, "false"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenFalse, "false", 0, 6},
+		token{tokenEOF, "", 0, 11},
 	})
 }
 
 func TestArrayNestedString(t *testing.T) {
-	testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenString, "hello"},
-		token{tokenComma, ","},
-		token{tokenString, "world"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
+		token{tokenKey, "a", 0, 0},
+		token{tokenEqual, "=", 0, 2},
+		token{tokenLeftBracket, "[", 0, 4},
+		token{tokenLeftBracket, "[", 0, 6},
+		token{tokenString, "hello", 0, 8},
+		token{tokenComma, ",", 0, 14},
+		token{tokenString, "world", 0, 17},
+		token{tokenRightBracket, "]", 0, 23},
+		token{tokenRightBracket, "]", 0, 25},
+		token{tokenEOF, "", 0, 26},
 	})
 }
 
 func TestArrayNestedInts(t *testing.T) {
 	testFlow(t, "a = [ [42, 21], [10] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenRightBracket, "]"},
-		token{tokenComma, ","},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "10"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "a", 0, 0},
+		token{tokenEqual, "=", 0, 2},
+		token{tokenLeftBracket, "[", 0, 4},
+		token{tokenLeftBracket, "[", 0, 6},
+		token{tokenInteger, "42", 0, 7},
+		token{tokenComma, ",", 0, 9},
+		token{tokenInteger, "21", 0, 11},
+		token{tokenRightBracket, "]", 0, 13},
+		token{tokenComma, ",", 0, 14},
+		token{tokenLeftBracket, "[", 0, 16},
+		token{tokenInteger, "10", 0, 17},
+		token{tokenRightBracket, "]", 0, 19},
+		token{tokenRightBracket, "]", 0, 21},
+		token{tokenEOF, "", 0, 22},
 	})
 }
 
 func TestArrayInts(t *testing.T) {
 	testFlow(t, "a = [ 42, 21, 10, ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenComma, ","},
-		token{tokenInteger, "10"},
-		token{tokenComma, ","},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "a", 0, 0},
+		token{tokenEqual, "=", 0, 2},
+		token{tokenLeftBracket, "[", 0, 4},
+		token{tokenInteger, "42", 0, 6},
+		token{tokenComma, ",", 0, 8},
+		token{tokenInteger, "21", 0, 10},
+		token{tokenComma, ",", 0, 12},
+		token{tokenInteger, "10", 0, 14},
+		token{tokenComma, ",", 0, 16},
+		token{tokenRightBracket, "]", 0, 18},
+		token{tokenEOF, "", 0, 19},
 	})
 }
 
 func TestMultilineArrayComments(t *testing.T) {
 	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "1"},
-		token{tokenComma, ","},
-		token{tokenInteger, "2"},
-		token{tokenComma, ","},
-		token{tokenInteger, "3"},
-		token{tokenComma, ","},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "a", 0, 0},
+		token{tokenEqual, "=", 0, 2},
+		token{tokenLeftBracket, "[", 0, 4},
+		token{tokenInteger, "1", 0, 5},
+		token{tokenComma, ",", 0, 6},
+		token{tokenInteger, "2", 1, 0},
+		token{tokenComma, ",", 1, 1},
+		token{tokenInteger, "3", 2, 0},
+		token{tokenComma, ",", 2, 1},
+		token{tokenRightBracket, "]", 3, 0},
+		token{tokenEOF, "", 3, 1},
 	})
 }
 
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenLeftBracket, "[", 0, 6},
+		token{tokenTrue, "true", 0, 7},
+		token{tokenComma, ",", 0, 11},
+		token{tokenFalse, "false", 0, 13},
+		token{tokenComma, ",", 0, 18},
+		token{tokenTrue, "true", 0, 20},
+		token{tokenRightBracket, "]", 0, 24},
+		token{tokenEOF, "", 0, 25},
 	})
 }
 
 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 	testFlow(t, "foo = [true, false, true] # YEAH", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenLeftBracket, "[", 0, 6},
+		token{tokenTrue, "true", 0, 7},
+		token{tokenComma, ",", 0, 11},
+		token{tokenFalse, "false", 0, 13},
+		token{tokenComma, ",", 0, 18},
+		token{tokenTrue, "true", 0, 20},
+		token{tokenRightBracket, "]", 0, 24},
+		token{tokenEOF, "", 0, 32},
 	})
 }
 
@@ -278,138 +282,138 @@ func TestDateRegexp(t *testing.T) {
 
 func TestKeyEqualDate(t *testing.T) {
 	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenDate, "1979-05-27T07:32:00Z"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenDate, "1979-05-27T07:32:00Z", 0, 6},
+		token{tokenEOF, "", 0, 26},
 	})
 }
 
 func TestFloatEndingWithDot(t *testing.T) {
 	testFlow(t, "foo = 42.", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "float cannot end with a dot"},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenError, "float cannot end with a dot", 0, 6},
 	})
 }
 
 func TestFloatWithTwoDots(t *testing.T) {
 	testFlow(t, "foo = 4.2.", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "cannot have two dots in one float"},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenError, "cannot have two dots in one float", 0, 6},
 	})
 }
 
 func TestDoubleEqualKey(t *testing.T) {
 	testFlow(t, "foo= = 2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "cannot have multiple equals for the same key"},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 3},
+		token{tokenError, "cannot have multiple equals for the same key", 0, 4},
 	})
 }
 
 func TestInvalidEsquapeSequence(t *testing.T) {
-	testFlow(t, "foo = \"\\x\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "invalid escape sequence: \\x"},
+	testFlow(t, `foo = "\x"`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenError, "invalid escape sequence: \\x", 0, 7},
 	})
 }
 
 func TestNestedArrays(t *testing.T) {
 	testFlow(t, "foo = [[[]]]", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenLeftBracket, "[", 0, 6},
+		token{tokenLeftBracket, "[", 0, 7},
+		token{tokenLeftBracket, "[", 0, 8},
+		token{tokenRightBracket, "]", 0, 9},
+		token{tokenRightBracket, "]", 0, 10},
+		token{tokenRightBracket, "]", 0, 11},
+		token{tokenEOF, "", 0, 12},
 	})
 }
 
 func TestKeyEqualNumber(t *testing.T) {
 	testFlow(t, "foo = 42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenInteger, "42", 0, 6},
+		token{tokenEOF, "", 0, 8},
 	})
 
 	testFlow(t, "foo = +42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "+42"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenInteger, "+42", 0, 6},
+		token{tokenEOF, "", 0, 9},
 	})
 
 	testFlow(t, "foo = -42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "-42"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenInteger, "-42", 0, 6},
+		token{tokenEOF, "", 0, 9},
 	})
 
 	testFlow(t, "foo = 4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "4.2"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenFloat, "4.2", 0, 6},
+		token{tokenEOF, "", 0, 9},
 	})
 
 	testFlow(t, "foo = +4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "+4.2"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenFloat, "+4.2", 0, 6},
+		token{tokenEOF, "", 0, 10},
 	})
 
 	testFlow(t, "foo = -4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "-4.2"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenFloat, "-4.2", 0, 6},
+		token{tokenEOF, "", 0, 10},
 	})
 }
 
 func TestMultiline(t *testing.T) {
 	testFlow(t, "foo = 42\nbar=21", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenKey, "bar"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "21"},
-		token{tokenEOF, ""},
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenInteger, "42", 0, 6},
+		token{tokenKey, "bar", 1, 0},
+		token{tokenEqual, "=", 1, 3},
+		token{tokenInteger, "21", 1, 4},
+		token{tokenEOF, "", 1, 6},
 	})
 }
 
 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello \\u2665\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello \u2665"`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenString, "hello ♥", 0, 7},
+		token{tokenEOF, "", 0, 20},
 	})
 }
 
 func TestUnicodeString(t *testing.T) {
-	testFlow(t, "foo = \"hello ♥ world\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥ world"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello ♥ world"`, []token{
+		token{tokenKey, "foo", 0, 0},
+		token{tokenEqual, "=", 0, 4},
+		token{tokenString, "hello ♥ world", 0, 7},
+		token{tokenEOF, "", 0, 21},
 	})
 }
 
 func TestKeyGroupArray(t *testing.T) {
 	testFlow(t, "[[foo]]", []token{
-		token{tokenDoubleLeftBracket, "[["},
-		token{tokenKeyGroupArray, "foo"},
-		token{tokenDoubleRightBracket, "]]"},
-		token{tokenEOF, ""},
+		token{tokenDoubleLeftBracket, "[[", 0, 0},
+		token{tokenKeyGroupArray, "foo", 0, 2},
+		token{tokenDoubleRightBracket, "]]", 0, 5},
+		token{tokenEOF, "", 0, 7},
 	})
 }

parser.go

@@ -20,6 +20,11 @@ type parser struct {
 
 type parserStateFn func(*parser) parserStateFn
 
+// Formats and panics an error message based on a token
+func (p *parser) raiseError(tok *token, msg string, args ...interface{}) {
+	panic(tok.Pos() + ": " + fmt.Sprintf(msg, args...))
+}
+
 func (p *parser) run() {
 	for state := parseStart; state != nil; {
 		state = state(p)
@@ -42,10 +47,10 @@ func (p *parser) peek() *token {
 func (p *parser) assume(typ tokenType) {
 	tok := p.getToken()
 	if tok == nil {
-		panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ))
+		p.raiseError(tok, "was expecting token %s, but token stream is empty", tok.typ)
 	}
 	if tok.typ != typ {
-		panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ))
+		p.raiseError(tok, "was expecting token %s, but got %s", typ, tok.typ)
 	}
 }
 
@@ -80,7 +85,7 @@ func parseStart(p *parser) parserStateFn {
 	case tokenEOF:
 		return nil
 	default:
-		panic("unexpected token")
+		p.raiseError(tok, "unexpected token")
 	}
 	return nil
 }
@@ -89,7 +94,7 @@ func parseGroupArray(p *parser) parserStateFn {
 	p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
 	}
 
 	// get or create group array element at the indicated part in the path
@@ -101,7 +106,7 @@ func parseGroupArray(p *parser) parserStateFn {
 	} else if dest_tree.([]*TomlTree) != nil {
 		array = dest_tree.([]*TomlTree)
 	} else {
-		panic(fmt.Sprintf("key %s is already assigned and not of type group array", key))
+		p.raiseError(key, "key %s is already assigned and not of type group array", key)
 	}
 
 	// add a new tree to the end of the group array
@@ -121,15 +126,17 @@ func parseGroup(p *parser) parserStateFn {
 	p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
 	}
 	for _, item := range p.seenGroupKeys {
 		if item == key.val {
-			panic("duplicated tables")
+			p.raiseError(key, "duplicated tables")
 		}
 	}
 	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
-	p.tree.createSubTree(key.val)
+	if err := p.tree.createSubTree(key.val); err != nil {
+		p.raiseError(key, "%s", err)
+	}
 	p.assume(tokenRightBracket)
 	p.currentGroup = strings.Split(key.val, ".")
 	return parseStart(p)
@@ -154,14 +161,14 @@ func parseAssign(p *parser) parserStateFn {
 	case *TomlTree:
 		target_node = node
 	default:
-		panic(fmt.Sprintf("Unknown group type for path %v", group_key))
+		p.raiseError(key, "Unknown group type for path %s", group_key)
 	}
 
 	// assign value to the found group
 	local_key := []string{key.val}
 	final_key := append(group_key, key.val)
 	if target_node.GetPath(local_key) != nil {
-		panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
+		p.raiseError(key, "the following key was defined twice: %s", strings.Join(final_key, "."))
 	}
 	target_node.SetPath(local_key, value)
 	return parseStart(p)
@@ -170,7 +177,7 @@ func parseAssign(p *parser) parserStateFn {
 func parseRvalue(p *parser) interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
-		panic("expecting a value")
+		p.raiseError(tok, "expecting a value")
 	}
 
 	switch tok.typ {
@@ -183,28 +190,28 @@ func parseRvalue(p *parser) interface{} {
 	case tokenInteger:
 		val, err := strconv.ParseInt(tok.val, 10, 64)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
 		val, err := strconv.ParseFloat(tok.val, 64)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenDate:
 		val, err := time.Parse(time.RFC3339, tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenLeftBracket:
 		return parseArray(p)
 	case tokenError:
-		panic(tok.val)
+		p.raiseError(tok, "%s", tok)
 	}
 
-	panic("never reached")
+	p.raiseError(tok, "never reached")
 
 	return nil
 }
@@ -215,7 +222,7 @@ func parseArray(p *parser) []interface{} {
 	for {
 		follow := p.peek()
 		if follow == nil || follow.typ == tokenEOF {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ == tokenRightBracket {
 			p.getToken()
@@ -226,15 +233,15 @@ func parseArray(p *parser) []interface{} {
 			arrayType = reflect.TypeOf(val)
 		}
 		if reflect.TypeOf(val) != arrayType {
-			panic("mixed types in array")
+			p.raiseError(follow, "mixed types in array")
 		}
 		array = append(array, val)
 		follow = p.peek()
 		if follow == nil {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
-			panic("missing comma")
+			p.raiseError(follow, "missing comma")
 		}
 		if follow.typ == tokenComma {
 			p.getToken()

parser_test.go

@@ -160,12 +160,12 @@ func TestNestedEmptyArrays(t *testing.T) {
 
 func TestArrayMixedTypes(t *testing.T) {
 	_, err := Load("a = [42, 16.0]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 10): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = [42, \"hello\"]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 11): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -179,14 +179,14 @@ func TestArrayNestedStrings(t *testing.T) {
 
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "expecting a value" {
+	if err.Error() != "(1, 4): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestUnterminatedArray(t *testing.T) {
 	_, err := Load("a = [1,")
-	if err.Error() != "unterminated array" {
+	if err.Error() != "(1, 8): unterminated array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -214,21 +214,21 @@ func TestArrayWithExtraCommaComment(t *testing.T) {
 
 func TestDuplicateGroups(t *testing.T) {
 	_, err := Load("[foo]\na=2\n[foo]b=3")
-	if err.Error() != "duplicated tables" {
+	if err.Error() != "(3, 2): duplicated tables" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestDuplicateKeys(t *testing.T) {
 	_, err := Load("foo = 2\nfoo = 3")
-	if err.Error() != "the following key was defined twice: foo" {
+	if err.Error() != "(2, 1): the following key was defined twice: foo" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "empty intermediate table" {
+	if err.Error() != "(1, 2): empty intermediate table" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -249,12 +249,12 @@ func TestImplicitDeclarationBefore(t *testing.T) {
 
 func TestFloatsWithoutLeadingZeros(t *testing.T) {
 	_, err := Load("a = .42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 4): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = -.42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 }

toml.go

@@ -142,11 +142,13 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 //
 // e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
 // and tree[a][b][c]
-func (t *TomlTree) createSubTree(key string) {
+//
+// Returns nil on success, error object on failure
+func (t *TomlTree) createSubTree(key string) error {
 	subtree := t
 	for _, intermediate_key := range strings.Split(key, ".") {
 		if intermediate_key == "" {
-			panic("empty intermediate table")
+			return fmt.Errorf("empty intermediate table")
 		}
 		_, exists := (*subtree)[intermediate_key]
 		if !exists {
@@ -155,6 +157,7 @@ func (t *TomlTree) createSubTree(key string) {
 		}
 		subtree = ((*subtree)[intermediate_key]).(*TomlTree)
 	}
+	return nil
 }
 
 // encodes a string to a TOML-compliant string value