Added line/col support to lexer

lexer.go

@@ -46,6 +46,8 @@ const (
 type token struct {
 	typ tokenType
 	val string
+  line int
+  col int
 }
 
 func (i token) String() string {
@@ -93,6 +95,8 @@ type lexer struct {
 	width  int
 	tokens chan token
 	depth  int
+  line   int
+  col    int
 }
 
 func (l *lexer) run() {
@@ -102,14 +106,32 @@ func (l *lexer) run() {
 	close(l.tokens)
 }
 
+func (l *lexer) nextStart() {
+  // iterate by runes (utf8 characters)
+  // search for newlines and advance line/col counts
+  for i:=l.start; i<l.pos; {
+	  r, width := utf8.DecodeRuneInString(l.input[i:])
+    if r == '\n' {
+      l.line += 1
+      l.col = 0
+    } else {
+      l.col += 1
+    }
+    i += width
+//    fmt.Printf("'%c'\n", r)
+  }
+  // advance start position to next token
+  l.start = l.pos
+}
+
 func (l *lexer) emit(t tokenType) {
-	l.tokens <- token{t, l.input[l.start:l.pos]}
-	l.start = l.pos
+	l.tokens <- token{t, l.input[l.start:l.pos], l.line, l.col}
+  l.nextStart()
 }
 
 func (l *lexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{t, value}
-	l.start = l.pos
+	l.tokens <- token{t, value, l.line, l.col}
+  l.nextStart()
 }
 
 func (l *lexer) next() rune {
@@ -124,7 +146,7 @@ func (l *lexer) next() rune {
 }
 
 func (l *lexer) ignore() {
-	l.start = l.pos
+  l.nextStart()
 }
 
 func (l *lexer) backup() {
@@ -135,6 +157,8 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn {
 	l.tokens <- token{
 		tokenError,
 		fmt.Sprintf(format, args...),
+    l.line,
+    l.col,
 	}
 	return nil
 }