Make lexComment jump back to the previous state (#122)

When a comment appears in an rvalue, the lexer needs to jump back to
lexRValue, not to lexVoid.

Fixes #120.

lexer.go

@@ -131,7 +131,7 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 		case '[':
 			return l.lexTableKey
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexVoid)
 		case '=':
 			return l.lexEqual
 		case '\r':
@@ -182,7 +182,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 		case '}':
 			return l.lexRightCurlyBrace
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexRvalue)
 		case '"':
 			return l.lexString
 		case '\'':
@@ -309,15 +309,17 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 	return l.lexVoid
 }
 
-func (l *tomlLexer) lexComment() tomlLexStateFn {
+func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn {
-	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+	return func() tomlLexStateFn {
-		if next == '\r' && l.follow("\r\n") {
+		for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
-			break
+			if next == '\r' && l.follow("\r\n") {
+				break
+			}
+			l.next()
 		}
-		l.next()
+		l.ignore()
+		return previousState
 	}
-	l.ignore()
-	return l.lexVoid
 }
 
 func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {

lexer_test.go

@@ -264,6 +264,24 @@ func TestMultilineArrayComments(t *testing.T) {
 	})
 }
 
+func TestNestedArraysComment(t *testing.T) {
+	toml := `
+someArray = [
+# does not work
+["entry1"]
+]`
+	testFlow(t, toml, []token{
+		{Position{2, 1}, tokenKey, "someArray"},
+		{Position{2, 11}, tokenEqual, "="},
+		{Position{2, 13}, tokenLeftBracket, "["},
+		{Position{4, 1}, tokenLeftBracket, "["},
+		{Position{4, 3}, tokenString, "entry1"},
+		{Position{4, 10}, tokenRightBracket, "]"},
+		{Position{5, 1}, tokenRightBracket, "]"},
+		{Position{5, 2}, tokenEOF, ""},
+	})
+}
+
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
 		{Position{1, 1}, tokenKey, "foo"},

parser_test.go

@@ -283,6 +283,17 @@ func TestArrayNested(t *testing.T) {
 	})
 }
 
+func TestNestedArrayComment(t *testing.T) {
+	tree, err := Load(`
+someArray = [
+# does not work
+["entry1"]
+]`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"someArray": [][]string{{"entry1"}},
+	})
+}
+
 func TestNestedEmptyArrays(t *testing.T) {
 	tree, err := Load("a = [[[]]]")
 	assertTree(t, tree, err, map[string]interface{}{