From 4e227cb34e700f0c6f4ddfb98ce9f0d4d11f1246 Mon Sep 17 00:00:00 2001
From: Alexander Staubo <alex@bengler.no>
Date: Tue, 19 Mar 2013 06:16:06 +0100
Subject: [PATCH] Support underscores and uppercase letters in key names, as
 well as international characters (not specified in TOML spec, but implied by
 Unicode and JSON compatibility). Fixes #3.

---
 lexer.go      | 13 +++++++------
 lexer_test.go | 21 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/lexer.go b/lexer.go
index 586263b..3db06eb 100644
--- a/lexer.go
+++ b/lexer.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 )
 
@@ -61,12 +62,12 @@ func isSpace(r rune) bool {
 	return r == ' ' || r == '\t'
 }
 
-func isAlpha(r rune) bool {
-	return r >= 'a' && r <= 'z'
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
 }
 
 func isDigit(r rune) bool {
-	return r >= '0' && r <= '9'
+	return unicode.IsNumber(r)
 }
 
 // Define lexer
@@ -156,7 +157,7 @@ func lexVoid(l *lexer) stateFn {
 			return lexEqual
 		}
 
-		if isAlpha(next) {
+		if isAlphanumeric(next) {
 			return lexKey
 		}
 
@@ -207,7 +208,7 @@ func lexRvalue(l *lexer) stateFn {
 			return lexFalse
 		}
 
-		if isAlpha(next) {
+		if isAlphanumeric(next) {
 			return lexKey
 		}
 
@@ -269,7 +270,7 @@ func lexComma(l *lexer) stateFn {
 
 func lexKey(l *lexer) stateFn {
 	l.ignore()
-	for isAlpha(l.next()) {
+	for isAlphanumeric(l.next()) {
 	}
 	l.backup()
 	l.emit(tokenKey)
diff --git a/lexer_test.go b/lexer_test.go
index b232705..03a1740 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -77,6 +77,27 @@ func TestBasicKey(t *testing.T) {
 	})
 }
 
+func TestBasicKeyWithUnderscore(t *testing.T) {
+	testFlow(t, "hello_hello", []token{
+		token{tokenKey, "hello_hello"},
+		token{tokenEOF, ""},
+	})
+}
+
+func TestBasicKeyWithUppercaseMix(t *testing.T) {
+	testFlow(t, "helloHELLOHello", []token{
+		token{tokenKey, "helloHELLOHello"},
+		token{tokenEOF, ""},
+	})
+}
+
+func TestBasicKeyWithInternationalCharacters(t *testing.T) {
+	testFlow(t, "héllÖ", []token{
+		token{tokenKey, "héllÖ"},
+		token{tokenEOF, ""},
+	})
+}
+
 func TestBasicKeyAndEqual(t *testing.T) {
 	testFlow(t, "hello =", []token{
 		token{tokenKey, "hello"},