Merge branch 'toml-0.3.1'

lexer.go

@@ -138,7 +138,7 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			return l.lexRvalue
 		}
 
-		if isKeyChar(next) {
+		if isKeyStartChar(next) {
 			return l.lexKey
 		}
 
@@ -169,6 +169,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexComment
 		case '"':
 			return l.lexString
+		case '\'':
+			return l.lexLiteralString
 		case ',':
 			return l.lexComma
 		case '\n':
@@ -192,7 +194,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexKey
 		}
 
-		if dateRegexp.FindString(l.input[l.pos:]) != "" {
+		dateMatch := dateRegexp.FindString(l.input[l.pos:])
+		if dateMatch != "" {
+			l.ignore()
+			l.pos += len(dateMatch)
 			return l.lexDate
 		}
 
@@ -214,8 +219,6 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexDate() tomlLexStateFn {
-	l.ignore()
-	l.pos += 20 // Fixed size of a date in TOML
 	l.emit(tokenDate)
 	return l.lexRvalue
 }
@@ -250,7 +253,10 @@ func (l *tomlLexer) lexComma() tomlLexStateFn {
 
 func (l *tomlLexer) lexKey() tomlLexStateFn {
 	l.ignore()
-	for isKeyChar(l.next()) {
+	for r := l.next(); isKeyChar(r); r = l.next() {
+		if (r == '#') {
+			return l.errorf("keys cannot contain # character")
+		}
 	}
 	l.backup()
 	l.emit(tokenKey)
@@ -275,6 +281,29 @@ func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
+	l.pos++
+	l.ignore()
+	growingString := ""
+
+	for {
+		if l.peek() == '\'' {
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
+			l.ignore()
+			return l.lexRvalue
+		}
+
+		growingString += string(l.peek())
+
+		if l.next() == eof {
+			break
+		}
+	}
+
+	return l.errorf("unclosed string")
+}
+
 func (l *tomlLexer) lexString() tomlLexStateFn {
 	l.pos++
 	l.ignore()
@@ -418,6 +447,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 		l.accept("-")
 	}
 	pointSeen := false
+	expSeen := false
 	digitSeen := false
 	for {
 		next := l.next()
@@ -429,6 +459,11 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 				return l.errorf("float cannot end with a dot")
 			}
 			pointSeen = true
+		} else if next == 'e' || next == 'E' {
+			expSeen = true
+			if !l.accept("+") {
+				l.accept("-")
+			}
 		} else if isDigit(next) {
 			digitSeen = true
 		} else {
@@ -443,7 +478,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	if !digitSeen {
 		return l.errorf("no digit in that number")
 	}
-	if pointSeen {
+	if pointSeen || expSeen {
 		l.emit(tokenFloat)
 	} else {
 		l.emit(tokenInteger)
@@ -452,7 +487,7 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 }
 
 func init() {
-	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z")
+	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})")
 }
 
 // Entry point

lexer_test.go

@@ -118,19 +118,16 @@ func TestBasicKeyAndEqual(t *testing.T) {
 
 func TestKeyWithSharpAndEqual(t *testing.T) {
 	testFlow(t, "key#name = 5", []token{
-		token{Position{1, 1}, tokenKey, "key#name"},
+		token{Position{1, 1}, tokenError, "keys cannot contain # character"},
-		token{Position{1, 10}, tokenEqual, "="},
-		token{Position{1, 12}, tokenInteger, "5"},
-		token{Position{1, 13}, tokenEOF, ""},
 	})
 }
 
 func TestKeyWithSymbolsAndEqual(t *testing.T) {
-	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
-		token{Position{1, 1}, tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+		token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"},
-		token{Position{1, 39}, tokenEqual, "="},
+		token{Position{1, 38}, tokenEqual, "="},
-		token{Position{1, 41}, tokenInteger, "5"},
+		token{Position{1, 40}, tokenInteger, "5"},
-		token{Position{1, 42}, tokenEOF, ""},
+		token{Position{1, 41}, tokenEOF, ""},
 	})
 }
 
@@ -276,7 +273,13 @@ func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 
 func TestDateRegexp(t *testing.T) {
 	if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
-		t.Fail()
+		t.Error("basic lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
+		t.Error("offset lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
+		t.Error("nano precision lexing")
 	}
 }
 
@@ -287,6 +290,18 @@ func TestKeyEqualDate(t *testing.T) {
 		token{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
 		token{Position{1, 27}, tokenEOF, ""},
 	})
+	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
+		token{Position{1, 32}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
+		token{Position{1, 39}, tokenEOF, ""},
+	})
 }
 
 func TestFloatEndingWithDot(t *testing.T) {
@@ -305,6 +320,51 @@ func TestFloatWithTwoDots(t *testing.T) {
 	})
 }
 
+func TestFloatWithExponent1(t *testing.T) {
+	testFlow(t, "a = 5e+22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "5e+22"},
+		token{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent2(t *testing.T) {
+	testFlow(t, "a = 5E+22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "5E+22"},
+		token{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent3(t *testing.T) {
+	testFlow(t, "a = -5e+22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "-5e+22"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent4(t *testing.T) {
+	testFlow(t, "a = -5e-22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "-5e-22"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent5(t *testing.T) {
+	testFlow(t, "a = 6.626e-34", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "6.626e-34"},
+		token{Position{1, 14}, tokenEOF, ""},
+	})
+}
+
 func TestDoubleEqualKey(t *testing.T) {
 	testFlow(t, "foo= = 2", []token{
 		token{Position{1, 1}, tokenKey, "foo"},
@@ -400,6 +460,33 @@ func TestKeyEqualStringUnicodeEscape(t *testing.T) {
 	})
 }
 
+func TestLiteralString(t *testing.T) {
+	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
+		token{Position{1, 35}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `<\i\c*\s*>`},
+		token{Position{1, 19}, tokenEOF, ""},
+	})
+}
+
 func TestUnicodeString(t *testing.T) {
 	testFlow(t, `foo = "hello ♥ world"`, []token{
 		token{Position{1, 1}, tokenKey, "foo"},

parser.go

@@ -222,7 +222,7 @@ func (p *tomlParser) parseRvalue() interface{} {
 		}
 		return val
 	case tokenDate:
-		val, err := time.Parse(time.RFC3339, tok.val)
+		val, err := time.Parse(time.RFC3339Nano, tok.val)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}

parser_test.go

@@ -54,9 +54,9 @@ func TestSimpleKV(t *testing.T) {
 // NOTE: from the BurntSushi test suite
 // NOTE: this test is pure evil due to the embedded '.'
 func TestSpecialKV(t *testing.T) {
-	tree, err := Load("~!@#$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
+	tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
 	assertTree(t, tree, err, map[string]interface{}{
-		"~!@#$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
+		"~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
 	})
 }
 
@@ -70,6 +70,17 @@ func TestSimpleNumbers(t *testing.T) {
 	})
 }
 
+func TestFloatsWithExponents(t *testing.T) {
+	tree, err := Load("a = 5e+22\nb = 5E+22\nc = -5e+22\nd = -5e-22\ne = 6.626e-34")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": float64(5e+22),
+		"b": float64(5E+22),
+		"c": float64(-5e+22),
+		"d": float64(-5e-22),
+		"e": float64(6.626e-34),
+	})
+}
+
 func TestSimpleDate(t *testing.T) {
 	tree, err := Load("a = 1979-05-27T07:32:00Z")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -77,6 +88,21 @@ func TestSimpleDate(t *testing.T) {
 	})
 }
 
+func TestDateOffset(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7 * 60 * 60)),
+	})
+}
+
+func TestDateNano(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7 * 60 * 60)),
+	})
+}
+
+
 func TestSimpleString(t *testing.T) {
 	tree, err := Load("a = \"hello world\"")
 	assertTree(t, tree, err, map[string]interface{}{

token.go

@@ -117,11 +117,16 @@ func isAlphanumeric(r rune) bool {
 }
 
 func isKeyChar(r rune) bool {
-	// "Keys start with the first non-whitespace character and end with the last
+	// Keys start with the first character that isn't whitespace or [ and end
-	// non-whitespace character before the equals sign."
+	// with the last non-whitespace character before the equals sign. Keys
+	// cannot contain a # character."
 	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
 }
 
+func isKeyStartChar(r rune) bool {
+	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
+}
+
 func isDigit(r rune) bool {
 	return unicode.IsNumber(r)
 }