Fix inline tables parsing

Inline tables were wrapped inside a TomlValue, although they should
just be part of the tree.

.travis.yml

@@ -1,11 +1,9 @@
 language: go
 script: "./test.sh"
-sudo: false
 go:
-    - 1.2.2
-    - 1.3.3
-    - 1.4.2
-    - 1.5.1
+    - 1.4.3
+    - 1.5.4
+    - 1.6.2
     - tip
 before_install:
   - go get github.com/axw/gocov/gocov

README.md

@@ -7,7 +7,7 @@ This library supports TOML version
 
 [![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
 [![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
-[![Coverage Status](https://coveralls.io/repos/pelletier/go-toml/badge.svg?branch=master&service=github)](https://coveralls.io/github/pelletier/go-toml?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master)
 
 ## Features
 
@@ -98,7 +98,7 @@ You can run both of them using `./test.sh`.
 
 ## License
 
-Copyright (c) 2013 - 2015 Thomas Pelletier, Eric Anderton
+Copyright (c) 2013 - 2016 Thomas Pelletier, Eric Anderton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

doc.go

@@ -1,7 +1,7 @@
 // Package toml is a TOML markup language parser.
 //
 // This version supports the specification as described in
-// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
+// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md
 //
 // TOML Parsing
 //
@@ -83,9 +83,9 @@
 // The idea behind a query path is to allow quick access to any element, or set
 // of elements within TOML document, with a single expression.
 //
-//   result := tree.Query("$.foo.bar.baz")  // result is 'nil' if the path is not present
+//   result, err := tree.Query("$.foo.bar.baz")
 //
-// This is equivalent to:
+// This is roughly equivalent to:
 //
 //   next := tree.Get("foo")
 //   if next != nil {
@@ -96,6 +96,11 @@
 //   }
 //   result := next
 //
+// err is nil if any parsing exception occurs.
+//
+// If no node in the tree matches the query, result will simply contain an empty list of
+// items.
+//
 // As illustrated above, the query path is much more efficient, especially since
 // the structure of the TOML file can vary.  Rather than making assumptions about
 // a document's structure, a query allows the programmer to make structured

doc_test.go

@@ -69,13 +69,13 @@ func Example_comprehensiveExample() {
 		fmt.Println("User is ", user, ". Password is ", password)
 
 		// show where elements are in the file
-		fmt.Println("User position: %v", configTree.GetPosition("user"))
-		fmt.Println("Password position: %v", configTree.GetPosition("password"))
+		fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
+		fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
 
 		// use a query to gather elements without walking the tree
 		results, _ := config.Query("$..[user,password]")
 		for ii, item := range results.Values() {
-			fmt.Println("Query result %d: %v", ii, item)
+			fmt.Printf("Query result %d: %v\n", ii, item)
 		}
 	}
 }

example-crlf.toml

@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it

keysparsing.go

@@ -70,6 +70,9 @@ func parseKey(key string) ([]string, error) {
 	if buffer.Len() > 0 {
 		groups = append(groups, buffer.String())
 	}
+	if len(groups) == 0 {
+		return nil, fmt.Errorf("empty key")
+	}
 	return groups, nil
 }
 

keysparsing_test.go

@@ -42,3 +42,8 @@ func TestDottedKeyBasic(t *testing.T) {
 func TestBaseKeyPound(t *testing.T) {
 	testError(t, "hello#world", "invalid bare character: #")
 }
+
+func TestEmptyKey(t *testing.T) {
+	testError(t, "", "empty key")
+	testError(t, " ", "empty key")
+}

lexer.go

@@ -6,11 +6,14 @@
 package toml
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode/utf8"
+
+	"github.com/pelletier/go-buffruneio"
 )
 
 var dateRegexp *regexp.Regexp
@@ -20,47 +23,56 @@ type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
 type tomlLexer struct {
-	input  string
-	start  int
-	pos    int
-	width  int
-	tokens chan token
-	depth  int
-	line   int
-	col    int
+	input         *buffruneio.Reader // Textual source
+	buffer        []rune             // Runes composing the current token
+	tokens        chan token
+	depth         int
+	line          int
+	col           int
+	endbufferLine int
+	endbufferCol  int
 }
 
-func (l *tomlLexer) run() {
-	for state := l.lexVoid; state != nil; {
-		state = state()
+// Basic read operations on input
+
+func (l *tomlLexer) read() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
 	}
-	close(l.tokens)
+	if r == '\n' {
+		l.endbufferLine++
+		l.endbufferCol = 1
+	} else {
+		l.endbufferCol++
+	}
+	return r
 }
 
-func (l *tomlLexer) nextStart() {
-	// iterate by runes (utf8 characters)
-	// search for newlines and advance line/col counts
-	for i := l.start; i < l.pos; {
-		r, width := utf8.DecodeRuneInString(l.input[i:])
-		if r == '\n' {
-			l.line++
-			l.col = 1
-		} else {
-			l.col++
-		}
-		i += width
+func (l *tomlLexer) next() rune {
+	r := l.read()
+
+	if r != eof {
+		l.buffer = append(l.buffer, r)
 	}
-	// advance start position to next token
-	l.start = l.pos
+	return r
 }
 
-func (l *tomlLexer) emit(t tokenType) {
-	l.tokens <- token{
-		Position: Position{l.line, l.col},
-		typ:      t,
-		val:      l.input[l.start:l.pos],
+func (l *tomlLexer) ignore() {
+	l.buffer = make([]rune, 0)
+	l.line = l.endbufferLine
+	l.col = l.endbufferCol
+}
+
+func (l *tomlLexer) skip() {
+	l.next()
+	l.ignore()
+}
+
+func (l *tomlLexer) fastForward(n int) {
+	for i := 0; i < n; i++ {
+		l.next()
 	}
-	l.nextStart()
 }
 
 func (l *tomlLexer) emitWithValue(t tokenType, value string) {
@@ -69,27 +81,37 @@ func (l *tomlLexer) emitWithValue(t tokenType, value string) {
 		typ:      t,
 		val:      value,
 	}
-	l.nextStart()
+	l.ignore()
 }
 
-func (l *tomlLexer) next() rune {
-	if l.pos >= len(l.input) {
-		l.width = 0
-		return eof
+func (l *tomlLexer) emit(t tokenType) {
+	l.emitWithValue(t, string(l.buffer))
+}
+
+func (l *tomlLexer) peek() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
 	}
-	var r rune
-	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
-	l.pos += l.width
+	l.input.UnreadRune()
 	return r
 }
 
-func (l *tomlLexer) ignore() {
-	l.nextStart()
+func (l *tomlLexer) follow(next string) bool {
+	for _, expectedRune := range next {
+		r, err := l.input.ReadRune()
+		defer l.input.UnreadRune()
+		if err != nil {
+			panic(err)
+		}
+		if expectedRune != r {
+			return false
+		}
+	}
+	return true
 }
 
-func (l *tomlLexer) backup() {
-	l.pos -= l.width
-}
+// Error management
 
 func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
 	l.tokens <- token{
@@ -100,23 +122,7 @@ func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
 	return nil
 }
 
-func (l *tomlLexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
-}
-
-func (l *tomlLexer) accept(valid string) bool {
-	if strings.IndexRune(valid, l.next()) >= 0 {
-		return true
-	}
-	l.backup()
-	return false
-}
-
-func (l *tomlLexer) follow(next string) bool {
-	return strings.HasPrefix(l.input[l.pos:], next)
-}
+// State functions
 
 func (l *tomlLexer) lexVoid() tomlLexStateFn {
 	for {
@@ -128,10 +134,15 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			return l.lexComment
 		case '=':
 			return l.lexEqual
+		case '\r':
+			fallthrough
+		case '\n':
+			l.skip()
+			continue
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
 		}
 
 		if l.depth > 0 {
@@ -142,7 +153,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			return l.lexKey
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
 	}
@@ -177,13 +189,16 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexLiteralString
 		case ',':
 			return l.lexComma
+		case '\r':
+			fallthrough
 		case '\n':
-			l.ignore()
-			l.pos++
+			l.skip()
 			if l.depth == 0 {
 				return l.lexVoid
 			}
 			return l.lexRvalue
+		case '_':
+			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
@@ -194,14 +209,20 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexFalse
 		}
 
-		if isAlphanumeric(next) {
-			return l.lexKey
+		if isSpace(next) {
+			l.skip()
+			continue
 		}
 
-		dateMatch := dateRegexp.FindString(l.input[l.pos:])
+		if next == eof {
+			l.next()
+			break
+		}
+
+		possibleDate := string(l.input.Peek(35))
+		dateMatch := dateRegexp.FindString(possibleDate)
 		if dateMatch != "" {
-			l.ignore()
-			l.pos += len(dateMatch)
+			l.fastForward(len(dateMatch))
 			return l.lexDate
 		}
 
@@ -209,13 +230,10 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexNumber
 		}
 
-		if isSpace(next) {
-			l.ignore()
+		if isAlphanumeric(next) {
+			return l.lexKey
 		}
 
-		if l.next() == eof {
-			break
-		}
 	}
 
 	l.emit(tokenEOF)
@@ -223,15 +241,13 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenLeftCurlyBrace)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenRightCurlyBrace)
 	return l.lexRvalue
 }
@@ -242,137 +258,147 @@ func (l *tomlLexer) lexDate() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexTrue() tomlLexStateFn {
-	l.ignore()
-	l.pos += 4
+	l.fastForward(4)
 	l.emit(tokenTrue)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexFalse() tomlLexStateFn {
-	l.ignore()
-	l.pos += 5
+	l.fastForward(5)
 	l.emit(tokenFalse)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexEqual() tomlLexStateFn {
-	l.ignore()
-	l.accept("=")
+	l.next()
 	l.emit(tokenEqual)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexComma() tomlLexStateFn {
-	l.ignore()
-	l.accept(",")
+	l.next()
 	l.emit(tokenComma)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexKey() tomlLexStateFn {
-	l.ignore()
-	inQuotes := false
-	for r := l.next(); isKeyChar(r) || r == '\n'; r = l.next() {
+	growingString := ""
+
+	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
-			inQuotes = !inQuotes
+			l.next()
+			str, err := l.lexStringAsString(`"`, false, true)
+			if err != nil {
+				return l.errorf(err.Error())
+			}
+			growingString += `"` + str + `"`
+			l.next()
+			continue
 		} else if r == '\n' {
 			return l.errorf("keys cannot contain new lines")
-		} else if isSpace(r) && !inQuotes {
+		} else if isSpace(r) {
 			break
-		} else if !isValidBareChar(r) && !inQuotes {
+		} else if !isValidBareChar(r) {
 			return l.errorf("keys cannot contain %c character", r)
 		}
+		growingString += string(r)
+		l.next()
 	}
-	l.backup()
-	l.emit(tokenKey)
+	l.emitWithValue(tokenKey, growingString)
 	return l.lexVoid
 }
 
 func (l *tomlLexer) lexComment() tomlLexStateFn {
-	for {
-		next := l.next()
-		if next == '\n' || next == eof {
+	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+		if next == '\r' && l.follow("\r\n") {
 			break
 		}
+		l.next()
 	}
 	l.ignore()
 	return l.lexVoid
 }
 
 func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenLeftBracket)
 	return l.lexRvalue
 }
 
-func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
-	l.pos++
-	l.ignore()
+func (l *tomlLexer) lexLiteralStringAsString(terminator string, discardLeadingNewLine bool) (string, error) {
 	growingString := ""
 
-	// handle special case for triple-quote
-	terminator := "'"
-	if l.follow("''") {
-		l.pos += 2
-		l.ignore()
-		terminator = "'''"
-
-		// special case: discard leading newline
-		if l.peek() == '\n' {
-			l.pos++
-			l.ignore()
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
 		}
 	}
 
 	// find end of string
 	for {
 		if l.follow(terminator) {
-			l.emitWithValue(tokenString, growingString)
-			l.pos += len(terminator)
-			l.ignore()
-			return l.lexRvalue
+			return growingString, nil
 		}
 
-		growingString += string(l.peek())
-
-		if l.next() == eof {
+		next := l.peek()
+		if next == eof {
 			break
 		}
+		growingString += string(l.next())
 	}
 
-	return l.errorf("unclosed string")
+	return "", errors.New("unclosed string")
 }
 
-func (l *tomlLexer) lexString() tomlLexStateFn {
-	l.pos++
-	l.ignore()
-	growingString := ""
+func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
+	l.skip()
 
 	// handle special case for triple-quote
-	terminator := "\""
-	if l.follow("\"\"") {
-		l.pos += 2
-		l.ignore()
-		terminator = "\"\"\""
+	terminator := "'"
+	discardLeadingNewLine := false
+	if l.follow("''") {
+		l.skip()
+		l.skip()
+		terminator = "'''"
+		discardLeadingNewLine = true
+	}
 
-		// special case: discard leading newline
-		if l.peek() == '\n' {
-			l.pos++
-			l.ignore()
+	str, err := l.lexLiteralStringAsString(terminator, discardLeadingNewLine)
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
+	l.ignore()
+	return l.lexRvalue
+}
+
+// Lex a string and return the results as a string.
+// Terminator is the substring indicating the end of the token.
+// The resulting string does not include the terminator.
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
+	growingString := ""
+
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
 		}
 	}
 
 	for {
 		if l.follow(terminator) {
-			l.emitWithValue(tokenString, growingString)
-			l.pos += len(terminator)
-			l.ignore()
-			return l.lexRvalue
+			return growingString, nil
 		}
 
 		if l.follow("\\") {
-			l.pos++
+			l.next()
 			switch l.peek() {
 			case '\r':
 				fallthrough
@@ -382,87 +408,119 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 				fallthrough
 			case ' ':
 				// skip all whitespace chars following backslash
-				l.pos++
 				for strings.ContainsRune("\r\n\t ", l.peek()) {
-					l.pos++
+					l.next()
 				}
-				l.pos--
 			case '"':
 				growingString += "\""
+				l.next()
 			case 'n':
 				growingString += "\n"
+				l.next()
 			case 'b':
 				growingString += "\b"
+				l.next()
 			case 'f':
 				growingString += "\f"
+				l.next()
 			case '/':
 				growingString += "/"
+				l.next()
 			case 't':
 				growingString += "\t"
+				l.next()
 			case 'r':
 				growingString += "\r"
+				l.next()
 			case '\\':
 				growingString += "\\"
+				l.next()
 			case 'u':
-				l.pos++
+				l.next()
 				code := ""
 				for i := 0; i < 4; i++ {
 					c := l.peek()
-					l.pos++
 					if !isHexDigit(c) {
-						return l.errorf("unfinished unicode escape")
+						return "", errors.New("unfinished unicode escape")
 					}
+					l.next()
 					code = code + string(c)
 				}
-				l.pos--
 				intcode, err := strconv.ParseInt(code, 16, 32)
 				if err != nil {
-					return l.errorf("invalid unicode escape: \\u" + code)
+					return "", errors.New("invalid unicode escape: \\u" + code)
 				}
 				growingString += string(rune(intcode))
 			case 'U':
-				l.pos++
+				l.next()
 				code := ""
 				for i := 0; i < 8; i++ {
 					c := l.peek()
-					l.pos++
 					if !isHexDigit(c) {
-						return l.errorf("unfinished unicode escape")
+						return "", errors.New("unfinished unicode escape")
 					}
+					l.next()
 					code = code + string(c)
 				}
-				l.pos--
 				intcode, err := strconv.ParseInt(code, 16, 64)
 				if err != nil {
-					return l.errorf("invalid unicode escape: \\U" + code)
+					return "", errors.New("invalid unicode escape: \\U" + code)
 				}
 				growingString += string(rune(intcode))
 			default:
-				return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+				return "", errors.New("invalid escape sequence: \\" + string(l.peek()))
 			}
 		} else {
 			r := l.peek()
-			if 0x00 <= r && r <= 0x1F {
-				return l.errorf("unescaped control character %U", r)
+
+			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) {
+				return "", fmt.Errorf("unescaped control character %U", r)
 			}
+			l.next()
 			growingString += string(r)
 		}
 
-		if l.next() == eof {
+		if l.peek() == eof {
 			break
 		}
 	}
 
-	return l.errorf("unclosed string")
+	return "", errors.New("unclosed string")
+}
+
+func (l *tomlLexer) lexString() tomlLexStateFn {
+	l.skip()
+
+	// handle special case for triple-quote
+	terminator := `"`
+	discardLeadingNewLine := false
+	acceptNewLines := false
+	if l.follow(`""`) {
+		l.skip()
+		l.skip()
+		terminator = `"""`
+		discardLeadingNewLine = true
+		acceptNewLines = true
+	}
+
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
+
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
+	l.ignore()
+	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 
 	if l.peek() == '[' {
 		// token '[[' signifies an array of anonymous key groups
-		l.pos++
+		l.next()
 		l.emit(tokenDoubleLeftBracket)
 		return l.lexInsideKeyGroupArray
 	}
@@ -472,87 +530,85 @@ func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroupArray)
 			}
-			l.ignore()
-			l.pos++
+			l.next()
 			if l.peek() != ']' {
-				break // error
+				break
 			}
-			l.pos++
+			l.next()
 			l.emit(tokenDoubleRightBracket)
 			return l.lexVoid
-		} else if l.peek() == '[' {
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		default:
+			l.next()
 		}
 	}
 	return l.errorf("unclosed key group array")
 }
 
 func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroup)
 			}
-			l.ignore()
-			l.pos++
+			l.next()
 			l.emit(tokenRightBracket)
 			return l.lexVoid
-		} else if l.peek() == '[' {
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		default:
+			l.next()
 		}
 	}
 	return l.errorf("unclosed key group")
 }
 
 func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenRightBracket)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexNumber() tomlLexStateFn {
-	l.ignore()
-	if !l.accept("+") {
-		l.accept("-")
+	r := l.peek()
+	if r == '+' || r == '-' {
+		l.next()
 	}
 	pointSeen := false
 	expSeen := false
 	digitSeen := false
 	for {
-		next := l.next()
+		next := l.peek()
 		if next == '.' {
 			if pointSeen {
 				return l.errorf("cannot have two dots in one float")
 			}
+			l.next()
 			if !isDigit(l.peek()) {
 				return l.errorf("float cannot end with a dot")
 			}
 			pointSeen = true
 		} else if next == 'e' || next == 'E' {
 			expSeen = true
-			if !l.accept("+") {
-				l.accept("-")
+			l.next()
+			r := l.peek()
+			if r == '+' || r == '-' {
+				l.next()
 			}
 		} else if isDigit(next) {
 			digitSeen = true
+			l.next()
 		} else if next == '_' {
-			l.pos++
+			l.next()
 		} else {
-			l.backup()
 			break
 		}
 		if pointSeen && !digitSeen {
@@ -571,17 +627,27 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+func (l *tomlLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
+}
+
 func init() {
-	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})")
+	dateRegexp = regexp.MustCompile(`^\d{1,4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})`)
 }
 
 // Entry point
-func lexToml(input string) chan token {
+func lexToml(input io.Reader) chan token {
+	bufferedInput := buffruneio.NewReader(input)
 	l := &tomlLexer{
-		input:  input,
-		tokens: make(chan token),
-		line:   1,
-		col:    1,
+		input:         bufferedInput,
+		tokens:        make(chan token),
+		line:          1,
+		col:           1,
+		endbufferLine: 1,
+		endbufferCol:  1,
 	}
 	go l.run()
 	return l.tokens

lexer_test.go

@@ -1,15 +1,19 @@
 package toml
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	ch := lexToml(input)
+	ch := lexToml(strings.NewReader(input))
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {
 			t.Log("While testing: ", input)
 			t.Log("compared (got)", token, "to (expected)", expected)
 			t.Log("\tvalue:", token.val, "<->", expected.val)
+			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
 			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
 			t.Log("\tline:", token.Line, "<->", expected.Line)
 			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
@@ -83,6 +87,18 @@ func TestMultipleKeyGroupsComment(t *testing.T) {
 	})
 }
 
+func TestSimpleWindowsCRLF(t *testing.T) {
+	testFlow(t, "a=4\r\nb=2", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenInteger, "4"},
+		token{Position{2, 1}, tokenKey, "b"},
+		token{Position{2, 2}, tokenEqual, "="},
+		token{Position{2, 3}, tokenInteger, "2"},
+		token{Position{2, 4}, tokenEOF, ""},
+	})
+}
+
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
 		token{Position{1, 1}, tokenKey, "hello"},
@@ -464,6 +480,12 @@ func TestKeyEqualNumber(t *testing.T) {
 		token{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
 		token{Position{1, 33}, tokenEOF, ""},
 	})
+
+	testFlow(t, "foo = +", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenError, "no digit in that number"},
+	})
 }
 
 func TestMultiline(t *testing.T) {
@@ -491,6 +513,16 @@ func TestKeyEqualStringUnicodeEscape(t *testing.T) {
 		token{Position{1, 8}, tokenString, "hello δ"},
 		token{Position{1, 25}, tokenEOF, ""},
 	})
+	testFlow(t, `foo = "\u2"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unfinished unicode escape"},
+	})
+	testFlow(t, `foo = "\U2"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unfinished unicode escape"},
+	})
 }
 
 func TestKeyEqualStringNoEscape(t *testing.T) {
@@ -531,6 +563,11 @@ func TestLiteralString(t *testing.T) {
 		token{Position{1, 8}, tokenString, `<\i\c*\s*>`},
 		token{Position{1, 19}, tokenEOF, ""},
 	})
+	testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unclosed string"},
+	})
 }
 
 func TestMultilineLiteralString(t *testing.T) {
@@ -547,6 +584,12 @@ func TestMultilineLiteralString(t *testing.T) {
 		token{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
 		token{Position{4, 9}, tokenEOF, ""},
 	})
+	testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
+		token{Position{4, 9}, tokenEOF, ""},
+	})
 }
 
 func TestMultilineString(t *testing.T) {
@@ -557,7 +600,7 @@ func TestMultilineString(t *testing.T) {
 		token{Position{1, 34}, tokenEOF, ""},
 	})
 
-	testFlow(t, "foo = \"\"\"\nhello\\\n\"literal\"\\\nworld\"\"\"", []token{
+	testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
 		token{Position{1, 1}, tokenKey, "foo"},
 		token{Position{1, 5}, tokenEqual, "="},
 		token{Position{2, 1}, tokenString, "hello\"literal\"world"},
@@ -584,6 +627,20 @@ func TestMultilineString(t *testing.T) {
 		token{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
 		token{Position{5, 11}, tokenEOF, ""},
 	})
+
+	testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
+		token{Position{1, 1}, tokenKey, "key2"},
+		token{Position{1, 6}, tokenEqual, "="},
+		token{Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
+		token{Position{1, 41}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
+		token{Position{1, 1}, tokenKey, "key2"},
+		token{Position{1, 6}, tokenEqual, "="},
+		token{Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
+		token{Position{3, 20}, tokenEOF, ""},
+	})
 }
 
 func TestUnicodeString(t *testing.T) {
@@ -594,6 +651,14 @@ func TestUnicodeString(t *testing.T) {
 		token{Position{1, 22}, tokenEOF, ""},
 	})
 }
+func TestEscapeInString(t *testing.T) {
+	testFlow(t, `foo = "\b\f\/"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "\b\f/"},
+		token{Position{1, 15}, tokenEOF, ""},
+	})
+}
 
 func TestKeyGroupArray(t *testing.T) {
 	testFlow(t, "[[foo]]", []token{
@@ -618,3 +683,12 @@ func TestKeyNewline(t *testing.T) {
 		token{Position{1, 1}, tokenError, "keys cannot contain new lines"},
 	})
 }
+
+func TestInvalidFloat(t *testing.T) {
+	testFlow(t, "a=7e1_", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenFloat, "7e1_"},
+		token{Position{1, 7}, tokenEOF, ""},
+	})
+}

match.go

@@ -67,7 +67,14 @@ func newMatchKeyFn(name string) *matchKeyFn {
 }
 
 func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
+	if array, ok := node.([]*TomlTree); ok {
+		for _, tree := range array {
+			item := tree.values[f.Name]
+			if item != nil {
+				f.next.call(item, ctx)
+			}
+		}
+	} else if tree, ok := node.(*TomlTree); ok {
 		item := tree.values[f.Name]
 		if item != nil {
 			f.next.call(item, ctx)
@@ -202,7 +209,7 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
 	fn, ok := (*ctx.filters)[f.Name]
 	if !ok {
 		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
-			f.Pos, f.Name))
+			f.Pos.String(), f.Name))
 	}
 	switch castNode := tomlValueCheck(node, ctx).(type) {
 	case *TomlTree:

match_test.go

@@ -109,7 +109,7 @@ func TestPathSliceStart(t *testing.T) {
 	assertPath(t,
 		"$[123:]",
 		buildPath(
-			newMatchSliceFn(123, MaxInt, 1),
+			newMatchSliceFn(123, maxInt, 1),
 		))
 }
 
@@ -133,7 +133,7 @@ func TestPathSliceStartStep(t *testing.T) {
 	assertPath(t,
 		"$[123::7]",
 		buildPath(
-			newMatchSliceFn(123, MaxInt, 7),
+			newMatchSliceFn(123, maxInt, 7),
 		))
 }
 
@@ -149,7 +149,7 @@ func TestPathSliceStep(t *testing.T) {
 	assertPath(t,
 		"$[::7]",
 		buildPath(
-			newMatchSliceFn(0, MaxInt, 7),
+			newMatchSliceFn(0, maxInt, 7),
 		))
 }
 

parser.go

@@ -5,6 +5,7 @@ package toml
 import (
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -107,7 +108,7 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn {
 	var array []*TomlTree
 	if destTree == nil {
 		array = make([]*TomlTree, 0)
-	} else if destTree.([]*TomlTree) != nil {
+	} else if target, ok := destTree.([]*TomlTree); ok && target != nil {
 		array = destTree.([]*TomlTree)
 	} else {
 		p.raiseError(key, "key %s is already assigned and not of type group array", key)
@@ -207,10 +208,28 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 		p.raiseError(key, "The following key was defined twice: %s",
 			strings.Join(finalKey, "."))
 	}
-	targetNode.values[keyVal] = &tomlValue{value, key.Position}
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *TomlTree:
+		toInsert = value
+	default:
+		toInsert = &tomlValue{value, key.Position}
+	}
+	targetNode.values[keyVal] = toInsert
 	return p.parseStart
 }
 
+var numberUnderscoreInvalidRegexp *regexp.Regexp
+
+func cleanupNumberToken(value string) (string, error) {
+	if numberUnderscoreInvalidRegexp.MatchString(value) {
+		return "", fmt.Errorf("invalid use of _ in number")
+	}
+	cleanedVal := strings.Replace(value, "_", "", -1)
+	return cleanedVal, nil
+}
+
 func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
@@ -225,14 +244,20 @@ func (p *tomlParser) parseRvalue() interface{} {
 	case tokenFalse:
 		return false
 	case tokenInteger:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
+		cleanedVal, err := cleanupNumberToken(tok.val)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
+		}
 		val, err := strconv.ParseInt(cleanedVal, 10, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
+		cleanedVal, err := cleanupNumberToken(tok.val)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
+		}
 		val, err := strconv.ParseFloat(cleanedVal, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
@@ -361,3 +386,7 @@ func parseToml(flow chan token) *TomlTree {
 	parser.run()
 	return result
 }
+
+func init() {
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`)
+}

parser_test.go

@@ -2,26 +2,34 @@ package toml
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
-func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref map[string]interface{}) {
 	if err != nil {
 		t.Error("Non-nil error:", err.Error())
 		return
 	}
 	for k, v := range ref {
+		nextPath := append(path, k)
+		t.Log("asserting path", nextPath)
 		// NOTE: directly access key instead of resolve by path
 		// NOTE: see TestSpecialKV
 		switch node := tree.GetPath([]string{k}).(type) {
 		case []*TomlTree:
+			t.Log("\tcomparing key", nextPath, "by array iteration")
 			for idx, item := range node {
-				assertTree(t, item, err, v.([]map[string]interface{})[idx])
+				assertSubTree(t, nextPath, item, err, v.([]map[string]interface{})[idx])
 			}
 		case *TomlTree:
-			assertTree(t, node, err, v.(map[string]interface{}))
+			t.Log("\tcomparing key", nextPath, "by subtree assestion")
+			assertSubTree(t, nextPath, node, err, v.(map[string]interface{}))
 		default:
+			t.Log("\tcomparing key", nextPath, "by string representation because it's of type", reflect.TypeOf(node))
 			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
 				t.Errorf("was expecting %v at %v but got %v", v, k, node)
 			}
@@ -29,6 +37,12 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 	}
 }
 
+func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+	t.Log("Asserting tree:\n", spew.Sdump(tree))
+	assertSubTree(t, []string{}, tree, err, ref)
+	t.Log("Finished tree assertion.")
+}
+
 func TestCreateSubTree(t *testing.T) {
 	tree := newTomlTree()
 	tree.createSubTree([]string{"a", "b", "c"}, Position{})
@@ -287,7 +301,7 @@ func TestArrayNestedStrings(t *testing.T) {
 
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "(1, 4): expecting a value" {
+	if err.Error() != "(1, 5): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -441,7 +455,7 @@ func TestImplicitDeclarationBefore(t *testing.T) {
 
 func TestFloatsWithoutLeadingZeros(t *testing.T) {
 	_, err := Load("a = .42")
-	if err.Error() != "(1, 4): cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 
@@ -494,6 +508,42 @@ func TestParseFile(t *testing.T) {
 	})
 }
 
+func TestParseFileCRLF(t *testing.T) {
+	tree, err := LoadFile("example-crlf.toml")
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
+	})
+}
+
 func TestParseKeyGroupArray(t *testing.T) {
 	tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -506,6 +556,40 @@ func TestParseKeyGroupArray(t *testing.T) {
 	})
 }
 
+func TestParseKeyGroupArrayUnfinished(t *testing.T) {
+	_, err := Load("[[foo.bar]\na = 42")
+	if err.Error() != "(1, 10): was expecting token [[, but got unclosed key group array instead" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("[[foo.[bar]\na = 42")
+	if err.Error() != "(1, 3): unexpected token group name cannot contain ']', was expecting a key group array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestParseKeyGroupArrayQueryExample(t *testing.T) {
+	tree, err := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"book": []map[string]interface{}{
+			{"title": "The Stand", "author": "Stephen King"},
+			{"title": "For Whom the Bell Tolls", "author": "Ernest Hemmingway"},
+			{"title": "Neuromancer", "author": "William Gibson"},
+		},
+	})
+}
+
 func TestParseKeyGroupArraySpec(t *testing.T) {
 	tree, err := Load("[[fruit]]\n name=\"apple\"\n [fruit.physical]\n color=\"red\"\n shape=\"round\"\n [[fruit]]\n name=\"banana\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -618,6 +702,11 @@ func TestInvalidGroupArray(t *testing.T) {
 	if err == nil {
 		t.Error("Should error")
 	}
+
+	_, err = Load("[foo.[bar]\na = 42")
+	if err.Error() != "(1, 2): unexpected token group name cannot contain ']', was expecting a key group" {
+		t.Error("Bad error message:", err.Error())
+	}
 }
 
 func TestDoubleEqual(t *testing.T) {
@@ -626,3 +715,32 @@ func TestDoubleEqual(t *testing.T) {
 		t.Error("Bad error message:", err.Error())
 	}
 }
+
+func TestGroupArrayReassign(t *testing.T) {
+	_, err := Load("[hello]\n[[hello]]")
+	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type group array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInvalidFloatParsing(t *testing.T) {
+	_, err := Load("a=1e_2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1e2_")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1__2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=_1_2")
+	if err.Error() != "(1, 3): cannot start number with underscore" {
+		t.Error("Bad error message:", err.Error())
+	}
+}

position.go

@@ -6,13 +6,11 @@ import (
 	"fmt"
 )
 
-/*
-  Position of a document element within a TOML document.
-
-  Line and Col are both 1-indexed positions for the element's line number and
-  column number, respectively.  Values of zero or less will cause Invalid(),
-  to return true.
-*/
+// Position of a document element within a TOML document.
+//
+// Line and Col are both 1-indexed positions for the element's line number and
+// column number, respectively.  Values of zero or less will cause Invalid(),
+// to return true.
 type Position struct {
 	Line int // line within the document
 	Col  int // column within the line
@@ -24,7 +22,7 @@ func (p *Position) String() string {
 	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
 }
 
-// Returns whether or not the position is valid (i.e. with negative or
+// Invalid returns whether or not the position is valid (i.e. with negative or
 // null values)
 func (p *Position) Invalid() bool {
 	return p.Line <= 0 || p.Col <= 0

query.go

@@ -4,36 +4,47 @@ import (
 	"time"
 )
 
-//  Type of a user-defined filter function, for use with Query.SetFilter().
+// NodeFilterFn represents a user-defined filter function, for use with
+// Query.SetFilter().
 //
-//  The return value of the function must indicate if 'node' is to be included
-//  at this stage of the TOML path.  Returning true will include the node, and
-//  returning false will exclude it.
+// The return value of the function must indicate if 'node' is to be included
+// at this stage of the TOML path.  Returning true will include the node, and
+// returning false will exclude it.
 //
-//  NOTE: Care should be taken to write script callbacks such that they are safe
-//  to use from multiple goroutines.
+// NOTE: Care should be taken to write script callbacks such that they are safe
+// to use from multiple goroutines.
 type NodeFilterFn func(node interface{}) bool
 
-// The result of Executing a Query
+// QueryResult is the result of Executing a Query.
 type QueryResult struct {
 	items     []interface{}
 	positions []Position
 }
 
-// appends a value/position pair to the result set
+// appends a value/position pair to the result set.
 func (r *QueryResult) appendResult(node interface{}, pos Position) {
 	r.items = append(r.items, node)
 	r.positions = append(r.positions, pos)
 }
 
-// Set of values within a QueryResult.  The order of values is not guaranteed
-// to be in document order, and may be different each time a query is executed.
+// Values is a set of values within a QueryResult.  The order of values is not
+// guaranteed to be in document order, and may be different each time a query is
+// executed.
 func (r *QueryResult) Values() []interface{} {
-	return r.items
+	values := make([]interface{}, len(r.items))
+	for i, v := range r.items {
+		o, ok := v.(*tomlValue)
+		if ok {
+			values[i] = o.value
+		} else {
+			values[i] = v
+		}
+	}
+	return values
 }
 
-// Set of positions for values within a QueryResult.  Each index in Positions()
-// corresponds to the entry in Value() of the same index.
+// Positions is a set of positions for values within a QueryResult.  Each index
+// in Positions() corresponds to the entry in Value() of the same index.
 func (r *QueryResult) Positions() []Position {
 	return r.positions
 }
@@ -77,13 +88,13 @@ func (q *Query) appendPath(next pathFn) {
 	next.setNext(newTerminatingFn()) // init the next functor
 }
 
-// Compiles a TOML path expression.  The returned Query can be used to match
-// elements within a TomlTree and its descendants.
+// CompileQuery compiles a TOML path expression.  The returned Query can be used
+// to match elements within a TomlTree and its descendants.
 func CompileQuery(path string) (*Query, error) {
 	return parseQuery(lexQuery(path))
 }
 
-// Executes a query against a TomlTree, and returns the result of the query.
+// Execute executes a query against a TomlTree, and returns the result of the query.
 func (q *Query) Execute(tree *TomlTree) *QueryResult {
 	result := &QueryResult{
 		items:     []interface{}{},
@@ -101,8 +112,8 @@ func (q *Query) Execute(tree *TomlTree) *QueryResult {
 	return result
 }
 
-// Sets a user-defined filter function.  These may be used inside "?(..)" query
-// expressions to filter TOML document elements within a query.
+// SetFilter sets a user-defined filter function.  These may be used inside
+// "?(..)" query expressions to filter TOML document elements within a query.
 func (q *Query) SetFilter(name string, fn NodeFilterFn) {
 	if q.filters == &defaultFilterFunctions {
 		// clone the static table

query_test.go

@@ -0,0 +1,70 @@
+package toml
+
+import (
+	"testing"
+)
+
+func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
+	if len(array) != len(objects) {
+		t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
+	}
+
+	for _, o := range objects {
+		found := false
+		for _, a := range array {
+			if a == o {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal(o, "not found in array", array)
+		}
+	}
+}
+
+func TestQueryExample(t *testing.T) {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	authors, _ := config.Query("$.book.author")
+	names := authors.Values()
+	if len(names) != 3 {
+		t.Fatalf("query should return 3 names but returned %d", len(names))
+	}
+	assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
+}
+
+func TestQueryReadmeExample(t *testing.T) {
+	config, _ := Load(`
+[postgres]
+user = "pelletier"
+password = "mypassword"
+`)
+	results, _ := config.Query("$..[user,password]")
+	values := results.Values()
+	if len(values) != 2 {
+		t.Fatalf("query should return 2 values but returned %d", len(values))
+	}
+	assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
+}
+
+func TestQueryPathNotPresent(t *testing.T) {
+	config, _ := Load(`a = "hello"`)
+	results, err := config.Query("$.foo.bar")
+	if err != nil {
+		t.Fatalf("err should be nil. got %s instead", err)
+	}
+	if len(results.items) != 0 {
+		t.Fatalf("no items should be matched. %d matched instead", len(results.items))
+	}
+}

querylexer.go

@@ -105,7 +105,7 @@ func (l *queryLexer) peek() rune {
 }
 
 func (l *queryLexer) accept(valid string) bool {
-	if strings.IndexRune(valid, l.next()) >= 0 {
+	if strings.ContainsRune(valid, l.next()) {
 		return true
 	}
 	l.backup()

queryparser.go

@@ -11,7 +11,7 @@ import (
 	"fmt"
 )
 
-const MaxInt = int(^uint(0) >> 1)
+const maxInt = int(^uint(0) >> 1)
 
 type queryParser struct {
 	flow         chan token
@@ -138,7 +138,6 @@ func (p *queryParser) parseMatchExpr() queryParserStateFn {
 		return nil // allow EOF at this stage
 	}
 	return p.parseError(tok, "expected match expression")
-	return nil
 }
 
 func (p *queryParser) parseBracketExpr() queryParserStateFn {
@@ -204,7 +203,7 @@ loop: // labeled loop for easy breaking
 
 func (p *queryParser) parseSliceExpr() queryParserStateFn {
 	// init slice to grab all elements
-	start, end, step := 0, MaxInt, 1
+	start, end, step := 0, maxInt, 1
 
 	// parse optional start
 	tok := p.getToken()

test.sh

@@ -19,6 +19,9 @@ function git_clone() {
   popd
 }
 
+go get github.com/pelletier/go-buffruneio
+go get github.com/davecgh/go-spew/spew
+
 # get code for BurntSushi TOML validation
 # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)
 git_clone github.com/BurntSushi/toml master HEAD
@@ -66,7 +69,7 @@ else
     echo "Invalid Test TOML for $test:"
     echo "===="
     cat "$invalid_test.toml"
-    
+
     echo "Go-TOML Output for $test:"
     echo "===="
     echo "go-toml Output:"

token.go

@@ -107,9 +107,6 @@ func (t token) String() string {
 		return t.val
 	}
 
-	if len(t.val) > 10 {
-		return fmt.Sprintf("%.10q...", t.val)
-	}
 	return fmt.Sprintf("%q", t.val)
 }
 

token_test.go

@@ -0,0 +1,67 @@
+package toml
+
+import "testing"
+
+func TestTokenStringer(t *testing.T) {
+	var tests = []struct {
+		tt     tokenType
+		expect string
+	}{
+		{tokenError, "Error"},
+		{tokenEOF, "EOF"},
+		{tokenComment, "Comment"},
+		{tokenKey, "Key"},
+		{tokenString, "String"},
+		{tokenInteger, "Integer"},
+		{tokenTrue, "True"},
+		{tokenFalse, "False"},
+		{tokenFloat, "Float"},
+		{tokenEqual, "="},
+		{tokenLeftBracket, "["},
+		{tokenRightBracket, "]"},
+		{tokenLeftCurlyBrace, "{"},
+		{tokenRightCurlyBrace, "}"},
+		{tokenLeftParen, "("},
+		{tokenRightParen, ")"},
+		{tokenDoubleLeftBracket, "]]"},
+		{tokenDoubleRightBracket, "[["},
+		{tokenDate, "Date"},
+		{tokenKeyGroup, "KeyGroup"},
+		{tokenKeyGroupArray, "KeyGroupArray"},
+		{tokenComma, ","},
+		{tokenColon, ":"},
+		{tokenDollar, "$"},
+		{tokenStar, "*"},
+		{tokenQuestion, "?"},
+		{tokenDot, "."},
+		{tokenDotDot, ".."},
+		{tokenEOL, "EOL"},
+		{tokenEOL + 1, "Unknown"},
+	}
+
+	for i, test := range tests {
+		got := test.tt.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid string of token type; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}
+
+func TestTokenString(t *testing.T) {
+	var tests = []struct {
+		tok    token
+		expect string
+	}{
+		{token{Position{1, 1}, tokenEOF, ""}, "EOF"},
+		{token{Position{1, 1}, tokenError, "Δt"}, "Δt"},
+		{token{Position{1, 1}, tokenString, "bar"}, `"bar"`},
+		{token{Position{1, 1}, tokenString, "123456789012345"}, `"123456789012345"`},
+	}
+
+	for i, test := range tests {
+		got := test.tok.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid of string token; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}

toml.go

@@ -3,11 +3,10 @@ package toml
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"os"
 	"runtime"
-	"strconv"
 	"strings"
-	"time"
 )
 
 type tomlValue struct {
@@ -28,6 +27,7 @@ func newTomlTree() *TomlTree {
 	}
 }
 
+// TreeFromMap initializes a new TomlTree object using the given map.
 func TreeFromMap(m map[string]interface{}) *TomlTree {
 	return &TomlTree{
 		values: m,
@@ -94,7 +94,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 			}
 			subtree = node[len(node)-1]
 		default:
-			return nil // cannot naigate through other node types
+			return nil // cannot navigate through other node types
 		}
 	}
 	// branch based on final node type
@@ -246,122 +246,17 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	return nil
 }
 
-// encodes a string to a TOML-compliant string value
-func encodeTomlString(value string) string {
-	result := ""
-	for _, rr := range value {
-		intRr := uint16(rr)
-		switch rr {
-		case '\b':
-			result += "\\b"
-		case '\t':
-			result += "\\t"
-		case '\n':
-			result += "\\n"
-		case '\f':
-			result += "\\f"
-		case '\r':
-			result += "\\r"
-		case '"':
-			result += "\\\""
-		case '\\':
-			result += "\\\\"
-		default:
-			if intRr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", intRr)
-			} else {
-				result += string(rr)
-			}
-		}
-	}
-	return result
-}
-
-// Value print support function for ToString()
-// Outputs the TOML compliant string representation of a value
-func toTomlValue(item interface{}, indent int) string {
-	tab := strings.Repeat(" ", indent)
-	switch value := item.(type) {
-	case int64:
-		return tab + strconv.FormatInt(value, 10)
-	case float64:
-		return tab + strconv.FormatFloat(value, 'f', -1, 64)
-	case string:
-		return tab + "\"" + encodeTomlString(value) + "\""
-	case bool:
-		if value {
-			return "true"
-		}
-		return "false"
-	case time.Time:
-		return tab + value.Format(time.RFC3339)
-	case []interface{}:
-		result := tab + "[\n"
-		for _, item := range value {
-			result += toTomlValue(item, indent+2) + ",\n"
-		}
-		return result + tab + "]"
-	default:
-		panic(fmt.Sprintf("unsupported value type: %v", value))
-	}
-}
-
-// Recursive support function for ToString()
-// Outputs a tree, using the provided keyspace to prefix group names
-func (t *TomlTree) toToml(indent, keyspace string) string {
-	result := ""
-	for k, v := range t.values {
-		// figure out the keyspace
-		combinedKey := k
-		if keyspace != "" {
-			combinedKey = keyspace + "." + combinedKey
-		}
-		// output based on type
-		switch node := v.(type) {
-		case []*TomlTree:
-			for _, item := range node {
-				if len(item.Keys()) > 0 {
-					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
-				}
-				result += item.toToml(indent+"  ", combinedKey)
-			}
-		case *TomlTree:
-			if len(node.Keys()) > 0 {
-				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			result += node.toToml(indent+"  ", combinedKey)
-		case map[string]interface{}:
-			sub := TreeFromMap(node)
-
-			if len(sub.Keys()) > 0 {
-				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			result += sub.toToml(indent+"  ", combinedKey)
-		case *tomlValue:
-			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
-		default:
-			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
-		}
-	}
-	return result
-}
-
+// Query compiles and executes a query on a tree and returns the query result.
 func (t *TomlTree) Query(query string) (*QueryResult, error) {
-	if q, err := CompileQuery(query); err != nil {
+	q, err := CompileQuery(query)
+	if err != nil {
 		return nil, err
-	} else {
-		return q.Execute(t), nil
 	}
+	return q.Execute(t), nil
 }
 
-// ToString generates a human-readable representation of the current tree.
-// Output spans multiple lines, and is suitable for ingest by a TOML parser
-func (t *TomlTree) ToString() string {
-	return t.toToml("", "")
-}
-
-// Load creates a TomlTree from a string.
-func Load(content string) (tree *TomlTree, err error) {
+// LoadReader creates a TomlTree from any io.Reader.
+func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -370,18 +265,21 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	tree = parseToml(lexToml(content))
+	tree = parseToml(lexToml(reader))
 	return
 }
 
+// Load creates a TomlTree from a string.
+func Load(content string) (tree *TomlTree, err error) {
+	return LoadReader(strings.NewReader(content))
+}
+
 // LoadFile creates a TomlTree from a file.
 func LoadFile(path string) (tree *TomlTree, err error) {
-	buff, ferr := ioutil.ReadFile(path)
-	if ferr != nil {
-		err = ferr
-	} else {
-		s := string(buff)
-		tree, err = Load(s)
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
 	}
-	return
+	defer file.Close()
+	return LoadReader(file)
 }

toml_test.go

@@ -15,6 +15,47 @@ func TestTomlHas(t *testing.T) {
 	if !tree.Has("test.key") {
 		t.Errorf("Has - expected test.key to exists")
 	}
+
+	if tree.Has("") {
+		t.Errorf("Should return false if the key is not provided")
+	}
+}
+
+func TestTomlGet(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.Get("") != tree {
+		t.Errorf("Get should return the tree itself when given an empty path")
+	}
+
+	if tree.Get("test.key") != "value" {
+		t.Errorf("Get should return the value")
+	}
+	if tree.Get(`\`) != nil {
+		t.Errorf("should return nil when the key is malformed")
+	}
+}
+
+func TestTomlGetDefault(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.GetDefault("", "hello") != tree {
+		t.Error("GetDefault should return the tree itself when given an empty path")
+	}
+
+	if tree.GetDefault("test.key", "hello") != "value" {
+		t.Error("Get should return the value")
+	}
+
+	if tree.GetDefault("whatever", "hello") != "hello" {
+		t.Error("GetDefault should return the default value if the key does not exist")
+	}
 }
 
 func TestTomlHasPath(t *testing.T) {
@@ -46,6 +87,11 @@ func TestTomlGetPath(t *testing.T) {
 			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
 		}
 	}
+
+	tree, _ := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if tree.GetPath([]string{"whatever"}) != nil {
+		t.Error("GetPath should return nil when the key does not exist")
+	}
 }
 
 func TestTomlQuery(t *testing.T) {
@@ -65,10 +111,18 @@ func TestTomlQuery(t *testing.T) {
 	}
 
 	if tt, ok := values[0].(*TomlTree); !ok {
-		t.Errorf("Expected type of TomlTree: %T Tv", values[0], values[0])
+		t.Errorf("Expected type of TomlTree: %T", values[0])
 	} else if tt.Get("a") != int64(1) {
 		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
 	} else if tt.Get("b") != int64(2) {
 		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
 	}
 }
+
+func TestTomlFromMap(t *testing.T) {
+	simpleMap := map[string]interface{}{"hello": 42}
+	tree := TreeFromMap(simpleMap)
+	if tree.Get("hello") != 42 {
+		t.Fatal("hello should be 42, not", tree.Get("hello"))
+	}
+}

tomltree_conversions.go

@@ -0,0 +1,139 @@
+// Tools to convert a TomlTree to different representations
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	result := ""
+	for _, rr := range value {
+		intRr := uint16(rr)
+		switch rr {
+		case '\b':
+			result += "\\b"
+		case '\t':
+			result += "\\t"
+		case '\n':
+			result += "\\n"
+		case '\f':
+			result += "\\f"
+		case '\r':
+			result += "\\r"
+		case '"':
+			result += "\\\""
+		case '\\':
+			result += "\\\\"
+		default:
+			if intRr < 0x001F {
+				result += fmt.Sprintf("\\u%0.4X", intRr)
+			} else {
+				result += string(rr)
+			}
+		}
+	}
+	return result
+}
+
+// Value print support function for ToString()
+// Outputs the TOML compliant string representation of a value
+func toTomlValue(item interface{}, indent int) string {
+	tab := strings.Repeat(" ", indent)
+	switch value := item.(type) {
+	case int64:
+		return tab + strconv.FormatInt(value, 10)
+	case float64:
+		return tab + strconv.FormatFloat(value, 'f', -1, 64)
+	case string:
+		return tab + "\"" + encodeTomlString(value) + "\""
+	case bool:
+		if value {
+			return "true"
+		}
+		return "false"
+	case time.Time:
+		return tab + value.Format(time.RFC3339)
+	case []interface{}:
+		result := tab + "[\n"
+		for _, item := range value {
+			result += toTomlValue(item, indent+2) + ",\n"
+		}
+		return result + tab + "]"
+	default:
+		panic(fmt.Sprintf("unsupported value type: %v", value))
+	}
+}
+
+// Recursive support function for ToString()
+// Outputs a tree, using the provided keyspace to prefix group names
+func (t *TomlTree) toToml(indent, keyspace string) string {
+	result := ""
+	for k, v := range t.values {
+		// figure out the keyspace
+		combinedKey := k
+		if keyspace != "" {
+			combinedKey = keyspace + "." + combinedKey
+		}
+		// output based on type
+		switch node := v.(type) {
+		case []*TomlTree:
+			for _, item := range node {
+				if len(item.Keys()) > 0 {
+					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
+				}
+				result += item.toToml(indent+"  ", combinedKey)
+			}
+		case *TomlTree:
+			if len(node.Keys()) > 0 {
+				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			result += node.toToml(indent+"  ", combinedKey)
+		case map[string]interface{}:
+			sub := TreeFromMap(node)
+
+			if len(sub.Keys()) > 0 {
+				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			result += sub.toToml(indent+"  ", combinedKey)
+		case *tomlValue:
+			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
+		default:
+			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
+		}
+	}
+	return result
+}
+
+// ToString generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser
+func (t *TomlTree) ToString() string {
+	return t.toToml("", "")
+}
+
+// ToMap recursively generates a representation of the current tree using map[string]interface{}.
+func (t *TomlTree) ToMap() map[string]interface{} {
+	result := map[string]interface{}{}
+
+	for k, v := range t.values {
+		switch node := v.(type) {
+		case []*TomlTree:
+			result[k] = make([]interface{}, 0)
+			for _, item := range node {
+				result[k] = item.ToMap()
+			}
+		case *TomlTree:
+			result[k] = node.ToMap()
+		case map[string]interface{}:
+			sub := TreeFromMap(node)
+			result[k] = sub.ToMap()
+		case *tomlValue:
+			result[k] = node.value
+		}
+	}
+
+	return result
+}

tomltree_conversions_test.go

@@ -0,0 +1,82 @@
+package toml
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestTomlTreeConversionToString(t *testing.T) {
+	toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+points = { x = 1, y = 2 }`)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	reparsedTree, err := Load(toml.ToString())
+
+	assertTree(t, reparsedTree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"points": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func testMaps(t *testing.T, actual, expected map[string]interface{}) {
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatal("trees aren't equal.\n", "Expected:\n", expected, "\nActual:\n", actual)
+	}
+}
+
+func TestTomlTreeConversionToMapSimple(t *testing.T) {
+	tree, _ := Load("a = 42\nb = 17")
+
+	expected := map[string]interface{}{
+		"a": int64(42),
+		"b": int64(17),
+	}
+
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTomlTreeConversionToMapExampleFile(t *testing.T) {
+	tree, _ := LoadFile("example.toml")
+	expected := map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+			"connection_max": int64(5000),
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]interface{}{"gamma", "delta"},
+				[]interface{}{int64(1), int64(2)},
+			},
+		},
+	}
+	testMaps(t, tree.ToMap(), expected)
+}