Bump test go patchlevels (#113)

* 1.6.4
* 1.7.4

.travis.yml

@@ -1,6 +1,18 @@
 language: go
-script: "./test.sh"
 go:
-    - 1.1
-    - 1.2
-    - tip
+  - 1.5.4
+  - 1.6.4
+  - 1.7.4
+  - tip
+matrix:
+  allow_failures:
+    - go: tip
+  fast_finish: true
+script:
+  - ./test.sh
+before_install:
+  - go get github.com/axw/gocov/gocov
+  - go get github.com/mattn/goveralls
+  - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+after_success:
+  - $HOME/gopath/bin/goveralls -service=travis-ci

LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 - 2016 Thomas Pelletier, Eric Anderton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -3,9 +3,30 @@
 Go library for the [TOML](https://github.com/mojombo/toml) format.
 
 This library supports TOML version
-[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md)
+[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
 
-[![Build Status](https://travis-ci.org/pelletier/go-toml.png?branch=master)](https://travis-ci.org/pelletier/go-toml)
+[![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/goadesign/goa/blob/master/LICENSE)
+[![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
+[![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pelletier/go-toml)](https://goreportcard.com/report/github.com/pelletier/go-toml)
+
+## Features
+
+Go-toml provides the following features for using data parsed from TOML documents:
+
+* Load TOML documents from files and string data
+* Easily navigate TOML structure using TomlTree
+* Line & column position data for all parsed elements
+* Query support similar to JSON-Path
+* Syntax errors contain line and column numbers
+
+Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
+
+* Semantic evaluation of parsed TOML
+* Informing a user of mistakes in the source document, after it has been parsed
+* Programatic handling of default values on a case-by-case basis
+* Using a TOML document as a flexible data-store
 
 ## Import
 
@@ -13,6 +34,8 @@ This library supports TOML version
 
 ## Usage
 
+### Example
+
 Say you have a TOML file that looks like this:
 
 ```toml
@@ -42,14 +65,41 @@ if err != nil {
     user = configTree.Get("user").(string)
     password = configTree.Get("password").(string)
     fmt.Println("User is ", user, ". Password is ", password)
+
+    // show where elements are in the file
+    fmt.Println("User position: %v", configTree.GetPosition("user"))
+    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+    // use a query to gather elements without walking the tree
+    results, _ := config.Query("$..[user,password]")
+    for ii, item := range results.Values() {
+      fmt.Println("Query result %d: %v", ii, item)
+    }
 }
 ```
 
 ## Documentation
 
-The documentation is available at
+The documentation and additional examples are available at
 [godoc.org](http://godoc.org/github.com/pelletier/go-toml).
 
+## Tools
+
+Go-toml provides two handy command line tools:
+
+* `tomll`: Reads TOML files and lint them.
+
+    ```
+    go install github.com/pelletier/go-toml/cmd/tomll
+    tomll --help
+    ```
+* `tomljson`: Reads a TOML file and outputs its JSON representation.
+
+    ```
+    go install github.com/pelletier/go-toml/cmd/tomjson
+    tomljson --help
+    ```
+
 ## Contribute
 
 Feel free to report bugs and patches using GitHub's pull requests system on
@@ -60,29 +110,11 @@ much appreciated!
 
 You have to make sure two kind of tests run:
 
-1. The Go unit tests: `go test`
-2. The TOML examples base: `./test_program/go-test.sh`
+1. The Go unit tests
+2. The TOML examples base
 
 You can run both of them using `./test.sh`.
 
 ## License
 
-Copyright (c) 2013, 2014 Thomas Pelletier, Eric Anderton
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+The MIT License (MIT). Read [LICENSE](LICENSE).

cmd/test_program.go

@@ -3,34 +3,37 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/pelletier/go-toml"
 	"io/ioutil"
 	"log"
 	"os"
 	"time"
+
+	"github.com/pelletier/go-toml"
 )
 
 func main() {
 	bytes, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
+		log.Fatalf("Error during TOML read: %s", err)
 		os.Exit(2)
 	}
 	tree, err := toml.Load(string(bytes))
 	if err != nil {
+		log.Fatalf("Error during TOML load: %s", err)
 		os.Exit(1)
 	}
 
-	typedTree := translate((map[string]interface{})(*tree))
+	typedTree := translate(*tree)
 
 	if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
 		log.Fatalf("Error encoding JSON: %s", err)
+		os.Exit(3)
 	}
 
 	os.Exit(0)
 }
 
 func translate(tomlData interface{}) interface{} {
-
 	switch orig := tomlData.(type) {
 	case map[string]interface{}:
 		typed := make(map[string]interface{}, len(orig))
@@ -39,7 +42,14 @@ func translate(tomlData interface{}) interface{} {
 		}
 		return typed
 	case *toml.TomlTree:
-		return translate((map[string]interface{})(*orig))
+		return translate(*orig)
+	case toml.TomlTree:
+		keys := orig.Keys()
+		typed := make(map[string]interface{}, len(keys))
+		for _, k := range keys {
+			typed[k] = translate(orig.GetPath([]string{k}))
+		}
+		return typed
 	case []*toml.TomlTree:
 		typed := make([]map[string]interface{}, len(orig))
 		for i, v := range orig {

cmd/tomljson/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, `tomljson can be used in two ways:
+Writing to STDIN and reading from STDOUT:
+  cat file.toml | tomljson > file.json
+
+Reading from a file name:
+  tomljson file.toml
+`)
+	}
+	flag.Parse()
+	os.Exit(processMain(flag.Args(), os.Stdin, os.Stdout, os.Stderr))
+}
+
+func processMain(files []string, defaultInput io.Reader, output io.Writer, errorOutput io.Writer) int {
+	// read from stdin and print to stdout
+	inputReader := defaultInput
+
+	if len(files) > 0 {
+		var err error
+		inputReader, err = os.Open(files[0])
+		if err != nil {
+			printError(err, errorOutput)
+			return -1
+		}
+	}
+	s, err := reader(inputReader)
+	if err != nil {
+		printError(err, errorOutput)
+		return -1
+	}
+	io.WriteString(output, s+"\n")
+	return 0
+}
+
+func printError(err error, output io.Writer) {
+	io.WriteString(output, err.Error()+"\n")
+}
+
+func reader(r io.Reader) (string, error) {
+	tree, err := toml.LoadReader(r)
+	if err != nil {
+		return "", err
+	}
+	return mapToJSON(tree)
+}
+
+func mapToJSON(tree *toml.TomlTree) (string, error) {
+	treeMap := tree.ToMap()
+	bytes, err := json.MarshalIndent(treeMap, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(bytes[:]), nil
+}

cmd/tomljson/main_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+func expectBufferEquality(t *testing.T, name string, buffer *bytes.Buffer, expected string) {
+	output := buffer.String()
+	if output != expected {
+		t.Errorf("incorrect %s:\n%s\n\nexpected %s:\n%s", name, output, name, expected)
+		t.Log([]rune(output))
+		t.Log([]rune(expected))
+	}
+}
+
+func expectProcessMainResults(t *testing.T, input string, args []string, exitCode int, expectedOutput string, expectedError string) {
+	inputReader := strings.NewReader(input)
+	outputBuffer := new(bytes.Buffer)
+	errorBuffer := new(bytes.Buffer)
+
+	returnCode := processMain(args, inputReader, outputBuffer, errorBuffer)
+
+	expectBufferEquality(t, "output", outputBuffer, expectedOutput)
+	expectBufferEquality(t, "error", errorBuffer, expectedError)
+
+	if returnCode != exitCode {
+		t.Error("incorrect return code:", returnCode, "expected", exitCode)
+	}
+}
+
+func TestProcessMainReadFromStdin(t *testing.T) {
+	input := `
+		[mytoml]
+		a = 42`
+	expectedOutput := `{
+  "mytoml": {
+    "a": 42
+  }
+}
+`
+	expectedError := ``
+	expectedExitCode := 0
+
+	expectProcessMainResults(t, input, []string{}, expectedExitCode, expectedOutput, expectedError)
+}
+
+func TestProcessMainReadFromFile(t *testing.T) {
+	input := `
+		[mytoml]
+		a = 42`
+
+	tmpfile, err := ioutil.TempFile("", "example.toml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tmpfile.Write([]byte(input)); err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.Remove(tmpfile.Name())
+
+	expectedOutput := `{
+  "mytoml": {
+    "a": 42
+  }
+}
+`
+	expectedError := ``
+	expectedExitCode := 0
+
+	expectProcessMainResults(t, ``, []string{tmpfile.Name()}, expectedExitCode, expectedOutput, expectedError)
+}
+
+func TestProcessMainReadFromMissingFile(t *testing.T) {
+	expectedError := `open /this/file/does/not/exist: no such file or directory
+`
+	expectProcessMainResults(t, ``, []string{"/this/file/does/not/exist"}, -1, ``, expectedError)
+}

cmd/tomll/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, `tomll can be used in two ways:
+Writing to STDIN and reading from STDOUT:
+  cat file.toml | tomll > file.toml
+
+Reading and updating a list of files:
+  tomll a.toml b.toml c.toml
+
+When given a list of files, tomll will modify all files in place without asking.
+`)
+	}
+	flag.Parse()
+	// read from stdin and print to stdout
+	if flag.NArg() == 0 {
+		s, err := lintReader(os.Stdin)
+		if err != nil {
+			io.WriteString(os.Stderr, err.Error())
+			os.Exit(-1)
+		}
+		io.WriteString(os.Stdout, s)
+	} else {
+		// otherwise modify a list of files
+		for _, filename := range flag.Args() {
+			s, err := lintFile(filename)
+			if err != nil {
+				io.WriteString(os.Stderr, err.Error())
+				os.Exit(-1)
+			}
+			ioutil.WriteFile(filename, []byte(s), 0644)
+		}
+	}
+}
+
+func lintFile(filename string) (string, error) {
+	tree, err := toml.LoadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	return tree.String(), nil
+}
+
+func lintReader(r io.Reader) (string, error) {
+	tree, err := toml.LoadReader(r)
+	if err != nil {
+		return "", err
+	}
+	return tree.String(), nil
+}

doc.go

@@ -0,0 +1,250 @@
+// Package toml is a TOML markup language parser.
+//
+// This version supports the specification as described in
+// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md
+//
+// TOML Parsing
+//
+// TOML data may be parsed in two ways: by file, or by string.
+//
+//   // load TOML data by filename
+//   tree, err := toml.LoadFile("filename.toml")
+//
+//   // load TOML data stored in a string
+//   tree, err := toml.Load(stringContainingTomlData)
+//
+// Either way, the result is a TomlTree object that can be used to navigate the
+// structure and data within the original document.
+//
+//
+// Getting data from the TomlTree
+//
+// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
+// methods on the returned TomlTree, to find your way through the document data.
+//
+//   if tree.Has('foo') {
+//     fmt.Prinln("foo is: %v", tree.Get('foo'))
+//   }
+//
+// Working with Paths
+//
+// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
+// and GetDefault() methods.  These are the same kind of key paths used within the
+// TOML specification for struct tames.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.Has("foo.bar.baz")
+//
+//   // returns the key at this path, if it is there
+//   tree.Get("foo.bar.baz")
+//
+// TOML allows keys to contain '.', which can cause this syntax to be problematic
+// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
+// methods to explicitly define the path.  This form is also faster, since
+// it avoids having to parse the passed key for '.' delimiters.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.HasPath(string{}{"foo","bar","baz"})
+//
+//   // returns the key at this path, if it is there
+//   tree.GetPath(string{}{"foo","bar","baz"})
+//
+// Note that this is distinct from the heavyweight query syntax supported by
+// TomlTree.Query() and the Query() struct (see below).
+//
+// Position Support
+//
+// Each element within the TomlTree is stored with position metadata, which is
+// invaluable for providing semantic feedback to a user.  This helps in
+// situations where the TOML file parses correctly, but contains data that is
+// not correct for the application.  In such cases, an error message can be
+// generated that indicates the problem line and column number in the source
+// TOML document.
+//
+//   // load TOML data
+//   tree, _ := toml.Load("filename.toml")
+//
+//   // get an entry and report an error if it's the wrong type
+//   element := tree.Get("foo")
+//   if value, ok := element.(int64); !ok {
+//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
+//   }
+//
+//   // report an error if an expected element is missing
+//   if !tree.Has("bar") {
+//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
+//   }
+//
+// Query Support
+//
+// The TOML query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result, err := tree.Query("$.foo.bar.baz")
+//
+// This is roughly equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// err is nil if any parsing exception occurs.
+//
+// If no node in the tree matches the query, result will simply contain an empty list of
+// items.
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   tree.Query("$.foo[-1]")
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   tree.Query("$.foo[0:5]")
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   tree.Query("$.foo[0:-1]")
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   tree.Query("$.foo[0:-1:2]")
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   tree.Query("$.foo[:]")
+//   tree.Query("$.foo[0:]")
+//   tree.Query("$.foo[:-1]")
+//   tree.Query("$.foo[0:-1:]")
+//   tree.Query("$.foo[::1]")
+//   tree.Query("$.foo[0::1]")
+//   tree.Query("$.foo[:-1:1]")
+//   tree.Query("$.foo[0:-1:1]")
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   tree.Query("$.foo[?(bar)]")
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type TomlTree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a QueryResult object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := tree.Query("$.foo.bar.baz")
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a TomlTree object, or compiled ahead
+// of time and executed discretely.  The former is more convienent, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := tree.Query("$.foo.bar.baz")
+//
+//   // compiled query
+//   query := toml.CompileQuery("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := CompileQuery("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*TomlTree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package toml

doc_test.go

@@ -0,0 +1,81 @@
+// code examples for godoc
+
+package toml
+
+import (
+	"fmt"
+)
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := CompileQuery("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*TomlTree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' TomlTree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	authors, _ := config.Query("$.book.author")
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func Example_comprehensiveExample() {
+	config, err := LoadFile("config.toml")
+
+	if err != nil {
+		fmt.Println("Error ", err.Error())
+	} else {
+		// retrieve data directly
+		user := config.Get("postgres.user").(string)
+		password := config.Get("postgres.password").(string)
+
+		// or using an intermediate object
+		configTree := config.Get("postgres").(*TomlTree)
+		user = configTree.Get("user").(string)
+		password = configTree.Get("password").(string)
+		fmt.Println("User is ", user, ". Password is ", password)
+
+		// show where elements are in the file
+		fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
+		fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
+
+		// use a query to gather elements without walking the tree
+		results, _ := config.Query("$..[user,password]")
+		for ii, item := range results.Values() {
+			fmt.Printf("Query result %d: %v\n", ii, item)
+		}
+	}
+}

example-crlf.toml

@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it

keysparsing.go

@@ -0,0 +1,93 @@
+// Parsing keys handling both bare and quoted keys.
+
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"unicode"
+)
+
+func parseKey(key string) ([]string, error) {
+	groups := []string{}
+	var buffer bytes.Buffer
+	inQuotes := false
+	wasInQuotes := false
+	escapeNext := false
+	ignoreSpace := true
+	expectDot := false
+
+	for _, char := range key {
+		if ignoreSpace {
+			if char == ' ' {
+				continue
+			}
+			ignoreSpace = false
+		}
+		if escapeNext {
+			buffer.WriteRune(char)
+			escapeNext = false
+			continue
+		}
+		switch char {
+		case '\\':
+			escapeNext = true
+			continue
+		case '"':
+			if inQuotes {
+				groups = append(groups, buffer.String())
+				buffer.Reset()
+				wasInQuotes = true
+			}
+			inQuotes = !inQuotes
+			expectDot = false
+		case '.':
+			if inQuotes {
+				buffer.WriteRune(char)
+			} else {
+				if !wasInQuotes {
+					if buffer.Len() == 0 {
+						return nil, fmt.Errorf("empty key group")
+					}
+					groups = append(groups, buffer.String())
+					buffer.Reset()
+				}
+				ignoreSpace = true
+				expectDot = false
+				wasInQuotes = false
+			}
+		case ' ':
+			if inQuotes {
+				buffer.WriteRune(char)
+			} else {
+				expectDot = true
+			}
+		default:
+			if !inQuotes && !isValidBareChar(char) {
+				return nil, fmt.Errorf("invalid bare character: %c", char)
+			}
+			if !inQuotes && expectDot {
+				return nil, fmt.Errorf("what?")
+			}
+			buffer.WriteRune(char)
+			expectDot = false
+		}
+	}
+	if inQuotes {
+		return nil, fmt.Errorf("mismatched quotes")
+	}
+	if escapeNext {
+		return nil, fmt.Errorf("unfinished escape sequence")
+	}
+	if buffer.Len() > 0 {
+		groups = append(groups, buffer.String())
+	}
+	if len(groups) == 0 {
+		return nil, fmt.Errorf("empty key")
+	}
+	return groups, nil
+}
+
+func isValidBareChar(r rune) bool {
+	return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r)
+}

keysparsing_test.go

@@ -0,0 +1,56 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+func testResult(t *testing.T, key string, expected []string) {
+	parsed, err := parseKey(key)
+	t.Logf("key=%s expected=%s parsed=%s", key, expected, parsed)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if len(expected) != len(parsed) {
+		t.Fatal("Expected length", len(expected), "but", len(parsed), "parsed")
+	}
+	for index, expectedKey := range expected {
+		if expectedKey != parsed[index] {
+			t.Fatal("Expected", expectedKey, "at index", index, "but found", parsed[index])
+		}
+	}
+}
+
+func testError(t *testing.T, key string, expectedError string) {
+	_, err := parseKey(key)
+	if fmt.Sprintf("%s", err) != expectedError {
+		t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
+	}
+}
+
+func TestBareKeyBasic(t *testing.T) {
+	testResult(t, "test", []string{"test"})
+}
+
+func TestBareKeyDotted(t *testing.T) {
+	testResult(t, "this.is.a.key", []string{"this", "is", "a", "key"})
+}
+
+func TestDottedKeyBasic(t *testing.T) {
+	testResult(t, "\"a.dotted.key\"", []string{"a.dotted.key"})
+}
+
+func TestBaseKeyPound(t *testing.T) {
+	testError(t, "hello#world", "invalid bare character: #")
+}
+
+func TestQuotedKeys(t *testing.T) {
+	testResult(t, `hello."foo".bar`, []string{"hello", "foo", "bar"})
+	testResult(t, `"hello!"`, []string{"hello!"})
+}
+
+func TestEmptyKey(t *testing.T) {
+	testError(t, "", "empty key")
+	testError(t, " ", "empty key")
+	testResult(t, `""`, []string{""})
+}

lexer.go

@@ -1,190 +1,160 @@
-// TOML lexer.// Written using the principles developped by Rob Pike in
+// TOML lexer.
+//
+// Written using the principles developed by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
 package toml
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
-	"unicode/utf8"
+
+	"github.com/pelletier/go-buffruneio"
 )
 
 var dateRegexp *regexp.Regexp
 
-// Define tokens
-type tokenType int
-
-const (
-	eof = -(iota + 1)
-)
-
-const (
-	tokenError tokenType = iota
-	tokenEOF
-	tokenComment
-	tokenKey
-	tokenEqual
-	tokenString
-	tokenInteger
-	tokenTrue
-	tokenFalse
-	tokenFloat
-	tokenLeftBracket
-	tokenRightBracket
-	tokenDoubleLeftBracket
-	tokenDoubleRightBracket
-	tokenDate
-	tokenKeyGroup
-	tokenKeyGroupArray
-	tokenComma
-	tokenEOL
-)
-
-type token struct {
-	typ tokenType
-	val string
-}
-
-func (i token) String() string {
-	switch i.typ {
-	case tokenEOF:
-		return "EOF"
-	case tokenError:
-		return i.val
-	}
-
-	if len(i.val) > 10 {
-		return fmt.Sprintf("%.10q...", i.val)
-	}
-	return fmt.Sprintf("%q", i.val)
-}
-
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
-}
-
-func isKeyChar(r rune) bool {
-	// "Keys start with the first non-whitespace character and end with the last
-	// non-whitespace character before the equals sign."
-	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
-}
-
-func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
-}
-
-func isHexDigit(r rune) bool {
-	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
-}
+// Define state functions
+type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
-type lexer struct {
-	input  string
-	start  int
-	pos    int
-	width  int
-	tokens chan token
-	depth  int
+type tomlLexer struct {
+	input         *buffruneio.Reader // Textual source
+	buffer        []rune             // Runes composing the current token
+	tokens        chan token
+	depth         int
+	line          int
+	col           int
+	endbufferLine int
+	endbufferCol  int
 }
 
-func (l *lexer) run() {
-	for state := lexVoid; state != nil; {
-		state = state(l)
+// Basic read operations on input
+
+func (l *tomlLexer) read() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
 	}
-	close(l.tokens)
-}
-
-func (l *lexer) emit(t tokenType) {
-	l.tokens <- token{t, l.input[l.start:l.pos]}
-	l.start = l.pos
-}
-
-func (l *lexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{t, value}
-	l.start = l.pos
-}
-
-func (l *lexer) next() rune {
-	if l.pos >= len(l.input) {
-		l.width = 0
-		return eof
+	if r == '\n' {
+		l.endbufferLine++
+		l.endbufferCol = 1
+	} else {
+		l.endbufferCol++
 	}
-	var r rune
-	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
-	l.pos += l.width
 	return r
 }
 
-func (l *lexer) ignore() {
-	l.start = l.pos
+func (l *tomlLexer) next() rune {
+	r := l.read()
+
+	if r != eof {
+		l.buffer = append(l.buffer, r)
+	}
+	return r
 }
 
-func (l *lexer) backup() {
-	l.pos -= l.width
+func (l *tomlLexer) ignore() {
+	l.buffer = make([]rune, 0)
+	l.line = l.endbufferLine
+	l.col = l.endbufferCol
 }
 
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+func (l *tomlLexer) skip() {
+	l.next()
+	l.ignore()
+}
+
+func (l *tomlLexer) fastForward(n int) {
+	for i := 0; i < n; i++ {
+		l.next()
+	}
+}
+
+func (l *tomlLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
-		tokenError,
-		fmt.Sprintf(format, args...),
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.ignore()
+}
+
+func (l *tomlLexer) emit(t tokenType) {
+	l.emitWithValue(t, string(l.buffer))
+}
+
+func (l *tomlLexer) peek() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
+	}
+	l.input.UnreadRune()
+	return r
+}
+
+func (l *tomlLexer) follow(next string) bool {
+	for _, expectedRune := range next {
+		r, err := l.input.ReadRune()
+		defer l.input.UnreadRune()
+		if err != nil {
+			panic(err)
+		}
+		if expectedRune != r {
+			return false
+		}
+	}
+	return true
+}
+
+// Error management
+
+func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
 	}
 	return nil
 }
 
-func (l *lexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
-}
+// State functions
 
-func (l *lexer) accept(valid string) bool {
-	if strings.IndexRune(valid, l.next()) >= 0 {
-		return true
-	}
-	l.backup()
-	return false
-}
-
-func (l *lexer) follow(next string) bool {
-	return strings.HasPrefix(l.input[l.pos:], next)
-}
-
-// Define state functions
-type stateFn func(*lexer) stateFn
-
-func lexVoid(l *lexer) stateFn {
+func (l *tomlLexer) lexVoid() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '[':
-			return lexKeyGroup
+			return l.lexKeyGroup
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '=':
-			return lexEqual
+			return l.lexEqual
+		case '\r':
+			fallthrough
+		case '\n':
+			l.skip()
+			continue
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
 		}
 
 		if l.depth > 0 {
-			return lexRvalue
+			return l.lexRvalue
 		}
 
-		if isKeyChar(next) {
-			return lexKey
+		if isKeyStartChar(next) {
+			return l.lexKey
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
 	}
@@ -193,318 +163,493 @@ func lexVoid(l *lexer) stateFn {
 	return nil
 }
 
-func lexRvalue(l *lexer) stateFn {
+func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '.':
 			return l.errorf("cannot start float with a dot")
 		case '=':
-			return l.errorf("cannot have multiple equals for the same key")
+			return l.lexEqual
 		case '[':
-			l.depth += 1
-			return lexLeftBracket
+			l.depth++
+			return l.lexLeftBracket
 		case ']':
-			l.depth -= 1
-			return lexRightBracket
+			l.depth--
+			return l.lexRightBracket
+		case '{':
+			return l.lexLeftCurlyBrace
+		case '}':
+			return l.lexRightCurlyBrace
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '"':
-			return lexString
+			return l.lexString
+		case '\'':
+			return l.lexLiteralString
 		case ',':
-			return lexComma
+			return l.lexComma
+		case '\r':
+			fallthrough
 		case '\n':
-			l.ignore()
-			l.pos += 1
+			l.skip()
 			if l.depth == 0 {
-				return lexVoid
-			} else {
-				return lexRvalue
+				return l.lexVoid
 			}
+			return l.lexRvalue
+		case '_':
+			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
-			return lexTrue
+			return l.lexTrue
 		}
 
 		if l.follow("false") {
-			return lexFalse
-		}
-
-		if isAlphanumeric(next) {
-			return lexKey
-		}
-
-		if dateRegexp.FindString(l.input[l.pos:]) != "" {
-			return lexDate
-		}
-
-		if next == '+' || next == '-' || isDigit(next) {
-			return lexNumber
+			return l.lexFalse
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
+			continue
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
+
+		possibleDate := string(l.input.Peek(35))
+		dateMatch := dateRegexp.FindString(possibleDate)
+		if dateMatch != "" {
+			l.fastForward(len(dateMatch))
+			return l.lexDate
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
+		return l.errorf("no value can start with %c", next)
 	}
 
 	l.emit(tokenEOF)
 	return nil
 }
 
-func lexDate(l *lexer) stateFn {
-	l.ignore()
-	l.pos += 20 // Fixed size of a date in TOML
+func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
+	l.next()
+	l.emit(tokenLeftCurlyBrace)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
+	l.next()
+	l.emit(tokenRightCurlyBrace)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexDate() tomlLexStateFn {
 	l.emit(tokenDate)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexTrue(l *lexer) stateFn {
-	l.ignore()
-	l.pos += 4
+func (l *tomlLexer) lexTrue() tomlLexStateFn {
+	l.fastForward(4)
 	l.emit(tokenTrue)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexFalse(l *lexer) stateFn {
-	l.ignore()
-	l.pos += 5
+func (l *tomlLexer) lexFalse() tomlLexStateFn {
+	l.fastForward(5)
 	l.emit(tokenFalse)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexEqual(l *lexer) stateFn {
-	l.ignore()
-	l.accept("=")
+func (l *tomlLexer) lexEqual() tomlLexStateFn {
+	l.next()
 	l.emit(tokenEqual)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexComma(l *lexer) stateFn {
-	l.ignore()
-	l.accept(",")
+func (l *tomlLexer) lexComma() tomlLexStateFn {
+	l.next()
 	l.emit(tokenComma)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexKey(l *lexer) stateFn {
-	l.ignore()
-	for isKeyChar(l.next()) {
-	}
-	l.backup()
-	l.emit(tokenKey)
-	return lexVoid
-}
+func (l *tomlLexer) lexKey() tomlLexStateFn {
+	growingString := ""
 
-func lexComment(l *lexer) stateFn {
-	for {
-		next := l.next()
-		if next == '\n' || next == eof {
-			break
-		}
-	}
-	l.ignore()
-	return lexVoid
-}
-
-func lexLeftBracket(l *lexer) stateFn {
-	l.ignore()
-	l.pos += 1
-	l.emit(tokenLeftBracket)
-	return lexRvalue
-}
-
-func lexString(l *lexer) stateFn {
-	l.pos += 1
-	l.ignore()
-	growing_string := ""
-
-	for {
-		if l.peek() == '"' {
-			l.emitWithValue(tokenString, growing_string)
-			l.pos += 1
-			l.ignore()
-			return lexRvalue
-		}
-
-		if l.follow("\\\"") {
-			l.pos += 1
-			growing_string += "\""
-		} else if l.follow("\\n") {
-			l.pos += 1
-			growing_string += "\n"
-		} else if l.follow("\\b") {
-			l.pos += 1
-			growing_string += "\b"
-		} else if l.follow("\\f") {
-			l.pos += 1
-			growing_string += "\f"
-		} else if l.follow("\\/") {
-			l.pos += 1
-			growing_string += "/"
-		} else if l.follow("\\t") {
-			l.pos += 1
-			growing_string += "\t"
-		} else if l.follow("\\r") {
-			l.pos += 1
-			growing_string += "\r"
-		} else if l.follow("\\\\") {
-			l.pos += 1
-			growing_string += "\\"
-		} else if l.follow("\\u") {
-			l.pos += 2
-			code := ""
-			for i := 0; i < 4; i++ {
-				c := l.peek()
-				l.pos += 1
-				if !isHexDigit(c) {
-					return l.errorf("unfinished unicode escape")
-				}
-				code = code + string(c)
-			}
-			l.pos -= 1
-			intcode, err := strconv.ParseInt(code, 16, 32)
+	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
+		if r == '"' {
+			l.next()
+			str, err := l.lexStringAsString(`"`, false, true)
 			if err != nil {
-				return l.errorf("invalid unicode escape: \\u" + code)
+				return l.errorf(err.Error())
 			}
-			growing_string += string(rune(intcode))
-		} else if l.follow("\\") {
-			l.pos += 1
-			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
-		} else {
-			growing_string += string(l.peek())
+			growingString += `"` + str + `"`
+			l.next()
+			continue
+		} else if r == '\n' {
+			return l.errorf("keys cannot contain new lines")
+		} else if isSpace(r) {
+			break
+		} else if !isValidBareChar(r) {
+			return l.errorf("keys cannot contain %c character", r)
+		}
+		growingString += string(r)
+		l.next()
+	}
+	l.emitWithValue(tokenKey, growingString)
+	return l.lexVoid
+}
+
+func (l *tomlLexer) lexComment() tomlLexStateFn {
+	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+		if next == '\r' && l.follow("\r\n") {
+			break
+		}
+		l.next()
+	}
+	l.ignore()
+	return l.lexVoid
+}
+
+func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
+	l.next()
+	l.emit(tokenLeftBracket)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexLiteralStringAsString(terminator string, discardLeadingNewLine bool) (string, error) {
+	growingString := ""
+
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
+		}
+	}
+
+	// find end of string
+	for {
+		if l.follow(terminator) {
+			return growingString, nil
 		}
 
-		if l.next() == eof {
+		next := l.peek()
+		if next == eof {
+			break
+		}
+		growingString += string(l.next())
+	}
+
+	return "", errors.New("unclosed string")
+}
+
+func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
+	l.skip()
+
+	// handle special case for triple-quote
+	terminator := "'"
+	discardLeadingNewLine := false
+	if l.follow("''") {
+		l.skip()
+		l.skip()
+		terminator = "'''"
+		discardLeadingNewLine = true
+	}
+
+	str, err := l.lexLiteralStringAsString(terminator, discardLeadingNewLine)
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
+	l.ignore()
+	return l.lexRvalue
+}
+
+// Lex a string and return the results as a string.
+// Terminator is the substring indicating the end of the token.
+// The resulting string does not include the terminator.
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
+	growingString := ""
+
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
+		}
+	}
+
+	for {
+		if l.follow(terminator) {
+			return growingString, nil
+		}
+
+		if l.follow("\\") {
+			l.next()
+			switch l.peek() {
+			case '\r':
+				fallthrough
+			case '\n':
+				fallthrough
+			case '\t':
+				fallthrough
+			case ' ':
+				// skip all whitespace chars following backslash
+				for strings.ContainsRune("\r\n\t ", l.peek()) {
+					l.next()
+				}
+			case '"':
+				growingString += "\""
+				l.next()
+			case 'n':
+				growingString += "\n"
+				l.next()
+			case 'b':
+				growingString += "\b"
+				l.next()
+			case 'f':
+				growingString += "\f"
+				l.next()
+			case '/':
+				growingString += "/"
+				l.next()
+			case 't':
+				growingString += "\t"
+				l.next()
+			case 'r':
+				growingString += "\r"
+				l.next()
+			case '\\':
+				growingString += "\\"
+				l.next()
+			case 'u':
+				l.next()
+				code := ""
+				for i := 0; i < 4; i++ {
+					c := l.peek()
+					if !isHexDigit(c) {
+						return "", errors.New("unfinished unicode escape")
+					}
+					l.next()
+					code = code + string(c)
+				}
+				intcode, err := strconv.ParseInt(code, 16, 32)
+				if err != nil {
+					return "", errors.New("invalid unicode escape: \\u" + code)
+				}
+				growingString += string(rune(intcode))
+			case 'U':
+				l.next()
+				code := ""
+				for i := 0; i < 8; i++ {
+					c := l.peek()
+					if !isHexDigit(c) {
+						return "", errors.New("unfinished unicode escape")
+					}
+					l.next()
+					code = code + string(c)
+				}
+				intcode, err := strconv.ParseInt(code, 16, 64)
+				if err != nil {
+					return "", errors.New("invalid unicode escape: \\U" + code)
+				}
+				growingString += string(rune(intcode))
+			default:
+				return "", errors.New("invalid escape sequence: \\" + string(l.peek()))
+			}
+		} else {
+			r := l.peek()
+
+			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) {
+				return "", fmt.Errorf("unescaped control character %U", r)
+			}
+			l.next()
+			growingString += string(r)
+		}
+
+		if l.peek() == eof {
 			break
 		}
 	}
 
-	return l.errorf("unclosed string")
+	return "", errors.New("unclosed string")
 }
 
-func lexKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexString() tomlLexStateFn {
+	l.skip()
+
+	// handle special case for triple-quote
+	terminator := `"`
+	discardLeadingNewLine := false
+	acceptNewLines := false
+	if l.follow(`""`) {
+		l.skip()
+		l.skip()
+		terminator = `"""`
+		discardLeadingNewLine = true
+		acceptNewLines = true
+	}
+
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
+
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
 	l.ignore()
-	l.pos += 1
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
+	l.next()
 
 	if l.peek() == '[' {
 		// token '[[' signifies an array of anonymous key groups
-		l.pos += 1
+		l.next()
 		l.emit(tokenDoubleLeftBracket)
-		return lexInsideKeyGroupArray
-	} else {
-		// vanilla key group
-		l.emit(tokenLeftBracket)
-		return lexInsideKeyGroup
+		return l.lexInsideKeyGroupArray
 	}
+	// vanilla key group
+	l.emit(tokenLeftBracket)
+	return l.lexInsideKeyGroup
 }
 
-func lexInsideKeyGroupArray(l *lexer) stateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroupArray)
 			}
-			l.ignore()
-			l.pos += 1
+			l.next()
 			if l.peek() != ']' {
-				break // error
+				break
 			}
-			l.pos += 1
+			l.next()
 			l.emit(tokenDoubleRightBracket)
-			return lexVoid
-		} else if l.peek() == '[' {
+			return l.lexVoid
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		default:
+			l.next()
 		}
 	}
 	return l.errorf("unclosed key group array")
 }
 
-func lexInsideKeyGroup(l *lexer) stateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroup)
 			}
-			l.ignore()
-			l.pos += 1
+			l.next()
 			l.emit(tokenRightBracket)
-			return lexVoid
-		} else if l.peek() == '[' {
+			return l.lexVoid
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		default:
+			l.next()
 		}
 	}
 	return l.errorf("unclosed key group")
 }
 
-func lexRightBracket(l *lexer) stateFn {
-	l.ignore()
-	l.pos += 1
+func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
+	l.next()
 	l.emit(tokenRightBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexNumber(l *lexer) stateFn {
-	l.ignore()
-	if !l.accept("+") {
-		l.accept("-")
+func (l *tomlLexer) lexNumber() tomlLexStateFn {
+	r := l.peek()
+	if r == '+' || r == '-' {
+		l.next()
 	}
-	point_seen := false
-	digit_seen := false
+	pointSeen := false
+	expSeen := false
+	digitSeen := false
 	for {
-		next := l.next()
+		next := l.peek()
 		if next == '.' {
-			if point_seen {
+			if pointSeen {
 				return l.errorf("cannot have two dots in one float")
 			}
+			l.next()
 			if !isDigit(l.peek()) {
 				return l.errorf("float cannot end with a dot")
 			}
-			point_seen = true
+			pointSeen = true
+		} else if next == 'e' || next == 'E' {
+			expSeen = true
+			l.next()
+			r := l.peek()
+			if r == '+' || r == '-' {
+				l.next()
+			}
 		} else if isDigit(next) {
-			digit_seen = true
+			digitSeen = true
+			l.next()
+		} else if next == '_' {
+			l.next()
 		} else {
-			l.backup()
 			break
 		}
-		if point_seen && !digit_seen {
+		if pointSeen && !digitSeen {
 			return l.errorf("cannot start float with a dot")
 		}
 	}
 
-	if !digit_seen {
+	if !digitSeen {
 		return l.errorf("no digit in that number")
 	}
-	if point_seen {
+	if pointSeen || expSeen {
 		l.emit(tokenFloat)
 	} else {
 		l.emit(tokenInteger)
 	}
-	return lexRvalue
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
 }
 
 func init() {
-	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z")
+	dateRegexp = regexp.MustCompile(`^\d{1,4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})`)
 }
 
 // Entry point
-func lex(input string) (*lexer, chan token) {
-	l := &lexer{
-		input:  input,
-		tokens: make(chan token),
+func lexToml(input io.Reader) chan token {
+	bufferedInput := buffruneio.NewReader(input)
+	l := &tomlLexer{
+		input:         bufferedInput,
+		tokens:        make(chan token),
+		line:          1,
+		col:           1,
+		endbufferLine: 1,
+		endbufferCol:  1,
 	}
 	go l.run()
-	return l, l.tokens
+	return l.tokens
 }

lexer_test.go

@@ -1,15 +1,23 @@
 package toml
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	_, ch := lex(input)
+	ch := lexToml(strings.NewReader(input))
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {
+			t.Log("While testing: ", input)
+			t.Log("compared (got)", token, "to (expected)", expected)
+			t.Log("\tvalue:", token.val, "<->", expected.val)
+			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
+			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
+			t.Log("\tline:", token.Line, "<->", expected.Line)
+			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
 			t.Log("compared", token, "to", expected)
-			t.Log(token.val, "<->", expected.val)
-			t.Log(token.typ, "<->", expected.typ)
 			t.FailNow()
 		}
 	}
@@ -29,387 +37,672 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
 
 func TestValidKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenLeftBracket, "["},
+		{Position{1, 2}, tokenKeyGroup, "hello world"},
+		{Position{1, 13}, tokenRightBracket, "]"},
+		{Position{1, 14}, tokenEOF, ""},
+	})
+}
+
+func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
+	testFlow(t, `[ j . "ʞ" . l ]`, []token{
+		{Position{1, 1}, tokenLeftBracket, "["},
+		{Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
+		{Position{1, 15}, tokenRightBracket, "]"},
+		{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestUnclosedKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenError, "unclosed key group"},
+		{Position{1, 1}, tokenLeftBracket, "["},
+		{Position{1, 2}, tokenError, "unclosed key group"},
 	})
 }
 
 func TestComment(t *testing.T) {
 	testFlow(t, "# blahblah", []token{
-		token{tokenEOF, ""},
+		{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyGroupComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenLeftBracket, "["},
+		{Position{1, 2}, tokenKeyGroup, "hello world"},
+		{Position{1, 13}, tokenRightBracket, "]"},
+		{Position{1, 25}, tokenEOF, ""},
 	})
 }
 
 func TestMultipleKeyGroupsComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah\n[test]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "test"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenLeftBracket, "["},
+		{Position{1, 2}, tokenKeyGroup, "hello world"},
+		{Position{1, 13}, tokenRightBracket, "]"},
+		{Position{2, 1}, tokenLeftBracket, "["},
+		{Position{2, 2}, tokenKeyGroup, "test"},
+		{Position{2, 6}, tokenRightBracket, "]"},
+		{Position{2, 7}, tokenEOF, ""},
+	})
+}
+
+func TestSimpleWindowsCRLF(t *testing.T) {
+	testFlow(t, "a=4\r\nb=2", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 2}, tokenEqual, "="},
+		{Position{1, 3}, tokenInteger, "4"},
+		{Position{2, 1}, tokenKey, "b"},
+		{Position{2, 2}, tokenEqual, "="},
+		{Position{2, 3}, tokenInteger, "2"},
+		{Position{2, 4}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
-		token{tokenKey, "hello"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "hello"},
+		{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUnderscore(t *testing.T) {
 	testFlow(t, "hello_hello", []token{
-		token{tokenKey, "hello_hello"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "hello_hello"},
+		{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithDash(t *testing.T) {
 	testFlow(t, "hello-world", []token{
-		token{tokenKey, "hello-world"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "hello-world"},
+		{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUppercaseMix(t *testing.T) {
 	testFlow(t, "helloHELLOHello", []token{
-		token{tokenKey, "helloHELLOHello"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "helloHELLOHello"},
+		{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
 	testFlow(t, "héllÖ", []token{
-		token{tokenKey, "héllÖ"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "héllÖ"},
+		{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyAndEqual(t *testing.T) {
 	testFlow(t, "hello =", []token{
-		token{tokenKey, "hello"},
-		token{tokenEqual, "="},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "hello"},
+		{Position{1, 7}, tokenEqual, "="},
+		{Position{1, 8}, tokenEOF, ""},
 	})
 }
 
 func TestKeyWithSharpAndEqual(t *testing.T) {
 	testFlow(t, "key#name = 5", []token{
-		token{tokenKey, "key#name"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenError, "keys cannot contain # character"},
 	})
 }
+
 func TestKeyWithSymbolsAndEqual(t *testing.T) {
-	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
-		token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
+	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+		{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
 	})
 }
 
 func TestKeyEqualStringEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello\\\"\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello\""},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello\""`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "hello\""},
+		{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnfinished(t *testing.T) {
-	testFlow(t, "foo = \"bar", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "unclosed string"},
+	testFlow(t, `foo = "bar`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unclosed string"},
 	})
 }
 
 func TestKeyEqualString(t *testing.T) {
-	testFlow(t, "foo = \"bar\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "bar"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "bar"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "bar"},
+		{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualTrue(t *testing.T) {
 	testFlow(t, "foo = true", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenTrue, "true"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenTrue, "true"},
+		{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualFalse(t *testing.T) {
 	testFlow(t, "foo = false", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFalse, "false"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenFalse, "false"},
+		{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedString(t *testing.T) {
-	testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenString, "hello"},
-		token{tokenComma, ","},
-		token{tokenString, "world"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenLeftBracket, "["},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 9}, tokenString, "hello"},
+		{Position{1, 15}, tokenComma, ","},
+		{Position{1, 18}, tokenString, "world"},
+		{Position{1, 24}, tokenRightBracket, "]"},
+		{Position{1, 26}, tokenRightBracket, "]"},
+		{Position{1, 27}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedInts(t *testing.T) {
 	testFlow(t, "a = [ [42, 21], [10] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenRightBracket, "]"},
-		token{tokenComma, ","},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "10"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenLeftBracket, "["},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 8}, tokenInteger, "42"},
+		{Position{1, 10}, tokenComma, ","},
+		{Position{1, 12}, tokenInteger, "21"},
+		{Position{1, 14}, tokenRightBracket, "]"},
+		{Position{1, 15}, tokenComma, ","},
+		{Position{1, 17}, tokenLeftBracket, "["},
+		{Position{1, 18}, tokenInteger, "10"},
+		{Position{1, 20}, tokenRightBracket, "]"},
+		{Position{1, 22}, tokenRightBracket, "]"},
+		{Position{1, 23}, tokenEOF, ""},
 	})
 }
 
 func TestArrayInts(t *testing.T) {
 	testFlow(t, "a = [ 42, 21, 10, ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenComma, ","},
-		token{tokenInteger, "10"},
-		token{tokenComma, ","},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenLeftBracket, "["},
+		{Position{1, 7}, tokenInteger, "42"},
+		{Position{1, 9}, tokenComma, ","},
+		{Position{1, 11}, tokenInteger, "21"},
+		{Position{1, 13}, tokenComma, ","},
+		{Position{1, 15}, tokenInteger, "10"},
+		{Position{1, 17}, tokenComma, ","},
+		{Position{1, 19}, tokenRightBracket, "]"},
+		{Position{1, 20}, tokenEOF, ""},
 	})
 }
 
 func TestMultilineArrayComments(t *testing.T) {
 	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "1"},
-		token{tokenComma, ","},
-		token{tokenInteger, "2"},
-		token{tokenComma, ","},
-		token{tokenInteger, "3"},
-		token{tokenComma, ","},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenLeftBracket, "["},
+		{Position{1, 6}, tokenInteger, "1"},
+		{Position{1, 7}, tokenComma, ","},
+		{Position{2, 1}, tokenInteger, "2"},
+		{Position{2, 2}, tokenComma, ","},
+		{Position{3, 1}, tokenInteger, "3"},
+		{Position{3, 2}, tokenComma, ","},
+		{Position{4, 1}, tokenRightBracket, "]"},
+		{Position{4, 2}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 8}, tokenTrue, "true"},
+		{Position{1, 12}, tokenComma, ","},
+		{Position{1, 14}, tokenFalse, "false"},
+		{Position{1, 19}, tokenComma, ","},
+		{Position{1, 21}, tokenTrue, "true"},
+		{Position{1, 25}, tokenRightBracket, "]"},
+		{Position{1, 26}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 	testFlow(t, "foo = [true, false, true] # YEAH", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 8}, tokenTrue, "true"},
+		{Position{1, 12}, tokenComma, ","},
+		{Position{1, 14}, tokenFalse, "false"},
+		{Position{1, 19}, tokenComma, ","},
+		{Position{1, 21}, tokenTrue, "true"},
+		{Position{1, 25}, tokenRightBracket, "]"},
+		{Position{1, 33}, tokenEOF, ""},
 	})
 }
 
 func TestDateRegexp(t *testing.T) {
 	if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
-		t.Fail()
+		t.Error("basic lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
+		t.Error("offset lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
+		t.Error("nano precision lexing")
 	}
 }
 
 func TestKeyEqualDate(t *testing.T) {
 	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenDate, "1979-05-27T07:32:00Z"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
+		{Position{1, 27}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
+		{Position{1, 32}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
+		{Position{1, 39}, tokenEOF, ""},
 	})
 }
 
 func TestFloatEndingWithDot(t *testing.T) {
 	testFlow(t, "foo = 42.", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "float cannot end with a dot"},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenError, "float cannot end with a dot"},
 	})
 }
 
 func TestFloatWithTwoDots(t *testing.T) {
 	testFlow(t, "foo = 4.2.", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "cannot have two dots in one float"},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenError, "cannot have two dots in one float"},
 	})
 }
 
-func TestDoubleEqualKey(t *testing.T) {
-	testFlow(t, "foo= = 2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "cannot have multiple equals for the same key"},
+func TestFloatWithExponent1(t *testing.T) {
+	testFlow(t, "a = 5e+22", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenFloat, "5e+22"},
+		{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent2(t *testing.T) {
+	testFlow(t, "a = 5E+22", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenFloat, "5E+22"},
+		{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent3(t *testing.T) {
+	testFlow(t, "a = -5e+22", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenFloat, "-5e+22"},
+		{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent4(t *testing.T) {
+	testFlow(t, "a = -5e-22", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenFloat, "-5e-22"},
+		{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent5(t *testing.T) {
+	testFlow(t, "a = 6.626e-34", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenFloat, "6.626e-34"},
+		{Position{1, 14}, tokenEOF, ""},
 	})
 }
 
 func TestInvalidEsquapeSequence(t *testing.T) {
-	testFlow(t, "foo = \"\\x\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "invalid escape sequence: \\x"},
+	testFlow(t, `foo = "\x"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
 	})
 }
 
 func TestNestedArrays(t *testing.T) {
 	testFlow(t, "foo = [[[]]]", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenLeftBracket, "["},
+		{Position{1, 8}, tokenLeftBracket, "["},
+		{Position{1, 9}, tokenLeftBracket, "["},
+		{Position{1, 10}, tokenRightBracket, "]"},
+		{Position{1, 11}, tokenRightBracket, "]"},
+		{Position{1, 12}, tokenRightBracket, "]"},
+		{Position{1, 13}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualNumber(t *testing.T) {
 	testFlow(t, "foo = 42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "42"},
+		{Position{1, 9}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "+42"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "+42"},
+		{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "-42"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "-42"},
+		{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = 4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "4.2"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenFloat, "4.2"},
+		{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "+4.2"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenFloat, "+4.2"},
+		{Position{1, 11}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "-4.2"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenFloat, "-4.2"},
+		{Position{1, 11}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 1_000", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "1_000"},
+		{Position{1, 12}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 5_349_221", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "5_349_221"},
+		{Position{1, 16}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 1_2_3_4_5", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "1_2_3_4_5"},
+		{Position{1, 16}, tokenEOF, ""},
+	})
+
+	testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
+		{Position{1, 1}, tokenKey, "flt8"},
+		{Position{1, 6}, tokenEqual, "="},
+		{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
+		{Position{1, 33}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = +", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenError, "no digit in that number"},
 	})
 }
 
 func TestMultiline(t *testing.T) {
 	testFlow(t, "foo = 42\nbar=21", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenKey, "bar"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "21"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 7}, tokenInteger, "42"},
+		{Position{2, 1}, tokenKey, "bar"},
+		{Position{2, 4}, tokenEqual, "="},
+		{Position{2, 5}, tokenInteger, "21"},
+		{Position{2, 7}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello \\u2665\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello \u2665"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "hello ♥"},
+		{Position{1, 21}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "hello \U000003B4"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "hello δ"},
+		{Position{1, 25}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "\u2"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unfinished unicode escape"},
+	})
+	testFlow(t, `foo = "\U2"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unfinished unicode escape"},
+	})
+}
+
+func TestKeyEqualStringNoEscape(t *testing.T) {
+	testFlow(t, "foo = \"hello \u0002\"", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unescaped control character U+0002"},
+	})
+	testFlow(t, "foo = \"hello \u001F\"", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unescaped control character U+001F"},
+	})
+}
+
+func TestLiteralString(t *testing.T) {
+	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
+		{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
+		{Position{1, 35}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
+		{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, `<\i\c*\s*>`},
+		{Position{1, 19}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenError, "unclosed string"},
+	})
+}
+
+func TestMultilineLiteralString(t *testing.T) {
+	testFlow(t, `foo = '''hello 'literal' world'''`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 10}, tokenString, `hello 'literal' world`},
+		{Position{1, 34}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
+		{Position{4, 9}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
+		{Position{4, 9}, tokenEOF, ""},
+	})
+}
+
+func TestMultilineString(t *testing.T) {
+	testFlow(t, `foo = """hello "literal" world"""`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 10}, tokenString, `hello "literal" world`},
+		{Position{1, 34}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{2, 1}, tokenString, "hello\"literal\"world"},
+		{Position{4, 9}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 10}, tokenString, "hellomultilineworld"},
+		{Position{6, 9}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
+		{Position{1, 1}, tokenKey, "key2"},
+		{Position{1, 6}, tokenEqual, "="},
+		{Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
+		{Position{6, 21}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
+		{Position{1, 1}, tokenKey, "key2"},
+		{Position{1, 6}, tokenEqual, "="},
+		{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
+		{Position{5, 11}, tokenEOF, ""},
+	})
+
+	testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
+		{Position{1, 1}, tokenKey, "key2"},
+		{Position{1, 6}, tokenEqual, "="},
+		{Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
+		{Position{1, 41}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
+		{Position{1, 1}, tokenKey, "key2"},
+		{Position{1, 6}, tokenEqual, "="},
+		{Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
+		{Position{3, 20}, tokenEOF, ""},
 	})
 }
 
 func TestUnicodeString(t *testing.T) {
-	testFlow(t, "foo = \"hello ♥ world\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥ world"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello ♥ world"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "hello ♥ world"},
+		{Position{1, 22}, tokenEOF, ""},
+	})
+}
+func TestEscapeInString(t *testing.T) {
+	testFlow(t, `foo = "\b\f\/"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "\b\f/"},
+		{Position{1, 15}, tokenEOF, ""},
 	})
 }
 
 func TestKeyGroupArray(t *testing.T) {
 	testFlow(t, "[[foo]]", []token{
-		token{tokenDoubleLeftBracket, "[["},
-		token{tokenKeyGroupArray, "foo"},
-		token{tokenDoubleRightBracket, "]]"},
-		token{tokenEOF, ""},
+		{Position{1, 1}, tokenDoubleLeftBracket, "[["},
+		{Position{1, 3}, tokenKeyGroupArray, "foo"},
+		{Position{1, 6}, tokenDoubleRightBracket, "]]"},
+		{Position{1, 8}, tokenEOF, ""},
+	})
+}
+
+func TestQuotedKey(t *testing.T) {
+	testFlow(t, "\"a b\" = 42", []token{
+		{Position{1, 1}, tokenKey, "\"a b\""},
+		{Position{1, 7}, tokenEqual, "="},
+		{Position{1, 9}, tokenInteger, "42"},
+		{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestKeyNewline(t *testing.T) {
+	testFlow(t, "a\n= 4", []token{
+		{Position{1, 1}, tokenError, "keys cannot contain new lines"},
+	})
+}
+
+func TestInvalidFloat(t *testing.T) {
+	testFlow(t, "a=7e1_", []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 2}, tokenEqual, "="},
+		{Position{1, 3}, tokenFloat, "7e1_"},
+		{Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexUnknownRvalue(t *testing.T) {
+	testFlow(t, `a = !b`, []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenError, "no value can start with !"},
+	})
+
+	testFlow(t, `a = \b`, []token{
+		{Position{1, 1}, tokenKey, "a"},
+		{Position{1, 3}, tokenEqual, "="},
+		{Position{1, 5}, tokenError, `no value can start with \`},
 	})
 }

match.go

@@ -0,0 +1,234 @@
+package toml
+
+import (
+	"fmt"
+)
+
+// support function to set positions for tomlValues
+// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
+// values returned by the query engines
+func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
+	switch castNode := node.(type) {
+	case *tomlValue:
+		ctx.lastPosition = castNode.position
+		return castNode.value
+	case []*TomlTree:
+		if len(castNode) > 0 {
+			ctx.lastPosition = castNode[0].position
+		}
+		return node
+	default:
+		return node
+	}
+}
+
+// base match
+type matchBase struct {
+	next pathFn
+}
+
+func (f *matchBase) setNext(next pathFn) {
+	f.next = next
+}
+
+// terminating functor - gathers results
+type terminatingFn struct {
+	// empty
+}
+
+func newTerminatingFn() *terminatingFn {
+	return &terminatingFn{}
+}
+
+func (f *terminatingFn) setNext(next pathFn) {
+	// do nothing
+}
+
+func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
+	switch castNode := node.(type) {
+	case *TomlTree:
+		ctx.result.appendResult(node, castNode.position)
+	case *tomlValue:
+		ctx.result.appendResult(node, castNode.position)
+	default:
+		// use last position for scalars
+		ctx.result.appendResult(node, ctx.lastPosition)
+	}
+}
+
+// match single key
+type matchKeyFn struct {
+	matchBase
+	Name string
+}
+
+func newMatchKeyFn(name string) *matchKeyFn {
+	return &matchKeyFn{Name: name}
+}
+
+func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
+	if array, ok := node.([]*TomlTree); ok {
+		for _, tree := range array {
+			item := tree.values[f.Name]
+			if item != nil {
+				f.next.call(item, ctx)
+			}
+		}
+	} else if tree, ok := node.(*TomlTree); ok {
+		item := tree.values[f.Name]
+		if item != nil {
+			f.next.call(item, ctx)
+		}
+	}
+}
+
+// match single index
+type matchIndexFn struct {
+	matchBase
+	Idx int
+}
+
+func newMatchIndexFn(idx int) *matchIndexFn {
+	return &matchIndexFn{Idx: idx}
+}
+
+func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		if f.Idx < len(arr) && f.Idx >= 0 {
+			f.next.call(arr[f.Idx], ctx)
+		}
+	}
+}
+
+// filter by slicing
+type matchSliceFn struct {
+	matchBase
+	Start, End, Step int
+}
+
+func newMatchSliceFn(start, end, step int) *matchSliceFn {
+	return &matchSliceFn{Start: start, End: end, Step: step}
+}
+
+func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		// adjust indexes for negative values, reverse ordering
+		realStart, realEnd := f.Start, f.End
+		if realStart < 0 {
+			realStart = len(arr) + realStart
+		}
+		if realEnd < 0 {
+			realEnd = len(arr) + realEnd
+		}
+		if realEnd < realStart {
+			realEnd, realStart = realStart, realEnd // swap
+		}
+		// loop and gather
+		for idx := realStart; idx < realEnd; idx += f.Step {
+			f.next.call(arr[idx], ctx)
+		}
+	}
+}
+
+// match anything
+type matchAnyFn struct {
+	matchBase
+}
+
+func newMatchAnyFn() *matchAnyFn {
+	return &matchAnyFn{}
+}
+
+func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		for _, v := range tree.values {
+			f.next.call(v, ctx)
+		}
+	}
+}
+
+// filter through union
+type matchUnionFn struct {
+	Union []pathFn
+}
+
+func (f *matchUnionFn) setNext(next pathFn) {
+	for _, fn := range f.Union {
+		fn.setNext(next)
+	}
+}
+
+func (f *matchUnionFn) call(node interface{}, ctx *queryContext) {
+	for _, fn := range f.Union {
+		fn.call(node, ctx)
+	}
+}
+
+// match every single last node in the tree
+type matchRecursiveFn struct {
+	matchBase
+}
+
+func newMatchRecursiveFn() *matchRecursiveFn {
+	return &matchRecursiveFn{}
+}
+
+func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		var visit func(tree *TomlTree)
+		visit = func(tree *TomlTree) {
+			for _, v := range tree.values {
+				f.next.call(v, ctx)
+				switch node := v.(type) {
+				case *TomlTree:
+					visit(node)
+				case []*TomlTree:
+					for _, subtree := range node {
+						visit(subtree)
+					}
+				}
+			}
+		}
+		f.next.call(tree, ctx)
+		visit(tree)
+	}
+}
+
+// match based on an externally provided functional filter
+type matchFilterFn struct {
+	matchBase
+	Pos  Position
+	Name string
+}
+
+func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+	return &matchFilterFn{Name: name, Pos: pos}
+}
+
+func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
+	fn, ok := (*ctx.filters)[f.Name]
+	if !ok {
+		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
+			f.Pos.String(), f.Name))
+	}
+	switch castNode := tomlValueCheck(node, ctx).(type) {
+	case *TomlTree:
+		for _, v := range castNode.values {
+			if tv, ok := v.(*tomlValue); ok {
+				if fn(tv.value) {
+					f.next.call(v, ctx)
+				}
+			} else {
+				if fn(v) {
+					f.next.call(v, ctx)
+				}
+			}
+		}
+	case []interface{}:
+		for _, v := range castNode {
+			if fn(v) {
+				f.next.call(v, ctx)
+			}
+		}
+	}
+}

match_test.go

@@ -0,0 +1,201 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+// dump path tree to a string
+func pathString(root pathFn) string {
+	result := fmt.Sprintf("%T:", root)
+	switch fn := root.(type) {
+	case *terminatingFn:
+		result += "{}"
+	case *matchKeyFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	case *matchIndexFn:
+		result += fmt.Sprintf("{%d}", fn.Idx)
+		result += pathString(fn.next)
+	case *matchSliceFn:
+		result += fmt.Sprintf("{%d:%d:%d}",
+			fn.Start, fn.End, fn.Step)
+		result += pathString(fn.next)
+	case *matchAnyFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchUnionFn:
+		result += "{["
+		for _, v := range fn.Union {
+			result += pathString(v) + ", "
+		}
+		result += "]}"
+	case *matchRecursiveFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchFilterFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	}
+	return result
+}
+
+func assertPathMatch(t *testing.T, path, ref *Query) bool {
+	pathStr := pathString(path.root)
+	refStr := pathString(ref.root)
+	if pathStr != refStr {
+		t.Errorf("paths do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+		return false
+	}
+	return true
+}
+
+func assertPath(t *testing.T, query string, ref *Query) {
+	path, _ := parseQuery(lexQuery(query))
+	assertPathMatch(t, path, ref)
+}
+
+func buildPath(parts ...pathFn) *Query {
+	query := newQuery()
+	for _, v := range parts {
+		query.appendPath(v)
+	}
+	return query
+}
+
+func TestPathRoot(t *testing.T) {
+	assertPath(t,
+		"$",
+		buildPath(
+		// empty
+		))
+}
+
+func TestPathKey(t *testing.T) {
+	assertPath(t,
+		"$.foo",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketKey(t *testing.T) {
+	assertPath(t,
+		"$[foo]",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketStringKey(t *testing.T) {
+	assertPath(t,
+		"$['foo']",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathIndex(t *testing.T) {
+	assertPath(t,
+		"$[123]",
+		buildPath(
+			newMatchIndexFn(123),
+		))
+}
+
+func TestPathSliceStart(t *testing.T) {
+	assertPath(t,
+		"$[123:]",
+		buildPath(
+			newMatchSliceFn(123, maxInt, 1),
+		))
+}
+
+func TestPathSliceStartEnd(t *testing.T) {
+	assertPath(t,
+		"$[123:456]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartEndColon(t *testing.T) {
+	assertPath(t,
+		"$[123:456:]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartStep(t *testing.T) {
+	assertPath(t,
+		"$[123::7]",
+		buildPath(
+			newMatchSliceFn(123, maxInt, 7),
+		))
+}
+
+func TestPathSliceEndStep(t *testing.T) {
+	assertPath(t,
+		"$[:456:7]",
+		buildPath(
+			newMatchSliceFn(0, 456, 7),
+		))
+}
+
+func TestPathSliceStep(t *testing.T) {
+	assertPath(t,
+		"$[::7]",
+		buildPath(
+			newMatchSliceFn(0, maxInt, 7),
+		))
+}
+
+func TestPathSliceAll(t *testing.T) {
+	assertPath(t,
+		"$[123:456:7]",
+		buildPath(
+			newMatchSliceFn(123, 456, 7),
+		))
+}
+
+func TestPathAny(t *testing.T) {
+	assertPath(t,
+		"$.*",
+		buildPath(
+			newMatchAnyFn(),
+		))
+}
+
+func TestPathUnion(t *testing.T) {
+	assertPath(t,
+		"$[foo, bar, baz]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchKeyFn("foo"),
+				newMatchKeyFn("bar"),
+				newMatchKeyFn("baz"),
+			}},
+		))
+}
+
+func TestPathRecurse(t *testing.T) {
+	assertPath(t,
+		"$..*",
+		buildPath(
+			newMatchRecursiveFn(),
+		))
+}
+
+func TestPathFilterExpr(t *testing.T) {
+	assertPath(t,
+		"$[?('foo'),?(bar)]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchFilterFn("foo", Position{}),
+				newMatchFilterFn("bar", Position{}),
+			}},
+		))
+}

parser.go

@@ -5,12 +5,13 @@ package toml
 import (
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 )
 
-type parser struct {
+type tomlParser struct {
 	flow          chan token
 	tree          *TomlTree
 	tokensBuffer  []token
@@ -18,15 +19,20 @@ type parser struct {
 	seenGroupKeys []string
 }
 
-type parserStateFn func(*parser) parserStateFn
+type tomlParserStateFn func() tomlParserStateFn
 
-func (p *parser) run() {
-	for state := parseStart; state != nil; {
-		state = state(p)
+// Formats and panics an error message based on a token
+func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) {
+	panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...))
+}
+
+func (p *tomlParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
 	}
 }
 
-func (p *parser) peek() *token {
+func (p *tomlParser) peek() *token {
 	if len(p.tokensBuffer) != 0 {
 		return &(p.tokensBuffer[0])
 	}
@@ -39,17 +45,17 @@ func (p *parser) peek() *token {
 	return &tok
 }
 
-func (p *parser) assume(typ tokenType) {
+func (p *tomlParser) assume(typ tokenType) {
 	tok := p.getToken()
 	if tok == nil {
-		panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ))
+		p.raiseError(tok, "was expecting token %s, but token stream is empty", tok)
 	}
 	if tok.typ != typ {
-		panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ))
+		p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok)
 	}
 }
 
-func (p *parser) getToken() *token {
+func (p *tomlParser) getToken() *token {
 	if len(p.tokensBuffer) != 0 {
 		tok := p.tokensBuffer[0]
 		p.tokensBuffer = p.tokensBuffer[1:]
@@ -62,7 +68,7 @@ func (p *parser) getToken() *token {
 	return &tok
 }
 
-func parseStart(p *parser) parserStateFn {
+func (p *tomlParser) parseStart() tomlParserStateFn {
 	tok := p.peek()
 
 	// end of stream, parsing is finished
@@ -72,105 +78,162 @@ func parseStart(p *parser) parserStateFn {
 
 	switch tok.typ {
 	case tokenDoubleLeftBracket:
-		return parseGroupArray
+		return p.parseGroupArray
 	case tokenLeftBracket:
-		return parseGroup
+		return p.parseGroup
 	case tokenKey:
-		return parseAssign
+		return p.parseAssign
 	case tokenEOF:
 		return nil
 	default:
-		panic("unexpected token")
+		p.raiseError(tok, "unexpected token")
 	}
 	return nil
 }
 
-func parseGroupArray(p *parser) parserStateFn {
-	p.getToken() // discard the [[
+func (p *tomlParser) parseGroupArray() tomlParserStateFn {
+	startToken := p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
 	}
 
 	// get or create group array element at the indicated part in the path
-	p.currentGroup = strings.Split(key.val, ".")
-	dest_tree := p.tree.GetPath(p.currentGroup)
-	var array []*TomlTree
-	if dest_tree == nil {
-		array = make([]*TomlTree, 0)
-	} else if dest_tree.([]*TomlTree) != nil {
-		array = dest_tree.([]*TomlTree)
-	} else {
-		panic(fmt.Sprintf("key %s is already assigned and not of type group array", key))
+	keys, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid group array key: %s", err)
 	}
+	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
+	destTree := p.tree.GetPath(keys)
+	var array []*TomlTree
+	if destTree == nil {
+		array = make([]*TomlTree, 0)
+	} else if target, ok := destTree.([]*TomlTree); ok && target != nil {
+		array = destTree.([]*TomlTree)
+	} else {
+		p.raiseError(key, "key %s is already assigned and not of type group array", key)
+	}
+	p.currentGroup = keys
 
 	// add a new tree to the end of the group array
-	new_tree := make(TomlTree)
-	array = append(array, &new_tree)
+	newTree := newTomlTree()
+	newTree.position = startToken.Position
+	array = append(array, newTree)
 	p.tree.SetPath(p.currentGroup, array)
 
+	// remove all keys that were children of this group array
+	prefix := key.val + "."
+	found := false
+	for ii := 0; ii < len(p.seenGroupKeys); {
+		groupKey := p.seenGroupKeys[ii]
+		if strings.HasPrefix(groupKey, prefix) {
+			p.seenGroupKeys = append(p.seenGroupKeys[:ii], p.seenGroupKeys[ii+1:]...)
+		} else {
+			found = (groupKey == key.val)
+			ii++
+		}
+	}
+
 	// keep this key name from use by other kinds of assignments
-	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	if !found {
+		p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	}
 
 	// move to next parser state
 	p.assume(tokenDoubleRightBracket)
-	return parseStart(p)
+	return p.parseStart
 }
 
-func parseGroup(p *parser) parserStateFn {
-	p.getToken() // discard the [
+func (p *tomlParser) parseGroup() tomlParserStateFn {
+	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
 	}
 	for _, item := range p.seenGroupKeys {
 		if item == key.val {
-			panic("duplicated tables")
+			p.raiseError(key, "duplicated tables")
 		}
 	}
+
 	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
-	p.tree.createSubTree(key.val)
+	keys, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid group array key: %s", err)
+	}
+	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
+		p.raiseError(key, "%s", err)
+	}
 	p.assume(tokenRightBracket)
-	p.currentGroup = strings.Split(key.val, ".")
-	return parseStart(p)
+	p.currentGroup = keys
+	return p.parseStart
 }
 
-func parseAssign(p *parser) parserStateFn {
+func (p *tomlParser) parseAssign() tomlParserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
-	value := parseRvalue(p)
-	var group_key []string
+
+	value := p.parseRvalue()
+	var groupKey []string
 	if len(p.currentGroup) > 0 {
-		group_key = p.currentGroup
+		groupKey = p.currentGroup
 	} else {
-		group_key = make([]string, 0)
+		groupKey = []string{}
 	}
 
 	// find the group to assign, looking out for arrays of groups
-	var target_node *TomlTree
-	switch node := p.tree.GetPath(group_key).(type) {
+	var targetNode *TomlTree
+	switch node := p.tree.GetPath(groupKey).(type) {
 	case []*TomlTree:
-		target_node = node[len(node)-1]
+		targetNode = node[len(node)-1]
 	case *TomlTree:
-		target_node = node
+		targetNode = node
 	default:
-		panic(fmt.Sprintf("Unknown group type for path %v", group_key))
+		p.raiseError(key, "Unknown group type for path: %s",
+			strings.Join(groupKey, "."))
 	}
 
 	// assign value to the found group
-	local_key := []string{key.val}
-	final_key := append(group_key, key.val)
-	if target_node.GetPath(local_key) != nil {
-		panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
+	keyVals, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "%s", err)
 	}
-	target_node.SetPath(local_key, value)
-	return parseStart(p)
+	if len(keyVals) != 1 {
+		p.raiseError(key, "Invalid key")
+	}
+	keyVal := keyVals[0]
+	localKey := []string{keyVal}
+	finalKey := append(groupKey, keyVal)
+	if targetNode.GetPath(localKey) != nil {
+		p.raiseError(key, "The following key was defined twice: %s",
+			strings.Join(finalKey, "."))
+	}
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *TomlTree, []*TomlTree:
+		toInsert = value
+	default:
+		toInsert = &tomlValue{value, key.Position}
+	}
+	targetNode.values[keyVal] = toInsert
+	return p.parseStart
 }
 
-func parseRvalue(p *parser) interface{} {
+var numberUnderscoreInvalidRegexp *regexp.Regexp
+
+func cleanupNumberToken(value string) (string, error) {
+	if numberUnderscoreInvalidRegexp.MatchString(value) {
+		return "", fmt.Errorf("invalid use of _ in number")
+	}
+	cleanedVal := strings.Replace(value, "_", "", -1)
+	return cleanedVal, nil
+}
+
+func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
-		panic("expecting a value")
+		p.raiseError(tok, "expecting a value")
 	}
 
 	switch tok.typ {
@@ -181,77 +244,149 @@ func parseRvalue(p *parser) interface{} {
 	case tokenFalse:
 		return false
 	case tokenInteger:
-		val, err := strconv.ParseInt(tok.val, 10, 64)
+		cleanedVal, err := cleanupNumberToken(tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
+		}
+		val, err := strconv.ParseInt(cleanedVal, 10, 64)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		val, err := strconv.ParseFloat(tok.val, 64)
+		cleanedVal, err := cleanupNumberToken(tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
+		}
+		val, err := strconv.ParseFloat(cleanedVal, 64)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenDate:
-		val, err := time.Parse(time.RFC3339, tok.val)
+		val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenLeftBracket:
-		return parseArray(p)
+		return p.parseArray()
+	case tokenLeftCurlyBrace:
+		return p.parseInlineTable()
+	case tokenEqual:
+		p.raiseError(tok, "cannot have multiple equals for the same key")
 	case tokenError:
-		panic(tok.val)
+		p.raiseError(tok, "%s", tok)
 	}
 
-	panic("never reached")
+	p.raiseError(tok, "never reached")
 
 	return nil
 }
 
-func parseArray(p *parser) []interface{} {
-	array := make([]interface{}, 0)
+func tokenIsComma(t *token) bool {
+	return t != nil && t.typ == tokenComma
+}
+
+func (p *tomlParser) parseInlineTable() *TomlTree {
+	tree := newTomlTree()
+	var previous *token
+Loop:
+	for {
+		follow := p.peek()
+		if follow == nil || follow.typ == tokenEOF {
+			p.raiseError(follow, "unterminated inline table")
+		}
+		switch follow.typ {
+		case tokenRightCurlyBrace:
+			p.getToken()
+			break Loop
+		case tokenKey:
+			if !tokenIsComma(previous) && previous != nil {
+				p.raiseError(follow, "comma expected between fields in inline table")
+			}
+			key := p.getToken()
+			p.assume(tokenEqual)
+			value := p.parseRvalue()
+			tree.Set(key.val, value)
+		case tokenComma:
+			if previous == nil {
+				p.raiseError(follow, "inline table cannot start with a comma")
+			}
+			if tokenIsComma(previous) {
+				p.raiseError(follow, "need field between two commas in inline table")
+			}
+			p.getToken()
+		default:
+			p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String())
+		}
+		previous = follow
+	}
+	if tokenIsComma(previous) {
+		p.raiseError(previous, "trailing comma at the end of inline table")
+	}
+	return tree
+}
+
+func (p *tomlParser) parseArray() interface{} {
+	var array []interface{}
 	arrayType := reflect.TypeOf(nil)
 	for {
 		follow := p.peek()
 		if follow == nil || follow.typ == tokenEOF {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ == tokenRightBracket {
 			p.getToken()
-			return array
+			break
 		}
-		val := parseRvalue(p)
+		val := p.parseRvalue()
 		if arrayType == nil {
 			arrayType = reflect.TypeOf(val)
 		}
 		if reflect.TypeOf(val) != arrayType {
-			panic("mixed types in array")
+			p.raiseError(follow, "mixed types in array")
 		}
 		array = append(array, val)
 		follow = p.peek()
-		if follow == nil {
-			panic("unterminated array")
+		if follow == nil || follow.typ == tokenEOF {
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
-			panic("missing comma")
+			p.raiseError(follow, "missing comma")
 		}
 		if follow.typ == tokenComma {
 			p.getToken()
 		}
 	}
+	// An array of TomlTrees is actually an array of inline
+	// tables, which is a shorthand for a table array. If the
+	// array was not converted from []interface{} to []*TomlTree,
+	// the two notations would not be equivalent.
+	if arrayType == reflect.TypeOf(newTomlTree()) {
+		tomlArray := make([]*TomlTree, len(array))
+		for i, v := range array {
+			tomlArray[i] = v.(*TomlTree)
+		}
+		return tomlArray
+	}
 	return array
 }
 
-func parse(flow chan token) *TomlTree {
-	result := make(TomlTree)
-	parser := &parser{
+func parseToml(flow chan token) *TomlTree {
+	result := newTomlTree()
+	result.position = Position{1, 1}
+	parser := &tomlParser{
 		flow:          flow,
-		tree:          &result,
+		tree:          result,
 		tokensBuffer:  make([]token, 0),
 		currentGroup:  make([]string, 0),
 		seenGroupKeys: make([]string, 0),
 	}
 	parser.run()
-	return parser.tree
+	return result
+}
+
+func init() {
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`)
 }

parser_test.go

@@ -2,25 +2,34 @@ package toml
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
-func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref map[string]interface{}) {
 	if err != nil {
 		t.Error("Non-nil error:", err.Error())
 		return
 	}
 	for k, v := range ref {
-		node := tree.Get(k)
-		switch cast_node := node.(type) {
+		nextPath := append(path, k)
+		t.Log("asserting path", nextPath)
+		// NOTE: directly access key instead of resolve by path
+		// NOTE: see TestSpecialKV
+		switch node := tree.GetPath([]string{k}).(type) {
 		case []*TomlTree:
-			for idx, item := range cast_node {
-				assertTree(t, item, err, v.([]map[string]interface{})[idx])
+			t.Log("\tcomparing key", nextPath, "by array iteration")
+			for idx, item := range node {
+				assertSubTree(t, nextPath, item, err, v.([]map[string]interface{})[idx])
 			}
 		case *TomlTree:
-			assertTree(t, cast_node, err, v.(map[string]interface{}))
+			t.Log("\tcomparing key", nextPath, "by subtree assestion")
+			assertSubTree(t, nextPath, node, err, v.(map[string]interface{}))
 		default:
+			t.Log("\tcomparing key", nextPath, "by string representation because it's of type", reflect.TypeOf(node))
 			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
 				t.Errorf("was expecting %v at %v but got %v", v, k, node)
 			}
@@ -28,9 +37,15 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 	}
 }
 
+func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+	t.Log("Asserting tree:\n", spew.Sdump(tree))
+	assertSubTree(t, []string{}, tree, err, ref)
+	t.Log("Finished tree assertion.")
+}
+
 func TestCreateSubTree(t *testing.T) {
-	tree := make(TomlTree)
-	tree.createSubTree("a.b.c")
+	tree := newTomlTree()
+	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
 		t.Fail()
@@ -50,6 +65,13 @@ func TestSimpleKV(t *testing.T) {
 	})
 }
 
+func TestNumberInKey(t *testing.T) {
+	tree, err := Load("hello2 = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"hello2": int64(42),
+	})
+}
+
 func TestSimpleNumbers(t *testing.T) {
 	tree, err := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -60,6 +82,44 @@ func TestSimpleNumbers(t *testing.T) {
 	})
 }
 
+func TestNumbersWithUnderscores(t *testing.T) {
+	tree, err := Load("a = 1_000")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(1000),
+	})
+
+	tree, err = Load("a = 5_349_221")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(5349221),
+	})
+
+	tree, err = Load("a = 1_2_3_4_5")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(12345),
+	})
+
+	tree, err = Load("flt8 = 9_224_617.445_991_228_313")
+	assertTree(t, tree, err, map[string]interface{}{
+		"flt8": float64(9224617.445991228313),
+	})
+
+	tree, err = Load("flt9 = 1e1_00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"flt9": float64(1e100),
+	})
+}
+
+func TestFloatsWithExponents(t *testing.T) {
+	tree, err := Load("a = 5e+22\nb = 5E+22\nc = -5e+22\nd = -5e-22\ne = 6.626e-34")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": float64(5e+22),
+		"b": float64(5E+22),
+		"c": float64(-5e+22),
+		"d": float64(-5e-22),
+		"e": float64(6.626e-34),
+	})
+}
+
 func TestSimpleDate(t *testing.T) {
 	tree, err := Load("a = 1979-05-27T07:32:00Z")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -67,6 +127,20 @@ func TestSimpleDate(t *testing.T) {
 	})
 }
 
+func TestDateOffset(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)),
+	})
+}
+
+func TestDateNano(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)),
+	})
+}
+
 func TestSimpleString(t *testing.T) {
 	tree, err := Load("a = \"hello world\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -74,6 +148,13 @@ func TestSimpleString(t *testing.T) {
 	})
 }
 
+func TestSpaceKey(t *testing.T) {
+	tree, err := Load("\"a b\" = \"hello world\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a b": "hello world",
+	})
+}
+
 func TestStringEscapables(t *testing.T) {
 	tree, err := Load("a = \"a \\n b\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -96,6 +177,16 @@ func TestStringEscapables(t *testing.T) {
 	})
 }
 
+func TestEmptyQuotedString(t *testing.T) {
+	tree, err := Load(`[""]
+"" = 1`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"": map[string]interface{}{
+			"": int64(1),
+		},
+	})
+}
+
 func TestBools(t *testing.T) {
 	tree, err := Load("a = true\nb = false")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -107,7 +198,48 @@ func TestBools(t *testing.T) {
 func TestNestedKeys(t *testing.T) {
 	tree, err := Load("[a.b.c]\nd = 42")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a.b.c.d": int64(42),
+		"a": map[string]interface{}{
+			"b": map[string]interface{}{
+				"c": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+}
+
+func TestNestedQuotedUnicodeKeys(t *testing.T) {
+	tree, err := Load("[ j . \"ʞ\" . l ]\nd = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"j": map[string]interface{}{
+			"ʞ": map[string]interface{}{
+				"l": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+
+	tree, err = Load("[ g . h . i ]\nd = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"g": map[string]interface{}{
+			"h": map[string]interface{}{
+				"i": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+
+	tree, err = Load("[ d.e.f ]\nk = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"d": map[string]interface{}{
+			"e": map[string]interface{}{
+				"f": map[string]interface{}{
+					"k": int64(42),
+				},
+			},
+		},
 	})
 }
 
@@ -147,25 +279,25 @@ func TestArrayMultiline(t *testing.T) {
 func TestArrayNested(t *testing.T) {
 	tree, err := Load("a = [[42, 21], [10]]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": [][]int64{[]int64{int64(42), int64(21)}, []int64{int64(10)}},
+		"a": [][]int64{{int64(42), int64(21)}, {int64(10)}},
 	})
 }
 
 func TestNestedEmptyArrays(t *testing.T) {
 	tree, err := Load("a = [[[]]]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": [][][]interface{}{[][]interface{}{[]interface{}{}}},
+		"a": [][][]interface{}{{{}}},
 	})
 }
 
 func TestArrayMixedTypes(t *testing.T) {
 	_, err := Load("a = [42, 16.0]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 10): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = [42, \"hello\"]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 11): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -173,20 +305,42 @@ func TestArrayMixedTypes(t *testing.T) {
 func TestArrayNestedStrings(t *testing.T) {
 	tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"data": [][]string{[]string{"gamma", "delta"}, []string{"Foo"}},
+		"data": [][]string{{"gamma", "delta"}, {"Foo"}},
 	})
 }
 
+func TestParseUnknownRvalue(t *testing.T) {
+	_, err := Load("a = !bssss")
+	if err == nil {
+		t.Error("Expecting a parse error")
+	}
+
+	_, err = Load("a = /b")
+	if err == nil {
+		t.Error("Expecting a parse error")
+	}
+}
+
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "expecting a value" {
+	if err.Error() != "(1, 5): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestUnterminatedArray(t *testing.T) {
 	_, err := Load("a = [1,")
-	if err.Error() != "unterminated array" {
+	if err.Error() != "(1, 8): unterminated array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [1")
+	if err.Error() != "(1, 7): unterminated array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [1 2")
+	if err.Error() != "(1, 8): missing comma" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -212,23 +366,97 @@ func TestArrayWithExtraCommaComment(t *testing.T) {
 	})
 }
 
+func TestSimpleInlineGroup(t *testing.T) {
+	tree, err := Load("key = {a = 42}")
+	assertTree(t, tree, err, map[string]interface{}{
+		"key": map[string]interface{}{
+			"a": int64(42),
+		},
+	})
+}
+
+func TestDoubleInlineGroup(t *testing.T) {
+	tree, err := Load("key = {a = 42, b = \"foo\"}")
+	assertTree(t, tree, err, map[string]interface{}{
+		"key": map[string]interface{}{
+			"a": int64(42),
+			"b": "foo",
+		},
+	})
+}
+
+func TestExampleInlineGroup(t *testing.T) {
+	tree, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"point": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func TestExampleInlineGroupInArray(t *testing.T) {
+	tree, err := Load(`points = [{ x = 1, y = 2 }]`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"points": []map[string]interface{}{
+			{
+				"x": int64(1),
+				"y": int64(2),
+			},
+		},
+	})
+}
+
+func TestInlineTableUnterminated(t *testing.T) {
+	_, err := Load("foo = {")
+	if err.Error() != "(1, 8): unterminated inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableCommaExpected(t *testing.T) {
+	_, err := Load("foo = {hello = 53 test = foo}")
+	if err.Error() != "(1, 19): comma expected between fields in inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableCommaStart(t *testing.T) {
+	_, err := Load("foo = {, hello = 53}")
+	if err.Error() != "(1, 8): inline table cannot start with a comma" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableDoubleComma(t *testing.T) {
+	_, err := Load("foo = {hello = 53,, foo = 17}")
+	if err.Error() != "(1, 19): need field between two commas in inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestDuplicateGroups(t *testing.T) {
 	_, err := Load("[foo]\na=2\n[foo]b=3")
-	if err.Error() != "duplicated tables" {
+	if err.Error() != "(3, 2): duplicated tables" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestDuplicateKeys(t *testing.T) {
 	_, err := Load("foo = 2\nfoo = 3")
-	if err.Error() != "the following key was defined twice: foo" {
+	if err.Error() != "(2, 1): The following key was defined twice: foo" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "empty intermediate table" {
+	if err.Error() != "(1, 2): invalid group array key: empty key group" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -249,12 +477,12 @@ func TestImplicitDeclarationBefore(t *testing.T) {
 
 func TestFloatsWithoutLeadingZeros(t *testing.T) {
 	_, err := Load("a = .42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = -.42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -270,20 +498,71 @@ func TestParseFile(t *testing.T) {
 	tree, err := LoadFile("example.toml")
 
 	assertTree(t, tree, err, map[string]interface{}{
-		"title":                   "TOML Example",
-		"owner.name":              "Tom Preston-Werner",
-		"owner.organization":      "GitHub",
-		"owner.bio":               "GitHub Cofounder & CEO\nLikes tater tots and beer.",
-		"owner.dob":               time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
-		"database.server":         "192.168.1.1",
-		"database.ports":          []int64{8001, 8001, 8002},
-		"database.connection_max": 5000,
-		"database.enabled":        true,
-		"servers.alpha.ip":        "10.0.0.1",
-		"servers.alpha.dc":        "eqdc10",
-		"servers.beta.ip":         "10.0.0.2",
-		"servers.beta.dc":         "eqdc10",
-		"clients.data":            []interface{}{[]string{"gamma", "delta"}, []int64{1, 2}},
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
+	})
+}
+
+func TestParseFileCRLF(t *testing.T) {
+	tree, err := LoadFile("example-crlf.toml")
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
 	})
 }
 
@@ -299,12 +578,66 @@ func TestParseKeyGroupArray(t *testing.T) {
 	})
 }
 
+func TestParseKeyGroupArrayUnfinished(t *testing.T) {
+	_, err := Load("[[foo.bar]\na = 42")
+	if err.Error() != "(1, 10): was expecting token [[, but got unclosed key group array instead" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("[[foo.[bar]\na = 42")
+	if err.Error() != "(1, 3): unexpected token group name cannot contain ']', was expecting a key group array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestParseKeyGroupArrayQueryExample(t *testing.T) {
+	tree, err := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"book": []map[string]interface{}{
+			{"title": "The Stand", "author": "Stephen King"},
+			{"title": "For Whom the Bell Tolls", "author": "Ernest Hemmingway"},
+			{"title": "Neuromancer", "author": "William Gibson"},
+		},
+	})
+}
+
+func TestParseKeyGroupArraySpec(t *testing.T) {
+	tree, err := Load("[[fruit]]\n name=\"apple\"\n [fruit.physical]\n color=\"red\"\n shape=\"round\"\n [[fruit]]\n name=\"banana\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"fruit": []map[string]interface{}{
+			{"name": "apple", "physical": map[string]interface{}{"color": "red", "shape": "round"}},
+			{"name": "banana"},
+		},
+	})
+}
+
 func TestToTomlValue(t *testing.T) {
 	for idx, item := range []struct {
 		Value  interface{}
 		Expect string
 	}{
+		{int(1), "1"},
+		{int8(2), "2"},
+		{int16(3), "3"},
+		{int32(4), "4"},
 		{int64(12345), "12345"},
+		{uint(10), "10"},
+		{uint8(20), "20"},
+		{uint16(30), "30"},
+		{uint32(40), "40"},
+		{uint64(50), "50"},
+		{float32(12.456), "12.456"},
 		{float64(123.45), "123.45"},
 		{bool(true), "true"},
 		{"hello world", "\"hello world\""},
@@ -314,6 +647,7 @@ func TestToTomlValue(t *testing.T) {
 			"1979-05-27T07:32:00Z"},
 		{[]interface{}{"gamma", "delta"},
 			"[\n  \"gamma\",\n  \"delta\",\n]"},
+		{nil, ""},
 	} {
 		result := toTomlValue(item.Value, 0)
 		if result != item.Expect {
@@ -323,17 +657,145 @@ func TestToTomlValue(t *testing.T) {
 }
 
 func TestToString(t *testing.T) {
-	tree := &TomlTree{
-		"foo": &TomlTree{
-			"bar": []*TomlTree{
-				{"a": int64(42)},
-				{"a": int64(69)},
-			},
-		},
+	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
+	if err != nil {
+		t.Errorf("Test failed to parse: %v", err)
+		return
 	}
 	result := tree.ToString()
-	expected := "\n[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n"
+	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
 	if result != expected {
 		t.Errorf("Expected got '%s', expected '%s'", result, expected)
 	}
 }
+
+func TestToStringMapStringString(t *testing.T) {
+	in := map[string]interface{}{"m": map[string]string{"v": "abc"}}
+	want := "\n[m]\n  v = \"abc\"\n"
+	tree := TreeFromMap(in)
+	got := tree.String()
+
+	if got != want {
+		t.Errorf("want:\n%q\ngot:\n%q", want, got)
+	}
+}
+
+func TestToStringMapInterfaceInterface(t *testing.T) {
+	in := map[string]interface{}{"m": map[interface{}]interface{}{"v": "abc"}}
+	want := "\n[m]\n  v = \"abc\"\n"
+	tree := TreeFromMap(in)
+	got := tree.String()
+
+	if got != want {
+		t.Errorf("want:\n%q\ngot:\n%q", want, got)
+	}
+}
+
+func assertPosition(t *testing.T, text string, ref map[string]Position) {
+	tree, err := Load(text)
+	if err != nil {
+		t.Errorf("Error loading document text: `%v`", text)
+		t.Errorf("Error: %v", err)
+	}
+	for path, pos := range ref {
+		testPos := tree.GetPosition(path)
+		if testPos.Invalid() {
+			t.Errorf("Failed to query tree path or path has invalid position: %s", path)
+		} else if pos != testPos {
+			t.Errorf("Expected position %v, got %v instead", pos, testPos)
+		}
+	}
+}
+
+func TestDocumentPositions(t *testing.T) {
+	assertPosition(t,
+		"[foo]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        {1, 1},
+			"foo":     {1, 1},
+			"foo.bar": {2, 1},
+			"foo.baz": {3, 1},
+		})
+}
+
+func TestDocumentPositionsWithSpaces(t *testing.T) {
+	assertPosition(t,
+		"  [foo]\n  bar=42\n  baz=69",
+		map[string]Position{
+			"":        {1, 1},
+			"foo":     {1, 3},
+			"foo.bar": {2, 3},
+			"foo.baz": {3, 3},
+		})
+}
+
+func TestDocumentPositionsWithGroupArray(t *testing.T) {
+	assertPosition(t,
+		"[[foo]]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        {1, 1},
+			"foo":     {1, 1},
+			"foo.bar": {2, 1},
+			"foo.baz": {3, 1},
+		})
+}
+
+func TestNestedTreePosition(t *testing.T) {
+	assertPosition(t,
+		"[foo.bar]\na=42\nb=69",
+		map[string]Position{
+			"":          {1, 1},
+			"foo":       {1, 1},
+			"foo.bar":   {1, 1},
+			"foo.bar.a": {2, 1},
+			"foo.bar.b": {3, 1},
+		})
+}
+
+func TestInvalidGroupArray(t *testing.T) {
+	_, err := Load("[key#group]\nanswer = 42")
+	if err == nil {
+		t.Error("Should error")
+	}
+
+	_, err = Load("[foo.[bar]\na = 42")
+	if err.Error() != "(1, 2): unexpected token group name cannot contain ']', was expecting a key group" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestDoubleEqual(t *testing.T) {
+	_, err := Load("foo= = 2")
+	if err.Error() != "(1, 6): cannot have multiple equals for the same key" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestGroupArrayReassign(t *testing.T) {
+	_, err := Load("[hello]\n[[hello]]")
+	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type group array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInvalidFloatParsing(t *testing.T) {
+	_, err := Load("a=1e_2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1e2_")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1__2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=_1_2")
+	if err.Error() != "(1, 3): cannot start number with underscore" {
+		t.Error("Bad error message:", err.Error())
+	}
+}

position.go

@@ -0,0 +1,29 @@
+// Position support for go-toml
+
+package toml
+
+import (
+	"fmt"
+)
+
+// Position of a document element within a TOML document.
+//
+// Line and Col are both 1-indexed positions for the element's line number and
+// column number, respectively.  Values of zero or less will cause Invalid(),
+// to return true.
+type Position struct {
+	Line int // line within the document
+	Col  int // column within the line
+}
+
+// String representation of the position.
+// Displays 1-indexed line and column numbers.
+func (p Position) String() string {
+	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
+}
+
+// Invalid returns whether or not the position is valid (i.e. with negative or
+// null values)
+func (p Position) Invalid() bool {
+	return p.Line <= 0 || p.Col <= 0
+}

position_test.go

@@ -0,0 +1,29 @@
+// Testing support for go-toml
+
+package toml
+
+import (
+	"testing"
+)
+
+func TestPositionString(t *testing.T) {
+	p := Position{123, 456}
+	expected := "(123, 456)"
+	value := p.String()
+
+	if value != expected {
+		t.Errorf("Expected %v, got %v instead", expected, value)
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	for i, v := range []Position{
+		{0, 1234},
+		{1234, 0},
+		{0, 0},
+	} {
+		if !v.Invalid() {
+			t.Errorf("Position at %v is valid: %v", i, v)
+		}
+	}
+}

query.go

@@ -0,0 +1,153 @@
+package toml
+
+import (
+	"time"
+)
+
+// NodeFilterFn represents a user-defined filter function, for use with
+// Query.SetFilter().
+//
+// The return value of the function must indicate if 'node' is to be included
+// at this stage of the TOML path.  Returning true will include the node, and
+// returning false will exclude it.
+//
+// NOTE: Care should be taken to write script callbacks such that they are safe
+// to use from multiple goroutines.
+type NodeFilterFn func(node interface{}) bool
+
+// QueryResult is the result of Executing a Query.
+type QueryResult struct {
+	items     []interface{}
+	positions []Position
+}
+
+// appends a value/position pair to the result set.
+func (r *QueryResult) appendResult(node interface{}, pos Position) {
+	r.items = append(r.items, node)
+	r.positions = append(r.positions, pos)
+}
+
+// Values is a set of values within a QueryResult.  The order of values is not
+// guaranteed to be in document order, and may be different each time a query is
+// executed.
+func (r QueryResult) Values() []interface{} {
+	values := make([]interface{}, len(r.items))
+	for i, v := range r.items {
+		o, ok := v.(*tomlValue)
+		if ok {
+			values[i] = o.value
+		} else {
+			values[i] = v
+		}
+	}
+	return values
+}
+
+// Positions is a set of positions for values within a QueryResult.  Each index
+// in Positions() corresponds to the entry in Value() of the same index.
+func (r QueryResult) Positions() []Position {
+	return r.positions
+}
+
+// runtime context for executing query paths
+type queryContext struct {
+	result       *QueryResult
+	filters      *map[string]NodeFilterFn
+	lastPosition Position
+}
+
+// generic path functor interface
+type pathFn interface {
+	setNext(next pathFn)
+	call(node interface{}, ctx *queryContext)
+}
+
+// A Query is the representation of a compiled TOML path.  A Query is safe
+// for concurrent use by multiple goroutines.
+type Query struct {
+	root    pathFn
+	tail    pathFn
+	filters *map[string]NodeFilterFn
+}
+
+func newQuery() *Query {
+	return &Query{
+		root:    nil,
+		tail:    nil,
+		filters: &defaultFilterFunctions,
+	}
+}
+
+func (q *Query) appendPath(next pathFn) {
+	if q.root == nil {
+		q.root = next
+	} else {
+		q.tail.setNext(next)
+	}
+	q.tail = next
+	next.setNext(newTerminatingFn()) // init the next functor
+}
+
+// CompileQuery compiles a TOML path expression.  The returned Query can be used
+// to match elements within a TomlTree and its descendants.
+func CompileQuery(path string) (*Query, error) {
+	return parseQuery(lexQuery(path))
+}
+
+// Execute executes a query against a TomlTree, and returns the result of the query.
+func (q *Query) Execute(tree *TomlTree) *QueryResult {
+	result := &QueryResult{
+		items:     []interface{}{},
+		positions: []Position{},
+	}
+	if q.root == nil {
+		result.appendResult(tree, tree.GetPosition(""))
+	} else {
+		ctx := &queryContext{
+			result:  result,
+			filters: q.filters,
+		}
+		q.root.call(tree, ctx)
+	}
+	return result
+}
+
+// SetFilter sets a user-defined filter function.  These may be used inside
+// "?(..)" query expressions to filter TOML document elements within a query.
+func (q *Query) SetFilter(name string, fn NodeFilterFn) {
+	if q.filters == &defaultFilterFunctions {
+		// clone the static table
+		q.filters = &map[string]NodeFilterFn{}
+		for k, v := range defaultFilterFunctions {
+			(*q.filters)[k] = v
+		}
+	}
+	(*q.filters)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]NodeFilterFn{
+	"tree": func(node interface{}) bool {
+		_, ok := node.(*TomlTree)
+		return ok
+	},
+	"int": func(node interface{}) bool {
+		_, ok := node.(int64)
+		return ok
+	},
+	"float": func(node interface{}) bool {
+		_, ok := node.(float64)
+		return ok
+	},
+	"string": func(node interface{}) bool {
+		_, ok := node.(string)
+		return ok
+	},
+	"time": func(node interface{}) bool {
+		_, ok := node.(time.Time)
+		return ok
+	},
+	"bool": func(node interface{}) bool {
+		_, ok := node.(bool)
+		return ok
+	},
+}

query_test.go

@@ -0,0 +1,70 @@
+package toml
+
+import (
+	"testing"
+)
+
+func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
+	if len(array) != len(objects) {
+		t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
+	}
+
+	for _, o := range objects {
+		found := false
+		for _, a := range array {
+			if a == o {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal(o, "not found in array", array)
+		}
+	}
+}
+
+func TestQueryExample(t *testing.T) {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	authors, _ := config.Query("$.book.author")
+	names := authors.Values()
+	if len(names) != 3 {
+		t.Fatalf("query should return 3 names but returned %d", len(names))
+	}
+	assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
+}
+
+func TestQueryReadmeExample(t *testing.T) {
+	config, _ := Load(`
+[postgres]
+user = "pelletier"
+password = "mypassword"
+`)
+	results, _ := config.Query("$..[user,password]")
+	values := results.Values()
+	if len(values) != 2 {
+		t.Fatalf("query should return 2 values but returned %d", len(values))
+	}
+	assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
+}
+
+func TestQueryPathNotPresent(t *testing.T) {
+	config, _ := Load(`a = "hello"`)
+	results, err := config.Query("$.foo.bar")
+	if err != nil {
+		t.Fatalf("err should be nil. got %s instead", err)
+	}
+	if len(results.items) != 0 {
+		t.Fatalf("no items should be matched. %d matched instead", len(results.items))
+	}
+}

querylexer.go

@@ -0,0 +1,356 @@
+// TOML JSONPath lexer.
+//
+// Written using the principles developed by Rob Pike in
+// http://www.youtube.com/watch?v=HxaD_trXwRE
+
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// Lexer state function
+type queryLexStateFn func() queryLexStateFn
+
+// Lexer definition
+type queryLexer struct {
+	input      string
+	start      int
+	pos        int
+	width      int
+	tokens     chan token
+	depth      int
+	line       int
+	col        int
+	stringTerm string
+}
+
+func (l *queryLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
+}
+
+func (l *queryLexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line++
+			l.col = 1
+		} else {
+			l.col++
+		}
+		i += width
+	}
+	// advance start position to next token
+	l.start = l.pos
+}
+
+func (l *queryLexer) emit(t tokenType) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      l.input[l.start:l.pos],
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) emitWithValue(t tokenType, value string) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) next() rune {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	var r rune
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *queryLexer) ignore() {
+	l.nextStart()
+}
+
+func (l *queryLexer) backup() {
+	l.pos -= l.width
+}
+
+func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func (l *queryLexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *queryLexer) accept(valid string) bool {
+	if strings.ContainsRune(valid, l.next()) {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+func (l *queryLexer) follow(next string) bool {
+	return strings.HasPrefix(l.input[l.pos:], next)
+}
+
+func (l *queryLexer) lexVoid() queryLexStateFn {
+	for {
+		next := l.peek()
+		switch next {
+		case '$':
+			l.pos++
+			l.emit(tokenDollar)
+			continue
+		case '.':
+			if l.follow("..") {
+				l.pos += 2
+				l.emit(tokenDotDot)
+			} else {
+				l.pos++
+				l.emit(tokenDot)
+			}
+			continue
+		case '[':
+			l.pos++
+			l.emit(tokenLeftBracket)
+			continue
+		case ']':
+			l.pos++
+			l.emit(tokenRightBracket)
+			continue
+		case ',':
+			l.pos++
+			l.emit(tokenComma)
+			continue
+		case '*':
+			l.pos++
+			l.emit(tokenStar)
+			continue
+		case '(':
+			l.pos++
+			l.emit(tokenLeftParen)
+			continue
+		case ')':
+			l.pos++
+			l.emit(tokenRightParen)
+			continue
+		case '?':
+			l.pos++
+			l.emit(tokenQuestion)
+			continue
+		case ':':
+			l.pos++
+			l.emit(tokenColon)
+			continue
+		case '\'':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		case '"':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		}
+
+		if isSpace(next) {
+			l.next()
+			l.ignore()
+			continue
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if l.next() == eof {
+			break
+		}
+
+		return l.errorf("unexpected char: '%v'", next)
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexKey() queryLexStateFn {
+	for {
+		next := l.peek()
+		if !isAlphanumeric(next) {
+			l.emit(tokenKey)
+			return l.lexVoid
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexString() queryLexStateFn {
+	l.pos++
+	l.ignore()
+	growingString := ""
+
+	for {
+		if l.follow(l.stringTerm) {
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
+			l.ignore()
+			return l.lexVoid
+		}
+
+		if l.follow("\\\"") {
+			l.pos++
+			growingString += "\""
+		} else if l.follow("\\'") {
+			l.pos++
+			growingString += "'"
+		} else if l.follow("\\n") {
+			l.pos++
+			growingString += "\n"
+		} else if l.follow("\\b") {
+			l.pos++
+			growingString += "\b"
+		} else if l.follow("\\f") {
+			l.pos++
+			growingString += "\f"
+		} else if l.follow("\\/") {
+			l.pos++
+			growingString += "/"
+		} else if l.follow("\\t") {
+			l.pos++
+			growingString += "\t"
+		} else if l.follow("\\r") {
+			l.pos++
+			growingString += "\r"
+		} else if l.follow("\\\\") {
+			l.pos++
+			growingString += "\\"
+		} else if l.follow("\\u") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 4; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
+		} else if l.follow("\\U") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 8; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos++
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+		} else {
+			growingString += string(l.peek())
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+
+	return l.errorf("unclosed string")
+}
+
+func (l *queryLexer) lexNumber() queryLexStateFn {
+	l.ignore()
+	if !l.accept("+") {
+		l.accept("-")
+	}
+	pointSeen := false
+	digitSeen := false
+	for {
+		next := l.next()
+		if next == '.' {
+			if pointSeen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
+			pointSeen = true
+		} else if isDigit(next) {
+			digitSeen = true
+		} else {
+			l.backup()
+			break
+		}
+		if pointSeen && !digitSeen {
+			return l.errorf("cannot start float with a dot")
+		}
+	}
+
+	if !digitSeen {
+		return l.errorf("no digit in that number")
+	}
+	if pointSeen {
+		l.emit(tokenFloat)
+	} else {
+		l.emit(tokenInteger)
+	}
+	return l.lexVoid
+}
+
+// Entry point
+func lexQuery(input string) chan token {
+	l := &queryLexer{
+		input:  input,
+		tokens: make(chan token),
+		line:   1,
+		col:    1,
+	}
+	go l.run()
+	return l.tokens
+}

querylexer_test.go

@@ -0,0 +1,178 @@
+package toml
+
+import (
+	"testing"
+)
+
+func testQLFlow(t *testing.T, input string, expectedFlow []token) {
+	ch := lexQuery(input)
+	for idx, expected := range expectedFlow {
+		token := <-ch
+		if token != expected {
+			t.Log("While testing #", idx, ":", input)
+			t.Log("compared (got)", token, "to (expected)", expected)
+			t.Log("\tvalue:", token.val, "<->", expected.val)
+			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
+			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
+			t.Log("\tline:", token.Line, "<->", expected.Line)
+			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
+			t.Log("compared", token, "to", expected)
+			t.FailNow()
+		}
+	}
+
+	tok, ok := <-ch
+	if ok {
+		t.Log("channel is not closed!")
+		t.Log(len(ch)+1, "tokens remaining:")
+
+		t.Log("token ->", tok)
+		for token := range ch {
+			t.Log("token ->", token)
+		}
+		t.FailNow()
+	}
+}
+
+func TestLexSpecialChars(t *testing.T) {
+	testQLFlow(t, " .$[]..()?*", []token{
+		{Position{1, 2}, tokenDot, "."},
+		{Position{1, 3}, tokenDollar, "$"},
+		{Position{1, 4}, tokenLeftBracket, "["},
+		{Position{1, 5}, tokenRightBracket, "]"},
+		{Position{1, 6}, tokenDotDot, ".."},
+		{Position{1, 8}, tokenLeftParen, "("},
+		{Position{1, 9}, tokenRightParen, ")"},
+		{Position{1, 10}, tokenQuestion, "?"},
+		{Position{1, 11}, tokenStar, "*"},
+		{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexString(t *testing.T) {
+	testQLFlow(t, "'foo\n'", []token{
+		{Position{1, 2}, tokenString, "foo\n"},
+		{Position{2, 2}, tokenEOF, ""},
+	})
+}
+
+func TestLexDoubleString(t *testing.T) {
+	testQLFlow(t, `"bar"`, []token{
+		{Position{1, 2}, tokenString, "bar"},
+		{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexStringEscapes(t *testing.T) {
+	testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{
+		{Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"},
+		{Position{1, 55}, tokenEOF, ""},
+	})
+}
+
+func TestLexStringUnfinishedUnicode4(t *testing.T) {
+	testQLFlow(t, `"\u000"`, []token{
+		{Position{1, 2}, tokenError, "unfinished unicode escape"},
+	})
+}
+
+func TestLexStringUnfinishedUnicode8(t *testing.T) {
+	testQLFlow(t, `"\U0000"`, []token{
+		{Position{1, 2}, tokenError, "unfinished unicode escape"},
+	})
+}
+
+func TestLexStringInvalidEscape(t *testing.T) {
+	testQLFlow(t, `"\x"`, []token{
+		{Position{1, 2}, tokenError, "invalid escape sequence: \\x"},
+	})
+}
+
+func TestLexStringUnfinished(t *testing.T) {
+	testQLFlow(t, `"bar`, []token{
+		{Position{1, 2}, tokenError, "unclosed string"},
+	})
+}
+
+func TestLexKey(t *testing.T) {
+	testQLFlow(t, "foo", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 4}, tokenEOF, ""},
+	})
+}
+
+func TestLexRecurse(t *testing.T) {
+	testQLFlow(t, "$..*", []token{
+		{Position{1, 1}, tokenDollar, "$"},
+		{Position{1, 2}, tokenDotDot, ".."},
+		{Position{1, 4}, tokenStar, "*"},
+		{Position{1, 5}, tokenEOF, ""},
+	})
+}
+
+func TestLexBracketKey(t *testing.T) {
+	testQLFlow(t, "$[foo]", []token{
+		{Position{1, 1}, tokenDollar, "$"},
+		{Position{1, 2}, tokenLeftBracket, "["},
+		{Position{1, 3}, tokenKey, "foo"},
+		{Position{1, 6}, tokenRightBracket, "]"},
+		{Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexSpace(t *testing.T) {
+	testQLFlow(t, "foo bar baz", []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenKey, "bar"},
+		{Position{1, 9}, tokenKey, "baz"},
+		{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexInteger(t *testing.T) {
+	testQLFlow(t, "100 +200 -300", []token{
+		{Position{1, 1}, tokenInteger, "100"},
+		{Position{1, 5}, tokenInteger, "+200"},
+		{Position{1, 10}, tokenInteger, "-300"},
+		{Position{1, 14}, tokenEOF, ""},
+	})
+}
+
+func TestLexFloat(t *testing.T) {
+	testQLFlow(t, "100.0 +200.0 -300.0", []token{
+		{Position{1, 1}, tokenFloat, "100.0"},
+		{Position{1, 7}, tokenFloat, "+200.0"},
+		{Position{1, 14}, tokenFloat, "-300.0"},
+		{Position{1, 20}, tokenEOF, ""},
+	})
+}
+
+func TestLexFloatWithMultipleDots(t *testing.T) {
+	testQLFlow(t, "4.2.", []token{
+		{Position{1, 1}, tokenError, "cannot have two dots in one float"},
+	})
+}
+
+func TestLexFloatLeadingDot(t *testing.T) {
+	testQLFlow(t, "+.1", []token{
+		{Position{1, 1}, tokenError, "cannot start float with a dot"},
+	})
+}
+
+func TestLexFloatWithTrailingDot(t *testing.T) {
+	testQLFlow(t, "42.", []token{
+		{Position{1, 1}, tokenError, "float cannot end with a dot"},
+	})
+}
+
+func TestLexNumberWithoutDigit(t *testing.T) {
+	testQLFlow(t, "+", []token{
+		{Position{1, 1}, tokenError, "no digit in that number"},
+	})
+}
+
+func TestLexUnknown(t *testing.T) {
+	testQLFlow(t, "^", []token{
+		{Position{1, 1}, tokenError, "unexpected char: '94'"},
+	})
+}

queryparser.go

@@ -0,0 +1,275 @@
+/*
+  Based on the "jsonpath" spec/concept.
+
+  http://goessner.net/articles/JsonPath/
+  https://code.google.com/p/json-path/
+*/
+
+package toml
+
+import (
+	"fmt"
+)
+
+const maxInt = int(^uint(0) >> 1)
+
+type queryParser struct {
+	flow         chan token
+	tokensBuffer []token
+	query        *Query
+	union        []pathFn
+	err          error
+}
+
+type queryParserStateFn func() queryParserStateFn
+
+// Formats and panics an error message based on a token
+func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn {
+	p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...)
+	return nil // trigger parse to end
+}
+
+func (p *queryParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
+	}
+}
+
+func (p *queryParser) backup(tok *token) {
+	p.tokensBuffer = append(p.tokensBuffer, *tok)
+}
+
+func (p *queryParser) peek() *token {
+	if len(p.tokensBuffer) != 0 {
+		return &(p.tokensBuffer[0])
+	}
+
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	p.backup(&tok)
+	return &tok
+}
+
+func (p *queryParser) lookahead(types ...tokenType) bool {
+	result := true
+	buffer := []token{}
+
+	for _, typ := range types {
+		tok := p.getToken()
+		if tok == nil {
+			result = false
+			break
+		}
+		buffer = append(buffer, *tok)
+		if tok.typ != typ {
+			result = false
+			break
+		}
+	}
+	// add the tokens back to the buffer, and return
+	p.tokensBuffer = append(p.tokensBuffer, buffer...)
+	return result
+}
+
+func (p *queryParser) getToken() *token {
+	if len(p.tokensBuffer) != 0 {
+		tok := p.tokensBuffer[0]
+		p.tokensBuffer = p.tokensBuffer[1:]
+		return &tok
+	}
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	return &tok
+}
+
+func (p *queryParser) parseStart() queryParserStateFn {
+	tok := p.getToken()
+
+	if tok == nil || tok.typ == tokenEOF {
+		return nil
+	}
+
+	if tok.typ != tokenDollar {
+		return p.parseError(tok, "Expected '$' at start of expression")
+	}
+
+	return p.parseMatchExpr
+}
+
+// handle '.' prefix, '[]', and '..'
+func (p *queryParser) parseMatchExpr() queryParserStateFn {
+	tok := p.getToken()
+	switch tok.typ {
+	case tokenDotDot:
+		p.query.appendPath(&matchRecursiveFn{})
+		// nested parse for '..'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenLeftBracket:
+			return p.parseBracketExpr
+		case tokenStar:
+			// do nothing - the recursive predicate is enough
+			return p.parseMatchExpr
+		}
+
+	case tokenDot:
+		// nested parse for '.'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenStar:
+			p.query.appendPath(&matchAnyFn{})
+			return p.parseMatchExpr
+		}
+
+	case tokenLeftBracket:
+		return p.parseBracketExpr
+
+	case tokenEOF:
+		return nil // allow EOF at this stage
+	}
+	return p.parseError(tok, "expected match expression")
+}
+
+func (p *queryParser) parseBracketExpr() queryParserStateFn {
+	if p.lookahead(tokenInteger, tokenColon) {
+		return p.parseSliceExpr
+	}
+	if p.peek().typ == tokenColon {
+		return p.parseSliceExpr
+	}
+	return p.parseUnionExpr
+}
+
+func (p *queryParser) parseUnionExpr() queryParserStateFn {
+	var tok *token
+
+	// this state can be traversed after some sub-expressions
+	// so be careful when setting up state in the parser
+	if p.union == nil {
+		p.union = []pathFn{}
+	}
+
+loop: // labeled loop for easy breaking
+	for {
+		if len(p.union) > 0 {
+			// parse delimiter or terminator
+			tok = p.getToken()
+			switch tok.typ {
+			case tokenComma:
+				// do nothing
+			case tokenRightBracket:
+				break loop
+			default:
+				return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val)
+			}
+		}
+
+		// parse sub expression
+		tok = p.getToken()
+		switch tok.typ {
+		case tokenInteger:
+			p.union = append(p.union, newMatchIndexFn(tok.Int()))
+		case tokenKey:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenString:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenQuestion:
+			return p.parseFilterExpr
+		default:
+			return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union))
+		}
+	}
+
+	// if there is only one sub-expression, use that instead
+	if len(p.union) == 1 {
+		p.query.appendPath(p.union[0])
+	} else {
+		p.query.appendPath(&matchUnionFn{p.union})
+	}
+
+	p.union = nil // clear out state
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseSliceExpr() queryParserStateFn {
+	// init slice to grab all elements
+	start, end, step := 0, maxInt, 1
+
+	// parse optional start
+	tok := p.getToken()
+	if tok.typ == tokenInteger {
+		start = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ':'")
+	}
+
+	// parse optional end
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		end = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ == tokenRightBracket {
+		p.query.appendPath(newMatchSliceFn(start, end, step))
+		return p.parseMatchExpr
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ']' or ':'")
+	}
+
+	// parse optional step
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		step = tok.Int()
+		if step < 0 {
+			return p.parseError(tok, "step must be a positive value")
+		}
+		tok = p.getToken()
+	}
+	if tok.typ != tokenRightBracket {
+		return p.parseError(tok, "expected ']'")
+	}
+
+	p.query.appendPath(newMatchSliceFn(start, end, step))
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseFilterExpr() queryParserStateFn {
+	tok := p.getToken()
+	if tok.typ != tokenLeftParen {
+		return p.parseError(tok, "expected left-parenthesis for filter expression")
+	}
+	tok = p.getToken()
+	if tok.typ != tokenKey && tok.typ != tokenString {
+		return p.parseError(tok, "expected key or string for filter funciton name")
+	}
+	name := tok.val
+	tok = p.getToken()
+	if tok.typ != tokenRightParen {
+		return p.parseError(tok, "expected right-parenthesis for filter expression")
+	}
+	p.union = append(p.union, newMatchFilterFn(name, tok.Position))
+	return p.parseUnionExpr
+}
+
+func parseQuery(flow chan token) (*Query, error) {
+	parser := &queryParser{
+		flow:         flow,
+		tokensBuffer: []token{},
+		query:        newQuery(),
+	}
+	parser.run()
+	return parser.query, parser.err
+}

queryparser_test.go

@@ -0,0 +1,483 @@
+package toml
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+type queryTestNode struct {
+	value    interface{}
+	position Position
+}
+
+func valueString(root interface{}) string {
+	result := "" //fmt.Sprintf("%T:", root)
+	switch node := root.(type) {
+	case *tomlValue:
+		return valueString(node.value)
+	case *QueryResult:
+		items := []string{}
+		for i, v := range node.Values() {
+			items = append(items, fmt.Sprintf("%s:%s",
+				node.Positions()[i].String(), valueString(v)))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case queryTestNode:
+		result = fmt.Sprintf("%s:%s",
+			node.position.String(), valueString(node.value))
+	case []interface{}:
+		items := []string{}
+		for _, v := range node {
+			items = append(items, valueString(v))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case *TomlTree:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for _, k := range node.Keys() {
+			v := node.GetPath([]string{k})
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case map[string]interface{}:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for k, v := range node {
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case int64:
+		result += fmt.Sprintf("%d", node)
+	case string:
+		result += "'" + node + "'"
+	case float64:
+		result += fmt.Sprintf("%f", node)
+	case bool:
+		result += fmt.Sprintf("%t", node)
+	case time.Time:
+		result += fmt.Sprintf("'%v'", node)
+	}
+	return result
+}
+
+func assertValue(t *testing.T, result, ref interface{}) {
+	pathStr := valueString(result)
+	refStr := valueString(ref)
+	if pathStr != refStr {
+		t.Errorf("values do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+	}
+}
+
+func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
+	tree, err := Load(toml)
+	if err != nil {
+		t.Errorf("Non-nil toml parse error: %v", err)
+		return
+	}
+	q, err := CompileQuery(query)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	results := q.Execute(tree)
+	assertValue(t, results, ref)
+}
+
+func TestQueryRoot(t *testing.T) {
+	assertQueryPositions(t,
+		"a = 42",
+		"$",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(42),
+				}, Position{1, 1},
+			},
+		})
+}
+
+func TestQueryKey(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo.a",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryKeyString(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo['a']",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryIndex(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[5]",
+		[]interface{}{
+			queryTestNode{
+				int64(6), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceRange(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(4), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceStep(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5:2]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryAny(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[foo.baz]\na=3\nb=4",
+		"$.foo.*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+		})
+}
+func TestQueryUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$.*[bar,foo]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryRecursionAll(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": map[string]interface{}{
+							"a": int64(1),
+							"b": int64(2),
+						},
+					},
+					"baz": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(3),
+							"b": int64(4),
+						},
+					},
+					"gorf": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(5),
+							"b": int64(6),
+						},
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{3, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(3),
+						"b": int64(4),
+					},
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				int64(3), Position{5, 1},
+			},
+			queryTestNode{
+				int64(4), Position{6, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(5),
+						"b": int64(6),
+					},
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				int64(5), Position{8, 1},
+			},
+			queryTestNode{
+				int64(6), Position{9, 1},
+			},
+		})
+}
+
+func TestQueryRecursionUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..['foo','bar']",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryFilterFn(t *testing.T) {
+	buff, err := ioutil.ReadFile("example.toml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(int)]",
+		[]interface{}{
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8002), Position{13, 1},
+			},
+			queryTestNode{
+				int64(5000), Position{14, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(string)]",
+		[]interface{}{
+			queryTestNode{
+				"TOML Example", Position{3, 1},
+			},
+			queryTestNode{
+				"Tom Preston-Werner", Position{6, 1},
+			},
+			queryTestNode{
+				"GitHub", Position{7, 1},
+			},
+			queryTestNode{
+				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
+				Position{8, 1},
+			},
+			queryTestNode{
+				"192.168.1.1", Position{12, 1},
+			},
+			queryTestNode{
+				"10.0.0.1", Position{21, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{22, 3},
+			},
+			queryTestNode{
+				"10.0.0.2", Position{25, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{26, 3},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(float)]",
+		[]interface{}{
+		// no float values in document
+		})
+
+	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
+	assertQueryPositions(t, string(buff),
+		"$..[?(tree)]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"name":         "Tom Preston-Werner",
+					"organization": "GitHub",
+					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+					"dob":          tv,
+				}, Position{5, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"server":         "192.168.1.1",
+					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+					"connection_max": int64(5000),
+					"enabled":        true,
+				}, Position{11, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"alpha": map[string]interface{}{
+						"ip": "10.0.0.1",
+						"dc": "eqdc10",
+					},
+					"beta": map[string]interface{}{
+						"ip": "10.0.0.2",
+						"dc": "eqdc10",
+					},
+				}, Position{17, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.1",
+					"dc": "eqdc10",
+				}, Position{20, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.2",
+					"dc": "eqdc10",
+				}, Position{24, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"data": []interface{}{
+						[]interface{}{"gamma", "delta"},
+						[]interface{}{int64(1), int64(2)},
+					},
+				}, Position{28, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(time)]",
+		[]interface{}{
+			queryTestNode{
+				tv, Position{9, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(bool)]",
+		[]interface{}{
+			queryTestNode{
+				true, Position{15, 1},
+			},
+		})
+}

test.sh

@@ -5,24 +5,75 @@ set -e
 # set the path to the present working directory
 export GOPATH=`pwd`
 
-# Vendorize the BurntSushi test suite
-# NOTE: this gets a specific release to avoid versioning issues
-if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then
-  mkdir -p src/github.com/BurntSushi
-  git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test
-fi
-pushd src/github.com/BurntSushi/toml-test
-git reset --hard '0.2.0'  # use the released version, NOT tip
-popd
+function git_clone() {
+  path=$1
+  branch=$2
+  version=$3
+  if [ ! -d "src/$path" ]; then
+    mkdir -p src/$path
+    git clone https://$path.git src/$path
+  fi
+  pushd src/$path
+  git checkout "$branch"
+  git reset --hard "$version"
+  popd
+}
+
+go get github.com/pelletier/go-buffruneio
+go get github.com/davecgh/go-spew/spew
+
+# get code for BurntSushi TOML validation
+# pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)
+git_clone github.com/BurntSushi/toml master HEAD
+git_clone github.com/BurntSushi/toml-test master HEAD #was: 0.2.0 HEAD
+
+# build the BurntSushi test application
 go build -o toml-test github.com/BurntSushi/toml-test
 
 # vendorize the current lib for testing
 # NOTE: this basically mocks an install without having to go back out to github for code
 mkdir -p src/github.com/pelletier/go-toml/cmd
 cp *.go *.toml src/github.com/pelletier/go-toml
-cp cmd/*.go src/github.com/pelletier/go-toml/cmd
+cp -R cmd/* src/github.com/pelletier/go-toml/cmd
 go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
 
-# Run basic unit tests and then the BurntSushi test suite
-go test -v github.com/pelletier/go-toml
-./toml-test ./test_program_bin | tee test_out
+# Run basic unit tests
+go test github.com/pelletier/go-toml \
+        github.com/pelletier/go-toml/cmd/tomljson
+
+# run the entire BurntSushi test suite
+if [[ $# -eq 0 ]] ; then
+  echo "Running all BurntSushi tests"
+  ./toml-test ./test_program_bin | tee test_out
+else
+  # run a specific test
+  test=$1
+  test_path='src/github.com/BurntSushi/toml-test/tests'
+  valid_test="$test_path/valid/$test"
+  invalid_test="$test_path/invalid/$test"
+
+  if [ -e "$valid_test.toml" ]; then
+    echo "Valid Test TOML for $test:"
+    echo "===="
+    cat "$valid_test.toml"
+
+    echo "Valid Test JSON for $test:"
+    echo "===="
+    cat "$valid_test.json"
+
+    echo "Go-TOML Output for $test:"
+    echo "===="
+    cat "$valid_test.toml" | ./test_program_bin
+  fi
+
+  if [ -e "$invalid_test.toml" ]; then
+    echo "Invalid Test TOML for $test:"
+    echo "===="
+    cat "$invalid_test.toml"
+
+    echo "Go-TOML Output for $test:"
+    echo "===="
+    echo "go-toml Output:"
+    cat "$invalid_test.toml" | ./test_program_bin
+  fi
+fi

token.go

@@ -0,0 +1,139 @@
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenComment
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenTrue
+	tokenFalse
+	tokenFloat
+	tokenEqual
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftCurlyBrace
+	tokenRightCurlyBrace
+	tokenLeftParen
+	tokenRightParen
+	tokenDoubleLeftBracket
+	tokenDoubleRightBracket
+	tokenDate
+	tokenKeyGroup
+	tokenKeyGroupArray
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+	tokenEOL
+)
+
+var tokenTypeNames = []string{
+	"Error",
+	"EOF",
+	"Comment",
+	"Key",
+	"String",
+	"Integer",
+	"True",
+	"False",
+	"Float",
+	"=",
+	"[",
+	"]",
+	"{",
+	"}",
+	"(",
+	")",
+	"]]",
+	"[[",
+	"Date",
+	"KeyGroup",
+	"KeyGroupArray",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+	"EOL",
+}
+
+type token struct {
+	Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isKeyChar(r rune) bool {
+	// Keys start with the first character that isn't whitespace or [ and end
+	// with the last non-whitespace character before the equals sign. Keys
+	// cannot contain a # character."
+	return !(r == '\r' || r == '\n' || r == eof || r == '=')
+}
+
+func isKeyStartChar(r rune) bool {
+	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+}

token_test.go

@@ -0,0 +1,67 @@
+package toml
+
+import "testing"
+
+func TestTokenStringer(t *testing.T) {
+	var tests = []struct {
+		tt     tokenType
+		expect string
+	}{
+		{tokenError, "Error"},
+		{tokenEOF, "EOF"},
+		{tokenComment, "Comment"},
+		{tokenKey, "Key"},
+		{tokenString, "String"},
+		{tokenInteger, "Integer"},
+		{tokenTrue, "True"},
+		{tokenFalse, "False"},
+		{tokenFloat, "Float"},
+		{tokenEqual, "="},
+		{tokenLeftBracket, "["},
+		{tokenRightBracket, "]"},
+		{tokenLeftCurlyBrace, "{"},
+		{tokenRightCurlyBrace, "}"},
+		{tokenLeftParen, "("},
+		{tokenRightParen, ")"},
+		{tokenDoubleLeftBracket, "]]"},
+		{tokenDoubleRightBracket, "[["},
+		{tokenDate, "Date"},
+		{tokenKeyGroup, "KeyGroup"},
+		{tokenKeyGroupArray, "KeyGroupArray"},
+		{tokenComma, ","},
+		{tokenColon, ":"},
+		{tokenDollar, "$"},
+		{tokenStar, "*"},
+		{tokenQuestion, "?"},
+		{tokenDot, "."},
+		{tokenDotDot, ".."},
+		{tokenEOL, "EOL"},
+		{tokenEOL + 1, "Unknown"},
+	}
+
+	for i, test := range tests {
+		got := test.tt.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid string of token type; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}
+
+func TestTokenString(t *testing.T) {
+	var tests = []struct {
+		tok    token
+		expect string
+	}{
+		{token{Position{1, 1}, tokenEOF, ""}, "EOF"},
+		{token{Position{1, 1}, tokenError, "Δt"}, "Δt"},
+		{token{Position{1, 1}, tokenString, "bar"}, `"bar"`},
+		{token{Position{1, 1}, tokenString, "123456789012345"}, `"123456789012345"`},
+	}
+
+	for i, test := range tests {
+		got := test.tok.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid of string token; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}

toml.go

@@ -1,41 +1,57 @@
-// TOML markup language parser.
-//
-// This version supports the specification as described in
-// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
 package toml
 
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"os"
 	"runtime"
-	"strconv"
 	"strings"
-	"time"
 )
 
-// Definition of a TomlTree.
-// This is the result of the parsing of a TOML file.
-type TomlTree map[string]interface{}
+type tomlValue struct {
+	value    interface{}
+	position Position
+}
 
-// Has returns a boolean indicating if the toplevel tree contains the given
-// key.
-func (t *TomlTree) Has(key string) bool {
-	mp := (map[string]interface{})(*t)
-	for k, _ := range mp {
-		if k == key {
-			return true
-		}
+// TomlTree is the result of the parsing of a TOML file.
+type TomlTree struct {
+	values   map[string]interface{}
+	position Position
+}
+
+func newTomlTree() *TomlTree {
+	return &TomlTree{
+		values:   make(map[string]interface{}),
+		position: Position{},
 	}
-	return false
+}
+
+// TreeFromMap initializes a new TomlTree object using the given map.
+func TreeFromMap(m map[string]interface{}) *TomlTree {
+	return &TomlTree{
+		values: m,
+	}
+}
+
+// Has returns a boolean indicating if the given key exists.
+func (t *TomlTree) Has(key string) bool {
+	if key == "" {
+		return false
+	}
+	return t.HasPath(strings.Split(key, "."))
+}
+
+// HasPath returns true if the given path of keys exists, false otherwise.
+func (t *TomlTree) HasPath(keys []string) bool {
+	return t.GetPath(keys) != nil
 }
 
 // Keys returns the keys of the toplevel tree.
 // Warning: this is a costly operation.
 func (t *TomlTree) Keys() []string {
-	keys := make([]string, 0)
-	mp := (map[string]interface{})(*t)
-	for k, _ := range mp {
+	var keys []string
+	for k := range t.values {
 		keys = append(keys, k)
 	}
 	return keys
@@ -49,36 +65,98 @@ func (t *TomlTree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
-	return t.GetPath(strings.Split(key, "."))
+	comps, err := parseKey(key)
+	if err != nil {
+		return nil
+	}
+	return t.GetPath(comps)
 }
 
-// Returns the element in the tree indicated by 'keys'.
+// GetPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
 func (t *TomlTree) GetPath(keys []string) interface{} {
 	if len(keys) == 0 {
 		return t
 	}
 	subtree := t
-	for _, intermediate_key := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		value, exists := subtree.values[intermediateKey]
 		if !exists {
 			return nil
 		}
-		switch node := (*subtree)[intermediate_key].(type) {
+		switch node := value.(type) {
 		case *TomlTree:
 			subtree = node
 		case []*TomlTree:
 			// go to most recent element
 			if len(node) == 0 {
-				return nil //(*subtree)[intermediate_key] = append(node, &TomlTree{})
+				return nil
 			}
 			subtree = node[len(node)-1]
+		default:
+			return nil // cannot navigate through other node types
 		}
 	}
-	return (*subtree)[keys[len(keys)-1]]
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.value
+	default:
+		return node
+	}
 }
 
-// Same as Get but with a default value
+// GetPosition returns the position of the given key.
+func (t *TomlTree) GetPosition(key string) Position {
+	if key == "" {
+		return t.position
+	}
+	return t.GetPositionPath(strings.Split(key, "."))
+}
+
+// GetPositionPath returns the element in the tree indicated by 'keys'.
+// If keys is of length zero, the current tree is returned.
+func (t *TomlTree) GetPositionPath(keys []string) Position {
+	if len(keys) == 0 {
+		return t.position
+	}
+	subtree := t
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		value, exists := subtree.values[intermediateKey]
+		if !exists {
+			return Position{0, 0}
+		}
+		switch node := value.(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				return Position{0, 0}
+			}
+			subtree = node[len(node)-1]
+		default:
+			return Position{0, 0}
+		}
+	}
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.position
+	case *TomlTree:
+		return node.position
+	case []*TomlTree:
+		// go to most recent element
+		if len(node) == 0 {
+			return Position{0, 0}
+		}
+		return node[len(node)-1].position
+	default:
+		return Position{0, 0}
+	}
+}
+
+// GetDefault works like Get but with a default value
 func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
 	val := t.Get(key)
 	if val == nil {
@@ -94,26 +172,44 @@ func (t *TomlTree) Set(key string, value interface{}) {
 	t.SetPath(strings.Split(key, "."), value)
 }
 
+// SetPath sets an element in the tree.
+// Keys is an array of path elements (e.g. {"a","b","c"}).
+// Creates all necessary intermediates trees, if needed.
 func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	subtree := t
-	for _, intermediate_key := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			var new_tree TomlTree = make(TomlTree)
-			(*subtree)[intermediate_key] = &new_tree
+			nextTree = newTomlTree()
+			subtree.values[intermediateKey] = nextTree // add new element here
 		}
-		switch node := (*subtree)[intermediate_key].(type) {
+		switch node := nextTree.(type) {
 		case *TomlTree:
 			subtree = node
 		case []*TomlTree:
 			// go to most recent element
 			if len(node) == 0 {
-				(*subtree)[intermediate_key] = append(node, &TomlTree{})
+				// create element if it does not exist
+				subtree.values[intermediateKey] = append(node, newTomlTree())
 			}
 			subtree = node[len(node)-1]
 		}
 	}
-	(*subtree)[keys[len(keys)-1]] = value
+
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *TomlTree:
+		toInsert = value
+	case []*TomlTree:
+		toInsert = value
+	case *tomlValue:
+		toInsert = value
+	default:
+		toInsert = &tomlValue{value: value}
+	}
+
+	subtree.values[keys[len(keys)-1]] = toInsert
 }
 
 // createSubTree takes a tree and a key and create the necessary intermediate
@@ -121,121 +217,43 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 //
 // e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
 // and tree[a][b][c]
-func (t *TomlTree) createSubTree(key string) {
+//
+// Returns nil on success, error object on failure
+func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	subtree := t
-	for _, intermediate_key := range strings.Split(key, ".") {
-		if intermediate_key == "" {
-			panic("empty intermediate table")
-		}
-		_, exists := (*subtree)[intermediate_key]
+	for _, intermediateKey := range keys {
+		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			var new_tree TomlTree = make(TomlTree)
-			(*subtree)[intermediate_key] = &new_tree
+			tree := newTomlTree()
+			tree.position = pos
+			subtree.values[intermediateKey] = tree
+			nextTree = tree
 		}
-		subtree = ((*subtree)[intermediate_key]).(*TomlTree)
-	}
-}
 
-// encodes a string to a TOML-compliant string value
-func encodeTomlString(value string) string {
-	result := ""
-	for _, rr := range value {
-		int_rr := uint16(rr)
-		switch rr {
-		case '\b':
-			result += "\\b"
-		case '\t':
-			result += "\\t"
-		case '\n':
-			result += "\\n"
-		case '\f':
-			result += "\\f"
-		case '\r':
-			result += "\\r"
-		case '"':
-			result += "\\\""
-		case '\\':
-			result += "\\\\"
-		default:
-			if int_rr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", int_rr)
-			} else {
-				result += string(rr)
-			}
-		}
-	}
-	return result
-}
-
-// Value print support function for ToString()
-// Outputs the TOML compliant string representation of a value
-func toTomlValue(item interface{}, indent int) string {
-	tab := strings.Repeat(" ", indent)
-	switch value := item.(type) {
-	case int64:
-		return tab + strconv.FormatInt(value, 10)
-	case float64:
-		return tab + strconv.FormatFloat(value, 'f', -1, 64)
-	case string:
-		return tab + "\"" + encodeTomlString(value) + "\""
-	case bool:
-		if value {
-			return "true"
-		} else {
-			return "false"
-		}
-	case time.Time:
-		return tab + value.Format(time.RFC3339)
-	case []interface{}:
-		result := tab + "[\n"
-		for _, item := range value {
-			result += toTomlValue(item, indent+2) + ",\n"
-		}
-		return result + tab + "]"
-	default:
-		panic(fmt.Sprintf("unsupported value type: %v", value))
-	}
-}
-
-// Recursive support function for ToString()
-// Outputs a tree, using the provided keyspace to prefix group names
-func (t *TomlTree) toToml(keyspace string) string {
-	result := ""
-	for k, v := range (map[string]interface{})(*t) {
-		// figure out the keyspace
-		combined_key := k
-		if keyspace != "" {
-			combined_key = keyspace + "." + combined_key
-		}
-		// output based on type
-		switch node := v.(type) {
+		switch node := nextTree.(type) {
 		case []*TomlTree:
-			for _, item := range node {
-				if len(item.Keys()) > 0 {
-					result += fmt.Sprintf("\n[[%s]]\n", combined_key)
-				}
-				result += item.toToml(combined_key)
-			}
+			subtree = node[len(node)-1]
 		case *TomlTree:
-			if len(node.Keys()) > 0 {
-				result += fmt.Sprintf("\n[%s]\n", combined_key)
-			}
-			result += node.toToml(combined_key)
+			subtree = node
 		default:
-			result += fmt.Sprintf("%s = %s\n", k, toTomlValue(node, 0))
+			return fmt.Errorf("unknown type for path %s (%s): %T (%#v)",
+				strings.Join(keys, "."), intermediateKey, nextTree, nextTree)
 		}
 	}
-	return result
+	return nil
 }
 
-// Generates a human-readable representation of the current tree.
-// Output spans multiple lines, and is suitable for ingest by a TOML parser
-func (t *TomlTree) ToString() string {
-	return t.toToml("")
+// Query compiles and executes a query on a tree and returns the query result.
+func (t *TomlTree) Query(query string) (*QueryResult, error) {
+	q, err := CompileQuery(query)
+	if err != nil {
+		return nil, err
+	}
+	return q.Execute(t), nil
 }
 
-// Create a TomlTree from a string.
-func Load(content string) (tree *TomlTree, err error) {
+// LoadReader creates a TomlTree from any io.Reader.
+func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -244,19 +262,21 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	_, flow := lex(content)
-	tree = parse(flow)
+	tree = parseToml(lexToml(reader))
 	return
 }
 
-// Create a TomlTree from a file.
-func LoadFile(path string) (tree *TomlTree, err error) {
-	buff, ferr := ioutil.ReadFile(path)
-	if ferr != nil {
-		err = ferr
-	} else {
-		s := string(buff)
-		tree, err = Load(s)
-	}
-	return
+// Load creates a TomlTree from a string.
+func Load(content string) (tree *TomlTree, err error) {
+	return LoadReader(strings.NewReader(content))
+}
+
+// LoadFile creates a TomlTree from a file.
+func LoadFile(path string) (tree *TomlTree, err error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	return LoadReader(file)
 }

toml_test.go

@@ -1,20 +1,85 @@
+// Testing support for go-toml
+
 package toml
 
 import (
 	"testing"
 )
 
+func TestTomlHas(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.Has("test.key") {
+		t.Errorf("Has - expected test.key to exists")
+	}
+
+	if tree.Has("") {
+		t.Errorf("Should return false if the key is not provided")
+	}
+}
+
+func TestTomlGet(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.Get("") != tree {
+		t.Errorf("Get should return the tree itself when given an empty path")
+	}
+
+	if tree.Get("test.key") != "value" {
+		t.Errorf("Get should return the value")
+	}
+	if tree.Get(`\`) != nil {
+		t.Errorf("should return nil when the key is malformed")
+	}
+}
+
+func TestTomlGetDefault(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.GetDefault("", "hello") != tree {
+		t.Error("GetDefault should return the tree itself when given an empty path")
+	}
+
+	if tree.GetDefault("test.key", "hello") != "value" {
+		t.Error("Get should return the value")
+	}
+
+	if tree.GetDefault("whatever", "hello") != "hello" {
+		t.Error("GetDefault should return the default value if the key does not exist")
+	}
+}
+
+func TestTomlHasPath(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.HasPath([]string{"test", "key"}) {
+		t.Errorf("HasPath - expected test.key to exists")
+	}
+}
+
 func TestTomlGetPath(t *testing.T) {
-	node := make(TomlTree)
+	node := newTomlTree()
 	//TODO: set other node data
 
 	for idx, item := range []struct {
 		Path     []string
-		Expected interface{}
+		Expected *TomlTree
 	}{
 		{ // empty path test
 			[]string{},
-			&node,
+			node,
 		},
 	} {
 		result := node.GetPath(item.Path)
@@ -22,4 +87,42 @@ func TestTomlGetPath(t *testing.T) {
 			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
 		}
 	}
+
+	tree, _ := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if tree.GetPath([]string{"whatever"}) != nil {
+		t.Error("GetPath should return nil when the key does not exist")
+	}
+}
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result, err := tree.Query("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*TomlTree); !ok {
+		t.Errorf("Expected type of TomlTree: %T", values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}
+
+func TestTomlFromMap(t *testing.T) {
+	simpleMap := map[string]interface{}{"hello": 42}
+	tree := TreeFromMap(simpleMap)
+	if tree.Get("hello") != 42 {
+		t.Fatal("hello should be 42, not", tree.Get("hello"))
+	}
 }

tomltree_conversions.go

@@ -0,0 +1,207 @@
+package toml
+
+// Tools to convert a TomlTree to different representations
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	result := ""
+	for _, rr := range value {
+		intRr := uint16(rr)
+		switch rr {
+		case '\b':
+			result += "\\b"
+		case '\t':
+			result += "\\t"
+		case '\n':
+			result += "\\n"
+		case '\f':
+			result += "\\f"
+		case '\r':
+			result += "\\r"
+		case '"':
+			result += "\\\""
+		case '\\':
+			result += "\\\\"
+		default:
+			if intRr < 0x001F {
+				result += fmt.Sprintf("\\u%0.4X", intRr)
+			} else {
+				result += string(rr)
+			}
+		}
+	}
+	return result
+}
+
+// Value print support function for ToString()
+// Outputs the TOML compliant string representation of a value
+func toTomlValue(item interface{}, indent int) string {
+	tab := strings.Repeat(" ", indent)
+	switch value := item.(type) {
+	case int:
+		return tab + strconv.FormatInt(int64(value), 10)
+	case int8:
+		return tab + strconv.FormatInt(int64(value), 10)
+	case int16:
+		return tab + strconv.FormatInt(int64(value), 10)
+	case int32:
+		return tab + strconv.FormatInt(int64(value), 10)
+	case int64:
+		return tab + strconv.FormatInt(value, 10)
+	case uint:
+		return tab + strconv.FormatUint(uint64(value), 10)
+	case uint8:
+		return tab + strconv.FormatUint(uint64(value), 10)
+	case uint16:
+		return tab + strconv.FormatUint(uint64(value), 10)
+	case uint32:
+		return tab + strconv.FormatUint(uint64(value), 10)
+	case uint64:
+		return tab + strconv.FormatUint(value, 10)
+	case float32:
+		return tab + strconv.FormatFloat(float64(value), 'f', -1, 32)
+	case float64:
+		return tab + strconv.FormatFloat(value, 'f', -1, 64)
+	case string:
+		return tab + "\"" + encodeTomlString(value) + "\""
+	case bool:
+		if value {
+			return "true"
+		}
+		return "false"
+	case time.Time:
+		return tab + value.Format(time.RFC3339)
+	case []interface{}:
+		result := tab + "[\n"
+		for _, item := range value {
+			result += toTomlValue(item, indent+2) + ",\n"
+		}
+		return result + tab + "]"
+	case nil:
+		return ""
+	default:
+		panic(fmt.Sprintf("unsupported value type %T: %v", value, value))
+	}
+}
+
+// Recursive support function for ToString()
+// Outputs a tree, using the provided keyspace to prefix group names
+func (t *TomlTree) toToml(indent, keyspace string) string {
+	resultChunks := []string{}
+	for k, v := range t.values {
+		// figure out the keyspace
+		combinedKey := k
+		if keyspace != "" {
+			combinedKey = keyspace + "." + combinedKey
+		}
+		resultChunk := ""
+		// output based on type
+		switch node := v.(type) {
+		case []*TomlTree:
+			for _, item := range node {
+				if len(item.Keys()) > 0 {
+					resultChunk += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
+				}
+				resultChunk += item.toToml(indent+"  ", combinedKey)
+			}
+			resultChunks = append(resultChunks, resultChunk)
+		case *TomlTree:
+			if len(node.Keys()) > 0 {
+				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			resultChunk += node.toToml(indent+"  ", combinedKey)
+			resultChunks = append(resultChunks, resultChunk)
+		case map[string]interface{}:
+			sub := TreeFromMap(node)
+
+			if len(sub.Keys()) > 0 {
+				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			resultChunk += sub.toToml(indent+"  ", combinedKey)
+			resultChunks = append(resultChunks, resultChunk)
+		case map[string]string:
+			sub := TreeFromMap(convertMapStringString(node))
+
+			if len(sub.Keys()) > 0 {
+				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			resultChunk += sub.toToml(indent+"  ", combinedKey)
+			resultChunks = append(resultChunks, resultChunk)
+		case map[interface{}]interface{}:
+			sub := TreeFromMap(convertMapInterfaceInterface(node))
+
+			if len(sub.Keys()) > 0 {
+				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			resultChunk += sub.toToml(indent+"  ", combinedKey)
+			resultChunks = append(resultChunks, resultChunk)
+		case *tomlValue:
+			resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
+			resultChunks = append([]string{resultChunk}, resultChunks...)
+		default:
+			resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
+			resultChunks = append([]string{resultChunk}, resultChunks...)
+		}
+
+	}
+	return strings.Join(resultChunks, "")
+}
+
+func convertMapStringString(in map[string]string) map[string]interface{} {
+	result := make(map[string]interface{}, len(in))
+	for k, v := range in {
+		result[k] = v
+	}
+	return result
+}
+
+func convertMapInterfaceInterface(in map[interface{}]interface{}) map[string]interface{} {
+	result := make(map[string]interface{}, len(in))
+	for k, v := range in {
+		result[k.(string)] = v
+	}
+	return result
+}
+
+// ToString is an alias for String
+func (t *TomlTree) ToString() string {
+	return t.String()
+}
+
+// String generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser
+func (t *TomlTree) String() string {
+	return t.toToml("", "")
+}
+
+// ToMap recursively generates a representation of the current tree using map[string]interface{}.
+func (t *TomlTree) ToMap() map[string]interface{} {
+	result := map[string]interface{}{}
+
+	for k, v := range t.values {
+		switch node := v.(type) {
+		case []*TomlTree:
+			var array []interface{}
+			for _, item := range node {
+				array = append(array, item.ToMap())
+			}
+			result[k] = array
+		case *TomlTree:
+			result[k] = node.ToMap()
+		case map[string]interface{}:
+			sub := TreeFromMap(node)
+			result[k] = sub.ToMap()
+		case *tomlValue:
+			result[k] = node.value
+		}
+	}
+
+	return result
+}

tomltree_conversions_test.go

@@ -0,0 +1,171 @@
+package toml
+
+import (
+	"reflect"
+	"testing"
+	"time"
+	"strings"
+)
+
+func TestTomlTreeConversionToString(t *testing.T) {
+	toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+points = { x = 1, y = 2 }`)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	reparsedTree, err := Load(toml.ToString())
+
+	assertTree(t, reparsedTree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"points": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func TestTomlTreeConversionToStringKeysOrders(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		tree, _ := Load(`
+		foobar = true
+		bar = "baz"
+		foo = 1
+		[qux]
+		  foo = 1
+		  bar = "baz2"`)
+
+		stringRepr := tree.ToString()
+
+		t.Log("Intermediate string representation:")
+		t.Log(stringRepr)
+
+		r := strings.NewReader(stringRepr)
+		toml, err := LoadReader(r)
+
+
+		if err != nil {
+			t.Fatal("Unexpected error:", err)
+		}
+
+		assertTree(t, toml, err, map[string]interface{}{
+			"foobar": true,
+			"bar": "baz",
+			"foo": 1,
+			"qux": map[string]interface{}{
+				"foo": 1,
+				"bar": "baz2",
+			},
+		})
+	}
+}
+
+func testMaps(t *testing.T, actual, expected map[string]interface{}) {
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatal("trees aren't equal.\n", "Expected:\n", expected, "\nActual:\n", actual)
+	}
+}
+
+func TestTomlTreeConversionToMapSimple(t *testing.T) {
+	tree, _ := Load("a = 42\nb = 17")
+
+	expected := map[string]interface{}{
+		"a": int64(42),
+		"b": int64(17),
+	}
+
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTomlTreeConversionToMapExampleFile(t *testing.T) {
+	tree, _ := LoadFile("example.toml")
+	expected := map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+			"connection_max": int64(5000),
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]interface{}{"gamma", "delta"},
+				[]interface{}{int64(1), int64(2)},
+			},
+		},
+	}
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) {
+	tree, _ := Load(`
+	[[menu.main]]
+        a = "menu 1"
+        b = "menu 2"
+        [[menu.main]]
+        c = "menu 3"
+        d = "menu 4"`)
+	expected := map[string]interface{}{
+		"menu": map[string]interface{}{
+			"main": []interface{}{
+				map[string]interface{}{"a": "menu 1", "b": "menu 2"},
+				map[string]interface{}{"c": "menu 3", "d": "menu 4"},
+			},
+		},
+	}
+	treeMap := tree.ToMap()
+
+	testMaps(t, treeMap, expected)
+}
+
+func TestTomlTreeConversionToMapWithArrayOfInlineTables(t *testing.T) {
+	tree, _ := Load(`
+    	[params]
+	language_tabs = [
+    		{ key = "shell", name = "Shell" },
+    		{ key = "ruby", name = "Ruby" },
+    		{ key = "python", name = "Python" }
+	]`)
+
+	expected := map[string]interface{}{
+		"params": map[string]interface{}{
+			"language_tabs": []interface{}{
+				map[string]interface{}{
+					"key": "shell",
+					"name": "Shell",
+				},
+				map[string]interface{}{
+					"key": "ruby",
+					"name": "Ruby",
+				},
+				map[string]interface{}{
+					"key": "python",
+					"name": "Python",
+				},
+			},
+		},
+	}
+
+	treeMap := tree.ToMap()
+	testMaps(t, treeMap, expected)
+}