Merge pull request #51 from pelletier/pelletier/fix-crlf-support

Fix support for CRLF line ending

.travis.yml

@@ -1,6 +1,13 @@
 language: go
 script: "./test.sh"
 go:
-    - 1.1
+    - 1.3.3
-    - 1.2
+    - 1.4.2
+    - 1.5.3
     - tip
+before_install:
+  - go get github.com/axw/gocov/gocov
+  - go get github.com/mattn/goveralls
+  - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+after_success:
+    - $HOME/gopath/bin/goveralls -service=travis-ci

README.md

@@ -3,9 +3,28 @@
 Go library for the [TOML](https://github.com/mojombo/toml) format.
 
 This library supports TOML version
-[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md)
+[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
 
-[![Build Status](https://travis-ci.org/pelletier/go-toml.png?branch=master)](https://travis-ci.org/pelletier/go-toml)
+[![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
+[![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
+[![Coverage Status](https://coveralls.io/repos/pelletier/go-toml/badge.svg?branch=master&service=github)](https://coveralls.io/github/pelletier/go-toml?branch=master)
+
+## Features
+
+Go-toml provides the following features for using data parsed from TOML documents:
+
+* Load TOML documents from files and string data
+* Easily navigate TOML structure using TomlTree
+* Line & column position data for all parsed elements
+* Query support similar to JSON-Path
+* Syntax errors contain line and column numbers
+
+Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
+
+* Semantic evaluation of parsed TOML
+* Informing a user of mistakes in the source document, after it has been parsed
+* Programatic handling of default values on a case-by-case basis
+* Using a TOML document as a flexible data-store
 
 ## Import
 
@@ -13,6 +32,8 @@ This library supports TOML version
 
 ## Usage
 
+### Example
+
 Say you have a TOML file that looks like this:
 
 ```toml
@@ -42,12 +63,22 @@ if err != nil {
     user = configTree.Get("user").(string)
     password = configTree.Get("password").(string)
     fmt.Println("User is ", user, ". Password is ", password)
+
+    // show where elements are in the file
+    fmt.Println("User position: %v", configTree.GetPosition("user"))
+    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+    // use a query to gather elements without walking the tree
+    results, _ := config.Query("$..[user,password]")
+    for ii, item := range results.Values() {
+      fmt.Println("Query result %d: %v", ii, item)
+    }
 }
 ```
 
 ## Documentation
 
-The documentation is available at
+The documentation and additional examples are available at
 [godoc.org](http://godoc.org/github.com/pelletier/go-toml).
 
 ## Contribute
@@ -60,14 +91,14 @@ much appreciated!
 
 You have to make sure two kind of tests run:
 
-1. The Go unit tests: `go test`
+1. The Go unit tests
-2. The TOML examples base: `./test_program/go-test.sh`
+2. The TOML examples base
 
 You can run both of them using `./test.sh`.
 
 ## License
 
-Copyright (c) 2013, 2014 Thomas Pelletier, Eric Anderton
+Copyright (c) 2013 - 2016 Thomas Pelletier, Eric Anderton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

cmd/test_program.go

@@ -13,24 +13,26 @@ import (
 func main() {
 	bytes, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
+		log.Fatalf("Error during TOML read: %s", err)
 		os.Exit(2)
 	}
 	tree, err := toml.Load(string(bytes))
 	if err != nil {
+		log.Fatalf("Error during TOML load: %s", err)
 		os.Exit(1)
 	}
 
-	typedTree := translate((map[string]interface{})(*tree))
+	typedTree := translate(*tree)
 
 	if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
 		log.Fatalf("Error encoding JSON: %s", err)
+		os.Exit(3)
 	}
 
 	os.Exit(0)
 }
 
 func translate(tomlData interface{}) interface{} {
-
 	switch orig := tomlData.(type) {
 	case map[string]interface{}:
 		typed := make(map[string]interface{}, len(orig))
@@ -39,7 +41,14 @@ func translate(tomlData interface{}) interface{} {
 		}
 		return typed
 	case *toml.TomlTree:
-		return translate((map[string]interface{})(*orig))
+		return translate(*orig)
+	case toml.TomlTree:
+		keys := orig.Keys()
+		typed := make(map[string]interface{}, len(keys))
+		for _, k := range keys {
+			typed[k] = translate(orig.GetPath([]string{k}))
+		}
+		return typed
 	case []*toml.TomlTree:
 		typed := make([]map[string]interface{}, len(orig))
 		for i, v := range orig {

doc.go

@@ -0,0 +1,245 @@
+// Package toml is a TOML markup language parser.
+//
+// This version supports the specification as described in
+// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md
+//
+// TOML Parsing
+//
+// TOML data may be parsed in two ways: by file, or by string.
+//
+//   // load TOML data by filename
+//   tree, err := toml.LoadFile("filename.toml")
+//
+//   // load TOML data stored in a string
+//   tree, err := toml.Load(stringContainingTomlData)
+//
+// Either way, the result is a TomlTree object that can be used to navigate the
+// structure and data within the original document.
+//
+//
+// Getting data from the TomlTree
+//
+// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
+// methods on the returned TomlTree, to find your way through the document data.
+//
+//   if tree.Has('foo') {
+//     fmt.Prinln("foo is: %v", tree.Get('foo'))
+//   }
+//
+// Working with Paths
+//
+// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
+// and GetDefault() methods.  These are the same kind of key paths used within the
+// TOML specification for struct tames.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.Has("foo.bar.baz")
+//
+//   // returns the key at this path, if it is there
+//   tree.Get("foo.bar.baz")
+//
+// TOML allows keys to contain '.', which can cause this syntax to be problematic
+// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
+// methods to explicitly define the path.  This form is also faster, since
+// it avoids having to parse the passed key for '.' delimiters.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.HasPath(string{}{"foo","bar","baz"})
+//
+//   // returns the key at this path, if it is there
+//   tree.GetPath(string{}{"foo","bar","baz"})
+//
+// Note that this is distinct from the heavyweight query syntax supported by
+// TomlTree.Query() and the Query() struct (see below).
+//
+// Position Support
+//
+// Each element within the TomlTree is stored with position metadata, which is
+// invaluable for providing semantic feedback to a user.  This helps in
+// situations where the TOML file parses correctly, but contains data that is
+// not correct for the application.  In such cases, an error message can be
+// generated that indicates the problem line and column number in the source
+// TOML document.
+//
+//   // load TOML data
+//   tree, _ := toml.Load("filename.toml")
+//
+//   // get an entry and report an error if it's the wrong type
+//   element := tree.Get("foo")
+//   if value, ok := element.(int64); !ok {
+//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
+//   }
+//
+//   // report an error if an expected element is missing
+//   if !tree.Has("bar") {
+//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
+//   }
+//
+// Query Support
+//
+// The TOML query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result := tree.Query("$.foo.bar.baz")  // result is 'nil' if the path is not present
+//
+// This is equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   tree.Query("$.foo[-1]")
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   tree.Query("$.foo[0:5]")
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   tree.Query("$.foo[0:-1]")
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   tree.Query("$.foo[0:-1:2]")
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   tree.Query("$.foo[:]")
+//   tree.Query("$.foo[0:]")
+//   tree.Query("$.foo[:-1]")
+//   tree.Query("$.foo[0:-1:]")
+//   tree.Query("$.foo[::1]")
+//   tree.Query("$.foo[0::1]")
+//   tree.Query("$.foo[:-1:1]")
+//   tree.Query("$.foo[0:-1:1]")
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   tree.Query("$.foo[?(bar)]")
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type TomlTree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a QueryResult object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := tree.Query("$.foo.bar.baz")
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a TomlTree object, or compiled ahead
+// of time and executed discretely.  The former is more convienent, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := tree.Query("$.foo.bar.baz")
+//
+//   // compiled query
+//   query := toml.CompileQuery("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := CompileQuery("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*TomlTree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package toml

doc_test.go

@@ -0,0 +1,81 @@
+// code examples for godoc
+
+package toml
+
+import (
+	"fmt"
+)
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := CompileQuery("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*TomlTree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' TomlTree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	authors, _ := config.Query("$.book.author")
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func Example_comprehensiveExample() {
+	config, err := LoadFile("config.toml")
+
+	if err != nil {
+		fmt.Println("Error ", err.Error())
+	} else {
+		// retrieve data directly
+		user := config.Get("postgres.user").(string)
+		password := config.Get("postgres.password").(string)
+
+		// or using an intermediate object
+		configTree := config.Get("postgres").(*TomlTree)
+		user = configTree.Get("user").(string)
+		password = configTree.Get("password").(string)
+		fmt.Println("User is ", user, ". Password is ", password)
+
+		// show where elements are in the file
+		fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
+		fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
+
+		// use a query to gather elements without walking the tree
+		results, _ := config.Query("$..[user,password]")
+		for ii, item := range results.Values() {
+			fmt.Printf("Query result %d: %v\n", ii, item)
+		}
+	}
+}

example-crlf.toml

@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it

keysparsing.go

@@ -0,0 +1,81 @@
+// Parsing keys handling both bare and quoted keys.
+
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"unicode"
+)
+
+func parseKey(key string) ([]string, error) {
+	groups := []string{}
+	var buffer bytes.Buffer
+	inQuotes := false
+	escapeNext := false
+	ignoreSpace := true
+	expectDot := false
+
+	for _, char := range key {
+		if ignoreSpace {
+			if char == ' ' {
+				continue
+			}
+			ignoreSpace = false
+		}
+		if escapeNext {
+			buffer.WriteRune(char)
+			escapeNext = false
+			continue
+		}
+		switch char {
+		case '\\':
+			escapeNext = true
+			continue
+		case '"':
+			inQuotes = !inQuotes
+			expectDot = false
+		case '.':
+			if inQuotes {
+				buffer.WriteRune(char)
+			} else {
+				groups = append(groups, buffer.String())
+				buffer.Reset()
+				ignoreSpace = true
+				expectDot = false
+			}
+		case ' ':
+			if inQuotes {
+				buffer.WriteRune(char)
+			} else {
+				expectDot = true
+			}
+		default:
+			if !inQuotes && !isValidBareChar(char) {
+				return nil, fmt.Errorf("invalid bare character: %c", char)
+			}
+			if !inQuotes && expectDot {
+				return nil, fmt.Errorf("what?")
+			}
+			buffer.WriteRune(char)
+			expectDot = false
+		}
+	}
+	if inQuotes {
+		return nil, fmt.Errorf("mismatched quotes")
+	}
+	if escapeNext {
+		return nil, fmt.Errorf("unfinished escape sequence")
+	}
+	if buffer.Len() > 0 {
+		groups = append(groups, buffer.String())
+	}
+	if len(groups) == 0 {
+		return nil, fmt.Errorf("empty key")
+	}
+	return groups, nil
+}
+
+func isValidBareChar(r rune) bool {
+	return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r)
+}

keysparsing_test.go

@@ -0,0 +1,49 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+func testResult(t *testing.T, key string, expected []string) {
+	parsed, err := parseKey(key)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if len(expected) != len(parsed) {
+		t.Fatal("Expected length", len(expected), "but", len(parsed), "parsed")
+	}
+	for index, expectedKey := range expected {
+		if expectedKey != parsed[index] {
+			t.Fatal("Expected", expectedKey, "at index", index, "but found", parsed[index])
+		}
+	}
+}
+
+func testError(t *testing.T, key string, expectedError string) {
+	_, err := parseKey(key)
+	if fmt.Sprintf("%s", err) != expectedError {
+		t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
+	}
+}
+
+func TestBareKeyBasic(t *testing.T) {
+	testResult(t, "test", []string{"test"})
+}
+
+func TestBareKeyDotted(t *testing.T) {
+	testResult(t, "this.is.a.key", []string{"this", "is", "a", "key"})
+}
+
+func TestDottedKeyBasic(t *testing.T) {
+	testResult(t, "\"a.dotted.key\"", []string{"a.dotted.key"})
+}
+
+func TestBaseKeyPound(t *testing.T) {
+	testError(t, "hello#world", "invalid bare character: #")
+}
+
+func TestEmptyKey(t *testing.T) {
+	testError(t, "", "empty key")
+	testError(t, " ", "empty key")
+}

lexer.go

@@ -1,190 +1,158 @@
-// TOML lexer.// Written using the principles developped by Rob Pike in
+// TOML lexer.
+//
+// Written using the principles developped by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
 package toml
 
 import (
 	"fmt"
+	"github.com/pelletier/go-buffruneio"
+	"io"
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
-	"unicode/utf8"
 )
 
 var dateRegexp *regexp.Regexp
 
-// Define tokens
+// Define state functions
-type tokenType int
+type tomlLexStateFn func() tomlLexStateFn
-
-const (
-	eof = -(iota + 1)
-)
-
-const (
-	tokenError tokenType = iota
-	tokenEOF
-	tokenComment
-	tokenKey
-	tokenEqual
-	tokenString
-	tokenInteger
-	tokenTrue
-	tokenFalse
-	tokenFloat
-	tokenLeftBracket
-	tokenRightBracket
-	tokenDoubleLeftBracket
-	tokenDoubleRightBracket
-	tokenDate
-	tokenKeyGroup
-	tokenKeyGroupArray
-	tokenComma
-	tokenEOL
-)
-
-type token struct {
-	typ tokenType
-	val string
-}
-
-func (i token) String() string {
-	switch i.typ {
-	case tokenEOF:
-		return "EOF"
-	case tokenError:
-		return i.val
-	}
-
-	if len(i.val) > 10 {
-		return fmt.Sprintf("%.10q...", i.val)
-	}
-	return fmt.Sprintf("%q", i.val)
-}
-
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
-}
-
-func isKeyChar(r rune) bool {
-	// "Keys start with the first non-whitespace character and end with the last
-	// non-whitespace character before the equals sign."
-	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
-}
-
-func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
-}
-
-func isHexDigit(r rune) bool {
-	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
-}
 
 // Define lexer
-type lexer struct {
+type tomlLexer struct {
-	input  string
+	input         *buffruneio.Reader // Textual source
-	start  int
+	buffer        []rune             // Runes composing the current token
-	pos    int
+	tokens        chan token
-	width  int
+	depth         int
-	tokens chan token
+	line          int
-	depth  int
+	col           int
+	endbufferLine int
+	endbufferCol  int
 }
 
-func (l *lexer) run() {
+// Basic read operations on input
-	for state := lexVoid; state != nil; {
+
-		state = state(l)
+func (l *tomlLexer) read() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
 	}
-	close(l.tokens)
+	if r == '\n' {
-}
+		l.endbufferLine++
-
+		l.endbufferCol = 1
-func (l *lexer) emit(t tokenType) {
+	} else {
-	l.tokens <- token{t, l.input[l.start:l.pos]}
+		l.endbufferCol++
-	l.start = l.pos
-}
-
-func (l *lexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{t, value}
-	l.start = l.pos
-}
-
-func (l *lexer) next() rune {
-	if l.pos >= len(l.input) {
-		l.width = 0
-		return eof
 	}
-	var r rune
-	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
-	l.pos += l.width
 	return r
 }
 
-func (l *lexer) ignore() {
+func (l *tomlLexer) next() rune {
-	l.start = l.pos
+	r := l.read()
+
+	if r != eof {
+		l.buffer = append(l.buffer, r)
+	}
+	return r
 }
 
-func (l *lexer) backup() {
+func (l *tomlLexer) ignore() {
-	l.pos -= l.width
+	l.buffer = make([]rune, 0)
+	l.line = l.endbufferLine
+	l.col = l.endbufferCol
 }
 
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+func (l *tomlLexer) skip() {
+	l.next()
+	l.ignore()
+}
+
+func (l *tomlLexer) fastForward(n int) {
+	for i := 0; i < n; i++ {
+		l.next()
+	}
+}
+
+func (l *tomlLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
-		tokenError,
+		Position: Position{l.line, l.col},
-		fmt.Sprintf(format, args...),
+		typ:      t,
+		val:      value,
+	}
+	l.ignore()
+}
+
+func (l *tomlLexer) emit(t tokenType) {
+	l.emitWithValue(t, string(l.buffer))
+}
+
+func (l *tomlLexer) peek() rune {
+	r, err := l.input.ReadRune()
+	if err != nil {
+		panic(err)
+	}
+	l.input.UnreadRune()
+	return r
+}
+
+func (l *tomlLexer) follow(next string) bool {
+	for _, expectedRune := range next {
+		r, err := l.input.ReadRune()
+		defer l.input.UnreadRune()
+		if err != nil {
+			panic(err)
+		}
+		if expectedRune != r {
+			return false
+		}
+	}
+	return true
+}
+
+// Error management
+
+func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
 	}
 	return nil
 }
 
-func (l *lexer) peek() rune {
+// State functions
-	r := l.next()
-	l.backup()
-	return r
-}
 
-func (l *lexer) accept(valid string) bool {
+func (l *tomlLexer) lexVoid() tomlLexStateFn {
-	if strings.IndexRune(valid, l.next()) >= 0 {
-		return true
-	}
-	l.backup()
-	return false
-}
-
-func (l *lexer) follow(next string) bool {
-	return strings.HasPrefix(l.input[l.pos:], next)
-}
-
-// Define state functions
-type stateFn func(*lexer) stateFn
-
-func lexVoid(l *lexer) stateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '[':
-			return lexKeyGroup
+			return l.lexKeyGroup
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '=':
-			return lexEqual
+			return l.lexEqual
+		case '\r':
+			fallthrough
+		case '\n':
+			l.skip()
+			continue
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
 		}
 
 		if l.depth > 0 {
-			return lexRvalue
+			return l.lexRvalue
 		}
 
-		if isKeyChar(next) {
+		if isKeyStartChar(next) {
-			return lexKey
+			return l.lexKey
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
 	}
@@ -193,193 +161,309 @@ func lexVoid(l *lexer) stateFn {
 	return nil
 }
 
-func lexRvalue(l *lexer) stateFn {
+func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '.':
 			return l.errorf("cannot start float with a dot")
 		case '=':
-			return l.errorf("cannot have multiple equals for the same key")
+			return l.lexEqual
 		case '[':
-			l.depth += 1
+			l.depth++
-			return lexLeftBracket
+			return l.lexLeftBracket
 		case ']':
-			l.depth -= 1
+			l.depth--
-			return lexRightBracket
+			return l.lexRightBracket
+		case '{':
+			return l.lexLeftCurlyBrace
+		case '}':
+			return l.lexRightCurlyBrace
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '"':
-			return lexString
+			return l.lexString
+		case '\'':
+			return l.lexLiteralString
 		case ',':
-			return lexComma
+			return l.lexComma
+		case '\r':
+			fallthrough
 		case '\n':
-			l.ignore()
+			l.skip()
-			l.pos += 1
 			if l.depth == 0 {
-				return lexVoid
+				return l.lexVoid
-			} else {
-				return lexRvalue
 			}
+			return l.lexRvalue
+		case '_':
+			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
-			return lexTrue
+			return l.lexTrue
 		}
 
 		if l.follow("false") {
-			return lexFalse
+			return l.lexFalse
-		}
-
-		if isAlphanumeric(next) {
-			return lexKey
-		}
-
-		if dateRegexp.FindString(l.input[l.pos:]) != "" {
-			return lexDate
-		}
-
-		if next == '+' || next == '-' || isDigit(next) {
-			return lexNumber
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
+			continue
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
+
+		possibleDate := string(l.input.Peek(35))
+		dateMatch := dateRegexp.FindString(possibleDate)
+		if dateMatch != "" {
+			l.fastForward(len(dateMatch))
+			return l.lexDate
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
 	}
 
 	l.emit(tokenEOF)
 	return nil
 }
 
-func lexDate(l *lexer) stateFn {
+func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.pos += 20 // Fixed size of a date in TOML
+	l.emit(tokenLeftCurlyBrace)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
+	l.next()
+	l.emit(tokenRightCurlyBrace)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexDate() tomlLexStateFn {
 	l.emit(tokenDate)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexTrue(l *lexer) stateFn {
+func (l *tomlLexer) lexTrue() tomlLexStateFn {
-	l.ignore()
+	l.fastForward(4)
-	l.pos += 4
 	l.emit(tokenTrue)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexFalse(l *lexer) stateFn {
+func (l *tomlLexer) lexFalse() tomlLexStateFn {
-	l.ignore()
+	l.fastForward(5)
-	l.pos += 5
 	l.emit(tokenFalse)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexEqual(l *lexer) stateFn {
+func (l *tomlLexer) lexEqual() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.accept("=")
 	l.emit(tokenEqual)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexComma(l *lexer) stateFn {
+func (l *tomlLexer) lexComma() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.accept(",")
 	l.emit(tokenComma)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexKey(l *lexer) stateFn {
+func (l *tomlLexer) lexKey() tomlLexStateFn {
-	l.ignore()
+	inQuotes := false
-	for isKeyChar(l.next()) {
+	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
+		if r == '"' {
+			inQuotes = !inQuotes
+		} else if r == '\n' {
+			return l.errorf("keys cannot contain new lines")
+		} else if isSpace(r) && !inQuotes {
+			break
+		} else if !isValidBareChar(r) && !inQuotes {
+			return l.errorf("keys cannot contain %c character", r)
+		}
+		l.next()
 	}
-	l.backup()
 	l.emit(tokenKey)
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexComment(l *lexer) stateFn {
+func (l *tomlLexer) lexComment() tomlLexStateFn {
-	for {
+	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
-		next := l.next()
+		if (next == '\r' && l.follow("\r\n")) {
-		if next == '\n' || next == eof {
 			break
 		}
+		l.next()
 	}
 	l.ignore()
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexLeftBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.pos += 1
 	l.emit(tokenLeftBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexString(l *lexer) stateFn {
+func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
-	l.pos += 1
+	l.skip()
-	l.ignore()
+	growingString := ""
-	growing_string := ""
+
+	// handle special case for triple-quote
+	terminator := "'"
+	if l.follow("''") {
+		l.skip()
+		l.skip()
+		terminator = "'''"
+
+		// special case: discard leading newline
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
+		}
+	}
+
+	// find end of string
+	for {
+		if l.follow(terminator) {
+			l.emitWithValue(tokenString, growingString)
+			l.fastForward(len(terminator))
+			l.ignore()
+			return l.lexRvalue
+		}
+
+		next := l.peek()
+		if next == eof {
+			break
+		}
+		growingString += string(l.next())
+	}
+
+	return l.errorf("unclosed string")
+}
+
+func (l *tomlLexer) lexString() tomlLexStateFn {
+	l.skip()
+	growingString := ""
+
+	// handle special case for triple-quote
+	terminator := "\""
+	if l.follow("\"\"") {
+		l.skip()
+		l.skip()
+		terminator = "\"\"\""
+
+		// special case: discard leading newline
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
+		}
+	}
 
 	for {
-		if l.peek() == '"' {
+		if l.follow(terminator) {
-			l.emitWithValue(tokenString, growing_string)
+			l.emitWithValue(tokenString, growingString)
-			l.pos += 1
+			l.fastForward(len(terminator))
 			l.ignore()
-			return lexRvalue
+			return l.lexRvalue
 		}
 
-		if l.follow("\\\"") {
+		if l.follow("\\") {
-			l.pos += 1
+			l.next()
-			growing_string += "\""
+			switch l.peek() {
-		} else if l.follow("\\n") {
+			case '\r':
-			l.pos += 1
+				fallthrough
-			growing_string += "\n"
+			case '\n':
-		} else if l.follow("\\b") {
+				fallthrough
-			l.pos += 1
+			case '\t':
-			growing_string += "\b"
+				fallthrough
-		} else if l.follow("\\f") {
+			case ' ':
-			l.pos += 1
+				// skip all whitespace chars following backslash
-			growing_string += "\f"
+				for strings.ContainsRune("\r\n\t ", l.peek()) {
-		} else if l.follow("\\/") {
+					l.next()
-			l.pos += 1
-			growing_string += "/"
-		} else if l.follow("\\t") {
-			l.pos += 1
-			growing_string += "\t"
-		} else if l.follow("\\r") {
-			l.pos += 1
-			growing_string += "\r"
-		} else if l.follow("\\\\") {
-			l.pos += 1
-			growing_string += "\\"
-		} else if l.follow("\\u") {
-			l.pos += 2
-			code := ""
-			for i := 0; i < 4; i++ {
-				c := l.peek()
-				l.pos += 1
-				if !isHexDigit(c) {
-					return l.errorf("unfinished unicode escape")
 				}
-				code = code + string(c)
+			case '"':
+				growingString += "\""
+				l.next()
+			case 'n':
+				growingString += "\n"
+				l.next()
+			case 'b':
+				growingString += "\b"
+				l.next()
+			case 'f':
+				growingString += "\f"
+				l.next()
+			case '/':
+				growingString += "/"
+				l.next()
+			case 't':
+				growingString += "\t"
+				l.next()
+			case 'r':
+				growingString += "\r"
+				l.next()
+			case '\\':
+				growingString += "\\"
+				l.next()
+			case 'u':
+				l.next()
+				code := ""
+				for i := 0; i < 4; i++ {
+					c := l.peek()
+					if !isHexDigit(c) {
+						return l.errorf("unfinished unicode escape")
+					}
+					l.next()
+					code = code + string(c)
+				}
+				intcode, err := strconv.ParseInt(code, 16, 32)
+				if err != nil {
+					return l.errorf("invalid unicode escape: \\u" + code)
+				}
+				growingString += string(rune(intcode))
+			case 'U':
+				l.next()
+				code := ""
+				for i := 0; i < 8; i++ {
+					c := l.peek()
+					if !isHexDigit(c) {
+						return l.errorf("unfinished unicode escape")
+					}
+					l.next()
+					code = code + string(c)
+				}
+				intcode, err := strconv.ParseInt(code, 16, 64)
+				if err != nil {
+					return l.errorf("invalid unicode escape: \\U" + code)
+				}
+				growingString += string(rune(intcode))
+			default:
+				return l.errorf("invalid escape sequence: \\" + string(l.peek()))
 			}
-			l.pos -= 1
-			intcode, err := strconv.ParseInt(code, 16, 32)
-			if err != nil {
-				return l.errorf("invalid unicode escape: \\u" + code)
-			}
-			growing_string += string(rune(intcode))
-		} else if l.follow("\\") {
-			l.pos += 1
-			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
 		} else {
-			growing_string += string(l.peek())
+			r := l.peek()
+			if 0x00 <= r && r <= 0x1F {
+				return l.errorf("unescaped control character %U", r)
+			}
+			l.next()
+			growingString += string(r)
 		}
 
-		if l.next() == eof {
+		if l.peek() == eof {
 			break
 		}
 	}
@@ -387,124 +471,140 @@ func lexString(l *lexer) stateFn {
 	return l.errorf("unclosed string")
 }
 
-func lexKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.pos += 1
 
 	if l.peek() == '[' {
 		// token '[[' signifies an array of anonymous key groups
-		l.pos += 1
+		l.next()
 		l.emit(tokenDoubleLeftBracket)
-		return lexInsideKeyGroupArray
+		return l.lexInsideKeyGroupArray
-	} else {
-		// vanilla key group
-		l.emit(tokenLeftBracket)
-		return lexInsideKeyGroup
 	}
+	// vanilla key group
+	l.emit(tokenLeftBracket)
+	return l.lexInsideKeyGroup
 }
 
-func lexInsideKeyGroupArray(l *lexer) stateFn {
+func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
-	for {
+	for r := l.peek(); r != eof; r = l.peek() {
-		if l.peek() == ']' {
+		switch r {
-			if l.pos > l.start {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroupArray)
 			}
-			l.ignore()
+			l.next()
-			l.pos += 1
 			if l.peek() != ']' {
-				break // error
+				break
 			}
-			l.pos += 1
+			l.next()
 			l.emit(tokenDoubleRightBracket)
-			return lexVoid
+			return l.lexVoid
-		} else if l.peek() == '[' {
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
+		default:
-
+			l.next()
-		if l.next() == eof {
-			break
 		}
 	}
 	return l.errorf("unclosed key group array")
 }
 
-func lexInsideKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
-	for {
+	for r := l.peek(); r != eof; r = l.peek() {
-		if l.peek() == ']' {
+		switch r {
-			if l.pos > l.start {
+		case ']':
+			if len(l.buffer) > 0 {
 				l.emit(tokenKeyGroup)
 			}
-			l.ignore()
+			l.next()
-			l.pos += 1
 			l.emit(tokenRightBracket)
-			return lexVoid
+			return l.lexVoid
-		} else if l.peek() == '[' {
+		case '[':
 			return l.errorf("group name cannot contain ']'")
-		}
+		default:
-
+			l.next()
-		if l.next() == eof {
-			break
 		}
 	}
 	return l.errorf("unclosed key group")
 }
 
-func lexRightBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
-	l.ignore()
+	l.next()
-	l.pos += 1
 	l.emit(tokenRightBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexNumber(l *lexer) stateFn {
+func (l *tomlLexer) lexNumber() tomlLexStateFn {
-	l.ignore()
+	r := l.peek()
-	if !l.accept("+") {
+	if r == '+' || r == '-' {
-		l.accept("-")
+		l.next()
 	}
-	point_seen := false
+	pointSeen := false
-	digit_seen := false
+	expSeen := false
+	digitSeen := false
 	for {
-		next := l.next()
+		next := l.peek()
 		if next == '.' {
-			if point_seen {
+			if pointSeen {
 				return l.errorf("cannot have two dots in one float")
 			}
+			l.next()
 			if !isDigit(l.peek()) {
 				return l.errorf("float cannot end with a dot")
 			}
-			point_seen = true
+			pointSeen = true
+		} else if next == 'e' || next == 'E' {
+			expSeen = true
+			l.next()
+			r := l.peek()
+			if r == '+' || r == '-' {
+				l.next()
+			}
 		} else if isDigit(next) {
-			digit_seen = true
+			digitSeen = true
+			l.next()
+		} else if next == '_' {
+			l.next()
 		} else {
-			l.backup()
 			break
 		}
-		if point_seen && !digit_seen {
+		if pointSeen && !digitSeen {
 			return l.errorf("cannot start float with a dot")
 		}
 	}
 
-	if !digit_seen {
+	if !digitSeen {
 		return l.errorf("no digit in that number")
 	}
-	if point_seen {
+	if pointSeen || expSeen {
 		l.emit(tokenFloat)
 	} else {
 		l.emit(tokenInteger)
 	}
-	return lexRvalue
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
 }
 
 func init() {
-	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z")
+	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})")
 }
 
 // Entry point
-func lex(input string) (*lexer, chan token) {
+func lexToml(input io.Reader) chan token {
-	l := &lexer{
+	bufferedInput := buffruneio.NewReader(input)
-		input:  input,
+	l := &tomlLexer{
-		tokens: make(chan token),
+		input:         bufferedInput,
+		tokens:        make(chan token),
+		line:          1,
+		col:           1,
+		endbufferLine: 1,
+		endbufferCol:  1,
 	}
 	go l.run()
-	return l, l.tokens
+	return l.tokens
 }

lexer_test.go

@@ -1,15 +1,23 @@
 package toml
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	_, ch := lex(input)
+	ch := lexToml(strings.NewReader(input))
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {
+			t.Log("While testing: ", input)
+			t.Log("compared (got)", token, "to (expected)", expected)
+			t.Log("\tvalue:", token.val, "<->", expected.val)
+			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
+			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
+			t.Log("\tline:", token.Line, "<->", expected.Line)
+			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
 			t.Log("compared", token, "to", expected)
-			t.Log(token.val, "<->", expected.val)
-			t.Log(token.typ, "<->", expected.typ)
 			t.FailNow()
 		}
 	}
@@ -29,387 +37,610 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
 
 func TestValidKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world]", []token{
-		token{tokenLeftBracket, "["},
+		token{Position{1, 1}, tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 14}, tokenEOF, ""},
+	})
+}
+
+func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
+	testFlow(t, `[ j . "ʞ" . l ]`, []token{
+		token{Position{1, 1}, tokenLeftBracket, "["},
+		token{Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
+		token{Position{1, 15}, tokenRightBracket, "]"},
+		token{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestUnclosedKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world", []token{
-		token{tokenLeftBracket, "["},
+		token{Position{1, 1}, tokenLeftBracket, "["},
-		token{tokenError, "unclosed key group"},
+		token{Position{1, 2}, tokenError, "unclosed key group"},
 	})
 }
 
 func TestComment(t *testing.T) {
 	testFlow(t, "# blahblah", []token{
-		token{tokenEOF, ""},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyGroupComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah", []token{
-		token{tokenLeftBracket, "["},
+		token{Position{1, 1}, tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 25}, tokenEOF, ""},
 	})
 }
 
 func TestMultipleKeyGroupsComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah\n[test]", []token{
-		token{tokenLeftBracket, "["},
+		token{Position{1, 1}, tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
-		token{tokenLeftBracket, "["},
+		token{Position{2, 1}, tokenLeftBracket, "["},
-		token{tokenKeyGroup, "test"},
+		token{Position{2, 2}, tokenKeyGroup, "test"},
-		token{tokenRightBracket, "]"},
+		token{Position{2, 6}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{2, 7}, tokenEOF, ""},
+	})
+}
+
+
+func TestSimpleWindowsCRLF(t *testing.T) {
+	testFlow(t, "a=4\r\nb=2", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenInteger, "4"},
+		token{Position{2, 1}, tokenKey, "b"},
+		token{Position{2, 2}, tokenEqual, "="},
+		token{Position{2, 3}, tokenInteger, "2"},
+		token{Position{2, 4}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
-		token{tokenKey, "hello"},
+		token{Position{1, 1}, tokenKey, "hello"},
-		token{tokenEOF, ""},
+		token{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUnderscore(t *testing.T) {
 	testFlow(t, "hello_hello", []token{
-		token{tokenKey, "hello_hello"},
+		token{Position{1, 1}, tokenKey, "hello_hello"},
-		token{tokenEOF, ""},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithDash(t *testing.T) {
 	testFlow(t, "hello-world", []token{
-		token{tokenKey, "hello-world"},
+		token{Position{1, 1}, tokenKey, "hello-world"},
-		token{tokenEOF, ""},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUppercaseMix(t *testing.T) {
 	testFlow(t, "helloHELLOHello", []token{
-		token{tokenKey, "helloHELLOHello"},
+		token{Position{1, 1}, tokenKey, "helloHELLOHello"},
-		token{tokenEOF, ""},
+		token{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
 	testFlow(t, "héllÖ", []token{
-		token{tokenKey, "héllÖ"},
+		token{Position{1, 1}, tokenKey, "héllÖ"},
-		token{tokenEOF, ""},
+		token{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyAndEqual(t *testing.T) {
 	testFlow(t, "hello =", []token{
-		token{tokenKey, "hello"},
+		token{Position{1, 1}, tokenKey, "hello"},
-		token{tokenEqual, "="},
+		token{Position{1, 7}, tokenEqual, "="},
-		token{tokenEOF, ""},
+		token{Position{1, 8}, tokenEOF, ""},
 	})
 }
 
 func TestKeyWithSharpAndEqual(t *testing.T) {
 	testFlow(t, "key#name = 5", []token{
-		token{tokenKey, "key#name"},
+		token{Position{1, 1}, tokenError, "keys cannot contain # character"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
 	})
 }
+
 func TestKeyWithSymbolsAndEqual(t *testing.T) {
-	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
-		token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+		token{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "5"},
-		token{tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello\\\"\"", []token{
+	testFlow(t, `foo = "hello\""`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenString, "hello\""},
+		token{Position{1, 8}, tokenString, "hello\""},
-		token{tokenEOF, ""},
+		token{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnfinished(t *testing.T) {
-	testFlow(t, "foo = \"bar", []token{
+	testFlow(t, `foo = "bar`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenError, "unclosed string"},
+		token{Position{1, 8}, tokenError, "unclosed string"},
 	})
 }
 
 func TestKeyEqualString(t *testing.T) {
-	testFlow(t, "foo = \"bar\"", []token{
+	testFlow(t, `foo = "bar"`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenString, "bar"},
+		token{Position{1, 8}, tokenString, "bar"},
-		token{tokenEOF, ""},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualTrue(t *testing.T) {
 	testFlow(t, "foo = true", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenTrue, "true"},
+		token{Position{1, 7}, tokenTrue, "true"},
-		token{tokenEOF, ""},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualFalse(t *testing.T) {
 	testFlow(t, "foo = false", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenFalse, "false"},
+		token{Position{1, 7}, tokenFalse, "false"},
-		token{tokenEOF, ""},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedString(t *testing.T) {
-	testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{
+	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
-		token{tokenKey, "a"},
+		token{Position{1, 1}, tokenKey, "a"},
-		token{tokenEqual, "="},
+		token{Position{1, 3}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
-		token{tokenString, "hello"},
+		token{Position{1, 9}, tokenString, "hello"},
-		token{tokenComma, ","},
+		token{Position{1, 15}, tokenComma, ","},
-		token{tokenString, "world"},
+		token{Position{1, 18}, tokenString, "world"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 24}, tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 26}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 27}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedInts(t *testing.T) {
 	testFlow(t, "a = [ [42, 21], [10] ]", []token{
-		token{tokenKey, "a"},
+		token{Position{1, 1}, tokenKey, "a"},
-		token{tokenEqual, "="},
+		token{Position{1, 3}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
+		token{Position{1, 8}, tokenInteger, "42"},
-		token{tokenComma, ","},
+		token{Position{1, 10}, tokenComma, ","},
-		token{tokenInteger, "21"},
+		token{Position{1, 12}, tokenInteger, "21"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 14}, tokenRightBracket, "]"},
-		token{tokenComma, ","},
+		token{Position{1, 15}, tokenComma, ","},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 17}, tokenLeftBracket, "["},
-		token{tokenInteger, "10"},
+		token{Position{1, 18}, tokenInteger, "10"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 20}, tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 22}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 23}, tokenEOF, ""},
 	})
 }
 
 func TestArrayInts(t *testing.T) {
 	testFlow(t, "a = [ 42, 21, 10, ]", []token{
-		token{tokenKey, "a"},
+		token{Position{1, 1}, tokenKey, "a"},
-		token{tokenEqual, "="},
+		token{Position{1, 3}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
+		token{Position{1, 7}, tokenInteger, "42"},
-		token{tokenComma, ","},
+		token{Position{1, 9}, tokenComma, ","},
-		token{tokenInteger, "21"},
+		token{Position{1, 11}, tokenInteger, "21"},
-		token{tokenComma, ","},
+		token{Position{1, 13}, tokenComma, ","},
-		token{tokenInteger, "10"},
+		token{Position{1, 15}, tokenInteger, "10"},
-		token{tokenComma, ","},
+		token{Position{1, 17}, tokenComma, ","},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 19}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 20}, tokenEOF, ""},
 	})
 }
 
 func TestMultilineArrayComments(t *testing.T) {
 	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
-		token{tokenKey, "a"},
+		token{Position{1, 1}, tokenKey, "a"},
-		token{tokenEqual, "="},
+		token{Position{1, 3}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenLeftBracket, "["},
-		token{tokenInteger, "1"},
+		token{Position{1, 6}, tokenInteger, "1"},
-		token{tokenComma, ","},
+		token{Position{1, 7}, tokenComma, ","},
-		token{tokenInteger, "2"},
+		token{Position{2, 1}, tokenInteger, "2"},
-		token{tokenComma, ","},
+		token{Position{2, 2}, tokenComma, ","},
-		token{tokenInteger, "3"},
+		token{Position{3, 1}, tokenInteger, "3"},
-		token{tokenComma, ","},
+		token{Position{3, 2}, tokenComma, ","},
-		token{tokenRightBracket, "]"},
+		token{Position{4, 1}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{4, 2}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
+		token{Position{1, 8}, tokenTrue, "true"},
-		token{tokenComma, ","},
+		token{Position{1, 12}, tokenComma, ","},
-		token{tokenFalse, "false"},
+		token{Position{1, 14}, tokenFalse, "false"},
-		token{tokenComma, ","},
+		token{Position{1, 19}, tokenComma, ","},
-		token{tokenTrue, "true"},
+		token{Position{1, 21}, tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 25}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 26}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 	testFlow(t, "foo = [true, false, true] # YEAH", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
+		token{Position{1, 8}, tokenTrue, "true"},
-		token{tokenComma, ","},
+		token{Position{1, 12}, tokenComma, ","},
-		token{tokenFalse, "false"},
+		token{Position{1, 14}, tokenFalse, "false"},
-		token{tokenComma, ","},
+		token{Position{1, 19}, tokenComma, ","},
-		token{tokenTrue, "true"},
+		token{Position{1, 21}, tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 25}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 33}, tokenEOF, ""},
 	})
 }
 
 func TestDateRegexp(t *testing.T) {
 	if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
-		t.Fail()
+		t.Error("basic lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
+		t.Error("offset lexing")
+	}
+	if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
+		t.Error("nano precision lexing")
 	}
 }
 
 func TestKeyEqualDate(t *testing.T) {
 	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenDate, "1979-05-27T07:32:00Z"},
+		token{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
-		token{tokenEOF, ""},
+		token{Position{1, 27}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
+		token{Position{1, 32}, tokenEOF, ""},
+	})
+	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
+		token{Position{1, 39}, tokenEOF, ""},
 	})
 }
 
 func TestFloatEndingWithDot(t *testing.T) {
 	testFlow(t, "foo = 42.", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenError, "float cannot end with a dot"},
+		token{Position{1, 7}, tokenError, "float cannot end with a dot"},
 	})
 }
 
 func TestFloatWithTwoDots(t *testing.T) {
 	testFlow(t, "foo = 4.2.", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenError, "cannot have two dots in one float"},
+		token{Position{1, 7}, tokenError, "cannot have two dots in one float"},
 	})
 }
 
-func TestDoubleEqualKey(t *testing.T) {
+func TestFloatWithExponent1(t *testing.T) {
-	testFlow(t, "foo= = 2", []token{
+	testFlow(t, "a = 5e+22", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "a"},
-		token{tokenEqual, "="},
+		token{Position{1, 3}, tokenEqual, "="},
-		token{tokenError, "cannot have multiple equals for the same key"},
+		token{Position{1, 5}, tokenFloat, "5e+22"},
+		token{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent2(t *testing.T) {
+	testFlow(t, "a = 5E+22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "5E+22"},
+		token{Position{1, 10}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent3(t *testing.T) {
+	testFlow(t, "a = -5e+22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "-5e+22"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent4(t *testing.T) {
+	testFlow(t, "a = -5e-22", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "-5e-22"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestFloatWithExponent5(t *testing.T) {
+	testFlow(t, "a = 6.626e-34", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenFloat, "6.626e-34"},
+		token{Position{1, 14}, tokenEOF, ""},
 	})
 }
 
 func TestInvalidEsquapeSequence(t *testing.T) {
-	testFlow(t, "foo = \"\\x\"", []token{
+	testFlow(t, `foo = "\x"`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenError, "invalid escape sequence: \\x"},
+		token{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
 	})
 }
 
 func TestNestedArrays(t *testing.T) {
 	testFlow(t, "foo = [[[]]]", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 8}, tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
+		token{Position{1, 9}, tokenLeftBracket, "["},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 10}, tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 11}, tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
+		token{Position{1, 12}, tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 13}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualNumber(t *testing.T) {
 	testFlow(t, "foo = 42", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenInteger, "42"},
+		token{Position{1, 7}, tokenInteger, "42"},
-		token{tokenEOF, ""},
+		token{Position{1, 9}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +42", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenInteger, "+42"},
+		token{Position{1, 7}, tokenInteger, "+42"},
-		token{tokenEOF, ""},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -42", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenInteger, "-42"},
+		token{Position{1, 7}, tokenInteger, "-42"},
-		token{tokenEOF, ""},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = 4.2", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenFloat, "4.2"},
+		token{Position{1, 7}, tokenFloat, "4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +4.2", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenFloat, "+4.2"},
+		token{Position{1, 7}, tokenFloat, "+4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -4.2", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenFloat, "-4.2"},
+		token{Position{1, 7}, tokenFloat, "-4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 1_000", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "1_000"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 5_349_221", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "5_349_221"},
+		token{Position{1, 16}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = 1_2_3_4_5", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "1_2_3_4_5"},
+		token{Position{1, 16}, tokenEOF, ""},
+	})
+
+	testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
+		token{Position{1, 1}, tokenKey, "flt8"},
+		token{Position{1, 6}, tokenEqual, "="},
+		token{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
+		token{Position{1, 33}, tokenEOF, ""},
 	})
 }
 
 func TestMultiline(t *testing.T) {
 	testFlow(t, "foo = 42\nbar=21", []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenInteger, "42"},
+		token{Position{1, 7}, tokenInteger, "42"},
-		token{tokenKey, "bar"},
+		token{Position{2, 1}, tokenKey, "bar"},
-		token{tokenEqual, "="},
+		token{Position{2, 4}, tokenEqual, "="},
-		token{tokenInteger, "21"},
+		token{Position{2, 5}, tokenInteger, "21"},
-		token{tokenEOF, ""},
+		token{Position{2, 7}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello \\u2665\"", []token{
+	testFlow(t, `foo = "hello \u2665"`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenString, "hello ♥"},
+		token{Position{1, 8}, tokenString, "hello ♥"},
-		token{tokenEOF, ""},
+		token{Position{1, 21}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "hello \U000003B4"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "hello δ"},
+		token{Position{1, 25}, tokenEOF, ""},
+	})
+}
+
+func TestKeyEqualStringNoEscape(t *testing.T) {
+	testFlow(t, "foo = \"hello \u0002\"", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unescaped control character U+0002"},
+	})
+	testFlow(t, "foo = \"hello \u001F\"", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unescaped control character U+001F"},
+	})
+}
+
+func TestLiteralString(t *testing.T) {
+	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
+		token{Position{1, 35}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, `<\i\c*\s*>`},
+		token{Position{1, 19}, tokenEOF, ""},
+	})
+}
+
+func TestMultilineLiteralString(t *testing.T) {
+	testFlow(t, `foo = '''hello 'literal' world'''`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 10}, tokenString, `hello 'literal' world`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
+		token{Position{4, 9}, tokenEOF, ""},
+	})
+}
+
+func TestMultilineString(t *testing.T) {
+	testFlow(t, `foo = """hello "literal" world"""`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 10}, tokenString, `hello "literal" world`},
+		token{Position{1, 34}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = \"\"\"\nhello\\\n\"literal\"\\\nworld\"\"\"", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{2, 1}, tokenString, "hello\"literal\"world"},
+		token{Position{4, 9}, tokenEOF, ""},
+	})
+
+	testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 10}, tokenString, "hellomultilineworld"},
+		token{Position{6, 9}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
+		token{Position{1, 1}, tokenKey, "key2"},
+		token{Position{1, 6}, tokenEqual, "="},
+		token{Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
+		token{Position{6, 21}, tokenEOF, ""},
+	})
+
+	testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
+		token{Position{1, 1}, tokenKey, "key2"},
+		token{Position{1, 6}, tokenEqual, "="},
+		token{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
+		token{Position{5, 11}, tokenEOF, ""},
 	})
 }
 
 func TestUnicodeString(t *testing.T) {
-	testFlow(t, "foo = \"hello ♥ world\"", []token{
+	testFlow(t, `foo = "hello ♥ world"`, []token{
-		token{tokenKey, "foo"},
+		token{Position{1, 1}, tokenKey, "foo"},
-		token{tokenEqual, "="},
+		token{Position{1, 5}, tokenEqual, "="},
-		token{tokenString, "hello ♥ world"},
+		token{Position{1, 8}, tokenString, "hello ♥ world"},
-		token{tokenEOF, ""},
+		token{Position{1, 22}, tokenEOF, ""},
 	})
 }
 
 func TestKeyGroupArray(t *testing.T) {
 	testFlow(t, "[[foo]]", []token{
-		token{tokenDoubleLeftBracket, "[["},
+		token{Position{1, 1}, tokenDoubleLeftBracket, "[["},
-		token{tokenKeyGroupArray, "foo"},
+		token{Position{1, 3}, tokenKeyGroupArray, "foo"},
-		token{tokenDoubleRightBracket, "]]"},
+		token{Position{1, 6}, tokenDoubleRightBracket, "]]"},
-		token{tokenEOF, ""},
+		token{Position{1, 8}, tokenEOF, ""},
+	})
+}
+
+func TestQuotedKey(t *testing.T) {
+	testFlow(t, "\"a b\" = 42", []token{
+		token{Position{1, 1}, tokenKey, "\"a b\""},
+		token{Position{1, 7}, tokenEqual, "="},
+		token{Position{1, 9}, tokenInteger, "42"},
+		token{Position{1, 11}, tokenEOF, ""},
+	})
+}
+
+func TestKeyNewline(t *testing.T) {
+	testFlow(t, "a\n= 4", []token{
+		token{Position{1, 1}, tokenError, "keys cannot contain new lines"},
+	})
+}
+
+func TestInvalidFloat(t *testing.T) {
+	testFlow(t, "a=7e1_", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenFloat, "7e1_"},
+		token{Position{1, 7}, tokenEOF, ""},
 	})
 }

match.go

@@ -0,0 +1,227 @@
+package toml
+
+import (
+	"fmt"
+)
+
+// support function to set positions for tomlValues
+// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
+// values returned by the query engines
+func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
+	switch castNode := node.(type) {
+	case *tomlValue:
+		ctx.lastPosition = castNode.position
+		return castNode.value
+	case []*TomlTree:
+		if len(castNode) > 0 {
+			ctx.lastPosition = castNode[0].position
+		}
+		return node
+	default:
+		return node
+	}
+}
+
+// base match
+type matchBase struct {
+	next pathFn
+}
+
+func (f *matchBase) setNext(next pathFn) {
+	f.next = next
+}
+
+// terminating functor - gathers results
+type terminatingFn struct {
+	// empty
+}
+
+func newTerminatingFn() *terminatingFn {
+	return &terminatingFn{}
+}
+
+func (f *terminatingFn) setNext(next pathFn) {
+	// do nothing
+}
+
+func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
+	switch castNode := node.(type) {
+	case *TomlTree:
+		ctx.result.appendResult(node, castNode.position)
+	case *tomlValue:
+		ctx.result.appendResult(node, castNode.position)
+	default:
+		// use last position for scalars
+		ctx.result.appendResult(node, ctx.lastPosition)
+	}
+}
+
+// match single key
+type matchKeyFn struct {
+	matchBase
+	Name string
+}
+
+func newMatchKeyFn(name string) *matchKeyFn {
+	return &matchKeyFn{Name: name}
+}
+
+func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		item := tree.values[f.Name]
+		if item != nil {
+			f.next.call(item, ctx)
+		}
+	}
+}
+
+// match single index
+type matchIndexFn struct {
+	matchBase
+	Idx int
+}
+
+func newMatchIndexFn(idx int) *matchIndexFn {
+	return &matchIndexFn{Idx: idx}
+}
+
+func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		if f.Idx < len(arr) && f.Idx >= 0 {
+			f.next.call(arr[f.Idx], ctx)
+		}
+	}
+}
+
+// filter by slicing
+type matchSliceFn struct {
+	matchBase
+	Start, End, Step int
+}
+
+func newMatchSliceFn(start, end, step int) *matchSliceFn {
+	return &matchSliceFn{Start: start, End: end, Step: step}
+}
+
+func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		// adjust indexes for negative values, reverse ordering
+		realStart, realEnd := f.Start, f.End
+		if realStart < 0 {
+			realStart = len(arr) + realStart
+		}
+		if realEnd < 0 {
+			realEnd = len(arr) + realEnd
+		}
+		if realEnd < realStart {
+			realEnd, realStart = realStart, realEnd // swap
+		}
+		// loop and gather
+		for idx := realStart; idx < realEnd; idx += f.Step {
+			f.next.call(arr[idx], ctx)
+		}
+	}
+}
+
+// match anything
+type matchAnyFn struct {
+	matchBase
+}
+
+func newMatchAnyFn() *matchAnyFn {
+	return &matchAnyFn{}
+}
+
+func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		for _, v := range tree.values {
+			f.next.call(v, ctx)
+		}
+	}
+}
+
+// filter through union
+type matchUnionFn struct {
+	Union []pathFn
+}
+
+func (f *matchUnionFn) setNext(next pathFn) {
+	for _, fn := range f.Union {
+		fn.setNext(next)
+	}
+}
+
+func (f *matchUnionFn) call(node interface{}, ctx *queryContext) {
+	for _, fn := range f.Union {
+		fn.call(node, ctx)
+	}
+}
+
+// match every single last node in the tree
+type matchRecursiveFn struct {
+	matchBase
+}
+
+func newMatchRecursiveFn() *matchRecursiveFn {
+	return &matchRecursiveFn{}
+}
+
+func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		var visit func(tree *TomlTree)
+		visit = func(tree *TomlTree) {
+			for _, v := range tree.values {
+				f.next.call(v, ctx)
+				switch node := v.(type) {
+				case *TomlTree:
+					visit(node)
+				case []*TomlTree:
+					for _, subtree := range node {
+						visit(subtree)
+					}
+				}
+			}
+		}
+		f.next.call(tree, ctx)
+		visit(tree)
+	}
+}
+
+// match based on an externally provided functional filter
+type matchFilterFn struct {
+	matchBase
+	Pos  Position
+	Name string
+}
+
+func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+	return &matchFilterFn{Name: name, Pos: pos}
+}
+
+func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
+	fn, ok := (*ctx.filters)[f.Name]
+	if !ok {
+		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
+			f.Pos.String(), f.Name))
+	}
+	switch castNode := tomlValueCheck(node, ctx).(type) {
+	case *TomlTree:
+		for _, v := range castNode.values {
+			if tv, ok := v.(*tomlValue); ok {
+				if fn(tv.value) {
+					f.next.call(v, ctx)
+				}
+			} else {
+				if fn(v) {
+					f.next.call(v, ctx)
+				}
+			}
+		}
+	case []interface{}:
+		for _, v := range castNode {
+			if fn(v) {
+				f.next.call(v, ctx)
+			}
+		}
+	}
+}

match_test.go

@@ -0,0 +1,201 @@
+package toml
+
+import (
+	"fmt"
+	"testing"
+)
+
+// dump path tree to a string
+func pathString(root pathFn) string {
+	result := fmt.Sprintf("%T:", root)
+	switch fn := root.(type) {
+	case *terminatingFn:
+		result += "{}"
+	case *matchKeyFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	case *matchIndexFn:
+		result += fmt.Sprintf("{%d}", fn.Idx)
+		result += pathString(fn.next)
+	case *matchSliceFn:
+		result += fmt.Sprintf("{%d:%d:%d}",
+			fn.Start, fn.End, fn.Step)
+		result += pathString(fn.next)
+	case *matchAnyFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchUnionFn:
+		result += "{["
+		for _, v := range fn.Union {
+			result += pathString(v) + ", "
+		}
+		result += "]}"
+	case *matchRecursiveFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchFilterFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	}
+	return result
+}
+
+func assertPathMatch(t *testing.T, path, ref *Query) bool {
+	pathStr := pathString(path.root)
+	refStr := pathString(ref.root)
+	if pathStr != refStr {
+		t.Errorf("paths do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+		return false
+	}
+	return true
+}
+
+func assertPath(t *testing.T, query string, ref *Query) {
+	path, _ := parseQuery(lexQuery(query))
+	assertPathMatch(t, path, ref)
+}
+
+func buildPath(parts ...pathFn) *Query {
+	query := newQuery()
+	for _, v := range parts {
+		query.appendPath(v)
+	}
+	return query
+}
+
+func TestPathRoot(t *testing.T) {
+	assertPath(t,
+		"$",
+		buildPath(
+		// empty
+		))
+}
+
+func TestPathKey(t *testing.T) {
+	assertPath(t,
+		"$.foo",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketKey(t *testing.T) {
+	assertPath(t,
+		"$[foo]",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketStringKey(t *testing.T) {
+	assertPath(t,
+		"$['foo']",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathIndex(t *testing.T) {
+	assertPath(t,
+		"$[123]",
+		buildPath(
+			newMatchIndexFn(123),
+		))
+}
+
+func TestPathSliceStart(t *testing.T) {
+	assertPath(t,
+		"$[123:]",
+		buildPath(
+			newMatchSliceFn(123, MaxInt, 1),
+		))
+}
+
+func TestPathSliceStartEnd(t *testing.T) {
+	assertPath(t,
+		"$[123:456]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartEndColon(t *testing.T) {
+	assertPath(t,
+		"$[123:456:]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartStep(t *testing.T) {
+	assertPath(t,
+		"$[123::7]",
+		buildPath(
+			newMatchSliceFn(123, MaxInt, 7),
+		))
+}
+
+func TestPathSliceEndStep(t *testing.T) {
+	assertPath(t,
+		"$[:456:7]",
+		buildPath(
+			newMatchSliceFn(0, 456, 7),
+		))
+}
+
+func TestPathSliceStep(t *testing.T) {
+	assertPath(t,
+		"$[::7]",
+		buildPath(
+			newMatchSliceFn(0, MaxInt, 7),
+		))
+}
+
+func TestPathSliceAll(t *testing.T) {
+	assertPath(t,
+		"$[123:456:7]",
+		buildPath(
+			newMatchSliceFn(123, 456, 7),
+		))
+}
+
+func TestPathAny(t *testing.T) {
+	assertPath(t,
+		"$.*",
+		buildPath(
+			newMatchAnyFn(),
+		))
+}
+
+func TestPathUnion(t *testing.T) {
+	assertPath(t,
+		"$[foo, bar, baz]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchKeyFn("foo"),
+				newMatchKeyFn("bar"),
+				newMatchKeyFn("baz"),
+			}},
+		))
+}
+
+func TestPathRecurse(t *testing.T) {
+	assertPath(t,
+		"$..*",
+		buildPath(
+			newMatchRecursiveFn(),
+		))
+}
+
+func TestPathFilterExpr(t *testing.T) {
+	assertPath(t,
+		"$[?('foo'),?(bar)]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchFilterFn("foo", Position{}),
+				newMatchFilterFn("bar", Position{}),
+			}},
+		))
+}

parser.go

@@ -5,12 +5,13 @@ package toml
 import (
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 )
 
-type parser struct {
+type tomlParser struct {
 	flow          chan token
 	tree          *TomlTree
 	tokensBuffer  []token
@@ -18,15 +19,20 @@ type parser struct {
 	seenGroupKeys []string
 }
 
-type parserStateFn func(*parser) parserStateFn
+type tomlParserStateFn func() tomlParserStateFn
 
-func (p *parser) run() {
+// Formats and panics an error message based on a token
-	for state := parseStart; state != nil; {
+func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) {
-		state = state(p)
+	panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...))
+}
+
+func (p *tomlParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
 	}
 }
 
-func (p *parser) peek() *token {
+func (p *tomlParser) peek() *token {
 	if len(p.tokensBuffer) != 0 {
 		return &(p.tokensBuffer[0])
 	}
@@ -39,17 +45,17 @@ func (p *parser) peek() *token {
 	return &tok
 }
 
-func (p *parser) assume(typ tokenType) {
+func (p *tomlParser) assume(typ tokenType) {
 	tok := p.getToken()
 	if tok == nil {
-		panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ))
+		p.raiseError(tok, "was expecting token %s, but token stream is empty", tok)
 	}
 	if tok.typ != typ {
-		panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ))
+		p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok)
 	}
 }
 
-func (p *parser) getToken() *token {
+func (p *tomlParser) getToken() *token {
 	if len(p.tokensBuffer) != 0 {
 		tok := p.tokensBuffer[0]
 		p.tokensBuffer = p.tokensBuffer[1:]
@@ -62,7 +68,7 @@ func (p *parser) getToken() *token {
 	return &tok
 }
 
-func parseStart(p *parser) parserStateFn {
+func (p *tomlParser) parseStart() tomlParserStateFn {
 	tok := p.peek()
 
 	// end of stream, parsing is finished
@@ -72,105 +78,154 @@ func parseStart(p *parser) parserStateFn {
 
 	switch tok.typ {
 	case tokenDoubleLeftBracket:
-		return parseGroupArray
+		return p.parseGroupArray
 	case tokenLeftBracket:
-		return parseGroup
+		return p.parseGroup
 	case tokenKey:
-		return parseAssign
+		return p.parseAssign
 	case tokenEOF:
 		return nil
 	default:
-		panic("unexpected token")
+		p.raiseError(tok, "unexpected token")
 	}
 	return nil
 }
 
-func parseGroupArray(p *parser) parserStateFn {
+func (p *tomlParser) parseGroupArray() tomlParserStateFn {
-	p.getToken() // discard the [[
+	startToken := p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group array", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
 	}
 
 	// get or create group array element at the indicated part in the path
-	p.currentGroup = strings.Split(key.val, ".")
+	keys, err := parseKey(key.val)
-	dest_tree := p.tree.GetPath(p.currentGroup)
+	if err != nil {
-	var array []*TomlTree
+		p.raiseError(key, "invalid group array key: %s", err)
-	if dest_tree == nil {
-		array = make([]*TomlTree, 0)
-	} else if dest_tree.([]*TomlTree) != nil {
-		array = dest_tree.([]*TomlTree)
-	} else {
-		panic(fmt.Sprintf("key %s is already assigned and not of type group array", key))
 	}
+	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
+	destTree := p.tree.GetPath(keys)
+	var array []*TomlTree
+	if destTree == nil {
+		array = make([]*TomlTree, 0)
+	} else if target, ok := destTree.([]*TomlTree); ok && target != nil {
+		array = destTree.([]*TomlTree)
+	} else {
+		p.raiseError(key, "key %s is already assigned and not of type group array", key)
+	}
+	p.currentGroup = keys
 
 	// add a new tree to the end of the group array
-	new_tree := make(TomlTree)
+	newTree := newTomlTree()
-	array = append(array, &new_tree)
+	newTree.position = startToken.Position
+	array = append(array, newTree)
 	p.tree.SetPath(p.currentGroup, array)
 
+	// remove all keys that were children of this group array
+	prefix := key.val + "."
+	found := false
+	for ii := 0; ii < len(p.seenGroupKeys); {
+		groupKey := p.seenGroupKeys[ii]
+		if strings.HasPrefix(groupKey, prefix) {
+			p.seenGroupKeys = append(p.seenGroupKeys[:ii], p.seenGroupKeys[ii+1:]...)
+		} else {
+			found = (groupKey == key.val)
+			ii++
+		}
+	}
+
 	// keep this key name from use by other kinds of assignments
-	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	if !found {
+		p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	}
 
 	// move to next parser state
 	p.assume(tokenDoubleRightBracket)
-	return parseStart(p)
+	return p.parseStart
 }
 
-func parseGroup(p *parser) parserStateFn {
+func (p *tomlParser) parseGroup() tomlParserStateFn {
-	p.getToken() // discard the [
+	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
 	}
 	for _, item := range p.seenGroupKeys {
 		if item == key.val {
-			panic("duplicated tables")
+			p.raiseError(key, "duplicated tables")
 		}
 	}
+
 	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
-	p.tree.createSubTree(key.val)
+	keys, err := parseKey(key.val)
+	if err != nil {
+		p.raiseError(key, "invalid group array key: %s", err)
+	}
+	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
+		p.raiseError(key, "%s", err)
+	}
 	p.assume(tokenRightBracket)
-	p.currentGroup = strings.Split(key.val, ".")
+	p.currentGroup = keys
-	return parseStart(p)
+	return p.parseStart
 }
 
-func parseAssign(p *parser) parserStateFn {
+func (p *tomlParser) parseAssign() tomlParserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
-	value := parseRvalue(p)
+
-	var group_key []string
+	value := p.parseRvalue()
+	var groupKey []string
 	if len(p.currentGroup) > 0 {
-		group_key = p.currentGroup
+		groupKey = p.currentGroup
 	} else {
-		group_key = make([]string, 0)
+		groupKey = []string{}
 	}
 
 	// find the group to assign, looking out for arrays of groups
-	var target_node *TomlTree
+	var targetNode *TomlTree
-	switch node := p.tree.GetPath(group_key).(type) {
+	switch node := p.tree.GetPath(groupKey).(type) {
 	case []*TomlTree:
-		target_node = node[len(node)-1]
+		targetNode = node[len(node)-1]
 	case *TomlTree:
-		target_node = node
+		targetNode = node
 	default:
-		panic(fmt.Sprintf("Unknown group type for path %v", group_key))
+		p.raiseError(key, "Unknown group type for path: %s",
+			strings.Join(groupKey, "."))
 	}
 
 	// assign value to the found group
-	local_key := []string{key.val}
+	keyVals, err := parseKey(key.val)
-	final_key := append(group_key, key.val)
+	if err != nil {
-	if target_node.GetPath(local_key) != nil {
+		p.raiseError(key, "%s", err)
-		panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
 	}
-	target_node.SetPath(local_key, value)
+	if len(keyVals) != 1 {
-	return parseStart(p)
+		p.raiseError(key, "Invalid key")
+	}
+	keyVal := keyVals[0]
+	localKey := []string{keyVal}
+	finalKey := append(groupKey, keyVal)
+	if targetNode.GetPath(localKey) != nil {
+		p.raiseError(key, "The following key was defined twice: %s",
+			strings.Join(finalKey, "."))
+	}
+	targetNode.values[keyVal] = &tomlValue{value, key.Position}
+	return p.parseStart
 }
 
-func parseRvalue(p *parser) interface{} {
+var numberUnderscoreInvalidRegexp *regexp.Regexp
+
+func cleanupNumberToken(value string) (string, error) {
+	if numberUnderscoreInvalidRegexp.MatchString(value) {
+		return "", fmt.Errorf("invalid use of _ in number")
+	}
+	cleanedVal := strings.Replace(value, "_", "", -1)
+	return cleanedVal, nil
+}
+
+func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
-		panic("expecting a value")
+		p.raiseError(tok, "expecting a value")
 	}
 
 	switch tok.typ {
@@ -181,77 +236,149 @@ func parseRvalue(p *parser) interface{} {
 	case tokenFalse:
 		return false
 	case tokenInteger:
-		val, err := strconv.ParseInt(tok.val, 10, 64)
+		cleanedVal, err := cleanupNumberToken(tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
+		}
+		val, err := strconv.ParseInt(cleanedVal, 10, 64)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		val, err := strconv.ParseFloat(tok.val, 64)
+		cleanedVal, err := cleanupNumberToken(tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
+		}
+		val, err := strconv.ParseFloat(cleanedVal, 64)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenDate:
-		val, err := time.Parse(time.RFC3339, tok.val)
+		val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenLeftBracket:
-		return parseArray(p)
+		return p.parseArray()
+	case tokenLeftCurlyBrace:
+		return p.parseInlineTable()
+	case tokenEqual:
+		p.raiseError(tok, "cannot have multiple equals for the same key")
 	case tokenError:
-		panic(tok.val)
+		p.raiseError(tok, "%s", tok)
 	}
 
-	panic("never reached")
+	p.raiseError(tok, "never reached")
 
 	return nil
 }
 
-func parseArray(p *parser) []interface{} {
+func tokenIsComma(t *token) bool {
-	array := make([]interface{}, 0)
+	return t != nil && t.typ == tokenComma
+}
+
+func (p *tomlParser) parseInlineTable() *TomlTree {
+	tree := newTomlTree()
+	var previous *token
+Loop:
+	for {
+		follow := p.peek()
+		if follow == nil || follow.typ == tokenEOF {
+			p.raiseError(follow, "unterminated inline table")
+		}
+		switch follow.typ {
+		case tokenRightCurlyBrace:
+			p.getToken()
+			break Loop
+		case tokenKey:
+			if !tokenIsComma(previous) && previous != nil {
+				p.raiseError(follow, "comma expected between fields in inline table")
+			}
+			key := p.getToken()
+			p.assume(tokenEqual)
+			value := p.parseRvalue()
+			tree.Set(key.val, value)
+		case tokenComma:
+			if previous == nil {
+				p.raiseError(follow, "inline table cannot start with a comma")
+			}
+			if tokenIsComma(previous) {
+				p.raiseError(follow, "need field between two commas in inline table")
+			}
+			p.getToken()
+		default:
+			p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String())
+		}
+		previous = follow
+	}
+	if tokenIsComma(previous) {
+		p.raiseError(previous, "trailing comma at the end of inline table")
+	}
+	return tree
+}
+
+func (p *tomlParser) parseArray() interface{} {
+	var array []interface{}
 	arrayType := reflect.TypeOf(nil)
 	for {
 		follow := p.peek()
 		if follow == nil || follow.typ == tokenEOF {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ == tokenRightBracket {
 			p.getToken()
-			return array
+			break
 		}
-		val := parseRvalue(p)
+		val := p.parseRvalue()
 		if arrayType == nil {
 			arrayType = reflect.TypeOf(val)
 		}
 		if reflect.TypeOf(val) != arrayType {
-			panic("mixed types in array")
+			p.raiseError(follow, "mixed types in array")
 		}
 		array = append(array, val)
 		follow = p.peek()
-		if follow == nil {
+		if follow == nil || follow.typ == tokenEOF {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
-			panic("missing comma")
+			p.raiseError(follow, "missing comma")
 		}
 		if follow.typ == tokenComma {
 			p.getToken()
 		}
 	}
+	// An array of TomlTrees is actually an array of inline
+	// tables, which is a shorthand for a table array. If the
+	// array was not converted from []interface{} to []*TomlTree,
+	// the two notations would not be equivalent.
+	if arrayType == reflect.TypeOf(newTomlTree()) {
+		tomlArray := make([]*TomlTree, len(array))
+		for i, v := range array {
+			tomlArray[i] = v.(*TomlTree)
+		}
+		return tomlArray
+	}
 	return array
 }
 
-func parse(flow chan token) *TomlTree {
+func parseToml(flow chan token) *TomlTree {
-	result := make(TomlTree)
+	result := newTomlTree()
-	parser := &parser{
+	result.position = Position{1, 1}
+	parser := &tomlParser{
 		flow:          flow,
-		tree:          &result,
+		tree:          result,
 		tokensBuffer:  make([]token, 0),
 		currentGroup:  make([]string, 0),
 		seenGroupKeys: make([]string, 0),
 	}
 	parser.run()
-	return parser.tree
+	return result
+}
+
+func init() {
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`)
 }

parser_test.go

@@ -12,14 +12,15 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 		return
 	}
 	for k, v := range ref {
-		node := tree.Get(k)
+		// NOTE: directly access key instead of resolve by path
-		switch cast_node := node.(type) {
+		// NOTE: see TestSpecialKV
+		switch node := tree.GetPath([]string{k}).(type) {
 		case []*TomlTree:
-			for idx, item := range cast_node {
+			for idx, item := range node {
 				assertTree(t, item, err, v.([]map[string]interface{})[idx])
 			}
 		case *TomlTree:
-			assertTree(t, cast_node, err, v.(map[string]interface{}))
+			assertTree(t, node, err, v.(map[string]interface{}))
 		default:
 			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
 				t.Errorf("was expecting %v at %v but got %v", v, k, node)
@@ -29,8 +30,8 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 }
 
 func TestCreateSubTree(t *testing.T) {
-	tree := make(TomlTree)
+	tree := newTomlTree()
-	tree.createSubTree("a.b.c")
+	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
 		t.Fail()
@@ -50,6 +51,13 @@ func TestSimpleKV(t *testing.T) {
 	})
 }
 
+func TestNumberInKey(t *testing.T) {
+	tree, err := Load("hello2 = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"hello2": int64(42),
+	})
+}
+
 func TestSimpleNumbers(t *testing.T) {
 	tree, err := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -60,6 +68,44 @@ func TestSimpleNumbers(t *testing.T) {
 	})
 }
 
+func TestNumbersWithUnderscores(t *testing.T) {
+	tree, err := Load("a = 1_000")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(1000),
+	})
+
+	tree, err = Load("a = 5_349_221")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(5349221),
+	})
+
+	tree, err = Load("a = 1_2_3_4_5")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": int64(12345),
+	})
+
+	tree, err = Load("flt8 = 9_224_617.445_991_228_313")
+	assertTree(t, tree, err, map[string]interface{}{
+		"flt8": float64(9224617.445991228313),
+	})
+
+	tree, err = Load("flt9 = 1e1_00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"flt9": float64(1e100),
+	})
+}
+
+func TestFloatsWithExponents(t *testing.T) {
+	tree, err := Load("a = 5e+22\nb = 5E+22\nc = -5e+22\nd = -5e-22\ne = 6.626e-34")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": float64(5e+22),
+		"b": float64(5E+22),
+		"c": float64(-5e+22),
+		"d": float64(-5e-22),
+		"e": float64(6.626e-34),
+	})
+}
+
 func TestSimpleDate(t *testing.T) {
 	tree, err := Load("a = 1979-05-27T07:32:00Z")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -67,6 +113,20 @@ func TestSimpleDate(t *testing.T) {
 	})
 }
 
+func TestDateOffset(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)),
+	})
+}
+
+func TestDateNano(t *testing.T) {
+	tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)),
+	})
+}
+
 func TestSimpleString(t *testing.T) {
 	tree, err := Load("a = \"hello world\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -74,6 +134,13 @@ func TestSimpleString(t *testing.T) {
 	})
 }
 
+func TestSpaceKey(t *testing.T) {
+	tree, err := Load("\"a b\" = \"hello world\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a b": "hello world",
+	})
+}
+
 func TestStringEscapables(t *testing.T) {
 	tree, err := Load("a = \"a \\n b\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -107,7 +174,48 @@ func TestBools(t *testing.T) {
 func TestNestedKeys(t *testing.T) {
 	tree, err := Load("[a.b.c]\nd = 42")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a.b.c.d": int64(42),
+		"a": map[string]interface{}{
+			"b": map[string]interface{}{
+				"c": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+}
+
+func TestNestedQuotedUnicodeKeys(t *testing.T) {
+	tree, err := Load("[ j . \"ʞ\" . l ]\nd = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"j": map[string]interface{}{
+			"ʞ": map[string]interface{}{
+				"l": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+
+	tree, err = Load("[ g . h . i ]\nd = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"g": map[string]interface{}{
+			"h": map[string]interface{}{
+				"i": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
+	})
+
+	tree, err = Load("[ d.e.f ]\nk = 42")
+	assertTree(t, tree, err, map[string]interface{}{
+		"d": map[string]interface{}{
+			"e": map[string]interface{}{
+				"f": map[string]interface{}{
+					"k": int64(42),
+				},
+			},
+		},
 	})
 }
 
@@ -160,12 +268,12 @@ func TestNestedEmptyArrays(t *testing.T) {
 
 func TestArrayMixedTypes(t *testing.T) {
 	_, err := Load("a = [42, 16.0]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 10): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = [42, \"hello\"]")
-	if err.Error() != "mixed types in array" {
+	if err.Error() != "(1, 11): mixed types in array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -179,14 +287,24 @@ func TestArrayNestedStrings(t *testing.T) {
 
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "expecting a value" {
+	if err.Error() != "(1, 5): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestUnterminatedArray(t *testing.T) {
 	_, err := Load("a = [1,")
-	if err.Error() != "unterminated array" {
+	if err.Error() != "(1, 8): unterminated array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [1")
+	if err.Error() != "(1, 7): unterminated array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [1 2")
+	if err.Error() != "(1, 8): missing comma" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -212,23 +330,97 @@ func TestArrayWithExtraCommaComment(t *testing.T) {
 	})
 }
 
+func TestSimpleInlineGroup(t *testing.T) {
+	tree, err := Load("key = {a = 42}")
+	assertTree(t, tree, err, map[string]interface{}{
+		"key": map[string]interface{}{
+			"a": int64(42),
+		},
+	})
+}
+
+func TestDoubleInlineGroup(t *testing.T) {
+	tree, err := Load("key = {a = 42, b = \"foo\"}")
+	assertTree(t, tree, err, map[string]interface{}{
+		"key": map[string]interface{}{
+			"a": int64(42),
+			"b": "foo",
+		},
+	})
+}
+
+func TestExampleInlineGroup(t *testing.T) {
+	tree, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"point": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func TestExampleInlineGroupInArray(t *testing.T) {
+	tree, err := Load(`points = [{ x = 1, y = 2 }]`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"points": []map[string]interface{}{
+			map[string]interface{}{
+				"x": int64(1),
+				"y": int64(2),
+			},
+		},
+	})
+}
+
+func TestInlineTableUnterminated(t *testing.T) {
+	_, err := Load("foo = {")
+	if err.Error() != "(1, 8): unterminated inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableCommaExpected(t *testing.T) {
+	_, err := Load("foo = {hello = 53 test = foo}")
+	if err.Error() != "(1, 19): comma expected between fields in inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableCommaStart(t *testing.T) {
+	_, err := Load("foo = {, hello = 53}")
+	if err.Error() != "(1, 8): inline table cannot start with a comma" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInlineTableDoubleComma(t *testing.T) {
+	_, err := Load("foo = {hello = 53,, foo = 17}")
+	if err.Error() != "(1, 19): need field between two commas in inline table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestDuplicateGroups(t *testing.T) {
 	_, err := Load("[foo]\na=2\n[foo]b=3")
-	if err.Error() != "duplicated tables" {
+	if err.Error() != "(3, 2): duplicated tables" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestDuplicateKeys(t *testing.T) {
 	_, err := Load("foo = 2\nfoo = 3")
-	if err.Error() != "the following key was defined twice: foo" {
+	if err.Error() != "(2, 1): The following key was defined twice: foo" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "empty intermediate table" {
+	if err.Error() != "(1, 2): empty intermediate table" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -249,12 +441,12 @@ func TestImplicitDeclarationBefore(t *testing.T) {
 
 func TestFloatsWithoutLeadingZeros(t *testing.T) {
 	_, err := Load("a = .42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("a = -.42")
-	if err.Error() != "cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -270,20 +462,71 @@ func TestParseFile(t *testing.T) {
 	tree, err := LoadFile("example.toml")
 
 	assertTree(t, tree, err, map[string]interface{}{
-		"title":                   "TOML Example",
+		"title": "TOML Example",
-		"owner.name":              "Tom Preston-Werner",
+		"owner": map[string]interface{}{
-		"owner.organization":      "GitHub",
+			"name":         "Tom Preston-Werner",
-		"owner.bio":               "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"organization": "GitHub",
-		"owner.dob":               time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
-		"database.server":         "192.168.1.1",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
-		"database.ports":          []int64{8001, 8001, 8002},
+		},
-		"database.connection_max": 5000,
+		"database": map[string]interface{}{
-		"database.enabled":        true,
+			"server":         "192.168.1.1",
-		"servers.alpha.ip":        "10.0.0.1",
+			"ports":          []int64{8001, 8001, 8002},
-		"servers.alpha.dc":        "eqdc10",
+			"connection_max": 5000,
-		"servers.beta.ip":         "10.0.0.2",
+			"enabled":        true,
-		"servers.beta.dc":         "eqdc10",
+		},
-		"clients.data":            []interface{}{[]string{"gamma", "delta"}, []int64{1, 2}},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
+	})
+}
+
+func TestParseFileCRLF(t *testing.T) {
+	tree, err := LoadFile("example-crlf.toml")
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
 	})
 }
 
@@ -299,6 +542,16 @@ func TestParseKeyGroupArray(t *testing.T) {
 	})
 }
 
+func TestParseKeyGroupArraySpec(t *testing.T) {
+	tree, err := Load("[[fruit]]\n name=\"apple\"\n [fruit.physical]\n color=\"red\"\n shape=\"round\"\n [[fruit]]\n name=\"banana\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"fruit": []map[string]interface{}{
+			{"name": "apple", "physical": map[string]interface{}{"color": "red", "shape": "round"}},
+			{"name": "banana"},
+		},
+	})
+}
+
 func TestToTomlValue(t *testing.T) {
 	for idx, item := range []struct {
 		Value  interface{}
@@ -323,17 +576,118 @@ func TestToTomlValue(t *testing.T) {
 }
 
 func TestToString(t *testing.T) {
-	tree := &TomlTree{
+	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
-		"foo": &TomlTree{
+	if err != nil {
-			"bar": []*TomlTree{
+		t.Errorf("Test failed to parse: %v", err)
-				{"a": int64(42)},
+		return
-				{"a": int64(69)},
-			},
-		},
 	}
 	result := tree.ToString()
-	expected := "\n[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n"
+	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
 	if result != expected {
 		t.Errorf("Expected got '%s', expected '%s'", result, expected)
 	}
 }
+
+func assertPosition(t *testing.T, text string, ref map[string]Position) {
+	tree, err := Load(text)
+	if err != nil {
+		t.Errorf("Error loading document text: `%v`", text)
+		t.Errorf("Error: %v", err)
+	}
+	for path, pos := range ref {
+		testPos := tree.GetPosition(path)
+		if testPos.Invalid() {
+			t.Errorf("Failed to query tree path or path has invalid position: %s", path)
+		} else if pos != testPos {
+			t.Errorf("Expected position %v, got %v instead", pos, testPos)
+		}
+	}
+}
+
+func TestDocumentPositions(t *testing.T) {
+	assertPosition(t,
+		"[foo]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 1},
+			"foo.bar": Position{2, 1},
+			"foo.baz": Position{3, 1},
+		})
+}
+
+func TestDocumentPositionsWithSpaces(t *testing.T) {
+	assertPosition(t,
+		"  [foo]\n  bar=42\n  baz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 3},
+			"foo.bar": Position{2, 3},
+			"foo.baz": Position{3, 3},
+		})
+}
+
+func TestDocumentPositionsWithGroupArray(t *testing.T) {
+	assertPosition(t,
+		"[[foo]]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 1},
+			"foo.bar": Position{2, 1},
+			"foo.baz": Position{3, 1},
+		})
+}
+
+func TestNestedTreePosition(t *testing.T) {
+	assertPosition(t,
+		"[foo.bar]\na=42\nb=69",
+		map[string]Position{
+			"":          Position{1, 1},
+			"foo":       Position{1, 1},
+			"foo.bar":   Position{1, 1},
+			"foo.bar.a": Position{2, 1},
+			"foo.bar.b": Position{3, 1},
+		})
+}
+
+func TestInvalidGroupArray(t *testing.T) {
+	_, err := Load("[key#group]\nanswer = 42")
+	if err == nil {
+		t.Error("Should error")
+	}
+}
+
+func TestDoubleEqual(t *testing.T) {
+	_, err := Load("foo= = 2")
+	if err.Error() != "(1, 6): cannot have multiple equals for the same key" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestGroupArrayReassign(t *testing.T) {
+	_, err := Load("[hello]\n[[hello]]")
+	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type group array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInvalidFloatParsing(t *testing.T) {
+	_, err := Load("a=1e_2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1e2_")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1__2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=_1_2")
+	if err.Error() != "(1, 3): cannot start number with underscore" {
+		t.Error("Bad error message:", err.Error())
+	}
+}

position.go

@@ -0,0 +1,29 @@
+// Position support for go-toml
+
+package toml
+
+import (
+	"fmt"
+)
+
+// Position of a document element within a TOML document.
+//
+// Line and Col are both 1-indexed positions for the element's line number and
+// column number, respectively.  Values of zero or less will cause Invalid(),
+// to return true.
+type Position struct {
+	Line int // line within the document
+	Col  int // column within the line
+}
+
+// String representation of the position.
+// Displays 1-indexed line and column numbers.
+func (p *Position) String() string {
+	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
+}
+
+// Invalid returns whether or not the position is valid (i.e. with negative or
+// null values)
+func (p *Position) Invalid() bool {
+	return p.Line <= 0 || p.Col <= 0
+}

position_test.go

@@ -0,0 +1,29 @@
+// Testing support for go-toml
+
+package toml
+
+import (
+	"testing"
+)
+
+func TestPositionString(t *testing.T) {
+	p := Position{123, 456}
+	expected := "(123, 456)"
+	value := p.String()
+
+	if value != expected {
+		t.Errorf("Expected %v, got %v instead", expected, value)
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	for i, v := range []Position{
+		Position{0, 1234},
+		Position{1234, 0},
+		Position{0, 0},
+	} {
+		if !v.Invalid() {
+			t.Errorf("Position at %v is valid: %v", i, v)
+		}
+	}
+}

query.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"time"
+)
+
+//  Type of a user-defined filter function, for use with Query.SetFilter().
+//
+//  The return value of the function must indicate if 'node' is to be included
+//  at this stage of the TOML path.  Returning true will include the node, and
+//  returning false will exclude it.
+//
+//  NOTE: Care should be taken to write script callbacks such that they are safe
+//  to use from multiple goroutines.
+type NodeFilterFn func(node interface{}) bool
+
+// The result of Executing a Query
+type QueryResult struct {
+	items     []interface{}
+	positions []Position
+}
+
+// appends a value/position pair to the result set
+func (r *QueryResult) appendResult(node interface{}, pos Position) {
+	r.items = append(r.items, node)
+	r.positions = append(r.positions, pos)
+}
+
+// Set of values within a QueryResult.  The order of values is not guaranteed
+// to be in document order, and may be different each time a query is executed.
+func (r *QueryResult) Values() []interface{} {
+	return r.items
+}
+
+// Set of positions for values within a QueryResult.  Each index in Positions()
+// corresponds to the entry in Value() of the same index.
+func (r *QueryResult) Positions() []Position {
+	return r.positions
+}
+
+// runtime context for executing query paths
+type queryContext struct {
+	result       *QueryResult
+	filters      *map[string]NodeFilterFn
+	lastPosition Position
+}
+
+// generic path functor interface
+type pathFn interface {
+	setNext(next pathFn)
+	call(node interface{}, ctx *queryContext)
+}
+
+// A Query is the representation of a compiled TOML path.  A Query is safe
+// for concurrent use by multiple goroutines.
+type Query struct {
+	root    pathFn
+	tail    pathFn
+	filters *map[string]NodeFilterFn
+}
+
+func newQuery() *Query {
+	return &Query{
+		root:    nil,
+		tail:    nil,
+		filters: &defaultFilterFunctions,
+	}
+}
+
+func (q *Query) appendPath(next pathFn) {
+	if q.root == nil {
+		q.root = next
+	} else {
+		q.tail.setNext(next)
+	}
+	q.tail = next
+	next.setNext(newTerminatingFn()) // init the next functor
+}
+
+// Compiles a TOML path expression.  The returned Query can be used to match
+// elements within a TomlTree and its descendants.
+func CompileQuery(path string) (*Query, error) {
+	return parseQuery(lexQuery(path))
+}
+
+// Executes a query against a TomlTree, and returns the result of the query.
+func (q *Query) Execute(tree *TomlTree) *QueryResult {
+	result := &QueryResult{
+		items:     []interface{}{},
+		positions: []Position{},
+	}
+	if q.root == nil {
+		result.appendResult(tree, tree.GetPosition(""))
+	} else {
+		ctx := &queryContext{
+			result:  result,
+			filters: q.filters,
+		}
+		q.root.call(tree, ctx)
+	}
+	return result
+}
+
+// Sets a user-defined filter function.  These may be used inside "?(..)" query
+// expressions to filter TOML document elements within a query.
+func (q *Query) SetFilter(name string, fn NodeFilterFn) {
+	if q.filters == &defaultFilterFunctions {
+		// clone the static table
+		q.filters = &map[string]NodeFilterFn{}
+		for k, v := range defaultFilterFunctions {
+			(*q.filters)[k] = v
+		}
+	}
+	(*q.filters)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]NodeFilterFn{
+	"tree": func(node interface{}) bool {
+		_, ok := node.(*TomlTree)
+		return ok
+	},
+	"int": func(node interface{}) bool {
+		_, ok := node.(int64)
+		return ok
+	},
+	"float": func(node interface{}) bool {
+		_, ok := node.(float64)
+		return ok
+	},
+	"string": func(node interface{}) bool {
+		_, ok := node.(string)
+		return ok
+	},
+	"time": func(node interface{}) bool {
+		_, ok := node.(time.Time)
+		return ok
+	},
+	"bool": func(node interface{}) bool {
+		_, ok := node.(bool)
+		return ok
+	},
+}

querylexer.go

@@ -0,0 +1,339 @@
+// TOML JSONPath lexer.
+//
+// Written using the principles developed by Rob Pike in
+// http://www.youtube.com/watch?v=HxaD_trXwRE
+
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// Lexer state function
+type queryLexStateFn func() queryLexStateFn
+
+// Lexer definition
+type queryLexer struct {
+	input      string
+	start      int
+	pos        int
+	width      int
+	tokens     chan token
+	depth      int
+	line       int
+	col        int
+	stringTerm string
+}
+
+func (l *queryLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
+}
+
+func (l *queryLexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line++
+			l.col = 1
+		} else {
+			l.col++
+		}
+		i += width
+	}
+	// advance start position to next token
+	l.start = l.pos
+}
+
+func (l *queryLexer) emit(t tokenType) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      l.input[l.start:l.pos],
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) emitWithValue(t tokenType, value string) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) next() rune {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	var r rune
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *queryLexer) ignore() {
+	l.nextStart()
+}
+
+func (l *queryLexer) backup() {
+	l.pos -= l.width
+}
+
+func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func (l *queryLexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *queryLexer) accept(valid string) bool {
+	if strings.IndexRune(valid, l.next()) >= 0 {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+func (l *queryLexer) follow(next string) bool {
+	return strings.HasPrefix(l.input[l.pos:], next)
+}
+
+func (l *queryLexer) lexVoid() queryLexStateFn {
+	for {
+		next := l.peek()
+		switch next {
+		case '$':
+			l.pos++
+			l.emit(tokenDollar)
+			continue
+		case '.':
+			if l.follow("..") {
+				l.pos += 2
+				l.emit(tokenDotDot)
+			} else {
+				l.pos++
+				l.emit(tokenDot)
+			}
+			continue
+		case '[':
+			l.pos++
+			l.emit(tokenLeftBracket)
+			continue
+		case ']':
+			l.pos++
+			l.emit(tokenRightBracket)
+			continue
+		case ',':
+			l.pos++
+			l.emit(tokenComma)
+			continue
+		case '*':
+			l.pos++
+			l.emit(tokenStar)
+			continue
+		case '(':
+			l.pos++
+			l.emit(tokenLeftParen)
+			continue
+		case ')':
+			l.pos++
+			l.emit(tokenRightParen)
+			continue
+		case '?':
+			l.pos++
+			l.emit(tokenQuestion)
+			continue
+		case ':':
+			l.pos++
+			l.emit(tokenColon)
+			continue
+		case '\'':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		case '"':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		}
+
+		if isSpace(next) {
+			l.next()
+			l.ignore()
+			continue
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if l.next() == eof {
+			break
+		}
+
+		return l.errorf("unexpected char: '%v'", next)
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexKey() queryLexStateFn {
+	for {
+		next := l.peek()
+		if !isAlphanumeric(next) {
+			l.emit(tokenKey)
+			return l.lexVoid
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexString() queryLexStateFn {
+	l.pos++
+	l.ignore()
+	growingString := ""
+
+	for {
+		if l.follow(l.stringTerm) {
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
+			l.ignore()
+			return l.lexVoid
+		}
+
+		if l.follow("\\\"") {
+			l.pos++
+			growingString += "\""
+		} else if l.follow("\\'") {
+			l.pos++
+			growingString += "'"
+		} else if l.follow("\\n") {
+			l.pos++
+			growingString += "\n"
+		} else if l.follow("\\b") {
+			l.pos++
+			growingString += "\b"
+		} else if l.follow("\\f") {
+			l.pos++
+			growingString += "\f"
+		} else if l.follow("\\/") {
+			l.pos++
+			growingString += "/"
+		} else if l.follow("\\t") {
+			l.pos++
+			growingString += "\t"
+		} else if l.follow("\\r") {
+			l.pos++
+			growingString += "\r"
+		} else if l.follow("\\\\") {
+			l.pos++
+			growingString += "\\"
+		} else if l.follow("\\u") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 4; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos++
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+		} else {
+			growingString += string(l.peek())
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+
+	return l.errorf("unclosed string")
+}
+
+func (l *queryLexer) lexNumber() queryLexStateFn {
+	l.ignore()
+	if !l.accept("+") {
+		l.accept("-")
+	}
+	pointSeen := false
+	digitSeen := false
+	for {
+		next := l.next()
+		if next == '.' {
+			if pointSeen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
+			pointSeen = true
+		} else if isDigit(next) {
+			digitSeen = true
+		} else {
+			l.backup()
+			break
+		}
+		if pointSeen && !digitSeen {
+			return l.errorf("cannot start float with a dot")
+		}
+	}
+
+	if !digitSeen {
+		return l.errorf("no digit in that number")
+	}
+	if pointSeen {
+		l.emit(tokenFloat)
+	} else {
+		l.emit(tokenInteger)
+	}
+	return l.lexVoid
+}
+
+// Entry point
+func lexQuery(input string) chan token {
+	l := &queryLexer{
+		input:  input,
+		tokens: make(chan token),
+		line:   1,
+		col:    1,
+	}
+	go l.run()
+	return l.tokens
+}

querylexer_test.go

@@ -0,0 +1,97 @@
+package toml
+
+import (
+	"testing"
+)
+
+func testQLFlow(t *testing.T, input string, expectedFlow []token) {
+	ch := lexQuery(input)
+	for idx, expected := range expectedFlow {
+		token := <-ch
+		if token != expected {
+			t.Log("While testing #", idx, ":", input)
+			t.Log("compared", token, "to", expected)
+			t.Log(token.val, "<->", expected.val)
+			t.Log(token.typ, "<->", expected.typ)
+			t.Log(token.Line, "<->", expected.Line)
+			t.Log(token.Col, "<->", expected.Col)
+			t.FailNow()
+		}
+	}
+
+	tok, ok := <-ch
+	if ok {
+		t.Log("channel is not closed!")
+		t.Log(len(ch)+1, "tokens remaining:")
+
+		t.Log("token ->", tok)
+		for token := range ch {
+			t.Log("token ->", token)
+		}
+		t.FailNow()
+	}
+}
+
+func TestLexSpecialChars(t *testing.T) {
+	testQLFlow(t, " .$[]..()?*", []token{
+		token{Position{1, 2}, tokenDot, "."},
+		token{Position{1, 3}, tokenDollar, "$"},
+		token{Position{1, 4}, tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenRightBracket, "]"},
+		token{Position{1, 6}, tokenDotDot, ".."},
+		token{Position{1, 8}, tokenLeftParen, "("},
+		token{Position{1, 9}, tokenRightParen, ")"},
+		token{Position{1, 10}, tokenQuestion, "?"},
+		token{Position{1, 11}, tokenStar, "*"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexString(t *testing.T) {
+	testQLFlow(t, "'foo'", []token{
+		token{Position{1, 2}, tokenString, "foo"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexDoubleString(t *testing.T) {
+	testQLFlow(t, `"bar"`, []token{
+		token{Position{1, 2}, tokenString, "bar"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexKey(t *testing.T) {
+	testQLFlow(t, "foo", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 4}, tokenEOF, ""},
+	})
+}
+
+func TestLexRecurse(t *testing.T) {
+	testQLFlow(t, "$..*", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenDotDot, ".."},
+		token{Position{1, 4}, tokenStar, "*"},
+		token{Position{1, 5}, tokenEOF, ""},
+	})
+}
+
+func TestLexBracketKey(t *testing.T) {
+	testQLFlow(t, "$[foo]", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenLeftBracket, "["},
+		token{Position{1, 3}, tokenKey, "foo"},
+		token{Position{1, 6}, tokenRightBracket, "]"},
+		token{Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexSpace(t *testing.T) {
+	testQLFlow(t, "foo bar baz", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenKey, "bar"},
+		token{Position{1, 9}, tokenKey, "baz"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}

queryparser.go

@@ -0,0 +1,275 @@
+/*
+  Based on the "jsonpath" spec/concept.
+
+  http://goessner.net/articles/JsonPath/
+  https://code.google.com/p/json-path/
+*/
+
+package toml
+
+import (
+	"fmt"
+)
+
+const MaxInt = int(^uint(0) >> 1)
+
+type queryParser struct {
+	flow         chan token
+	tokensBuffer []token
+	query        *Query
+	union        []pathFn
+	err          error
+}
+
+type queryParserStateFn func() queryParserStateFn
+
+// Formats and panics an error message based on a token
+func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn {
+	p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...)
+	return nil // trigger parse to end
+}
+
+func (p *queryParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
+	}
+}
+
+func (p *queryParser) backup(tok *token) {
+	p.tokensBuffer = append(p.tokensBuffer, *tok)
+}
+
+func (p *queryParser) peek() *token {
+	if len(p.tokensBuffer) != 0 {
+		return &(p.tokensBuffer[0])
+	}
+
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	p.backup(&tok)
+	return &tok
+}
+
+func (p *queryParser) lookahead(types ...tokenType) bool {
+	result := true
+	buffer := []token{}
+
+	for _, typ := range types {
+		tok := p.getToken()
+		if tok == nil {
+			result = false
+			break
+		}
+		buffer = append(buffer, *tok)
+		if tok.typ != typ {
+			result = false
+			break
+		}
+	}
+	// add the tokens back to the buffer, and return
+	p.tokensBuffer = append(p.tokensBuffer, buffer...)
+	return result
+}
+
+func (p *queryParser) getToken() *token {
+	if len(p.tokensBuffer) != 0 {
+		tok := p.tokensBuffer[0]
+		p.tokensBuffer = p.tokensBuffer[1:]
+		return &tok
+	}
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	return &tok
+}
+
+func (p *queryParser) parseStart() queryParserStateFn {
+	tok := p.getToken()
+
+	if tok == nil || tok.typ == tokenEOF {
+		return nil
+	}
+
+	if tok.typ != tokenDollar {
+		return p.parseError(tok, "Expected '$' at start of expression")
+	}
+
+	return p.parseMatchExpr
+}
+
+// handle '.' prefix, '[]', and '..'
+func (p *queryParser) parseMatchExpr() queryParserStateFn {
+	tok := p.getToken()
+	switch tok.typ {
+	case tokenDotDot:
+		p.query.appendPath(&matchRecursiveFn{})
+		// nested parse for '..'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenLeftBracket:
+			return p.parseBracketExpr
+		case tokenStar:
+			// do nothing - the recursive predicate is enough
+			return p.parseMatchExpr
+		}
+
+	case tokenDot:
+		// nested parse for '.'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenStar:
+			p.query.appendPath(&matchAnyFn{})
+			return p.parseMatchExpr
+		}
+
+	case tokenLeftBracket:
+		return p.parseBracketExpr
+
+	case tokenEOF:
+		return nil // allow EOF at this stage
+	}
+	return p.parseError(tok, "expected match expression")
+}
+
+func (p *queryParser) parseBracketExpr() queryParserStateFn {
+	if p.lookahead(tokenInteger, tokenColon) {
+		return p.parseSliceExpr
+	}
+	if p.peek().typ == tokenColon {
+		return p.parseSliceExpr
+	}
+	return p.parseUnionExpr
+}
+
+func (p *queryParser) parseUnionExpr() queryParserStateFn {
+	var tok *token
+
+	// this state can be traversed after some sub-expressions
+	// so be careful when setting up state in the parser
+	if p.union == nil {
+		p.union = []pathFn{}
+	}
+
+loop: // labeled loop for easy breaking
+	for {
+		if len(p.union) > 0 {
+			// parse delimiter or terminator
+			tok = p.getToken()
+			switch tok.typ {
+			case tokenComma:
+				// do nothing
+			case tokenRightBracket:
+				break loop
+			default:
+				return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val)
+			}
+		}
+
+		// parse sub expression
+		tok = p.getToken()
+		switch tok.typ {
+		case tokenInteger:
+			p.union = append(p.union, newMatchIndexFn(tok.Int()))
+		case tokenKey:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenString:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenQuestion:
+			return p.parseFilterExpr
+		default:
+			return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union))
+		}
+	}
+
+	// if there is only one sub-expression, use that instead
+	if len(p.union) == 1 {
+		p.query.appendPath(p.union[0])
+	} else {
+		p.query.appendPath(&matchUnionFn{p.union})
+	}
+
+	p.union = nil // clear out state
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseSliceExpr() queryParserStateFn {
+	// init slice to grab all elements
+	start, end, step := 0, MaxInt, 1
+
+	// parse optional start
+	tok := p.getToken()
+	if tok.typ == tokenInteger {
+		start = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ':'")
+	}
+
+	// parse optional end
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		end = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ == tokenRightBracket {
+		p.query.appendPath(newMatchSliceFn(start, end, step))
+		return p.parseMatchExpr
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ']' or ':'")
+	}
+
+	// parse optional step
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		step = tok.Int()
+		if step < 0 {
+			return p.parseError(tok, "step must be a positive value")
+		}
+		tok = p.getToken()
+	}
+	if tok.typ != tokenRightBracket {
+		return p.parseError(tok, "expected ']'")
+	}
+
+	p.query.appendPath(newMatchSliceFn(start, end, step))
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseFilterExpr() queryParserStateFn {
+	tok := p.getToken()
+	if tok.typ != tokenLeftParen {
+		return p.parseError(tok, "expected left-parenthesis for filter expression")
+	}
+	tok = p.getToken()
+	if tok.typ != tokenKey && tok.typ != tokenString {
+		return p.parseError(tok, "expected key or string for filter funciton name")
+	}
+	name := tok.val
+	tok = p.getToken()
+	if tok.typ != tokenRightParen {
+		return p.parseError(tok, "expected right-parenthesis for filter expression")
+	}
+	p.union = append(p.union, newMatchFilterFn(name, tok.Position))
+	return p.parseUnionExpr
+}
+
+func parseQuery(flow chan token) (*Query, error) {
+	parser := &queryParser{
+		flow:         flow,
+		tokensBuffer: []token{},
+		query:        newQuery(),
+	}
+	parser.run()
+	return parser.query, parser.err
+}

queryparser_test.go

@@ -0,0 +1,483 @@
+package toml
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+type queryTestNode struct {
+	value    interface{}
+	position Position
+}
+
+func valueString(root interface{}) string {
+	result := "" //fmt.Sprintf("%T:", root)
+	switch node := root.(type) {
+	case *tomlValue:
+		return valueString(node.value)
+	case *QueryResult:
+		items := []string{}
+		for i, v := range node.Values() {
+			items = append(items, fmt.Sprintf("%s:%s",
+				node.Positions()[i].String(), valueString(v)))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case queryTestNode:
+		result = fmt.Sprintf("%s:%s",
+			node.position.String(), valueString(node.value))
+	case []interface{}:
+		items := []string{}
+		for _, v := range node {
+			items = append(items, valueString(v))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case *TomlTree:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for _, k := range node.Keys() {
+			v := node.GetPath([]string{k})
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case map[string]interface{}:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for k, v := range node {
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case int64:
+		result += fmt.Sprintf("%d", node)
+	case string:
+		result += "'" + node + "'"
+	case float64:
+		result += fmt.Sprintf("%f", node)
+	case bool:
+		result += fmt.Sprintf("%t", node)
+	case time.Time:
+		result += fmt.Sprintf("'%v'", node)
+	}
+	return result
+}
+
+func assertValue(t *testing.T, result, ref interface{}) {
+	pathStr := valueString(result)
+	refStr := valueString(ref)
+	if pathStr != refStr {
+		t.Errorf("values do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+	}
+}
+
+func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
+	tree, err := Load(toml)
+	if err != nil {
+		t.Errorf("Non-nil toml parse error: %v", err)
+		return
+	}
+	q, err := CompileQuery(query)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	results := q.Execute(tree)
+	assertValue(t, results, ref)
+}
+
+func TestQueryRoot(t *testing.T) {
+	assertQueryPositions(t,
+		"a = 42",
+		"$",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(42),
+				}, Position{1, 1},
+			},
+		})
+}
+
+func TestQueryKey(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo.a",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryKeyString(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo['a']",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryIndex(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[5]",
+		[]interface{}{
+			queryTestNode{
+				int64(6), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceRange(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(4), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceStep(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5:2]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryAny(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[foo.baz]\na=3\nb=4",
+		"$.foo.*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+		})
+}
+func TestQueryUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$.*[bar,foo]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryRecursionAll(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": map[string]interface{}{
+							"a": int64(1),
+							"b": int64(2),
+						},
+					},
+					"baz": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(3),
+							"b": int64(4),
+						},
+					},
+					"gorf": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(5),
+							"b": int64(6),
+						},
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{3, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(3),
+						"b": int64(4),
+					},
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				int64(3), Position{5, 1},
+			},
+			queryTestNode{
+				int64(4), Position{6, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(5),
+						"b": int64(6),
+					},
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				int64(5), Position{8, 1},
+			},
+			queryTestNode{
+				int64(6), Position{9, 1},
+			},
+		})
+}
+
+func TestQueryRecursionUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..['foo','bar']",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryFilterFn(t *testing.T) {
+	buff, err := ioutil.ReadFile("example.toml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(int)]",
+		[]interface{}{
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8002), Position{13, 1},
+			},
+			queryTestNode{
+				int64(5000), Position{14, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(string)]",
+		[]interface{}{
+			queryTestNode{
+				"TOML Example", Position{3, 1},
+			},
+			queryTestNode{
+				"Tom Preston-Werner", Position{6, 1},
+			},
+			queryTestNode{
+				"GitHub", Position{7, 1},
+			},
+			queryTestNode{
+				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
+				Position{8, 1},
+			},
+			queryTestNode{
+				"192.168.1.1", Position{12, 1},
+			},
+			queryTestNode{
+				"10.0.0.1", Position{21, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{22, 3},
+			},
+			queryTestNode{
+				"10.0.0.2", Position{25, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{26, 3},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(float)]",
+		[]interface{}{
+		// no float values in document
+		})
+
+	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
+	assertQueryPositions(t, string(buff),
+		"$..[?(tree)]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"name":         "Tom Preston-Werner",
+					"organization": "GitHub",
+					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+					"dob":          tv,
+				}, Position{5, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"server":         "192.168.1.1",
+					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+					"connection_max": int64(5000),
+					"enabled":        true,
+				}, Position{11, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"alpha": map[string]interface{}{
+						"ip": "10.0.0.1",
+						"dc": "eqdc10",
+					},
+					"beta": map[string]interface{}{
+						"ip": "10.0.0.2",
+						"dc": "eqdc10",
+					},
+				}, Position{17, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.1",
+					"dc": "eqdc10",
+				}, Position{20, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.2",
+					"dc": "eqdc10",
+				}, Position{24, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"data": []interface{}{
+						[]interface{}{"gamma", "delta"},
+						[]interface{}{int64(1), int64(2)},
+					},
+				}, Position{28, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(time)]",
+		[]interface{}{
+			queryTestNode{
+				tv, Position{9, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(bool)]",
+		[]interface{}{
+			queryTestNode{
+				true, Position{15, 1},
+			},
+		})
+}

test.sh

@@ -5,15 +5,28 @@ set -e
 # set the path to the present working directory
 export GOPATH=`pwd`
 
-# Vendorize the BurntSushi test suite
+function git_clone() {
-# NOTE: this gets a specific release to avoid versioning issues
+  path=$1
-if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then
+  branch=$2
-  mkdir -p src/github.com/BurntSushi
+  version=$3
-  git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test
+  if [ ! -d "src/$path" ]; then
-fi
+    mkdir -p src/$path
-pushd src/github.com/BurntSushi/toml-test
+    git clone https://$path.git src/$path
-git reset --hard '0.2.0'  # use the released version, NOT tip
+  fi
-popd
+  pushd src/$path
+  git checkout "$branch"
+  git reset --hard "$version"
+  popd
+}
+
+go get github.com/pelletier/go-buffruneio
+
+# get code for BurntSushi TOML validation
+# pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)
+git_clone github.com/BurntSushi/toml master HEAD
+git_clone github.com/BurntSushi/toml-test master HEAD #was: 0.2.0 HEAD
+
+# build the BurntSushi test application
 go build -o toml-test github.com/BurntSushi/toml-test
 
 # vendorize the current lib for testing
@@ -23,6 +36,42 @@ cp *.go *.toml src/github.com/pelletier/go-toml
 cp cmd/*.go src/github.com/pelletier/go-toml/cmd
 go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
 
-# Run basic unit tests and then the BurntSushi test suite
+# Run basic unit tests
 go test -v github.com/pelletier/go-toml
-./toml-test ./test_program_bin | tee test_out
+
+# run the entire BurntSushi test suite
+if [[ $# -eq 0 ]] ; then
+  echo "Running all BurntSushi tests"
+  ./toml-test ./test_program_bin | tee test_out
+else
+  # run a specific test
+  test=$1
+  test_path='src/github.com/BurntSushi/toml-test/tests'
+  valid_test="$test_path/valid/$test"
+  invalid_test="$test_path/invalid/$test"
+
+  if [ -e "$valid_test.toml" ]; then
+    echo "Valid Test TOML for $test:"
+    echo "===="
+    cat "$valid_test.toml"
+
+    echo "Valid Test JSON for $test:"
+    echo "===="
+    cat "$valid_test.json"
+
+    echo "Go-TOML Output for $test:"
+    echo "===="
+    cat "$valid_test.toml" | ./test_program_bin
+  fi
+
+  if [ -e "$invalid_test.toml" ]; then
+    echo "Invalid Test TOML for $test:"
+    echo "===="
+    cat "$invalid_test.toml"
+
+    echo "Go-TOML Output for $test:"
+    echo "===="
+    echo "go-toml Output:"
+    cat "$invalid_test.toml" | ./test_program_bin
+  fi
+fi

token.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenComment
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenTrue
+	tokenFalse
+	tokenFloat
+	tokenEqual
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftCurlyBrace
+	tokenRightCurlyBrace
+	tokenLeftParen
+	tokenRightParen
+	tokenDoubleLeftBracket
+	tokenDoubleRightBracket
+	tokenDate
+	tokenKeyGroup
+	tokenKeyGroupArray
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+	tokenEOL
+)
+
+var tokenTypeNames = []string{
+	"Error",
+	"EOF",
+	"Comment",
+	"Key",
+	"String",
+	"Integer",
+	"True",
+	"False",
+	"Float",
+	"=",
+	"[",
+	"]",
+	"{",
+	"}",
+	"(",
+	")",
+	"]]",
+	"[[",
+	"Date",
+	"KeyGroup",
+	"KeyGroupArray",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+	"EOL",
+}
+
+type token struct {
+	Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	if len(t.val) > 10 {
+		return fmt.Sprintf("%.10q...", t.val)
+	}
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isKeyChar(r rune) bool {
+	// Keys start with the first character that isn't whitespace or [ and end
+	// with the last non-whitespace character before the equals sign. Keys
+	// cannot contain a # character."
+	return !(r == '\r' || r == '\n' || r == eof || r == '=')
+}
+
+func isKeyStartChar(r rune) bool {
+	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+}

toml.go

@@ -1,41 +1,58 @@
-// TOML markup language parser.
-//
-// This version supports the specification as described in
-// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
 package toml
 
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"os"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 )
 
-// Definition of a TomlTree.
+type tomlValue struct {
-// This is the result of the parsing of a TOML file.
+	value    interface{}
-type TomlTree map[string]interface{}
+	position Position
+}
 
-// Has returns a boolean indicating if the toplevel tree contains the given
+// TomlTree is the result of the parsing of a TOML file.
-// key.
+type TomlTree struct {
-func (t *TomlTree) Has(key string) bool {
+	values   map[string]interface{}
-	mp := (map[string]interface{})(*t)
+	position Position
-	for k, _ := range mp {
+}
-		if k == key {
+
-			return true
+func newTomlTree() *TomlTree {
-		}
+	return &TomlTree{
+		values:   make(map[string]interface{}),
+		position: Position{},
 	}
-	return false
+}
+
+func TreeFromMap(m map[string]interface{}) *TomlTree {
+	return &TomlTree{
+		values: m,
+	}
+}
+
+// Has returns a boolean indicating if the given key exists.
+func (t *TomlTree) Has(key string) bool {
+	if key == "" {
+		return false
+	}
+	return t.HasPath(strings.Split(key, "."))
+}
+
+// HasPath returns true if the given path of keys exists, false otherwise.
+func (t *TomlTree) HasPath(keys []string) bool {
+	return t.GetPath(keys) != nil
 }
 
 // Keys returns the keys of the toplevel tree.
 // Warning: this is a costly operation.
 func (t *TomlTree) Keys() []string {
-	keys := make([]string, 0)
+	var keys []string
-	mp := (map[string]interface{})(*t)
+	for k := range t.values {
-	for k, _ := range mp {
 		keys = append(keys, k)
 	}
 	return keys
@@ -49,36 +66,98 @@ func (t *TomlTree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
-	return t.GetPath(strings.Split(key, "."))
+	comps, err := parseKey(key)
+	if err != nil {
+		return nil
+	}
+	return t.GetPath(comps)
 }
 
-// Returns the element in the tree indicated by 'keys'.
+// GetPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
 func (t *TomlTree) GetPath(keys []string) interface{} {
 	if len(keys) == 0 {
 		return t
 	}
 	subtree := t
-	for _, intermediate_key := range keys[:len(keys)-1] {
+	for _, intermediateKey := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+		value, exists := subtree.values[intermediateKey]
 		if !exists {
 			return nil
 		}
-		switch node := (*subtree)[intermediate_key].(type) {
+		switch node := value.(type) {
 		case *TomlTree:
 			subtree = node
 		case []*TomlTree:
 			// go to most recent element
 			if len(node) == 0 {
-				return nil //(*subtree)[intermediate_key] = append(node, &TomlTree{})
+				return nil
 			}
 			subtree = node[len(node)-1]
+		default:
+			return nil // cannot naigate through other node types
 		}
 	}
-	return (*subtree)[keys[len(keys)-1]]
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.value
+	default:
+		return node
+	}
 }
 
-// Same as Get but with a default value
+// GetPosition returns the position of the given key.
+func (t *TomlTree) GetPosition(key string) Position {
+	if key == "" {
+		return t.position
+	}
+	return t.GetPositionPath(strings.Split(key, "."))
+}
+
+// GetPositionPath returns the element in the tree indicated by 'keys'.
+// If keys is of length zero, the current tree is returned.
+func (t *TomlTree) GetPositionPath(keys []string) Position {
+	if len(keys) == 0 {
+		return t.position
+	}
+	subtree := t
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		value, exists := subtree.values[intermediateKey]
+		if !exists {
+			return Position{0, 0}
+		}
+		switch node := value.(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				return Position{0, 0}
+			}
+			subtree = node[len(node)-1]
+		default:
+			return Position{0, 0}
+		}
+	}
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.position
+	case *TomlTree:
+		return node.position
+	case []*TomlTree:
+		// go to most recent element
+		if len(node) == 0 {
+			return Position{0, 0}
+		}
+		return node[len(node)-1].position
+	default:
+		return Position{0, 0}
+	}
+}
+
+// GetDefault works like Get but with a default value
 func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
 	val := t.Get(key)
 	if val == nil {
@@ -94,26 +173,44 @@ func (t *TomlTree) Set(key string, value interface{}) {
 	t.SetPath(strings.Split(key, "."), value)
 }
 
+// SetPath sets an element in the tree.
+// Keys is an array of path elements (e.g. {"a","b","c"}).
+// Creates all necessary intermediates trees, if needed.
 func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	subtree := t
-	for _, intermediate_key := range keys[:len(keys)-1] {
+	for _, intermediateKey := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			var new_tree TomlTree = make(TomlTree)
+			nextTree = newTomlTree()
-			(*subtree)[intermediate_key] = &new_tree
+			subtree.values[intermediateKey] = nextTree // add new element here
 		}
-		switch node := (*subtree)[intermediate_key].(type) {
+		switch node := nextTree.(type) {
 		case *TomlTree:
 			subtree = node
 		case []*TomlTree:
 			// go to most recent element
 			if len(node) == 0 {
-				(*subtree)[intermediate_key] = append(node, &TomlTree{})
+				// create element if it does not exist
+				subtree.values[intermediateKey] = append(node, newTomlTree())
 			}
 			subtree = node[len(node)-1]
 		}
 	}
-	(*subtree)[keys[len(keys)-1]] = value
+
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *TomlTree:
+		toInsert = value
+	case []*TomlTree:
+		toInsert = value
+	case *tomlValue:
+		toInsert = value
+	default:
+		toInsert = &tomlValue{value: value}
+	}
+
+	subtree.values[keys[len(keys)-1]] = toInsert
 }
 
 // createSubTree takes a tree and a key and create the necessary intermediate
@@ -121,26 +218,40 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 //
 // e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
 // and tree[a][b][c]
-func (t *TomlTree) createSubTree(key string) {
+//
+// Returns nil on success, error object on failure
+func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	subtree := t
-	for _, intermediate_key := range strings.Split(key, ".") {
+	for _, intermediateKey := range keys {
-		if intermediate_key == "" {
+		if intermediateKey == "" {
-			panic("empty intermediate table")
+			return fmt.Errorf("empty intermediate table")
 		}
-		_, exists := (*subtree)[intermediate_key]
+		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			var new_tree TomlTree = make(TomlTree)
+			tree := newTomlTree()
-			(*subtree)[intermediate_key] = &new_tree
+			tree.position = pos
+			subtree.values[intermediateKey] = tree
+			nextTree = tree
+		}
+
+		switch node := nextTree.(type) {
+		case []*TomlTree:
+			subtree = node[len(node)-1]
+		case *TomlTree:
+			subtree = node
+		default:
+			return fmt.Errorf("unknown type for path %s (%s): %T (%#v)",
+				strings.Join(keys, "."), intermediateKey, nextTree, nextTree)
 		}
-		subtree = ((*subtree)[intermediate_key]).(*TomlTree)
 	}
+	return nil
 }
 
 // encodes a string to a TOML-compliant string value
 func encodeTomlString(value string) string {
 	result := ""
 	for _, rr := range value {
-		int_rr := uint16(rr)
+		intRr := uint16(rr)
 		switch rr {
 		case '\b':
 			result += "\\b"
@@ -157,8 +268,8 @@ func encodeTomlString(value string) string {
 		case '\\':
 			result += "\\\\"
 		default:
-			if int_rr < 0x001F {
+			if intRr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", int_rr)
+				result += fmt.Sprintf("\\u%0.4X", intRr)
 			} else {
 				result += string(rr)
 			}
@@ -181,9 +292,8 @@ func toTomlValue(item interface{}, indent int) string {
 	case bool:
 		if value {
 			return "true"
-		} else {
-			return "false"
 		}
+		return "false"
 	case time.Time:
 		return tab + value.Format(time.RFC3339)
 	case []interface{}:
@@ -199,43 +309,60 @@ func toTomlValue(item interface{}, indent int) string {
 
 // Recursive support function for ToString()
 // Outputs a tree, using the provided keyspace to prefix group names
-func (t *TomlTree) toToml(keyspace string) string {
+func (t *TomlTree) toToml(indent, keyspace string) string {
 	result := ""
-	for k, v := range (map[string]interface{})(*t) {
+	for k, v := range t.values {
 		// figure out the keyspace
-		combined_key := k
+		combinedKey := k
 		if keyspace != "" {
-			combined_key = keyspace + "." + combined_key
+			combinedKey = keyspace + "." + combinedKey
 		}
 		// output based on type
 		switch node := v.(type) {
 		case []*TomlTree:
 			for _, item := range node {
 				if len(item.Keys()) > 0 {
-					result += fmt.Sprintf("\n[[%s]]\n", combined_key)
+					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
 				}
-				result += item.toToml(combined_key)
+				result += item.toToml(indent+"  ", combinedKey)
 			}
 		case *TomlTree:
 			if len(node.Keys()) > 0 {
-				result += fmt.Sprintf("\n[%s]\n", combined_key)
+				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
 			}
-			result += node.toToml(combined_key)
+			result += node.toToml(indent+"  ", combinedKey)
+		case map[string]interface{}:
+			sub := TreeFromMap(node)
+
+			if len(sub.Keys()) > 0 {
+				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			result += sub.toToml(indent+"  ", combinedKey)
+		case *tomlValue:
+			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
 		default:
-			result += fmt.Sprintf("%s = %s\n", k, toTomlValue(node, 0))
+			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
 		}
 	}
 	return result
 }
 
-// Generates a human-readable representation of the current tree.
+func (t *TomlTree) Query(query string) (*QueryResult, error) {
-// Output spans multiple lines, and is suitable for ingest by a TOML parser
+	if q, err := CompileQuery(query); err != nil {
-func (t *TomlTree) ToString() string {
+		return nil, err
-	return t.toToml("")
+	} else {
+		return q.Execute(t), nil
+	}
 }
 
-// Create a TomlTree from a string.
+// ToString generates a human-readable representation of the current tree.
-func Load(content string) (tree *TomlTree, err error) {
+// Output spans multiple lines, and is suitable for ingest by a TOML parser
+func (t *TomlTree) ToString() string {
+	return t.toToml("", "")
+}
+
+// LoadReader creates a TomlTree from any io.Reader.
+func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -244,19 +371,21 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	_, flow := lex(content)
+	tree = parseToml(lexToml(reader))
-	tree = parse(flow)
 	return
 }
 
-// Create a TomlTree from a file.
+// Load creates a TomlTree from a string.
-func LoadFile(path string) (tree *TomlTree, err error) {
+func Load(content string) (tree *TomlTree, err error) {
-	buff, ferr := ioutil.ReadFile(path)
+	return LoadReader(strings.NewReader(content))
-	if ferr != nil {
+}
-		err = ferr
+
-	} else {
+// LoadFile creates a TomlTree from a file.
-		s := string(buff)
+func LoadFile(path string) (tree *TomlTree, err error) {
-		tree, err = Load(s)
+	file, err := os.Open(path)
-	}
+	if err != nil {
-	return
+		return nil, err
+	}
+	defer file.Close()
+	return LoadReader(file)
 }

toml_test.go

@@ -1,20 +1,44 @@
+// Testing support for go-toml
+
 package toml
 
 import (
 	"testing"
 )
 
+func TestTomlHas(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.Has("test.key") {
+		t.Errorf("Has - expected test.key to exists")
+	}
+}
+
+func TestTomlHasPath(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.HasPath([]string{"test", "key"}) {
+		t.Errorf("HasPath - expected test.key to exists")
+	}
+}
+
 func TestTomlGetPath(t *testing.T) {
-	node := make(TomlTree)
+	node := newTomlTree()
 	//TODO: set other node data
 
 	for idx, item := range []struct {
 		Path     []string
-		Expected interface{}
+		Expected *TomlTree
 	}{
 		{ // empty path test
 			[]string{},
-			&node,
+			node,
 		},
 	} {
 		result := node.GetPath(item.Path)
@@ -23,3 +47,28 @@ func TestTomlGetPath(t *testing.T) {
 		}
 	}
 }
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result, err := tree.Query("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*TomlTree); !ok {
+		t.Errorf("Expected type of TomlTree: %T", values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}