Merge pull request #23 from pelletier/jsonpath

Powerful querying interface

.gitignore

@@ -0,0 +1 @@
+test_program/test_program_bin

.travis.yml

@@ -1 +1,7 @@
 language: go
+script: "./test.sh"
+go:
+    - 1.1
+    - 1.2
+    - 1.3
+    - tip

README.md

@@ -3,9 +3,27 @@
 Go library for the [TOML](https://github.com/mojombo/toml) format.
 
 This library supports TOML version
-[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md)
+[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md)
 
-[![Build Status](https://travis-ci.org/pelletier/go-toml.png?branch=master)](https://travis-ci.org/pelletier/go-toml)
+[![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
+[![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
+
+## Features
+
+Go-toml provides the following features for using data parsed from TOML documents:
+
+* Load TOML documents from files and string data
+* Easily navigate TOML structure using TomlTree
+* Line & column position data for all parsed elements
+* Query support similar to JSON-Path
+* Syntax errors contain line and column numbers
+
+Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
+
+* Semantic evaluation of parsed TOML
+* Informing a user of mistakes in the source document, after it has been parsed
+* Programatic handling of default values on a case-by-case basis
+* Using a TOML document as a flexible data-store
 
 ## Import
 
@@ -13,6 +31,8 @@ This library supports TOML version
 
 ## Usage
 
+### Example
+
 Say you have a TOML file that looks like this:
 
 ```toml
@@ -42,12 +62,22 @@ if err != nil {
     user = configTree.Get("user").(string)
     password = configTree.Get("password").(string)
     fmt.Println("User is ", user, ". Password is ", password)
+
+    // show where elements are in the file
+    fmt.Println("User position: %v", configTree.GetPosition("user"))
+    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+    // use a query to gather elements without walking the tree
+    results, _ := config.Query("$..[user,password]")
+    for ii, item := range results.Values() {
+      fmt.Println("Query result %d: %v", ii, item)
+    }
 }
 ```
 
 ## Documentation
 
-The documentation is available at
+The documentation and additional examples are available at
 [godoc.org](http://godoc.org/github.com/pelletier/go-toml).
 
 ## Contribute
@@ -56,10 +86,18 @@ Feel free to report bugs and patches using GitHub's pull requests system on
 [pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be
 much appreciated!
 
+### Run tests
+
+You have to make sure two kind of tests run:
+
+1. The Go unit tests
+2. The TOML examples base
+
+You can run both of them using `./test.sh`.
 
 ## License
 
-Copyright (c) 2013 Thomas Pelletier
+Copyright (c) 2013, 2014 Thomas Pelletier, Eric Anderton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# fail out of the script if anything here fails
+set -e
+
+# clear out stuff generated by test.sh
+rm -rf src test_program_bin toml-test

cmd/test_program.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/pelletier/go-toml"
+	"io/ioutil"
+	"log"
+	"os"
+	"time"
+)
+
+func main() {
+	bytes, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		os.Exit(2)
+	}
+	tree, err := toml.Load(string(bytes))
+	if err != nil {
+		os.Exit(1)
+	}
+
+	typedTree := translate(*tree)
+
+	if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
+		log.Fatalf("Error encoding JSON: %s", err)
+	}
+
+	os.Exit(0)
+}
+
+func translate(tomlData interface{}) interface{} {
+	switch orig := tomlData.(type) {
+	case map[string]interface{}:
+		typed := make(map[string]interface{}, len(orig))
+		for k, v := range orig {
+			typed[k] = translate(v)
+		}
+		return typed
+	case *toml.TomlTree:
+		return translate(*orig)
+	case toml.TomlTree:
+		keys := orig.Keys()
+		typed := make(map[string]interface{}, len(keys))
+		for _, k := range keys {
+			typed[k] = translate(orig.GetPath([]string{k}))
+		}
+		return typed
+	case []*toml.TomlTree:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v).(map[string]interface{})
+		}
+		return typed
+	case []map[string]interface{}:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v).(map[string]interface{})
+		}
+		return typed
+	case []interface{}:
+		typed := make([]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v)
+		}
+		return tag("array", typed)
+	case time.Time:
+		return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
+	case bool:
+		return tag("bool", fmt.Sprintf("%v", orig))
+	case int64:
+		return tag("integer", fmt.Sprintf("%d", orig))
+	case float64:
+		return tag("float", fmt.Sprintf("%v", orig))
+	case string:
+		return tag("string", orig)
+	}
+
+	panic(fmt.Sprintf("Unknown type: %T", tomlData))
+}
+
+func tag(typeName string, data interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"type":  typeName,
+		"value": data,
+	}
+}

doc.go

@@ -0,0 +1,245 @@
+// Package toml is a TOML markup language parser.
+//
+// This version supports the specification as described in
+// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
+//
+// TOML Parsing
+//
+// TOML data may be parsed in two ways: by file, or by string.
+//
+//   // load TOML data by filename
+//   tree, err := toml.LoadFile("filename.toml")
+//
+//   // load TOML data stored in a string
+//   tree, err := toml.Load(stringContainingTomlData)
+//
+// Either way, the result is a TomlTree object that can be used to navigate the
+// structure and data within the original document.
+//
+//
+// Getting data from the TomlTree
+//
+// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
+// methods on the returned TomlTree, to find your way through the document data.
+//
+//   if tree.Has('foo') {
+//     fmt.Prinln("foo is: %v", tree.Get('foo'))
+//   }
+//
+// Working with Paths
+//
+// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
+// and GetDefault() methods.  These are the same kind of key paths used within the
+// TOML specification for struct tames.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.Has("foo.bar.baz")
+//
+//   // returns the key at this path, if it is there
+//   tree.Get("foo.bar.baz")
+//
+// TOML allows keys to contain '.', which can cause this syntax to be problematic
+// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
+// methods to explicitly define the path.  This form is also faster, since
+// it avoids having to parse the passed key for '.' delimiters.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.HasPath(string{}{"foo","bar","baz"})
+//
+//   // returns the key at this path, if it is there
+//   tree.GetPath(string{}{"foo","bar","baz"})
+//
+// Note that this is distinct from the heavyweight query syntax supported by
+// TomlTree.Query() and the Query() struct (see below).
+//
+// Position Support
+//
+// Each element within the TomlTree is stored with position metadata, which is
+// invaluable for providing semantic feedback to a user.  This helps in
+// situations where the TOML file parses correctly, but contains data that is
+// not correct for the application.  In such cases, an error message can be
+// generated that indicates the problem line and column number in the source
+// TOML document.
+//
+//   // load TOML data
+//   tree, _ := toml.Load("filename.toml")
+//
+//   // get an entry and report an error if it's the wrong type
+//   element := tree.Get("foo")
+//   if value, ok := element.(int64); !ok {
+//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
+//   }
+//
+//   // report an error if an expected element is missing
+//   if !tree.Has("bar") {
+//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
+//   }
+//
+// Query Support
+//
+// The TOML query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result := tree.Query("$.foo.bar.baz")  // result is 'nil' if the path is not present
+//
+// This is equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   tree.Query("$.foo[-1]")
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   tree.Query("$.foo[0:5]")
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   tree.Query("$.foo[0:-1]")
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   tree.Query("$.foo[0:-1:2]")
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   tree.Query("$.foo[:]")
+//   tree.Query("$.foo[0:]")
+//   tree.Query("$.foo[:-1]")
+//   tree.Query("$.foo[0:-1:]")
+//   tree.Query("$.foo[::1]")
+//   tree.Query("$.foo[0::1]")
+//   tree.Query("$.foo[:-1:1]")
+//   tree.Query("$.foo[0:-1:1]")
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   tree.Query("$.foo[?(bar)]")
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type TomlTree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a QueryResult object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := tree.Query("$.foo.bar.baz")
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a TomlTree object, or compiled ahead
+// of time and executed discretely.  The former is more convienent, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := tree.Query("$.foo.bar.baz")
+//
+//   // compiled query
+//   query := toml.CompileQuery("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := CompileQuery("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*TomlTree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package toml

doc_test.go

@@ -0,0 +1,81 @@
+// code examples for godoc
+
+package toml
+
+import (
+	"fmt"
+)
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := CompileQuery("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*TomlTree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' TomlTree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	authors, _ := config.Query("$.book.author")
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func Example_comprehensiveExample() {
+	config, err := LoadFile("config.toml")
+
+	if err != nil {
+		fmt.Println("Error ", err.Error())
+	} else {
+		// retrieve data directly
+		user := config.Get("postgres.user").(string)
+		password := config.Get("postgres.password").(string)
+
+		// or using an intermediate object
+		configTree := config.Get("postgres").(*TomlTree)
+		user = configTree.Get("user").(string)
+		password = configTree.Get("password").(string)
+		fmt.Println("User is ", user, ". Password is ", password)
+
+		// show where elements are in the file
+		fmt.Println("User position: %v", configTree.GetPosition("user"))
+		fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+		// use a query to gather elements without walking the tree
+		results, _ := config.Query("$..[user,password]")
+		for ii, item := range results.Values() {
+			fmt.Println("Query result %d: %v", ii, item)
+		}
+	}
+}

lexer.go

@@ -1,4 +1,6 @@
-// TOML lexer.// Written using the principles developped by Rob Pike in
+// TOML lexer.
+//
+// Written using the principles developped by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
 package toml
@@ -8,102 +10,69 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 )
 
 var dateRegexp *regexp.Regexp
 
-// Define tokens
-type tokenType int
-
-const (
-	eof = -(iota + 1)
-)
-
-const (
-	tokenError tokenType = iota
-	tokenEOF
-	tokenComment
-	tokenKey
-	tokenEqual
-	tokenString
-	tokenInteger
-	tokenTrue
-	tokenFalse
-	tokenFloat
-	tokenLeftBracket
-	tokenRightBracket
-	tokenDate
-	tokenKeyGroup
-	tokenComma
-	tokenEOL
-)
-
-type token struct {
-	typ tokenType
-	val string
-}
-
-func (i token) String() string {
-	switch i.typ {
-	case tokenEOF:
-		return "EOF"
-	case tokenError:
-		return i.val
-	}
-
-	if len(i.val) > 10 {
-		return fmt.Sprintf("%.10q...", i.val)
-	}
-	return fmt.Sprintf("%q", i.val)
-}
-
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
-}
-
-func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
-}
-
-func isHexDigit(r rune) bool {
-	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
-}
+// Define state functions
+type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
-type lexer struct {
+type tomlLexer struct {
 	input  string
 	start  int
 	pos    int
 	width  int
 	tokens chan token
 	depth  int
+	line   int
+	col    int
 }
 
-func (l *lexer) run() {
-	for state := lexVoid; state != nil; {
-		state = state(l)
+func (l *tomlLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
 	}
 	close(l.tokens)
 }
 
-func (l *lexer) emit(t tokenType) {
-	l.tokens <- token{t, l.input[l.start:l.pos]}
+func (l *tomlLexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line++
+			l.col = 1
+		} else {
+			l.col++
+		}
+		i += width
+	}
+	// advance start position to next token
 	l.start = l.pos
 }
 
-func (l *lexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{t, value}
-	l.start = l.pos
+func (l *tomlLexer) emit(t tokenType) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      l.input[l.start:l.pos],
+	}
+	l.nextStart()
 }
 
-func (l *lexer) next() rune {
+func (l *tomlLexer) emitWithValue(t tokenType, value string) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.nextStart()
+}
+
+func (l *tomlLexer) next() rune {
 	if l.pos >= len(l.input) {
 		l.width = 0
 		return eof
@@ -114,29 +83,30 @@ func (l *lexer) next() rune {
 	return r
 }
 
-func (l *lexer) ignore() {
-	l.start = l.pos
+func (l *tomlLexer) ignore() {
+	l.nextStart()
 }
 
-func (l *lexer) backup() {
+func (l *tomlLexer) backup() {
 	l.pos -= l.width
 }
 
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
 	l.tokens <- token{
-		tokenError,
-		fmt.Sprintf(format, args...),
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
 	}
 	return nil
 }
 
-func (l *lexer) peek() rune {
+func (l *tomlLexer) peek() rune {
 	r := l.next()
 	l.backup()
 	return r
 }
 
-func (l *lexer) accept(valid string) bool {
+func (l *tomlLexer) accept(valid string) bool {
 	if strings.IndexRune(valid, l.next()) >= 0 {
 		return true
 	}
@@ -144,33 +114,34 @@ func (l *lexer) accept(valid string) bool {
 	return false
 }
 
-func (l *lexer) follow(next string) bool {
+func (l *tomlLexer) follow(next string) bool {
 	return strings.HasPrefix(l.input[l.pos:], next)
 }
 
-// Define state functions
-type stateFn func(*lexer) stateFn
-
-func lexVoid(l *lexer) stateFn {
+func (l *tomlLexer) lexVoid() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '[':
-			return lexKeyGroup
+			return l.lexKeyGroup
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '=':
-			return lexEqual
-		}
-
-		if isAlphanumeric(next) {
-			return lexKey
+			return l.lexEqual
 		}
 
 		if isSpace(next) {
 			l.ignore()
 		}
 
+		if l.depth > 0 {
+			return l.lexRvalue
+		}
+
+		if isKeyChar(next) {
+			return l.lexKey
+		}
+
 		if l.next() == eof {
 			break
 		}
@@ -180,50 +151,53 @@ func lexVoid(l *lexer) stateFn {
 	return nil
 }
 
-func lexRvalue(l *lexer) stateFn {
+func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
+		case '.':
+			return l.errorf("cannot start float with a dot")
+		case '=':
+			return l.errorf("cannot have multiple equals for the same key")
 		case '[':
-			l.depth += 1
-			return lexLeftBracket
+			l.depth++
+			return l.lexLeftBracket
 		case ']':
-			l.depth -= 1
-			return lexRightBracket
+			l.depth--
+			return l.lexRightBracket
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '"':
-			return lexString
+			return l.lexString
 		case ',':
-			return lexComma
+			return l.lexComma
 		case '\n':
 			l.ignore()
-			l.pos += 1
+			l.pos++
 			if l.depth == 0 {
-				return lexVoid
-			} else {
-				return lexRvalue
+				return l.lexVoid
 			}
+			return l.lexRvalue
 		}
 
 		if l.follow("true") {
-			return lexTrue
+			return l.lexTrue
 		}
 
 		if l.follow("false") {
-			return lexFalse
+			return l.lexFalse
 		}
 
 		if isAlphanumeric(next) {
-			return lexKey
+			return l.lexKey
 		}
 
 		if dateRegexp.FindString(l.input[l.pos:]) != "" {
-			return lexDate
+			return l.lexDate
 		}
 
 		if next == '+' || next == '-' || isDigit(next) {
-			return lexNumber
+			return l.lexNumber
 		}
 
 		if isSpace(next) {
@@ -239,51 +213,51 @@ func lexRvalue(l *lexer) stateFn {
 	return nil
 }
 
-func lexDate(l *lexer) stateFn {
+func (l *tomlLexer) lexDate() tomlLexStateFn {
 	l.ignore()
 	l.pos += 20 // Fixed size of a date in TOML
 	l.emit(tokenDate)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexTrue(l *lexer) stateFn {
+func (l *tomlLexer) lexTrue() tomlLexStateFn {
 	l.ignore()
 	l.pos += 4
 	l.emit(tokenTrue)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexFalse(l *lexer) stateFn {
+func (l *tomlLexer) lexFalse() tomlLexStateFn {
 	l.ignore()
 	l.pos += 5
 	l.emit(tokenFalse)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexEqual(l *lexer) stateFn {
+func (l *tomlLexer) lexEqual() tomlLexStateFn {
 	l.ignore()
 	l.accept("=")
 	l.emit(tokenEqual)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexComma(l *lexer) stateFn {
+func (l *tomlLexer) lexComma() tomlLexStateFn {
 	l.ignore()
 	l.accept(",")
 	l.emit(tokenComma)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexKey(l *lexer) stateFn {
+func (l *tomlLexer) lexKey() tomlLexStateFn {
 	l.ignore()
-	for isAlphanumeric(l.next()) {
+	for isKeyChar(l.next()) {
 	}
 	l.backup()
 	l.emit(tokenKey)
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexComment(l *lexer) stateFn {
+func (l *tomlLexer) lexComment() tomlLexStateFn {
 	for {
 		next := l.next()
 		if next == '\n' || next == eof {
@@ -291,63 +265,75 @@ func lexComment(l *lexer) stateFn {
 		}
 	}
 	l.ignore()
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexLeftBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
 	l.ignore()
-	l.pos += 1
+	l.pos++
 	l.emit(tokenLeftBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexString(l *lexer) stateFn {
-	l.pos += 1
+func (l *tomlLexer) lexString() tomlLexStateFn {
+	l.pos++
 	l.ignore()
-	growing_string := ""
+	growingString := ""
 
 	for {
 		if l.peek() == '"' {
-			l.emitWithValue(tokenString, growing_string)
-			l.pos += 1
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
 			l.ignore()
-			return lexRvalue
+			return l.lexRvalue
 		}
 
 		if l.follow("\\\"") {
-			l.pos += 1
-			growing_string += "\""
+			l.pos++
+			growingString += "\""
 		} else if l.follow("\\n") {
-			l.pos += 1
-			growing_string += "\n"
+			l.pos++
+			growingString += "\n"
+		} else if l.follow("\\b") {
+			l.pos++
+			growingString += "\b"
+		} else if l.follow("\\f") {
+			l.pos++
+			growingString += "\f"
+		} else if l.follow("\\/") {
+			l.pos++
+			growingString += "/"
 		} else if l.follow("\\t") {
-			l.pos += 1
-			growing_string += "\t"
+			l.pos++
+			growingString += "\t"
 		} else if l.follow("\\r") {
-			l.pos += 1
-			growing_string += "\r"
+			l.pos++
+			growingString += "\r"
 		} else if l.follow("\\\\") {
-			l.pos += 1
-			growing_string += "\\"
+			l.pos++
+			growingString += "\\"
 		} else if l.follow("\\u") {
 			l.pos += 2
 			code := ""
 			for i := 0; i < 4; i++ {
 				c := l.peek()
-				l.pos += 1
+				l.pos++
 				if !isHexDigit(c) {
 					return l.errorf("unfinished unicode escape")
 				}
 				code = code + string(c)
 			}
-			l.pos -= 1
+			l.pos--
 			intcode, err := strconv.ParseInt(code, 16, 32)
 			if err != nil {
 				return l.errorf("invalid unicode escape: \\u" + code)
 			}
-			growing_string += string(rune(intcode))
+			growingString += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos++
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
 		} else {
-			growing_string += string(l.peek())
+			growingString += string(l.peek())
 		}
 
 		if l.next() == eof {
@@ -358,23 +344,58 @@ func lexString(l *lexer) stateFn {
 	return l.errorf("unclosed string")
 }
 
-func lexKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
 	l.ignore()
-	l.pos += 1
+	l.pos++
+
+	if l.peek() == '[' {
+		// token '[[' signifies an array of anonymous key groups
+		l.pos++
+		l.emit(tokenDoubleLeftBracket)
+		return l.lexInsideKeyGroupArray
+	}
+	// vanilla key group
 	l.emit(tokenLeftBracket)
-	return lexInsideKeyGroup
+	return l.lexInsideKeyGroup
 }
 
-func lexInsideKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
+	for {
+		if l.peek() == ']' {
+			if l.pos > l.start {
+				l.emit(tokenKeyGroupArray)
+			}
+			l.ignore()
+			l.pos++
+			if l.peek() != ']' {
+				break // error
+			}
+			l.pos++
+			l.emit(tokenDoubleRightBracket)
+			return l.lexVoid
+		} else if l.peek() == '[' {
+			return l.errorf("group name cannot contain ']'")
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	return l.errorf("unclosed key group array")
+}
+
+func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
 	for {
 		if l.peek() == ']' {
 			if l.pos > l.start {
 				l.emit(tokenKeyGroup)
 			}
 			l.ignore()
-			l.pos += 1
+			l.pos++
 			l.emit(tokenRightBracket)
-			return lexVoid
+			return l.lexVoid
+		} else if l.peek() == '[' {
+			return l.errorf("group name cannot contain ']'")
 		}
 
 		if l.next() == eof {
@@ -384,41 +405,50 @@ func lexInsideKeyGroup(l *lexer) stateFn {
 	return l.errorf("unclosed key group")
 }
 
-func lexRightBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
 	l.ignore()
-	l.pos += 1
+	l.pos++
 	l.emit(tokenRightBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexNumber(l *lexer) stateFn {
+func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	l.ignore()
 	if !l.accept("+") {
 		l.accept("-")
 	}
-	point_seen := false
-	digit_seen := false
+	pointSeen := false
+	digitSeen := false
 	for {
 		next := l.next()
 		if next == '.' {
-			point_seen = true
+			if pointSeen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
+			pointSeen = true
 		} else if isDigit(next) {
-			digit_seen = true
+			digitSeen = true
 		} else {
 			l.backup()
 			break
 		}
+		if pointSeen && !digitSeen {
+			return l.errorf("cannot start float with a dot")
+		}
 	}
 
-	if !digit_seen {
+	if !digitSeen {
 		return l.errorf("no digit in that number")
 	}
-	if point_seen {
+	if pointSeen {
 		l.emit(tokenFloat)
 	} else {
 		l.emit(tokenInteger)
 	}
-	return lexRvalue
+	return l.lexRvalue
 }
 
 func init() {
@@ -426,11 +456,13 @@ func init() {
 }
 
 // Entry point
-func lex(input string) (*lexer, chan token) {
-	l := &lexer{
+func lexToml(input string) chan token {
+	l := &tomlLexer{
 		input:  input,
 		tokens: make(chan token),
+		line:   1,
+		col:    1,
 	}
 	go l.run()
-	return l, l.tokens
+	return l.tokens
 }

lexer_test.go

@@ -3,13 +3,16 @@ package toml
 import "testing"
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	_, ch := lex(input)
+	ch := lexToml(input)
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {
+			t.Log("While testing: ", input)
 			t.Log("compared", token, "to", expected)
 			t.Log(token.val, "<->", expected.val)
 			t.Log(token.typ, "<->", expected.typ)
+			t.Log(token.Line, "<->", expected.Line)
+			t.Log(token.Col, "<->", expected.Col)
 			t.FailNow()
 		}
 	}
@@ -29,204 +32,245 @@ func testFlow(t *testing.T, input string, expectedFlow []token) {
 
 func TestValidKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenLeftBracket, "["},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
+		token{Position{1, 14}, tokenEOF, ""},
 	})
 }
 
 func TestUnclosedKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenError, "unclosed key group"},
+		token{Position{1, 1}, tokenLeftBracket, "["},
+		token{Position{1, 2}, tokenError, "unclosed key group"},
 	})
 }
 
 func TestComment(t *testing.T) {
 	testFlow(t, "# blahblah", []token{
-		token{tokenEOF, ""},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyGroupComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenLeftBracket, "["},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
+		token{Position{1, 25}, tokenEOF, ""},
 	})
 }
 
 func TestMultipleKeyGroupsComment(t *testing.T) {
 	testFlow(t, "[hello world] # blahblah\n[test]", []token{
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "hello world"},
-		token{tokenRightBracket, "]"},
-		token{tokenLeftBracket, "["},
-		token{tokenKeyGroup, "test"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenLeftBracket, "["},
+		token{Position{1, 2}, tokenKeyGroup, "hello world"},
+		token{Position{1, 13}, tokenRightBracket, "]"},
+		token{Position{2, 1}, tokenLeftBracket, "["},
+		token{Position{2, 2}, tokenKeyGroup, "test"},
+		token{Position{2, 6}, tokenRightBracket, "]"},
+		token{Position{2, 7}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
-		token{tokenKey, "hello"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "hello"},
+		token{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUnderscore(t *testing.T) {
 	testFlow(t, "hello_hello", []token{
-		token{tokenKey, "hello_hello"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "hello_hello"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestBasicKeyWithDash(t *testing.T) {
+	testFlow(t, "hello-world", []token{
+		token{Position{1, 1}, tokenKey, "hello-world"},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithUppercaseMix(t *testing.T) {
 	testFlow(t, "helloHELLOHello", []token{
-		token{tokenKey, "helloHELLOHello"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "helloHELLOHello"},
+		token{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
 	testFlow(t, "héllÖ", []token{
-		token{tokenKey, "héllÖ"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "héllÖ"},
+		token{Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestBasicKeyAndEqual(t *testing.T) {
 	testFlow(t, "hello =", []token{
-		token{tokenKey, "hello"},
-		token{tokenEqual, "="},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "hello"},
+		token{Position{1, 7}, tokenEqual, "="},
+		token{Position{1, 8}, tokenEOF, ""},
+	})
+}
+
+func TestKeyWithSharpAndEqual(t *testing.T) {
+	testFlow(t, "key#name = 5", []token{
+		token{Position{1, 1}, tokenKey, "key#name"},
+		token{Position{1, 10}, tokenEqual, "="},
+		token{Position{1, 12}, tokenInteger, "5"},
+		token{Position{1, 13}, tokenEOF, ""},
+	})
+}
+
+func TestKeyWithSymbolsAndEqual(t *testing.T) {
+	testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+		token{Position{1, 1}, tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+		token{Position{1, 39}, tokenEqual, "="},
+		token{Position{1, 41}, tokenInteger, "5"},
+		token{Position{1, 42}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello\\\"\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello\""},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello\""`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "hello\""},
+		token{Position{1, 16}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnfinished(t *testing.T) {
-	testFlow(t, "foo = \"bar", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenError, "unclosed string"},
+	testFlow(t, `foo = "bar`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "unclosed string"},
 	})
 }
 
 func TestKeyEqualString(t *testing.T) {
-	testFlow(t, "foo = \"bar\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "bar"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "bar"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "bar"},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualTrue(t *testing.T) {
 	testFlow(t, "foo = true", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenTrue, "true"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenTrue, "true"},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualFalse(t *testing.T) {
 	testFlow(t, "foo = false", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFalse, "false"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenFalse, "false"},
+		token{Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedString(t *testing.T) {
-	testFlow(t, "a = [ [\"hello\", \"world\"] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenString, "hello"},
-		token{tokenComma, ","},
-		token{tokenString, "world"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
+		token{Position{1, 9}, tokenString, "hello"},
+		token{Position{1, 15}, tokenComma, ","},
+		token{Position{1, 18}, tokenString, "world"},
+		token{Position{1, 24}, tokenRightBracket, "]"},
+		token{Position{1, 26}, tokenRightBracket, "]"},
+		token{Position{1, 27}, tokenEOF, ""},
 	})
 }
 
 func TestArrayNestedInts(t *testing.T) {
 	testFlow(t, "a = [ [42, 21], [10] ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenRightBracket, "]"},
-		token{tokenComma, ","},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "10"},
-		token{tokenRightBracket, "]"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenLeftBracket, "["},
+		token{Position{1, 8}, tokenInteger, "42"},
+		token{Position{1, 10}, tokenComma, ","},
+		token{Position{1, 12}, tokenInteger, "21"},
+		token{Position{1, 14}, tokenRightBracket, "]"},
+		token{Position{1, 15}, tokenComma, ","},
+		token{Position{1, 17}, tokenLeftBracket, "["},
+		token{Position{1, 18}, tokenInteger, "10"},
+		token{Position{1, 20}, tokenRightBracket, "]"},
+		token{Position{1, 22}, tokenRightBracket, "]"},
+		token{Position{1, 23}, tokenEOF, ""},
 	})
 }
 
 func TestArrayInts(t *testing.T) {
 	testFlow(t, "a = [ 42, 21, 10, ]", []token{
-		token{tokenKey, "a"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenInteger, "42"},
-		token{tokenComma, ","},
-		token{tokenInteger, "21"},
-		token{tokenComma, ","},
-		token{tokenInteger, "10"},
-		token{tokenComma, ","},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenLeftBracket, "["},
+		token{Position{1, 7}, tokenInteger, "42"},
+		token{Position{1, 9}, tokenComma, ","},
+		token{Position{1, 11}, tokenInteger, "21"},
+		token{Position{1, 13}, tokenComma, ","},
+		token{Position{1, 15}, tokenInteger, "10"},
+		token{Position{1, 17}, tokenComma, ","},
+		token{Position{1, 19}, tokenRightBracket, "]"},
+		token{Position{1, 20}, tokenEOF, ""},
+	})
+}
+
+func TestMultilineArrayComments(t *testing.T) {
+	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 3}, tokenEqual, "="},
+		token{Position{1, 5}, tokenLeftBracket, "["},
+		token{Position{1, 6}, tokenInteger, "1"},
+		token{Position{1, 7}, tokenComma, ","},
+		token{Position{2, 1}, tokenInteger, "2"},
+		token{Position{2, 2}, tokenComma, ","},
+		token{Position{3, 1}, tokenInteger, "3"},
+		token{Position{3, 2}, tokenComma, ","},
+		token{Position{4, 1}, tokenRightBracket, "]"},
+		token{Position{4, 2}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenLeftBracket, "["},
+		token{Position{1, 8}, tokenTrue, "true"},
+		token{Position{1, 12}, tokenComma, ","},
+		token{Position{1, 14}, tokenFalse, "false"},
+		token{Position{1, 19}, tokenComma, ","},
+		token{Position{1, 21}, tokenTrue, "true"},
+		token{Position{1, 25}, tokenRightBracket, "]"},
+		token{Position{1, 26}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
 	testFlow(t, "foo = [true, false, true] # YEAH", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenLeftBracket, "["},
-		token{tokenTrue, "true"},
-		token{tokenComma, ","},
-		token{tokenFalse, "false"},
-		token{tokenComma, ","},
-		token{tokenTrue, "true"},
-		token{tokenRightBracket, "]"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenLeftBracket, "["},
+		token{Position{1, 8}, tokenTrue, "true"},
+		token{Position{1, 12}, tokenComma, ","},
+		token{Position{1, 14}, tokenFalse, "false"},
+		token{Position{1, 19}, tokenComma, ","},
+		token{Position{1, 21}, tokenTrue, "true"},
+		token{Position{1, 25}, tokenRightBracket, "]"},
+		token{Position{1, 33}, tokenEOF, ""},
 	})
 }
 
@@ -238,83 +282,138 @@ func TestDateRegexp(t *testing.T) {
 
 func TestKeyEqualDate(t *testing.T) {
 	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenDate, "1979-05-27T07:32:00Z"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
+		token{Position{1, 27}, tokenEOF, ""},
+	})
+}
+
+func TestFloatEndingWithDot(t *testing.T) {
+	testFlow(t, "foo = 42.", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenError, "float cannot end with a dot"},
+	})
+}
+
+func TestFloatWithTwoDots(t *testing.T) {
+	testFlow(t, "foo = 4.2.", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenError, "cannot have two dots in one float"},
+	})
+}
+
+func TestDoubleEqualKey(t *testing.T) {
+	testFlow(t, "foo= = 2", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 4}, tokenEqual, "="},
+		token{Position{1, 5}, tokenError, "cannot have multiple equals for the same key"},
+	})
+}
+
+func TestInvalidEsquapeSequence(t *testing.T) {
+	testFlow(t, `foo = "\x"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
+	})
+}
+
+func TestNestedArrays(t *testing.T) {
+	testFlow(t, "foo = [[[]]]", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenLeftBracket, "["},
+		token{Position{1, 8}, tokenLeftBracket, "["},
+		token{Position{1, 9}, tokenLeftBracket, "["},
+		token{Position{1, 10}, tokenRightBracket, "]"},
+		token{Position{1, 11}, tokenRightBracket, "]"},
+		token{Position{1, 12}, tokenRightBracket, "]"},
+		token{Position{1, 13}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualNumber(t *testing.T) {
 	testFlow(t, "foo = 42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "42"},
+		token{Position{1, 9}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "+42"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "+42"},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -42", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "-42"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "-42"},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = 4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenFloat, "4.2"},
+		token{Position{1, 10}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = +4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "+4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenFloat, "+4.2"},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 
 	testFlow(t, "foo = -4.2", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenFloat, "-4.2"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenFloat, "-4.2"},
+		token{Position{1, 11}, tokenEOF, ""},
 	})
 }
 
 func TestMultiline(t *testing.T) {
 	testFlow(t, "foo = 42\nbar=21", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "42"},
-		token{tokenKey, "bar"},
-		token{tokenEqual, "="},
-		token{tokenInteger, "21"},
-		token{tokenEOF, ""},
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 7}, tokenInteger, "42"},
+		token{Position{2, 1}, tokenKey, "bar"},
+		token{Position{2, 4}, tokenEqual, "="},
+		token{Position{2, 5}, tokenInteger, "21"},
+		token{Position{2, 7}, tokenEOF, ""},
 	})
 }
 
 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
-	testFlow(t, "foo = \"hello \\u2665\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello \u2665"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "hello ♥"},
+		token{Position{1, 21}, tokenEOF, ""},
 	})
 }
 
 func TestUnicodeString(t *testing.T) {
-	testFlow(t, "foo = \"hello ♥ world\"", []token{
-		token{tokenKey, "foo"},
-		token{tokenEqual, "="},
-		token{tokenString, "hello ♥ world"},
-		token{tokenEOF, ""},
+	testFlow(t, `foo = "hello ♥ world"`, []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenEqual, "="},
+		token{Position{1, 8}, tokenString, "hello ♥ world"},
+		token{Position{1, 22}, tokenEOF, ""},
+	})
+}
+
+func TestKeyGroupArray(t *testing.T) {
+	testFlow(t, "[[foo]]", []token{
+		token{Position{1, 1}, tokenDoubleLeftBracket, "[["},
+		token{Position{1, 3}, tokenKeyGroupArray, "foo"},
+		token{Position{1, 6}, tokenDoubleRightBracket, "]]"},
+		token{Position{1, 8}, tokenEOF, ""},
 	})
 }

match.go

@@ -0,0 +1,227 @@
+package toml
+
+import (
+	"fmt"
+)
+
+// support function to set positions for tomlValues
+// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
+// values returned by the query engines
+func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
+	switch castNode := node.(type) {
+	case *tomlValue:
+		ctx.lastPosition = castNode.position
+		return castNode.value
+	case []*TomlTree:
+		if len(castNode) > 0 {
+			ctx.lastPosition = castNode[0].position
+		}
+		return node
+	default:
+		return node
+	}
+}
+
+// base match
+type matchBase struct {
+	next pathFn
+}
+
+func (f *matchBase) setNext(next pathFn) {
+	f.next = next
+}
+
+// terminating functor - gathers results
+type terminatingFn struct {
+	// empty
+}
+
+func newTerminatingFn() *terminatingFn {
+	return &terminatingFn{}
+}
+
+func (f *terminatingFn) setNext(next pathFn) {
+	// do nothing
+}
+
+func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
+	switch castNode := node.(type) {
+	case *TomlTree:
+		ctx.result.appendResult(node, castNode.position)
+	case *tomlValue:
+		ctx.result.appendResult(node, castNode.position)
+	default:
+		// use last position for scalars
+		ctx.result.appendResult(node, ctx.lastPosition)
+	}
+}
+
+// match single key
+type matchKeyFn struct {
+	matchBase
+	Name string
+}
+
+func newMatchKeyFn(name string) *matchKeyFn {
+	return &matchKeyFn{Name: name}
+}
+
+func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		item := tree.values[f.Name]
+		if item != nil {
+			f.next.call(item, ctx)
+		}
+	}
+}
+
+// match single index
+type matchIndexFn struct {
+	matchBase
+	Idx int
+}
+
+func newMatchIndexFn(idx int) *matchIndexFn {
+	return &matchIndexFn{Idx: idx}
+}
+
+func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		if f.Idx < len(arr) && f.Idx >= 0 {
+			f.next.call(arr[f.Idx], ctx)
+		}
+	}
+}
+
+// filter by slicing
+type matchSliceFn struct {
+	matchBase
+	Start, End, Step int
+}
+
+func newMatchSliceFn(start, end, step int) *matchSliceFn {
+	return &matchSliceFn{Start: start, End: end, Step: step}
+}
+
+func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		// adjust indexes for negative values, reverse ordering
+		realStart, realEnd := f.Start, f.End
+		if realStart < 0 {
+			realStart = len(arr) + realStart
+		}
+		if realEnd < 0 {
+			realEnd = len(arr) + realEnd
+		}
+		if realEnd < realStart {
+			realEnd, realStart = realStart, realEnd // swap
+		}
+		// loop and gather
+		for idx := realStart; idx < realEnd; idx += f.Step {
+			f.next.call(arr[idx], ctx)
+		}
+	}
+}
+
+// match anything
+type matchAnyFn struct {
+	matchBase
+}
+
+func newMatchAnyFn() *matchAnyFn {
+	return &matchAnyFn{}
+}
+
+func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		for _, v := range tree.values {
+			f.next.call(v, ctx)
+		}
+	}
+}
+
+// filter through union
+type matchUnionFn struct {
+	Union []pathFn
+}
+
+func (f *matchUnionFn) setNext(next pathFn) {
+	for _, fn := range f.Union {
+		fn.setNext(next)
+	}
+}
+
+func (f *matchUnionFn) call(node interface{}, ctx *queryContext) {
+	for _, fn := range f.Union {
+		fn.call(node, ctx)
+	}
+}
+
+// match every single last node in the tree
+type matchRecursiveFn struct {
+	matchBase
+}
+
+func newMatchRecursiveFn() *matchRecursiveFn {
+	return &matchRecursiveFn{}
+}
+
+func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		var visit func(tree *TomlTree)
+		visit = func(tree *TomlTree) {
+			for _, v := range tree.values {
+				f.next.call(v, ctx)
+				switch node := v.(type) {
+				case *TomlTree:
+					visit(node)
+				case []*TomlTree:
+					for _, subtree := range node {
+						visit(subtree)
+					}
+				}
+			}
+		}
+		f.next.call(tree, ctx)
+		visit(tree)
+	}
+}
+
+// match based on an externally provided functional filter
+type matchFilterFn struct {
+	matchBase
+	Pos  Position
+	Name string
+}
+
+func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+	return &matchFilterFn{Name: name, Pos: pos}
+}
+
+func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
+	fn, ok := (*ctx.filters)[f.Name]
+	if !ok {
+		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
+			f.Pos, f.Name))
+	}
+	switch castNode := tomlValueCheck(node, ctx).(type) {
+	case *TomlTree:
+		for _, v := range castNode.values {
+			if tv, ok := v.(*tomlValue); ok {
+				if fn(tv.value) {
+					f.next.call(v, ctx)
+				}
+			} else {
+				if fn(v) {
+					f.next.call(v, ctx)
+				}
+			}
+		}
+	case []interface{}:
+		for _, v := range castNode {
+			if fn(v) {
+				f.next.call(v, ctx)
+			}
+		}
+	}
+}

match_test.go

@@ -0,0 +1,202 @@
+package toml
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+// dump path tree to a string
+func pathString(root pathFn) string {
+	result := fmt.Sprintf("%T:", root)
+	switch fn := root.(type) {
+	case *terminatingFn:
+		result += "{}"
+	case *matchKeyFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	case *matchIndexFn:
+		result += fmt.Sprintf("{%d}", fn.Idx)
+		result += pathString(fn.next)
+	case *matchSliceFn:
+		result += fmt.Sprintf("{%d:%d:%d}",
+			fn.Start, fn.End, fn.Step)
+		result += pathString(fn.next)
+	case *matchAnyFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchUnionFn:
+		result += "{["
+		for _, v := range fn.Union {
+			result += pathString(v) + ", "
+		}
+		result += "]}"
+	case *matchRecursiveFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchFilterFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	}
+	return result
+}
+
+func assertPathMatch(t *testing.T, path, ref *Query) bool {
+	pathStr := pathString(path.root)
+	refStr := pathString(ref.root)
+	if pathStr != refStr {
+		t.Errorf("paths do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+		return false
+	}
+	return true
+}
+
+func assertPath(t *testing.T, query string, ref *Query) {
+	path, _ := parseQuery(lexQuery(query))
+	assertPathMatch(t, path, ref)
+}
+
+func buildPath(parts ...pathFn) *Query {
+	query := newQuery()
+	for _, v := range parts {
+		query.appendPath(v)
+	}
+	return query
+}
+
+func TestPathRoot(t *testing.T) {
+	assertPath(t,
+		"$",
+		buildPath(
+		// empty
+		))
+}
+
+func TestPathKey(t *testing.T) {
+	assertPath(t,
+		"$.foo",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketKey(t *testing.T) {
+	assertPath(t,
+		"$[foo]",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketStringKey(t *testing.T) {
+	assertPath(t,
+		"$['foo']",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathIndex(t *testing.T) {
+	assertPath(t,
+		"$[123]",
+		buildPath(
+			newMatchIndexFn(123),
+		))
+}
+
+func TestPathSliceStart(t *testing.T) {
+	assertPath(t,
+		"$[123:]",
+		buildPath(
+			newMatchSliceFn(123, math.MaxInt64, 1),
+		))
+}
+
+func TestPathSliceStartEnd(t *testing.T) {
+	assertPath(t,
+		"$[123:456]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartEndColon(t *testing.T) {
+	assertPath(t,
+		"$[123:456:]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartStep(t *testing.T) {
+	assertPath(t,
+		"$[123::7]",
+		buildPath(
+			newMatchSliceFn(123, math.MaxInt64, 7),
+		))
+}
+
+func TestPathSliceEndStep(t *testing.T) {
+	assertPath(t,
+		"$[:456:7]",
+		buildPath(
+			newMatchSliceFn(0, 456, 7),
+		))
+}
+
+func TestPathSliceStep(t *testing.T) {
+	assertPath(t,
+		"$[::7]",
+		buildPath(
+			newMatchSliceFn(0, math.MaxInt64, 7),
+		))
+}
+
+func TestPathSliceAll(t *testing.T) {
+	assertPath(t,
+		"$[123:456:7]",
+		buildPath(
+			newMatchSliceFn(123, 456, 7),
+		))
+}
+
+func TestPathAny(t *testing.T) {
+	assertPath(t,
+		"$.*",
+		buildPath(
+			newMatchAnyFn(),
+		))
+}
+
+func TestPathUnion(t *testing.T) {
+	assertPath(t,
+		"$[foo, bar, baz]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchKeyFn("foo"),
+				newMatchKeyFn("bar"),
+				newMatchKeyFn("baz"),
+			}},
+		))
+}
+
+func TestPathRecurse(t *testing.T) {
+	assertPath(t,
+		"$..*",
+		buildPath(
+			newMatchRecursiveFn(),
+		))
+}
+
+func TestPathFilterExpr(t *testing.T) {
+	assertPath(t,
+		"$[?('foo'),?(bar)]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchFilterFn("foo", Position{}),
+				newMatchFilterFn("bar", Position{}),
+			}},
+		))
+}

parser.go

@@ -4,26 +4,34 @@ package toml
 
 import (
 	"fmt"
+	"reflect"
 	"strconv"
+	"strings"
 	"time"
 )
 
-type parser struct {
-	flow         chan token
-	tree         *TomlTree
-	tokensBuffer []token
-	currentGroup string
+type tomlParser struct {
+	flow          chan token
+	tree          *TomlTree
+	tokensBuffer  []token
+	currentGroup  []string
+	seenGroupKeys []string
 }
 
-type parserStateFn func(*parser) parserStateFn
+type tomlParserStateFn func() tomlParserStateFn
 
-func (p *parser) run() {
-	for state := parseStart; state != nil; {
-		state = state(p)
+// Formats and panics an error message based on a token
+func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) {
+	panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...))
+}
+
+func (p *tomlParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
 	}
 }
 
-func (p *parser) peek() *token {
+func (p *tomlParser) peek() *token {
 	if len(p.tokensBuffer) != 0 {
 		return &(p.tokensBuffer[0])
 	}
@@ -36,17 +44,17 @@ func (p *parser) peek() *token {
 	return &tok
 }
 
-func (p *parser) assume(typ tokenType) {
+func (p *tomlParser) assume(typ tokenType) {
 	tok := p.getToken()
 	if tok == nil {
-		panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ))
+		p.raiseError(tok, "was expecting token %s, but token stream is empty", tok)
 	}
 	if tok.typ != typ {
-		panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ))
+		p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok)
 	}
 }
 
-func (p *parser) getToken() *token {
+func (p *tomlParser) getToken() *token {
 	if len(p.tokensBuffer) != 0 {
 		tok := p.tokensBuffer[0]
 		p.tokensBuffer = p.tokensBuffer[1:]
@@ -59,7 +67,7 @@ func (p *parser) getToken() *token {
 	return &tok
 }
 
-func parseStart(p *parser) parserStateFn {
+func (p *tomlParser) parseStart() tomlParserStateFn {
 	tok := p.peek()
 
 	// end of stream, parsing is finished
@@ -68,46 +76,130 @@ func parseStart(p *parser) parserStateFn {
 	}
 
 	switch tok.typ {
+	case tokenDoubleLeftBracket:
+		return p.parseGroupArray
 	case tokenLeftBracket:
-		return parseGroup
+		return p.parseGroup
 	case tokenKey:
-		return parseAssign
+		return p.parseAssign
 	case tokenEOF:
 		return nil
 	default:
-		panic("unexpected token")
+		p.raiseError(tok, "unexpected token")
 	}
 	return nil
 }
 
-func parseGroup(p *parser) parserStateFn {
-	p.getToken() // discard the [
+func (p *tomlParser) parseGroupArray() tomlParserStateFn {
+	startToken := p.getToken() // discard the [[
+	key := p.getToken()
+	if key.typ != tokenKeyGroupArray {
+		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
+	}
+
+	// get or create group array element at the indicated part in the path
+	keys := strings.Split(key.val, ".")
+	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
+	destTree := p.tree.GetPath(keys)
+	var array []*TomlTree
+	if destTree == nil {
+		array = make([]*TomlTree, 0)
+	} else if destTree.([]*TomlTree) != nil {
+		array = destTree.([]*TomlTree)
+	} else {
+		p.raiseError(key, "key %s is already assigned and not of type group array", key)
+	}
+	p.currentGroup = keys
+
+	// add a new tree to the end of the group array
+	newTree := newTomlTree()
+	newTree.position = startToken.Position
+	array = append(array, newTree)
+	p.tree.SetPath(p.currentGroup, array)
+
+	// remove all keys that were children of this group array
+	prefix := key.val + "."
+	found := false
+	for ii := 0; ii < len(p.seenGroupKeys); {
+		groupKey := p.seenGroupKeys[ii]
+		if strings.HasPrefix(groupKey, prefix) {
+			p.seenGroupKeys = append(p.seenGroupKeys[:ii], p.seenGroupKeys[ii+1:]...)
+		} else {
+			found = (groupKey == key.val)
+			ii++
+		}
+	}
+
+	// keep this key name from use by other kinds of assignments
+	if !found {
+		p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	}
+
+	// move to next parser state
+	p.assume(tokenDoubleRightBracket)
+	return p.parseStart
+}
+
+func (p *tomlParser) parseGroup() tomlParserStateFn {
+	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
+		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
+	}
+	for _, item := range p.seenGroupKeys {
+		if item == key.val {
+			p.raiseError(key, "duplicated tables")
+		}
+	}
+
+	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	keys := strings.Split(key.val, ".")
+	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
+		p.raiseError(key, "%s", err)
 	}
-	p.tree.createSubTree(key.val)
 	p.assume(tokenRightBracket)
-	p.currentGroup = key.val
-	return parseStart(p)
+	p.currentGroup = keys
+	return p.parseStart
 }
 
-func parseAssign(p *parser) parserStateFn {
+func (p *tomlParser) parseAssign() tomlParserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
-	value := parseRvalue(p)
-	final_key := key.val
-	if p.currentGroup != "" {
-		final_key = p.currentGroup + "." + key.val
+	value := p.parseRvalue()
+	var groupKey []string
+	if len(p.currentGroup) > 0 {
+		groupKey = p.currentGroup
+	} else {
+		groupKey = []string{}
 	}
-	p.tree.Set(final_key, value)
-	return parseStart(p)
+
+	// find the group to assign, looking out for arrays of groups
+	var targetNode *TomlTree
+	switch node := p.tree.GetPath(groupKey).(type) {
+	case []*TomlTree:
+		targetNode = node[len(node)-1]
+	case *TomlTree:
+		targetNode = node
+	default:
+		p.raiseError(key, "Unknown group type for path: %s",
+			strings.Join(groupKey, "."))
+	}
+
+	// assign value to the found group
+	localKey := []string{key.val}
+	finalKey := append(groupKey, key.val)
+	if targetNode.GetPath(localKey) != nil {
+		p.raiseError(key, "The following key was defined twice: %s",
+			strings.Join(finalKey, "."))
+	}
+	targetNode.values[key.val] = &tomlValue{value, key.Position}
+	return p.parseStart
 }
 
-func parseRvalue(p *parser) interface{} {
+func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
-		panic("expecting a value")
+		p.raiseError(tok, "expecting a value")
 	}
 
 	switch tok.typ {
@@ -120,51 +212,58 @@ func parseRvalue(p *parser) interface{} {
 	case tokenInteger:
 		val, err := strconv.ParseInt(tok.val, 10, 64)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
 		val, err := strconv.ParseFloat(tok.val, 64)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenDate:
 		val, err := time.Parse(time.RFC3339, tok.val)
 		if err != nil {
-			panic(err)
+			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenLeftBracket:
-		return parseArray(p)
+		return p.parseArray()
+	case tokenError:
+		p.raiseError(tok, "%s", tok)
 	}
 
-	panic("never reached")
+	p.raiseError(tok, "never reached")
 
 	return nil
 }
 
-func parseArray(p *parser) []interface{} {
-	array := make([]interface{}, 0)
+func (p *tomlParser) parseArray() []interface{} {
+	var array []interface{}
+	arrayType := reflect.TypeOf(nil)
 	for {
 		follow := p.peek()
 		if follow == nil || follow.typ == tokenEOF {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ == tokenRightBracket {
 			p.getToken()
 			return array
 		}
-		val := parseRvalue(p)
+		val := p.parseRvalue()
+		if arrayType == nil {
+			arrayType = reflect.TypeOf(val)
+		}
+		if reflect.TypeOf(val) != arrayType {
+			p.raiseError(follow, "mixed types in array")
+		}
 		array = append(array, val)
 		follow = p.peek()
 		if follow == nil {
-			panic("unterminated array")
+			p.raiseError(follow, "unterminated array")
 		}
 		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
-			fmt.Println(follow.typ)
-			fmt.Println(follow.val)
-			panic("missing comma")
+			p.raiseError(follow, "missing comma")
 		}
 		if follow.typ == tokenComma {
 			p.getToken()
@@ -173,14 +272,16 @@ func parseArray(p *parser) []interface{} {
 	return array
 }
 
-func parse(flow chan token) *TomlTree {
-	result := make(TomlTree)
-	parser := &parser{
-		flow:         flow,
-		tree:         &result,
-		tokensBuffer: make([]token, 0),
-		currentGroup: "",
+func parseToml(flow chan token) *TomlTree {
+	result := newTomlTree()
+	result.position = Position{1, 1}
+	parser := &tomlParser{
+		flow:          flow,
+		tree:          result,
+		tokensBuffer:  make([]token, 0),
+		currentGroup:  make([]string, 0),
+		seenGroupKeys: make([]string, 0),
 	}
 	parser.run()
-	return parser.tree
+	return result
 }

parser_test.go

@@ -12,16 +12,26 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 		return
 	}
 	for k, v := range ref {
-		if fmt.Sprintf("%v", tree.Get(k)) != fmt.Sprintf("%v", v) {
-			t.Log("was expecting", v, "at", k, "but got", tree.Get(k))
-			t.Error()
+		// NOTE: directly access key instead of resolve by path
+		// NOTE: see TestSpecialKV
+		switch node := tree.GetPath([]string{k}).(type) {
+		case []*TomlTree:
+			for idx, item := range node {
+				assertTree(t, item, err, v.([]map[string]interface{})[idx])
+			}
+		case *TomlTree:
+			assertTree(t, node, err, v.(map[string]interface{}))
+		default:
+			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
+				t.Errorf("was expecting %v at %v but got %v", v, k, node)
+			}
 		}
 	}
 }
 
 func TestCreateSubTree(t *testing.T) {
-	tree := make(TomlTree)
-	tree.createSubTree("a.b.c")
+	tree := newTomlTree()
+	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
 		t.Fail()
@@ -41,6 +51,15 @@ func TestSimpleKV(t *testing.T) {
 	})
 }
 
+// NOTE: from the BurntSushi test suite
+// NOTE: this test is pure evil due to the embedded '.'
+func TestSpecialKV(t *testing.T) {
+	tree, err := Load("~!@#$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
+	assertTree(t, tree, err, map[string]interface{}{
+		"~!@#$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
+	})
+}
+
 func TestSimpleNumbers(t *testing.T) {
 	tree, err := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -98,7 +117,13 @@ func TestBools(t *testing.T) {
 func TestNestedKeys(t *testing.T) {
 	tree, err := Load("[a.b.c]\nd = 42")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a.b.c.d": int64(42),
+		"a": map[string]interface{}{
+			"b": map[string]interface{}{
+				"c": map[string]interface{}{
+					"d": int64(42),
+				},
+			},
+		},
 	})
 }
 
@@ -142,6 +167,25 @@ func TestArrayNested(t *testing.T) {
 	})
 }
 
+func TestNestedEmptyArrays(t *testing.T) {
+	tree, err := Load("a = [[[]]]")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": [][][]interface{}{[][]interface{}{[]interface{}{}}},
+	})
+}
+
+func TestArrayMixedTypes(t *testing.T) {
+	_, err := Load("a = [42, 16.0]")
+	if err.Error() != "(1, 10): mixed types in array" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = [42, \"hello\"]")
+	if err.Error() != "(1, 11): mixed types in array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestArrayNestedStrings(t *testing.T) {
 	tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -151,14 +195,14 @@ func TestArrayNestedStrings(t *testing.T) {
 
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "expecting a value" {
+	if err.Error() != "(1, 4): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
 
 func TestUnterminatedArray(t *testing.T) {
 	_, err := Load("a = [1,")
-	if err.Error() != "unterminated array" {
+	if err.Error() != "(1, 8): unterminated array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -170,6 +214,67 @@ func TestNewlinesInArrays(t *testing.T) {
 	})
 }
 
+func TestArrayWithExtraComma(t *testing.T) {
+	tree, err := Load("a = [1,\n2,\n3,\n]")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": []int64{int64(1), int64(2), int64(3)},
+	})
+}
+
+func TestArrayWithExtraCommaComment(t *testing.T) {
+	tree, err := Load("a = [1, # wow\n2, # such items\n3, # so array\n]")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": []int64{int64(1), int64(2), int64(3)},
+	})
+}
+
+func TestDuplicateGroups(t *testing.T) {
+	_, err := Load("[foo]\na=2\n[foo]b=3")
+	if err.Error() != "(3, 2): duplicated tables" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestDuplicateKeys(t *testing.T) {
+	_, err := Load("foo = 2\nfoo = 3")
+	if err.Error() != "(2, 1): The following key was defined twice: foo" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestEmptyIntermediateTable(t *testing.T) {
+	_, err := Load("[foo..bar]")
+	if err.Error() != "(1, 2): empty intermediate table" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestImplicitDeclarationBefore(t *testing.T) {
+	tree, err := Load("[a.b.c]\nanswer = 42\n[a]\nbetter = 43")
+	assertTree(t, tree, err, map[string]interface{}{
+		"a": map[string]interface{}{
+			"b": map[string]interface{}{
+				"c": map[string]interface{}{
+					"answer": int64(42),
+				},
+			},
+			"better": int64(43),
+		},
+	})
+}
+
+func TestFloatsWithoutLeadingZeros(t *testing.T) {
+	_, err := Load("a = .42")
+	if err.Error() != "(1, 4): cannot start float with a dot" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a = -.42")
+	if err.Error() != "(1, 5): cannot start float with a dot" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestMissingFile(t *testing.T) {
 	_, err := LoadFile("foo.toml")
 	if err.Error() != "open foo.toml: no such file or directory" {
@@ -181,19 +286,153 @@ func TestParseFile(t *testing.T) {
 	tree, err := LoadFile("example.toml")
 
 	assertTree(t, tree, err, map[string]interface{}{
-		"title":                   "TOML Example",
-		"owner.name":              "Tom Preston-Werner",
-		"owner.organization":      "GitHub",
-		"owner.bio":               "GitHub Cofounder & CEO\nLikes tater tots and beer.",
-		"owner.dob":               time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
-		"database.server":         "192.168.1.1",
-		"database.ports":          []int64{8001, 8001, 8002},
-		"database.connection_max": 5000,
-		"database.enabled":        true,
-		"servers.alpha.ip":        "10.0.0.1",
-		"servers.alpha.dc":        "eqdc10",
-		"servers.beta.ip":         "10.0.0.2",
-		"servers.beta.dc":         "eqdc10",
-		"clients.data":            []interface{}{[]string{"gamma", "delta"}, []int64{1, 2}},
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
 	})
 }
+
+func TestParseKeyGroupArray(t *testing.T) {
+	tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
+	assertTree(t, tree, err, map[string]interface{}{
+		"foo": map[string]interface{}{
+			"bar": []map[string]interface{}{
+				{"a": int64(42)},
+				{"a": int64(69)},
+			},
+		},
+	})
+}
+
+func TestParseKeyGroupArraySpec(t *testing.T) {
+	tree, err := Load("[[fruit]]\n name=\"apple\"\n [fruit.physical]\n color=\"red\"\n shape=\"round\"\n [[fruit]]\n name=\"banana\"")
+	assertTree(t, tree, err, map[string]interface{}{
+		"fruit": []map[string]interface{}{
+			{"name": "apple", "physical": map[string]interface{}{"color": "red", "shape": "round"}},
+			{"name": "banana"},
+		},
+	})
+}
+
+func TestToTomlValue(t *testing.T) {
+	for idx, item := range []struct {
+		Value  interface{}
+		Expect string
+	}{
+		{int64(12345), "12345"},
+		{float64(123.45), "123.45"},
+		{bool(true), "true"},
+		{"hello world", "\"hello world\""},
+		{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
+		{"\x05", "\"\\u0005\""},
+		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+			"1979-05-27T07:32:00Z"},
+		{[]interface{}{"gamma", "delta"},
+			"[\n  \"gamma\",\n  \"delta\",\n]"},
+	} {
+		result := toTomlValue(item.Value, 0)
+		if result != item.Expect {
+			t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect)
+		}
+	}
+}
+
+func TestToString(t *testing.T) {
+	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
+	if err != nil {
+		t.Errorf("Test failed to parse: %v", err)
+		return
+	}
+	result := tree.ToString()
+	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
+	if result != expected {
+		t.Errorf("Expected got '%s', expected '%s'", result, expected)
+	}
+}
+
+func assertPosition(t *testing.T, text string, ref map[string]Position) {
+	tree, err := Load(text)
+	if err != nil {
+		t.Errorf("Error loading document text: `%v`", text)
+		t.Errorf("Error: %v", err)
+	}
+	for path, pos := range ref {
+		testPos := tree.GetPosition(path)
+		if testPos.Invalid() {
+			t.Errorf("Failed to query tree path or path has invalid position: %s", path)
+		} else if pos != testPos {
+			t.Errorf("Expected position %v, got %v instead", pos, testPos)
+		}
+	}
+}
+
+func TestDocumentPositions(t *testing.T) {
+	assertPosition(t,
+		"[foo]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 1},
+			"foo.bar": Position{2, 1},
+			"foo.baz": Position{3, 1},
+		})
+}
+
+func TestDocumentPositionsWithSpaces(t *testing.T) {
+	assertPosition(t,
+		"  [foo]\n  bar=42\n  baz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 3},
+			"foo.bar": Position{2, 3},
+			"foo.baz": Position{3, 3},
+		})
+}
+
+func TestDocumentPositionsWithGroupArray(t *testing.T) {
+	assertPosition(t,
+		"[[foo]]\nbar=42\nbaz=69",
+		map[string]Position{
+			"":        Position{1, 1},
+			"foo":     Position{1, 1},
+			"foo.bar": Position{2, 1},
+			"foo.baz": Position{3, 1},
+		})
+}
+
+func TestNestedTreePosition(t *testing.T) {
+	assertPosition(t,
+		"[foo.bar]\na=42\nb=69",
+		map[string]Position{
+			"":          Position{1, 1},
+			"foo":       Position{1, 1},
+			"foo.bar":   Position{1, 1},
+			"foo.bar.a": Position{2, 1},
+			"foo.bar.b": Position{3, 1},
+		})
+}

position.go

@@ -0,0 +1,31 @@
+// Position support for go-toml
+
+package toml
+
+import (
+	"fmt"
+)
+
+/*
+  Position of a document element within a TOML document.
+
+  Line and Col are both 1-indexed positions for the element's line number and
+  column number, respectively.  Values of zero or less will cause Invalid(),
+  to return true.
+*/
+type Position struct {
+	Line int // line within the document
+	Col  int // column within the line
+}
+
+// String representation of the position.
+// Displays 1-indexed line and column numbers.
+func (p *Position) String() string {
+	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
+}
+
+// Returns whether or not the position is valid (i.e. with negative or
+// null values)
+func (p *Position) Invalid() bool {
+	return p.Line <= 0 || p.Col <= 0
+}

position_test.go

@@ -0,0 +1,29 @@
+// Testing support for go-toml
+
+package toml
+
+import (
+	"testing"
+)
+
+func TestPositionString(t *testing.T) {
+	p := Position{123, 456}
+	expected := "(123, 456)"
+	value := p.String()
+
+	if value != expected {
+		t.Errorf("Expected %v, got %v instead", expected, value)
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	for i, v := range []Position{
+		Position{0, 1234},
+		Position{1234, 0},
+		Position{0, 0},
+	} {
+		if !v.Invalid() {
+			t.Errorf("Position at %v is valid: %v", i, v)
+		}
+	}
+}

query.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"time"
+)
+
+//  Type of a user-defined filter function, for use with Query.SetFilter().
+//
+//  The return value of the function must indicate if 'node' is to be included
+//  at this stage of the TOML path.  Returning true will include the node, and
+//  returning false will exclude it.
+//
+//  NOTE: Care should be taken to write script callbacks such that they are safe
+//  to use from multiple goroutines.
+type NodeFilterFn func(node interface{}) bool
+
+// The result of Executing a Query
+type QueryResult struct {
+	items     []interface{}
+	positions []Position
+}
+
+// appends a value/position pair to the result set
+func (r *QueryResult) appendResult(node interface{}, pos Position) {
+	r.items = append(r.items, node)
+	r.positions = append(r.positions, pos)
+}
+
+// Set of values within a QueryResult.  The order of values is not guaranteed
+// to be in document order, and may be different each time a query is executed.
+func (r *QueryResult) Values() []interface{} {
+	return r.items
+}
+
+// Set of positions for values within a QueryResult.  Each index in Positions()
+// corresponds to the entry in Value() of the same index.
+func (r *QueryResult) Positions() []Position {
+	return r.positions
+}
+
+// runtime context for executing query paths
+type queryContext struct {
+	result       *QueryResult
+	filters      *map[string]NodeFilterFn
+	lastPosition Position
+}
+
+// generic path functor interface
+type pathFn interface {
+	setNext(next pathFn)
+	call(node interface{}, ctx *queryContext)
+}
+
+// A Query is the representation of a compiled TOML path.  A Query is safe
+// for concurrent use by multiple goroutines.
+type Query struct {
+	root    pathFn
+	tail    pathFn
+	filters *map[string]NodeFilterFn
+}
+
+func newQuery() *Query {
+	return &Query{
+		root:    nil,
+		tail:    nil,
+		filters: &defaultFilterFunctions,
+	}
+}
+
+func (q *Query) appendPath(next pathFn) {
+	if q.root == nil {
+		q.root = next
+	} else {
+		q.tail.setNext(next)
+	}
+	q.tail = next
+	next.setNext(newTerminatingFn()) // init the next functor
+}
+
+// Compiles a TOML path expression.  The returned Query can be used to match
+// elements within a TomlTree and its descendants.
+func CompileQuery(path string) (*Query, error) {
+	return parseQuery(lexQuery(path))
+}
+
+// Executes a query against a TomlTree, and returns the result of the query.
+func (q *Query) Execute(tree *TomlTree) *QueryResult {
+	result := &QueryResult{
+		items:     []interface{}{},
+		positions: []Position{},
+	}
+	if q.root == nil {
+		result.appendResult(tree, tree.GetPosition(""))
+	} else {
+		ctx := &queryContext{
+			result:  result,
+			filters: q.filters,
+		}
+		q.root.call(tree, ctx)
+	}
+	return result
+}
+
+// Sets a user-defined filter function.  These may be used inside "?(..)" query
+// expressions to filter TOML document elements within a query.
+func (q *Query) SetFilter(name string, fn NodeFilterFn) {
+	if q.filters == &defaultFilterFunctions {
+		// clone the static table
+		q.filters = &map[string]NodeFilterFn{}
+		for k, v := range defaultFilterFunctions {
+			(*q.filters)[k] = v
+		}
+	}
+	(*q.filters)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]NodeFilterFn{
+	"tree": func(node interface{}) bool {
+		_, ok := node.(*TomlTree)
+		return ok
+	},
+	"int": func(node interface{}) bool {
+		_, ok := node.(int64)
+		return ok
+	},
+	"float": func(node interface{}) bool {
+		_, ok := node.(float64)
+		return ok
+	},
+	"string": func(node interface{}) bool {
+		_, ok := node.(string)
+		return ok
+	},
+	"time": func(node interface{}) bool {
+		_, ok := node.(time.Time)
+		return ok
+	},
+	"bool": func(node interface{}) bool {
+		_, ok := node.(bool)
+		return ok
+	},
+}

querylexer.go

@@ -0,0 +1,339 @@
+// TOML JSONPath lexer.
+//
+// Written using the principles developed by Rob Pike in
+// http://www.youtube.com/watch?v=HxaD_trXwRE
+
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// Lexer state function
+type queryLexStateFn func() queryLexStateFn
+
+// Lexer definition
+type queryLexer struct {
+	input      string
+	start      int
+	pos        int
+	width      int
+	tokens     chan token
+	depth      int
+	line       int
+	col        int
+	stringTerm string
+}
+
+func (l *queryLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
+}
+
+func (l *queryLexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line++
+			l.col = 1
+		} else {
+			l.col++
+		}
+		i += width
+	}
+	// advance start position to next token
+	l.start = l.pos
+}
+
+func (l *queryLexer) emit(t tokenType) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      l.input[l.start:l.pos],
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) emitWithValue(t tokenType, value string) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) next() rune {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	var r rune
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *queryLexer) ignore() {
+	l.nextStart()
+}
+
+func (l *queryLexer) backup() {
+	l.pos -= l.width
+}
+
+func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func (l *queryLexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *queryLexer) accept(valid string) bool {
+	if strings.IndexRune(valid, l.next()) >= 0 {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+func (l *queryLexer) follow(next string) bool {
+	return strings.HasPrefix(l.input[l.pos:], next)
+}
+
+func (l *queryLexer) lexVoid() queryLexStateFn {
+	for {
+		next := l.peek()
+		switch next {
+		case '$':
+			l.pos++
+			l.emit(tokenDollar)
+			continue
+		case '.':
+			if l.follow("..") {
+				l.pos += 2
+				l.emit(tokenDotDot)
+			} else {
+				l.pos++
+				l.emit(tokenDot)
+			}
+			continue
+		case '[':
+			l.pos++
+			l.emit(tokenLeftBracket)
+			continue
+		case ']':
+			l.pos++
+			l.emit(tokenRightBracket)
+			continue
+		case ',':
+			l.pos++
+			l.emit(tokenComma)
+			continue
+		case '*':
+			l.pos++
+			l.emit(tokenStar)
+			continue
+		case '(':
+			l.pos++
+			l.emit(tokenLeftParen)
+			continue
+		case ')':
+			l.pos++
+			l.emit(tokenRightParen)
+			continue
+		case '?':
+			l.pos++
+			l.emit(tokenQuestion)
+			continue
+		case ':':
+			l.pos++
+			l.emit(tokenColon)
+			continue
+		case '\'':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		case '"':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		}
+
+		if isSpace(next) {
+			l.next()
+			l.ignore()
+			continue
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if l.next() == eof {
+			break
+		}
+
+		return l.errorf("unexpected char: '%v'", next)
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexKey() queryLexStateFn {
+	for {
+		next := l.peek()
+		if !isAlphanumeric(next) {
+			l.emit(tokenKey)
+			return l.lexVoid
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexString() queryLexStateFn {
+	l.pos++
+	l.ignore()
+	growingString := ""
+
+	for {
+		if l.follow(l.stringTerm) {
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
+			l.ignore()
+			return l.lexVoid
+		}
+
+		if l.follow("\\\"") {
+			l.pos++
+			growingString += "\""
+		} else if l.follow("\\'") {
+			l.pos++
+			growingString += "'"
+		} else if l.follow("\\n") {
+			l.pos++
+			growingString += "\n"
+		} else if l.follow("\\b") {
+			l.pos++
+			growingString += "\b"
+		} else if l.follow("\\f") {
+			l.pos++
+			growingString += "\f"
+		} else if l.follow("\\/") {
+			l.pos++
+			growingString += "/"
+		} else if l.follow("\\t") {
+			l.pos++
+			growingString += "\t"
+		} else if l.follow("\\r") {
+			l.pos++
+			growingString += "\r"
+		} else if l.follow("\\\\") {
+			l.pos++
+			growingString += "\\"
+		} else if l.follow("\\u") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 4; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos++
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+		} else {
+			growingString += string(l.peek())
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+
+	return l.errorf("unclosed string")
+}
+
+func (l *queryLexer) lexNumber() queryLexStateFn {
+	l.ignore()
+	if !l.accept("+") {
+		l.accept("-")
+	}
+	pointSeen := false
+	digitSeen := false
+	for {
+		next := l.next()
+		if next == '.' {
+			if pointSeen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
+			pointSeen = true
+		} else if isDigit(next) {
+			digitSeen = true
+		} else {
+			l.backup()
+			break
+		}
+		if pointSeen && !digitSeen {
+			return l.errorf("cannot start float with a dot")
+		}
+	}
+
+	if !digitSeen {
+		return l.errorf("no digit in that number")
+	}
+	if pointSeen {
+		l.emit(tokenFloat)
+	} else {
+		l.emit(tokenInteger)
+	}
+	return l.lexVoid
+}
+
+// Entry point
+func lexQuery(input string) chan token {
+	l := &queryLexer{
+		input:  input,
+		tokens: make(chan token),
+		line:   1,
+		col:    1,
+	}
+	go l.run()
+	return l.tokens
+}

querylexer_test.go

@@ -0,0 +1,97 @@
+package toml
+
+import (
+	"testing"
+)
+
+func testQLFlow(t *testing.T, input string, expectedFlow []token) {
+	ch := lexQuery(input)
+	for idx, expected := range expectedFlow {
+		token := <-ch
+		if token != expected {
+			t.Log("While testing #", idx, ":", input)
+			t.Log("compared", token, "to", expected)
+			t.Log(token.val, "<->", expected.val)
+			t.Log(token.typ, "<->", expected.typ)
+			t.Log(token.Line, "<->", expected.Line)
+			t.Log(token.Col, "<->", expected.Col)
+			t.FailNow()
+		}
+	}
+
+	tok, ok := <-ch
+	if ok {
+		t.Log("channel is not closed!")
+		t.Log(len(ch)+1, "tokens remaining:")
+
+		t.Log("token ->", tok)
+		for token := range ch {
+			t.Log("token ->", token)
+		}
+		t.FailNow()
+	}
+}
+
+func TestLexSpecialChars(t *testing.T) {
+	testQLFlow(t, " .$[]..()?*", []token{
+		token{Position{1, 2}, tokenDot, "."},
+		token{Position{1, 3}, tokenDollar, "$"},
+		token{Position{1, 4}, tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenRightBracket, "]"},
+		token{Position{1, 6}, tokenDotDot, ".."},
+		token{Position{1, 8}, tokenLeftParen, "("},
+		token{Position{1, 9}, tokenRightParen, ")"},
+		token{Position{1, 10}, tokenQuestion, "?"},
+		token{Position{1, 11}, tokenStar, "*"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexString(t *testing.T) {
+	testQLFlow(t, "'foo'", []token{
+		token{Position{1, 2}, tokenString, "foo"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexDoubleString(t *testing.T) {
+	testQLFlow(t, `"bar"`, []token{
+		token{Position{1, 2}, tokenString, "bar"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexKey(t *testing.T) {
+	testQLFlow(t, "foo", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 4}, tokenEOF, ""},
+	})
+}
+
+func TestLexRecurse(t *testing.T) {
+	testQLFlow(t, "$..*", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenDotDot, ".."},
+		token{Position{1, 4}, tokenStar, "*"},
+		token{Position{1, 5}, tokenEOF, ""},
+	})
+}
+
+func TestLexBracketKey(t *testing.T) {
+	testQLFlow(t, "$[foo]", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenLeftBracket, "["},
+		token{Position{1, 3}, tokenKey, "foo"},
+		token{Position{1, 6}, tokenRightBracket, "]"},
+		token{Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexSpace(t *testing.T) {
+	testQLFlow(t, "foo bar baz", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenKey, "bar"},
+		token{Position{1, 9}, tokenKey, "baz"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}

queryparser.go

@@ -0,0 +1,275 @@
+/*
+  Based on the "jsonpath" spec/concept.
+
+  http://goessner.net/articles/JsonPath/
+  https://code.google.com/p/json-path/
+*/
+
+package toml
+
+import (
+	"fmt"
+	"math"
+)
+
+type queryParser struct {
+	flow         chan token
+	tokensBuffer []token
+	query        *Query
+	union        []pathFn
+	err          error
+}
+
+type queryParserStateFn func() queryParserStateFn
+
+// Formats and panics an error message based on a token
+func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn {
+	p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...)
+	return nil // trigger parse to end
+}
+
+func (p *queryParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
+	}
+}
+
+func (p *queryParser) backup(tok *token) {
+	p.tokensBuffer = append(p.tokensBuffer, *tok)
+}
+
+func (p *queryParser) peek() *token {
+	if len(p.tokensBuffer) != 0 {
+		return &(p.tokensBuffer[0])
+	}
+
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	p.backup(&tok)
+	return &tok
+}
+
+func (p *queryParser) lookahead(types ...tokenType) bool {
+	result := true
+	buffer := []token{}
+
+	for _, typ := range types {
+		tok := p.getToken()
+		if tok == nil {
+			result = false
+			break
+		}
+		buffer = append(buffer, *tok)
+		if tok.typ != typ {
+			result = false
+			break
+		}
+	}
+	// add the tokens back to the buffer, and return
+	p.tokensBuffer = append(p.tokensBuffer, buffer...)
+	return result
+}
+
+func (p *queryParser) getToken() *token {
+	if len(p.tokensBuffer) != 0 {
+		tok := p.tokensBuffer[0]
+		p.tokensBuffer = p.tokensBuffer[1:]
+		return &tok
+	}
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	return &tok
+}
+
+func (p *queryParser) parseStart() queryParserStateFn {
+	tok := p.getToken()
+
+	if tok == nil || tok.typ == tokenEOF {
+		return nil
+	}
+
+	if tok.typ != tokenDollar {
+		return p.parseError(tok, "Expected '$' at start of expression")
+	}
+
+	return p.parseMatchExpr
+}
+
+// handle '.' prefix, '[]', and '..'
+func (p *queryParser) parseMatchExpr() queryParserStateFn {
+	tok := p.getToken()
+	switch tok.typ {
+	case tokenDotDot:
+		p.query.appendPath(&matchRecursiveFn{})
+		// nested parse for '..'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenLeftBracket:
+			return p.parseBracketExpr
+		case tokenStar:
+			// do nothing - the recursive predicate is enough
+			return p.parseMatchExpr
+		}
+
+	case tokenDot:
+		// nested parse for '.'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenStar:
+			p.query.appendPath(&matchAnyFn{})
+			return p.parseMatchExpr
+		}
+
+	case tokenLeftBracket:
+		return p.parseBracketExpr
+
+	case tokenEOF:
+		return nil // allow EOF at this stage
+	}
+	return p.parseError(tok, "expected match expression")
+	return nil
+}
+
+func (p *queryParser) parseBracketExpr() queryParserStateFn {
+	if p.lookahead(tokenInteger, tokenColon) {
+		return p.parseSliceExpr
+	}
+	if p.peek().typ == tokenColon {
+		return p.parseSliceExpr
+	}
+	return p.parseUnionExpr
+}
+
+func (p *queryParser) parseUnionExpr() queryParserStateFn {
+	var tok *token
+
+	// this state can be traversed after some sub-expressions
+	// so be careful when setting up state in the parser
+	if p.union == nil {
+		p.union = []pathFn{}
+	}
+
+loop: // labeled loop for easy breaking
+	for {
+		if len(p.union) > 0 {
+			// parse delimiter or terminator
+			tok = p.getToken()
+			switch tok.typ {
+			case tokenComma:
+				// do nothing
+			case tokenRightBracket:
+				break loop
+			default:
+				return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val)
+			}
+		}
+
+		// parse sub expression
+		tok = p.getToken()
+		switch tok.typ {
+		case tokenInteger:
+			p.union = append(p.union, newMatchIndexFn(tok.Int()))
+		case tokenKey:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenString:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenQuestion:
+			return p.parseFilterExpr
+		default:
+			return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union))
+		}
+	}
+
+	// if there is only one sub-expression, use that instead
+	if len(p.union) == 1 {
+		p.query.appendPath(p.union[0])
+	} else {
+		p.query.appendPath(&matchUnionFn{p.union})
+	}
+
+	p.union = nil // clear out state
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseSliceExpr() queryParserStateFn {
+	// init slice to grab all elements
+	start, end, step := 0, math.MaxInt64, 1
+
+	// parse optional start
+	tok := p.getToken()
+	if tok.typ == tokenInteger {
+		start = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ':'")
+	}
+
+	// parse optional end
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		end = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ == tokenRightBracket {
+		p.query.appendPath(newMatchSliceFn(start, end, step))
+		return p.parseMatchExpr
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ']' or ':'")
+	}
+
+	// parse optional step
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		step = tok.Int()
+		if step < 0 {
+			return p.parseError(tok, "step must be a positive value")
+		}
+		tok = p.getToken()
+	}
+	if tok.typ != tokenRightBracket {
+		return p.parseError(tok, "expected ']'")
+	}
+
+	p.query.appendPath(newMatchSliceFn(start, end, step))
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseFilterExpr() queryParserStateFn {
+	tok := p.getToken()
+	if tok.typ != tokenLeftParen {
+		return p.parseError(tok, "expected left-parenthesis for filter expression")
+	}
+	tok = p.getToken()
+	if tok.typ != tokenKey && tok.typ != tokenString {
+		return p.parseError(tok, "expected key or string for filter funciton name")
+	}
+	name := tok.val
+	tok = p.getToken()
+	if tok.typ != tokenRightParen {
+		return p.parseError(tok, "expected right-parenthesis for filter expression")
+	}
+	p.union = append(p.union, newMatchFilterFn(name, tok.Position))
+	return p.parseUnionExpr
+}
+
+func parseQuery(flow chan token) (*Query, error) {
+	parser := &queryParser{
+		flow:         flow,
+		tokensBuffer: []token{},
+		query:        newQuery(),
+	}
+	parser.run()
+	return parser.query, parser.err
+}

queryparser_test.go

@@ -0,0 +1,483 @@
+package toml
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+type queryTestNode struct {
+	value    interface{}
+	position Position
+}
+
+func valueString(root interface{}) string {
+	result := "" //fmt.Sprintf("%T:", root)
+	switch node := root.(type) {
+	case *tomlValue:
+		return valueString(node.value)
+	case *QueryResult:
+		items := []string{}
+		for i, v := range node.Values() {
+			items = append(items, fmt.Sprintf("%s:%s",
+				node.Positions()[i].String(), valueString(v)))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case queryTestNode:
+		result = fmt.Sprintf("%s:%s",
+			node.position.String(), valueString(node.value))
+	case []interface{}:
+		items := []string{}
+		for _, v := range node {
+			items = append(items, valueString(v))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case *TomlTree:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for _, k := range node.Keys() {
+			v := node.GetPath([]string{k})
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case map[string]interface{}:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for k, v := range node {
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case int64:
+		result += fmt.Sprintf("%d", node)
+	case string:
+		result += "'" + node + "'"
+	case float64:
+		result += fmt.Sprintf("%f", node)
+	case bool:
+		result += fmt.Sprintf("%t", node)
+	case time.Time:
+		result += fmt.Sprintf("'%v'", node)
+	}
+	return result
+}
+
+func assertValue(t *testing.T, result, ref interface{}) {
+	pathStr := valueString(result)
+	refStr := valueString(ref)
+	if pathStr != refStr {
+		t.Errorf("values do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+	}
+}
+
+func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
+	tree, err := Load(toml)
+	if err != nil {
+		t.Errorf("Non-nil toml parse error: %v", err)
+		return
+	}
+	q, err := CompileQuery(query)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	results := q.Execute(tree)
+	assertValue(t, results, ref)
+}
+
+func TestQueryRoot(t *testing.T) {
+	assertQueryPositions(t,
+		"a = 42",
+		"$",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(42),
+				}, Position{1, 1},
+			},
+		})
+}
+
+func TestQueryKey(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo.a",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryKeyString(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo['a']",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryIndex(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[5]",
+		[]interface{}{
+			queryTestNode{
+				int64(6), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceRange(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(4), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceStep(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5:2]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryAny(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[foo.baz]\na=3\nb=4",
+		"$.foo.*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+		})
+}
+func TestQueryUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$.*[bar,foo]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryRecursionAll(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": map[string]interface{}{
+							"a": int64(1),
+							"b": int64(2),
+						},
+					},
+					"baz": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(3),
+							"b": int64(4),
+						},
+					},
+					"gorf": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(5),
+							"b": int64(6),
+						},
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{3, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(3),
+						"b": int64(4),
+					},
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				int64(3), Position{5, 1},
+			},
+			queryTestNode{
+				int64(4), Position{6, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(5),
+						"b": int64(6),
+					},
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				int64(5), Position{8, 1},
+			},
+			queryTestNode{
+				int64(6), Position{9, 1},
+			},
+		})
+}
+
+func TestQueryRecursionUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..['foo','bar']",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryFilterFn(t *testing.T) {
+	buff, err := ioutil.ReadFile("example.toml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(int)]",
+		[]interface{}{
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8002), Position{13, 1},
+			},
+			queryTestNode{
+				int64(5000), Position{14, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(string)]",
+		[]interface{}{
+			queryTestNode{
+				"TOML Example", Position{3, 1},
+			},
+			queryTestNode{
+				"Tom Preston-Werner", Position{6, 1},
+			},
+			queryTestNode{
+				"GitHub", Position{7, 1},
+			},
+			queryTestNode{
+				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
+				Position{8, 1},
+			},
+			queryTestNode{
+				"192.168.1.1", Position{12, 1},
+			},
+			queryTestNode{
+				"10.0.0.1", Position{21, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{22, 3},
+			},
+			queryTestNode{
+				"10.0.0.2", Position{25, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{26, 3},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(float)]",
+		[]interface{}{
+		// no float values in document
+		})
+
+	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
+	assertQueryPositions(t, string(buff),
+		"$..[?(tree)]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"name":         "Tom Preston-Werner",
+					"organization": "GitHub",
+					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+					"dob":          tv,
+				}, Position{5, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"server":         "192.168.1.1",
+					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+					"connection_max": int64(5000),
+					"enabled":        true,
+				}, Position{11, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"alpha": map[string]interface{}{
+						"ip": "10.0.0.1",
+						"dc": "eqdc10",
+					},
+					"beta": map[string]interface{}{
+						"ip": "10.0.0.2",
+						"dc": "eqdc10",
+					},
+				}, Position{17, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.1",
+					"dc": "eqdc10",
+				}, Position{20, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.2",
+					"dc": "eqdc10",
+				}, Position{24, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"data": []interface{}{
+						[]interface{}{"gamma", "delta"},
+						[]interface{}{int64(1), int64(2)},
+					},
+				}, Position{28, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(time)]",
+		[]interface{}{
+			queryTestNode{
+				tv, Position{9, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(bool)]",
+		[]interface{}{
+			queryTestNode{
+				true, Position{15, 1},
+			},
+		})
+}

test.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# fail out of the script if anything here fails
+set -e
+
+# set the path to the present working directory
+export GOPATH=`pwd`
+
+# Vendorize the BurntSushi test suite
+# NOTE: this gets a specific release to avoid versioning issues
+if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then
+  mkdir -p src/github.com/BurntSushi
+  git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test
+fi
+pushd src/github.com/BurntSushi/toml-test
+git reset --hard '0.2.0'  # use the released version, NOT tip
+popd
+go build -o toml-test github.com/BurntSushi/toml-test
+
+# vendorize the current lib for testing
+# NOTE: this basically mocks an install without having to go back out to github for code
+mkdir -p src/github.com/pelletier/go-toml/cmd
+cp *.go *.toml src/github.com/pelletier/go-toml
+cp cmd/*.go src/github.com/pelletier/go-toml/cmd
+go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
+
+# Run basic unit tests and then the BurntSushi test suite
+go test -v github.com/pelletier/go-toml
+./toml-test ./test_program_bin | tee test_out

token.go

@@ -0,0 +1,132 @@
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenComment
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenTrue
+	tokenFalse
+	tokenFloat
+	tokenEqual
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftParen
+	tokenRightParen
+	tokenDoubleLeftBracket
+	tokenDoubleRightBracket
+	tokenDate
+	tokenKeyGroup
+	tokenKeyGroupArray
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+	tokenEOL
+)
+
+var tokenTypeNames = []string{
+	"EOF",
+	"Comment",
+	"Key",
+	"String",
+	"Integer",
+	"True",
+	"False",
+	"Float",
+	"=",
+	"[",
+	"[",
+	"(",
+	")",
+	"]]",
+	"[[",
+	"Date",
+	"KeyGroup",
+	"KeyGroupArray",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+	"EOL",
+}
+
+type token struct {
+	Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	if len(t.val) > 10 {
+		return fmt.Sprintf("%.10q...", t.val)
+	}
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isKeyChar(r rune) bool {
+	// "Keys start with the first non-whitespace character and end with the last
+	// non-whitespace character before the equals sign."
+	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+}

toml.go

@@ -1,26 +1,51 @@
-// TOML markup language parser.
-//
-// This version supports the specification as described in
-// https://github.com/mojombo/toml/tree/e3656ad493400895f4460f1244a25f8f8e31a32a
 package toml
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 )
 
-// Definition of a TomlTree.
-// This is the result of the parsing of a TOML file.
-type TomlTree map[string]interface{}
+type tomlValue struct {
+	value    interface{}
+	position Position
+}
+
+// TomlTree is the result of the parsing of a TOML file.
+type TomlTree struct {
+	values   map[string]interface{}
+	position Position
+}
+
+func newTomlTree() *TomlTree {
+	return &TomlTree{
+		values:   make(map[string]interface{}),
+		position: Position{},
+	}
+}
+
+// Has returns a boolean indicating if the given key exists.
+func (t *TomlTree) Has(key string) bool {
+	if key == "" {
+		return false
+	}
+	return t.HasPath(strings.Split(key, "."))
+}
+
+// HasPath returns true if the given path of keys exists, false otherwise.
+func (t *TomlTree) HasPath(keys []string) bool {
+	return t.GetPath(keys) != nil
+}
 
 // Keys returns the keys of the toplevel tree.
 // Warning: this is a costly operation.
 func (t *TomlTree) Keys() []string {
-	keys := make([]string, 0)
-	mp := (map[string]interface{})(*t)
-	for k, _ := range mp {
+	var keys []string
+	for k := range t.values {
 		keys = append(keys, k)
 	}
 	return keys
@@ -29,43 +54,138 @@ func (t *TomlTree) Keys() []string {
 // Get the value at key in the TomlTree.
 // Key is a dot-separated path (e.g. a.b.c).
 // Returns nil if the path does not exist in the tree.
+// If keys is of length zero, the current tree is returned.
 func (t *TomlTree) Get(key string) interface{} {
+	if key == "" {
+		return t
+	}
+	return t.GetPath(strings.Split(key, "."))
+}
+
+// GetPath returns the element in the tree indicated by 'keys'.
+// If keys is of length zero, the current tree is returned.
+func (t *TomlTree) GetPath(keys []string) interface{} {
+	if len(keys) == 0 {
+		return t
+	}
 	subtree := t
-	keys := strings.Split(key, ".")
-	for _, intermediate_key := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		value, exists := subtree.values[intermediateKey]
 		if !exists {
 			return nil
 		}
-		subtree = (*subtree)[intermediate_key].(*TomlTree)
+		switch node := value.(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				return nil
+			}
+			subtree = node[len(node)-1]
+		default:
+			return nil // cannot naigate through other node types
+		}
+	}
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.value
+	default:
+		return node
 	}
-	return (*subtree)[keys[len(keys)-1]]
 }
 
-// Same as Get but with a default value
+// GetPosition returns the position of the given key.
+func (t *TomlTree) GetPosition(key string) Position {
+	if key == "" {
+		return t.position
+	}
+	return t.GetPositionPath(strings.Split(key, "."))
+}
+
+// GetPositionPath returns the element in the tree indicated by 'keys'.
+// If keys is of length zero, the current tree is returned.
+func (t *TomlTree) GetPositionPath(keys []string) Position {
+	if len(keys) == 0 {
+		return t.position
+	}
+	subtree := t
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		value, exists := subtree.values[intermediateKey]
+		if !exists {
+			return Position{0, 0}
+		}
+		switch node := value.(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				return Position{0, 0}
+			}
+			subtree = node[len(node)-1]
+		default:
+			return Position{0, 0}
+		}
+	}
+	// branch based on final node type
+	switch node := subtree.values[keys[len(keys)-1]].(type) {
+	case *tomlValue:
+		return node.position
+	case *TomlTree:
+		return node.position
+	case []*TomlTree:
+		// go to most recent element
+		if len(node) == 0 {
+			return Position{0, 0}
+		}
+		return node[len(node)-1].position
+	default:
+		return Position{0, 0}
+	}
+}
+
+// GetDefault works like Get but with a default value
 func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
-    val := t.Get(key)
-    if val == nil {
-        return def
-    }
-    return val;
+	val := t.Get(key)
+	if val == nil {
+		return def
+	}
+	return val
 }
 
 // Set an element in the tree.
 // Key is a dot-separated path (e.g. a.b.c).
 // Creates all necessary intermediates trees, if needed.
 func (t *TomlTree) Set(key string, value interface{}) {
+	t.SetPath(strings.Split(key, "."), value)
+}
+
+// SetPath sets an element in the tree.
+// Keys is an array of path elements (e.g. {"a","b","c"}).
+// Creates all necessary intermediates trees, if needed.
+func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	subtree := t
-	keys := strings.Split(key, ".")
-	for _, intermediate_key := range keys[:len(keys)-1] {
-		_, exists := (*subtree)[intermediate_key]
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			var new_tree TomlTree = make(TomlTree)
-			(*subtree)[intermediate_key] = &new_tree
+			nextTree = newTomlTree()
+			subtree.values[intermediateKey] = &nextTree // add new element here
+		}
+		switch node := nextTree.(type) {
+		case *TomlTree:
+			subtree = node
+		case []*TomlTree:
+			// go to most recent element
+			if len(node) == 0 {
+				// create element if it does not exist
+				subtree.values[intermediateKey] = append(node, newTomlTree())
+			}
+			subtree = node[len(node)-1]
 		}
-		subtree = (*subtree)[intermediate_key].(*TomlTree)
 	}
-	(*subtree)[keys[len(keys)-1]] = value
+	subtree.values[keys[len(keys)-1]] = value
 }
 
 // createSubTree takes a tree and a key and create the necessary intermediate
@@ -73,19 +193,143 @@ func (t *TomlTree) Set(key string, value interface{}) {
 //
 // e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
 // and tree[a][b][c]
-func (t *TomlTree) createSubTree(key string) {
+//
+// Returns nil on success, error object on failure
+func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	subtree := t
-	for _, intermediate_key := range strings.Split(key, ".") {
-		_, exists := (*subtree)[intermediate_key]
-		if !exists {
-			var new_tree TomlTree = make(TomlTree)
-			(*subtree)[intermediate_key] = &new_tree
+	for _, intermediateKey := range keys {
+		if intermediateKey == "" {
+			return fmt.Errorf("empty intermediate table")
 		}
-		subtree = ((*subtree)[intermediate_key]).(*TomlTree)
+		nextTree, exists := subtree.values[intermediateKey]
+		if !exists {
+			tree := newTomlTree()
+			tree.position = pos
+			subtree.values[intermediateKey] = tree
+			nextTree = tree
+		}
+
+		switch node := nextTree.(type) {
+		case []*TomlTree:
+			subtree = node[len(node)-1]
+		case *TomlTree:
+			subtree = node
+		default:
+			return fmt.Errorf("unknown type for path %s (%s)",
+				strings.Join(keys, "."), intermediateKey)
+		}
+	}
+	return nil
+}
+
+// encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	result := ""
+	for _, rr := range value {
+		intRr := uint16(rr)
+		switch rr {
+		case '\b':
+			result += "\\b"
+		case '\t':
+			result += "\\t"
+		case '\n':
+			result += "\\n"
+		case '\f':
+			result += "\\f"
+		case '\r':
+			result += "\\r"
+		case '"':
+			result += "\\\""
+		case '\\':
+			result += "\\\\"
+		default:
+			if intRr < 0x001F {
+				result += fmt.Sprintf("\\u%0.4X", intRr)
+			} else {
+				result += string(rr)
+			}
+		}
+	}
+	return result
+}
+
+// Value print support function for ToString()
+// Outputs the TOML compliant string representation of a value
+func toTomlValue(item interface{}, indent int) string {
+	tab := strings.Repeat(" ", indent)
+	switch value := item.(type) {
+	case int64:
+		return tab + strconv.FormatInt(value, 10)
+	case float64:
+		return tab + strconv.FormatFloat(value, 'f', -1, 64)
+	case string:
+		return tab + "\"" + encodeTomlString(value) + "\""
+	case bool:
+		if value {
+			return "true"
+		}
+		return "false"
+	case time.Time:
+		return tab + value.Format(time.RFC3339)
+	case []interface{}:
+		result := tab + "[\n"
+		for _, item := range value {
+			result += toTomlValue(item, indent+2) + ",\n"
+		}
+		return result + tab + "]"
+	default:
+		panic(fmt.Sprintf("unsupported value type: %v", value))
 	}
 }
 
-// Create a TomlTree from a string.
+// Recursive support function for ToString()
+// Outputs a tree, using the provided keyspace to prefix group names
+func (t *TomlTree) toToml(indent, keyspace string) string {
+	result := ""
+	for k, v := range t.values {
+		// figure out the keyspace
+		combinedKey := k
+		if keyspace != "" {
+			combinedKey = keyspace + "." + combinedKey
+		}
+		// output based on type
+		switch node := v.(type) {
+		case []*TomlTree:
+			for _, item := range node {
+				if len(item.Keys()) > 0 {
+					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
+				}
+				result += item.toToml(indent+"  ", combinedKey)
+			}
+		case *TomlTree:
+			if len(node.Keys()) > 0 {
+				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			}
+			result += node.toToml(indent+"  ", combinedKey)
+		case *tomlValue:
+			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
+		default:
+			panic(fmt.Sprintf("unsupported node type: %v", node))
+		}
+	}
+	return result
+}
+
+func (t *TomlTree) Query(query string) (*QueryResult, error) {
+	if q, err := CompileQuery(query); err != nil {
+		return nil, err
+	} else {
+		return q.Execute(t), nil
+	}
+}
+
+// ToString generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser
+func (t *TomlTree) ToString() string {
+	return t.toToml("", "")
+}
+
+// Load creates a TomlTree from a string.
 func Load(content string) (tree *TomlTree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -95,12 +339,11 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	_, flow := lex(content)
-	tree = parse(flow)
+	tree = parseToml(lexToml(content))
 	return
 }
 
-// Create a TomlTree from a file.
+// LoadFile creates a TomlTree from a file.
 func LoadFile(path string) (tree *TomlTree, err error) {
 	buff, ferr := ioutil.ReadFile(path)
 	if ferr != nil {
@@ -109,6 +352,5 @@ func LoadFile(path string) (tree *TomlTree, err error) {
 		s := string(buff)
 		tree, err = Load(s)
 	}
-
 	return
 }

toml_test.go

@@ -1 +1,74 @@
+// Testing support for go-toml
+
 package toml
+
+import (
+	"testing"
+)
+
+func TestTomlHas(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.Has("test.key") {
+		t.Errorf("Has - expected test.key to exists")
+	}
+}
+
+func TestTomlHasPath(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if !tree.HasPath([]string{"test", "key"}) {
+		t.Errorf("HasPath - expected test.key to exists")
+	}
+}
+
+func TestTomlGetPath(t *testing.T) {
+	node := newTomlTree()
+	//TODO: set other node data
+
+	for idx, item := range []struct {
+		Path     []string
+		Expected *TomlTree
+	}{
+		{ // empty path test
+			[]string{},
+			node,
+		},
+	} {
+		result := node.GetPath(item.Path)
+		if result != item.Expected {
+			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
+		}
+	}
+}
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result, err := tree.Query("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*TomlTree); !ok {
+		t.Errorf("Expected type of TomlTree: %T Tv", values[0], values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}