Merge pull request #23 from pelletier/jsonpath

Powerful querying interface

README.md

@@ -8,6 +8,23 @@ This library supports TOML version
 [![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
 [![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
 
+## Features
+
+Go-toml provides the following features for using data parsed from TOML documents:
+
+* Load TOML documents from files and string data
+* Easily navigate TOML structure using TomlTree
+* Line & column position data for all parsed elements
+* Query support similar to JSON-Path
+* Syntax errors contain line and column numbers
+
+Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
+
+* Semantic evaluation of parsed TOML
+* Informing a user of mistakes in the source document, after it has been parsed
+* Programatic handling of default values on a case-by-case basis
+* Using a TOML document as a flexible data-store
+
 ## Import
 
     import "github.com/pelletier/go-toml"
@@ -45,34 +62,22 @@ if err != nil {
     user = configTree.Get("user").(string)
     password = configTree.Get("password").(string)
     fmt.Println("User is ", user, ". Password is ", password)
+
+    // show where elements are in the file
+    fmt.Println("User position: %v", configTree.GetPosition("user"))
+    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+    // use a query to gather elements without walking the tree
+    results, _ := config.Query("$..[user,password]")
+    for ii, item := range results.Values() {
+      fmt.Println("Query result %d: %v", ii, item)
+    }
 }
 ```
 
-### Dealing with values
-
-Here are some important functions you need to know in order to work with the
-values in a TOML tree:
-
-* `tree.Get("comma.separated.path")` Returns the value at the given path in the
-  tree as an `interface{}`. It's up to you to cast the result into the right
-  type.
-* `tree.Set("comma.separated.path", value)` Sets the value at the given path in
-  the tree, creating all necessary intermediate subtrees.
-
-### Dealing with positions
-
-Since
-[e118479061](https://github.com/pelletier/go-toml/commit/e1184790610b20d0541fc9f57c181cc5b1fc78be),
-go-toml supports positions. This feature allows you to track the positions of
-the values inside the source document, for example to provide better feedback in
-your application. Using positions works much like values:
-
-* `tree.GetPosition("comma.separated.path")` Returns the position of the given
-  path in the source.
-
 ## Documentation
 
-The documentation is available at
+The documentation and additional examples are available at
 [godoc.org](http://godoc.org/github.com/pelletier/go-toml).
 
 ## Contribute

doc.go

@@ -0,0 +1,245 @@
+// Package toml is a TOML markup language parser.
+//
+// This version supports the specification as described in
+// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
+//
+// TOML Parsing
+//
+// TOML data may be parsed in two ways: by file, or by string.
+//
+//   // load TOML data by filename
+//   tree, err := toml.LoadFile("filename.toml")
+//
+//   // load TOML data stored in a string
+//   tree, err := toml.Load(stringContainingTomlData)
+//
+// Either way, the result is a TomlTree object that can be used to navigate the
+// structure and data within the original document.
+//
+//
+// Getting data from the TomlTree
+//
+// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
+// methods on the returned TomlTree, to find your way through the document data.
+//
+//   if tree.Has('foo') {
+//     fmt.Prinln("foo is: %v", tree.Get('foo'))
+//   }
+//
+// Working with Paths
+//
+// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
+// and GetDefault() methods.  These are the same kind of key paths used within the
+// TOML specification for struct tames.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.Has("foo.bar.baz")
+//
+//   // returns the key at this path, if it is there
+//   tree.Get("foo.bar.baz")
+//
+// TOML allows keys to contain '.', which can cause this syntax to be problematic
+// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
+// methods to explicitly define the path.  This form is also faster, since
+// it avoids having to parse the passed key for '.' delimiters.
+//
+//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
+//   tree.HasPath(string{}{"foo","bar","baz"})
+//
+//   // returns the key at this path, if it is there
+//   tree.GetPath(string{}{"foo","bar","baz"})
+//
+// Note that this is distinct from the heavyweight query syntax supported by
+// TomlTree.Query() and the Query() struct (see below).
+//
+// Position Support
+//
+// Each element within the TomlTree is stored with position metadata, which is
+// invaluable for providing semantic feedback to a user.  This helps in
+// situations where the TOML file parses correctly, but contains data that is
+// not correct for the application.  In such cases, an error message can be
+// generated that indicates the problem line and column number in the source
+// TOML document.
+//
+//   // load TOML data
+//   tree, _ := toml.Load("filename.toml")
+//
+//   // get an entry and report an error if it's the wrong type
+//   element := tree.Get("foo")
+//   if value, ok := element.(int64); !ok {
+//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
+//   }
+//
+//   // report an error if an expected element is missing
+//   if !tree.Has("bar") {
+//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
+//   }
+//
+// Query Support
+//
+// The TOML query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result := tree.Query("$.foo.bar.baz")  // result is 'nil' if the path is not present
+//
+// This is equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   tree.Query("$.foo[-1]")
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   tree.Query("$.foo[0:5]")
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   tree.Query("$.foo[0:-1]")
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   tree.Query("$.foo[0:-1:2]")
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   tree.Query("$.foo[:]")
+//   tree.Query("$.foo[0:]")
+//   tree.Query("$.foo[:-1]")
+//   tree.Query("$.foo[0:-1:]")
+//   tree.Query("$.foo[::1]")
+//   tree.Query("$.foo[0::1]")
+//   tree.Query("$.foo[:-1:1]")
+//   tree.Query("$.foo[0:-1:1]")
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   tree.Query("$.foo[?(bar)]")
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type TomlTree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a QueryResult object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := tree.Query("$.foo.bar.baz")
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a TomlTree object, or compiled ahead
+// of time and executed discretely.  The former is more convienent, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := tree.Query("$.foo.bar.baz")
+//
+//   // compiled query
+//   query := toml.CompileQuery("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := CompileQuery("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*TomlTree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package toml

doc_test.go

@@ -0,0 +1,81 @@
+// code examples for godoc
+
+package toml
+
+import (
+	"fmt"
+)
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := CompileQuery("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*TomlTree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' TomlTree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	authors, _ := config.Query("$.book.author")
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func Example_comprehensiveExample() {
+	config, err := LoadFile("config.toml")
+
+	if err != nil {
+		fmt.Println("Error ", err.Error())
+	} else {
+		// retrieve data directly
+		user := config.Get("postgres.user").(string)
+		password := config.Get("postgres.password").(string)
+
+		// or using an intermediate object
+		configTree := config.Get("postgres").(*TomlTree)
+		user = configTree.Get("user").(string)
+		password = configTree.Get("password").(string)
+		fmt.Println("User is ", user, ". Password is ", password)
+
+		// show where elements are in the file
+		fmt.Println("User position: %v", configTree.GetPosition("user"))
+		fmt.Println("Password position: %v", configTree.GetPosition("password"))
+
+		// use a query to gather elements without walking the tree
+		results, _ := config.Query("$..[user,password]")
+		for ii, item := range results.Values() {
+			fmt.Println("Query result %d: %v", ii, item)
+		}
+	}
+}

lexer.go

@@ -10,115 +10,16 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 )
 
 var dateRegexp *regexp.Regexp
 
-// Define tokens
+// Define state functions
-type tokenType int
+type tomlLexStateFn func() tomlLexStateFn
-
-const (
-	eof = -(iota + 1)
-)
-
-const (
-	tokenError tokenType = iota
-	tokenEOF
-	tokenComment
-	tokenKey
-	tokenEqual
-	tokenString
-	tokenInteger
-	tokenTrue
-	tokenFalse
-	tokenFloat
-	tokenLeftBracket
-	tokenRightBracket
-	tokenDoubleLeftBracket
-	tokenDoubleRightBracket
-	tokenDate
-	tokenKeyGroup
-	tokenKeyGroupArray
-	tokenComma
-	tokenEOL
-)
-
-var tokenTypeNames = []string{
-	"EOF",
-	"Comment",
-	"Key",
-	"=",
-	"\"",
-	"Integer",
-	"True",
-	"False",
-	"Float",
-	"[",
-	"[",
-	"]]",
-	"[[",
-	"Date",
-	"KeyGroup",
-	"KeyGroupArray",
-	",",
-	"EOL",
-}
-
-type token struct {
-	Position
-	typ tokenType
-	val string
-}
-
-func (tt tokenType) String() string {
-	idx := int(tt)
-	if idx < len(tokenTypeNames) {
-		return tokenTypeNames[idx]
-	}
-	return "Unknown"
-}
-
-func (i token) String() string {
-	switch i.typ {
-	case tokenEOF:
-		return "EOF"
-	case tokenError:
-		return i.val
-	}
-
-	if len(i.val) > 10 {
-		return fmt.Sprintf("%.10q...", i.val)
-	}
-	return fmt.Sprintf("%q", i.val)
-}
-
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-func isAlphanumeric(r rune) bool {
-	return unicode.IsLetter(r) || r == '_'
-}
-
-func isKeyChar(r rune) bool {
-	// "Keys start with the first non-whitespace character and end with the last
-	// non-whitespace character before the equals sign."
-	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
-}
-
-func isDigit(r rune) bool {
-	return unicode.IsNumber(r)
-}
-
-func isHexDigit(r rune) bool {
-	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
-}
 
 // Define lexer
-type lexer struct {
+type tomlLexer struct {
 	input  string
 	start  int
 	pos    int
@@ -129,14 +30,14 @@ type lexer struct {
 	col    int
 }
 
-func (l *lexer) run() {
+func (l *tomlLexer) run() {
-	for state := lexVoid; state != nil; {
+	for state := l.lexVoid; state != nil; {
-		state = state(l)
+		state = state()
 	}
 	close(l.tokens)
 }
 
-func (l *lexer) nextStart() {
+func (l *tomlLexer) nextStart() {
 	// iterate by runes (utf8 characters)
 	// search for newlines and advance line/col counts
 	for i := l.start; i < l.pos; {
@@ -153,7 +54,7 @@ func (l *lexer) nextStart() {
 	l.start = l.pos
 }
 
-func (l *lexer) emit(t tokenType) {
+func (l *tomlLexer) emit(t tokenType) {
 	l.tokens <- token{
 		Position: Position{l.line, l.col},
 		typ:      t,
@@ -162,7 +63,7 @@ func (l *lexer) emit(t tokenType) {
 	l.nextStart()
 }
 
-func (l *lexer) emitWithValue(t tokenType, value string) {
+func (l *tomlLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
 		Position: Position{l.line, l.col},
 		typ:      t,
@@ -171,7 +72,7 @@ func (l *lexer) emitWithValue(t tokenType, value string) {
 	l.nextStart()
 }
 
-func (l *lexer) next() rune {
+func (l *tomlLexer) next() rune {
 	if l.pos >= len(l.input) {
 		l.width = 0
 		return eof
@@ -182,15 +83,15 @@ func (l *lexer) next() rune {
 	return r
 }
 
-func (l *lexer) ignore() {
+func (l *tomlLexer) ignore() {
 	l.nextStart()
 }
 
-func (l *lexer) backup() {
+func (l *tomlLexer) backup() {
 	l.pos -= l.width
 }
 
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
 	l.tokens <- token{
 		Position: Position{l.line, l.col},
 		typ:      tokenError,
@@ -199,13 +100,13 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn {
 	return nil
 }
 
-func (l *lexer) peek() rune {
+func (l *tomlLexer) peek() rune {
 	r := l.next()
 	l.backup()
 	return r
 }
 
-func (l *lexer) accept(valid string) bool {
+func (l *tomlLexer) accept(valid string) bool {
 	if strings.IndexRune(valid, l.next()) >= 0 {
 		return true
 	}
@@ -213,23 +114,20 @@ func (l *lexer) accept(valid string) bool {
 	return false
 }
 
-func (l *lexer) follow(next string) bool {
+func (l *tomlLexer) follow(next string) bool {
 	return strings.HasPrefix(l.input[l.pos:], next)
 }
 
-// Define state functions
+func (l *tomlLexer) lexVoid() tomlLexStateFn {
-type stateFn func(*lexer) stateFn
-
-func lexVoid(l *lexer) stateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '[':
-			return lexKeyGroup
+			return l.lexKeyGroup
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '=':
-			return lexEqual
+			return l.lexEqual
 		}
 
 		if isSpace(next) {
@@ -237,11 +135,11 @@ func lexVoid(l *lexer) stateFn {
 		}
 
 		if l.depth > 0 {
-			return lexRvalue
+			return l.lexRvalue
 		}
 
 		if isKeyChar(next) {
-			return lexKey
+			return l.lexKey
 		}
 
 		if l.next() == eof {
@@ -253,7 +151,7 @@ func lexVoid(l *lexer) stateFn {
 	return nil
 }
 
-func lexRvalue(l *lexer) stateFn {
+func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
@@ -263,43 +161,43 @@ func lexRvalue(l *lexer) stateFn {
 			return l.errorf("cannot have multiple equals for the same key")
 		case '[':
 			l.depth++
-			return lexLeftBracket
+			return l.lexLeftBracket
 		case ']':
 			l.depth--
-			return lexRightBracket
+			return l.lexRightBracket
 		case '#':
-			return lexComment
+			return l.lexComment
 		case '"':
-			return lexString
+			return l.lexString
 		case ',':
-			return lexComma
+			return l.lexComma
 		case '\n':
 			l.ignore()
 			l.pos++
 			if l.depth == 0 {
-				return lexVoid
+				return l.lexVoid
 			}
-			return lexRvalue
+			return l.lexRvalue
 		}
 
 		if l.follow("true") {
-			return lexTrue
+			return l.lexTrue
 		}
 
 		if l.follow("false") {
-			return lexFalse
+			return l.lexFalse
 		}
 
 		if isAlphanumeric(next) {
-			return lexKey
+			return l.lexKey
 		}
 
 		if dateRegexp.FindString(l.input[l.pos:]) != "" {
-			return lexDate
+			return l.lexDate
 		}
 
 		if next == '+' || next == '-' || isDigit(next) {
-			return lexNumber
+			return l.lexNumber
 		}
 
 		if isSpace(next) {
@@ -315,51 +213,51 @@ func lexRvalue(l *lexer) stateFn {
 	return nil
 }
 
-func lexDate(l *lexer) stateFn {
+func (l *tomlLexer) lexDate() tomlLexStateFn {
 	l.ignore()
 	l.pos += 20 // Fixed size of a date in TOML
 	l.emit(tokenDate)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexTrue(l *lexer) stateFn {
+func (l *tomlLexer) lexTrue() tomlLexStateFn {
 	l.ignore()
 	l.pos += 4
 	l.emit(tokenTrue)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexFalse(l *lexer) stateFn {
+func (l *tomlLexer) lexFalse() tomlLexStateFn {
 	l.ignore()
 	l.pos += 5
 	l.emit(tokenFalse)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexEqual(l *lexer) stateFn {
+func (l *tomlLexer) lexEqual() tomlLexStateFn {
 	l.ignore()
 	l.accept("=")
 	l.emit(tokenEqual)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexComma(l *lexer) stateFn {
+func (l *tomlLexer) lexComma() tomlLexStateFn {
 	l.ignore()
 	l.accept(",")
 	l.emit(tokenComma)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexKey(l *lexer) stateFn {
+func (l *tomlLexer) lexKey() tomlLexStateFn {
 	l.ignore()
 	for isKeyChar(l.next()) {
 	}
 	l.backup()
 	l.emit(tokenKey)
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexComment(l *lexer) stateFn {
+func (l *tomlLexer) lexComment() tomlLexStateFn {
 	for {
 		next := l.next()
 		if next == '\n' || next == eof {
@@ -367,17 +265,17 @@ func lexComment(l *lexer) stateFn {
 		}
 	}
 	l.ignore()
-	return lexVoid
+	return l.lexVoid
 }
 
-func lexLeftBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
 	l.ignore()
 	l.pos++
 	l.emit(tokenLeftBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexString(l *lexer) stateFn {
+func (l *tomlLexer) lexString() tomlLexStateFn {
 	l.pos++
 	l.ignore()
 	growingString := ""
@@ -387,7 +285,7 @@ func lexString(l *lexer) stateFn {
 			l.emitWithValue(tokenString, growingString)
 			l.pos++
 			l.ignore()
-			return lexRvalue
+			return l.lexRvalue
 		}
 
 		if l.follow("\\\"") {
@@ -446,7 +344,7 @@ func lexString(l *lexer) stateFn {
 	return l.errorf("unclosed string")
 }
 
-func lexKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
 	l.ignore()
 	l.pos++
 
@@ -454,14 +352,14 @@ func lexKeyGroup(l *lexer) stateFn {
 		// token '[[' signifies an array of anonymous key groups
 		l.pos++
 		l.emit(tokenDoubleLeftBracket)
-		return lexInsideKeyGroupArray
+		return l.lexInsideKeyGroupArray
 	}
 	// vanilla key group
 	l.emit(tokenLeftBracket)
-	return lexInsideKeyGroup
+	return l.lexInsideKeyGroup
 }
 
-func lexInsideKeyGroupArray(l *lexer) stateFn {
+func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
 	for {
 		if l.peek() == ']' {
 			if l.pos > l.start {
@@ -474,7 +372,7 @@ func lexInsideKeyGroupArray(l *lexer) stateFn {
 			}
 			l.pos++
 			l.emit(tokenDoubleRightBracket)
-			return lexVoid
+			return l.lexVoid
 		} else if l.peek() == '[' {
 			return l.errorf("group name cannot contain ']'")
 		}
@@ -486,7 +384,7 @@ func lexInsideKeyGroupArray(l *lexer) stateFn {
 	return l.errorf("unclosed key group array")
 }
 
-func lexInsideKeyGroup(l *lexer) stateFn {
+func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
 	for {
 		if l.peek() == ']' {
 			if l.pos > l.start {
@@ -495,7 +393,7 @@ func lexInsideKeyGroup(l *lexer) stateFn {
 			l.ignore()
 			l.pos++
 			l.emit(tokenRightBracket)
-			return lexVoid
+			return l.lexVoid
 		} else if l.peek() == '[' {
 			return l.errorf("group name cannot contain ']'")
 		}
@@ -507,14 +405,14 @@ func lexInsideKeyGroup(l *lexer) stateFn {
 	return l.errorf("unclosed key group")
 }
 
-func lexRightBracket(l *lexer) stateFn {
+func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
 	l.ignore()
 	l.pos++
 	l.emit(tokenRightBracket)
-	return lexRvalue
+	return l.lexRvalue
 }
 
-func lexNumber(l *lexer) stateFn {
+func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	l.ignore()
 	if !l.accept("+") {
 		l.accept("-")
@@ -550,7 +448,7 @@ func lexNumber(l *lexer) stateFn {
 	} else {
 		l.emit(tokenInteger)
 	}
-	return lexRvalue
+	return l.lexRvalue
 }
 
 func init() {
@@ -558,13 +456,13 @@ func init() {
 }
 
 // Entry point
-func lex(input string) (*lexer, chan token) {
+func lexToml(input string) chan token {
-	l := &lexer{
+	l := &tomlLexer{
 		input:  input,
 		tokens: make(chan token),
 		line:   1,
 		col:    1,
 	}
 	go l.run()
-	return l, l.tokens
+	return l.tokens
 }

lexer_test.go

@@ -3,7 +3,7 @@ package toml
 import "testing"
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	_, ch := lex(input)
+	ch := lexToml(input)
 	for _, expected := range expectedFlow {
 		token := <-ch
 		if token != expected {

match.go

@@ -0,0 +1,227 @@
+package toml
+
+import (
+	"fmt"
+)
+
+// support function to set positions for tomlValues
+// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
+// values returned by the query engines
+func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
+	switch castNode := node.(type) {
+	case *tomlValue:
+		ctx.lastPosition = castNode.position
+		return castNode.value
+	case []*TomlTree:
+		if len(castNode) > 0 {
+			ctx.lastPosition = castNode[0].position
+		}
+		return node
+	default:
+		return node
+	}
+}
+
+// base match
+type matchBase struct {
+	next pathFn
+}
+
+func (f *matchBase) setNext(next pathFn) {
+	f.next = next
+}
+
+// terminating functor - gathers results
+type terminatingFn struct {
+	// empty
+}
+
+func newTerminatingFn() *terminatingFn {
+	return &terminatingFn{}
+}
+
+func (f *terminatingFn) setNext(next pathFn) {
+	// do nothing
+}
+
+func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
+	switch castNode := node.(type) {
+	case *TomlTree:
+		ctx.result.appendResult(node, castNode.position)
+	case *tomlValue:
+		ctx.result.appendResult(node, castNode.position)
+	default:
+		// use last position for scalars
+		ctx.result.appendResult(node, ctx.lastPosition)
+	}
+}
+
+// match single key
+type matchKeyFn struct {
+	matchBase
+	Name string
+}
+
+func newMatchKeyFn(name string) *matchKeyFn {
+	return &matchKeyFn{Name: name}
+}
+
+func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		item := tree.values[f.Name]
+		if item != nil {
+			f.next.call(item, ctx)
+		}
+	}
+}
+
+// match single index
+type matchIndexFn struct {
+	matchBase
+	Idx int
+}
+
+func newMatchIndexFn(idx int) *matchIndexFn {
+	return &matchIndexFn{Idx: idx}
+}
+
+func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		if f.Idx < len(arr) && f.Idx >= 0 {
+			f.next.call(arr[f.Idx], ctx)
+		}
+	}
+}
+
+// filter by slicing
+type matchSliceFn struct {
+	matchBase
+	Start, End, Step int
+}
+
+func newMatchSliceFn(start, end, step int) *matchSliceFn {
+	return &matchSliceFn{Start: start, End: end, Step: step}
+}
+
+func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
+	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+		// adjust indexes for negative values, reverse ordering
+		realStart, realEnd := f.Start, f.End
+		if realStart < 0 {
+			realStart = len(arr) + realStart
+		}
+		if realEnd < 0 {
+			realEnd = len(arr) + realEnd
+		}
+		if realEnd < realStart {
+			realEnd, realStart = realStart, realEnd // swap
+		}
+		// loop and gather
+		for idx := realStart; idx < realEnd; idx += f.Step {
+			f.next.call(arr[idx], ctx)
+		}
+	}
+}
+
+// match anything
+type matchAnyFn struct {
+	matchBase
+}
+
+func newMatchAnyFn() *matchAnyFn {
+	return &matchAnyFn{}
+}
+
+func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		for _, v := range tree.values {
+			f.next.call(v, ctx)
+		}
+	}
+}
+
+// filter through union
+type matchUnionFn struct {
+	Union []pathFn
+}
+
+func (f *matchUnionFn) setNext(next pathFn) {
+	for _, fn := range f.Union {
+		fn.setNext(next)
+	}
+}
+
+func (f *matchUnionFn) call(node interface{}, ctx *queryContext) {
+	for _, fn := range f.Union {
+		fn.call(node, ctx)
+	}
+}
+
+// match every single last node in the tree
+type matchRecursiveFn struct {
+	matchBase
+}
+
+func newMatchRecursiveFn() *matchRecursiveFn {
+	return &matchRecursiveFn{}
+}
+
+func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
+	if tree, ok := node.(*TomlTree); ok {
+		var visit func(tree *TomlTree)
+		visit = func(tree *TomlTree) {
+			for _, v := range tree.values {
+				f.next.call(v, ctx)
+				switch node := v.(type) {
+				case *TomlTree:
+					visit(node)
+				case []*TomlTree:
+					for _, subtree := range node {
+						visit(subtree)
+					}
+				}
+			}
+		}
+		f.next.call(tree, ctx)
+		visit(tree)
+	}
+}
+
+// match based on an externally provided functional filter
+type matchFilterFn struct {
+	matchBase
+	Pos  Position
+	Name string
+}
+
+func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+	return &matchFilterFn{Name: name, Pos: pos}
+}
+
+func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
+	fn, ok := (*ctx.filters)[f.Name]
+	if !ok {
+		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
+			f.Pos, f.Name))
+	}
+	switch castNode := tomlValueCheck(node, ctx).(type) {
+	case *TomlTree:
+		for _, v := range castNode.values {
+			if tv, ok := v.(*tomlValue); ok {
+				if fn(tv.value) {
+					f.next.call(v, ctx)
+				}
+			} else {
+				if fn(v) {
+					f.next.call(v, ctx)
+				}
+			}
+		}
+	case []interface{}:
+		for _, v := range castNode {
+			if fn(v) {
+				f.next.call(v, ctx)
+			}
+		}
+	}
+}

match_test.go

@@ -0,0 +1,202 @@
+package toml
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+// dump path tree to a string
+func pathString(root pathFn) string {
+	result := fmt.Sprintf("%T:", root)
+	switch fn := root.(type) {
+	case *terminatingFn:
+		result += "{}"
+	case *matchKeyFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	case *matchIndexFn:
+		result += fmt.Sprintf("{%d}", fn.Idx)
+		result += pathString(fn.next)
+	case *matchSliceFn:
+		result += fmt.Sprintf("{%d:%d:%d}",
+			fn.Start, fn.End, fn.Step)
+		result += pathString(fn.next)
+	case *matchAnyFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchUnionFn:
+		result += "{["
+		for _, v := range fn.Union {
+			result += pathString(v) + ", "
+		}
+		result += "]}"
+	case *matchRecursiveFn:
+		result += "{}"
+		result += pathString(fn.next)
+	case *matchFilterFn:
+		result += fmt.Sprintf("{%s}", fn.Name)
+		result += pathString(fn.next)
+	}
+	return result
+}
+
+func assertPathMatch(t *testing.T, path, ref *Query) bool {
+	pathStr := pathString(path.root)
+	refStr := pathString(ref.root)
+	if pathStr != refStr {
+		t.Errorf("paths do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+		return false
+	}
+	return true
+}
+
+func assertPath(t *testing.T, query string, ref *Query) {
+	path, _ := parseQuery(lexQuery(query))
+	assertPathMatch(t, path, ref)
+}
+
+func buildPath(parts ...pathFn) *Query {
+	query := newQuery()
+	for _, v := range parts {
+		query.appendPath(v)
+	}
+	return query
+}
+
+func TestPathRoot(t *testing.T) {
+	assertPath(t,
+		"$",
+		buildPath(
+		// empty
+		))
+}
+
+func TestPathKey(t *testing.T) {
+	assertPath(t,
+		"$.foo",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketKey(t *testing.T) {
+	assertPath(t,
+		"$[foo]",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathBracketStringKey(t *testing.T) {
+	assertPath(t,
+		"$['foo']",
+		buildPath(
+			newMatchKeyFn("foo"),
+		))
+}
+
+func TestPathIndex(t *testing.T) {
+	assertPath(t,
+		"$[123]",
+		buildPath(
+			newMatchIndexFn(123),
+		))
+}
+
+func TestPathSliceStart(t *testing.T) {
+	assertPath(t,
+		"$[123:]",
+		buildPath(
+			newMatchSliceFn(123, math.MaxInt64, 1),
+		))
+}
+
+func TestPathSliceStartEnd(t *testing.T) {
+	assertPath(t,
+		"$[123:456]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartEndColon(t *testing.T) {
+	assertPath(t,
+		"$[123:456:]",
+		buildPath(
+			newMatchSliceFn(123, 456, 1),
+		))
+}
+
+func TestPathSliceStartStep(t *testing.T) {
+	assertPath(t,
+		"$[123::7]",
+		buildPath(
+			newMatchSliceFn(123, math.MaxInt64, 7),
+		))
+}
+
+func TestPathSliceEndStep(t *testing.T) {
+	assertPath(t,
+		"$[:456:7]",
+		buildPath(
+			newMatchSliceFn(0, 456, 7),
+		))
+}
+
+func TestPathSliceStep(t *testing.T) {
+	assertPath(t,
+		"$[::7]",
+		buildPath(
+			newMatchSliceFn(0, math.MaxInt64, 7),
+		))
+}
+
+func TestPathSliceAll(t *testing.T) {
+	assertPath(t,
+		"$[123:456:7]",
+		buildPath(
+			newMatchSliceFn(123, 456, 7),
+		))
+}
+
+func TestPathAny(t *testing.T) {
+	assertPath(t,
+		"$.*",
+		buildPath(
+			newMatchAnyFn(),
+		))
+}
+
+func TestPathUnion(t *testing.T) {
+	assertPath(t,
+		"$[foo, bar, baz]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchKeyFn("foo"),
+				newMatchKeyFn("bar"),
+				newMatchKeyFn("baz"),
+			}},
+		))
+}
+
+func TestPathRecurse(t *testing.T) {
+	assertPath(t,
+		"$..*",
+		buildPath(
+			newMatchRecursiveFn(),
+		))
+}
+
+func TestPathFilterExpr(t *testing.T) {
+	assertPath(t,
+		"$[?('foo'),?(bar)]",
+		buildPath(
+			&matchUnionFn{[]pathFn{
+				newMatchFilterFn("foo", Position{}),
+				newMatchFilterFn("bar", Position{}),
+			}},
+		))
+}

parser.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-type parser struct {
+type tomlParser struct {
 	flow          chan token
 	tree          *TomlTree
 	tokensBuffer  []token
@@ -18,20 +18,20 @@ type parser struct {
 	seenGroupKeys []string
 }
 
-type parserStateFn func(*parser) parserStateFn
+type tomlParserStateFn func() tomlParserStateFn
 
 // Formats and panics an error message based on a token
-func (p *parser) raiseError(tok *token, msg string, args ...interface{}) {
+func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) {
 	panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...))
 }
 
-func (p *parser) run() {
+func (p *tomlParser) run() {
-	for state := parseStart; state != nil; {
+	for state := p.parseStart; state != nil; {
-		state = state(p)
+		state = state()
 	}
 }
 
-func (p *parser) peek() *token {
+func (p *tomlParser) peek() *token {
 	if len(p.tokensBuffer) != 0 {
 		return &(p.tokensBuffer[0])
 	}
@@ -44,7 +44,7 @@ func (p *parser) peek() *token {
 	return &tok
 }
 
-func (p *parser) assume(typ tokenType) {
+func (p *tomlParser) assume(typ tokenType) {
 	tok := p.getToken()
 	if tok == nil {
 		p.raiseError(tok, "was expecting token %s, but token stream is empty", tok)
@@ -54,7 +54,7 @@ func (p *parser) assume(typ tokenType) {
 	}
 }
 
-func (p *parser) getToken() *token {
+func (p *tomlParser) getToken() *token {
 	if len(p.tokensBuffer) != 0 {
 		tok := p.tokensBuffer[0]
 		p.tokensBuffer = p.tokensBuffer[1:]
@@ -67,12 +67,9 @@ func (p *parser) getToken() *token {
 	return &tok
 }
 
-func parseStart(p *parser) parserStateFn {
+func (p *tomlParser) parseStart() tomlParserStateFn {
 	tok := p.peek()
 
-	// prime position data with root tree instance
-	p.tree.position = tok.Position
-
 	// end of stream, parsing is finished
 	if tok == nil {
 		return nil
@@ -80,11 +77,11 @@ func parseStart(p *parser) parserStateFn {
 
 	switch tok.typ {
 	case tokenDoubleLeftBracket:
-		return parseGroupArray
+		return p.parseGroupArray
 	case tokenLeftBracket:
-		return parseGroup
+		return p.parseGroup
 	case tokenKey:
-		return parseAssign
+		return p.parseAssign
 	case tokenEOF:
 		return nil
 	default:
@@ -93,7 +90,7 @@ func parseStart(p *parser) parserStateFn {
 	return nil
 }
 
-func parseGroupArray(p *parser) parserStateFn {
+func (p *tomlParser) parseGroupArray() tomlParserStateFn {
 	startToken := p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
@@ -102,7 +99,7 @@ func parseGroupArray(p *parser) parserStateFn {
 
 	// get or create group array element at the indicated part in the path
 	keys := strings.Split(key.val, ".")
-	p.tree.createSubTree(keys[:len(keys)-1]) // create parent entries
+	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
 	destTree := p.tree.GetPath(keys)
 	var array []*TomlTree
 	if destTree == nil {
@@ -140,10 +137,10 @@ func parseGroupArray(p *parser) parserStateFn {
 
 	// move to next parser state
 	p.assume(tokenDoubleRightBracket)
-	return parseStart(p)
+	return p.parseStart
 }
 
-func parseGroup(p *parser) parserStateFn {
+func (p *tomlParser) parseGroup() tomlParserStateFn {
 	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
@@ -157,20 +154,18 @@ func parseGroup(p *parser) parserStateFn {
 
 	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
 	keys := strings.Split(key.val, ".")
-	if err := p.tree.createSubTree(keys); err != nil {
+	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
 		p.raiseError(key, "%s", err)
 	}
 	p.assume(tokenRightBracket)
 	p.currentGroup = keys
-	targetTree := p.tree.GetPath(p.currentGroup).(*TomlTree)
+	return p.parseStart
-	targetTree.position = startToken.Position
-	return parseStart(p)
 }
 
-func parseAssign(p *parser) parserStateFn {
+func (p *tomlParser) parseAssign() tomlParserStateFn {
 	key := p.getToken()
 	p.assume(tokenEqual)
-	value := parseRvalue(p)
+	value := p.parseRvalue()
 	var groupKey []string
 	if len(p.currentGroup) > 0 {
 		groupKey = p.currentGroup
@@ -198,10 +193,10 @@ func parseAssign(p *parser) parserStateFn {
 			strings.Join(finalKey, "."))
 	}
 	targetNode.values[key.val] = &tomlValue{value, key.Position}
-	return parseStart(p)
+	return p.parseStart
 }
 
-func parseRvalue(p *parser) interface{} {
+func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
 		p.raiseError(tok, "expecting a value")
@@ -233,7 +228,7 @@ func parseRvalue(p *parser) interface{} {
 		}
 		return val
 	case tokenLeftBracket:
-		return parseArray(p)
+		return p.parseArray()
 	case tokenError:
 		p.raiseError(tok, "%s", tok)
 	}
@@ -243,7 +238,7 @@ func parseRvalue(p *parser) interface{} {
 	return nil
 }
 
-func parseArray(p *parser) []interface{} {
+func (p *tomlParser) parseArray() []interface{} {
 	var array []interface{}
 	arrayType := reflect.TypeOf(nil)
 	for {
@@ -255,7 +250,7 @@ func parseArray(p *parser) []interface{} {
 			p.getToken()
 			return array
 		}
-		val := parseRvalue(p)
+		val := p.parseRvalue()
 		if arrayType == nil {
 			arrayType = reflect.TypeOf(val)
 		}
@@ -277,9 +272,10 @@ func parseArray(p *parser) []interface{} {
 	return array
 }
 
-func parse(flow chan token) *TomlTree {
+func parseToml(flow chan token) *TomlTree {
 	result := newTomlTree()
-	parser := &parser{
+	result.position = Position{1, 1}
+	parser := &tomlParser{
 		flow:          flow,
 		tree:          result,
 		tokensBuffer:  make([]token, 0),

parser_test.go

@@ -31,7 +31,7 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 
 func TestCreateSubTree(t *testing.T) {
 	tree := newTomlTree()
-	tree.createSubTree([]string{"a", "b", "c"})
+	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
 		t.Fail()
@@ -385,7 +385,7 @@ func assertPosition(t *testing.T, text string, ref map[string]Position) {
 	for path, pos := range ref {
 		testPos := tree.GetPosition(path)
 		if testPos.Invalid() {
-			t.Errorf("Failed to query tree path: %s", path)
+			t.Errorf("Failed to query tree path or path has invalid position: %s", path)
 		} else if pos != testPos {
 			t.Errorf("Expected position %v, got %v instead", pos, testPos)
 		}
@@ -396,6 +396,7 @@ func TestDocumentPositions(t *testing.T) {
 	assertPosition(t,
 		"[foo]\nbar=42\nbaz=69",
 		map[string]Position{
+			"":        Position{1, 1},
 			"foo":     Position{1, 1},
 			"foo.bar": Position{2, 1},
 			"foo.baz": Position{3, 1},
@@ -406,6 +407,7 @@ func TestDocumentPositionsWithSpaces(t *testing.T) {
 	assertPosition(t,
 		"  [foo]\n  bar=42\n  baz=69",
 		map[string]Position{
+			"":        Position{1, 1},
 			"foo":     Position{1, 3},
 			"foo.bar": Position{2, 3},
 			"foo.baz": Position{3, 3},
@@ -416,20 +418,21 @@ func TestDocumentPositionsWithGroupArray(t *testing.T) {
 	assertPosition(t,
 		"[[foo]]\nbar=42\nbaz=69",
 		map[string]Position{
+			"":        Position{1, 1},
 			"foo":     Position{1, 1},
 			"foo.bar": Position{2, 1},
 			"foo.baz": Position{3, 1},
 		})
 }
 
-func TestDocumentPositionsEmptyPath(t *testing.T) {
+func TestNestedTreePosition(t *testing.T) {
-	text := "[foo]\nbar=42\nbaz=69"
+	assertPosition(t,
-	tree, err := Load(text)
+		"[foo.bar]\na=42\nb=69",
-	if err != nil {
+		map[string]Position{
-		t.Errorf("Error loading document text: `%v`", text)
+			"":          Position{1, 1},
-		t.Errorf("Error: %v", err)
+			"foo":       Position{1, 1},
-	}
+			"foo.bar":   Position{1, 1},
-	if pos := tree.GetPosition(""); !pos.Invalid() {
+			"foo.bar.a": Position{2, 1},
-		t.Errorf("Valid position was returned for empty path")
+			"foo.bar.b": Position{3, 1},
-	}
+		})
 }

position.go

@@ -6,7 +6,13 @@ import (
 	"fmt"
 )
 
-// Position within a TOML document
+/*
+  Position of a document element within a TOML document.
+
+  Line and Col are both 1-indexed positions for the element's line number and
+  column number, respectively.  Values of zero or less will cause Invalid(),
+  to return true.
+*/
 type Position struct {
 	Line int // line within the document
 	Col  int // column within the line
@@ -18,7 +24,7 @@ func (p *Position) String() string {
 	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
 }
 
-// Invalid returns wheter or not the position is valid (i.e. with negative or
+// Returns whether or not the position is valid (i.e. with negative or
 // null values)
 func (p *Position) Invalid() bool {
 	return p.Line <= 0 || p.Col <= 0

query.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"time"
+)
+
+//  Type of a user-defined filter function, for use with Query.SetFilter().
+//
+//  The return value of the function must indicate if 'node' is to be included
+//  at this stage of the TOML path.  Returning true will include the node, and
+//  returning false will exclude it.
+//
+//  NOTE: Care should be taken to write script callbacks such that they are safe
+//  to use from multiple goroutines.
+type NodeFilterFn func(node interface{}) bool
+
+// The result of Executing a Query
+type QueryResult struct {
+	items     []interface{}
+	positions []Position
+}
+
+// appends a value/position pair to the result set
+func (r *QueryResult) appendResult(node interface{}, pos Position) {
+	r.items = append(r.items, node)
+	r.positions = append(r.positions, pos)
+}
+
+// Set of values within a QueryResult.  The order of values is not guaranteed
+// to be in document order, and may be different each time a query is executed.
+func (r *QueryResult) Values() []interface{} {
+	return r.items
+}
+
+// Set of positions for values within a QueryResult.  Each index in Positions()
+// corresponds to the entry in Value() of the same index.
+func (r *QueryResult) Positions() []Position {
+	return r.positions
+}
+
+// runtime context for executing query paths
+type queryContext struct {
+	result       *QueryResult
+	filters      *map[string]NodeFilterFn
+	lastPosition Position
+}
+
+// generic path functor interface
+type pathFn interface {
+	setNext(next pathFn)
+	call(node interface{}, ctx *queryContext)
+}
+
+// A Query is the representation of a compiled TOML path.  A Query is safe
+// for concurrent use by multiple goroutines.
+type Query struct {
+	root    pathFn
+	tail    pathFn
+	filters *map[string]NodeFilterFn
+}
+
+func newQuery() *Query {
+	return &Query{
+		root:    nil,
+		tail:    nil,
+		filters: &defaultFilterFunctions,
+	}
+}
+
+func (q *Query) appendPath(next pathFn) {
+	if q.root == nil {
+		q.root = next
+	} else {
+		q.tail.setNext(next)
+	}
+	q.tail = next
+	next.setNext(newTerminatingFn()) // init the next functor
+}
+
+// Compiles a TOML path expression.  The returned Query can be used to match
+// elements within a TomlTree and its descendants.
+func CompileQuery(path string) (*Query, error) {
+	return parseQuery(lexQuery(path))
+}
+
+// Executes a query against a TomlTree, and returns the result of the query.
+func (q *Query) Execute(tree *TomlTree) *QueryResult {
+	result := &QueryResult{
+		items:     []interface{}{},
+		positions: []Position{},
+	}
+	if q.root == nil {
+		result.appendResult(tree, tree.GetPosition(""))
+	} else {
+		ctx := &queryContext{
+			result:  result,
+			filters: q.filters,
+		}
+		q.root.call(tree, ctx)
+	}
+	return result
+}
+
+// Sets a user-defined filter function.  These may be used inside "?(..)" query
+// expressions to filter TOML document elements within a query.
+func (q *Query) SetFilter(name string, fn NodeFilterFn) {
+	if q.filters == &defaultFilterFunctions {
+		// clone the static table
+		q.filters = &map[string]NodeFilterFn{}
+		for k, v := range defaultFilterFunctions {
+			(*q.filters)[k] = v
+		}
+	}
+	(*q.filters)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]NodeFilterFn{
+	"tree": func(node interface{}) bool {
+		_, ok := node.(*TomlTree)
+		return ok
+	},
+	"int": func(node interface{}) bool {
+		_, ok := node.(int64)
+		return ok
+	},
+	"float": func(node interface{}) bool {
+		_, ok := node.(float64)
+		return ok
+	},
+	"string": func(node interface{}) bool {
+		_, ok := node.(string)
+		return ok
+	},
+	"time": func(node interface{}) bool {
+		_, ok := node.(time.Time)
+		return ok
+	},
+	"bool": func(node interface{}) bool {
+		_, ok := node.(bool)
+		return ok
+	},
+}

querylexer.go

@@ -0,0 +1,339 @@
+// TOML JSONPath lexer.
+//
+// Written using the principles developed by Rob Pike in
+// http://www.youtube.com/watch?v=HxaD_trXwRE
+
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// Lexer state function
+type queryLexStateFn func() queryLexStateFn
+
+// Lexer definition
+type queryLexer struct {
+	input      string
+	start      int
+	pos        int
+	width      int
+	tokens     chan token
+	depth      int
+	line       int
+	col        int
+	stringTerm string
+}
+
+func (l *queryLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+	close(l.tokens)
+}
+
+func (l *queryLexer) nextStart() {
+	// iterate by runes (utf8 characters)
+	// search for newlines and advance line/col counts
+	for i := l.start; i < l.pos; {
+		r, width := utf8.DecodeRuneInString(l.input[i:])
+		if r == '\n' {
+			l.line++
+			l.col = 1
+		} else {
+			l.col++
+		}
+		i += width
+	}
+	// advance start position to next token
+	l.start = l.pos
+}
+
+func (l *queryLexer) emit(t tokenType) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      l.input[l.start:l.pos],
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) emitWithValue(t tokenType, value string) {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	}
+	l.nextStart()
+}
+
+func (l *queryLexer) next() rune {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	var r rune
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *queryLexer) ignore() {
+	l.nextStart()
+}
+
+func (l *queryLexer) backup() {
+	l.pos -= l.width
+}
+
+func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
+	l.tokens <- token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func (l *queryLexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *queryLexer) accept(valid string) bool {
+	if strings.IndexRune(valid, l.next()) >= 0 {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+func (l *queryLexer) follow(next string) bool {
+	return strings.HasPrefix(l.input[l.pos:], next)
+}
+
+func (l *queryLexer) lexVoid() queryLexStateFn {
+	for {
+		next := l.peek()
+		switch next {
+		case '$':
+			l.pos++
+			l.emit(tokenDollar)
+			continue
+		case '.':
+			if l.follow("..") {
+				l.pos += 2
+				l.emit(tokenDotDot)
+			} else {
+				l.pos++
+				l.emit(tokenDot)
+			}
+			continue
+		case '[':
+			l.pos++
+			l.emit(tokenLeftBracket)
+			continue
+		case ']':
+			l.pos++
+			l.emit(tokenRightBracket)
+			continue
+		case ',':
+			l.pos++
+			l.emit(tokenComma)
+			continue
+		case '*':
+			l.pos++
+			l.emit(tokenStar)
+			continue
+		case '(':
+			l.pos++
+			l.emit(tokenLeftParen)
+			continue
+		case ')':
+			l.pos++
+			l.emit(tokenRightParen)
+			continue
+		case '?':
+			l.pos++
+			l.emit(tokenQuestion)
+			continue
+		case ':':
+			l.pos++
+			l.emit(tokenColon)
+			continue
+		case '\'':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		case '"':
+			l.ignore()
+			l.stringTerm = string(next)
+			return l.lexString
+		}
+
+		if isSpace(next) {
+			l.next()
+			l.ignore()
+			continue
+		}
+
+		if isAlphanumeric(next) {
+			return l.lexKey
+		}
+
+		if next == '+' || next == '-' || isDigit(next) {
+			return l.lexNumber
+		}
+
+		if l.next() == eof {
+			break
+		}
+
+		return l.errorf("unexpected char: '%v'", next)
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexKey() queryLexStateFn {
+	for {
+		next := l.peek()
+		if !isAlphanumeric(next) {
+			l.emit(tokenKey)
+			return l.lexVoid
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+	l.emit(tokenEOF)
+	return nil
+}
+
+func (l *queryLexer) lexString() queryLexStateFn {
+	l.pos++
+	l.ignore()
+	growingString := ""
+
+	for {
+		if l.follow(l.stringTerm) {
+			l.emitWithValue(tokenString, growingString)
+			l.pos++
+			l.ignore()
+			return l.lexVoid
+		}
+
+		if l.follow("\\\"") {
+			l.pos++
+			growingString += "\""
+		} else if l.follow("\\'") {
+			l.pos++
+			growingString += "'"
+		} else if l.follow("\\n") {
+			l.pos++
+			growingString += "\n"
+		} else if l.follow("\\b") {
+			l.pos++
+			growingString += "\b"
+		} else if l.follow("\\f") {
+			l.pos++
+			growingString += "\f"
+		} else if l.follow("\\/") {
+			l.pos++
+			growingString += "/"
+		} else if l.follow("\\t") {
+			l.pos++
+			growingString += "\t"
+		} else if l.follow("\\r") {
+			l.pos++
+			growingString += "\r"
+		} else if l.follow("\\\\") {
+			l.pos++
+			growingString += "\\"
+		} else if l.follow("\\u") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 4; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
+		} else if l.follow("\\") {
+			l.pos++
+			return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+		} else {
+			growingString += string(l.peek())
+		}
+
+		if l.next() == eof {
+			break
+		}
+	}
+
+	return l.errorf("unclosed string")
+}
+
+func (l *queryLexer) lexNumber() queryLexStateFn {
+	l.ignore()
+	if !l.accept("+") {
+		l.accept("-")
+	}
+	pointSeen := false
+	digitSeen := false
+	for {
+		next := l.next()
+		if next == '.' {
+			if pointSeen {
+				return l.errorf("cannot have two dots in one float")
+			}
+			if !isDigit(l.peek()) {
+				return l.errorf("float cannot end with a dot")
+			}
+			pointSeen = true
+		} else if isDigit(next) {
+			digitSeen = true
+		} else {
+			l.backup()
+			break
+		}
+		if pointSeen && !digitSeen {
+			return l.errorf("cannot start float with a dot")
+		}
+	}
+
+	if !digitSeen {
+		return l.errorf("no digit in that number")
+	}
+	if pointSeen {
+		l.emit(tokenFloat)
+	} else {
+		l.emit(tokenInteger)
+	}
+	return l.lexVoid
+}
+
+// Entry point
+func lexQuery(input string) chan token {
+	l := &queryLexer{
+		input:  input,
+		tokens: make(chan token),
+		line:   1,
+		col:    1,
+	}
+	go l.run()
+	return l.tokens
+}

querylexer_test.go

@@ -0,0 +1,97 @@
+package toml
+
+import (
+	"testing"
+)
+
+func testQLFlow(t *testing.T, input string, expectedFlow []token) {
+	ch := lexQuery(input)
+	for idx, expected := range expectedFlow {
+		token := <-ch
+		if token != expected {
+			t.Log("While testing #", idx, ":", input)
+			t.Log("compared", token, "to", expected)
+			t.Log(token.val, "<->", expected.val)
+			t.Log(token.typ, "<->", expected.typ)
+			t.Log(token.Line, "<->", expected.Line)
+			t.Log(token.Col, "<->", expected.Col)
+			t.FailNow()
+		}
+	}
+
+	tok, ok := <-ch
+	if ok {
+		t.Log("channel is not closed!")
+		t.Log(len(ch)+1, "tokens remaining:")
+
+		t.Log("token ->", tok)
+		for token := range ch {
+			t.Log("token ->", token)
+		}
+		t.FailNow()
+	}
+}
+
+func TestLexSpecialChars(t *testing.T) {
+	testQLFlow(t, " .$[]..()?*", []token{
+		token{Position{1, 2}, tokenDot, "."},
+		token{Position{1, 3}, tokenDollar, "$"},
+		token{Position{1, 4}, tokenLeftBracket, "["},
+		token{Position{1, 5}, tokenRightBracket, "]"},
+		token{Position{1, 6}, tokenDotDot, ".."},
+		token{Position{1, 8}, tokenLeftParen, "("},
+		token{Position{1, 9}, tokenRightParen, ")"},
+		token{Position{1, 10}, tokenQuestion, "?"},
+		token{Position{1, 11}, tokenStar, "*"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexString(t *testing.T) {
+	testQLFlow(t, "'foo'", []token{
+		token{Position{1, 2}, tokenString, "foo"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexDoubleString(t *testing.T) {
+	testQLFlow(t, `"bar"`, []token{
+		token{Position{1, 2}, tokenString, "bar"},
+		token{Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexKey(t *testing.T) {
+	testQLFlow(t, "foo", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 4}, tokenEOF, ""},
+	})
+}
+
+func TestLexRecurse(t *testing.T) {
+	testQLFlow(t, "$..*", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenDotDot, ".."},
+		token{Position{1, 4}, tokenStar, "*"},
+		token{Position{1, 5}, tokenEOF, ""},
+	})
+}
+
+func TestLexBracketKey(t *testing.T) {
+	testQLFlow(t, "$[foo]", []token{
+		token{Position{1, 1}, tokenDollar, "$"},
+		token{Position{1, 2}, tokenLeftBracket, "["},
+		token{Position{1, 3}, tokenKey, "foo"},
+		token{Position{1, 6}, tokenRightBracket, "]"},
+		token{Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexSpace(t *testing.T) {
+	testQLFlow(t, "foo bar baz", []token{
+		token{Position{1, 1}, tokenKey, "foo"},
+		token{Position{1, 5}, tokenKey, "bar"},
+		token{Position{1, 9}, tokenKey, "baz"},
+		token{Position{1, 12}, tokenEOF, ""},
+	})
+}

queryparser.go

@@ -0,0 +1,275 @@
+/*
+  Based on the "jsonpath" spec/concept.
+
+  http://goessner.net/articles/JsonPath/
+  https://code.google.com/p/json-path/
+*/
+
+package toml
+
+import (
+	"fmt"
+	"math"
+)
+
+type queryParser struct {
+	flow         chan token
+	tokensBuffer []token
+	query        *Query
+	union        []pathFn
+	err          error
+}
+
+type queryParserStateFn func() queryParserStateFn
+
+// Formats and panics an error message based on a token
+func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn {
+	p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...)
+	return nil // trigger parse to end
+}
+
+func (p *queryParser) run() {
+	for state := p.parseStart; state != nil; {
+		state = state()
+	}
+}
+
+func (p *queryParser) backup(tok *token) {
+	p.tokensBuffer = append(p.tokensBuffer, *tok)
+}
+
+func (p *queryParser) peek() *token {
+	if len(p.tokensBuffer) != 0 {
+		return &(p.tokensBuffer[0])
+	}
+
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	p.backup(&tok)
+	return &tok
+}
+
+func (p *queryParser) lookahead(types ...tokenType) bool {
+	result := true
+	buffer := []token{}
+
+	for _, typ := range types {
+		tok := p.getToken()
+		if tok == nil {
+			result = false
+			break
+		}
+		buffer = append(buffer, *tok)
+		if tok.typ != typ {
+			result = false
+			break
+		}
+	}
+	// add the tokens back to the buffer, and return
+	p.tokensBuffer = append(p.tokensBuffer, buffer...)
+	return result
+}
+
+func (p *queryParser) getToken() *token {
+	if len(p.tokensBuffer) != 0 {
+		tok := p.tokensBuffer[0]
+		p.tokensBuffer = p.tokensBuffer[1:]
+		return &tok
+	}
+	tok, ok := <-p.flow
+	if !ok {
+		return nil
+	}
+	return &tok
+}
+
+func (p *queryParser) parseStart() queryParserStateFn {
+	tok := p.getToken()
+
+	if tok == nil || tok.typ == tokenEOF {
+		return nil
+	}
+
+	if tok.typ != tokenDollar {
+		return p.parseError(tok, "Expected '$' at start of expression")
+	}
+
+	return p.parseMatchExpr
+}
+
+// handle '.' prefix, '[]', and '..'
+func (p *queryParser) parseMatchExpr() queryParserStateFn {
+	tok := p.getToken()
+	switch tok.typ {
+	case tokenDotDot:
+		p.query.appendPath(&matchRecursiveFn{})
+		// nested parse for '..'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenLeftBracket:
+			return p.parseBracketExpr
+		case tokenStar:
+			// do nothing - the recursive predicate is enough
+			return p.parseMatchExpr
+		}
+
+	case tokenDot:
+		// nested parse for '.'
+		tok := p.getToken()
+		switch tok.typ {
+		case tokenKey:
+			p.query.appendPath(newMatchKeyFn(tok.val))
+			return p.parseMatchExpr
+		case tokenStar:
+			p.query.appendPath(&matchAnyFn{})
+			return p.parseMatchExpr
+		}
+
+	case tokenLeftBracket:
+		return p.parseBracketExpr
+
+	case tokenEOF:
+		return nil // allow EOF at this stage
+	}
+	return p.parseError(tok, "expected match expression")
+	return nil
+}
+
+func (p *queryParser) parseBracketExpr() queryParserStateFn {
+	if p.lookahead(tokenInteger, tokenColon) {
+		return p.parseSliceExpr
+	}
+	if p.peek().typ == tokenColon {
+		return p.parseSliceExpr
+	}
+	return p.parseUnionExpr
+}
+
+func (p *queryParser) parseUnionExpr() queryParserStateFn {
+	var tok *token
+
+	// this state can be traversed after some sub-expressions
+	// so be careful when setting up state in the parser
+	if p.union == nil {
+		p.union = []pathFn{}
+	}
+
+loop: // labeled loop for easy breaking
+	for {
+		if len(p.union) > 0 {
+			// parse delimiter or terminator
+			tok = p.getToken()
+			switch tok.typ {
+			case tokenComma:
+				// do nothing
+			case tokenRightBracket:
+				break loop
+			default:
+				return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val)
+			}
+		}
+
+		// parse sub expression
+		tok = p.getToken()
+		switch tok.typ {
+		case tokenInteger:
+			p.union = append(p.union, newMatchIndexFn(tok.Int()))
+		case tokenKey:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenString:
+			p.union = append(p.union, newMatchKeyFn(tok.val))
+		case tokenQuestion:
+			return p.parseFilterExpr
+		default:
+			return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union))
+		}
+	}
+
+	// if there is only one sub-expression, use that instead
+	if len(p.union) == 1 {
+		p.query.appendPath(p.union[0])
+	} else {
+		p.query.appendPath(&matchUnionFn{p.union})
+	}
+
+	p.union = nil // clear out state
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseSliceExpr() queryParserStateFn {
+	// init slice to grab all elements
+	start, end, step := 0, math.MaxInt64, 1
+
+	// parse optional start
+	tok := p.getToken()
+	if tok.typ == tokenInteger {
+		start = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ':'")
+	}
+
+	// parse optional end
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		end = tok.Int()
+		tok = p.getToken()
+	}
+	if tok.typ == tokenRightBracket {
+		p.query.appendPath(newMatchSliceFn(start, end, step))
+		return p.parseMatchExpr
+	}
+	if tok.typ != tokenColon {
+		return p.parseError(tok, "expected ']' or ':'")
+	}
+
+	// parse optional step
+	tok = p.getToken()
+	if tok.typ == tokenInteger {
+		step = tok.Int()
+		if step < 0 {
+			return p.parseError(tok, "step must be a positive value")
+		}
+		tok = p.getToken()
+	}
+	if tok.typ != tokenRightBracket {
+		return p.parseError(tok, "expected ']'")
+	}
+
+	p.query.appendPath(newMatchSliceFn(start, end, step))
+	return p.parseMatchExpr
+}
+
+func (p *queryParser) parseFilterExpr() queryParserStateFn {
+	tok := p.getToken()
+	if tok.typ != tokenLeftParen {
+		return p.parseError(tok, "expected left-parenthesis for filter expression")
+	}
+	tok = p.getToken()
+	if tok.typ != tokenKey && tok.typ != tokenString {
+		return p.parseError(tok, "expected key or string for filter funciton name")
+	}
+	name := tok.val
+	tok = p.getToken()
+	if tok.typ != tokenRightParen {
+		return p.parseError(tok, "expected right-parenthesis for filter expression")
+	}
+	p.union = append(p.union, newMatchFilterFn(name, tok.Position))
+	return p.parseUnionExpr
+}
+
+func parseQuery(flow chan token) (*Query, error) {
+	parser := &queryParser{
+		flow:         flow,
+		tokensBuffer: []token{},
+		query:        newQuery(),
+	}
+	parser.run()
+	return parser.query, parser.err
+}

queryparser_test.go

@@ -0,0 +1,483 @@
+package toml
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+type queryTestNode struct {
+	value    interface{}
+	position Position
+}
+
+func valueString(root interface{}) string {
+	result := "" //fmt.Sprintf("%T:", root)
+	switch node := root.(type) {
+	case *tomlValue:
+		return valueString(node.value)
+	case *QueryResult:
+		items := []string{}
+		for i, v := range node.Values() {
+			items = append(items, fmt.Sprintf("%s:%s",
+				node.Positions()[i].String(), valueString(v)))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case queryTestNode:
+		result = fmt.Sprintf("%s:%s",
+			node.position.String(), valueString(node.value))
+	case []interface{}:
+		items := []string{}
+		for _, v := range node {
+			items = append(items, valueString(v))
+		}
+		sort.Strings(items)
+		result = "[" + strings.Join(items, ", ") + "]"
+	case *TomlTree:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for _, k := range node.Keys() {
+			v := node.GetPath([]string{k})
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case map[string]interface{}:
+		// workaround for unreliable map key ordering
+		items := []string{}
+		for k, v := range node {
+			items = append(items, k+":"+valueString(v))
+		}
+		sort.Strings(items)
+		result = "{" + strings.Join(items, ", ") + "}"
+	case int64:
+		result += fmt.Sprintf("%d", node)
+	case string:
+		result += "'" + node + "'"
+	case float64:
+		result += fmt.Sprintf("%f", node)
+	case bool:
+		result += fmt.Sprintf("%t", node)
+	case time.Time:
+		result += fmt.Sprintf("'%v'", node)
+	}
+	return result
+}
+
+func assertValue(t *testing.T, result, ref interface{}) {
+	pathStr := valueString(result)
+	refStr := valueString(ref)
+	if pathStr != refStr {
+		t.Errorf("values do not match")
+		t.Log("test:", pathStr)
+		t.Log("ref: ", refStr)
+	}
+}
+
+func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
+	tree, err := Load(toml)
+	if err != nil {
+		t.Errorf("Non-nil toml parse error: %v", err)
+		return
+	}
+	q, err := CompileQuery(query)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	results := q.Execute(tree)
+	assertValue(t, results, ref)
+}
+
+func TestQueryRoot(t *testing.T) {
+	assertQueryPositions(t,
+		"a = 42",
+		"$",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(42),
+				}, Position{1, 1},
+			},
+		})
+}
+
+func TestQueryKey(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo.a",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryKeyString(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = 42",
+		"$.foo['a']",
+		[]interface{}{
+			queryTestNode{
+				int64(42), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryIndex(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[5]",
+		[]interface{}{
+			queryTestNode{
+				int64(6), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceRange(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(4), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQuerySliceStep(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo]\na = [1,2,3,4,5,6,7,8,9,0]",
+		"$.foo.a[0:5:2]",
+		[]interface{}{
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(3), Position{2, 1},
+			},
+			queryTestNode{
+				int64(5), Position{2, 1},
+			},
+		})
+}
+
+func TestQueryAny(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[foo.baz]\na=3\nb=4",
+		"$.foo.*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+		})
+}
+func TestQueryUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$.*[bar,foo]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryRecursionAll(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..*",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": map[string]interface{}{
+							"a": int64(1),
+							"b": int64(2),
+						},
+					},
+					"baz": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(3),
+							"b": int64(4),
+						},
+					},
+					"gorf": map[string]interface{}{
+						"foo": map[string]interface{}{
+							"a": int64(5),
+							"b": int64(6),
+						},
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				int64(1), Position{2, 1},
+			},
+			queryTestNode{
+				int64(2), Position{3, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(3),
+						"b": int64(4),
+					},
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				int64(3), Position{5, 1},
+			},
+			queryTestNode{
+				int64(4), Position{6, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"a": int64(5),
+						"b": int64(6),
+					},
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+			queryTestNode{
+				int64(5), Position{8, 1},
+			},
+			queryTestNode{
+				int64(6), Position{9, 1},
+			},
+		})
+}
+
+func TestQueryRecursionUnionSimple(t *testing.T) {
+	assertQueryPositions(t,
+		"[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6",
+		"$..['foo','bar']",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"bar": map[string]interface{}{
+						"a": int64(1),
+						"b": int64(2),
+					},
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(3),
+					"b": int64(4),
+				}, Position{4, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(1),
+					"b": int64(2),
+				}, Position{1, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"a": int64(5),
+					"b": int64(6),
+				}, Position{7, 1},
+			},
+		})
+}
+
+func TestQueryFilterFn(t *testing.T) {
+	buff, err := ioutil.ReadFile("example.toml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(int)]",
+		[]interface{}{
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8001), Position{13, 1},
+			},
+			queryTestNode{
+				int64(8002), Position{13, 1},
+			},
+			queryTestNode{
+				int64(5000), Position{14, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(string)]",
+		[]interface{}{
+			queryTestNode{
+				"TOML Example", Position{3, 1},
+			},
+			queryTestNode{
+				"Tom Preston-Werner", Position{6, 1},
+			},
+			queryTestNode{
+				"GitHub", Position{7, 1},
+			},
+			queryTestNode{
+				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
+				Position{8, 1},
+			},
+			queryTestNode{
+				"192.168.1.1", Position{12, 1},
+			},
+			queryTestNode{
+				"10.0.0.1", Position{21, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{22, 3},
+			},
+			queryTestNode{
+				"10.0.0.2", Position{25, 3},
+			},
+			queryTestNode{
+				"eqdc10", Position{26, 3},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(float)]",
+		[]interface{}{
+		// no float values in document
+		})
+
+	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
+	assertQueryPositions(t, string(buff),
+		"$..[?(tree)]",
+		[]interface{}{
+			queryTestNode{
+				map[string]interface{}{
+					"name":         "Tom Preston-Werner",
+					"organization": "GitHub",
+					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+					"dob":          tv,
+				}, Position{5, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"server":         "192.168.1.1",
+					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+					"connection_max": int64(5000),
+					"enabled":        true,
+				}, Position{11, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"alpha": map[string]interface{}{
+						"ip": "10.0.0.1",
+						"dc": "eqdc10",
+					},
+					"beta": map[string]interface{}{
+						"ip": "10.0.0.2",
+						"dc": "eqdc10",
+					},
+				}, Position{17, 1},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.1",
+					"dc": "eqdc10",
+				}, Position{20, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"ip": "10.0.0.2",
+					"dc": "eqdc10",
+				}, Position{24, 3},
+			},
+			queryTestNode{
+				map[string]interface{}{
+					"data": []interface{}{
+						[]interface{}{"gamma", "delta"},
+						[]interface{}{int64(1), int64(2)},
+					},
+				}, Position{28, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(time)]",
+		[]interface{}{
+			queryTestNode{
+				tv, Position{9, 1},
+			},
+		})
+
+	assertQueryPositions(t, string(buff),
+		"$..[?(bool)]",
+		[]interface{}{
+			queryTestNode{
+				true, Position{15, 1},
+			},
+		})
+}

token.go

@@ -0,0 +1,132 @@
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenComment
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenTrue
+	tokenFalse
+	tokenFloat
+	tokenEqual
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftParen
+	tokenRightParen
+	tokenDoubleLeftBracket
+	tokenDoubleRightBracket
+	tokenDate
+	tokenKeyGroup
+	tokenKeyGroupArray
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+	tokenEOL
+)
+
+var tokenTypeNames = []string{
+	"EOF",
+	"Comment",
+	"Key",
+	"String",
+	"Integer",
+	"True",
+	"False",
+	"Float",
+	"=",
+	"[",
+	"[",
+	"(",
+	")",
+	"]]",
+	"[[",
+	"Date",
+	"KeyGroup",
+	"KeyGroupArray",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+	"EOL",
+}
+
+type token struct {
+	Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	if len(t.val) > 10 {
+		return fmt.Sprintf("%.10q...", t.val)
+	}
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isKeyChar(r rune) bool {
+	// "Keys start with the first non-whitespace character and end with the last
+	// non-whitespace character before the equals sign."
+	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+}

toml.go

@@ -1,7 +1,3 @@
-// Package toml is a TOML markup language parser.
-//
-// This version supports the specification as described in
-// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
 package toml
 
 import (
@@ -28,7 +24,7 @@ type TomlTree struct {
 func newTomlTree() *TomlTree {
 	return &TomlTree{
 		values:   make(map[string]interface{}),
-		position: Position{0, 0},
+		position: Position{},
 	}
 }
 
@@ -103,7 +99,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 // GetPosition returns the position of the given key.
 func (t *TomlTree) GetPosition(key string) Position {
 	if key == "" {
-		return Position{0, 0}
+		return t.position
 	}
 	return t.GetPositionPath(strings.Split(key, "."))
 }
@@ -199,7 +195,7 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 // and tree[a][b][c]
 //
 // Returns nil on success, error object on failure
-func (t *TomlTree) createSubTree(keys []string) error {
+func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	subtree := t
 	for _, intermediateKey := range keys {
 		if intermediateKey == "" {
@@ -207,8 +203,10 @@ func (t *TomlTree) createSubTree(keys []string) error {
 		}
 		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			nextTree = newTomlTree()
+			tree := newTomlTree()
-			subtree.values[intermediateKey] = nextTree
+			tree.position = pos
+			subtree.values[intermediateKey] = tree
+			nextTree = tree
 		}
 
 		switch node := nextTree.(type) {
@@ -317,6 +315,14 @@ func (t *TomlTree) toToml(indent, keyspace string) string {
 	return result
 }
 
+func (t *TomlTree) Query(query string) (*QueryResult, error) {
+	if q, err := CompileQuery(query); err != nil {
+		return nil, err
+	} else {
+		return q.Execute(t), nil
+	}
+}
+
 // ToString generates a human-readable representation of the current tree.
 // Output spans multiple lines, and is suitable for ingest by a TOML parser
 func (t *TomlTree) ToString() string {
@@ -333,8 +339,7 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	_, flow := lex(content)
+	tree = parseToml(lexToml(content))
-	tree = parse(flow)
 	return
 }
 

toml_test.go

@@ -47,3 +47,28 @@ func TestTomlGetPath(t *testing.T) {
 		}
 	}
 }
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result, err := tree.Query("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*TomlTree); !ok {
+		t.Errorf("Expected type of TomlTree: %T Tv", values[0], values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}