ostiwe/go-toml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move query to its own subpackage (#152)

Browse Source 
Move all the query system to its own package. The reason is to
avoid it to rely on unexported methods and structures, and move
it out of the main package since this is really not a core
feature. It is still tied to the toml.TomlTree and toml.Position
structures for now.

* Move query mechanism to its own subpackage
* Rename QueryResult to Result to avoid stutter
* Add query.CompileAndExecute

Fixes #116
This commit is contained in:
Thomas Pelletier
2017-05-07 17:14:13 -07:00
committed by GitHub
parent 64bc956d5e
commit 23f644976a
17 changed files with 665 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -18,7 +18,7 @@ Go-toml provides the following features for using data parsed from TOML document
* Load TOML documents from files and string data * Load TOML documents from files and string data
* Easily navigate TOML structure using TomlTree * Easily navigate TOML structure using TomlTree
* Line & column position data for all parsed elements * Line & column position data for all parsed elements
* Query support similar to JSON-Path * [Query support similar to JSON-Path](query/)
* Syntax errors contain line and column numbers * Syntax errors contain line and column numbers
Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing: Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
doc.go
+4 -170
View File
@@ -75,176 +75,10 @@
// return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition("")) // return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
// } // }
// //
// Query Support // JSONPath-like queries
// //
// The TOML query path implementation is based loosely on the JSONPath specification: // The package github.com/pelletier/go-toml/query implements a system
// http://goessner.net/articles/JsonPath/ // similar to JSONPath to quickly retrive elements of a TOML document using a
// // single expression. See the package documentation for more information.
// The idea behind a query path is to allow quick access to any element, or set
// of elements within TOML document, with a single expression.
//
// result, err := tree.Query("$.foo.bar.baz")
//
// This is roughly equivalent to:
//
// next := tree.Get("foo")
// if next != nil {
// next = next.Get("bar")
// if next != nil {
// next = next.Get("baz")
// }
// }
// result := next
//
// err is nil if any parsing exception occurs.
//
// If no node in the tree matches the query, result will simply contain an empty list of
// items.
//
// As illustrated above, the query path is much more efficient, especially since
// the structure of the TOML file can vary. Rather than making assumptions about
// a document's structure, a query allows the programmer to make structured
// requests into the document, and get zero or more values as a result.
//
// The syntax of a query begins with a root token, followed by any number
// sub-expressions:
//
// $
// Root of the TOML tree. This must always come first.
// .name
// Selects child of this node, where 'name' is a TOML key
// name.
// ['name']
// Selects child of this node, where 'name' is a string
// containing a TOML key name.
// [index]
// Selcts child array element at 'index'.
// ..expr
// Recursively selects all children, filtered by an a union,
// index, or slice expression.
// ..*
// Recursive selection of all nodes at this point in the
// tree.
// .*
// Selects all children of the current node.
// [expr,expr]
// Union operator - a logical 'or' grouping of two or more
// sub-expressions: index, key name, or filter.
// [start:end:step]
// Slice operator - selects array elements from start to
// end-1, at the given step. All three arguments are
// optional.
// [?(filter)]
// Named filter expression - the function 'filter' is
// used to filter children at this node.
//
// Query Indexes And Slices
//
// Index expressions perform no bounds checking, and will contribute no
// values to the result set if the provided index or index range is invalid.
// Negative indexes represent values from the end of the array, counting backwards.
//
// // select the last index of the array named 'foo'
// tree.Query("$.foo[-1]")
//
// Slice expressions are supported, by using ':' to separate a start/end index pair.
//
// // select up to the first five elements in the array
// tree.Query("$.foo[0:5]")
//
// Slice expressions also allow negative indexes for the start and stop
// arguments.
//
// // select all array elements.
// tree.Query("$.foo[0:-1]")
//
// Slice expressions may have an optional stride/step parameter:
//
// // select every other element
// tree.Query("$.foo[0:-1:2]")
//
// Slice start and end parameters are also optional:
//
// // these are all equivalent and select all the values in the array
// tree.Query("$.foo[:]")
// tree.Query("$.foo[0:]")
// tree.Query("$.foo[:-1]")
// tree.Query("$.foo[0:-1:]")
// tree.Query("$.foo[::1]")
// tree.Query("$.foo[0::1]")
// tree.Query("$.foo[:-1:1]")
// tree.Query("$.foo[0:-1:1]")
//
// Query Filters
//
// Query filters are used within a Union [,] or single Filter [] expression.
// A filter only allows nodes that qualify through to the next expression,
// and/or into the result set.
//
// // returns children of foo that are permitted by the 'bar' filter.
// tree.Query("$.foo[?(bar)]")
//
// There are several filters provided with the library:
//
// tree
// Allows nodes of type TomlTree.
// int
// Allows nodes of type int64.
// float
// Allows nodes of type float64.
// string
// Allows nodes of type string.
// time
// Allows nodes of type time.Time.
// bool
// Allows nodes of type bool.
//
// Query Results
//
// An executed query returns a QueryResult object. This contains the nodes
// in the TOML tree that qualify the query expression. Position information
// is also available for each value in the set.
//
// // display the results of a query
// results := tree.Query("$.foo.bar.baz")
// for idx, value := results.Values() {
// fmt.Println("%v: %v", results.Positions()[idx], value)
// }
//
// Compiled Queries
//
// Queries may be executed directly on a TomlTree object, or compiled ahead
// of time and executed discretely. The former is more convienent, but has the
// penalty of having to recompile the query expression each time.
//
// // basic query
// results := tree.Query("$.foo.bar.baz")
//
// // compiled query
// query := toml.CompileQuery("$.foo.bar.baz")
// results := query.Execute(tree)
//
// // run the compiled query again on a different tree
// moreResults := query.Execute(anotherTree)
//
// User Defined Query Filters
//
// Filter expressions may also be user defined by using the SetFilter()
// function on the Query object. The function must return true/false, which
// signifies if the passed node is kept or discarded, respectively.
//
// // create a query that references a user-defined filter
// query, _ := CompileQuery("$[?(bazOnly)]")
//
// // define the filter, and assign it to the query
// query.SetFilter("bazOnly", func(node interface{}) bool{
// if tree, ok := node.(*TomlTree); ok {
// return tree.Has("baz")
// }
// return false // reject all other node types
// })
//
// // run the query
// query.Execute(tree)
// //
package toml package toml
doc_test.go
-52
View File
@@ -6,52 +6,6 @@ import (
"fmt" "fmt"
) )
func ExampleNodeFilterFn_filterExample() {
tree, _ := Load(`
[struct_one]
foo = "foo"
bar = "bar"
[struct_two]
baz = "baz"
gorf = "gorf"
`)
// create a query that references a user-defined-filter
query, _ := CompileQuery("$[?(bazOnly)]")
// define the filter, and assign it to the query
query.SetFilter("bazOnly", func(node interface{}) bool {
if tree, ok := node.(*TomlTree); ok {
return tree.Has("baz")
}
return false // reject all other node types
})
// results contain only the 'struct_two' TomlTree
query.Execute(tree)
}
func ExampleQuery_queryExample() {
config, _ := Load(`
[[book]]
title = "The Stand"
author = "Stephen King"
[[book]]
title = "For Whom the Bell Tolls"
author = "Ernest Hemmingway"
[[book]]
title = "Neuromancer"
author = "William Gibson"
`)
// find and print all the authors in the document
authors, _ := config.Query("$.book.author")
for _, name := range authors.Values() {
fmt.Println(name)
}
}
func Example_comprehensiveExample() { func Example_comprehensiveExample() {
config, err := LoadFile("config.toml") config, err := LoadFile("config.toml")
@@ -71,11 +25,5 @@ func Example_comprehensiveExample() {
// show where elements are in the file // show where elements are in the file
fmt.Printf("User position: %v\n", configTree.GetPosition("user")) fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
fmt.Printf("Password position: %v\n", configTree.GetPosition("password")) fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
// use a query to gather elements without walking the tree
results, _ := config.Query("$..[user,password]")
for ii, item := range results.Values() {
fmt.Printf("Query result %d: %v\n", ii, item)
}
} }
} }
query/doc.go
+175
View File
@@ -0,0 +1,175 @@
// Package query performs JSONPath-like queries on a TOML document.
//
// The query path implementation is based loosely on the JSONPath specification:
// http://goessner.net/articles/JsonPath/.
//
// The idea behind a query path is to allow quick access to any element, or set
// of elements within TOML document, with a single expression.
//
// result, err := query.CompileAndExecute("$.foo.bar.baz", tree)
//
// This is roughly equivalent to:
//
// next := tree.Get("foo")
// if next != nil {
// next = next.Get("bar")
// if next != nil {
// next = next.Get("baz")
// }
// }
// result := next
//
// err is nil if any parsing exception occurs.
//
// If no node in the tree matches the query, result will simply contain an empty list of
// items.
//
// As illustrated above, the query path is much more efficient, especially since
// the structure of the TOML file can vary. Rather than making assumptions about
// a document's structure, a query allows the programmer to make structured
// requests into the document, and get zero or more values as a result.
//
// Query syntax
//
// The syntax of a query begins with a root token, followed by any number
// sub-expressions:
//
// $
// Root of the TOML tree. This must always come first.
// .name
// Selects child of this node, where 'name' is a TOML key
// name.
// ['name']
// Selects child of this node, where 'name' is a string
// containing a TOML key name.
// [index]
// Selcts child array element at 'index'.
// ..expr
// Recursively selects all children, filtered by an a union,
// index, or slice expression.
// ..*
// Recursive selection of all nodes at this point in the
// tree.
// .*
// Selects all children of the current node.
// [expr,expr]
// Union operator - a logical 'or' grouping of two or more
// sub-expressions: index, key name, or filter.
// [start:end:step]
// Slice operator - selects array elements from start to
// end-1, at the given step. All three arguments are
// optional.
// [?(filter)]
// Named filter expression - the function 'filter' is
// used to filter children at this node.
//
// Query Indexes And Slices
//
// Index expressions perform no bounds checking, and will contribute no
// values to the result set if the provided index or index range is invalid.
// Negative indexes represent values from the end of the array, counting backwards.
//
// // select the last index of the array named 'foo'
// query.CompileAndExecute("$.foo[-1]", tree)
//
// Slice expressions are supported, by using ':' to separate a start/end index pair.
//
// // select up to the first five elements in the array
// query.CompileAndExecute("$.foo[0:5]", tree)
//
// Slice expressions also allow negative indexes for the start and stop
// arguments.
//
// // select all array elements.
// query.CompileAndExecute("$.foo[0:-1]", tree)
//
// Slice expressions may have an optional stride/step parameter:
//
// // select every other element
// query.CompileAndExecute("$.foo[0:-1:2]", tree)
//
// Slice start and end parameters are also optional:
//
// // these are all equivalent and select all the values in the array
// query.CompileAndExecute("$.foo[:]", tree)
// query.CompileAndExecute("$.foo[0:]", tree)
// query.CompileAndExecute("$.foo[:-1]", tree)
// query.CompileAndExecute("$.foo[0:-1:]", tree)
// query.CompileAndExecute("$.foo[::1]", tree)
// query.CompileAndExecute("$.foo[0::1]", tree)
// query.CompileAndExecute("$.foo[:-1:1]", tree)
// query.CompileAndExecute("$.foo[0:-1:1]", tree)
//
// Query Filters
//
// Query filters are used within a Union [,] or single Filter [] expression.
// A filter only allows nodes that qualify through to the next expression,
// and/or into the result set.
//
// // returns children of foo that are permitted by the 'bar' filter.
// query.CompileAndExecute("$.foo[?(bar)]", tree)
//
// There are several filters provided with the library:
//
// tree
// Allows nodes of type TomlTree.
// int
// Allows nodes of type int64.
// float
// Allows nodes of type float64.
// string
// Allows nodes of type string.
// time
// Allows nodes of type time.Time.
// bool
// Allows nodes of type bool.
//
// Query Results
//
// An executed query returns a Result object. This contains the nodes
// in the TOML tree that qualify the query expression. Position information
// is also available for each value in the set.
//
// // display the results of a query
// results := query.CompileAndExecute("$.foo.bar.baz", tree)
// for idx, value := results.Values() {
// fmt.Println("%v: %v", results.Positions()[idx], value)
// }
//
// Compiled Queries
//
// Queries may be executed directly on a TomlTree object, or compiled ahead
// of time and executed discretely. The former is more convienent, but has the
// penalty of having to recompile the query expression each time.
//
// // basic query
// results := query.CompileAndExecute("$.foo.bar.baz", tree)
//
// // compiled query
// query, err := toml.Compile("$.foo.bar.baz")
// results := query.Execute(tree)
//
// // run the compiled query again on a different tree
// moreResults := query.Execute(anotherTree)
//
// User Defined Query Filters
//
// Filter expressions may also be user defined by using the SetFilter()
// function on the Query object. The function must return true/false, which
// signifies if the passed node is kept or discarded, respectively.
//
// // create a query that references a user-defined filter
// query, _ := query.Compile("$[?(bazOnly)]")
//
// // define the filter, and assign it to the query
// query.SetFilter("bazOnly", func(node interface{}) bool{
// if tree, ok := node.(*TomlTree); ok {
// return tree.Has("baz")
// }
// return false // reject all other node types
// })
//
// // run the query
// query.Execute(tree)
//
package query
querylexer.go → query/lexer.go
+5 -4
View File
@@ -3,13 +3,14 @@
// Written using the principles developed by Rob Pike in // Written using the principles developed by Rob Pike in
// http://www.youtube.com/watch?v=HxaD_trXwRE // http://www.youtube.com/watch?v=HxaD_trXwRE
package toml package query
import ( import (
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
"unicode/utf8" "unicode/utf8"
"github.com/pelletier/go-toml"
) )
// Lexer state function // Lexer state function
@@ -54,7 +55,7 @@ func (l *queryLexer) nextStart() {
func (l *queryLexer) emit(t tokenType) { func (l *queryLexer) emit(t tokenType) {
l.tokens <- token{ l.tokens <- token{
Position: Position{l.line, l.col}, Position: toml.Position{Line:l.line, Col:l.col},
typ: t, typ: t,
val: l.input[l.start:l.pos], val: l.input[l.start:l.pos],
} }
@@ -63,7 +64,7 @@ func (l *queryLexer) emit(t tokenType) {
func (l *queryLexer) emitWithValue(t tokenType, value string) { func (l *queryLexer) emitWithValue(t tokenType, value string) {
l.tokens <- token{ l.tokens <- token{
Position: Position{l.line, l.col}, Position: toml.Position{Line:l.line, Col:l.col},
typ: t, typ: t,
val: value, val: value,
} }
@@ -91,7 +92,7 @@ func (l *queryLexer) backup() {
func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn { func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
l.tokens <- token{ l.tokens <- token{
Position: Position{l.line, l.col}, Position: toml.Position{Line:l.line, Col:l.col},
typ: tokenError, typ: tokenError,
val: fmt.Sprintf(format, args...), val: fmt.Sprintf(format, args...),
} }
querylexer_test.go → query/lexer_test.go
+50 -49
View File
@@ -1,7 +1,8 @@
package toml package query
import ( import (
"testing" "testing"
"github.com/pelletier/go-toml"
) )
func testQLFlow(t *testing.T, input string, expectedFlow []token) { func testQLFlow(t *testing.T, input string, expectedFlow []token) {
@@ -36,143 +37,143 @@ func testQLFlow(t *testing.T, input string, expectedFlow []token) {
func TestLexSpecialChars(t *testing.T) { func TestLexSpecialChars(t *testing.T) {
testQLFlow(t, " .$[]..()?*", []token{ testQLFlow(t, " .$[]..()?*", []token{
{Position{1, 2}, tokenDot, "."}, {toml.Position{1, 2}, tokenDot, "."},
{Position{1, 3}, tokenDollar, "$"}, {toml.Position{1, 3}, tokenDollar, "$"},
{Position{1, 4}, tokenLeftBracket, "["}, {toml.Position{1, 4}, tokenLeftBracket, "["},
{Position{1, 5}, tokenRightBracket, "]"}, {toml.Position{1, 5}, tokenRightBracket, "]"},
{Position{1, 6}, tokenDotDot, ".."}, {toml.Position{1, 6}, tokenDotDot, ".."},
{Position{1, 8}, tokenLeftParen, "("}, {toml.Position{1, 8}, tokenLeftParen, "("},
{Position{1, 9}, tokenRightParen, ")"}, {toml.Position{1, 9}, tokenRightParen, ")"},
{Position{1, 10}, tokenQuestion, "?"}, {toml.Position{1, 10}, tokenQuestion, "?"},
{Position{1, 11}, tokenStar, "*"}, {toml.Position{1, 11}, tokenStar, "*"},
{Position{1, 12}, tokenEOF, ""}, {toml.Position{1, 12}, tokenEOF, ""},
}) })
} }
func TestLexString(t *testing.T) { func TestLexString(t *testing.T) {
testQLFlow(t, "'foo\n'", []token{ testQLFlow(t, "'foo\n'", []token{
{Position{1, 2}, tokenString, "foo\n"}, {toml.Position{1, 2}, tokenString, "foo\n"},
{Position{2, 2}, tokenEOF, ""}, {toml.Position{2, 2}, tokenEOF, ""},
}) })
} }
func TestLexDoubleString(t *testing.T) { func TestLexDoubleString(t *testing.T) {
testQLFlow(t, `"bar"`, []token{ testQLFlow(t, `"bar"`, []token{
{Position{1, 2}, tokenString, "bar"}, {toml.Position{1, 2}, tokenString, "bar"},
{Position{1, 6}, tokenEOF, ""}, {toml.Position{1, 6}, tokenEOF, ""},
}) })
} }
func TestLexStringEscapes(t *testing.T) { func TestLexStringEscapes(t *testing.T) {
testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{ testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{
{Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"}, {toml.Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"},
{Position{1, 55}, tokenEOF, ""}, {toml.Position{1, 55}, tokenEOF, ""},
}) })
} }
func TestLexStringUnfinishedUnicode4(t *testing.T) { func TestLexStringUnfinishedUnicode4(t *testing.T) {
testQLFlow(t, `"\u000"`, []token{ testQLFlow(t, `"\u000"`, []token{
{Position{1, 2}, tokenError, "unfinished unicode escape"}, {toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
}) })
} }
func TestLexStringUnfinishedUnicode8(t *testing.T) { func TestLexStringUnfinishedUnicode8(t *testing.T) {
testQLFlow(t, `"\U0000"`, []token{ testQLFlow(t, `"\U0000"`, []token{
{Position{1, 2}, tokenError, "unfinished unicode escape"}, {toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
}) })
} }
func TestLexStringInvalidEscape(t *testing.T) { func TestLexStringInvalidEscape(t *testing.T) {
testQLFlow(t, `"\x"`, []token{ testQLFlow(t, `"\x"`, []token{
{Position{1, 2}, tokenError, "invalid escape sequence: \\x"}, {toml.Position{1, 2}, tokenError, "invalid escape sequence: \\x"},
}) })
} }
func TestLexStringUnfinished(t *testing.T) { func TestLexStringUnfinished(t *testing.T) {
testQLFlow(t, `"bar`, []token{ testQLFlow(t, `"bar`, []token{
{Position{1, 2}, tokenError, "unclosed string"}, {toml.Position{1, 2}, tokenError, "unclosed string"},
}) })
} }
func TestLexKey(t *testing.T) { func TestLexKey(t *testing.T) {
testQLFlow(t, "foo", []token{ testQLFlow(t, "foo", []token{
{Position{1, 1}, tokenKey, "foo"}, {toml.Position{1, 1}, tokenKey, "foo"},
{Position{1, 4}, tokenEOF, ""}, {toml.Position{1, 4}, tokenEOF, ""},
}) })
} }
func TestLexRecurse(t *testing.T) { func TestLexRecurse(t *testing.T) {
testQLFlow(t, "$..*", []token{ testQLFlow(t, "$..*", []token{
{Position{1, 1}, tokenDollar, "$"}, {toml.Position{1, 1}, tokenDollar, "$"},
{Position{1, 2}, tokenDotDot, ".."}, {toml.Position{1, 2}, tokenDotDot, ".."},
{Position{1, 4}, tokenStar, "*"}, {toml.Position{1, 4}, tokenStar, "*"},
{Position{1, 5}, tokenEOF, ""}, {toml.Position{1, 5}, tokenEOF, ""},
}) })
} }
func TestLexBracketKey(t *testing.T) { func TestLexBracketKey(t *testing.T) {
testQLFlow(t, "$[foo]", []token{ testQLFlow(t, "$[foo]", []token{
{Position{1, 1}, tokenDollar, "$"}, {toml.Position{1, 1}, tokenDollar, "$"},
{Position{1, 2}, tokenLeftBracket, "["}, {toml.Position{1, 2}, tokenLeftBracket, "["},
{Position{1, 3}, tokenKey, "foo"}, {toml.Position{1, 3}, tokenKey, "foo"},
{Position{1, 6}, tokenRightBracket, "]"}, {toml.Position{1, 6}, tokenRightBracket, "]"},
{Position{1, 7}, tokenEOF, ""}, {toml.Position{1, 7}, tokenEOF, ""},
}) })
} }
func TestLexSpace(t *testing.T) { func TestLexSpace(t *testing.T) {
testQLFlow(t, "foo bar baz", []token{ testQLFlow(t, "foo bar baz", []token{
{Position{1, 1}, tokenKey, "foo"}, {toml.Position{1, 1}, tokenKey, "foo"},
{Position{1, 5}, tokenKey, "bar"}, {toml.Position{1, 5}, tokenKey, "bar"},
{Position{1, 9}, tokenKey, "baz"}, {toml.Position{1, 9}, tokenKey, "baz"},
{Position{1, 12}, tokenEOF, ""}, {toml.Position{1, 12}, tokenEOF, ""},
}) })
} }
func TestLexInteger(t *testing.T) { func TestLexInteger(t *testing.T) {
testQLFlow(t, "100 +200 -300", []token{ testQLFlow(t, "100 +200 -300", []token{
{Position{1, 1}, tokenInteger, "100"}, {toml.Position{1, 1}, tokenInteger, "100"},
{Position{1, 5}, tokenInteger, "+200"}, {toml.Position{1, 5}, tokenInteger, "+200"},
{Position{1, 10}, tokenInteger, "-300"}, {toml.Position{1, 10}, tokenInteger, "-300"},
{Position{1, 14}, tokenEOF, ""}, {toml.Position{1, 14}, tokenEOF, ""},
}) })
} }
func TestLexFloat(t *testing.T) { func TestLexFloat(t *testing.T) {
testQLFlow(t, "100.0 +200.0 -300.0", []token{ testQLFlow(t, "100.0 +200.0 -300.0", []token{
{Position{1, 1}, tokenFloat, "100.0"}, {toml.Position{1, 1}, tokenFloat, "100.0"},
{Position{1, 7}, tokenFloat, "+200.0"}, {toml.Position{1, 7}, tokenFloat, "+200.0"},
{Position{1, 14}, tokenFloat, "-300.0"}, {toml.Position{1, 14}, tokenFloat, "-300.0"},
{Position{1, 20}, tokenEOF, ""}, {toml.Position{1, 20}, tokenEOF, ""},
}) })
} }
func TestLexFloatWithMultipleDots(t *testing.T) { func TestLexFloatWithMultipleDots(t *testing.T) {
testQLFlow(t, "4.2.", []token{ testQLFlow(t, "4.2.", []token{
{Position{1, 1}, tokenError, "cannot have two dots in one float"}, {toml.Position{1, 1}, tokenError, "cannot have two dots in one float"},
}) })
} }
func TestLexFloatLeadingDot(t *testing.T) { func TestLexFloatLeadingDot(t *testing.T) {
testQLFlow(t, "+.1", []token{ testQLFlow(t, "+.1", []token{
{Position{1, 1}, tokenError, "cannot start float with a dot"}, {toml.Position{1, 1}, tokenError, "cannot start float with a dot"},
}) })
} }
func TestLexFloatWithTrailingDot(t *testing.T) { func TestLexFloatWithTrailingDot(t *testing.T) {
testQLFlow(t, "42.", []token{ testQLFlow(t, "42.", []token{
{Position{1, 1}, tokenError, "float cannot end with a dot"}, {toml.Position{1, 1}, tokenError, "float cannot end with a dot"},
}) })
} }
func TestLexNumberWithoutDigit(t *testing.T) { func TestLexNumberWithoutDigit(t *testing.T) {
testQLFlow(t, "+", []token{ testQLFlow(t, "+", []token{
{Position{1, 1}, tokenError, "no digit in that number"}, {toml.Position{1, 1}, tokenError, "no digit in that number"},
}) })
} }
func TestLexUnknown(t *testing.T) { func TestLexUnknown(t *testing.T) {
testQLFlow(t, "^", []token{ testQLFlow(t, "^", []token{
{Position{1, 1}, tokenError, "unexpected char: '94'"}, {toml.Position{1, 1}, tokenError, "unexpected char: '94'"},
}) })
} }
match.go → query/match.go
+52 -54
View File
@@ -1,27 +1,10 @@
package toml package query
import ( import (
"fmt" "fmt"
"github.com/pelletier/go-toml"
) )
// support function to set positions for tomlValues
// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
// values returned by the query engines
func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
switch castNode := node.(type) {
case *tomlValue:
ctx.lastPosition = castNode.position
return castNode.value
case []*TomlTree:
if len(castNode) > 0 {
ctx.lastPosition = castNode[0].position
}
return node
default:
return node
}
}
// base match // base match
type matchBase struct { type matchBase struct {
next pathFn next pathFn
@@ -45,15 +28,7 @@ func (f *terminatingFn) setNext(next pathFn) {
} }
func (f *terminatingFn) call(node interface{}, ctx *queryContext) { func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
switch castNode := node.(type) { ctx.result.appendResult(node, ctx.lastPosition)
case *TomlTree:
ctx.result.appendResult(node, castNode.position)
case *tomlValue:
ctx.result.appendResult(node, castNode.position)
default:
// use last position for scalars
ctx.result.appendResult(node, ctx.lastPosition)
}
} }
// match single key // match single key
@@ -67,16 +42,18 @@ func newMatchKeyFn(name string) *matchKeyFn {
} }
func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
if array, ok := node.([]*TomlTree); ok { if array, ok := node.([]*toml.TomlTree); ok {
for _, tree := range array { for _, tree := range array {
item := tree.values[f.Name] item := tree.Get(f.Name)
if item != nil { if item != nil {
ctx.lastPosition = tree.GetPosition(f.Name)
f.next.call(item, ctx) f.next.call(item, ctx)
} }
} }
} else if tree, ok := node.(*TomlTree); ok { } else if tree, ok := node.(*toml.TomlTree); ok {
item := tree.values[f.Name] item := tree.Get(f.Name)
if item != nil { if item != nil {
ctx.lastPosition = tree.GetPosition(f.Name)
f.next.call(item, ctx) f.next.call(item, ctx)
} }
} }
@@ -93,8 +70,13 @@ func newMatchIndexFn(idx int) *matchIndexFn {
} }
func (f *matchIndexFn) call(node interface{}, ctx *queryContext) { func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { if arr, ok := node.([]interface{}); ok {
if f.Idx < len(arr) && f.Idx >= 0 { if f.Idx < len(arr) && f.Idx >= 0 {
if treesArray, ok := node.([]*toml.TomlTree); ok {
if len(treesArray) > 0 {
ctx.lastPosition = treesArray[0].Position()
}
}
f.next.call(arr[f.Idx], ctx) f.next.call(arr[f.Idx], ctx)
} }
} }
@@ -111,7 +93,7 @@ func newMatchSliceFn(start, end, step int) *matchSliceFn {
} }
func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { if arr, ok := node.([]interface{}); ok {
// adjust indexes for negative values, reverse ordering // adjust indexes for negative values, reverse ordering
realStart, realEnd := f.Start, f.End realStart, realEnd := f.Start, f.End
if realStart < 0 { if realStart < 0 {
@@ -125,6 +107,11 @@ func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
} }
// loop and gather // loop and gather
for idx := realStart; idx < realEnd; idx += f.Step { for idx := realStart; idx < realEnd; idx += f.Step {
if treesArray, ok := node.([]*toml.TomlTree); ok {
if len(treesArray) > 0 {
ctx.lastPosition = treesArray[0].Position()
}
}
f.next.call(arr[idx], ctx) f.next.call(arr[idx], ctx)
} }
} }
@@ -140,8 +127,10 @@ func newMatchAnyFn() *matchAnyFn {
} }
func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
if tree, ok := node.(*TomlTree); ok { if tree, ok := node.(*toml.TomlTree); ok {
for _, v := range tree.values { for _, k := range tree.Keys() {
v := tree.Get(k)
ctx.lastPosition = tree.GetPosition(k)
f.next.call(v, ctx) f.next.call(v, ctx)
} }
} }
@@ -174,21 +163,25 @@ func newMatchRecursiveFn() *matchRecursiveFn {
} }
func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
if tree, ok := node.(*TomlTree); ok { originalPosition := ctx.lastPosition
var visit func(tree *TomlTree) if tree, ok := node.(*toml.TomlTree); ok {
visit = func(tree *TomlTree) { var visit func(tree *toml.TomlTree)
for _, v := range tree.values { visit = func(tree *toml.TomlTree) {
for _, k := range tree.Keys() {
v := tree.Get(k)
ctx.lastPosition = tree.GetPosition(k)
f.next.call(v, ctx) f.next.call(v, ctx)
switch node := v.(type) { switch node := v.(type) {
case *TomlTree: case *toml.TomlTree:
visit(node) visit(node)
case []*TomlTree: case []*toml.TomlTree:
for _, subtree := range node { for _, subtree := range node {
visit(subtree) visit(subtree)
} }
} }
} }
} }
ctx.lastPosition = originalPosition
f.next.call(tree, ctx) f.next.call(tree, ctx)
visit(tree) visit(tree)
} }
@@ -197,11 +190,11 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
// match based on an externally provided functional filter // match based on an externally provided functional filter
type matchFilterFn struct { type matchFilterFn struct {
matchBase matchBase
Pos Position Pos toml.Position
Name string Name string
} }
func newMatchFilterFn(name string, pos Position) *matchFilterFn { func newMatchFilterFn(name string, pos toml.Position) *matchFilterFn {
return &matchFilterFn{Name: name, Pos: pos} return &matchFilterFn{Name: name, Pos: pos}
} }
@@ -211,17 +204,22 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
panic(fmt.Sprintf("%s: query context does not have filter '%s'", panic(fmt.Sprintf("%s: query context does not have filter '%s'",
f.Pos.String(), f.Name)) f.Pos.String(), f.Name))
} }
switch castNode := tomlValueCheck(node, ctx).(type) { switch castNode := node.(type) {
case *TomlTree: case *toml.TomlTree:
for _, v := range castNode.values { for _, k := range castNode.Keys() {
if tv, ok := v.(*tomlValue); ok { v := castNode.Get(k)
if fn(tv.value) { if fn(v) {
f.next.call(v, ctx) ctx.lastPosition = castNode.GetPosition(k)
} f.next.call(v, ctx)
} else { }
if fn(v) { }
f.next.call(v, ctx) case []*toml.TomlTree:
for _, v := range castNode {
if fn(v) {
if len(castNode) > 0 {
ctx.lastPosition = castNode[0].Position()
} }
f.next.call(v, ctx)
} }
} }
case []interface{}: case []interface{}:
match_test.go → query/match_test.go
+4 -3
View File
@@ -1,8 +1,9 @@
package toml package query
import ( import (
"fmt" "fmt"
"testing" "testing"
"github.com/pelletier/go-toml"
) )
// dump path tree to a string // dump path tree to a string
@@ -194,8 +195,8 @@ func TestPathFilterExpr(t *testing.T) {
"$[?('foo'),?(bar)]", "$[?('foo'),?(bar)]",
buildPath( buildPath(
&matchUnionFn{[]pathFn{ &matchUnionFn{[]pathFn{
newMatchFilterFn("foo", Position{}), newMatchFilterFn("foo", toml.Position{}),
newMatchFilterFn("bar", Position{}), newMatchFilterFn("bar", toml.Position{}),
}}, }},
)) ))
} }
queryparser.go → query/parser.go
+1 -1
View File
@@ -5,7 +5,7 @@
https://code.google.com/p/json-path/ https://code.google.com/p/json-path/
*/ */
package toml package query
import ( import (
"fmt" "fmt"
queryparser_test.go → query/parser_test.go
+64 -65
View File
@@ -1,4 +1,4 @@
package toml package query
import ( import (
"fmt" "fmt"
@@ -7,19 +7,18 @@ import (
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/pelletier/go-toml"
) )
type queryTestNode struct { type queryTestNode struct {
value interface{} value interface{}
position Position position toml.Position
} }
func valueString(root interface{}) string { func valueString(root interface{}) string {
result := "" //fmt.Sprintf("%T:", root) result := "" //fmt.Sprintf("%T:", root)
switch node := root.(type) { switch node := root.(type) {
case *tomlValue: case *Result:
return valueString(node.value)
case *QueryResult:
items := []string{} items := []string{}
for i, v := range node.Values() { for i, v := range node.Values() {
items = append(items, fmt.Sprintf("%s:%s", items = append(items, fmt.Sprintf("%s:%s",
@@ -37,7 +36,7 @@ func valueString(root interface{}) string {
} }
sort.Strings(items) sort.Strings(items)
result = "[" + strings.Join(items, ", ") + "]" result = "[" + strings.Join(items, ", ") + "]"
case *TomlTree: case *toml.TomlTree:
// workaround for unreliable map key ordering // workaround for unreliable map key ordering
items := []string{} items := []string{}
for _, k := range node.Keys() { for _, k := range node.Keys() {
@@ -78,13 +77,13 @@ func assertValue(t *testing.T, result, ref interface{}) {
} }
} }
func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) { func assertQueryPositions(t *testing.T, tomlDoc string, query string, ref []interface{}) {
tree, err := Load(toml) tree, err := toml.Load(tomlDoc)
if err != nil { if err != nil {
t.Errorf("Non-nil toml parse error: %v", err) t.Errorf("Non-nil toml parse error: %v", err)
return return
} }
q, err := CompileQuery(query) q, err := Compile(query)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@@ -101,7 +100,7 @@ func TestQueryRoot(t *testing.T) {
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(42), "a": int64(42),
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
}) })
} }
@@ -112,7 +111,7 @@ func TestQueryKey(t *testing.T) {
"$.foo.a", "$.foo.a",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(42), Position{2, 1}, int64(42), toml.Position{2, 1},
}, },
}) })
} }
@@ -123,7 +122,7 @@ func TestQueryKeyString(t *testing.T) {
"$.foo['a']", "$.foo['a']",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(42), Position{2, 1}, int64(42), toml.Position{2, 1},
}, },
}) })
} }
@@ -134,7 +133,7 @@ func TestQueryIndex(t *testing.T) {
"$.foo.a[5]", "$.foo.a[5]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(6), Position{2, 1}, int64(6), toml.Position{2, 1},
}, },
}) })
} }
@@ -145,19 +144,19 @@ func TestQuerySliceRange(t *testing.T) {
"$.foo.a[0:5]", "$.foo.a[0:5]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(1), Position{2, 1}, int64(1), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(2), Position{2, 1}, int64(2), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(3), Position{2, 1}, int64(3), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(4), Position{2, 1}, int64(4), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(5), Position{2, 1}, int64(5), toml.Position{2, 1},
}, },
}) })
} }
@@ -168,13 +167,13 @@ func TestQuerySliceStep(t *testing.T) {
"$.foo.a[0:5:2]", "$.foo.a[0:5:2]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(1), Position{2, 1}, int64(1), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(3), Position{2, 1}, int64(3), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(5), Position{2, 1}, int64(5), toml.Position{2, 1},
}, },
}) })
} }
@@ -188,13 +187,13 @@ func TestQueryAny(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(3), "a": int64(3),
"b": int64(4), "b": int64(4),
}, Position{4, 1}, }, toml.Position{4, 1},
}, },
}) })
} }
@@ -207,19 +206,19 @@ func TestQueryUnionSimple(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(3), "a": int64(3),
"b": int64(4), "b": int64(4),
}, Position{4, 1}, }, toml.Position{4, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(5), "a": int64(5),
"b": int64(6), "b": int64(6),
}, Position{7, 1}, }, toml.Position{7, 1},
}, },
}) })
} }
@@ -249,7 +248,7 @@ func TestQueryRecursionAll(t *testing.T) {
"b": int64(6), "b": int64(6),
}, },
}, },
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -257,19 +256,19 @@ func TestQueryRecursionAll(t *testing.T) {
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, },
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
int64(1), Position{2, 1}, int64(1), toml.Position{2, 1},
}, },
queryTestNode{ queryTestNode{
int64(2), Position{3, 1}, int64(2), toml.Position{3, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -277,19 +276,19 @@ func TestQueryRecursionAll(t *testing.T) {
"a": int64(3), "a": int64(3),
"b": int64(4), "b": int64(4),
}, },
}, Position{4, 1}, }, toml.Position{4, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(3), "a": int64(3),
"b": int64(4), "b": int64(4),
}, Position{4, 1}, }, toml.Position{4, 1},
}, },
queryTestNode{ queryTestNode{
int64(3), Position{5, 1}, int64(3), toml.Position{5, 1},
}, },
queryTestNode{ queryTestNode{
int64(4), Position{6, 1}, int64(4), toml.Position{6, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -297,19 +296,19 @@ func TestQueryRecursionAll(t *testing.T) {
"a": int64(5), "a": int64(5),
"b": int64(6), "b": int64(6),
}, },
}, Position{7, 1}, }, toml.Position{7, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(5), "a": int64(5),
"b": int64(6), "b": int64(6),
}, Position{7, 1}, }, toml.Position{7, 1},
}, },
queryTestNode{ queryTestNode{
int64(5), Position{8, 1}, int64(5), toml.Position{8, 1},
}, },
queryTestNode{ queryTestNode{
int64(6), Position{9, 1}, int64(6), toml.Position{9, 1},
}, },
}) })
} }
@@ -325,31 +324,31 @@ func TestQueryRecursionUnionSimple(t *testing.T) {
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, },
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(3), "a": int64(3),
"b": int64(4), "b": int64(4),
}, Position{4, 1}, }, toml.Position{4, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(1), "a": int64(1),
"b": int64(2), "b": int64(2),
}, Position{1, 1}, }, toml.Position{1, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"a": int64(5), "a": int64(5),
"b": int64(6), "b": int64(6),
}, Position{7, 1}, }, toml.Position{7, 1},
}, },
}) })
} }
func TestQueryFilterFn(t *testing.T) { func TestQueryFilterFn(t *testing.T) {
buff, err := ioutil.ReadFile("example.toml") buff, err := ioutil.ReadFile("../example.toml")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@@ -359,16 +358,16 @@ func TestQueryFilterFn(t *testing.T) {
"$..[?(int)]", "$..[?(int)]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
int64(8001), Position{13, 1}, int64(8001), toml.Position{13, 1},
}, },
queryTestNode{ queryTestNode{
int64(8001), Position{13, 1}, int64(8001), toml.Position{13, 1},
}, },
queryTestNode{ queryTestNode{
int64(8002), Position{13, 1}, int64(8002), toml.Position{13, 1},
}, },
queryTestNode{ queryTestNode{
int64(5000), Position{14, 1}, int64(5000), toml.Position{14, 1},
}, },
}) })
@@ -376,32 +375,32 @@ func TestQueryFilterFn(t *testing.T) {
"$..[?(string)]", "$..[?(string)]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
"TOML Example", Position{3, 1}, "TOML Example", toml.Position{3, 1},
}, },
queryTestNode{ queryTestNode{
"Tom Preston-Werner", Position{6, 1}, "Tom Preston-Werner", toml.Position{6, 1},
}, },
queryTestNode{ queryTestNode{
"GitHub", Position{7, 1}, "GitHub", toml.Position{7, 1},
}, },
queryTestNode{ queryTestNode{
"GitHub Cofounder & CEO\nLikes tater tots and beer.", "GitHub Cofounder & CEO\nLikes tater tots and beer.",
Position{8, 1}, toml.Position{8, 1},
}, },
queryTestNode{ queryTestNode{
"192.168.1.1", Position{12, 1}, "192.168.1.1", toml.Position{12, 1},
}, },
queryTestNode{ queryTestNode{
"10.0.0.1", Position{21, 3}, "10.0.0.1", toml.Position{21, 3},
}, },
queryTestNode{ queryTestNode{
"eqdc10", Position{22, 3}, "eqdc10", toml.Position{22, 3},
}, },
queryTestNode{ queryTestNode{
"10.0.0.2", Position{25, 3}, "10.0.0.2", toml.Position{25, 3},
}, },
queryTestNode{ queryTestNode{
"eqdc10", Position{26, 3}, "eqdc10", toml.Position{26, 3},
}, },
}) })
@@ -421,7 +420,7 @@ func TestQueryFilterFn(t *testing.T) {
"organization": "GitHub", "organization": "GitHub",
"bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.",
"dob": tv, "dob": tv,
}, Position{5, 1}, }, toml.Position{5, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -429,7 +428,7 @@ func TestQueryFilterFn(t *testing.T) {
"ports": []interface{}{int64(8001), int64(8001), int64(8002)}, "ports": []interface{}{int64(8001), int64(8001), int64(8002)},
"connection_max": int64(5000), "connection_max": int64(5000),
"enabled": true, "enabled": true,
}, Position{11, 1}, }, toml.Position{11, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -441,19 +440,19 @@ func TestQueryFilterFn(t *testing.T) {
"ip": "10.0.0.2", "ip": "10.0.0.2",
"dc": "eqdc10", "dc": "eqdc10",
}, },
}, Position{17, 1}, }, toml.Position{17, 1},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"ip": "10.0.0.1", "ip": "10.0.0.1",
"dc": "eqdc10", "dc": "eqdc10",
}, Position{20, 3}, }, toml.Position{20, 3},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
"ip": "10.0.0.2", "ip": "10.0.0.2",
"dc": "eqdc10", "dc": "eqdc10",
}, Position{24, 3}, }, toml.Position{24, 3},
}, },
queryTestNode{ queryTestNode{
map[string]interface{}{ map[string]interface{}{
@@ -461,7 +460,7 @@ func TestQueryFilterFn(t *testing.T) {
[]interface{}{"gamma", "delta"}, []interface{}{"gamma", "delta"},
[]interface{}{int64(1), int64(2)}, []interface{}{int64(1), int64(2)},
}, },
}, Position{28, 1}, }, toml.Position{28, 1},
}, },
}) })
@@ -469,7 +468,7 @@ func TestQueryFilterFn(t *testing.T) {
"$..[?(time)]", "$..[?(time)]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
tv, Position{9, 1}, tv, toml.Position{9, 1},
}, },
}) })
@@ -477,7 +476,7 @@ func TestQueryFilterFn(t *testing.T) {
"$..[?(bool)]", "$..[?(bool)]",
[]interface{}{ []interface{}{
queryTestNode{ queryTestNode{
true, Position{15, 1}, true, toml.Position{15, 1},
}, },
}) })
} }
query.go → query/query.go
+33 -28
View File
@@ -1,7 +1,9 @@
package toml package query
import ( import (
"time" "time"
"github.com/pelletier/go-toml"
) )
// NodeFilterFn represents a user-defined filter function, for use with // NodeFilterFn represents a user-defined filter function, for use with
@@ -15,50 +17,43 @@ import (
// to use from multiple goroutines. // to use from multiple goroutines.
type NodeFilterFn func(node interface{}) bool type NodeFilterFn func(node interface{}) bool
// QueryResult is the result of Executing a Query. // Result is the result of Executing a Query.
type QueryResult struct { type Result struct {
items []interface{} items []interface{}
positions []Position positions []toml.Position
} }
// appends a value/position pair to the result set. // appends a value/position pair to the result set.
func (r *QueryResult) appendResult(node interface{}, pos Position) { func (r *Result) appendResult(node interface{}, pos toml.Position) {
r.items = append(r.items, node) r.items = append(r.items, node)
r.positions = append(r.positions, pos) r.positions = append(r.positions, pos)
} }
// Values is a set of values within a QueryResult. The order of values is not // Values is a set of values within a Result. The order of values is not
// guaranteed to be in document order, and may be different each time a query is // guaranteed to be in document order, and may be different each time a query is
// executed. // executed.
func (r QueryResult) Values() []interface{} { func (r Result) Values() []interface{} {
values := make([]interface{}, len(r.items)) return r.items
for i, v := range r.items {
o, ok := v.(*tomlValue)
if ok {
values[i] = o.value
} else {
values[i] = v
}
}
return values
} }
// Positions is a set of positions for values within a QueryResult. Each index // Positions is a set of positions for values within a Result. Each index
// in Positions() corresponds to the entry in Value() of the same index. // in Positions() corresponds to the entry in Value() of the same index.
func (r QueryResult) Positions() []Position { func (r Result) Positions() []toml.Position {
return r.positions return r.positions
} }
// runtime context for executing query paths // runtime context for executing query paths
type queryContext struct { type queryContext struct {
result *QueryResult result *Result
filters *map[string]NodeFilterFn filters *map[string]NodeFilterFn
lastPosition Position lastPosition toml.Position
} }
// generic path functor interface // generic path functor interface
type pathFn interface { type pathFn interface {
setNext(next pathFn) setNext(next pathFn)
// it is the caller's responsibility to set the ctx.lastPosition before invoking call()
// node can be one of: *toml.TomlTree, []*toml.TomlTree, or a scalar
call(node interface{}, ctx *queryContext) call(node interface{}, ctx *queryContext)
} }
@@ -88,17 +83,17 @@ func (q *Query) appendPath(next pathFn) {
next.setNext(newTerminatingFn()) // init the next functor next.setNext(newTerminatingFn()) // init the next functor
} }
// CompileQuery compiles a TOML path expression. The returned Query can be used // Compile compiles a TOML path expression. The returned Query can be used
// to match elements within a TomlTree and its descendants. // to match elements within a TomlTree and its descendants. See Execute.
func CompileQuery(path string) (*Query, error) { func Compile(path string) (*Query, error) {
return parseQuery(lexQuery(path)) return parseQuery(lexQuery(path))
} }
// Execute executes a query against a TomlTree, and returns the result of the query. // Execute executes a query against a TomlTree, and returns the result of the query.
func (q *Query) Execute(tree *TomlTree) *QueryResult { func (q *Query) Execute(tree *toml.TomlTree) *Result {
result := &QueryResult{ result := &Result{
items: []interface{}{}, items: []interface{}{},
positions: []Position{}, positions: []toml.Position{},
} }
if q.root == nil { if q.root == nil {
result.appendResult(tree, tree.GetPosition("")) result.appendResult(tree, tree.GetPosition(""))
@@ -107,11 +102,21 @@ func (q *Query) Execute(tree *TomlTree) *QueryResult {
result: result, result: result,
filters: q.filters, filters: q.filters,
} }
ctx.lastPosition = tree.Position()
q.root.call(tree, ctx) q.root.call(tree, ctx)
} }
return result return result
} }
// CompileAndExecute is a shorthand for Compile(path) followed by Execute(tree).
func CompileAndExecute(path string, tree *toml.TomlTree) (*Result, error) {
query, err := Compile(path)
if err != nil {
return nil, err
}
return query.Execute(tree), nil
}
// SetFilter sets a user-defined filter function. These may be used inside // SetFilter sets a user-defined filter function. These may be used inside
// "?(..)" query expressions to filter TOML document elements within a query. // "?(..)" query expressions to filter TOML document elements within a query.
func (q *Query) SetFilter(name string, fn NodeFilterFn) { func (q *Query) SetFilter(name string, fn NodeFilterFn) {
@@ -127,7 +132,7 @@ func (q *Query) SetFilter(name string, fn NodeFilterFn) {
var defaultFilterFunctions = map[string]NodeFilterFn{ var defaultFilterFunctions = map[string]NodeFilterFn{
"tree": func(node interface{}) bool { "tree": func(node interface{}) bool {
_, ok := node.(*TomlTree) _, ok := node.(*toml.TomlTree)
return ok return ok
}, },
"int": func(node interface{}) bool { "int": func(node interface{}) bool {
query/query_test.go
+157
View File
@@ -0,0 +1,157 @@
package query
import (
"fmt"
"testing"
"github.com/pelletier/go-toml"
)
func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
if len(array) != len(objects) {
t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
}
for _, o := range objects {
found := false
for _, a := range array {
if a == o {
found = true
break
}
}
if !found {
t.Fatal(o, "not found in array", array)
}
}
}
func TestQueryExample(t *testing.T) {
config, _ := toml.Load(`
[[book]]
title = "The Stand"
author = "Stephen King"
[[book]]
title = "For Whom the Bell Tolls"
author = "Ernest Hemmingway"
[[book]]
title = "Neuromancer"
author = "William Gibson"
`)
authors, err := CompileAndExecute("$.book.author", config)
if err != nil {
t.Fatal("unexpected error:", err)
}
names := authors.Values()
if len(names) != 3 {
t.Fatalf("query should return 3 names but returned %d", len(names))
}
assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
}
func TestQueryReadmeExample(t *testing.T) {
config, _ := toml.Load(`
[postgres]
user = "pelletier"
password = "mypassword"
`)
query, err := Compile("$..[user,password]")
if err != nil {
t.Fatal("unexpected error:", err)
}
results := query.Execute(config)
values := results.Values()
if len(values) != 2 {
t.Fatalf("query should return 2 values but returned %d", len(values))
}
assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
}
func TestQueryPathNotPresent(t *testing.T) {
config, _ := toml.Load(`a = "hello"`)
query, err := Compile("$.foo.bar")
if err != nil {
t.Fatal("unexpected error:", err)
}
results := query.Execute(config)
if err != nil {
t.Fatalf("err should be nil. got %s instead", err)
}
if len(results.items) != 0 {
t.Fatalf("no items should be matched. %d matched instead", len(results.items))
}
}
func ExampleNodeFilterFn_filterExample() {
tree, _ := toml.Load(`
[struct_one]
foo = "foo"
bar = "bar"
[struct_two]
baz = "baz"
gorf = "gorf"
`)
// create a query that references a user-defined-filter
query, _ := Compile("$[?(bazOnly)]")
// define the filter, and assign it to the query
query.SetFilter("bazOnly", func(node interface{}) bool {
if tree, ok := node.(*toml.TomlTree); ok {
return tree.Has("baz")
}
return false // reject all other node types
})
// results contain only the 'struct_two' TomlTree
query.Execute(tree)
}
func ExampleQuery_queryExample() {
config, _ := toml.Load(`
[[book]]
title = "The Stand"
author = "Stephen King"
[[book]]
title = "For Whom the Bell Tolls"
author = "Ernest Hemmingway"
[[book]]
title = "Neuromancer"
author = "William Gibson"
`)
// find and print all the authors in the document
query, _ := Compile("$.book.author")
authors := query.Execute(config)
for _, name := range authors.Values() {
fmt.Println(name)
}
}
func TestTomlQuery(t *testing.T) {
tree, err := toml.Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
if err != nil {
t.Error(err)
return
}
query, err := Compile("$.foo.bar")
if err != nil {
t.Error(err)
return
}
result := query.Execute(tree)
values := result.Values()
if len(values) != 1 {
t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
}
if tt, ok := values[0].(*toml.TomlTree); !ok {
t.Errorf("Expected type of TomlTree: %T", values[0])
} else if tt.Get("a") != int64(1) {
t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
} else if tt.Get("b") != int64(2) {
t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
}
}
query/tokens.go
+107
View File
@@ -0,0 +1,107 @@
package query
import (
"fmt"
"strconv"
"unicode"
"github.com/pelletier/go-toml"
)
// Define tokens
type tokenType int
const (
eof = -(iota + 1)
)
const (
tokenError tokenType = iota
tokenEOF
tokenKey
tokenString
tokenInteger
tokenFloat
tokenLeftBracket
tokenRightBracket
tokenLeftParen
tokenRightParen
tokenComma
tokenColon
tokenDollar
tokenStar
tokenQuestion
tokenDot
tokenDotDot
)
var tokenTypeNames = []string{
"Error",
"EOF",
"Key",
"String",
"Integer",
"Float",
"[",
"]",
"(",
")",
",",
":",
"$",
"*",
"?",
".",
"..",
}
type token struct {
toml.Position
typ tokenType
val string
}
func (tt tokenType) String() string {
idx := int(tt)
if idx < len(tokenTypeNames) {
return tokenTypeNames[idx]
}
return "Unknown"
}
func (t token) Int() int {
if result, err := strconv.Atoi(t.val); err != nil {
panic(err)
} else {
return result
}
}
func (t token) String() string {
switch t.typ {
case tokenEOF:
return "EOF"
case tokenError:
return t.val
}
return fmt.Sprintf("%q", t.val)
}
func isSpace(r rune) bool {
return r == ' ' || r == '\t'
}
func isAlphanumeric(r rune) bool {
return unicode.IsLetter(r) || r == '_'
}
func isDigit(r rune) bool {
return unicode.IsNumber(r)
}
func isHexDigit(r rune) bool {
return isDigit(r) ||
(r >= 'a' && r <= 'f') ||
(r >= 'A' && r <= 'F')
}
query_test.go
-70
View File
@@ -1,70 +0,0 @@
package toml
import (
"testing"
)
func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
if len(array) != len(objects) {
t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
}
for _, o := range objects {
found := false
for _, a := range array {
if a == o {
found = true
break
}
}
if !found {
t.Fatal(o, "not found in array", array)
}
}
}
func TestQueryExample(t *testing.T) {
config, _ := Load(`
[[book]]
title = "The Stand"
author = "Stephen King"
[[book]]
title = "For Whom the Bell Tolls"
author = "Ernest Hemmingway"
[[book]]
title = "Neuromancer"
author = "William Gibson"
`)
authors, _ := config.Query("$.book.author")
names := authors.Values()
if len(names) != 3 {
t.Fatalf("query should return 3 names but returned %d", len(names))
}
assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
}
func TestQueryReadmeExample(t *testing.T) {
config, _ := Load(`
[postgres]
user = "pelletier"
password = "mypassword"
`)
results, _ := config.Query("$..[user,password]")
values := results.Values()
if len(values) != 2 {
t.Fatalf("query should return 2 values but returned %d", len(values))
}
assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
}
func TestQueryPathNotPresent(t *testing.T) {
config, _ := Load(`a = "hello"`)
results, err := config.Query("$.foo.bar")
if err != nil {
t.Fatalf("err should be nil. got %s instead", err)
}
if len(results.items) != 0 {
t.Fatalf("no items should be matched. %d matched instead", len(results.items))
}
}
test.sh
+7 -1
View File
@@ -19,6 +19,9 @@ function git_clone() {
popd popd
} }
# Remove potential previous runs
rm -rf src test_program_bin toml-test
# Run go vet # Run go vet
go vet ./... go vet ./...
@@ -36,13 +39,16 @@ go build -o toml-test github.com/BurntSushi/toml-test
# vendorize the current lib for testing # vendorize the current lib for testing
# NOTE: this basically mocks an install without having to go back out to github for code # NOTE: this basically mocks an install without having to go back out to github for code
mkdir -p src/github.com/pelletier/go-toml/cmd mkdir -p src/github.com/pelletier/go-toml/cmd
mkdir -p src/github.com/pelletier/go-toml/query
cp *.go *.toml src/github.com/pelletier/go-toml cp *.go *.toml src/github.com/pelletier/go-toml
cp -R cmd/* src/github.com/pelletier/go-toml/cmd cp -R cmd/* src/github.com/pelletier/go-toml/cmd
cp -R query/* src/github.com/pelletier/go-toml/query
go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
# Run basic unit tests # Run basic unit tests
go test github.com/pelletier/go-toml -v -covermode=count -coverprofile=coverage.out go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out
go test github.com/pelletier/go-toml/cmd/tomljson go test github.com/pelletier/go-toml/cmd/tomljson
go test github.com/pelletier/go-toml/query
# run the entire BurntSushi test suite # run the entire BurntSushi test suite
if [[ $# -eq 0 ]] ; then if [[ $# -eq 0 ]] ; then
toml.go
+5 -9
View File
@@ -36,6 +36,11 @@ func TreeFromMap(m map[string]interface{}) (*TomlTree, error) {
return result.(*TomlTree), nil return result.(*TomlTree), nil
} }
// Position returns the position of the tree.
func (t *TomlTree) Position() Position {
return t.position
}
// Has returns a boolean indicating if the given key exists. // Has returns a boolean indicating if the given key exists.
func (t *TomlTree) Has(key string) bool { func (t *TomlTree) Has(key string) bool {
if key == "" { if key == "" {
@@ -247,15 +252,6 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error {
return nil return nil
} }
// Query compiles and executes a query on a tree and returns the query result.
func (t *TomlTree) Query(query string) (*QueryResult, error) {
q, err := CompileQuery(query)
if err != nil {
return nil, err
}
return q.Execute(t), nil
}
// LoadReader creates a TomlTree from any io.Reader. // LoadReader creates a TomlTree from any io.Reader.
func LoadReader(reader io.Reader) (tree *TomlTree, err error) { func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
defer func() { defer func() {
toml_test.go
-25
View File
@@ -94,31 +94,6 @@ func TestTomlGetPath(t *testing.T) {
} }
} }
func TestTomlQuery(t *testing.T) {
tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
if err != nil {
t.Error(err)
return
}
result, err := tree.Query("$.foo.bar")
if err != nil {
t.Error(err)
return
}
values := result.Values()
if len(values) != 1 {
t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
}
if tt, ok := values[0].(*TomlTree); !ok {
t.Errorf("Expected type of TomlTree: %T", values[0])
} else if tt.Get("a") != int64(1) {
t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
} else if tt.Get("b") != int64(2) {
t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
}
}
func TestTomlFromMap(t *testing.T) { func TestTomlFromMap(t *testing.T) {
simpleMap := map[string]interface{}{"hello": 42} simpleMap := map[string]interface{}{"hello": 42}
tree, err := TreeFromMap(simpleMap) tree, err := TreeFromMap(simpleMap)