diff --git a/README.md b/README.md index b8137e0..d622596 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Go-toml provides the following features for using data parsed from TOML document * Load TOML documents from files and string data * Easily navigate TOML structure using TomlTree * Line & column position data for all parsed elements -* Query support similar to JSON-Path +* [Query support similar to JSON-Path](query/) * Syntax errors contain line and column numbers Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing: diff --git a/doc.go b/doc.go index 9156b73..395d4f4 100644 --- a/doc.go +++ b/doc.go @@ -75,176 +75,10 @@ // return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition("")) // } // -// Query Support +// JSONPath-like queries // -// The TOML query path implementation is based loosely on the JSONPath specification: -// http://goessner.net/articles/JsonPath/ -// -// The idea behind a query path is to allow quick access to any element, or set -// of elements within TOML document, with a single expression. -// -// result, err := tree.Query("$.foo.bar.baz") -// -// This is roughly equivalent to: -// -// next := tree.Get("foo") -// if next != nil { -// next = next.Get("bar") -// if next != nil { -// next = next.Get("baz") -// } -// } -// result := next -// -// err is nil if any parsing exception occurs. -// -// If no node in the tree matches the query, result will simply contain an empty list of -// items. -// -// As illustrated above, the query path is much more efficient, especially since -// the structure of the TOML file can vary. Rather than making assumptions about -// a document's structure, a query allows the programmer to make structured -// requests into the document, and get zero or more values as a result. -// -// The syntax of a query begins with a root token, followed by any number -// sub-expressions: -// -// $ -// Root of the TOML tree. This must always come first. -// .name -// Selects child of this node, where 'name' is a TOML key -// name. -// ['name'] -// Selects child of this node, where 'name' is a string -// containing a TOML key name. -// [index] -// Selcts child array element at 'index'. -// ..expr -// Recursively selects all children, filtered by an a union, -// index, or slice expression. -// ..* -// Recursive selection of all nodes at this point in the -// tree. -// .* -// Selects all children of the current node. -// [expr,expr] -// Union operator - a logical 'or' grouping of two or more -// sub-expressions: index, key name, or filter. -// [start:end:step] -// Slice operator - selects array elements from start to -// end-1, at the given step. All three arguments are -// optional. -// [?(filter)] -// Named filter expression - the function 'filter' is -// used to filter children at this node. -// -// Query Indexes And Slices -// -// Index expressions perform no bounds checking, and will contribute no -// values to the result set if the provided index or index range is invalid. -// Negative indexes represent values from the end of the array, counting backwards. -// -// // select the last index of the array named 'foo' -// tree.Query("$.foo[-1]") -// -// Slice expressions are supported, by using ':' to separate a start/end index pair. -// -// // select up to the first five elements in the array -// tree.Query("$.foo[0:5]") -// -// Slice expressions also allow negative indexes for the start and stop -// arguments. -// -// // select all array elements. -// tree.Query("$.foo[0:-1]") -// -// Slice expressions may have an optional stride/step parameter: -// -// // select every other element -// tree.Query("$.foo[0:-1:2]") -// -// Slice start and end parameters are also optional: -// -// // these are all equivalent and select all the values in the array -// tree.Query("$.foo[:]") -// tree.Query("$.foo[0:]") -// tree.Query("$.foo[:-1]") -// tree.Query("$.foo[0:-1:]") -// tree.Query("$.foo[::1]") -// tree.Query("$.foo[0::1]") -// tree.Query("$.foo[:-1:1]") -// tree.Query("$.foo[0:-1:1]") -// -// Query Filters -// -// Query filters are used within a Union [,] or single Filter [] expression. -// A filter only allows nodes that qualify through to the next expression, -// and/or into the result set. -// -// // returns children of foo that are permitted by the 'bar' filter. -// tree.Query("$.foo[?(bar)]") -// -// There are several filters provided with the library: -// -// tree -// Allows nodes of type TomlTree. -// int -// Allows nodes of type int64. -// float -// Allows nodes of type float64. -// string -// Allows nodes of type string. -// time -// Allows nodes of type time.Time. -// bool -// Allows nodes of type bool. -// -// Query Results -// -// An executed query returns a QueryResult object. This contains the nodes -// in the TOML tree that qualify the query expression. Position information -// is also available for each value in the set. -// -// // display the results of a query -// results := tree.Query("$.foo.bar.baz") -// for idx, value := results.Values() { -// fmt.Println("%v: %v", results.Positions()[idx], value) -// } -// -// Compiled Queries -// -// Queries may be executed directly on a TomlTree object, or compiled ahead -// of time and executed discretely. The former is more convienent, but has the -// penalty of having to recompile the query expression each time. -// -// // basic query -// results := tree.Query("$.foo.bar.baz") -// -// // compiled query -// query := toml.CompileQuery("$.foo.bar.baz") -// results := query.Execute(tree) -// -// // run the compiled query again on a different tree -// moreResults := query.Execute(anotherTree) -// -// User Defined Query Filters -// -// Filter expressions may also be user defined by using the SetFilter() -// function on the Query object. The function must return true/false, which -// signifies if the passed node is kept or discarded, respectively. -// -// // create a query that references a user-defined filter -// query, _ := CompileQuery("$[?(bazOnly)]") -// -// // define the filter, and assign it to the query -// query.SetFilter("bazOnly", func(node interface{}) bool{ -// if tree, ok := node.(*TomlTree); ok { -// return tree.Has("baz") -// } -// return false // reject all other node types -// }) -// -// // run the query -// query.Execute(tree) +// The package github.com/pelletier/go-toml/query implements a system +// similar to JSONPath to quickly retrive elements of a TOML document using a +// single expression. See the package documentation for more information. // package toml diff --git a/doc_test.go b/doc_test.go index 6945241..6553e4e 100644 --- a/doc_test.go +++ b/doc_test.go @@ -6,52 +6,6 @@ import ( "fmt" ) -func ExampleNodeFilterFn_filterExample() { - tree, _ := Load(` - [struct_one] - foo = "foo" - bar = "bar" - - [struct_two] - baz = "baz" - gorf = "gorf" - `) - - // create a query that references a user-defined-filter - query, _ := CompileQuery("$[?(bazOnly)]") - - // define the filter, and assign it to the query - query.SetFilter("bazOnly", func(node interface{}) bool { - if tree, ok := node.(*TomlTree); ok { - return tree.Has("baz") - } - return false // reject all other node types - }) - - // results contain only the 'struct_two' TomlTree - query.Execute(tree) -} - -func ExampleQuery_queryExample() { - config, _ := Load(` - [[book]] - title = "The Stand" - author = "Stephen King" - [[book]] - title = "For Whom the Bell Tolls" - author = "Ernest Hemmingway" - [[book]] - title = "Neuromancer" - author = "William Gibson" - `) - - // find and print all the authors in the document - authors, _ := config.Query("$.book.author") - for _, name := range authors.Values() { - fmt.Println(name) - } -} - func Example_comprehensiveExample() { config, err := LoadFile("config.toml") @@ -71,11 +25,5 @@ func Example_comprehensiveExample() { // show where elements are in the file fmt.Printf("User position: %v\n", configTree.GetPosition("user")) fmt.Printf("Password position: %v\n", configTree.GetPosition("password")) - - // use a query to gather elements without walking the tree - results, _ := config.Query("$..[user,password]") - for ii, item := range results.Values() { - fmt.Printf("Query result %d: %v\n", ii, item) - } } } diff --git a/query/doc.go b/query/doc.go new file mode 100644 index 0000000..fa2e457 --- /dev/null +++ b/query/doc.go @@ -0,0 +1,175 @@ +// Package query performs JSONPath-like queries on a TOML document. +// +// The query path implementation is based loosely on the JSONPath specification: +// http://goessner.net/articles/JsonPath/. +// +// The idea behind a query path is to allow quick access to any element, or set +// of elements within TOML document, with a single expression. +// +// result, err := query.CompileAndExecute("$.foo.bar.baz", tree) +// +// This is roughly equivalent to: +// +// next := tree.Get("foo") +// if next != nil { +// next = next.Get("bar") +// if next != nil { +// next = next.Get("baz") +// } +// } +// result := next +// +// err is nil if any parsing exception occurs. +// +// If no node in the tree matches the query, result will simply contain an empty list of +// items. +// +// As illustrated above, the query path is much more efficient, especially since +// the structure of the TOML file can vary. Rather than making assumptions about +// a document's structure, a query allows the programmer to make structured +// requests into the document, and get zero or more values as a result. +// +// Query syntax +// +// The syntax of a query begins with a root token, followed by any number +// sub-expressions: +// +// $ +// Root of the TOML tree. This must always come first. +// .name +// Selects child of this node, where 'name' is a TOML key +// name. +// ['name'] +// Selects child of this node, where 'name' is a string +// containing a TOML key name. +// [index] +// Selcts child array element at 'index'. +// ..expr +// Recursively selects all children, filtered by an a union, +// index, or slice expression. +// ..* +// Recursive selection of all nodes at this point in the +// tree. +// .* +// Selects all children of the current node. +// [expr,expr] +// Union operator - a logical 'or' grouping of two or more +// sub-expressions: index, key name, or filter. +// [start:end:step] +// Slice operator - selects array elements from start to +// end-1, at the given step. All three arguments are +// optional. +// [?(filter)] +// Named filter expression - the function 'filter' is +// used to filter children at this node. +// +// Query Indexes And Slices +// +// Index expressions perform no bounds checking, and will contribute no +// values to the result set if the provided index or index range is invalid. +// Negative indexes represent values from the end of the array, counting backwards. +// +// // select the last index of the array named 'foo' +// query.CompileAndExecute("$.foo[-1]", tree) +// +// Slice expressions are supported, by using ':' to separate a start/end index pair. +// +// // select up to the first five elements in the array +// query.CompileAndExecute("$.foo[0:5]", tree) +// +// Slice expressions also allow negative indexes for the start and stop +// arguments. +// +// // select all array elements. +// query.CompileAndExecute("$.foo[0:-1]", tree) +// +// Slice expressions may have an optional stride/step parameter: +// +// // select every other element +// query.CompileAndExecute("$.foo[0:-1:2]", tree) +// +// Slice start and end parameters are also optional: +// +// // these are all equivalent and select all the values in the array +// query.CompileAndExecute("$.foo[:]", tree) +// query.CompileAndExecute("$.foo[0:]", tree) +// query.CompileAndExecute("$.foo[:-1]", tree) +// query.CompileAndExecute("$.foo[0:-1:]", tree) +// query.CompileAndExecute("$.foo[::1]", tree) +// query.CompileAndExecute("$.foo[0::1]", tree) +// query.CompileAndExecute("$.foo[:-1:1]", tree) +// query.CompileAndExecute("$.foo[0:-1:1]", tree) +// +// Query Filters +// +// Query filters are used within a Union [,] or single Filter [] expression. +// A filter only allows nodes that qualify through to the next expression, +// and/or into the result set. +// +// // returns children of foo that are permitted by the 'bar' filter. +// query.CompileAndExecute("$.foo[?(bar)]", tree) +// +// There are several filters provided with the library: +// +// tree +// Allows nodes of type TomlTree. +// int +// Allows nodes of type int64. +// float +// Allows nodes of type float64. +// string +// Allows nodes of type string. +// time +// Allows nodes of type time.Time. +// bool +// Allows nodes of type bool. +// +// Query Results +// +// An executed query returns a Result object. This contains the nodes +// in the TOML tree that qualify the query expression. Position information +// is also available for each value in the set. +// +// // display the results of a query +// results := query.CompileAndExecute("$.foo.bar.baz", tree) +// for idx, value := results.Values() { +// fmt.Println("%v: %v", results.Positions()[idx], value) +// } +// +// Compiled Queries +// +// Queries may be executed directly on a TomlTree object, or compiled ahead +// of time and executed discretely. The former is more convienent, but has the +// penalty of having to recompile the query expression each time. +// +// // basic query +// results := query.CompileAndExecute("$.foo.bar.baz", tree) +// +// // compiled query +// query, err := toml.Compile("$.foo.bar.baz") +// results := query.Execute(tree) +// +// // run the compiled query again on a different tree +// moreResults := query.Execute(anotherTree) +// +// User Defined Query Filters +// +// Filter expressions may also be user defined by using the SetFilter() +// function on the Query object. The function must return true/false, which +// signifies if the passed node is kept or discarded, respectively. +// +// // create a query that references a user-defined filter +// query, _ := query.Compile("$[?(bazOnly)]") +// +// // define the filter, and assign it to the query +// query.SetFilter("bazOnly", func(node interface{}) bool{ +// if tree, ok := node.(*TomlTree); ok { +// return tree.Has("baz") +// } +// return false // reject all other node types +// }) +// +// // run the query +// query.Execute(tree) +// +package query diff --git a/querylexer.go b/query/lexer.go similarity index 96% rename from querylexer.go rename to query/lexer.go index 960681d..6336d52 100644 --- a/querylexer.go +++ b/query/lexer.go @@ -3,13 +3,14 @@ // Written using the principles developed by Rob Pike in // http://www.youtube.com/watch?v=HxaD_trXwRE -package toml +package query import ( "fmt" "strconv" "strings" "unicode/utf8" + "github.com/pelletier/go-toml" ) // Lexer state function @@ -54,7 +55,7 @@ func (l *queryLexer) nextStart() { func (l *queryLexer) emit(t tokenType) { l.tokens <- token{ - Position: Position{l.line, l.col}, + Position: toml.Position{Line:l.line, Col:l.col}, typ: t, val: l.input[l.start:l.pos], } @@ -63,7 +64,7 @@ func (l *queryLexer) emit(t tokenType) { func (l *queryLexer) emitWithValue(t tokenType, value string) { l.tokens <- token{ - Position: Position{l.line, l.col}, + Position: toml.Position{Line:l.line, Col:l.col}, typ: t, val: value, } @@ -91,7 +92,7 @@ func (l *queryLexer) backup() { func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn { l.tokens <- token{ - Position: Position{l.line, l.col}, + Position: toml.Position{Line:l.line, Col:l.col}, typ: tokenError, val: fmt.Sprintf(format, args...), } diff --git a/querylexer_test.go b/query/lexer_test.go similarity index 51% rename from querylexer_test.go rename to query/lexer_test.go index 2d0803f..e2b733a 100644 --- a/querylexer_test.go +++ b/query/lexer_test.go @@ -1,7 +1,8 @@ -package toml +package query import ( "testing" + "github.com/pelletier/go-toml" ) func testQLFlow(t *testing.T, input string, expectedFlow []token) { @@ -36,143 +37,143 @@ func testQLFlow(t *testing.T, input string, expectedFlow []token) { func TestLexSpecialChars(t *testing.T) { testQLFlow(t, " .$[]..()?*", []token{ - {Position{1, 2}, tokenDot, "."}, - {Position{1, 3}, tokenDollar, "$"}, - {Position{1, 4}, tokenLeftBracket, "["}, - {Position{1, 5}, tokenRightBracket, "]"}, - {Position{1, 6}, tokenDotDot, ".."}, - {Position{1, 8}, tokenLeftParen, "("}, - {Position{1, 9}, tokenRightParen, ")"}, - {Position{1, 10}, tokenQuestion, "?"}, - {Position{1, 11}, tokenStar, "*"}, - {Position{1, 12}, tokenEOF, ""}, + {toml.Position{1, 2}, tokenDot, "."}, + {toml.Position{1, 3}, tokenDollar, "$"}, + {toml.Position{1, 4}, tokenLeftBracket, "["}, + {toml.Position{1, 5}, tokenRightBracket, "]"}, + {toml.Position{1, 6}, tokenDotDot, ".."}, + {toml.Position{1, 8}, tokenLeftParen, "("}, + {toml.Position{1, 9}, tokenRightParen, ")"}, + {toml.Position{1, 10}, tokenQuestion, "?"}, + {toml.Position{1, 11}, tokenStar, "*"}, + {toml.Position{1, 12}, tokenEOF, ""}, }) } func TestLexString(t *testing.T) { testQLFlow(t, "'foo\n'", []token{ - {Position{1, 2}, tokenString, "foo\n"}, - {Position{2, 2}, tokenEOF, ""}, + {toml.Position{1, 2}, tokenString, "foo\n"}, + {toml.Position{2, 2}, tokenEOF, ""}, }) } func TestLexDoubleString(t *testing.T) { testQLFlow(t, `"bar"`, []token{ - {Position{1, 2}, tokenString, "bar"}, - {Position{1, 6}, tokenEOF, ""}, + {toml.Position{1, 2}, tokenString, "bar"}, + {toml.Position{1, 6}, tokenEOF, ""}, }) } func TestLexStringEscapes(t *testing.T) { testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{ - {Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"}, - {Position{1, 55}, tokenEOF, ""}, + {toml.Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"}, + {toml.Position{1, 55}, tokenEOF, ""}, }) } func TestLexStringUnfinishedUnicode4(t *testing.T) { testQLFlow(t, `"\u000"`, []token{ - {Position{1, 2}, tokenError, "unfinished unicode escape"}, + {toml.Position{1, 2}, tokenError, "unfinished unicode escape"}, }) } func TestLexStringUnfinishedUnicode8(t *testing.T) { testQLFlow(t, `"\U0000"`, []token{ - {Position{1, 2}, tokenError, "unfinished unicode escape"}, + {toml.Position{1, 2}, tokenError, "unfinished unicode escape"}, }) } func TestLexStringInvalidEscape(t *testing.T) { testQLFlow(t, `"\x"`, []token{ - {Position{1, 2}, tokenError, "invalid escape sequence: \\x"}, + {toml.Position{1, 2}, tokenError, "invalid escape sequence: \\x"}, }) } func TestLexStringUnfinished(t *testing.T) { testQLFlow(t, `"bar`, []token{ - {Position{1, 2}, tokenError, "unclosed string"}, + {toml.Position{1, 2}, tokenError, "unclosed string"}, }) } func TestLexKey(t *testing.T) { testQLFlow(t, "foo", []token{ - {Position{1, 1}, tokenKey, "foo"}, - {Position{1, 4}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenKey, "foo"}, + {toml.Position{1, 4}, tokenEOF, ""}, }) } func TestLexRecurse(t *testing.T) { testQLFlow(t, "$..*", []token{ - {Position{1, 1}, tokenDollar, "$"}, - {Position{1, 2}, tokenDotDot, ".."}, - {Position{1, 4}, tokenStar, "*"}, - {Position{1, 5}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenDollar, "$"}, + {toml.Position{1, 2}, tokenDotDot, ".."}, + {toml.Position{1, 4}, tokenStar, "*"}, + {toml.Position{1, 5}, tokenEOF, ""}, }) } func TestLexBracketKey(t *testing.T) { testQLFlow(t, "$[foo]", []token{ - {Position{1, 1}, tokenDollar, "$"}, - {Position{1, 2}, tokenLeftBracket, "["}, - {Position{1, 3}, tokenKey, "foo"}, - {Position{1, 6}, tokenRightBracket, "]"}, - {Position{1, 7}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenDollar, "$"}, + {toml.Position{1, 2}, tokenLeftBracket, "["}, + {toml.Position{1, 3}, tokenKey, "foo"}, + {toml.Position{1, 6}, tokenRightBracket, "]"}, + {toml.Position{1, 7}, tokenEOF, ""}, }) } func TestLexSpace(t *testing.T) { testQLFlow(t, "foo bar baz", []token{ - {Position{1, 1}, tokenKey, "foo"}, - {Position{1, 5}, tokenKey, "bar"}, - {Position{1, 9}, tokenKey, "baz"}, - {Position{1, 12}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenKey, "foo"}, + {toml.Position{1, 5}, tokenKey, "bar"}, + {toml.Position{1, 9}, tokenKey, "baz"}, + {toml.Position{1, 12}, tokenEOF, ""}, }) } func TestLexInteger(t *testing.T) { testQLFlow(t, "100 +200 -300", []token{ - {Position{1, 1}, tokenInteger, "100"}, - {Position{1, 5}, tokenInteger, "+200"}, - {Position{1, 10}, tokenInteger, "-300"}, - {Position{1, 14}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenInteger, "100"}, + {toml.Position{1, 5}, tokenInteger, "+200"}, + {toml.Position{1, 10}, tokenInteger, "-300"}, + {toml.Position{1, 14}, tokenEOF, ""}, }) } func TestLexFloat(t *testing.T) { testQLFlow(t, "100.0 +200.0 -300.0", []token{ - {Position{1, 1}, tokenFloat, "100.0"}, - {Position{1, 7}, tokenFloat, "+200.0"}, - {Position{1, 14}, tokenFloat, "-300.0"}, - {Position{1, 20}, tokenEOF, ""}, + {toml.Position{1, 1}, tokenFloat, "100.0"}, + {toml.Position{1, 7}, tokenFloat, "+200.0"}, + {toml.Position{1, 14}, tokenFloat, "-300.0"}, + {toml.Position{1, 20}, tokenEOF, ""}, }) } func TestLexFloatWithMultipleDots(t *testing.T) { testQLFlow(t, "4.2.", []token{ - {Position{1, 1}, tokenError, "cannot have two dots in one float"}, + {toml.Position{1, 1}, tokenError, "cannot have two dots in one float"}, }) } func TestLexFloatLeadingDot(t *testing.T) { testQLFlow(t, "+.1", []token{ - {Position{1, 1}, tokenError, "cannot start float with a dot"}, + {toml.Position{1, 1}, tokenError, "cannot start float with a dot"}, }) } func TestLexFloatWithTrailingDot(t *testing.T) { testQLFlow(t, "42.", []token{ - {Position{1, 1}, tokenError, "float cannot end with a dot"}, + {toml.Position{1, 1}, tokenError, "float cannot end with a dot"}, }) } func TestLexNumberWithoutDigit(t *testing.T) { testQLFlow(t, "+", []token{ - {Position{1, 1}, tokenError, "no digit in that number"}, + {toml.Position{1, 1}, tokenError, "no digit in that number"}, }) } func TestLexUnknown(t *testing.T) { testQLFlow(t, "^", []token{ - {Position{1, 1}, tokenError, "unexpected char: '94'"}, + {toml.Position{1, 1}, tokenError, "unexpected char: '94'"}, }) } diff --git a/match.go b/query/match.go similarity index 66% rename from match.go rename to query/match.go index 48b0f2a..1aac035 100644 --- a/match.go +++ b/query/match.go @@ -1,27 +1,10 @@ -package toml +package query import ( "fmt" + "github.com/pelletier/go-toml" ) -// support function to set positions for tomlValues -// NOTE: this is done to allow ctx.lastPosition to indicate the start of any -// values returned by the query engines -func tomlValueCheck(node interface{}, ctx *queryContext) interface{} { - switch castNode := node.(type) { - case *tomlValue: - ctx.lastPosition = castNode.position - return castNode.value - case []*TomlTree: - if len(castNode) > 0 { - ctx.lastPosition = castNode[0].position - } - return node - default: - return node - } -} - // base match type matchBase struct { next pathFn @@ -45,15 +28,7 @@ func (f *terminatingFn) setNext(next pathFn) { } func (f *terminatingFn) call(node interface{}, ctx *queryContext) { - switch castNode := node.(type) { - case *TomlTree: - ctx.result.appendResult(node, castNode.position) - case *tomlValue: - ctx.result.appendResult(node, castNode.position) - default: - // use last position for scalars - ctx.result.appendResult(node, ctx.lastPosition) - } + ctx.result.appendResult(node, ctx.lastPosition) } // match single key @@ -67,16 +42,18 @@ func newMatchKeyFn(name string) *matchKeyFn { } func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { - if array, ok := node.([]*TomlTree); ok { + if array, ok := node.([]*toml.TomlTree); ok { for _, tree := range array { - item := tree.values[f.Name] + item := tree.Get(f.Name) if item != nil { + ctx.lastPosition = tree.GetPosition(f.Name) f.next.call(item, ctx) } } - } else if tree, ok := node.(*TomlTree); ok { - item := tree.values[f.Name] + } else if tree, ok := node.(*toml.TomlTree); ok { + item := tree.Get(f.Name) if item != nil { + ctx.lastPosition = tree.GetPosition(f.Name) f.next.call(item, ctx) } } @@ -93,8 +70,13 @@ func newMatchIndexFn(idx int) *matchIndexFn { } func (f *matchIndexFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { + if arr, ok := node.([]interface{}); ok { if f.Idx < len(arr) && f.Idx >= 0 { + if treesArray, ok := node.([]*toml.TomlTree); ok { + if len(treesArray) > 0 { + ctx.lastPosition = treesArray[0].Position() + } + } f.next.call(arr[f.Idx], ctx) } } @@ -111,7 +93,7 @@ func newMatchSliceFn(start, end, step int) *matchSliceFn { } func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { + if arr, ok := node.([]interface{}); ok { // adjust indexes for negative values, reverse ordering realStart, realEnd := f.Start, f.End if realStart < 0 { @@ -125,6 +107,11 @@ func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { } // loop and gather for idx := realStart; idx < realEnd; idx += f.Step { + if treesArray, ok := node.([]*toml.TomlTree); ok { + if len(treesArray) > 0 { + ctx.lastPosition = treesArray[0].Position() + } + } f.next.call(arr[idx], ctx) } } @@ -140,8 +127,10 @@ func newMatchAnyFn() *matchAnyFn { } func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - for _, v := range tree.values { + if tree, ok := node.(*toml.TomlTree); ok { + for _, k := range tree.Keys() { + v := tree.Get(k) + ctx.lastPosition = tree.GetPosition(k) f.next.call(v, ctx) } } @@ -174,21 +163,25 @@ func newMatchRecursiveFn() *matchRecursiveFn { } func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - var visit func(tree *TomlTree) - visit = func(tree *TomlTree) { - for _, v := range tree.values { + originalPosition := ctx.lastPosition + if tree, ok := node.(*toml.TomlTree); ok { + var visit func(tree *toml.TomlTree) + visit = func(tree *toml.TomlTree) { + for _, k := range tree.Keys() { + v := tree.Get(k) + ctx.lastPosition = tree.GetPosition(k) f.next.call(v, ctx) switch node := v.(type) { - case *TomlTree: + case *toml.TomlTree: visit(node) - case []*TomlTree: + case []*toml.TomlTree: for _, subtree := range node { visit(subtree) } } } } + ctx.lastPosition = originalPosition f.next.call(tree, ctx) visit(tree) } @@ -197,11 +190,11 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { // match based on an externally provided functional filter type matchFilterFn struct { matchBase - Pos Position + Pos toml.Position Name string } -func newMatchFilterFn(name string, pos Position) *matchFilterFn { +func newMatchFilterFn(name string, pos toml.Position) *matchFilterFn { return &matchFilterFn{Name: name, Pos: pos} } @@ -211,17 +204,22 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) { panic(fmt.Sprintf("%s: query context does not have filter '%s'", f.Pos.String(), f.Name)) } - switch castNode := tomlValueCheck(node, ctx).(type) { - case *TomlTree: - for _, v := range castNode.values { - if tv, ok := v.(*tomlValue); ok { - if fn(tv.value) { - f.next.call(v, ctx) - } - } else { - if fn(v) { - f.next.call(v, ctx) + switch castNode := node.(type) { + case *toml.TomlTree: + for _, k := range castNode.Keys() { + v := castNode.Get(k) + if fn(v) { + ctx.lastPosition = castNode.GetPosition(k) + f.next.call(v, ctx) + } + } + case []*toml.TomlTree: + for _, v := range castNode { + if fn(v) { + if len(castNode) > 0 { + ctx.lastPosition = castNode[0].Position() } + f.next.call(v, ctx) } } case []interface{}: diff --git a/match_test.go b/query/match_test.go similarity index 96% rename from match_test.go rename to query/match_test.go index b63654a..567b11c 100644 --- a/match_test.go +++ b/query/match_test.go @@ -1,8 +1,9 @@ -package toml +package query import ( "fmt" "testing" + "github.com/pelletier/go-toml" ) // dump path tree to a string @@ -194,8 +195,8 @@ func TestPathFilterExpr(t *testing.T) { "$[?('foo'),?(bar)]", buildPath( &matchUnionFn{[]pathFn{ - newMatchFilterFn("foo", Position{}), - newMatchFilterFn("bar", Position{}), + newMatchFilterFn("foo", toml.Position{}), + newMatchFilterFn("bar", toml.Position{}), }}, )) } diff --git a/queryparser.go b/query/parser.go similarity index 99% rename from queryparser.go rename to query/parser.go index 1cbfc83..e4f91b9 100644 --- a/queryparser.go +++ b/query/parser.go @@ -5,7 +5,7 @@ https://code.google.com/p/json-path/ */ -package toml +package query import ( "fmt" diff --git a/queryparser_test.go b/query/parser_test.go similarity index 79% rename from queryparser_test.go rename to query/parser_test.go index b2b85ce..5057853 100644 --- a/queryparser_test.go +++ b/query/parser_test.go @@ -1,4 +1,4 @@ -package toml +package query import ( "fmt" @@ -7,19 +7,18 @@ import ( "strings" "testing" "time" + "github.com/pelletier/go-toml" ) type queryTestNode struct { value interface{} - position Position + position toml.Position } func valueString(root interface{}) string { result := "" //fmt.Sprintf("%T:", root) switch node := root.(type) { - case *tomlValue: - return valueString(node.value) - case *QueryResult: + case *Result: items := []string{} for i, v := range node.Values() { items = append(items, fmt.Sprintf("%s:%s", @@ -37,7 +36,7 @@ func valueString(root interface{}) string { } sort.Strings(items) result = "[" + strings.Join(items, ", ") + "]" - case *TomlTree: + case *toml.TomlTree: // workaround for unreliable map key ordering items := []string{} for _, k := range node.Keys() { @@ -78,13 +77,13 @@ func assertValue(t *testing.T, result, ref interface{}) { } } -func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) { - tree, err := Load(toml) +func assertQueryPositions(t *testing.T, tomlDoc string, query string, ref []interface{}) { + tree, err := toml.Load(tomlDoc) if err != nil { t.Errorf("Non-nil toml parse error: %v", err) return } - q, err := CompileQuery(query) + q, err := Compile(query) if err != nil { t.Error(err) return @@ -101,7 +100,7 @@ func TestQueryRoot(t *testing.T) { queryTestNode{ map[string]interface{}{ "a": int64(42), - }, Position{1, 1}, + }, toml.Position{1, 1}, }, }) } @@ -112,7 +111,7 @@ func TestQueryKey(t *testing.T) { "$.foo.a", []interface{}{ queryTestNode{ - int64(42), Position{2, 1}, + int64(42), toml.Position{2, 1}, }, }) } @@ -123,7 +122,7 @@ func TestQueryKeyString(t *testing.T) { "$.foo['a']", []interface{}{ queryTestNode{ - int64(42), Position{2, 1}, + int64(42), toml.Position{2, 1}, }, }) } @@ -134,7 +133,7 @@ func TestQueryIndex(t *testing.T) { "$.foo.a[5]", []interface{}{ queryTestNode{ - int64(6), Position{2, 1}, + int64(6), toml.Position{2, 1}, }, }) } @@ -145,19 +144,19 @@ func TestQuerySliceRange(t *testing.T) { "$.foo.a[0:5]", []interface{}{ queryTestNode{ - int64(1), Position{2, 1}, + int64(1), toml.Position{2, 1}, }, queryTestNode{ - int64(2), Position{2, 1}, + int64(2), toml.Position{2, 1}, }, queryTestNode{ - int64(3), Position{2, 1}, + int64(3), toml.Position{2, 1}, }, queryTestNode{ - int64(4), Position{2, 1}, + int64(4), toml.Position{2, 1}, }, queryTestNode{ - int64(5), Position{2, 1}, + int64(5), toml.Position{2, 1}, }, }) } @@ -168,13 +167,13 @@ func TestQuerySliceStep(t *testing.T) { "$.foo.a[0:5:2]", []interface{}{ queryTestNode{ - int64(1), Position{2, 1}, + int64(1), toml.Position{2, 1}, }, queryTestNode{ - int64(3), Position{2, 1}, + int64(3), toml.Position{2, 1}, }, queryTestNode{ - int64(5), Position{2, 1}, + int64(5), toml.Position{2, 1}, }, }) } @@ -188,13 +187,13 @@ func TestQueryAny(t *testing.T) { map[string]interface{}{ "a": int64(1), "b": int64(2), - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(3), "b": int64(4), - }, Position{4, 1}, + }, toml.Position{4, 1}, }, }) } @@ -207,19 +206,19 @@ func TestQueryUnionSimple(t *testing.T) { map[string]interface{}{ "a": int64(1), "b": int64(2), - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(3), "b": int64(4), - }, Position{4, 1}, + }, toml.Position{4, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(5), "b": int64(6), - }, Position{7, 1}, + }, toml.Position{7, 1}, }, }) } @@ -249,7 +248,7 @@ func TestQueryRecursionAll(t *testing.T) { "b": int64(6), }, }, - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ @@ -257,19 +256,19 @@ func TestQueryRecursionAll(t *testing.T) { "a": int64(1), "b": int64(2), }, - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(1), "b": int64(2), - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ - int64(1), Position{2, 1}, + int64(1), toml.Position{2, 1}, }, queryTestNode{ - int64(2), Position{3, 1}, + int64(2), toml.Position{3, 1}, }, queryTestNode{ map[string]interface{}{ @@ -277,19 +276,19 @@ func TestQueryRecursionAll(t *testing.T) { "a": int64(3), "b": int64(4), }, - }, Position{4, 1}, + }, toml.Position{4, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(3), "b": int64(4), - }, Position{4, 1}, + }, toml.Position{4, 1}, }, queryTestNode{ - int64(3), Position{5, 1}, + int64(3), toml.Position{5, 1}, }, queryTestNode{ - int64(4), Position{6, 1}, + int64(4), toml.Position{6, 1}, }, queryTestNode{ map[string]interface{}{ @@ -297,19 +296,19 @@ func TestQueryRecursionAll(t *testing.T) { "a": int64(5), "b": int64(6), }, - }, Position{7, 1}, + }, toml.Position{7, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(5), "b": int64(6), - }, Position{7, 1}, + }, toml.Position{7, 1}, }, queryTestNode{ - int64(5), Position{8, 1}, + int64(5), toml.Position{8, 1}, }, queryTestNode{ - int64(6), Position{9, 1}, + int64(6), toml.Position{9, 1}, }, }) } @@ -325,31 +324,31 @@ func TestQueryRecursionUnionSimple(t *testing.T) { "a": int64(1), "b": int64(2), }, - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(3), "b": int64(4), - }, Position{4, 1}, + }, toml.Position{4, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(1), "b": int64(2), - }, Position{1, 1}, + }, toml.Position{1, 1}, }, queryTestNode{ map[string]interface{}{ "a": int64(5), "b": int64(6), - }, Position{7, 1}, + }, toml.Position{7, 1}, }, }) } func TestQueryFilterFn(t *testing.T) { - buff, err := ioutil.ReadFile("example.toml") + buff, err := ioutil.ReadFile("../example.toml") if err != nil { t.Error(err) return @@ -359,16 +358,16 @@ func TestQueryFilterFn(t *testing.T) { "$..[?(int)]", []interface{}{ queryTestNode{ - int64(8001), Position{13, 1}, + int64(8001), toml.Position{13, 1}, }, queryTestNode{ - int64(8001), Position{13, 1}, + int64(8001), toml.Position{13, 1}, }, queryTestNode{ - int64(8002), Position{13, 1}, + int64(8002), toml.Position{13, 1}, }, queryTestNode{ - int64(5000), Position{14, 1}, + int64(5000), toml.Position{14, 1}, }, }) @@ -376,32 +375,32 @@ func TestQueryFilterFn(t *testing.T) { "$..[?(string)]", []interface{}{ queryTestNode{ - "TOML Example", Position{3, 1}, + "TOML Example", toml.Position{3, 1}, }, queryTestNode{ - "Tom Preston-Werner", Position{6, 1}, + "Tom Preston-Werner", toml.Position{6, 1}, }, queryTestNode{ - "GitHub", Position{7, 1}, + "GitHub", toml.Position{7, 1}, }, queryTestNode{ "GitHub Cofounder & CEO\nLikes tater tots and beer.", - Position{8, 1}, + toml.Position{8, 1}, }, queryTestNode{ - "192.168.1.1", Position{12, 1}, + "192.168.1.1", toml.Position{12, 1}, }, queryTestNode{ - "10.0.0.1", Position{21, 3}, + "10.0.0.1", toml.Position{21, 3}, }, queryTestNode{ - "eqdc10", Position{22, 3}, + "eqdc10", toml.Position{22, 3}, }, queryTestNode{ - "10.0.0.2", Position{25, 3}, + "10.0.0.2", toml.Position{25, 3}, }, queryTestNode{ - "eqdc10", Position{26, 3}, + "eqdc10", toml.Position{26, 3}, }, }) @@ -421,7 +420,7 @@ func TestQueryFilterFn(t *testing.T) { "organization": "GitHub", "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", "dob": tv, - }, Position{5, 1}, + }, toml.Position{5, 1}, }, queryTestNode{ map[string]interface{}{ @@ -429,7 +428,7 @@ func TestQueryFilterFn(t *testing.T) { "ports": []interface{}{int64(8001), int64(8001), int64(8002)}, "connection_max": int64(5000), "enabled": true, - }, Position{11, 1}, + }, toml.Position{11, 1}, }, queryTestNode{ map[string]interface{}{ @@ -441,19 +440,19 @@ func TestQueryFilterFn(t *testing.T) { "ip": "10.0.0.2", "dc": "eqdc10", }, - }, Position{17, 1}, + }, toml.Position{17, 1}, }, queryTestNode{ map[string]interface{}{ "ip": "10.0.0.1", "dc": "eqdc10", - }, Position{20, 3}, + }, toml.Position{20, 3}, }, queryTestNode{ map[string]interface{}{ "ip": "10.0.0.2", "dc": "eqdc10", - }, Position{24, 3}, + }, toml.Position{24, 3}, }, queryTestNode{ map[string]interface{}{ @@ -461,7 +460,7 @@ func TestQueryFilterFn(t *testing.T) { []interface{}{"gamma", "delta"}, []interface{}{int64(1), int64(2)}, }, - }, Position{28, 1}, + }, toml.Position{28, 1}, }, }) @@ -469,7 +468,7 @@ func TestQueryFilterFn(t *testing.T) { "$..[?(time)]", []interface{}{ queryTestNode{ - tv, Position{9, 1}, + tv, toml.Position{9, 1}, }, }) @@ -477,7 +476,7 @@ func TestQueryFilterFn(t *testing.T) { "$..[?(bool)]", []interface{}{ queryTestNode{ - true, Position{15, 1}, + true, toml.Position{15, 1}, }, }) } diff --git a/query.go b/query/query.go similarity index 69% rename from query.go rename to query/query.go index 307a1ec..2f4de78 100644 --- a/query.go +++ b/query/query.go @@ -1,7 +1,9 @@ -package toml +package query import ( "time" + + "github.com/pelletier/go-toml" ) // NodeFilterFn represents a user-defined filter function, for use with @@ -15,50 +17,43 @@ import ( // to use from multiple goroutines. type NodeFilterFn func(node interface{}) bool -// QueryResult is the result of Executing a Query. -type QueryResult struct { +// Result is the result of Executing a Query. +type Result struct { items []interface{} - positions []Position + positions []toml.Position } // appends a value/position pair to the result set. -func (r *QueryResult) appendResult(node interface{}, pos Position) { +func (r *Result) appendResult(node interface{}, pos toml.Position) { r.items = append(r.items, node) r.positions = append(r.positions, pos) } -// Values is a set of values within a QueryResult. The order of values is not +// Values is a set of values within a Result. The order of values is not // guaranteed to be in document order, and may be different each time a query is // executed. -func (r QueryResult) Values() []interface{} { - values := make([]interface{}, len(r.items)) - for i, v := range r.items { - o, ok := v.(*tomlValue) - if ok { - values[i] = o.value - } else { - values[i] = v - } - } - return values +func (r Result) Values() []interface{} { + return r.items } -// Positions is a set of positions for values within a QueryResult. Each index +// Positions is a set of positions for values within a Result. Each index // in Positions() corresponds to the entry in Value() of the same index. -func (r QueryResult) Positions() []Position { +func (r Result) Positions() []toml.Position { return r.positions } // runtime context for executing query paths type queryContext struct { - result *QueryResult + result *Result filters *map[string]NodeFilterFn - lastPosition Position + lastPosition toml.Position } // generic path functor interface type pathFn interface { setNext(next pathFn) + // it is the caller's responsibility to set the ctx.lastPosition before invoking call() + // node can be one of: *toml.TomlTree, []*toml.TomlTree, or a scalar call(node interface{}, ctx *queryContext) } @@ -88,17 +83,17 @@ func (q *Query) appendPath(next pathFn) { next.setNext(newTerminatingFn()) // init the next functor } -// CompileQuery compiles a TOML path expression. The returned Query can be used -// to match elements within a TomlTree and its descendants. -func CompileQuery(path string) (*Query, error) { +// Compile compiles a TOML path expression. The returned Query can be used +// to match elements within a TomlTree and its descendants. See Execute. +func Compile(path string) (*Query, error) { return parseQuery(lexQuery(path)) } // Execute executes a query against a TomlTree, and returns the result of the query. -func (q *Query) Execute(tree *TomlTree) *QueryResult { - result := &QueryResult{ +func (q *Query) Execute(tree *toml.TomlTree) *Result { + result := &Result{ items: []interface{}{}, - positions: []Position{}, + positions: []toml.Position{}, } if q.root == nil { result.appendResult(tree, tree.GetPosition("")) @@ -107,11 +102,21 @@ func (q *Query) Execute(tree *TomlTree) *QueryResult { result: result, filters: q.filters, } + ctx.lastPosition = tree.Position() q.root.call(tree, ctx) } return result } +// CompileAndExecute is a shorthand for Compile(path) followed by Execute(tree). +func CompileAndExecute(path string, tree *toml.TomlTree) (*Result, error) { + query, err := Compile(path) + if err != nil { + return nil, err + } + return query.Execute(tree), nil +} + // SetFilter sets a user-defined filter function. These may be used inside // "?(..)" query expressions to filter TOML document elements within a query. func (q *Query) SetFilter(name string, fn NodeFilterFn) { @@ -127,7 +132,7 @@ func (q *Query) SetFilter(name string, fn NodeFilterFn) { var defaultFilterFunctions = map[string]NodeFilterFn{ "tree": func(node interface{}) bool { - _, ok := node.(*TomlTree) + _, ok := node.(*toml.TomlTree) return ok }, "int": func(node interface{}) bool { diff --git a/query/query_test.go b/query/query_test.go new file mode 100644 index 0000000..4433f5f --- /dev/null +++ b/query/query_test.go @@ -0,0 +1,157 @@ +package query + +import ( + "fmt" + "testing" + + "github.com/pelletier/go-toml" +) + +func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) { + if len(array) != len(objects) { + t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects)) + } + + for _, o := range objects { + found := false + for _, a := range array { + if a == o { + found = true + break + } + } + if !found { + t.Fatal(o, "not found in array", array) + } + } +} + +func TestQueryExample(t *testing.T) { + config, _ := toml.Load(` + [[book]] + title = "The Stand" + author = "Stephen King" + [[book]] + title = "For Whom the Bell Tolls" + author = "Ernest Hemmingway" + [[book]] + title = "Neuromancer" + author = "William Gibson" + `) + authors, err := CompileAndExecute("$.book.author", config) + if err != nil { + t.Fatal("unexpected error:", err) + } + names := authors.Values() + if len(names) != 3 { + t.Fatalf("query should return 3 names but returned %d", len(names)) + } + assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson") +} + +func TestQueryReadmeExample(t *testing.T) { + config, _ := toml.Load(` +[postgres] +user = "pelletier" +password = "mypassword" +`) + + query, err := Compile("$..[user,password]") + if err != nil { + t.Fatal("unexpected error:", err) + } + results := query.Execute(config) + values := results.Values() + if len(values) != 2 { + t.Fatalf("query should return 2 values but returned %d", len(values)) + } + assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword") +} + +func TestQueryPathNotPresent(t *testing.T) { + config, _ := toml.Load(`a = "hello"`) + query, err := Compile("$.foo.bar") + if err != nil { + t.Fatal("unexpected error:", err) + } + results := query.Execute(config) + if err != nil { + t.Fatalf("err should be nil. got %s instead", err) + } + if len(results.items) != 0 { + t.Fatalf("no items should be matched. %d matched instead", len(results.items)) + } +} + +func ExampleNodeFilterFn_filterExample() { + tree, _ := toml.Load(` + [struct_one] + foo = "foo" + bar = "bar" + + [struct_two] + baz = "baz" + gorf = "gorf" + `) + + // create a query that references a user-defined-filter + query, _ := Compile("$[?(bazOnly)]") + + // define the filter, and assign it to the query + query.SetFilter("bazOnly", func(node interface{}) bool { + if tree, ok := node.(*toml.TomlTree); ok { + return tree.Has("baz") + } + return false // reject all other node types + }) + + // results contain only the 'struct_two' TomlTree + query.Execute(tree) +} + +func ExampleQuery_queryExample() { + config, _ := toml.Load(` + [[book]] + title = "The Stand" + author = "Stephen King" + [[book]] + title = "For Whom the Bell Tolls" + author = "Ernest Hemmingway" + [[book]] + title = "Neuromancer" + author = "William Gibson" + `) + + // find and print all the authors in the document + query, _ := Compile("$.book.author") + authors := query.Execute(config) + for _, name := range authors.Values() { + fmt.Println(name) + } +} + +func TestTomlQuery(t *testing.T) { + tree, err := toml.Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6") + if err != nil { + t.Error(err) + return + } + query, err := Compile("$.foo.bar") + if err != nil { + t.Error(err) + return + } + result := query.Execute(tree) + values := result.Values() + if len(values) != 1 { + t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values) + } + + if tt, ok := values[0].(*toml.TomlTree); !ok { + t.Errorf("Expected type of TomlTree: %T", values[0]) + } else if tt.Get("a") != int64(1) { + t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a")) + } else if tt.Get("b") != int64(2) { + t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b")) + } +} diff --git a/query/tokens.go b/query/tokens.go new file mode 100644 index 0000000..429e289 --- /dev/null +++ b/query/tokens.go @@ -0,0 +1,107 @@ +package query + +import ( +"fmt" +"strconv" +"unicode" + "github.com/pelletier/go-toml" +) + +// Define tokens +type tokenType int + +const ( + eof = -(iota + 1) +) + +const ( + tokenError tokenType = iota + tokenEOF + tokenKey + tokenString + tokenInteger + tokenFloat + tokenLeftBracket + tokenRightBracket + tokenLeftParen + tokenRightParen + tokenComma + tokenColon + tokenDollar + tokenStar + tokenQuestion + tokenDot + tokenDotDot +) + +var tokenTypeNames = []string{ + "Error", + "EOF", + "Key", + "String", + "Integer", + "Float", + "[", + "]", + "(", + ")", + ",", + ":", + "$", + "*", + "?", + ".", + "..", +} + +type token struct { + toml.Position + typ tokenType + val string +} + +func (tt tokenType) String() string { + idx := int(tt) + if idx < len(tokenTypeNames) { + return tokenTypeNames[idx] + } + return "Unknown" +} + +func (t token) Int() int { + if result, err := strconv.Atoi(t.val); err != nil { + panic(err) + } else { + return result + } +} + +func (t token) String() string { + switch t.typ { + case tokenEOF: + return "EOF" + case tokenError: + return t.val + } + + return fmt.Sprintf("%q", t.val) +} + +func isSpace(r rune) bool { + return r == ' ' || r == '\t' +} + +func isAlphanumeric(r rune) bool { + return unicode.IsLetter(r) || r == '_' +} + +func isDigit(r rune) bool { + return unicode.IsNumber(r) +} + +func isHexDigit(r rune) bool { + return isDigit(r) || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} + diff --git a/query_test.go b/query_test.go deleted file mode 100644 index 0d9f383..0000000 --- a/query_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package toml - -import ( - "testing" -) - -func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) { - if len(array) != len(objects) { - t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects)) - } - - for _, o := range objects { - found := false - for _, a := range array { - if a == o { - found = true - break - } - } - if !found { - t.Fatal(o, "not found in array", array) - } - } -} - -func TestQueryExample(t *testing.T) { - config, _ := Load(` - [[book]] - title = "The Stand" - author = "Stephen King" - [[book]] - title = "For Whom the Bell Tolls" - author = "Ernest Hemmingway" - [[book]] - title = "Neuromancer" - author = "William Gibson" - `) - - authors, _ := config.Query("$.book.author") - names := authors.Values() - if len(names) != 3 { - t.Fatalf("query should return 3 names but returned %d", len(names)) - } - assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson") -} - -func TestQueryReadmeExample(t *testing.T) { - config, _ := Load(` -[postgres] -user = "pelletier" -password = "mypassword" -`) - results, _ := config.Query("$..[user,password]") - values := results.Values() - if len(values) != 2 { - t.Fatalf("query should return 2 values but returned %d", len(values)) - } - assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword") -} - -func TestQueryPathNotPresent(t *testing.T) { - config, _ := Load(`a = "hello"`) - results, err := config.Query("$.foo.bar") - if err != nil { - t.Fatalf("err should be nil. got %s instead", err) - } - if len(results.items) != 0 { - t.Fatalf("no items should be matched. %d matched instead", len(results.items)) - } -} diff --git a/test.sh b/test.sh index 436d2fb..d2d2aed 100755 --- a/test.sh +++ b/test.sh @@ -19,6 +19,9 @@ function git_clone() { popd } +# Remove potential previous runs +rm -rf src test_program_bin toml-test + # Run go vet go vet ./... @@ -36,13 +39,16 @@ go build -o toml-test github.com/BurntSushi/toml-test # vendorize the current lib for testing # NOTE: this basically mocks an install without having to go back out to github for code mkdir -p src/github.com/pelletier/go-toml/cmd +mkdir -p src/github.com/pelletier/go-toml/query cp *.go *.toml src/github.com/pelletier/go-toml cp -R cmd/* src/github.com/pelletier/go-toml/cmd +cp -R query/* src/github.com/pelletier/go-toml/query go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go # Run basic unit tests -go test github.com/pelletier/go-toml -v -covermode=count -coverprofile=coverage.out +go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out go test github.com/pelletier/go-toml/cmd/tomljson +go test github.com/pelletier/go-toml/query # run the entire BurntSushi test suite if [[ $# -eq 0 ]] ; then diff --git a/toml.go b/toml.go index ae3a165..d07c1b4 100644 --- a/toml.go +++ b/toml.go @@ -36,6 +36,11 @@ func TreeFromMap(m map[string]interface{}) (*TomlTree, error) { return result.(*TomlTree), nil } +// Position returns the position of the tree. +func (t *TomlTree) Position() Position { + return t.position +} + // Has returns a boolean indicating if the given key exists. func (t *TomlTree) Has(key string) bool { if key == "" { @@ -247,15 +252,6 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error { return nil } -// Query compiles and executes a query on a tree and returns the query result. -func (t *TomlTree) Query(query string) (*QueryResult, error) { - q, err := CompileQuery(query) - if err != nil { - return nil, err - } - return q.Execute(t), nil -} - // LoadReader creates a TomlTree from any io.Reader. func LoadReader(reader io.Reader) (tree *TomlTree, err error) { defer func() { diff --git a/toml_test.go b/toml_test.go index 7c7f9ef..871063a 100644 --- a/toml_test.go +++ b/toml_test.go @@ -94,31 +94,6 @@ func TestTomlGetPath(t *testing.T) { } } -func TestTomlQuery(t *testing.T) { - tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6") - if err != nil { - t.Error(err) - return - } - result, err := tree.Query("$.foo.bar") - if err != nil { - t.Error(err) - return - } - values := result.Values() - if len(values) != 1 { - t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values) - } - - if tt, ok := values[0].(*TomlTree); !ok { - t.Errorf("Expected type of TomlTree: %T", values[0]) - } else if tt.Get("a") != int64(1) { - t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a")) - } else if tt.Get("b") != int64(2) { - t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b")) - } -} - func TestTomlFromMap(t *testing.T) { simpleMap := map[string]interface{}{"hello": 42} tree, err := TreeFromMap(simpleMap)