Query interface with callback functions

* Added public Query interface
* Added filter function callback support
* Added "script" function callback support

Queries are generated via Compile(), which then may be run via Execute()
as many times as needed.  Much like compiling a regex, this is done to
elide the need to re-parse and build the funciton tree for each
execution.

The distinction between 'filter' and 'script' is borrowed from their
syntactic equivalents in jsonpath.  Right now, these accept no arguments
in the query, and instead merely pass the current node to the callback.
Filters return a bool and determine if the node is kept or culled out.
'Scripts' return a string or an in64, which is in turn used in an index
or key filter (respectively) on the current node's data.

A few callbacks are provided by default, with the ability to add
additional callbacks before calling Execute() on a compiled query.

jpath/match.go

@@ -2,61 +2,9 @@ package jpath
 
 import (
 	. "github.com/pelletier/go-toml"
+  "fmt"
 )
 
-// result set for storage of results
-type pathResult struct {
-  Values []interface{}
-}
-
-func newPathResult() *pathResult {
-  return &pathResult {
-    Values: []interface{}{},
-  }
-}
-
-func (r *pathResult) Append(value interface{}) {
-  r.Values = append(r.Values, value)
-}
-
-// generic path functor interface
-type PathFn interface{
-  SetNext(next PathFn)
-  Call(context interface{}, results *pathResult)
-}
-
-// contains a functor chain
-type QueryPath struct {
-  root PathFn
-  tail PathFn
-}
-
-func newQueryPath() *QueryPath {
-  return &QueryPath {
-    root: nil,
-    tail: nil,
-  }
-}
-
-func (path *QueryPath) Append(next PathFn) {
-  if path.root == nil {
-    path.root = next
-  } else {
-    path.tail.SetNext(next)
-  }
-  path.tail = next
-  next.SetNext(newTerminatingFn()) // init the next functor
-}
-
-func (path *QueryPath) Call(context interface{}) []interface{} {
-  results := newPathResult()
-  if path.root == nil {
-    results.Append(context)  // identity query for no predicates
-  } else {
-    path.root.Call(context, results)
-  }
-  return results.Values
-}
 
 // base match
 type matchBase struct {
@@ -80,8 +28,8 @@ func (f *terminatingFn) SetNext(next PathFn) {
   // do nothing
 }
 
-func (f *terminatingFn) Call(context interface{}, results *pathResult) {
+func (f *terminatingFn) Call(node interface{}, ctx *queryContext) {
-  results.Append(context)
+  ctx.appendResult(node)
 }
 
 // shim to ease functor writing
@@ -99,11 +47,11 @@ func newMatchKeyFn(name string) *matchKeyFn {
   return &matchKeyFn{ Name: name }
 }
 
-func (f *matchKeyFn) Call(context interface{}, results *pathResult) {
+func (f *matchKeyFn) Call(node interface{}, ctx *queryContext) {
-  if tree, ok := context.(*TomlTree); ok {
+  if tree, ok := node.(*TomlTree); ok {
     item := treeValue(tree, f.Name)
     if item != nil {
-      f.next.Call(item, results)
+      f.next.Call(item, ctx)
     }
   }
 }
@@ -118,10 +66,10 @@ func newMatchIndexFn(idx int) *matchIndexFn {
   return &matchIndexFn{ Idx: idx }
 }
 
-func (f *matchIndexFn) Call(context interface{}, results *pathResult) {
+func (f *matchIndexFn) Call(node interface{}, ctx *queryContext) {
-  if arr, ok := context.([]interface{}); ok {
+  if arr, ok := node.([]interface{}); ok {
     if f.Idx < len(arr) && f.Idx >= 0 {
-      f.next.Call(arr[f.Idx], results)
+      f.next.Call(arr[f.Idx], ctx)
     }
   }
 }
@@ -136,8 +84,8 @@ func newMatchSliceFn(start, end, step int) *matchSliceFn {
   return &matchSliceFn{ Start: start, End: end, Step: step }
 }
 
-func (f *matchSliceFn) Call(context interface{}, results *pathResult) {
+func (f *matchSliceFn) Call(node interface{}, ctx *queryContext) {
-  if arr, ok := context.([]interface{}); ok {
+  if arr, ok := node.([]interface{}); ok {
     // adjust indexes for negative values, reverse ordering
     realStart, realEnd := f.Start, f.End
     if realStart < 0 {
@@ -151,7 +99,7 @@ func (f *matchSliceFn) Call(context interface{}, results *pathResult) {
     }
     // loop and gather
     for idx := realStart; idx < realEnd; idx += f.Step {
-      f.next.Call(arr[idx], results)
+      f.next.Call(arr[idx], ctx)
     }
   }
 }
@@ -159,18 +107,17 @@ func (f *matchSliceFn) Call(context interface{}, results *pathResult) {
 // match anything
 type matchAnyFn struct {
   matchBase
-  // empty
 }
 
 func newMatchAnyFn() *matchAnyFn {
   return &matchAnyFn{}
 }
 
-func (f *matchAnyFn) Call(context interface{}, results *pathResult) {
+func (f *matchAnyFn) Call(node interface{}, ctx *queryContext) {
-  if tree, ok := context.(*TomlTree); ok {
+  if tree, ok := node.(*TomlTree); ok {
     for _, key := range tree.Keys() {
       item := treeValue(tree, key)
-      f.next.Call(item, results)
+      f.next.Call(item, ctx)
     }
   }
 }
@@ -186,9 +133,9 @@ func (f *matchUnionFn) SetNext(next PathFn) {
   }
 }
 
-func (f *matchUnionFn) Call(context interface{}, results *pathResult) {
+func (f *matchUnionFn) Call(node interface{}, ctx *queryContext) {
   for _, fn := range f.Union {
-    fn.Call(context, results)
+    fn.Call(node, ctx)
   }
 }
 
@@ -201,13 +148,13 @@ func newMatchRecursiveFn() *matchRecursiveFn{
   return &matchRecursiveFn{}
 }
 
-func (f *matchRecursiveFn) Call(context interface{}, results *pathResult) {
+func (f *matchRecursiveFn) Call(node interface{}, ctx *queryContext) {
-  if tree, ok := context.(*TomlTree); ok {
+  if tree, ok := node.(*TomlTree); ok {
     var visit func(tree *TomlTree)
     visit = func(tree *TomlTree) {
       for _, key := range tree.Keys() {
         item := treeValue(tree, key)
-        f.next.Call(item, results)
+        f.next.Call(item, ctx)
         switch node := item.(type) {
         case *TomlTree:
           visit(node)
@@ -221,3 +168,67 @@ func (f *matchRecursiveFn) Call(context interface{}, results *pathResult) {
     visit(tree)
   }
 }
+
+// match based on an externally provided functional filter
+type matchFilterFn struct {
+  matchBase
+  Pos Position
+  Name string
+}
+
+func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+  return &matchFilterFn{ Name: name, Pos: pos }
+}
+
+func (f *matchFilterFn) Call(node interface{}, ctx *queryContext) {
+  fn, ok := (*ctx.filters)[f.Name]
+  if !ok {
+    panic(fmt.Sprintf("%s: query context does not have filter '%s'",
+      f.Pos, f.Name))
+  }
+  switch castNode := node.(type) {
+  case *TomlTree:
+    for _, k := range castNode.Keys() {
+      v := castNode.GetPath([]string{k})
+      if fn(v) {
+        f.next.Call(v, ctx)
+      }
+    }
+  case []interface{}:
+    for _, v := range castNode {
+      if fn(v) {
+        f.next.Call(v, ctx)
+      }
+    }
+  }
+}
+
+// match based using result of an externally provided functional filter
+type matchScriptFn struct {
+  matchBase
+  Pos Position
+  Name string
+}
+
+func newMatchScriptFn(name string, pos Position) *matchScriptFn {
+  return &matchScriptFn{ Name: name, Pos: pos }
+}
+
+func (f *matchScriptFn) Call(node interface{}, ctx *queryContext) {
+  fn, ok := (*ctx.scripts)[f.Name]
+  if !ok {
+    panic(fmt.Sprintf("%s: query context does not have script '%s'",
+      f.Pos, f.Name))
+  }
+  switch result := fn(node).(type) {
+  case string:
+    nextMatch := newMatchKeyFn(result)
+    nextMatch.SetNext(f.next)
+    nextMatch.Call(node, ctx)
+  case int:
+    nextMatch := newMatchIndexFn(result)
+    nextMatch.SetNext(f.next)
+    nextMatch.Call(node, ctx)
+  //TODO: support other return types?
+  }
+}

jpath/match_test.go

@@ -1,6 +1,7 @@
 package jpath
 
 import (
+	. "github.com/pelletier/go-toml"
   "fmt"
   "math"
 	"testing"
@@ -8,7 +9,7 @@ import (
 
 // dump path tree to a string
 func pathString(root PathFn) string {
-  result := fmt.Sprintf("%T:")
+  result := fmt.Sprintf("%T:", root)
   switch fn := root.(type) {
   case *terminatingFn:
     result += "{}"
@@ -28,21 +29,27 @@ func pathString(root PathFn) string {
   case *matchUnionFn:
     result += "{["
     for _, v := range fn.Union {
-      result += pathString(v)
+      result += pathString(v) + ", "
     }
     result += "]}"
   case *matchRecursiveFn:
     result += "{}"
     result += pathString(fn.next)
+  case *matchFilterFn:
+    result += fmt.Sprintf("{%s}", fn.Name)
+    result += pathString(fn.next)
+  case *matchScriptFn:
+    result += fmt.Sprintf("{%s}", fn.Name)
+    result += pathString(fn.next)
   }
   return result
 }
 
-func assertPathMatch(t *testing.T, path, ref *QueryPath) bool {
+func assertPathMatch(t *testing.T, path, ref *Query) bool {
   pathStr := pathString(path.root)
   refStr := pathString(ref.root)
   if pathStr != refStr {
-    t.Errorf("paths do not match: %v vs %v")
+    t.Errorf("paths do not match")
     t.Log("test:", pathStr)
     t.Log("ref: ", refStr)
     return false
@@ -50,18 +57,18 @@ func assertPathMatch(t *testing.T, path, ref *QueryPath) bool {
   return true
 }
 
-func assertPath(t *testing.T, query string, ref *QueryPath) {
+func assertPath(t *testing.T, query string, ref *Query) {
 	_, flow := lex(query)
 	path := parse(flow)
   assertPathMatch(t, path, ref)
 }
 
-func buildPath(parts... PathFn) *QueryPath {
+func buildPath(parts... PathFn) *Query {
-  path := newQueryPath()
+  query := newQuery()
   for _, v := range parts {
-    path.Append(v)
+    query.appendPath(v)
   }
-  return path
+  return query
 }
 
 func TestPathRoot(t *testing.T) {
@@ -187,3 +194,25 @@ func TestPathRecurse(t *testing.T) {
       newMatchRecursiveFn(),
     ))
 }
+
+func TestPathFilterExpr(t *testing.T) {
+	assertPath(t,
+		"$[?('foo'),?(bar)]",
+		buildPath(
+      &matchUnionFn{ []PathFn {
+        newMatchFilterFn("foo", Position{}),
+        newMatchFilterFn("bar", Position{}),
+      }},
+    ))
+}
+
+func TestPathScriptExpr(t *testing.T) {
+	assertPath(t,
+		"$[('foo'),(bar)]",
+		buildPath(
+      &matchUnionFn{ []PathFn {
+        newMatchScriptFn("foo", Position{}),
+        newMatchScriptFn("bar", Position{}),
+      }},
+    ))
+}

jpath/parser.go

@@ -15,7 +15,7 @@ import (
 type parser struct {
 	flow         chan token
 	tokensBuffer []token
-	path         *QueryPath
+	path         *Query
   union        []PathFn
 }
 
@@ -102,12 +102,12 @@ func parseMatchExpr(p *parser) parserStateFn {
 	tok := p.getToken()
 	switch tok.typ {
 	case tokenDotDot:
-    p.path.Append(&matchRecursiveFn{})
+    p.path.appendPath(&matchRecursiveFn{})
     // nested parse for '..'
     tok := p.getToken()
     switch tok.typ {
     case tokenKey:
-      p.path.Append(newMatchKeyFn(tok.val))
+      p.path.appendPath(newMatchKeyFn(tok.val))
       return parseMatchExpr
     case tokenLBracket:
       return parseBracketExpr
@@ -121,10 +121,10 @@ func parseMatchExpr(p *parser) parserStateFn {
     tok := p.getToken()
     switch tok.typ {
     case tokenKey:
-      p.path.Append(newMatchKeyFn(tok.val))
+      p.path.appendPath(newMatchKeyFn(tok.val))
       return parseMatchExpr
     case tokenStar:
-      p.path.Append(&matchAnyFn{})
+      p.path.appendPath(&matchAnyFn{})
       return parseMatchExpr
     }
 
@@ -149,6 +149,8 @@ func parseBracketExpr(p *parser) parserStateFn {
 }
 
 func parseUnionExpr(p *parser) parserStateFn {
+  var tok *token
+
   // this state can be traversed after some sub-expressions
   // so be careful when setting up state in the parser
 	if p.union == nil {
@@ -157,8 +159,21 @@ func parseUnionExpr(p *parser) parserStateFn {
 
 loop: // labeled loop for easy breaking
   for {
+    if len(p.union) > 0 {
+      // parse delimiter or terminator
+      tok = p.getToken()
+      switch tok.typ {
+      case tokenComma:
+        // do nothing
+      case tokenRBracket:
+        break loop
+      default:
+        p.raiseError(tok, "expected ',' or ']', not '%s'", tok.val)
+      }
+    }
+
 		// parse sub expression
-		tok := p.getToken()
+		tok = p.getToken()
 		switch tok.typ {
 		case tokenInteger:
 			p.union = append(p.union, newMatchIndexFn(tok.Int()))
@@ -171,25 +186,15 @@ loop: // labeled loop for easy breaking
 		case tokenLParen:
 			return parseScriptExpr
 		default:
-			p.raiseError(tok, "expected union sub expression, not '%s'", tok.val)
+			p.raiseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union))
-		}
-		// parse delimiter or terminator
-		tok = p.getToken()
-		switch tok.typ {
-		case tokenComma:
-			continue
-		case tokenRBracket:
-			break loop
-		default:
-			p.raiseError(tok, "expected ',' or ']'")
 		}
 	}
 
   // if there is only one sub-expression, use that instead
   if len(p.union) == 1 {
-    p.path.Append(p.union[0])
+    p.path.appendPath(p.union[0])
   }else {
-    p.path.Append(&matchUnionFn{p.union})
+    p.path.appendPath(&matchUnionFn{p.union})
   }
 
   p.union = nil // clear out state
@@ -217,7 +222,7 @@ func parseSliceExpr(p *parser) parserStateFn {
 		tok = p.getToken()
 	}
   if tok.typ == tokenRBracket {
-	  p.path.Append(newMatchSliceFn(start, end, step))
+	  p.path.appendPath(newMatchSliceFn(start, end, step))
     return parseMatchExpr
   }
   if tok.typ != tokenColon {
@@ -237,25 +242,47 @@ func parseSliceExpr(p *parser) parserStateFn {
 		p.raiseError(tok, "expected ']'")
 	}
 
-	p.path.Append(newMatchSliceFn(start, end, step))
+	p.path.appendPath(newMatchSliceFn(start, end, step))
 	return parseMatchExpr
 }
 
 func parseFilterExpr(p *parser) parserStateFn {
-	p.raiseError(p.peek(), "filter expressions are unsupported")
+  tok := p.getToken()
-	return nil
+  if tok.typ != tokenLParen {
+    p.raiseError(tok, "expected left-parenthesis for filter expression")
+  }
+  tok = p.getToken()
+  if tok.typ != tokenKey && tok.typ != tokenString {
+    p.raiseError(tok, "expected key or string for filter funciton name")
+  }
+  name := tok.val
+  tok = p.getToken()
+  if tok.typ != tokenRParen {
+    p.raiseError(tok, "expected right-parenthesis for filter expression")
+  }
+	p.union = append(p.union, newMatchFilterFn(name, tok.Position))
+	return parseUnionExpr
 }
 
 func parseScriptExpr(p *parser) parserStateFn {
-	p.raiseError(p.peek(), "script expressions are unsupported")
+  tok := p.getToken()
-	return nil
+  if tok.typ != tokenKey && tok.typ != tokenString {
+    p.raiseError(tok, "expected key or string for script funciton name")
+  }
+  name := tok.val
+  tok = p.getToken()
+  if tok.typ != tokenRParen {
+    p.raiseError(tok, "expected right-parenthesis for script expression")
+  }
+	p.union = append(p.union, newMatchScriptFn(name, tok.Position))
+	return parseUnionExpr
 }
 
-func parse(flow chan token) *QueryPath {
+func parse(flow chan token) *Query {
 	parser := &parser{
 		flow:         flow,
 		tokensBuffer: []token{},
-		path:         newQueryPath(),
+		path:         newQuery(),
 	}
 	parser.run()
 	return parser.path

jpath/parser_test.go

@@ -12,14 +12,8 @@ func assertQuery(t *testing.T, toml, query string, ref []interface{}) {
 		t.Errorf("Non-nil toml parse error: %v", err)
 		return
 	}
-	_, flow := lex(query)
+  results := Compile(query).Execute(tree)
-	if err != nil {
+	assertValue(t, results, ref, "((" + query + ")) -> ")
-		t.Errorf("Non-nil query lex error: %v", err)
-		return
-	}
-	path := parse(flow)
-	result := path.Call(tree)
-	assertValue(t, result, ref, "((" + query + ")) -> ")
 }
 
 func assertValue(t *testing.T, result, ref interface{}, location string) {
@@ -233,3 +227,38 @@ func TestQueryRecursionUnionSimple(t *testing.T) {
       },
 	  })
 }
+
+func TestQueryScriptFnLast(t *testing.T) {
+  assertQuery(t,
+    "[foo]\na = [0,1,2,3,4,5,6,7,8,9]",
+    "$.foo.a[(last)]",
+		[]interface{}{
+		  int64(9),
+	  })
+}
+
+func TestQueryFilterFnOdd(t *testing.T) {
+  assertQuery(t,
+    "[foo]\na = [0,1,2,3,4,5,6,7,8,9]",
+    "$.foo.a[?(odd)]",
+		[]interface{}{
+		  int64(1),
+		  int64(3),
+		  int64(5),
+		  int64(7),
+		  int64(9),
+	  })
+}
+
+func TestQueryFilterFnEven(t *testing.T) {
+  assertQuery(t,
+    "[foo]\na = [0,1,2,3,4,5,6,7,8,9]",
+    "$.foo.a[?(even)]",
+		[]interface{}{
+		  int64(0),
+		  int64(2),
+		  int64(4),
+		  int64(6),
+		  int64(8),
+	  })
+}

jpath/query.go

@@ -0,0 +1,117 @@
+package jpath
+
+import (
+  _ "github.com/pelletier/go-toml"
+)
+
+type nodeFilterFn func(node interface{}) bool
+type nodeFn func(node interface{}) interface{}
+
+// runtime context for executing query paths
+type queryContext struct {
+  filters *map[string]nodeFilterFn
+  scripts *map[string]nodeFn
+  results []interface{}
+}
+
+func (c *queryContext) appendResult(value interface{}) {
+  c.results = append(c.results, value)
+}
+
+// generic path functor interface
+type PathFn interface {
+  SetNext(next PathFn)
+  Call(node interface{}, ctx *queryContext)
+}
+
+// encapsulates a query functor chain and script callbacks
+type Query struct {
+  root PathFn
+  tail PathFn
+  filters *map[string]nodeFilterFn
+  scripts *map[string]nodeFn
+}
+
+func newQuery() *Query {
+  return &Query {
+    root: nil,
+    tail: nil,
+    filters: &defaultFilterFunctions,
+    scripts: &defaultScriptFunctions,
+  }
+}
+
+func (q *Query) appendPath(next PathFn) {
+  if q.root == nil {
+    q.root = next
+  } else {
+    q.tail.SetNext(next)
+  }
+  q.tail = next
+  next.SetNext(newTerminatingFn()) // init the next functor
+}
+
+func Compile(path string) *Query {
+  _, flow := lex(path)
+  return parse(flow)
+}
+
+func (q *Query) Execute(node interface{}) interface{} {
+  if q.root == nil {
+    return []interface{}{node} // identity query for no predicates
+  }
+  ctx := &queryContext {
+    filters: q.filters,
+    scripts: q.scripts,
+    results: []interface{}{},
+  }
+  q.root.Call(node, ctx)
+  return ctx.results
+}
+
+func (q *Query) SetFilter(name string, fn nodeFilterFn) {
+  if q.filters == &defaultFilterFunctions {
+    // clone the static table
+    q.filters = &map[string]nodeFilterFn{}
+    for k, v := range defaultFilterFunctions {
+      (*q.filters)[k] = v
+    }
+  }
+  (*q.filters)[name] = fn
+}
+
+func (q *Query) SetScript(name string, fn nodeFn) {
+  if q.scripts == &defaultScriptFunctions {
+    // clone the static table
+    q.scripts = &map[string]nodeFn{}
+    for k, v := range defaultScriptFunctions {
+      (*q.scripts)[k] = v
+    }
+  }
+  (*q.scripts)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]nodeFilterFn {
+  "odd":  func(node interface{}) bool {
+    if ii, ok := node.(int64); ok {
+      return (ii & 1) == 1
+    }
+    return false
+  },
+  "even": func(node interface{}) bool {
+    if ii, ok := node.(int64); ok {
+      return (ii & 1) == 0
+    }
+    return false
+  },
+}
+
+var defaultScriptFunctions = map[string]nodeFn {
+  "last": func(node interface{}) interface{} {
+    if arr, ok := node.([]interface{}); ok {
+      return len(arr)-1
+    }
+    return nil
+  },
+}
+