Fix example code in README (#187)

* Fix Marshal examples
* Move ExampleMarshal to doc test

.travis.yml

@@ -1,18 +1,23 @@
+sudo: false
 language: go
 go:
-  - 1.5.4
+  - 1.7.6
-  - 1.6.4
+  - 1.8.3
-  - 1.7.4
+  - 1.9
   - tip
 matrix:
   allow_failures:
     - go: tip
   fast_finish: true
 script:
+  - if [ -n "$(go fmt ./...)" ]; then exit 1; fi
   - ./test.sh
+  - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git
 before_install:
   - go get github.com/axw/gocov/gocov
   - go get github.com/mattn/goveralls
   - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+branches:
+  only: [master]
 after_success:
-  - $HOME/gopath/bin/goveralls -service=travis-ci
+  - $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=coverage.out -repotoken $COVERALLS_TOKEN

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013 - 2016 Thomas Pelletier, Eric Anderton
+Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-

README.md

@@ -6,7 +6,7 @@ This library supports TOML version
 [v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
 
 [![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
-[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/goadesign/goa/blob/master/LICENSE)
+[![license](https://img.shields.io/github/license/pelletier/go-toml.svg)](https://github.com/pelletier/go-toml/blob/master/LICENSE)
 [![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
 [![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/pelletier/go-toml)](https://goreportcard.com/report/github.com/pelletier/go-toml)
@@ -16,65 +16,64 @@ This library supports TOML version
 Go-toml provides the following features for using data parsed from TOML documents:
 
 * Load TOML documents from files and string data
-* Easily navigate TOML structure using TomlTree
+* Easily navigate TOML structure using Tree
+* Mashaling and unmarshaling to and from data structures
 * Line & column position data for all parsed elements
-* Query support similar to JSON-Path
+* [Query support similar to JSON-Path](query/)
 * Syntax errors contain line and column numbers
 
-Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
-
-* Semantic evaluation of parsed TOML
-* Informing a user of mistakes in the source document, after it has been parsed
-* Programatic handling of default values on a case-by-case basis
-* Using a TOML document as a flexible data-store
-
 ## Import
 
-    import "github.com/pelletier/go-toml"
+```go
-
+import "github.com/pelletier/go-toml"
-## Usage
-
-### Example
-
-Say you have a TOML file that looks like this:
-
-```toml
-[postgres]
-user = "pelletier"
-password = "mypassword"
 ```
 
-Read the username and password like this:
+## Usage example
+
+Read a TOML document:
 
 ```go
-import (
+config, _ := toml.Load(`
-    "fmt"
+[postgres]
-    "github.com/pelletier/go-toml"
+user = "pelletier"
-)
+password = "mypassword"`)
+// retrieve data directly
+user := config.Get("postgres.user").(string)
 
-config, err := toml.LoadFile("config.toml")
+// or using an intermediate object
-if err != nil {
+postgresConfig := config.Get("postgres").(*toml.Tree)
-    fmt.Println("Error ", err.Error())
+password := postgresConfig.Get("password").(string)
-} else {
+```
-    // retrieve data directly
-    user := config.Get("postgres.user").(string)
-    password := config.Get("postgres.password").(string)
 
-    // or using an intermediate object
+Or use Unmarshal:
-    configTree := config.Get("postgres").(*toml.TomlTree)
-    user = configTree.Get("user").(string)
-    password = configTree.Get("password").(string)
-    fmt.Println("User is ", user, ". Password is ", password)
 
-    // show where elements are in the file
+```go
-    fmt.Println("User position: %v", configTree.GetPosition("user"))
+type Postgres struct {
-    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+    User     string
+    Password string
+}
+type Config struct {
+    Postgres Postgres
+}
 
-    // use a query to gather elements without walking the tree
+doc := []byte(`
-    results, _ := config.Query("$..[user,password]")
+[postgres]
-    for ii, item := range results.Values() {
+user = "pelletier"
-      fmt.Println("Query result %d: %v", ii, item)
+password = "mypassword"`)
-    }
+
+config := Config{}
+toml.Unmarshal(doc, &config)
+fmt.Println("user=", config.Postgres.User)
+```
+
+Or use a query:
+
+```go
+// use a query to gather elements without walking the tree
+q, _ := query.Compile("$..[user,password]")
+results := q.Execute(config)
+for ii, item := range results.Values() {
+    fmt.Println("Query result %d: %v", ii, item)
 }
 ```
 
@@ -96,7 +95,7 @@ Go-toml provides two handy command line tools:
 * `tomljson`: Reads a TOML file and outputs its JSON representation.
 
     ```
-    go install github.com/pelletier/go-toml/cmd/tomjson
+    go install github.com/pelletier/go-toml/cmd/tomljson
     tomljson --help
     ```
 

benchmark.json

@@ -0,0 +1,164 @@
+{
+    "array": {
+        "key1": [
+            1,
+            2,
+            3
+        ],
+        "key2": [
+            "red",
+            "yellow",
+            "green"
+        ],
+        "key3": [
+            [
+                1,
+                2
+            ],
+            [
+                3,
+                4,
+                5
+            ]
+        ],
+        "key4": [
+            [
+                1,
+                2
+            ],
+            [
+                "a",
+                "b",
+                "c"
+            ]
+        ],
+        "key5": [
+            1,
+            2,
+            3
+        ],
+        "key6": [
+            1,
+            2
+        ]
+    },
+    "boolean": {
+        "False": false,
+        "True": true
+    },
+    "datetime": {
+        "key1": "1979-05-27T07:32:00Z",
+        "key2": "1979-05-27T00:32:00-07:00",
+        "key3": "1979-05-27T00:32:00.999999-07:00"
+    },
+    "float": {
+        "both": {
+            "key": 6.626e-34
+        },
+        "exponent": {
+            "key1": 5e+22,
+            "key2": 1000000,
+            "key3": -0.02
+        },
+        "fractional": {
+            "key1": 1,
+            "key2": 3.1415,
+            "key3": -0.01
+        },
+        "underscores": {
+            "key1": 9224617.445991227,
+            "key2": 1e+100
+        }
+    },
+    "fruit": [{
+            "name": "apple",
+            "physical": {
+                "color": "red",
+                "shape": "round"
+            },
+            "variety": [{
+                    "name": "red delicious"
+                },
+                {
+                    "name": "granny smith"
+                }
+            ]
+        },
+        {
+            "name": "banana",
+            "variety": [{
+                "name": "plantain"
+            }]
+        }
+    ],
+    "integer": {
+        "key1": 99,
+        "key2": 42,
+        "key3": 0,
+        "key4": -17,
+        "underscores": {
+            "key1": 1000,
+            "key2": 5349221,
+            "key3": 12345
+        }
+    },
+    "products": [{
+            "name": "Hammer",
+            "sku": 738594937
+        },
+        {},
+        {
+            "color": "gray",
+            "name": "Nail",
+            "sku": 284758393
+        }
+    ],
+    "string": {
+        "basic": {
+            "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+        },
+        "literal": {
+            "multiline": {
+                "lines": "The first newline is\ntrimmed in raw strings.\n   All other whitespace\n   is preserved.\n",
+                "regex2": "I [dw]on't need \\d{2} apples"
+            },
+            "quoted": "Tom \"Dubs\" Preston-Werner",
+            "regex": "\u003c\\i\\c*\\s*\u003e",
+            "winpath": "C:\\Users\\nodejs\\templates",
+            "winpath2": "\\\\ServerX\\admin$\\system32\\"
+        },
+        "multiline": {
+            "continued": {
+                "key1": "The quick brown fox jumps over the lazy dog.",
+                "key2": "The quick brown fox jumps over the lazy dog.",
+                "key3": "The quick brown fox jumps over the lazy dog."
+            },
+            "key1": "One\nTwo",
+            "key2": "One\nTwo",
+            "key3": "One\nTwo"
+        }
+    },
+    "table": {
+        "inline": {
+            "name": {
+                "first": "Tom",
+                "last": "Preston-Werner"
+            },
+            "point": {
+                "x": 1,
+                "y": 2
+            }
+        },
+        "key": "value",
+        "subtable": {
+            "key": "another value"
+        }
+    },
+    "x": {
+        "y": {
+            "z": {
+                "w": {}
+            }
+        }
+    }
+}

benchmark.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+reference_ref=${1:-master}
+reference_git=${2:-.}
+
+if ! `hash benchstat 2>/dev/null`; then
+    echo "Installing benchstat"
+    go get golang.org/x/perf/cmd/benchstat
+    go install golang.org/x/perf/cmd/benchstat
+fi
+
+tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX`
+ref_tempdir="${tempdir}/ref"
+ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt"
+local_benchmark="`pwd`/benchmark-local.txt"
+
+echo "=== ${reference_ref} (${ref_tempdir})"
+git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null
+pushd ${ref_tempdir} >/dev/null
+git checkout ${reference_ref} >/dev/null 2>/dev/null
+go test -bench=. -benchmem | tee ${ref_benchmark}
+popd >/dev/null
+
+echo ""
+echo "=== local"
+go test -bench=. -benchmem  | tee ${local_benchmark}
+
+echo ""
+echo "=== diff"
+benchstat -delta-test=none ${ref_benchmark} ${local_benchmark}

benchmark.toml

@@ -0,0 +1,244 @@
+################################################################################
+## Comment
+
+# Speak your mind with the hash symbol. They go from the symbol to the end of
+# the line.
+
+
+################################################################################
+## Table
+
+# Tables (also known as hash tables or dictionaries) are collections of
+# key/value pairs. They appear in square brackets on a line by themselves.
+
+[table]
+
+key = "value" # Yeah, you can do this.
+
+# Nested tables are denoted by table names with dots in them. Name your tables
+# whatever crap you please, just don't use #, ., [ or ].
+
+[table.subtable]
+
+key = "another value"
+
+# You don't need to specify all the super-tables if you don't want to. TOML
+# knows how to do it for you.
+
+# [x] you
+# [x.y] don't
+# [x.y.z] need these
+[x.y.z.w] # for this to work
+
+
+################################################################################
+## Inline Table
+
+# Inline tables provide a more compact syntax for expressing tables. They are
+# especially useful for grouped data that can otherwise quickly become verbose.
+# Inline tables are enclosed in curly braces `{` and `}`. No newlines are
+# allowed between the curly braces unless they are valid within a value.
+
+[table.inline]
+
+name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }
+
+
+################################################################################
+## String
+
+# There are four ways to express strings: basic, multi-line basic, literal, and
+# multi-line literal. All strings must contain only valid UTF-8 characters.
+
+[string.basic]
+
+basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+[string.multiline]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "One\nTwo"
+key2 = """One\nTwo"""
+key3 = """
+One
+Two"""
+
+[string.multiline.continued]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "The quick brown fox jumps over the lazy dog."
+
+key2 = """
+The quick brown \
+
+
+  fox jumps over \
+    the lazy dog."""
+
+key3 = """\
+       The quick brown \
+       fox jumps over \
+       the lazy dog.\
+       """
+
+[string.literal]
+
+# What you see is what you get.
+winpath  = 'C:\Users\nodejs\templates'
+winpath2 = '\\ServerX\admin$\system32\'
+quoted   = 'Tom "Dubs" Preston-Werner'
+regex    = '<\i\c*\s*>'
+
+
+[string.literal.multiline]
+
+regex2 = '''I [dw]on't need \d{2} apples'''
+lines  = '''
+The first newline is
+trimmed in raw strings.
+   All other whitespace
+   is preserved.
+'''
+
+
+################################################################################
+## Integer
+
+# Integers are whole numbers. Positive numbers may be prefixed with a plus sign.
+# Negative numbers are prefixed with a minus sign.
+
+[integer]
+
+key1 = +99
+key2 = 42
+key3 = 0
+key4 = -17
+
+[integer.underscores]
+
+# For large numbers, you may use underscores to enhance readability. Each
+# underscore must be surrounded by at least one digit.
+key1 = 1_000
+key2 = 5_349_221
+key3 = 1_2_3_4_5     # valid but inadvisable
+
+
+################################################################################
+## Float
+
+# A float consists of an integer part (which may be prefixed with a plus or
+# minus sign) followed by a fractional part and/or an exponent part.
+
+[float.fractional]
+
+key1 = +1.0
+key2 = 3.1415
+key3 = -0.01
+
+[float.exponent]
+
+key1 = 5e+22
+key2 = 1e6
+key3 = -2E-2
+
+[float.both]
+
+key = 6.626e-34
+
+[float.underscores]
+
+key1 = 9_224_617.445_991_228_313
+key2 = 1e1_00
+
+
+################################################################################
+## Boolean
+
+# Booleans are just the tokens you're used to. Always lowercase.
+
+[boolean]
+
+True = true
+False = false
+
+
+################################################################################
+## Datetime
+
+# Datetimes are RFC 3339 dates.
+
+[datetime]
+
+key1 = 1979-05-27T07:32:00Z
+key2 = 1979-05-27T00:32:00-07:00
+key3 = 1979-05-27T00:32:00.999999-07:00
+
+
+################################################################################
+## Array
+
+# Arrays are square brackets with other primitives inside. Whitespace is
+# ignored. Elements are separated by commas. Data types may not be mixed.
+
+[array]
+
+key1 = [ 1, 2, 3 ]
+key2 = [ "red", "yellow", "green" ]
+key3 = [ [ 1, 2 ], [3, 4, 5] ]
+#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok
+
+# Arrays can also be multiline. So in addition to ignoring whitespace, arrays
+# also ignore newlines between the brackets.  Terminating commas are ok before
+# the closing bracket.
+
+key5 = [
+  1, 2, 3
+]
+key6 = [
+  1,
+  2, # this is ok
+]
+
+
+################################################################################
+## Array of Tables
+
+# These can be expressed by using a table name in double brackets. Each table
+# with the same double bracketed name will be an element in the array. The
+# tables are inserted in the order encountered.
+
+[[products]]
+
+name = "Hammer"
+sku = 738594937
+
+[[products]]
+
+[[products]]
+
+name = "Nail"
+sku = 284758393
+color = "gray"
+
+
+# You can create nested arrays of tables as well.
+
+[[fruit]]
+  name = "apple"
+
+  [fruit.physical]
+    color = "red"
+    shape = "round"
+
+  [[fruit.variety]]
+    name = "red delicious"
+
+  [[fruit.variety]]
+    name = "granny smith"
+
+[[fruit]]
+  name = "banana"
+
+  [[fruit.variety]]
+    name = "plantain"

benchmark.yml

@@ -0,0 +1,121 @@
+---
+array:
+  key1:
+  - 1
+  - 2
+  - 3
+  key2:
+  - red
+  - yellow
+  - green
+  key3:
+  - - 1
+    - 2
+  - - 3
+    - 4
+    - 5
+  key4:
+  - - 1
+    - 2
+  - - a
+    - b
+    - c
+  key5:
+  - 1
+  - 2
+  - 3
+  key6:
+  - 1
+  - 2
+boolean:
+  'False': false
+  'True': true
+datetime:
+  key1: '1979-05-27T07:32:00Z'
+  key2: '1979-05-27T00:32:00-07:00'
+  key3: '1979-05-27T00:32:00.999999-07:00'
+float:
+  both:
+    key: 6.626e-34
+  exponent:
+    key1: 5.0e+22
+    key2: 1000000
+    key3: -0.02
+  fractional:
+    key1: 1
+    key2: 3.1415
+    key3: -0.01
+  underscores:
+    key1: 9224617.445991227
+    key2: 1.0e+100
+fruit:
+- name: apple
+  physical:
+    color: red
+    shape: round
+  variety:
+  - name: red delicious
+  - name: granny smith
+- name: banana
+  variety:
+  - name: plantain
+integer:
+  key1: 99
+  key2: 42
+  key3: 0
+  key4: -17
+  underscores:
+    key1: 1000
+    key2: 5349221
+    key3: 12345
+products:
+- name: Hammer
+  sku: 738594937
+- {}
+- color: gray
+  name: Nail
+  sku: 284758393
+string:
+  basic:
+    basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+  literal:
+    multiline:
+      lines: |
+        The first newline is
+        trimmed in raw strings.
+           All other whitespace
+           is preserved.
+      regex2: I [dw]on't need \d{2} apples
+    quoted: Tom "Dubs" Preston-Werner
+    regex: "<\\i\\c*\\s*>"
+    winpath: C:\Users\nodejs\templates
+    winpath2: "\\\\ServerX\\admin$\\system32\\"
+  multiline:
+    continued:
+      key1: The quick brown fox jumps over the lazy dog.
+      key2: The quick brown fox jumps over the lazy dog.
+      key3: The quick brown fox jumps over the lazy dog.
+    key1: |-
+      One
+      Two
+    key2: |-
+      One
+      Two
+    key3: |-
+      One
+      Two
+table:
+  inline:
+    name:
+      first: Tom
+      last: Preston-Werner
+    point:
+      x: 1
+      y: 2
+  key: value
+  subtable:
+    key: another value
+x:
+  y:
+    z:
+      w: {}

benchmark_test.go

@@ -0,0 +1,192 @@
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	burntsushi "github.com/BurntSushi/toml"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type benchmarkDoc struct {
+	Table struct {
+		Key      string
+		Subtable struct {
+			Key string
+		}
+		Inline struct {
+			Name struct {
+				First string
+				Last  string
+			}
+			Point struct {
+				X int64
+				U int64
+			}
+		}
+	}
+	String struct {
+		Basic struct {
+			Basic string
+		}
+		Multiline struct {
+			Key1      string
+			Key2      string
+			Key3      string
+			Continued struct {
+				Key1 string
+				Key2 string
+				Key3 string
+			}
+		}
+		Literal struct {
+			Winpath   string
+			Winpath2  string
+			Quoted    string
+			Regex     string
+			Multiline struct {
+				Regex2 string
+				Lines  string
+			}
+		}
+	}
+	Integer struct {
+		Key1        int64
+		Key2        int64
+		Key3        int64
+		Key4        int64
+		Underscores struct {
+			Key1 int64
+			Key2 int64
+			Key3 int64
+		}
+	}
+	Float struct {
+		Fractional struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Exponent struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Both struct {
+			Key float64
+		}
+		Underscores struct {
+			Key1 float64
+			Key2 float64
+		}
+	}
+	Boolean struct {
+		True  bool
+		False bool
+	}
+	Datetime struct {
+		Key1 time.Time
+		Key2 time.Time
+		Key3 time.Time
+	}
+	Array struct {
+		Key1 []int64
+		Key2 []string
+		Key3 [][]int64
+		// TODO: Key4 not supported by go-toml's Unmarshal
+		Key5 []int64
+		Key6 []int64
+	}
+	Products []struct {
+		Name  string
+		Sku   int64
+		Color string
+	}
+	Fruit []struct {
+		Name     string
+		Physical struct {
+			Color   string
+			Shape   string
+			Variety []struct {
+				Name string
+			}
+		}
+	}
+}
+
+func BenchmarkParseToml(b *testing.B) {
+	fileBytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := LoadReader(bytes.NewReader(fileBytes))
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalBurntSushiToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := burntsushi.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalJson(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := json.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalYaml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.yml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := yaml.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

clean.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-# fail out of the script if anything here fails
-set -e
-
-# clear out stuff generated by test.sh
-rm -rf src test_program_bin toml-test

cmd/test_program.go

@@ -41,16 +41,16 @@ func translate(tomlData interface{}) interface{} {
 			typed[k] = translate(v)
 		}
 		return typed
-	case *toml.TomlTree:
+	case *toml.Tree:
 		return translate(*orig)
-	case toml.TomlTree:
+	case toml.Tree:
 		keys := orig.Keys()
 		typed := make(map[string]interface{}, len(keys))
 		for _, k := range keys {
 			typed[k] = translate(orig.GetPath([]string{k}))
 		}
 		return typed
-	case []*toml.TomlTree:
+	case []*toml.Tree:
 		typed := make([]map[string]interface{}, len(orig))
 		for i, v := range orig {
 			typed[i] = translate(v).(map[string]interface{})

cmd/tomljson/main.go

@@ -1,3 +1,8 @@
+// Tomljson reads TOML and converts to JSON.
+//
+// Usage:
+//   cat file.toml | tomljson > file.json
+//   tomljson file1.toml > file.json
 package main
 
 import (
@@ -57,7 +62,7 @@ func reader(r io.Reader) (string, error) {
 	return mapToJSON(tree)
 }
 
-func mapToJSON(tree *toml.TomlTree) (string, error) {
+func mapToJSON(tree *toml.Tree) (string, error) {
 	treeMap := tree.ToMap()
 	bytes, err := json.MarshalIndent(treeMap, "", "  ")
 	if err != nil {

cmd/tomll/main.go

@@ -1,3 +1,8 @@
+// Tomll is a linter for TOML
+//
+// Usage:
+//   cat file.toml | tomll > file_linted.toml
+//   tomll file1.toml file2.toml # lint the two files in place
 package main
 
 import (

doc.go

@@ -1,250 +1,23 @@
-// Package toml is a TOML markup language parser.
+// Package toml is a TOML parser and manipulation library.
 //
 // This version supports the specification as described in
 // https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md
 //
-// TOML Parsing
+// Marshaling
 //
-// TOML data may be parsed in two ways: by file, or by string.
+// Go-toml can marshal and unmarshal TOML documents from and to data
+// structures.
 //
-//   // load TOML data by filename
+// TOML document as a tree
-//   tree, err := toml.LoadFile("filename.toml")
 //
-//   // load TOML data stored in a string
+// Go-toml can operate on a TOML document as a tree. Use one of the Load*
-//   tree, err := toml.Load(stringContainingTomlData)
+// functions to parse TOML data and obtain a Tree instance, then one of its
+// methods to manipulate the tree.
 //
-// Either way, the result is a TomlTree object that can be used to navigate the
+// JSONPath-like queries
-// structure and data within the original document.
 //
-//
+// The package github.com/pelletier/go-toml/query implements a system
-// Getting data from the TomlTree
+// similar to JSONPath to quickly retrive elements of a TOML document using a
-//
+// single expression. See the package documentation for more information.
-// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
-// methods on the returned TomlTree, to find your way through the document data.
-//
-//   if tree.Has('foo') {
-//     fmt.Prinln("foo is: %v", tree.Get('foo'))
-//   }
-//
-// Working with Paths
-//
-// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
-// and GetDefault() methods.  These are the same kind of key paths used within the
-// TOML specification for struct tames.
-//
-//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
-//   tree.Has("foo.bar.baz")
-//
-//   // returns the key at this path, if it is there
-//   tree.Get("foo.bar.baz")
-//
-// TOML allows keys to contain '.', which can cause this syntax to be problematic
-// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
-// methods to explicitly define the path.  This form is also faster, since
-// it avoids having to parse the passed key for '.' delimiters.
-//
-//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
-//   tree.HasPath(string{}{"foo","bar","baz"})
-//
-//   // returns the key at this path, if it is there
-//   tree.GetPath(string{}{"foo","bar","baz"})
-//
-// Note that this is distinct from the heavyweight query syntax supported by
-// TomlTree.Query() and the Query() struct (see below).
-//
-// Position Support
-//
-// Each element within the TomlTree is stored with position metadata, which is
-// invaluable for providing semantic feedback to a user.  This helps in
-// situations where the TOML file parses correctly, but contains data that is
-// not correct for the application.  In such cases, an error message can be
-// generated that indicates the problem line and column number in the source
-// TOML document.
-//
-//   // load TOML data
-//   tree, _ := toml.Load("filename.toml")
-//
-//   // get an entry and report an error if it's the wrong type
-//   element := tree.Get("foo")
-//   if value, ok := element.(int64); !ok {
-//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
-//   }
-//
-//   // report an error if an expected element is missing
-//   if !tree.Has("bar") {
-//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
-//   }
-//
-// Query Support
-//
-// The TOML query path implementation is based loosely on the JSONPath specification:
-// http://goessner.net/articles/JsonPath/
-//
-// The idea behind a query path is to allow quick access to any element, or set
-// of elements within TOML document, with a single expression.
-//
-//   result, err := tree.Query("$.foo.bar.baz")
-//
-// This is roughly equivalent to:
-//
-//   next := tree.Get("foo")
-//   if next != nil {
-//     next = next.Get("bar")
-//     if next != nil {
-//       next = next.Get("baz")
-//     }
-//   }
-//   result := next
-//
-// err is nil if any parsing exception occurs.
-//
-// If no node in the tree matches the query, result will simply contain an empty list of
-// items.
-//
-// As illustrated above, the query path is much more efficient, especially since
-// the structure of the TOML file can vary.  Rather than making assumptions about
-// a document's structure, a query allows the programmer to make structured
-// requests into the document, and get zero or more values as a result.
-//
-// The syntax of a query begins with a root token, followed by any number
-// sub-expressions:
-//
-//   $
-//                    Root of the TOML tree.  This must always come first.
-//   .name
-//                    Selects child of this node, where 'name' is a TOML key
-//                    name.
-//   ['name']
-//                    Selects child of this node, where 'name' is a string
-//                    containing a TOML key name.
-//   [index]
-//                    Selcts child array element at 'index'.
-//   ..expr
-//                    Recursively selects all children, filtered by an a union,
-//                    index, or slice expression.
-//   ..*
-//                    Recursive selection of all nodes at this point in the
-//                    tree.
-//   .*
-//                    Selects all children of the current node.
-//   [expr,expr]
-//                    Union operator - a logical 'or' grouping of two or more
-//                    sub-expressions: index, key name, or filter.
-//   [start:end:step]
-//                    Slice operator - selects array elements from start to
-//                    end-1, at the given step.  All three arguments are
-//                    optional.
-//   [?(filter)]
-//                    Named filter expression - the function 'filter' is
-//                    used to filter children at this node.
-//
-// Query Indexes And Slices
-//
-// Index expressions perform no bounds checking, and will contribute no
-// values to the result set if the provided index or index range is invalid.
-// Negative indexes represent values from the end of the array, counting backwards.
-//
-//   // select the last index of the array named 'foo'
-//   tree.Query("$.foo[-1]")
-//
-// Slice expressions are supported, by using ':' to separate a start/end index pair.
-//
-//   // select up to the first five elements in the array
-//   tree.Query("$.foo[0:5]")
-//
-// Slice expressions also allow negative indexes for the start and stop
-// arguments.
-//
-//   // select all array elements.
-//   tree.Query("$.foo[0:-1]")
-//
-// Slice expressions may have an optional stride/step parameter:
-//
-//   // select every other element
-//   tree.Query("$.foo[0:-1:2]")
-//
-// Slice start and end parameters are also optional:
-//
-//   // these are all equivalent and select all the values in the array
-//   tree.Query("$.foo[:]")
-//   tree.Query("$.foo[0:]")
-//   tree.Query("$.foo[:-1]")
-//   tree.Query("$.foo[0:-1:]")
-//   tree.Query("$.foo[::1]")
-//   tree.Query("$.foo[0::1]")
-//   tree.Query("$.foo[:-1:1]")
-//   tree.Query("$.foo[0:-1:1]")
-//
-// Query Filters
-//
-// Query filters are used within a Union [,] or single Filter [] expression.
-// A filter only allows nodes that qualify through to the next expression,
-// and/or into the result set.
-//
-//   // returns children of foo that are permitted by the 'bar' filter.
-//   tree.Query("$.foo[?(bar)]")
-//
-// There are several filters provided with the library:
-//
-//   tree
-//          Allows nodes of type TomlTree.
-//   int
-//          Allows nodes of type int64.
-//   float
-//          Allows nodes of type float64.
-//   string
-//          Allows nodes of type string.
-//   time
-//          Allows nodes of type time.Time.
-//   bool
-//          Allows nodes of type bool.
-//
-// Query Results
-//
-// An executed query returns a QueryResult object.  This contains the nodes
-// in the TOML tree that qualify the query expression.  Position information
-// is also available for each value in the set.
-//
-//   // display the results of a query
-//   results := tree.Query("$.foo.bar.baz")
-//   for idx, value := results.Values() {
-//       fmt.Println("%v: %v", results.Positions()[idx], value)
-//   }
-//
-// Compiled Queries
-//
-// Queries may be executed directly on a TomlTree object, or compiled ahead
-// of time and executed discretely.  The former is more convienent, but has the
-// penalty of having to recompile the query expression each time.
-//
-//   // basic query
-//   results := tree.Query("$.foo.bar.baz")
-//
-//   // compiled query
-//   query := toml.CompileQuery("$.foo.bar.baz")
-//   results := query.Execute(tree)
-//
-//   // run the compiled query again on a different tree
-//   moreResults := query.Execute(anotherTree)
-//
-// User Defined Query Filters
-//
-// Filter expressions may also be user defined by using the SetFilter()
-// function on the Query object.  The function must return true/false, which
-// signifies if the passed node is kept or discarded, respectively.
-//
-//   // create a query that references a user-defined filter
-//   query, _ := CompileQuery("$[?(bazOnly)]")
-//
-//   // define the filter, and assign it to the query
-//   query.SetFilter("bazOnly", func(node interface{}) bool{
-//       if tree, ok := node.(*TomlTree); ok {
-//           return tree.Has("baz")
-//       }
-//       return false  // reject all other node types
-//   })
-//
-//   // run the query
-//   query.Execute(tree)
 //
 package toml

doc_test.go

@@ -1,59 +1,16 @@
 // code examples for godoc
 
-package toml
+package toml_test
 
 import (
 	"fmt"
+	"log"
+
+	toml "github.com/pelletier/go-toml"
 )
 
-func ExampleNodeFilterFn_filterExample() {
+func Example_tree() {
-	tree, _ := Load(`
+	config, err := toml.LoadFile("config.toml")
-      [struct_one]
-      foo = "foo"
-      bar = "bar"
-
-      [struct_two]
-      baz = "baz"
-      gorf = "gorf"
-    `)
-
-	// create a query that references a user-defined-filter
-	query, _ := CompileQuery("$[?(bazOnly)]")
-
-	// define the filter, and assign it to the query
-	query.SetFilter("bazOnly", func(node interface{}) bool {
-		if tree, ok := node.(*TomlTree); ok {
-			return tree.Has("baz")
-		}
-		return false // reject all other node types
-	})
-
-	// results contain only the 'struct_two' TomlTree
-	query.Execute(tree)
-}
-
-func ExampleQuery_queryExample() {
-	config, _ := Load(`
-      [[book]]
-      title = "The Stand"
-      author = "Stephen King"
-      [[book]]
-      title = "For Whom the Bell Tolls"
-      author = "Ernest Hemmingway"
-      [[book]]
-      title = "Neuromancer"
-      author = "William Gibson"
-    `)
-
-	// find and print all the authors in the document
-	authors, _ := config.Query("$.book.author")
-	for _, name := range authors.Values() {
-		fmt.Println(name)
-	}
-}
-
-func Example_comprehensiveExample() {
-	config, err := LoadFile("config.toml")
 
 	if err != nil {
 		fmt.Println("Error ", err.Error())
@@ -63,19 +20,81 @@ func Example_comprehensiveExample() {
 		password := config.Get("postgres.password").(string)
 
 		// or using an intermediate object
-		configTree := config.Get("postgres").(*TomlTree)
+		configTree := config.Get("postgres").(*toml.Tree)
 		user = configTree.Get("user").(string)
 		password = configTree.Get("password").(string)
-		fmt.Println("User is ", user, ". Password is ", password)
+		fmt.Println("User is", user, " and password is", password)
 
 		// show where elements are in the file
 		fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
 		fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
-
-		// use a query to gather elements without walking the tree
-		results, _ := config.Query("$..[user,password]")
-		for ii, item := range results.Values() {
-			fmt.Printf("Query result %d: %v\n", ii, item)
-		}
 	}
 }
+
+func Example_unmarshal() {
+	type Employer struct {
+		Name  string
+		Phone string
+	}
+	type Person struct {
+		Name     string
+		Age      int64
+		Employer Employer
+	}
+
+	document := []byte(`
+	name = "John"
+	age = 30
+	[employer]
+		name = "Company Inc."
+		phone = "+1 234 567 89012"
+	`)
+
+	person := Person{}
+	toml.Unmarshal(document, &person)
+	fmt.Println(person.Name, "is", person.Age, "and works at", person.Employer.Name)
+	// Output:
+	// John is 30 and works at Company Inc.
+}
+
+func ExampleMarshal() {
+	type Postgres struct {
+		User     string `toml:"user"`
+		Password string `toml:"password"`
+	}
+	type Config struct {
+		Postgres Postgres `toml:"postgres"`
+	}
+
+	config := Config{Postgres{User: "pelletier", Password: "mypassword"}}
+	b, err := toml.Marshal(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b))
+	// Output:
+	// [postgres]
+	//   password = "mypassword"
+	//   user = "pelletier"
+}
+
+func ExampleUnmarshal() {
+	type Postgres struct {
+		User     string
+		Password string
+	}
+	type Config struct {
+		Postgres Postgres
+	}
+
+	doc := []byte(`
+	[postgres]
+	user = "pelletier"
+	password = "mypassword"`)
+
+	config := Config{}
+	toml.Unmarshal(doc, &config)
+	fmt.Println("user=", config.Postgres.User)
+	// Output:
+	// user= pelletier
+}

keysparsing.go

@@ -4,6 +4,7 @@ package toml
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"unicode"
 )
@@ -47,7 +48,7 @@ func parseKey(key string) ([]string, error) {
 			} else {
 				if !wasInQuotes {
 					if buffer.Len() == 0 {
-						return nil, fmt.Errorf("empty key group")
+						return nil, errors.New("empty table key")
 					}
 					groups = append(groups, buffer.String())
 					buffer.Reset()
@@ -67,23 +68,23 @@ func parseKey(key string) ([]string, error) {
 				return nil, fmt.Errorf("invalid bare character: %c", char)
 			}
 			if !inQuotes && expectDot {
-				return nil, fmt.Errorf("what?")
+				return nil, errors.New("what?")
 			}
 			buffer.WriteRune(char)
 			expectDot = false
 		}
 	}
 	if inQuotes {
-		return nil, fmt.Errorf("mismatched quotes")
+		return nil, errors.New("mismatched quotes")
 	}
 	if escapeNext {
-		return nil, fmt.Errorf("unfinished escape sequence")
+		return nil, errors.New("unfinished escape sequence")
 	}
 	if buffer.Len() > 0 {
 		groups = append(groups, buffer.String())
 	}
 	if len(groups) == 0 {
-		return nil, fmt.Errorf("empty key")
+		return nil, errors.New("empty key")
 	}
 	return groups, nil
 }

lexer.go

@@ -6,14 +6,12 @@
 package toml
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"regexp"
 	"strconv"
 	"strings"
-
-	"github.com/pelletier/go-buffruneio"
 )
 
 var dateRegexp *regexp.Regexp
@@ -23,29 +21,29 @@ type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
 type tomlLexer struct {
-	input         *buffruneio.Reader // Textual source
+	inputIdx          int
-	buffer        []rune             // Runes composing the current token
+	input             []rune // Textual source
-	tokens        chan token
+	currentTokenStart int
-	depth         int
+	currentTokenStop  int
-	line          int
+	tokens            []token
-	col           int
+	depth             int
-	endbufferLine int
+	line              int
-	endbufferCol  int
+	col               int
+	endbufferLine     int
+	endbufferCol      int
 }
 
 // Basic read operations on input
 
 func (l *tomlLexer) read() rune {
-	r, err := l.input.ReadRune()
+	r := l.peek()
-	if err != nil {
-		panic(err)
-	}
 	if r == '\n' {
 		l.endbufferLine++
 		l.endbufferCol = 1
 	} else {
 		l.endbufferCol++
 	}
+	l.inputIdx++
 	return r
 }
 
@@ -53,13 +51,13 @@ func (l *tomlLexer) next() rune {
 	r := l.read()
 
 	if r != eof {
-		l.buffer = append(l.buffer, r)
+		l.currentTokenStop++
 	}
 	return r
 }
 
 func (l *tomlLexer) ignore() {
-	l.buffer = make([]rune, 0)
+	l.currentTokenStart = l.currentTokenStop
 	l.line = l.endbufferLine
 	l.col = l.endbufferCol
 }
@@ -76,49 +74,46 @@ func (l *tomlLexer) fastForward(n int) {
 }
 
 func (l *tomlLexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{
+	l.tokens = append(l.tokens, token{
 		Position: Position{l.line, l.col},
 		typ:      t,
 		val:      value,
-	}
+	})
 	l.ignore()
 }
 
 func (l *tomlLexer) emit(t tokenType) {
-	l.emitWithValue(t, string(l.buffer))
+	l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop]))
 }
 
 func (l *tomlLexer) peek() rune {
-	r, err := l.input.ReadRune()
+	if l.inputIdx >= len(l.input) {
-	if err != nil {
+		return eof
-		panic(err)
 	}
-	l.input.UnreadRune()
+	return l.input[l.inputIdx]
-	return r
+}
+
+func (l *tomlLexer) peekString(size int) string {
+	maxIdx := len(l.input)
+	upperIdx := l.inputIdx + size // FIXME: potential overflow
+	if upperIdx > maxIdx {
+		upperIdx = maxIdx
+	}
+	return string(l.input[l.inputIdx:upperIdx])
 }
 
 func (l *tomlLexer) follow(next string) bool {
-	for _, expectedRune := range next {
+	return next == l.peekString(len(next))
-		r, err := l.input.ReadRune()
-		defer l.input.UnreadRune()
-		if err != nil {
-			panic(err)
-		}
-		if expectedRune != r {
-			return false
-		}
-	}
-	return true
 }
 
 // Error management
 
 func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
-	l.tokens <- token{
+	l.tokens = append(l.tokens, token{
 		Position: Position{l.line, l.col},
 		typ:      tokenError,
 		val:      fmt.Sprintf(format, args...),
-	}
+	})
 	return nil
 }
 
@@ -129,9 +124,9 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 		next := l.peek()
 		switch next {
 		case '[':
-			return l.lexKeyGroup
+			return l.lexTableKey
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexVoid)
 		case '=':
 			return l.lexEqual
 		case '\r':
@@ -182,7 +177,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 		case '}':
 			return l.lexRightCurlyBrace
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexRvalue)
 		case '"':
 			return l.lexString
 		case '\'':
@@ -219,7 +214,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			break
 		}
 
-		possibleDate := string(l.input.Peek(35))
+		possibleDate := l.peekString(35)
 		dateMatch := dateRegexp.FindString(possibleDate)
 		if dateMatch != "" {
 			l.fastForward(len(dateMatch))
@@ -309,15 +304,17 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 	return l.lexVoid
 }
 
-func (l *tomlLexer) lexComment() tomlLexStateFn {
+func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn {
-	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+	return func() tomlLexStateFn {
-		if next == '\r' && l.follow("\r\n") {
+		for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
-			break
+			if next == '\r' && l.follow("\r\n") {
+				break
+			}
+			l.next()
 		}
-		l.next()
+		l.ignore()
+		return previousState
 	}
-	l.ignore()
-	return l.lexVoid
 }
 
 func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
@@ -516,25 +513,25 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 	return l.lexRvalue
 }
 
-func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
+func (l *tomlLexer) lexTableKey() tomlLexStateFn {
 	l.next()
 
 	if l.peek() == '[' {
-		// token '[[' signifies an array of anonymous key groups
+		// token '[[' signifies an array of tables
 		l.next()
 		l.emit(tokenDoubleLeftBracket)
-		return l.lexInsideKeyGroupArray
+		return l.lexInsideTableArrayKey
 	}
-	// vanilla key group
+	// vanilla table key
 	l.emit(tokenLeftBracket)
-	return l.lexInsideKeyGroup
+	return l.lexInsideTableKey
 }
 
-func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
+func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn {
 	for r := l.peek(); r != eof; r = l.peek() {
 		switch r {
 		case ']':
-			if len(l.buffer) > 0 {
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroupArray)
 			}
 			l.next()
@@ -545,31 +542,31 @@ func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
 			l.emit(tokenDoubleRightBracket)
 			return l.lexVoid
 		case '[':
-			return l.errorf("group name cannot contain ']'")
+			return l.errorf("table array key cannot contain ']'")
 		default:
 			l.next()
 		}
 	}
-	return l.errorf("unclosed key group array")
+	return l.errorf("unclosed table array key")
 }
 
-func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
+func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn {
 	for r := l.peek(); r != eof; r = l.peek() {
 		switch r {
 		case ']':
-			if len(l.buffer) > 0 {
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroup)
 			}
 			l.next()
 			l.emit(tokenRightBracket)
 			return l.lexVoid
 		case '[':
-			return l.errorf("group name cannot contain ']'")
+			return l.errorf("table key cannot contain ']'")
 		default:
 			l.next()
 		}
 	}
-	return l.errorf("unclosed key group")
+	return l.errorf("unclosed table key")
 }
 
 func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
@@ -632,7 +629,6 @@ func (l *tomlLexer) run() {
 	for state := l.lexVoid; state != nil; {
 		state = state()
 	}
-	close(l.tokens)
 }
 
 func init() {
@@ -640,16 +636,16 @@ func init() {
 }
 
 // Entry point
-func lexToml(input io.Reader) chan token {
+func lexToml(inputBytes []byte) []token {
-	bufferedInput := buffruneio.NewReader(input)
+	runes := bytes.Runes(inputBytes)
 	l := &tomlLexer{
-		input:         bufferedInput,
+		input:         runes,
-		tokens:        make(chan token),
+		tokens:        make([]token, 0, 256),
 		line:          1,
 		col:           1,
 		endbufferLine: 1,
 		endbufferCol:  1,
 	}
-	go l.run()
+	l.run()
 	return l.tokens
 }

lexer_test.go

@@ -1,37 +1,14 @@
 package toml
 
 import (
-	"strings"
+	"reflect"
 	"testing"
 )
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	ch := lexToml(strings.NewReader(input))
+	tokens := lexToml([]byte(input))
-	for _, expected := range expectedFlow {
+	if !reflect.DeepEqual(tokens, expectedFlow) {
-		token := <-ch
+		t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens)
-		if token != expected {
-			t.Log("While testing: ", input)
-			t.Log("compared (got)", token, "to (expected)", expected)
-			t.Log("\tvalue:", token.val, "<->", expected.val)
-			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
-			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
-			t.Log("\tline:", token.Line, "<->", expected.Line)
-			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
-			t.Log("compared", token, "to", expected)
-			t.FailNow()
-		}
-	}
-
-	tok, ok := <-ch
-	if ok {
-		t.Log("channel is not closed!")
-		t.Log(len(ch)+1, "tokens remaining:")
-
-		t.Log("token ->", tok)
-		for token := range ch {
-			t.Log("token ->", token)
-		}
-		t.FailNow()
 	}
 }
 
@@ -56,7 +33,7 @@ func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
 func TestUnclosedKeyGroup(t *testing.T) {
 	testFlow(t, "[hello world", []token{
 		{Position{1, 1}, tokenLeftBracket, "["},
-		{Position{1, 2}, tokenError, "unclosed key group"},
+		{Position{1, 2}, tokenError, "unclosed table key"},
 	})
 }
 
@@ -264,6 +241,24 @@ func TestMultilineArrayComments(t *testing.T) {
 	})
 }
 
+func TestNestedArraysComment(t *testing.T) {
+	toml := `
+someArray = [
+# does not work
+["entry1"]
+]`
+	testFlow(t, toml, []token{
+		{Position{2, 1}, tokenKey, "someArray"},
+		{Position{2, 11}, tokenEqual, "="},
+		{Position{2, 13}, tokenLeftBracket, "["},
+		{Position{4, 1}, tokenLeftBracket, "["},
+		{Position{4, 3}, tokenString, "entry1"},
+		{Position{4, 10}, tokenRightBracket, "]"},
+		{Position{5, 1}, tokenRightBracket, "]"},
+		{Position{5, 2}, tokenEOF, ""},
+	})
+}
+
 func TestKeyEqualArrayBools(t *testing.T) {
 	testFlow(t, "foo = [true, false, true]", []token{
 		{Position{1, 1}, tokenKey, "foo"},
@@ -513,6 +508,30 @@ func TestKeyEqualStringUnicodeEscape(t *testing.T) {
 		{Position{1, 8}, tokenString, "hello δ"},
 		{Position{1, 25}, tokenEOF, ""},
 	})
+	testFlow(t, `foo = "\uabcd"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "\uabcd"},
+		{Position{1, 15}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "\uABCD"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "\uABCD"},
+		{Position{1, 15}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "\U000bcdef"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "\U000bcdef"},
+		{Position{1, 19}, tokenEOF, ""},
+	})
+	testFlow(t, `foo = "\U000BCDEF"`, []token{
+		{Position{1, 1}, tokenKey, "foo"},
+		{Position{1, 5}, tokenEqual, "="},
+		{Position{1, 8}, tokenString, "\U000BCDEF"},
+		{Position{1, 19}, tokenEOF, ""},
+	})
 	testFlow(t, `foo = "\u2"`, []token{
 		{Position{1, 1}, tokenKey, "foo"},
 		{Position{1, 5}, tokenEqual, "="},
@@ -706,3 +725,26 @@ func TestLexUnknownRvalue(t *testing.T) {
 		{Position{1, 5}, tokenError, `no value can start with \`},
 	})
 }
+
+func BenchmarkLexer(b *testing.B) {
+	sample := `title = "Hugo: A Fast and Flexible Website Generator"
+baseurl = "http://gohugo.io/"
+MetaDataFormat = "yaml"
+pluralizeListTitles = false
+
+[params]
+  description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go"
+  author = "Steve Francia (spf13) and friends"
+  release = "0.22-DEV"
+
+[[menu.main]]
+	name = "Download Hugo"
+	pre = "<i class='fa fa-download'></i>"
+	url = "https://github.com/spf13/hugo/releases"
+	weight = -200
+`
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		lexToml([]byte(sample))
+	}
+}

marshal.go

@@ -0,0 +1,489 @@
+package toml
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+)
+
+type tomlOpts struct {
+	name      string
+	include   bool
+	omitempty bool
+}
+
+var timeType = reflect.TypeOf(time.Time{})
+var marshalerType = reflect.TypeOf(new(Marshaler)).Elem()
+
+// Check if the given marshall type maps to a Tree primitive
+func isPrimitive(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Ptr:
+		return isPrimitive(mtype.Elem())
+	case reflect.Bool:
+		return true
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return true
+	case reflect.Float32, reflect.Float64:
+		return true
+	case reflect.String:
+		return true
+	case reflect.Struct:
+		return mtype == timeType || isCustomMarshaler(mtype)
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a Tree slice
+func isTreeSlice(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Slice:
+		return !isOtherSlice(mtype)
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a non-Tree slice
+func isOtherSlice(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Ptr:
+		return isOtherSlice(mtype.Elem())
+	case reflect.Slice:
+		return isPrimitive(mtype.Elem()) || isOtherSlice(mtype.Elem())
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a Tree
+func isTree(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Map:
+		return true
+	case reflect.Struct:
+		return !isPrimitive(mtype)
+	default:
+		return false
+	}
+}
+
+func isCustomMarshaler(mtype reflect.Type) bool {
+	return mtype.Implements(marshalerType)
+}
+
+func callCustomMarshaler(mval reflect.Value) ([]byte, error) {
+	return mval.Interface().(Marshaler).MarshalTOML()
+}
+
+// Marshaler is the interface implemented by types that
+// can marshal themselves into valid TOML.
+type Marshaler interface {
+	MarshalTOML() ([]byte, error)
+}
+
+/*
+Marshal returns the TOML encoding of v.  Behavior is similar to the Go json
+encoder, except that there is no concept of a Marshaler interface or MarshalTOML
+function for sub-structs, and currently only definite types can be marshaled
+(i.e. no `interface{}`).
+
+Note that pointers are automatically assigned the "omitempty" option, as TOML
+explicity does not handle null values (saying instead the label should be
+dropped).
+
+Tree structural types and corresponding marshal types:
+
+  *Tree                            (*)struct, (*)map[string]interface{}
+  []*Tree                          (*)[](*)struct, (*)[](*)map[string]interface{}
+  []interface{} (as interface{})   (*)[]primitive, (*)[]([]interface{})
+  interface{}                      (*)primitive
+
+Tree primitive types and corresponding marshal types:
+
+  uint64     uint, uint8-uint64, pointers to same
+  int64      int, int8-uint64, pointers to same
+  float64    float32, float64, pointers to same
+  string     string, pointers to same
+  bool       bool, pointers to same
+  time.Time  time.Time{}, pointers to same
+*/
+func Marshal(v interface{}) ([]byte, error) {
+	mtype := reflect.TypeOf(v)
+	if mtype.Kind() != reflect.Struct {
+		return []byte{}, errors.New("Only a struct can be marshaled to TOML")
+	}
+	sval := reflect.ValueOf(v)
+	if isCustomMarshaler(mtype) {
+		return callCustomMarshaler(sval)
+	}
+	t, err := valueToTree(mtype, sval)
+	if err != nil {
+		return []byte{}, err
+	}
+	s, err := t.ToTomlString()
+	return []byte(s), err
+}
+
+// Convert given marshal struct or map value to toml tree
+func valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return valueToTree(mtype.Elem(), mval.Elem())
+	}
+	tval := newTree()
+	switch mtype.Kind() {
+	case reflect.Struct:
+		for i := 0; i < mtype.NumField(); i++ {
+			mtypef, mvalf := mtype.Field(i), mval.Field(i)
+			opts := tomlOptions(mtypef)
+			if opts.include && (!opts.omitempty || !isZero(mvalf)) {
+				val, err := valueToToml(mtypef.Type, mvalf)
+				if err != nil {
+					return nil, err
+				}
+				tval.Set(opts.name, val)
+			}
+		}
+	case reflect.Map:
+		for _, key := range mval.MapKeys() {
+			mvalf := mval.MapIndex(key)
+			val, err := valueToToml(mtype.Elem(), mvalf)
+			if err != nil {
+				return nil, err
+			}
+			tval.Set(key.String(), val)
+		}
+	}
+	return tval, nil
+}
+
+// Convert given marshal slice to slice of Toml trees
+func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) {
+	tval := make([]*Tree, mval.Len(), mval.Len())
+	for i := 0; i < mval.Len(); i++ {
+		val, err := valueToTree(mtype.Elem(), mval.Index(i))
+		if err != nil {
+			return nil, err
+		}
+		tval[i] = val
+	}
+	return tval, nil
+}
+
+// Convert given marshal slice to slice of toml values
+func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+	tval := make([]interface{}, mval.Len(), mval.Len())
+	for i := 0; i < mval.Len(); i++ {
+		val, err := valueToToml(mtype.Elem(), mval.Index(i))
+		if err != nil {
+			return nil, err
+		}
+		tval[i] = val
+	}
+	return tval, nil
+}
+
+// Convert given marshal value to toml value
+func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return valueToToml(mtype.Elem(), mval.Elem())
+	}
+	switch {
+	case isCustomMarshaler(mtype):
+		return callCustomMarshaler(mval)
+	case isTree(mtype):
+		return valueToTree(mtype, mval)
+	case isTreeSlice(mtype):
+		return valueToTreeSlice(mtype, mval)
+	case isOtherSlice(mtype):
+		return valueToOtherSlice(mtype, mval)
+	default:
+		switch mtype.Kind() {
+		case reflect.Bool:
+			return mval.Bool(), nil
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			return mval.Int(), nil
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			return mval.Uint(), nil
+		case reflect.Float32, reflect.Float64:
+			return mval.Float(), nil
+		case reflect.String:
+			return mval.String(), nil
+		case reflect.Struct:
+			return mval.Interface().(time.Time), nil
+		default:
+			return nil, fmt.Errorf("Marshal can't handle %v(%v)", mtype, mtype.Kind())
+		}
+	}
+}
+
+// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v.
+// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for
+// sub-structs, and only definite types can be unmarshaled.
+func (t *Tree) Unmarshal(v interface{}) error {
+	mtype := reflect.TypeOf(v)
+	if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct {
+		return errors.New("Only a pointer to struct can be unmarshaled from TOML")
+	}
+
+	sval, err := valueFromTree(mtype.Elem(), t)
+	if err != nil {
+		return err
+	}
+	reflect.ValueOf(v).Elem().Set(sval)
+	return nil
+}
+
+// Unmarshal parses the TOML-encoded data and stores the result in the value
+// pointed to by v. Behavior is similar to the Go json encoder, except that there
+// is no concept of an Unmarshaler interface or UnmarshalTOML function for
+// sub-structs, and currently only definite types can be unmarshaled to (i.e. no
+// `interface{}`).
+//
+// See Marshal() documentation for types mapping table.
+func Unmarshal(data []byte, v interface{}) error {
+	t, err := LoadReader(bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	return t.Unmarshal(v)
+}
+
+// Convert toml tree to marshal struct or map, using marshal type
+func valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return unwrapPointer(mtype, tval)
+	}
+	var mval reflect.Value
+	switch mtype.Kind() {
+	case reflect.Struct:
+		mval = reflect.New(mtype).Elem()
+		for i := 0; i < mtype.NumField(); i++ {
+			mtypef := mtype.Field(i)
+			opts := tomlOptions(mtypef)
+			if opts.include {
+				baseKey := opts.name
+				keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)}
+				for _, key := range keysToTry {
+					exists := tval.Has(key)
+					if !exists {
+						continue
+					}
+					val := tval.Get(key)
+					mvalf, err := valueFromToml(mtypef.Type, val)
+					if err != nil {
+						return mval, formatError(err, tval.GetPosition(key))
+					}
+					mval.Field(i).Set(mvalf)
+					break
+				}
+			}
+		}
+	case reflect.Map:
+		mval = reflect.MakeMap(mtype)
+		for _, key := range tval.Keys() {
+			val := tval.Get(key)
+			mvalf, err := valueFromToml(mtype.Elem(), val)
+			if err != nil {
+				return mval, formatError(err, tval.GetPosition(key))
+			}
+			mval.SetMapIndex(reflect.ValueOf(key), mvalf)
+		}
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal struct/map slice, using marshal type
+func valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) {
+	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
+	for i := 0; i < len(tval); i++ {
+		val, err := valueFromTree(mtype.Elem(), tval[i])
+		if err != nil {
+			return mval, err
+		}
+		mval.Index(i).Set(val)
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal primitive slice, using marshal type
+func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) {
+	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
+	for i := 0; i < len(tval); i++ {
+		val, err := valueFromToml(mtype.Elem(), tval[i])
+		if err != nil {
+			return mval, err
+		}
+		mval.Index(i).Set(val)
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal value, using marshal type
+func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return unwrapPointer(mtype, tval)
+	}
+	switch {
+	case isTree(mtype):
+		return valueFromTree(mtype, tval.(*Tree))
+	case isTreeSlice(mtype):
+		return valueFromTreeSlice(mtype, tval.([]*Tree))
+	case isOtherSlice(mtype):
+		return valueFromOtherSlice(mtype, tval.([]interface{}))
+	default:
+		switch mtype.Kind() {
+		case reflect.Bool:
+			val, ok := tval.(bool)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to bool", tval, tval)
+			}
+			return reflect.ValueOf(val), nil
+		case reflect.Int:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
+			}
+			return reflect.ValueOf(int(val)), nil
+		case reflect.Int8:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
+			}
+			return reflect.ValueOf(int8(val)), nil
+		case reflect.Int16:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
+			}
+			return reflect.ValueOf(int16(val)), nil
+		case reflect.Int32:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
+			}
+			return reflect.ValueOf(int32(val)), nil
+		case reflect.Int64:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
+			}
+			return reflect.ValueOf(val), nil
+		case reflect.Uint:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
+			}
+			return reflect.ValueOf(uint(val)), nil
+		case reflect.Uint8:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
+			}
+			return reflect.ValueOf(uint8(val)), nil
+		case reflect.Uint16:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
+			}
+			return reflect.ValueOf(uint16(val)), nil
+		case reflect.Uint32:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
+			}
+			return reflect.ValueOf(uint32(val)), nil
+		case reflect.Uint64:
+			val, ok := tval.(int64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
+			}
+			return reflect.ValueOf(uint64(val)), nil
+		case reflect.Float32:
+			val, ok := tval.(float64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval)
+			}
+			return reflect.ValueOf(float32(val)), nil
+		case reflect.Float64:
+			val, ok := tval.(float64)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval)
+			}
+			return reflect.ValueOf(val), nil
+		case reflect.String:
+			val, ok := tval.(string)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to string", tval, tval)
+			}
+			return reflect.ValueOf(val), nil
+		case reflect.Struct:
+			val, ok := tval.(time.Time)
+			if !ok {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to time", tval, tval)
+			}
+			return reflect.ValueOf(val), nil
+		default:
+			return reflect.ValueOf(nil), fmt.Errorf("Unmarshal can't handle %v(%v)", mtype, mtype.Kind())
+		}
+	}
+}
+
+func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+	val, err := valueFromToml(mtype.Elem(), tval)
+	if err != nil {
+		return reflect.ValueOf(nil), err
+	}
+	mval := reflect.New(mtype.Elem())
+	mval.Elem().Set(val)
+	return mval, nil
+}
+
+func tomlOptions(vf reflect.StructField) tomlOpts {
+	tag := vf.Tag.Get("toml")
+	parse := strings.Split(tag, ",")
+	result := tomlOpts{vf.Name, true, false}
+	if parse[0] != "" {
+		if parse[0] == "-" && len(parse) == 1 {
+			result.include = false
+		} else {
+			result.name = strings.Trim(parse[0], " ")
+		}
+	}
+	if vf.PkgPath != "" {
+		result.include = false
+	}
+	if len(parse) > 1 && strings.Trim(parse[1], " ") == "omitempty" {
+		result.omitempty = true
+	}
+	if vf.Type.Kind() == reflect.Ptr {
+		result.omitempty = true
+	}
+	return result
+}
+
+func isZero(val reflect.Value) bool {
+	switch val.Type().Kind() {
+	case reflect.Map:
+		fallthrough
+	case reflect.Array:
+		fallthrough
+	case reflect.Slice:
+		return val.Len() == 0
+	default:
+		return reflect.DeepEqual(val.Interface(), reflect.Zero(val.Type()).Interface())
+	}
+}
+
+func formatError(err error, pos Position) error {
+	if err.Error()[0] == '(' { // Error already contains position information
+		return err
+	}
+	return fmt.Errorf("%s: %s", pos, err)
+}

marshal_test.go

@@ -0,0 +1,600 @@
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+	"time"
+)
+
+type basicMarshalTestStruct struct {
+	String     string                      `toml:"string"`
+	StringList []string                    `toml:"strlist"`
+	Sub        basicMarshalTestSubStruct   `toml:"subdoc"`
+	SubList    []basicMarshalTestSubStruct `toml:"sublist"`
+}
+
+type basicMarshalTestSubStruct struct {
+	String2 string
+}
+
+var basicTestData = basicMarshalTestStruct{
+	String:     "Hello",
+	StringList: []string{"Howdy", "Hey There"},
+	Sub:        basicMarshalTestSubStruct{"One"},
+	SubList:    []basicMarshalTestSubStruct{{"Two"}, {"Three"}},
+}
+
+var basicTestToml = []byte(`string = "Hello"
+strlist = ["Howdy","Hey There"]
+
+[subdoc]
+  String2 = "One"
+
+[[sublist]]
+  String2 = "Two"
+
+[[sublist]]
+  String2 = "Three"
+`)
+
+func TestBasicMarshal(t *testing.T) {
+	result, err := Marshal(basicTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := basicTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestBasicUnmarshal(t *testing.T) {
+	result := basicMarshalTestStruct{}
+	err := Unmarshal(basicTestToml, &result)
+	expected := basicTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type testDoc struct {
+	Title       string            `toml:"title"`
+	Basics      testDocBasics     `toml:"basic"`
+	BasicLists  testDocBasicLists `toml:"basic_lists"`
+	BasicMap    map[string]string `toml:"basic_map"`
+	Subdocs     testDocSubs       `toml:"subdoc"`
+	SubDocList  []testSubDoc      `toml:"subdoclist"`
+	SubDocPtrs  []*testSubDoc     `toml:"subdocptrs"`
+	err         int               `toml:"shouldntBeHere"`
+	unexported  int               `toml:"shouldntBeHere"`
+	Unexported2 int               `toml:"-"`
+}
+
+type testDocBasics struct {
+	Bool       bool      `toml:"bool"`
+	Date       time.Time `toml:"date"`
+	Float      float32   `toml:"float"`
+	Int        int       `toml:"int"`
+	Uint       uint      `toml:"uint"`
+	String     *string   `toml:"string"`
+	unexported int       `toml:"shouldntBeHere"`
+}
+
+type testDocBasicLists struct {
+	Bools   []bool      `toml:"bools"`
+	Dates   []time.Time `toml:"dates"`
+	Floats  []*float32  `toml:"floats"`
+	Ints    []int       `toml:"ints"`
+	Strings []string    `toml:"strings"`
+	UInts   []uint      `toml:"uints"`
+}
+
+type testDocSubs struct {
+	First  testSubDoc  `toml:"first"`
+	Second *testSubDoc `toml:"second"`
+}
+
+type testSubDoc struct {
+	Name       string `toml:"name"`
+	unexported int    `toml:"shouldntBeHere"`
+}
+
+var biteMe = "Bite me"
+var float1 float32 = 12.3
+var float2 float32 = 45.6
+var float3 float32 = 78.9
+var subdoc = testSubDoc{"Second", 0}
+
+var docData = testDoc{
+	Title:       "TOML Marshal Testing",
+	unexported:  0,
+	Unexported2: 0,
+	Basics: testDocBasics{
+		Bool:       true,
+		Date:       time.Date(1979, 5, 27, 7, 32, 0, 0, time.UTC),
+		Float:      123.4,
+		Int:        5000,
+		Uint:       5001,
+		String:     &biteMe,
+		unexported: 0,
+	},
+	BasicLists: testDocBasicLists{
+		Bools: []bool{true, false, true},
+		Dates: []time.Time{
+			time.Date(1979, 5, 27, 7, 32, 0, 0, time.UTC),
+			time.Date(1980, 5, 27, 7, 32, 0, 0, time.UTC),
+		},
+		Floats:  []*float32{&float1, &float2, &float3},
+		Ints:    []int{8001, 8001, 8002},
+		Strings: []string{"One", "Two", "Three"},
+		UInts:   []uint{5002, 5003},
+	},
+	BasicMap: map[string]string{
+		"one": "one",
+		"two": "two",
+	},
+	Subdocs: testDocSubs{
+		First:  testSubDoc{"First", 0},
+		Second: &subdoc,
+	},
+	SubDocList: []testSubDoc{
+		testSubDoc{"List.First", 0},
+		testSubDoc{"List.Second", 0},
+	},
+	SubDocPtrs: []*testSubDoc{&subdoc},
+}
+
+func TestDocMarshal(t *testing.T) {
+	result, err := Marshal(docData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected, _ := ioutil.ReadFile("marshal_test.toml")
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestDocUnmarshal(t *testing.T) {
+	result := testDoc{}
+	tomlData, _ := ioutil.ReadFile("marshal_test.toml")
+	err := Unmarshal(tomlData, &result)
+	expected := docData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		resStr, _ := json.MarshalIndent(result, "", "  ")
+		expStr, _ := json.MarshalIndent(expected, "", "  ")
+		t.Errorf("Bad unmarshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expStr, resStr)
+	}
+}
+
+func TestDocPartialUnmarshal(t *testing.T) {
+	result := testDocSubs{}
+
+	tree, _ := LoadFile("marshal_test.toml")
+	subTree := tree.Get("subdoc").(*Tree)
+	err := subTree.Unmarshal(&result)
+	expected := docData.Subdocs
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		resStr, _ := json.MarshalIndent(result, "", "  ")
+		expStr, _ := json.MarshalIndent(expected, "", "  ")
+		t.Errorf("Bad partial unmartial: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expStr, resStr)
+	}
+}
+
+type tomlTypeCheckTest struct {
+	name string
+	item interface{}
+	typ  int //0=primitive, 1=otherslice, 2=treeslice, 3=tree
+}
+
+func TestTypeChecks(t *testing.T) {
+	tests := []tomlTypeCheckTest{
+		{"integer", 2, 0},
+		{"time", time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC), 0},
+		{"stringlist", []string{"hello", "hi"}, 1},
+		{"timelist", []time.Time{time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)}, 1},
+		{"objectlist", []tomlTypeCheckTest{}, 2},
+		{"object", tomlTypeCheckTest{}, 3},
+	}
+
+	for _, test := range tests {
+		expected := []bool{false, false, false, false}
+		expected[test.typ] = true
+		result := []bool{
+			isPrimitive(reflect.TypeOf(test.item)),
+			isOtherSlice(reflect.TypeOf(test.item)),
+			isTreeSlice(reflect.TypeOf(test.item)),
+			isTree(reflect.TypeOf(test.item)),
+		}
+		if !reflect.DeepEqual(expected, result) {
+			t.Errorf("Bad type check on %q: expected %v, got %v", test.name, expected, result)
+		}
+	}
+}
+
+type unexportedMarshalTestStruct struct {
+	String      string                      `toml:"string"`
+	StringList  []string                    `toml:"strlist"`
+	Sub         basicMarshalTestSubStruct   `toml:"subdoc"`
+	SubList     []basicMarshalTestSubStruct `toml:"sublist"`
+	unexported  int                         `toml:"shouldntBeHere"`
+	Unexported2 int                         `toml:"-"`
+}
+
+var unexportedTestData = unexportedMarshalTestStruct{
+	String:      "Hello",
+	StringList:  []string{"Howdy", "Hey There"},
+	Sub:         basicMarshalTestSubStruct{"One"},
+	SubList:     []basicMarshalTestSubStruct{{"Two"}, {"Three"}},
+	unexported:  0,
+	Unexported2: 0,
+}
+
+var unexportedTestToml = []byte(`string = "Hello"
+strlist = ["Howdy","Hey There"]
+unexported = 1
+shouldntBeHere = 2
+
+[subdoc]
+  String2 = "One"
+
+[[sublist]]
+  String2 = "Two"
+
+[[sublist]]
+  String2 = "Three"
+`)
+
+func TestUnexportedUnmarshal(t *testing.T) {
+	result := unexportedMarshalTestStruct{}
+	err := Unmarshal(unexportedTestToml, &result)
+	expected := unexportedTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unexported unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type errStruct struct {
+	Bool   bool      `toml:"bool"`
+	Date   time.Time `toml:"date"`
+	Float  float64   `toml:"float"`
+	Int    int16     `toml:"int"`
+	String *string   `toml:"string"`
+}
+
+var errTomls = []string{
+	"bool = truly\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:3200Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123a4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = j000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = Bite me",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = Bite me",
+	"bool = 1\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\n\"sorry\"\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = \"sorry\"\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = 1",
+}
+
+type mapErr struct {
+	Vals map[string]float64
+}
+
+type intErr struct {
+	Int1  int
+	Int2  int8
+	Int3  int16
+	Int4  int32
+	Int5  int64
+	UInt1 uint
+	UInt2 uint8
+	UInt3 uint16
+	UInt4 uint32
+	UInt5 uint64
+	Flt1  float32
+	Flt2  float64
+}
+
+var intErrTomls = []string{
+	"Int1 = []\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = []\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = []\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = []\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = []\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = []\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = []\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = []\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = []\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = []\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = []\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = []",
+}
+
+func TestErrUnmarshal(t *testing.T) {
+	for ind, toml := range errTomls {
+		result := errStruct{}
+		err := Unmarshal([]byte(toml), &result)
+		if err == nil {
+			t.Errorf("Expected err from case %d\n", ind)
+		}
+	}
+	result2 := mapErr{}
+	err := Unmarshal([]byte("[Vals]\nfred=\"1.2\""), &result2)
+	if err == nil {
+		t.Errorf("Expected err from map")
+	}
+	for ind, toml := range intErrTomls {
+		result3 := intErr{}
+		err := Unmarshal([]byte(toml), &result3)
+		if err == nil {
+			t.Errorf("Expected int err from case %d\n", ind)
+		}
+	}
+}
+
+type emptyMarshalTestStruct struct {
+	Title      string                  `toml:"title"`
+	Bool       bool                    `toml:"bool"`
+	Int        int                     `toml:"int"`
+	String     string                  `toml:"string"`
+	StringList []string                `toml:"stringlist"`
+	Ptr        *basicMarshalTestStruct `toml:"ptr"`
+	Map        map[string]string       `toml:"map"`
+}
+
+var emptyTestData = emptyMarshalTestStruct{
+	Title:      "Placeholder",
+	Bool:       false,
+	Int:        0,
+	String:     "",
+	StringList: []string{},
+	Ptr:        nil,
+	Map:        map[string]string{},
+}
+
+var emptyTestToml = []byte(`bool = false
+int = 0
+string = ""
+stringlist = []
+title = "Placeholder"
+
+[map]
+`)
+
+type emptyMarshalTestStruct2 struct {
+	Title      string                  `toml:"title"`
+	Bool       bool                    `toml:"bool,omitempty"`
+	Int        int                     `toml:"int, omitempty"`
+	String     string                  `toml:"string,omitempty "`
+	StringList []string                `toml:"stringlist,omitempty"`
+	Ptr        *basicMarshalTestStruct `toml:"ptr,omitempty"`
+	Map        map[string]string       `toml:"map,omitempty"`
+}
+
+var emptyTestData2 = emptyMarshalTestStruct2{
+	Title:      "Placeholder",
+	Bool:       false,
+	Int:        0,
+	String:     "",
+	StringList: []string{},
+	Ptr:        nil,
+	Map:        map[string]string{},
+}
+
+var emptyTestToml2 = []byte(`title = "Placeholder"
+`)
+
+func TestEmptyMarshal(t *testing.T) {
+	result, err := Marshal(emptyTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := emptyTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad empty marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestEmptyMarshalOmit(t *testing.T) {
+	result, err := Marshal(emptyTestData2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := emptyTestToml2
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad empty omit marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestEmptyUnmarshal(t *testing.T) {
+	result := emptyMarshalTestStruct{}
+	err := Unmarshal(emptyTestToml, &result)
+	expected := emptyTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad empty unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+func TestEmptyUnmarshalOmit(t *testing.T) {
+	result := emptyMarshalTestStruct2{}
+	err := Unmarshal(emptyTestToml, &result)
+	expected := emptyTestData2
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad empty omit unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type pointerMarshalTestStruct struct {
+	Str       *string
+	List      *[]string
+	ListPtr   *[]*string
+	Map       *map[string]string
+	MapPtr    *map[string]*string
+	EmptyStr  *string
+	EmptyList *[]string
+	EmptyMap  *map[string]string
+	DblPtr    *[]*[]*string
+}
+
+var pointerStr = "Hello"
+var pointerList = []string{"Hello back"}
+var pointerListPtr = []*string{&pointerStr}
+var pointerMap = map[string]string{"response": "Goodbye"}
+var pointerMapPtr = map[string]*string{"alternate": &pointerStr}
+var pointerTestData = pointerMarshalTestStruct{
+	Str:       &pointerStr,
+	List:      &pointerList,
+	ListPtr:   &pointerListPtr,
+	Map:       &pointerMap,
+	MapPtr:    &pointerMapPtr,
+	EmptyStr:  nil,
+	EmptyList: nil,
+	EmptyMap:  nil,
+}
+
+var pointerTestToml = []byte(`List = ["Hello back"]
+ListPtr = ["Hello"]
+Str = "Hello"
+
+[Map]
+  response = "Goodbye"
+
+[MapPtr]
+  alternate = "Hello"
+`)
+
+func TestPointerMarshal(t *testing.T) {
+	result, err := Marshal(pointerTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := pointerTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad pointer marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestPointerUnmarshal(t *testing.T) {
+	result := pointerMarshalTestStruct{}
+	err := Unmarshal(pointerTestToml, &result)
+	expected := pointerTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad pointer unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type nestedMarshalTestStruct struct {
+	String [][]string
+	//Struct [][]basicMarshalTestSubStruct
+	StringPtr *[]*[]*string
+	// StructPtr *[]*[]*basicMarshalTestSubStruct
+}
+
+var str1 = "Three"
+var str2 = "Four"
+var strPtr = []*string{&str1, &str2}
+var strPtr2 = []*[]*string{&strPtr}
+
+var nestedTestData = nestedMarshalTestStruct{
+	String:    [][]string{[]string{"Five", "Six"}, []string{"One", "Two"}},
+	StringPtr: &strPtr2,
+}
+
+var nestedTestToml = []byte(`String = [["Five","Six"],["One","Two"]]
+StringPtr = [["Three","Four"]]
+`)
+
+func TestNestedMarshal(t *testing.T) {
+	result, err := Marshal(nestedTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := nestedTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad nested marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestNestedUnmarshal(t *testing.T) {
+	result := nestedMarshalTestStruct{}
+	err := Unmarshal(nestedTestToml, &result)
+	expected := nestedTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad nested unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type customMarshalerParent struct {
+	Self    customMarshaler   `toml:"me"`
+	Friends []customMarshaler `toml:"friends"`
+}
+
+type customMarshaler struct {
+	FirsName string
+	LastName string
+}
+
+func (c customMarshaler) MarshalTOML() ([]byte, error) {
+	fullName := fmt.Sprintf("%s %s", c.FirsName, c.LastName)
+	return []byte(fullName), nil
+}
+
+var customMarshalerData = customMarshaler{FirsName: "Sally", LastName: "Fields"}
+var customMarshalerToml = []byte(`Sally Fields`)
+var nestedCustomMarshalerData = customMarshalerParent{
+	Self:    customMarshaler{FirsName: "Maiku", LastName: "Suteda"},
+	Friends: []customMarshaler{customMarshalerData},
+}
+var nestedCustomMarshalerToml = []byte(`friends = ["Sally Fields"]
+me = "Maiku Suteda"
+`)
+
+func TestCustomMarshaler(t *testing.T) {
+	result, err := Marshal(customMarshalerData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := customMarshalerToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestNestedCustomMarshaler(t *testing.T) {
+	result, err := Marshal(nestedCustomMarshalerData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := nestedCustomMarshalerToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad nested custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}

marshal_test.toml

@@ -0,0 +1,38 @@
+title = "TOML Marshal Testing"
+
+[basic]
+  bool = true
+  date = 1979-05-27T07:32:00Z
+  float = 123.4
+  int = 5000
+  string = "Bite me"
+  uint = 5001
+
+[basic_lists]
+  bools = [true,false,true]
+  dates = [1979-05-27T07:32:00Z,1980-05-27T07:32:00Z]
+  floats = [12.3,45.6,78.9]
+  ints = [8001,8001,8002]
+  strings = ["One","Two","Three"]
+  uints = [5002,5003]
+
+[basic_map]
+  one = "one"
+  two = "two"
+
+[subdoc]
+
+  [subdoc.first]
+    name = "First"
+
+  [subdoc.second]
+    name = "Second"
+
+[[subdoclist]]
+  name = "List.First"
+
+[[subdoclist]]
+  name = "List.Second"
+
+[[subdocptrs]]
+  name = "Second"

parser.go

@@ -3,6 +3,7 @@
 package toml
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"regexp"
@@ -12,11 +13,11 @@ import (
 )
 
 type tomlParser struct {
-	flow          chan token
+	flowIdx       int
-	tree          *TomlTree
+	flow          []token
-	tokensBuffer  []token
+	tree          *Tree
-	currentGroup  []string
+	currentTable  []string
-	seenGroupKeys []string
+	seenTableKeys []string
 }
 
 type tomlParserStateFn func() tomlParserStateFn
@@ -33,16 +34,10 @@ func (p *tomlParser) run() {
 }
 
 func (p *tomlParser) peek() *token {
-	if len(p.tokensBuffer) != 0 {
+	if p.flowIdx >= len(p.flow) {
-		return &(p.tokensBuffer[0])
-	}
-
-	tok, ok := <-p.flow
-	if !ok {
 		return nil
 	}
-	p.tokensBuffer = append(p.tokensBuffer, tok)
+	return &p.flow[p.flowIdx]
-	return &tok
 }
 
 func (p *tomlParser) assume(typ tokenType) {
@@ -56,16 +51,12 @@ func (p *tomlParser) assume(typ tokenType) {
 }
 
 func (p *tomlParser) getToken() *token {
-	if len(p.tokensBuffer) != 0 {
+	tok := p.peek()
-		tok := p.tokensBuffer[0]
+	if tok == nil {
-		p.tokensBuffer = p.tokensBuffer[1:]
-		return &tok
-	}
-	tok, ok := <-p.flow
-	if !ok {
 		return nil
 	}
-	return &tok
+	p.flowIdx++
+	return tok
 }
 
 func (p *tomlParser) parseStart() tomlParserStateFn {
@@ -95,48 +86,48 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn {
 	startToken := p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
-		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
+		p.raiseError(key, "unexpected token %s, was expecting a table array key", key)
 	}
 
-	// get or create group array element at the indicated part in the path
+	// get or create table array element at the indicated part in the path
 	keys, err := parseKey(key.val)
 	if err != nil {
-		p.raiseError(key, "invalid group array key: %s", err)
+		p.raiseError(key, "invalid table array key: %s", err)
 	}
 	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
 	destTree := p.tree.GetPath(keys)
-	var array []*TomlTree
+	var array []*Tree
 	if destTree == nil {
-		array = make([]*TomlTree, 0)
+		array = make([]*Tree, 0)
-	} else if target, ok := destTree.([]*TomlTree); ok && target != nil {
+	} else if target, ok := destTree.([]*Tree); ok && target != nil {
-		array = destTree.([]*TomlTree)
+		array = destTree.([]*Tree)
 	} else {
-		p.raiseError(key, "key %s is already assigned and not of type group array", key)
+		p.raiseError(key, "key %s is already assigned and not of type table array", key)
 	}
-	p.currentGroup = keys
+	p.currentTable = keys
 
-	// add a new tree to the end of the group array
+	// add a new tree to the end of the table array
-	newTree := newTomlTree()
+	newTree := newTree()
 	newTree.position = startToken.Position
 	array = append(array, newTree)
-	p.tree.SetPath(p.currentGroup, array)
+	p.tree.SetPath(p.currentTable, array)
 
-	// remove all keys that were children of this group array
+	// remove all keys that were children of this table array
 	prefix := key.val + "."
 	found := false
-	for ii := 0; ii < len(p.seenGroupKeys); {
+	for ii := 0; ii < len(p.seenTableKeys); {
-		groupKey := p.seenGroupKeys[ii]
+		tableKey := p.seenTableKeys[ii]
-		if strings.HasPrefix(groupKey, prefix) {
+		if strings.HasPrefix(tableKey, prefix) {
-			p.seenGroupKeys = append(p.seenGroupKeys[:ii], p.seenGroupKeys[ii+1:]...)
+			p.seenTableKeys = append(p.seenTableKeys[:ii], p.seenTableKeys[ii+1:]...)
 		} else {
-			found = (groupKey == key.val)
+			found = (tableKey == key.val)
 			ii++
 		}
 	}
 
 	// keep this key name from use by other kinds of assignments
 	if !found {
-		p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+		p.seenTableKeys = append(p.seenTableKeys, key.val)
 	}
 
 	// move to next parser state
@@ -148,24 +139,24 @@ func (p *tomlParser) parseGroup() tomlParserStateFn {
 	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
+		p.raiseError(key, "unexpected token %s, was expecting a table key", key)
 	}
-	for _, item := range p.seenGroupKeys {
+	for _, item := range p.seenTableKeys {
 		if item == key.val {
 			p.raiseError(key, "duplicated tables")
 		}
 	}
 
-	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	p.seenTableKeys = append(p.seenTableKeys, key.val)
 	keys, err := parseKey(key.val)
 	if err != nil {
-		p.raiseError(key, "invalid group array key: %s", err)
+		p.raiseError(key, "invalid table array key: %s", err)
 	}
 	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
 		p.raiseError(key, "%s", err)
 	}
 	p.assume(tokenRightBracket)
-	p.currentGroup = keys
+	p.currentTable = keys
 	return p.parseStart
 }
 
@@ -174,26 +165,26 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	p.assume(tokenEqual)
 
 	value := p.parseRvalue()
-	var groupKey []string
+	var tableKey []string
-	if len(p.currentGroup) > 0 {
+	if len(p.currentTable) > 0 {
-		groupKey = p.currentGroup
+		tableKey = p.currentTable
 	} else {
-		groupKey = []string{}
+		tableKey = []string{}
 	}
 
-	// find the group to assign, looking out for arrays of groups
+	// find the table to assign, looking out for arrays of tables
-	var targetNode *TomlTree
+	var targetNode *Tree
-	switch node := p.tree.GetPath(groupKey).(type) {
+	switch node := p.tree.GetPath(tableKey).(type) {
-	case []*TomlTree:
+	case []*Tree:
 		targetNode = node[len(node)-1]
-	case *TomlTree:
+	case *Tree:
 		targetNode = node
 	default:
-		p.raiseError(key, "Unknown group type for path: %s",
+		p.raiseError(key, "Unknown table type for path: %s",
-			strings.Join(groupKey, "."))
+			strings.Join(tableKey, "."))
 	}
 
-	// assign value to the found group
+	// assign value to the found table
 	keyVals, err := parseKey(key.val)
 	if err != nil {
 		p.raiseError(key, "%s", err)
@@ -203,7 +194,7 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	}
 	keyVal := keyVals[0]
 	localKey := []string{keyVal}
-	finalKey := append(groupKey, keyVal)
+	finalKey := append(tableKey, keyVal)
 	if targetNode.GetPath(localKey) != nil {
 		p.raiseError(key, "The following key was defined twice: %s",
 			strings.Join(finalKey, "."))
@@ -211,7 +202,7 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	var toInsert interface{}
 
 	switch value.(type) {
-	case *TomlTree, []*TomlTree:
+	case *Tree, []*Tree:
 		toInsert = value
 	default:
 		toInsert = &tomlValue{value, key.Position}
@@ -224,7 +215,7 @@ var numberUnderscoreInvalidRegexp *regexp.Regexp
 
 func cleanupNumberToken(value string) (string, error) {
 	if numberUnderscoreInvalidRegexp.MatchString(value) {
-		return "", fmt.Errorf("invalid use of _ in number")
+		return "", errors.New("invalid use of _ in number")
 	}
 	cleanedVal := strings.Replace(value, "_", "", -1)
 	return cleanedVal, nil
@@ -288,8 +279,8 @@ func tokenIsComma(t *token) bool {
 	return t != nil && t.typ == tokenComma
 }
 
-func (p *tomlParser) parseInlineTable() *TomlTree {
+func (p *tomlParser) parseInlineTable() *Tree {
-	tree := newTomlTree()
+	tree := newTree()
 	var previous *token
 Loop:
 	for {
@@ -359,29 +350,29 @@ func (p *tomlParser) parseArray() interface{} {
 			p.getToken()
 		}
 	}
-	// An array of TomlTrees is actually an array of inline
+	// An array of Trees is actually an array of inline
 	// tables, which is a shorthand for a table array. If the
-	// array was not converted from []interface{} to []*TomlTree,
+	// array was not converted from []interface{} to []*Tree,
 	// the two notations would not be equivalent.
-	if arrayType == reflect.TypeOf(newTomlTree()) {
+	if arrayType == reflect.TypeOf(newTree()) {
-		tomlArray := make([]*TomlTree, len(array))
+		tomlArray := make([]*Tree, len(array))
 		for i, v := range array {
-			tomlArray[i] = v.(*TomlTree)
+			tomlArray[i] = v.(*Tree)
 		}
 		return tomlArray
 	}
 	return array
 }
 
-func parseToml(flow chan token) *TomlTree {
+func parseToml(flow []token) *Tree {
-	result := newTomlTree()
+	result := newTree()
 	result.position = Position{1, 1}
 	parser := &tomlParser{
+		flowIdx:       0,
 		flow:          flow,
 		tree:          result,
-		tokensBuffer:  make([]token, 0),
+		currentTable:  make([]string, 0),
-		currentGroup:  make([]string, 0),
+		seenTableKeys: make([]string, 0),
-		seenGroupKeys: make([]string, 0),
 	}
 	parser.run()
 	return result

parser_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 )
 
-func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref map[string]interface{}) {
+func assertSubTree(t *testing.T, path []string, tree *Tree, err error, ref map[string]interface{}) {
 	if err != nil {
 		t.Error("Non-nil error:", err.Error())
 		return
@@ -20,12 +20,12 @@ func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref m
 		// NOTE: directly access key instead of resolve by path
 		// NOTE: see TestSpecialKV
 		switch node := tree.GetPath([]string{k}).(type) {
-		case []*TomlTree:
+		case []*Tree:
 			t.Log("\tcomparing key", nextPath, "by array iteration")
 			for idx, item := range node {
 				assertSubTree(t, nextPath, item, err, v.([]map[string]interface{})[idx])
 			}
-		case *TomlTree:
+		case *Tree:
 			t.Log("\tcomparing key", nextPath, "by subtree assestion")
 			assertSubTree(t, nextPath, node, err, v.(map[string]interface{}))
 		default:
@@ -37,14 +37,14 @@ func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref m
 	}
 }
 
-func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+func assertTree(t *testing.T, tree *Tree, err error, ref map[string]interface{}) {
 	t.Log("Asserting tree:\n", spew.Sdump(tree))
 	assertSubTree(t, []string{}, tree, err, ref)
 	t.Log("Finished tree assertion.")
 }
 
 func TestCreateSubTree(t *testing.T) {
-	tree := newTomlTree()
+	tree := newTree()
 	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
@@ -283,6 +283,17 @@ func TestArrayNested(t *testing.T) {
 	})
 }
 
+func TestNestedArrayComment(t *testing.T) {
+	tree, err := Load(`
+someArray = [
+# does not work
+["entry1"]
+]`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"someArray": [][]string{{"entry1"}},
+	})
+}
+
 func TestNestedEmptyArrays(t *testing.T) {
 	tree, err := Load("a = [[[]]]")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -456,7 +467,7 @@ func TestDuplicateKeys(t *testing.T) {
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "(1, 2): invalid group array key: empty key group" {
+	if err.Error() != "(1, 2): invalid table array key: empty table key" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -489,7 +500,8 @@ func TestFloatsWithoutLeadingZeros(t *testing.T) {
 
 func TestMissingFile(t *testing.T) {
 	_, err := LoadFile("foo.toml")
-	if err.Error() != "open foo.toml: no such file or directory" {
+	if err.Error() != "open foo.toml: no such file or directory" &&
+		err.Error() != "open foo.toml: The system cannot find the file specified." {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -580,12 +592,12 @@ func TestParseKeyGroupArray(t *testing.T) {
 
 func TestParseKeyGroupArrayUnfinished(t *testing.T) {
 	_, err := Load("[[foo.bar]\na = 42")
-	if err.Error() != "(1, 10): was expecting token [[, but got unclosed key group array instead" {
+	if err.Error() != "(1, 10): was expecting token [[, but got unclosed table array key instead" {
 		t.Error("Bad error message:", err.Error())
 	}
 
 	_, err = Load("[[foo.[bar]\na = 42")
-	if err.Error() != "(1, 3): unexpected token group name cannot contain ']', was expecting a key group array" {
+	if err.Error() != "(1, 3): unexpected token table array key cannot contain ']', was expecting a table array key" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -622,22 +634,13 @@ func TestParseKeyGroupArraySpec(t *testing.T) {
 	})
 }
 
-func TestToTomlValue(t *testing.T) {
+func TestTomlValueStringRepresentation(t *testing.T) {
 	for idx, item := range []struct {
 		Value  interface{}
 		Expect string
 	}{
-		{int(1), "1"},
-		{int8(2), "2"},
-		{int16(3), "3"},
-		{int32(4), "4"},
 		{int64(12345), "12345"},
-		{uint(10), "10"},
-		{uint8(20), "20"},
-		{uint16(30), "30"},
-		{uint32(40), "40"},
 		{uint64(50), "50"},
-		{float32(12.456), "12.456"},
 		{float64(123.45), "123.45"},
 		{bool(true), "true"},
 		{"hello world", "\"hello world\""},
@@ -646,44 +649,25 @@ func TestToTomlValue(t *testing.T) {
 		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
 			"1979-05-27T07:32:00Z"},
 		{[]interface{}{"gamma", "delta"},
-			"[\n  \"gamma\",\n  \"delta\",\n]"},
+			"[\"gamma\",\"delta\"]"},
 		{nil, ""},
 	} {
-		result := toTomlValue(item.Value, 0)
+		result, err := tomlValueStringRepresentation(item.Value)
+		if err != nil {
+			t.Errorf("Test %d - unexpected error: %s", idx, err)
+		}
 		if result != item.Expect {
 			t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect)
 		}
 	}
 }
 
-func TestToString(t *testing.T) {
-	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
-	if err != nil {
-		t.Errorf("Test failed to parse: %v", err)
-		return
-	}
-	result := tree.ToString()
-	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
-	if result != expected {
-		t.Errorf("Expected got '%s', expected '%s'", result, expected)
-	}
-}
-
 func TestToStringMapStringString(t *testing.T) {
-	in := map[string]interface{}{"m": map[string]string{"v": "abc"}}
+	tree, err := TreeFromMap(map[string]interface{}{"m": map[string]interface{}{"v": "abc"}})
-	want := "\n[m]\n  v = \"abc\"\n"
+	if err != nil {
-	tree := TreeFromMap(in)
+		t.Fatalf("unexpected error: %s", err)
-	got := tree.String()
-
-	if got != want {
-		t.Errorf("want:\n%q\ngot:\n%q", want, got)
 	}
-}
-
-func TestToStringMapInterfaceInterface(t *testing.T) {
-	in := map[string]interface{}{"m": map[interface{}]interface{}{"v": "abc"}}
 	want := "\n[m]\n  v = \"abc\"\n"
-	tree := TreeFromMap(in)
 	got := tree.String()
 
 	if got != want {
@@ -753,13 +737,13 @@ func TestNestedTreePosition(t *testing.T) {
 }
 
 func TestInvalidGroupArray(t *testing.T) {
-	_, err := Load("[key#group]\nanswer = 42")
+	_, err := Load("[table#key]\nanswer = 42")
 	if err == nil {
 		t.Error("Should error")
 	}
 
 	_, err = Load("[foo.[bar]\na = 42")
-	if err.Error() != "(1, 2): unexpected token group name cannot contain ']', was expecting a key group" {
+	if err.Error() != "(1, 2): unexpected token table key cannot contain ']', was expecting a table key" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -773,7 +757,7 @@ func TestDoubleEqual(t *testing.T) {
 
 func TestGroupArrayReassign(t *testing.T) {
 	_, err := Load("[hello]\n[[hello]]")
-	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type group array" {
+	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type table array" {
 		t.Error("Bad error message:", err.Error())
 	}
 }

query/doc.go

@@ -0,0 +1,175 @@
+// Package query performs JSONPath-like queries on a TOML document.
+//
+// The query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/.
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result, err := query.CompileAndExecute("$.foo.bar.baz", tree)
+//
+// This is roughly equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// err is nil if any parsing exception occurs.
+//
+// If no node in the tree matches the query, result will simply contain an empty list of
+// items.
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// Query syntax
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   query.CompileAndExecute("$.foo[-1]", tree)
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   query.CompileAndExecute("$.foo[0:5]", tree)
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   query.CompileAndExecute("$.foo[0:-1]", tree)
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   query.CompileAndExecute("$.foo[0:-1:2]", tree)
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   query.CompileAndExecute("$.foo[:]", tree)
+//   query.CompileAndExecute("$.foo[0:]", tree)
+//   query.CompileAndExecute("$.foo[:-1]", tree)
+//   query.CompileAndExecute("$.foo[0:-1:]", tree)
+//   query.CompileAndExecute("$.foo[::1]", tree)
+//   query.CompileAndExecute("$.foo[0::1]", tree)
+//   query.CompileAndExecute("$.foo[:-1:1]", tree)
+//   query.CompileAndExecute("$.foo[0:-1:1]", tree)
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   query.CompileAndExecute("$.foo[?(bar)]", tree)
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type Tree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a Result object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := query.CompileAndExecute("$.foo.bar.baz", tree)
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a Tree object, or compiled ahead
+// of time and executed discretely.  The former is more convienent, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := query.CompileAndExecute("$.foo.bar.baz", tree)
+//
+//   // compiled query
+//   query, err := toml.Compile("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := query.Compile("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*Tree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package query

query/lexer.go

@@ -3,10 +3,11 @@
 // Written using the principles developed by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -54,7 +55,7 @@ func (l *queryLexer) nextStart() {
 
 func (l *queryLexer) emit(t tokenType) {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      l.input[l.start:l.pos],
 	}
@@ -63,7 +64,7 @@ func (l *queryLexer) emit(t tokenType) {
 
 func (l *queryLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      value,
 	}
@@ -91,7 +92,7 @@ func (l *queryLexer) backup() {
 
 func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      tokenError,
 		val:      fmt.Sprintf(format, args...),
 	}

query/lexer_test.go

@@ -1,6 +1,7 @@
-package toml
+package query
 
 import (
+	"github.com/pelletier/go-toml"
 	"testing"
 )
 
@@ -36,143 +37,143 @@ func testQLFlow(t *testing.T, input string, expectedFlow []token) {
 
 func TestLexSpecialChars(t *testing.T) {
 	testQLFlow(t, " .$[]..()?*", []token{
-		{Position{1, 2}, tokenDot, "."},
+		{toml.Position{1, 2}, tokenDot, "."},
-		{Position{1, 3}, tokenDollar, "$"},
+		{toml.Position{1, 3}, tokenDollar, "$"},
-		{Position{1, 4}, tokenLeftBracket, "["},
+		{toml.Position{1, 4}, tokenLeftBracket, "["},
-		{Position{1, 5}, tokenRightBracket, "]"},
+		{toml.Position{1, 5}, tokenRightBracket, "]"},
-		{Position{1, 6}, tokenDotDot, ".."},
+		{toml.Position{1, 6}, tokenDotDot, ".."},
-		{Position{1, 8}, tokenLeftParen, "("},
+		{toml.Position{1, 8}, tokenLeftParen, "("},
-		{Position{1, 9}, tokenRightParen, ")"},
+		{toml.Position{1, 9}, tokenRightParen, ")"},
-		{Position{1, 10}, tokenQuestion, "?"},
+		{toml.Position{1, 10}, tokenQuestion, "?"},
-		{Position{1, 11}, tokenStar, "*"},
+		{toml.Position{1, 11}, tokenStar, "*"},
-		{Position{1, 12}, tokenEOF, ""},
+		{toml.Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestLexString(t *testing.T) {
 	testQLFlow(t, "'foo\n'", []token{
-		{Position{1, 2}, tokenString, "foo\n"},
+		{toml.Position{1, 2}, tokenString, "foo\n"},
-		{Position{2, 2}, tokenEOF, ""},
+		{toml.Position{2, 2}, tokenEOF, ""},
 	})
 }
 
 func TestLexDoubleString(t *testing.T) {
 	testQLFlow(t, `"bar"`, []token{
-		{Position{1, 2}, tokenString, "bar"},
+		{toml.Position{1, 2}, tokenString, "bar"},
-		{Position{1, 6}, tokenEOF, ""},
+		{toml.Position{1, 6}, tokenEOF, ""},
 	})
 }
 
 func TestLexStringEscapes(t *testing.T) {
 	testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{
-		{Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"},
+		{toml.Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"},
-		{Position{1, 55}, tokenEOF, ""},
+		{toml.Position{1, 55}, tokenEOF, ""},
 	})
 }
 
 func TestLexStringUnfinishedUnicode4(t *testing.T) {
 	testQLFlow(t, `"\u000"`, []token{
-		{Position{1, 2}, tokenError, "unfinished unicode escape"},
+		{toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
 	})
 }
 
 func TestLexStringUnfinishedUnicode8(t *testing.T) {
 	testQLFlow(t, `"\U0000"`, []token{
-		{Position{1, 2}, tokenError, "unfinished unicode escape"},
+		{toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
 	})
 }
 
 func TestLexStringInvalidEscape(t *testing.T) {
 	testQLFlow(t, `"\x"`, []token{
-		{Position{1, 2}, tokenError, "invalid escape sequence: \\x"},
+		{toml.Position{1, 2}, tokenError, "invalid escape sequence: \\x"},
 	})
 }
 
 func TestLexStringUnfinished(t *testing.T) {
 	testQLFlow(t, `"bar`, []token{
-		{Position{1, 2}, tokenError, "unclosed string"},
+		{toml.Position{1, 2}, tokenError, "unclosed string"},
 	})
 }
 
 func TestLexKey(t *testing.T) {
 	testQLFlow(t, "foo", []token{
-		{Position{1, 1}, tokenKey, "foo"},
+		{toml.Position{1, 1}, tokenKey, "foo"},
-		{Position{1, 4}, tokenEOF, ""},
+		{toml.Position{1, 4}, tokenEOF, ""},
 	})
 }
 
 func TestLexRecurse(t *testing.T) {
 	testQLFlow(t, "$..*", []token{
-		{Position{1, 1}, tokenDollar, "$"},
+		{toml.Position{1, 1}, tokenDollar, "$"},
-		{Position{1, 2}, tokenDotDot, ".."},
+		{toml.Position{1, 2}, tokenDotDot, ".."},
-		{Position{1, 4}, tokenStar, "*"},
+		{toml.Position{1, 4}, tokenStar, "*"},
-		{Position{1, 5}, tokenEOF, ""},
+		{toml.Position{1, 5}, tokenEOF, ""},
 	})
 }
 
 func TestLexBracketKey(t *testing.T) {
 	testQLFlow(t, "$[foo]", []token{
-		{Position{1, 1}, tokenDollar, "$"},
+		{toml.Position{1, 1}, tokenDollar, "$"},
-		{Position{1, 2}, tokenLeftBracket, "["},
+		{toml.Position{1, 2}, tokenLeftBracket, "["},
-		{Position{1, 3}, tokenKey, "foo"},
+		{toml.Position{1, 3}, tokenKey, "foo"},
-		{Position{1, 6}, tokenRightBracket, "]"},
+		{toml.Position{1, 6}, tokenRightBracket, "]"},
-		{Position{1, 7}, tokenEOF, ""},
+		{toml.Position{1, 7}, tokenEOF, ""},
 	})
 }
 
 func TestLexSpace(t *testing.T) {
 	testQLFlow(t, "foo bar baz", []token{
-		{Position{1, 1}, tokenKey, "foo"},
+		{toml.Position{1, 1}, tokenKey, "foo"},
-		{Position{1, 5}, tokenKey, "bar"},
+		{toml.Position{1, 5}, tokenKey, "bar"},
-		{Position{1, 9}, tokenKey, "baz"},
+		{toml.Position{1, 9}, tokenKey, "baz"},
-		{Position{1, 12}, tokenEOF, ""},
+		{toml.Position{1, 12}, tokenEOF, ""},
 	})
 }
 
 func TestLexInteger(t *testing.T) {
 	testQLFlow(t, "100 +200 -300", []token{
-		{Position{1, 1}, tokenInteger, "100"},
+		{toml.Position{1, 1}, tokenInteger, "100"},
-		{Position{1, 5}, tokenInteger, "+200"},
+		{toml.Position{1, 5}, tokenInteger, "+200"},
-		{Position{1, 10}, tokenInteger, "-300"},
+		{toml.Position{1, 10}, tokenInteger, "-300"},
-		{Position{1, 14}, tokenEOF, ""},
+		{toml.Position{1, 14}, tokenEOF, ""},
 	})
 }
 
 func TestLexFloat(t *testing.T) {
 	testQLFlow(t, "100.0 +200.0 -300.0", []token{
-		{Position{1, 1}, tokenFloat, "100.0"},
+		{toml.Position{1, 1}, tokenFloat, "100.0"},
-		{Position{1, 7}, tokenFloat, "+200.0"},
+		{toml.Position{1, 7}, tokenFloat, "+200.0"},
-		{Position{1, 14}, tokenFloat, "-300.0"},
+		{toml.Position{1, 14}, tokenFloat, "-300.0"},
-		{Position{1, 20}, tokenEOF, ""},
+		{toml.Position{1, 20}, tokenEOF, ""},
 	})
 }
 
 func TestLexFloatWithMultipleDots(t *testing.T) {
 	testQLFlow(t, "4.2.", []token{
-		{Position{1, 1}, tokenError, "cannot have two dots in one float"},
+		{toml.Position{1, 1}, tokenError, "cannot have two dots in one float"},
 	})
 }
 
 func TestLexFloatLeadingDot(t *testing.T) {
 	testQLFlow(t, "+.1", []token{
-		{Position{1, 1}, tokenError, "cannot start float with a dot"},
+		{toml.Position{1, 1}, tokenError, "cannot start float with a dot"},
 	})
 }
 
 func TestLexFloatWithTrailingDot(t *testing.T) {
 	testQLFlow(t, "42.", []token{
-		{Position{1, 1}, tokenError, "float cannot end with a dot"},
+		{toml.Position{1, 1}, tokenError, "float cannot end with a dot"},
 	})
 }
 
 func TestLexNumberWithoutDigit(t *testing.T) {
 	testQLFlow(t, "+", []token{
-		{Position{1, 1}, tokenError, "no digit in that number"},
+		{toml.Position{1, 1}, tokenError, "no digit in that number"},
 	})
 }
 
 func TestLexUnknown(t *testing.T) {
 	testQLFlow(t, "^", []token{
-		{Position{1, 1}, tokenError, "unexpected char: '94'"},
+		{toml.Position{1, 1}, tokenError, "unexpected char: '94'"},
 	})
 }

query/match.go

@@ -1,27 +1,10 @@
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 )
 
-// support function to set positions for tomlValues
-// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
-// values returned by the query engines
-func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
-	switch castNode := node.(type) {
-	case *tomlValue:
-		ctx.lastPosition = castNode.position
-		return castNode.value
-	case []*TomlTree:
-		if len(castNode) > 0 {
-			ctx.lastPosition = castNode[0].position
-		}
-		return node
-	default:
-		return node
-	}
-}
-
 // base match
 type matchBase struct {
 	next pathFn
@@ -45,15 +28,7 @@ func (f *terminatingFn) setNext(next pathFn) {
 }
 
 func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
-	switch castNode := node.(type) {
+	ctx.result.appendResult(node, ctx.lastPosition)
-	case *TomlTree:
-		ctx.result.appendResult(node, castNode.position)
-	case *tomlValue:
-		ctx.result.appendResult(node, castNode.position)
-	default:
-		// use last position for scalars
-		ctx.result.appendResult(node, ctx.lastPosition)
-	}
 }
 
 // match single key
@@ -67,16 +42,18 @@ func newMatchKeyFn(name string) *matchKeyFn {
 }
 
 func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
-	if array, ok := node.([]*TomlTree); ok {
+	if array, ok := node.([]*toml.Tree); ok {
 		for _, tree := range array {
-			item := tree.values[f.Name]
+			item := tree.Get(f.Name)
 			if item != nil {
+				ctx.lastPosition = tree.GetPosition(f.Name)
 				f.next.call(item, ctx)
 			}
 		}
-	} else if tree, ok := node.(*TomlTree); ok {
+	} else if tree, ok := node.(*toml.Tree); ok {
-		item := tree.values[f.Name]
+		item := tree.Get(f.Name)
 		if item != nil {
+			ctx.lastPosition = tree.GetPosition(f.Name)
 			f.next.call(item, ctx)
 		}
 	}
@@ -93,8 +70,13 @@ func newMatchIndexFn(idx int) *matchIndexFn {
 }
 
 func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
-	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+	if arr, ok := node.([]interface{}); ok {
 		if f.Idx < len(arr) && f.Idx >= 0 {
+			if treesArray, ok := node.([]*toml.Tree); ok {
+				if len(treesArray) > 0 {
+					ctx.lastPosition = treesArray[0].Position()
+				}
+			}
 			f.next.call(arr[f.Idx], ctx)
 		}
 	}
@@ -111,7 +93,7 @@ func newMatchSliceFn(start, end, step int) *matchSliceFn {
 }
 
 func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
-	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+	if arr, ok := node.([]interface{}); ok {
 		// adjust indexes for negative values, reverse ordering
 		realStart, realEnd := f.Start, f.End
 		if realStart < 0 {
@@ -125,6 +107,11 @@ func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
 		}
 		// loop and gather
 		for idx := realStart; idx < realEnd; idx += f.Step {
+			if treesArray, ok := node.([]*toml.Tree); ok {
+				if len(treesArray) > 0 {
+					ctx.lastPosition = treesArray[0].Position()
+				}
+			}
 			f.next.call(arr[idx], ctx)
 		}
 	}
@@ -140,8 +127,10 @@ func newMatchAnyFn() *matchAnyFn {
 }
 
 func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
+	if tree, ok := node.(*toml.Tree); ok {
-		for _, v := range tree.values {
+		for _, k := range tree.Keys() {
+			v := tree.Get(k)
+			ctx.lastPosition = tree.GetPosition(k)
 			f.next.call(v, ctx)
 		}
 	}
@@ -174,21 +163,25 @@ func newMatchRecursiveFn() *matchRecursiveFn {
 }
 
 func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
+	originalPosition := ctx.lastPosition
-		var visit func(tree *TomlTree)
+	if tree, ok := node.(*toml.Tree); ok {
-		visit = func(tree *TomlTree) {
+		var visit func(tree *toml.Tree)
-			for _, v := range tree.values {
+		visit = func(tree *toml.Tree) {
+			for _, k := range tree.Keys() {
+				v := tree.Get(k)
+				ctx.lastPosition = tree.GetPosition(k)
 				f.next.call(v, ctx)
 				switch node := v.(type) {
-				case *TomlTree:
+				case *toml.Tree:
 					visit(node)
-				case []*TomlTree:
+				case []*toml.Tree:
 					for _, subtree := range node {
 						visit(subtree)
 					}
 				}
 			}
 		}
+		ctx.lastPosition = originalPosition
 		f.next.call(tree, ctx)
 		visit(tree)
 	}
@@ -197,11 +190,11 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
 // match based on an externally provided functional filter
 type matchFilterFn struct {
 	matchBase
-	Pos  Position
+	Pos  toml.Position
 	Name string
 }
 
-func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+func newMatchFilterFn(name string, pos toml.Position) *matchFilterFn {
 	return &matchFilterFn{Name: name, Pos: pos}
 }
 
@@ -211,17 +204,22 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
 		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
 			f.Pos.String(), f.Name))
 	}
-	switch castNode := tomlValueCheck(node, ctx).(type) {
+	switch castNode := node.(type) {
-	case *TomlTree:
+	case *toml.Tree:
-		for _, v := range castNode.values {
+		for _, k := range castNode.Keys() {
-			if tv, ok := v.(*tomlValue); ok {
+			v := castNode.Get(k)
-				if fn(tv.value) {
+			if fn(v) {
-					f.next.call(v, ctx)
+				ctx.lastPosition = castNode.GetPosition(k)
-				}
+				f.next.call(v, ctx)
-			} else {
+			}
-				if fn(v) {
+		}
-					f.next.call(v, ctx)
+	case []*toml.Tree:
+		for _, v := range castNode {
+			if fn(v) {
+				if len(castNode) > 0 {
+					ctx.lastPosition = castNode[0].Position()
 				}
+				f.next.call(v, ctx)
 			}
 		}
 	case []interface{}:

query/match_test.go

@@ -1,7 +1,8 @@
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"testing"
 )
 
@@ -194,8 +195,8 @@ func TestPathFilterExpr(t *testing.T) {
 		"$[?('foo'),?(bar)]",
 		buildPath(
 			&matchUnionFn{[]pathFn{
-				newMatchFilterFn("foo", Position{}),
+				newMatchFilterFn("foo", toml.Position{}),
-				newMatchFilterFn("bar", Position{}),
+				newMatchFilterFn("bar", toml.Position{}),
 			}},
 		))
 }

query/parser.go

@@ -5,7 +5,7 @@
   https://code.google.com/p/json-path/
 */
 
-package toml
+package query
 
 import (
 	"fmt"
@@ -253,7 +253,7 @@ func (p *queryParser) parseFilterExpr() queryParserStateFn {
 	}
 	tok = p.getToken()
 	if tok.typ != tokenKey && tok.typ != tokenString {
-		return p.parseError(tok, "expected key or string for filter funciton name")
+		return p.parseError(tok, "expected key or string for filter function name")
 	}
 	name := tok.val
 	tok = p.getToken()

query/parser_test.go

@@ -1,7 +1,8 @@
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"io/ioutil"
 	"sort"
 	"strings"
@@ -11,15 +12,13 @@ import (
 
 type queryTestNode struct {
 	value    interface{}
-	position Position
+	position toml.Position
 }
 
 func valueString(root interface{}) string {
 	result := "" //fmt.Sprintf("%T:", root)
 	switch node := root.(type) {
-	case *tomlValue:
+	case *Result:
-		return valueString(node.value)
-	case *QueryResult:
 		items := []string{}
 		for i, v := range node.Values() {
 			items = append(items, fmt.Sprintf("%s:%s",
@@ -37,7 +36,7 @@ func valueString(root interface{}) string {
 		}
 		sort.Strings(items)
 		result = "[" + strings.Join(items, ", ") + "]"
-	case *TomlTree:
+	case *toml.Tree:
 		// workaround for unreliable map key ordering
 		items := []string{}
 		for _, k := range node.Keys() {
@@ -78,13 +77,13 @@ func assertValue(t *testing.T, result, ref interface{}) {
 	}
 }
 
-func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
+func assertQueryPositions(t *testing.T, tomlDoc string, query string, ref []interface{}) {
-	tree, err := Load(toml)
+	tree, err := toml.Load(tomlDoc)
 	if err != nil {
 		t.Errorf("Non-nil toml parse error: %v", err)
 		return
 	}
-	q, err := CompileQuery(query)
+	q, err := Compile(query)
 	if err != nil {
 		t.Error(err)
 		return
@@ -101,7 +100,7 @@ func TestQueryRoot(t *testing.T) {
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(42),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 		})
 }
@@ -112,7 +111,7 @@ func TestQueryKey(t *testing.T) {
 		"$.foo.a",
 		[]interface{}{
 			queryTestNode{
-				int64(42), Position{2, 1},
+				int64(42), toml.Position{2, 1},
 			},
 		})
 }
@@ -123,7 +122,7 @@ func TestQueryKeyString(t *testing.T) {
 		"$.foo['a']",
 		[]interface{}{
 			queryTestNode{
-				int64(42), Position{2, 1},
+				int64(42), toml.Position{2, 1},
 			},
 		})
 }
@@ -134,7 +133,7 @@ func TestQueryIndex(t *testing.T) {
 		"$.foo.a[5]",
 		[]interface{}{
 			queryTestNode{
-				int64(6), Position{2, 1},
+				int64(6), toml.Position{2, 1},
 			},
 		})
 }
@@ -145,19 +144,19 @@ func TestQuerySliceRange(t *testing.T) {
 		"$.foo.a[0:5]",
 		[]interface{}{
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(2), Position{2, 1},
+				int64(2), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(3), Position{2, 1},
+				int64(3), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(4), Position{2, 1},
+				int64(4), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(5), Position{2, 1},
+				int64(5), toml.Position{2, 1},
 			},
 		})
 }
@@ -168,13 +167,13 @@ func TestQuerySliceStep(t *testing.T) {
 		"$.foo.a[0:5:2]",
 		[]interface{}{
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(3), Position{2, 1},
+				int64(3), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(5), Position{2, 1},
+				int64(5), toml.Position{2, 1},
 			},
 		})
 }
@@ -188,13 +187,13 @@ func TestQueryAny(t *testing.T) {
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 		})
 }
@@ -207,19 +206,19 @@ func TestQueryUnionSimple(t *testing.T) {
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 		})
 }
@@ -249,7 +248,7 @@ func TestQueryRecursionAll(t *testing.T) {
 							"b": int64(6),
 						},
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -257,19 +256,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(1),
 						"b": int64(2),
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(2), Position{3, 1},
+				int64(2), toml.Position{3, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -277,19 +276,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(3),
 						"b": int64(4),
 					},
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
-				int64(3), Position{5, 1},
+				int64(3), toml.Position{5, 1},
 			},
 			queryTestNode{
-				int64(4), Position{6, 1},
+				int64(4), toml.Position{6, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -297,19 +296,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(5),
 						"b": int64(6),
 					},
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 			queryTestNode{
-				int64(5), Position{8, 1},
+				int64(5), toml.Position{8, 1},
 			},
 			queryTestNode{
-				int64(6), Position{9, 1},
+				int64(6), toml.Position{9, 1},
 			},
 		})
 }
@@ -325,31 +324,31 @@ func TestQueryRecursionUnionSimple(t *testing.T) {
 						"a": int64(1),
 						"b": int64(2),
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 		})
 }
 
 func TestQueryFilterFn(t *testing.T) {
-	buff, err := ioutil.ReadFile("example.toml")
+	buff, err := ioutil.ReadFile("../example.toml")
 	if err != nil {
 		t.Error(err)
 		return
@@ -359,16 +358,16 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(int)]",
 		[]interface{}{
 			queryTestNode{
-				int64(8001), Position{13, 1},
+				int64(8001), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(8001), Position{13, 1},
+				int64(8001), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(8002), Position{13, 1},
+				int64(8002), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(5000), Position{14, 1},
+				int64(5000), toml.Position{14, 1},
 			},
 		})
 
@@ -376,32 +375,32 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(string)]",
 		[]interface{}{
 			queryTestNode{
-				"TOML Example", Position{3, 1},
+				"TOML Example", toml.Position{3, 1},
 			},
 			queryTestNode{
-				"Tom Preston-Werner", Position{6, 1},
+				"Tom Preston-Werner", toml.Position{6, 1},
 			},
 			queryTestNode{
-				"GitHub", Position{7, 1},
+				"GitHub", toml.Position{7, 1},
 			},
 			queryTestNode{
 				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
-				Position{8, 1},
+				toml.Position{8, 1},
 			},
 			queryTestNode{
-				"192.168.1.1", Position{12, 1},
+				"192.168.1.1", toml.Position{12, 1},
 			},
 			queryTestNode{
-				"10.0.0.1", Position{21, 3},
+				"10.0.0.1", toml.Position{21, 3},
 			},
 			queryTestNode{
-				"eqdc10", Position{22, 3},
+				"eqdc10", toml.Position{22, 3},
 			},
 			queryTestNode{
-				"10.0.0.2", Position{25, 3},
+				"10.0.0.2", toml.Position{25, 3},
 			},
 			queryTestNode{
-				"eqdc10", Position{26, 3},
+				"eqdc10", toml.Position{26, 3},
 			},
 		})
 
@@ -421,7 +420,7 @@ func TestQueryFilterFn(t *testing.T) {
 					"organization": "GitHub",
 					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
 					"dob":          tv,
-				}, Position{5, 1},
+				}, toml.Position{5, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -429,7 +428,7 @@ func TestQueryFilterFn(t *testing.T) {
 					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
 					"connection_max": int64(5000),
 					"enabled":        true,
-				}, Position{11, 1},
+				}, toml.Position{11, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -441,19 +440,19 @@ func TestQueryFilterFn(t *testing.T) {
 						"ip": "10.0.0.2",
 						"dc": "eqdc10",
 					},
-				}, Position{17, 1},
+				}, toml.Position{17, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"ip": "10.0.0.1",
 					"dc": "eqdc10",
-				}, Position{20, 3},
+				}, toml.Position{20, 3},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"ip": "10.0.0.2",
 					"dc": "eqdc10",
-				}, Position{24, 3},
+				}, toml.Position{24, 3},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -461,7 +460,7 @@ func TestQueryFilterFn(t *testing.T) {
 						[]interface{}{"gamma", "delta"},
 						[]interface{}{int64(1), int64(2)},
 					},
-				}, Position{28, 1},
+				}, toml.Position{28, 1},
 			},
 		})
 
@@ -469,7 +468,7 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(time)]",
 		[]interface{}{
 			queryTestNode{
-				tv, Position{9, 1},
+				tv, toml.Position{9, 1},
 			},
 		})
 
@@ -477,7 +476,7 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(bool)]",
 		[]interface{}{
 			queryTestNode{
-				true, Position{15, 1},
+				true, toml.Position{15, 1},
 			},
 		})
 }

query/query.go

@@ -1,7 +1,9 @@
-package toml
+package query
 
 import (
 	"time"
+
+	"github.com/pelletier/go-toml"
 )
 
 // NodeFilterFn represents a user-defined filter function, for use with
@@ -15,50 +17,43 @@ import (
 // to use from multiple goroutines.
 type NodeFilterFn func(node interface{}) bool
 
-// QueryResult is the result of Executing a Query.
+// Result is the result of Executing a Query.
-type QueryResult struct {
+type Result struct {
 	items     []interface{}
-	positions []Position
+	positions []toml.Position
 }
 
 // appends a value/position pair to the result set.
-func (r *QueryResult) appendResult(node interface{}, pos Position) {
+func (r *Result) appendResult(node interface{}, pos toml.Position) {
 	r.items = append(r.items, node)
 	r.positions = append(r.positions, pos)
 }
 
-// Values is a set of values within a QueryResult.  The order of values is not
+// Values is a set of values within a Result.  The order of values is not
 // guaranteed to be in document order, and may be different each time a query is
 // executed.
-func (r QueryResult) Values() []interface{} {
+func (r Result) Values() []interface{} {
-	values := make([]interface{}, len(r.items))
+	return r.items
-	for i, v := range r.items {
-		o, ok := v.(*tomlValue)
-		if ok {
-			values[i] = o.value
-		} else {
-			values[i] = v
-		}
-	}
-	return values
 }
 
-// Positions is a set of positions for values within a QueryResult.  Each index
+// Positions is a set of positions for values within a Result.  Each index
 // in Positions() corresponds to the entry in Value() of the same index.
-func (r QueryResult) Positions() []Position {
+func (r Result) Positions() []toml.Position {
 	return r.positions
 }
 
 // runtime context for executing query paths
 type queryContext struct {
-	result       *QueryResult
+	result       *Result
 	filters      *map[string]NodeFilterFn
-	lastPosition Position
+	lastPosition toml.Position
 }
 
 // generic path functor interface
 type pathFn interface {
 	setNext(next pathFn)
+	// it is the caller's responsibility to set the ctx.lastPosition before invoking call()
+	// node can be one of: *toml.Tree, []*toml.Tree, or a scalar
 	call(node interface{}, ctx *queryContext)
 }
 
@@ -88,17 +83,17 @@ func (q *Query) appendPath(next pathFn) {
 	next.setNext(newTerminatingFn()) // init the next functor
 }
 
-// CompileQuery compiles a TOML path expression.  The returned Query can be used
+// Compile compiles a TOML path expression. The returned Query can be used
-// to match elements within a TomlTree and its descendants.
+// to match elements within a Tree and its descendants. See Execute.
-func CompileQuery(path string) (*Query, error) {
+func Compile(path string) (*Query, error) {
 	return parseQuery(lexQuery(path))
 }
 
-// Execute executes a query against a TomlTree, and returns the result of the query.
+// Execute executes a query against a Tree, and returns the result of the query.
-func (q *Query) Execute(tree *TomlTree) *QueryResult {
+func (q *Query) Execute(tree *toml.Tree) *Result {
-	result := &QueryResult{
+	result := &Result{
 		items:     []interface{}{},
-		positions: []Position{},
+		positions: []toml.Position{},
 	}
 	if q.root == nil {
 		result.appendResult(tree, tree.GetPosition(""))
@@ -107,11 +102,21 @@ func (q *Query) Execute(tree *TomlTree) *QueryResult {
 			result:  result,
 			filters: q.filters,
 		}
+		ctx.lastPosition = tree.Position()
 		q.root.call(tree, ctx)
 	}
 	return result
 }
 
+// CompileAndExecute is a shorthand for Compile(path) followed by Execute(tree).
+func CompileAndExecute(path string, tree *toml.Tree) (*Result, error) {
+	query, err := Compile(path)
+	if err != nil {
+		return nil, err
+	}
+	return query.Execute(tree), nil
+}
+
 // SetFilter sets a user-defined filter function.  These may be used inside
 // "?(..)" query expressions to filter TOML document elements within a query.
 func (q *Query) SetFilter(name string, fn NodeFilterFn) {
@@ -127,7 +132,7 @@ func (q *Query) SetFilter(name string, fn NodeFilterFn) {
 
 var defaultFilterFunctions = map[string]NodeFilterFn{
 	"tree": func(node interface{}) bool {
-		_, ok := node.(*TomlTree)
+		_, ok := node.(*toml.Tree)
 		return ok
 	},
 	"int": func(node interface{}) bool {

query/query_test.go

@@ -0,0 +1,157 @@
+package query
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pelletier/go-toml"
+)
+
+func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
+	if len(array) != len(objects) {
+		t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
+	}
+
+	for _, o := range objects {
+		found := false
+		for _, a := range array {
+			if a == o {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal(o, "not found in array", array)
+		}
+	}
+}
+
+func TestQueryExample(t *testing.T) {
+	config, _ := toml.Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+	authors, err := CompileAndExecute("$.book.author", config)
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	names := authors.Values()
+	if len(names) != 3 {
+		t.Fatalf("query should return 3 names but returned %d", len(names))
+	}
+	assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
+}
+
+func TestQueryReadmeExample(t *testing.T) {
+	config, _ := toml.Load(`
+[postgres]
+user = "pelletier"
+password = "mypassword"
+`)
+
+	query, err := Compile("$..[user,password]")
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	results := query.Execute(config)
+	values := results.Values()
+	if len(values) != 2 {
+		t.Fatalf("query should return 2 values but returned %d", len(values))
+	}
+	assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
+}
+
+func TestQueryPathNotPresent(t *testing.T) {
+	config, _ := toml.Load(`a = "hello"`)
+	query, err := Compile("$.foo.bar")
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	results := query.Execute(config)
+	if err != nil {
+		t.Fatalf("err should be nil. got %s instead", err)
+	}
+	if len(results.items) != 0 {
+		t.Fatalf("no items should be matched. %d matched instead", len(results.items))
+	}
+}
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := toml.Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := Compile("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*toml.Tree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' Tree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := toml.Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	query, _ := Compile("$.book.author")
+	authors := query.Execute(config)
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := toml.Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	query, err := Compile("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result := query.Execute(tree)
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*toml.Tree); !ok {
+		t.Errorf("Expected type of Tree: %T", values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}

query/tokens.go

@@ -0,0 +1,106 @@
+package query
+
+import (
+	"fmt"
+	"github.com/pelletier/go-toml"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenFloat
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftParen
+	tokenRightParen
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+)
+
+var tokenTypeNames = []string{
+	"Error",
+	"EOF",
+	"Key",
+	"String",
+	"Integer",
+	"Float",
+	"[",
+	"]",
+	"(",
+	")",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+}
+
+type token struct {
+	toml.Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
+}

query_test.go

@@ -1,70 +0,0 @@
-package toml
-
-import (
-	"testing"
-)
-
-func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
-	if len(array) != len(objects) {
-		t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
-	}
-
-	for _, o := range objects {
-		found := false
-		for _, a := range array {
-			if a == o {
-				found = true
-				break
-			}
-		}
-		if !found {
-			t.Fatal(o, "not found in array", array)
-		}
-	}
-}
-
-func TestQueryExample(t *testing.T) {
-	config, _ := Load(`
-      [[book]]
-      title = "The Stand"
-      author = "Stephen King"
-      [[book]]
-      title = "For Whom the Bell Tolls"
-      author = "Ernest Hemmingway"
-      [[book]]
-      title = "Neuromancer"
-      author = "William Gibson"
-    `)
-
-	authors, _ := config.Query("$.book.author")
-	names := authors.Values()
-	if len(names) != 3 {
-		t.Fatalf("query should return 3 names but returned %d", len(names))
-	}
-	assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
-}
-
-func TestQueryReadmeExample(t *testing.T) {
-	config, _ := Load(`
-[postgres]
-user = "pelletier"
-password = "mypassword"
-`)
-	results, _ := config.Query("$..[user,password]")
-	values := results.Values()
-	if len(values) != 2 {
-		t.Fatalf("query should return 2 values but returned %d", len(values))
-	}
-	assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
-}
-
-func TestQueryPathNotPresent(t *testing.T) {
-	config, _ := Load(`a = "hello"`)
-	results, err := config.Query("$.foo.bar")
-	if err != nil {
-		t.Fatalf("err should be nil. got %s instead", err)
-	}
-	if len(results.items) != 0 {
-		t.Fatalf("no items should be matched. %d matched instead", len(results.items))
-	}
-}

test.sh

@@ -19,8 +19,16 @@ function git_clone() {
   popd
 }
 
+# Remove potential previous runs
+rm -rf src test_program_bin toml-test
+
+# Run go vet
+go vet ./...
+
 go get github.com/pelletier/go-buffruneio
 go get github.com/davecgh/go-spew/spew
+go get gopkg.in/yaml.v2
+go get github.com/BurntSushi/toml
 
 # get code for BurntSushi TOML validation
 # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)
@@ -33,13 +41,16 @@ go build -o toml-test github.com/BurntSushi/toml-test
 # vendorize the current lib for testing
 # NOTE: this basically mocks an install without having to go back out to github for code
 mkdir -p src/github.com/pelletier/go-toml/cmd
+mkdir -p src/github.com/pelletier/go-toml/query
 cp *.go *.toml src/github.com/pelletier/go-toml
 cp -R cmd/* src/github.com/pelletier/go-toml/cmd
+cp -R query/* src/github.com/pelletier/go-toml/query
 go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
 
 # Run basic unit tests
-go test github.com/pelletier/go-toml \
+go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out
-        github.com/pelletier/go-toml/cmd/tomljson
+go test github.com/pelletier/go-toml/cmd/tomljson
+go test github.com/pelletier/go-toml/query
 
 # run the entire BurntSushi test suite
 if [[ $# -eq 0 ]] ; then

token.go

@@ -135,5 +135,6 @@ func isDigit(r rune) bool {
 
 func isHexDigit(r rune) bool {
 	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
 }

toml.go

@@ -4,38 +4,46 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
 )
 
 type tomlValue struct {
-	value    interface{}
+	value    interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list
 	position Position
 }
 
-// TomlTree is the result of the parsing of a TOML file.
+// Tree is the result of the parsing of a TOML file.
-type TomlTree struct {
+type Tree struct {
-	values   map[string]interface{}
+	values   map[string]interface{} // string -> *tomlValue, *Tree, []*Tree
 	position Position
 }
 
-func newTomlTree() *TomlTree {
+func newTree() *Tree {
-	return &TomlTree{
+	return &Tree{
 		values:   make(map[string]interface{}),
 		position: Position{},
 	}
 }
 
-// TreeFromMap initializes a new TomlTree object using the given map.
+// TreeFromMap initializes a new Tree object using the given map.
-func TreeFromMap(m map[string]interface{}) *TomlTree {
+func TreeFromMap(m map[string]interface{}) (*Tree, error) {
-	return &TomlTree{
+	result, err := toTree(m)
-		values: m,
+	if err != nil {
+		return nil, err
 	}
+	return result.(*Tree), nil
+}
+
+// Position returns the position of the tree.
+func (t *Tree) Position() Position {
+	return t.position
 }
 
 // Has returns a boolean indicating if the given key exists.
-func (t *TomlTree) Has(key string) bool {
+func (t *Tree) Has(key string) bool {
 	if key == "" {
 		return false
 	}
@@ -43,25 +51,26 @@ func (t *TomlTree) Has(key string) bool {
 }
 
 // HasPath returns true if the given path of keys exists, false otherwise.
-func (t *TomlTree) HasPath(keys []string) bool {
+func (t *Tree) HasPath(keys []string) bool {
 	return t.GetPath(keys) != nil
 }
 
-// Keys returns the keys of the toplevel tree.
+// Keys returns the keys of the toplevel tree (does not recurse).
-// Warning: this is a costly operation.
+func (t *Tree) Keys() []string {
-func (t *TomlTree) Keys() []string {
+	keys := make([]string, len(t.values))
-	var keys []string
+	i := 0
 	for k := range t.values {
-		keys = append(keys, k)
+		keys[i] = k
+		i++
 	}
 	return keys
 }
 
-// Get the value at key in the TomlTree.
+// Get the value at key in the Tree.
 // Key is a dot-separated path (e.g. a.b.c).
 // Returns nil if the path does not exist in the tree.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) Get(key string) interface{} {
+func (t *Tree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
@@ -74,7 +83,7 @@ func (t *TomlTree) Get(key string) interface{} {
 
 // GetPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) GetPath(keys []string) interface{} {
+func (t *Tree) GetPath(keys []string) interface{} {
 	if len(keys) == 0 {
 		return t
 	}
@@ -85,9 +94,9 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 			return nil
 		}
 		switch node := value.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				return nil
@@ -107,7 +116,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 }
 
 // GetPosition returns the position of the given key.
-func (t *TomlTree) GetPosition(key string) Position {
+func (t *Tree) GetPosition(key string) Position {
 	if key == "" {
 		return t.position
 	}
@@ -116,7 +125,7 @@ func (t *TomlTree) GetPosition(key string) Position {
 
 // GetPositionPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) GetPositionPath(keys []string) Position {
+func (t *Tree) GetPositionPath(keys []string) Position {
 	if len(keys) == 0 {
 		return t.position
 	}
@@ -127,9 +136,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 			return Position{0, 0}
 		}
 		switch node := value.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				return Position{0, 0}
@@ -143,9 +152,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 	switch node := subtree.values[keys[len(keys)-1]].(type) {
 	case *tomlValue:
 		return node.position
-	case *TomlTree:
+	case *Tree:
 		return node.position
-	case []*TomlTree:
+	case []*Tree:
 		// go to most recent element
 		if len(node) == 0 {
 			return Position{0, 0}
@@ -157,7 +166,7 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 }
 
 // GetDefault works like Get but with a default value
-func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
+func (t *Tree) GetDefault(key string, def interface{}) interface{} {
 	val := t.Get(key)
 	if val == nil {
 		return def
@@ -167,30 +176,30 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
 
 // Set an element in the tree.
 // Key is a dot-separated path (e.g. a.b.c).
-// Creates all necessary intermediates trees, if needed.
+// Creates all necessary intermediate trees, if needed.
-func (t *TomlTree) Set(key string, value interface{}) {
+func (t *Tree) Set(key string, value interface{}) {
 	t.SetPath(strings.Split(key, "."), value)
 }
 
 // SetPath sets an element in the tree.
 // Keys is an array of path elements (e.g. {"a","b","c"}).
-// Creates all necessary intermediates trees, if needed.
+// Creates all necessary intermediate trees, if needed.
-func (t *TomlTree) SetPath(keys []string, value interface{}) {
+func (t *Tree) SetPath(keys []string, value interface{}) {
 	subtree := t
 	for _, intermediateKey := range keys[:len(keys)-1] {
 		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			nextTree = newTomlTree()
+			nextTree = newTree()
 			subtree.values[intermediateKey] = nextTree // add new element here
 		}
 		switch node := nextTree.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				// create element if it does not exist
-				subtree.values[intermediateKey] = append(node, newTomlTree())
+				subtree.values[intermediateKey] = append(node, newTree())
 			}
 			subtree = node[len(node)-1]
 		}
@@ -199,9 +208,9 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	var toInsert interface{}
 
 	switch value.(type) {
-	case *TomlTree:
+	case *Tree:
 		toInsert = value
-	case []*TomlTree:
+	case []*Tree:
 		toInsert = value
 	case *tomlValue:
 		toInsert = value
@@ -219,21 +228,21 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 // and tree[a][b][c]
 //
 // Returns nil on success, error object on failure
-func (t *TomlTree) createSubTree(keys []string, pos Position) error {
+func (t *Tree) createSubTree(keys []string, pos Position) error {
 	subtree := t
 	for _, intermediateKey := range keys {
 		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			tree := newTomlTree()
+			tree := newTree()
 			tree.position = pos
 			subtree.values[intermediateKey] = tree
 			nextTree = tree
 		}
 
 		switch node := nextTree.(type) {
-		case []*TomlTree:
+		case []*Tree:
 			subtree = node[len(node)-1]
-		case *TomlTree:
+		case *Tree:
 			subtree = node
 		default:
 			return fmt.Errorf("unknown type for path %s (%s): %T (%#v)",
@@ -243,17 +252,8 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	return nil
 }
 
-// Query compiles and executes a query on a tree and returns the query result.
+// LoadBytes creates a Tree from a []byte.
-func (t *TomlTree) Query(query string) (*QueryResult, error) {
+func LoadBytes(b []byte) (tree *Tree, err error) {
-	q, err := CompileQuery(query)
-	if err != nil {
-		return nil, err
-	}
-	return q.Execute(t), nil
-}
-
-// LoadReader creates a TomlTree from any io.Reader.
-func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -262,17 +262,27 @@ func LoadReader(reader io.Reader) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	tree = parseToml(lexToml(reader))
+	tree = parseToml(lexToml(b))
 	return
 }
 
-// Load creates a TomlTree from a string.
+// LoadReader creates a Tree from any io.Reader.
-func Load(content string) (tree *TomlTree, err error) {
+func LoadReader(reader io.Reader) (tree *Tree, err error) {
-	return LoadReader(strings.NewReader(content))
+	inputBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return
+	}
+	tree, err = LoadBytes(inputBytes)
+	return
 }
 
-// LoadFile creates a TomlTree from a file.
+// Load creates a Tree from a string.
-func LoadFile(path string) (tree *TomlTree, err error) {
+func Load(content string) (tree *Tree, err error) {
+	return LoadBytes([]byte(content))
+}
+
+// LoadFile creates a Tree from a file.
+func LoadFile(path string) (tree *Tree, err error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, err

toml_test.go

@@ -70,12 +70,12 @@ func TestTomlHasPath(t *testing.T) {
 }
 
 func TestTomlGetPath(t *testing.T) {
-	node := newTomlTree()
+	node := newTree()
 	//TODO: set other node data
 
 	for idx, item := range []struct {
 		Path     []string
-		Expected *TomlTree
+		Expected *Tree
 	}{
 		{ // empty path test
 			[]string{},
@@ -94,35 +94,13 @@ func TestTomlGetPath(t *testing.T) {
 	}
 }
 
-func TestTomlQuery(t *testing.T) {
-	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	result, err := tree.Query("$.foo.bar")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	values := result.Values()
-	if len(values) != 1 {
-		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
-	}
-
-	if tt, ok := values[0].(*TomlTree); !ok {
-		t.Errorf("Expected type of TomlTree: %T", values[0])
-	} else if tt.Get("a") != int64(1) {
-		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
-	} else if tt.Get("b") != int64(2) {
-		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
-	}
-}
-
 func TestTomlFromMap(t *testing.T) {
 	simpleMap := map[string]interface{}{"hello": 42}
-	tree := TreeFromMap(simpleMap)
+	tree, err := TreeFromMap(simpleMap)
-	if tree.Get("hello") != 42 {
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	if tree.Get("hello") != int64(42) {
 		t.Fatal("hello should be 42, not", tree.Get("hello"))
 	}
 }

tomltree_conversions.go

@@ -1,207 +0,0 @@
-package toml
-
-// Tools to convert a TomlTree to different representations
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// encodes a string to a TOML-compliant string value
-func encodeTomlString(value string) string {
-	result := ""
-	for _, rr := range value {
-		intRr := uint16(rr)
-		switch rr {
-		case '\b':
-			result += "\\b"
-		case '\t':
-			result += "\\t"
-		case '\n':
-			result += "\\n"
-		case '\f':
-			result += "\\f"
-		case '\r':
-			result += "\\r"
-		case '"':
-			result += "\\\""
-		case '\\':
-			result += "\\\\"
-		default:
-			if intRr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", intRr)
-			} else {
-				result += string(rr)
-			}
-		}
-	}
-	return result
-}
-
-// Value print support function for ToString()
-// Outputs the TOML compliant string representation of a value
-func toTomlValue(item interface{}, indent int) string {
-	tab := strings.Repeat(" ", indent)
-	switch value := item.(type) {
-	case int:
-		return tab + strconv.FormatInt(int64(value), 10)
-	case int8:
-		return tab + strconv.FormatInt(int64(value), 10)
-	case int16:
-		return tab + strconv.FormatInt(int64(value), 10)
-	case int32:
-		return tab + strconv.FormatInt(int64(value), 10)
-	case int64:
-		return tab + strconv.FormatInt(value, 10)
-	case uint:
-		return tab + strconv.FormatUint(uint64(value), 10)
-	case uint8:
-		return tab + strconv.FormatUint(uint64(value), 10)
-	case uint16:
-		return tab + strconv.FormatUint(uint64(value), 10)
-	case uint32:
-		return tab + strconv.FormatUint(uint64(value), 10)
-	case uint64:
-		return tab + strconv.FormatUint(value, 10)
-	case float32:
-		return tab + strconv.FormatFloat(float64(value), 'f', -1, 32)
-	case float64:
-		return tab + strconv.FormatFloat(value, 'f', -1, 64)
-	case string:
-		return tab + "\"" + encodeTomlString(value) + "\""
-	case bool:
-		if value {
-			return "true"
-		}
-		return "false"
-	case time.Time:
-		return tab + value.Format(time.RFC3339)
-	case []interface{}:
-		result := tab + "[\n"
-		for _, item := range value {
-			result += toTomlValue(item, indent+2) + ",\n"
-		}
-		return result + tab + "]"
-	case nil:
-		return ""
-	default:
-		panic(fmt.Sprintf("unsupported value type %T: %v", value, value))
-	}
-}
-
-// Recursive support function for ToString()
-// Outputs a tree, using the provided keyspace to prefix group names
-func (t *TomlTree) toToml(indent, keyspace string) string {
-	resultChunks := []string{}
-	for k, v := range t.values {
-		// figure out the keyspace
-		combinedKey := k
-		if keyspace != "" {
-			combinedKey = keyspace + "." + combinedKey
-		}
-		resultChunk := ""
-		// output based on type
-		switch node := v.(type) {
-		case []*TomlTree:
-			for _, item := range node {
-				if len(item.Keys()) > 0 {
-					resultChunk += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
-				}
-				resultChunk += item.toToml(indent+"  ", combinedKey)
-			}
-			resultChunks = append(resultChunks, resultChunk)
-		case *TomlTree:
-			if len(node.Keys()) > 0 {
-				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			resultChunk += node.toToml(indent+"  ", combinedKey)
-			resultChunks = append(resultChunks, resultChunk)
-		case map[string]interface{}:
-			sub := TreeFromMap(node)
-
-			if len(sub.Keys()) > 0 {
-				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			resultChunk += sub.toToml(indent+"  ", combinedKey)
-			resultChunks = append(resultChunks, resultChunk)
-		case map[string]string:
-			sub := TreeFromMap(convertMapStringString(node))
-
-			if len(sub.Keys()) > 0 {
-				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			resultChunk += sub.toToml(indent+"  ", combinedKey)
-			resultChunks = append(resultChunks, resultChunk)
-		case map[interface{}]interface{}:
-			sub := TreeFromMap(convertMapInterfaceInterface(node))
-
-			if len(sub.Keys()) > 0 {
-				resultChunk += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			resultChunk += sub.toToml(indent+"  ", combinedKey)
-			resultChunks = append(resultChunks, resultChunk)
-		case *tomlValue:
-			resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
-			resultChunks = append([]string{resultChunk}, resultChunks...)
-		default:
-			resultChunk = fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
-			resultChunks = append([]string{resultChunk}, resultChunks...)
-		}
-
-	}
-	return strings.Join(resultChunks, "")
-}
-
-func convertMapStringString(in map[string]string) map[string]interface{} {
-	result := make(map[string]interface{}, len(in))
-	for k, v := range in {
-		result[k] = v
-	}
-	return result
-}
-
-func convertMapInterfaceInterface(in map[interface{}]interface{}) map[string]interface{} {
-	result := make(map[string]interface{}, len(in))
-	for k, v := range in {
-		result[k.(string)] = v
-	}
-	return result
-}
-
-// ToString is an alias for String
-func (t *TomlTree) ToString() string {
-	return t.String()
-}
-
-// String generates a human-readable representation of the current tree.
-// Output spans multiple lines, and is suitable for ingest by a TOML parser
-func (t *TomlTree) String() string {
-	return t.toToml("", "")
-}
-
-// ToMap recursively generates a representation of the current tree using map[string]interface{}.
-func (t *TomlTree) ToMap() map[string]interface{} {
-	result := map[string]interface{}{}
-
-	for k, v := range t.values {
-		switch node := v.(type) {
-		case []*TomlTree:
-			var array []interface{}
-			for _, item := range node {
-				array = append(array, item.ToMap())
-			}
-			result[k] = array
-		case *TomlTree:
-			result[k] = node.ToMap()
-		case map[string]interface{}:
-			sub := TreeFromMap(node)
-			result[k] = sub.ToMap()
-		case *tomlValue:
-			result[k] = node.value
-		}
-	}
-
-	return result
-}

tomltree_conversions_test.go

@@ -1,171 +0,0 @@
-package toml
-
-import (
-	"reflect"
-	"testing"
-	"time"
-	"strings"
-)
-
-func TestTomlTreeConversionToString(t *testing.T) {
-	toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
-points = { x = 1, y = 2 }`)
-
-	if err != nil {
-		t.Fatal("Unexpected error:", err)
-	}
-
-	reparsedTree, err := Load(toml.ToString())
-
-	assertTree(t, reparsedTree, err, map[string]interface{}{
-		"name": map[string]interface{}{
-			"first": "Tom",
-			"last":  "Preston-Werner",
-		},
-		"points": map[string]interface{}{
-			"x": int64(1),
-			"y": int64(2),
-		},
-	})
-}
-
-func TestTomlTreeConversionToStringKeysOrders(t *testing.T) {
-	for i := 0; i < 100; i++ {
-		tree, _ := Load(`
-		foobar = true
-		bar = "baz"
-		foo = 1
-		[qux]
-		  foo = 1
-		  bar = "baz2"`)
-
-		stringRepr := tree.ToString()
-
-		t.Log("Intermediate string representation:")
-		t.Log(stringRepr)
-
-		r := strings.NewReader(stringRepr)
-		toml, err := LoadReader(r)
-
-
-		if err != nil {
-			t.Fatal("Unexpected error:", err)
-		}
-
-		assertTree(t, toml, err, map[string]interface{}{
-			"foobar": true,
-			"bar": "baz",
-			"foo": 1,
-			"qux": map[string]interface{}{
-				"foo": 1,
-				"bar": "baz2",
-			},
-		})
-	}
-}
-
-func testMaps(t *testing.T, actual, expected map[string]interface{}) {
-	if !reflect.DeepEqual(actual, expected) {
-		t.Fatal("trees aren't equal.\n", "Expected:\n", expected, "\nActual:\n", actual)
-	}
-}
-
-func TestTomlTreeConversionToMapSimple(t *testing.T) {
-	tree, _ := Load("a = 42\nb = 17")
-
-	expected := map[string]interface{}{
-		"a": int64(42),
-		"b": int64(17),
-	}
-
-	testMaps(t, tree.ToMap(), expected)
-}
-
-func TestTomlTreeConversionToMapExampleFile(t *testing.T) {
-	tree, _ := LoadFile("example.toml")
-	expected := map[string]interface{}{
-		"title": "TOML Example",
-		"owner": map[string]interface{}{
-			"name":         "Tom Preston-Werner",
-			"organization": "GitHub",
-			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
-			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
-		},
-		"database": map[string]interface{}{
-			"server":         "192.168.1.1",
-			"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
-			"connection_max": int64(5000),
-			"enabled":        true,
-		},
-		"servers": map[string]interface{}{
-			"alpha": map[string]interface{}{
-				"ip": "10.0.0.1",
-				"dc": "eqdc10",
-			},
-			"beta": map[string]interface{}{
-				"ip": "10.0.0.2",
-				"dc": "eqdc10",
-			},
-		},
-		"clients": map[string]interface{}{
-			"data": []interface{}{
-				[]interface{}{"gamma", "delta"},
-				[]interface{}{int64(1), int64(2)},
-			},
-		},
-	}
-	testMaps(t, tree.ToMap(), expected)
-}
-
-func TestTomlTreeConversionToMapWithTablesInMultipleChunks(t *testing.T) {
-	tree, _ := Load(`
-	[[menu.main]]
-        a = "menu 1"
-        b = "menu 2"
-        [[menu.main]]
-        c = "menu 3"
-        d = "menu 4"`)
-	expected := map[string]interface{}{
-		"menu": map[string]interface{}{
-			"main": []interface{}{
-				map[string]interface{}{"a": "menu 1", "b": "menu 2"},
-				map[string]interface{}{"c": "menu 3", "d": "menu 4"},
-			},
-		},
-	}
-	treeMap := tree.ToMap()
-
-	testMaps(t, treeMap, expected)
-}
-
-func TestTomlTreeConversionToMapWithArrayOfInlineTables(t *testing.T) {
-	tree, _ := Load(`
-    	[params]
-	language_tabs = [
-    		{ key = "shell", name = "Shell" },
-    		{ key = "ruby", name = "Ruby" },
-    		{ key = "python", name = "Python" }
-	]`)
-
-	expected := map[string]interface{}{
-		"params": map[string]interface{}{
-			"language_tabs": []interface{}{
-				map[string]interface{}{
-					"key": "shell",
-					"name": "Shell",
-				},
-				map[string]interface{}{
-					"key": "ruby",
-					"name": "Ruby",
-				},
-				map[string]interface{}{
-					"key": "python",
-					"name": "Python",
-				},
-			},
-		},
-	}
-
-	treeMap := tree.ToMap()
-	testMaps(t, treeMap, expected)
-}

tomltree_create.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+)
+
+var kindToType = [reflect.String + 1]reflect.Type{
+	reflect.Bool:    reflect.TypeOf(true),
+	reflect.String:  reflect.TypeOf(""),
+	reflect.Float32: reflect.TypeOf(float64(1)),
+	reflect.Float64: reflect.TypeOf(float64(1)),
+	reflect.Int:     reflect.TypeOf(int64(1)),
+	reflect.Int8:    reflect.TypeOf(int64(1)),
+	reflect.Int16:   reflect.TypeOf(int64(1)),
+	reflect.Int32:   reflect.TypeOf(int64(1)),
+	reflect.Int64:   reflect.TypeOf(int64(1)),
+	reflect.Uint:    reflect.TypeOf(uint64(1)),
+	reflect.Uint8:   reflect.TypeOf(uint64(1)),
+	reflect.Uint16:  reflect.TypeOf(uint64(1)),
+	reflect.Uint32:  reflect.TypeOf(uint64(1)),
+	reflect.Uint64:  reflect.TypeOf(uint64(1)),
+}
+
+// typeFor returns a reflect.Type for a reflect.Kind, or nil if none is found.
+// supported values:
+// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32
+func typeFor(k reflect.Kind) reflect.Type {
+	if k > 0 && int(k) < len(kindToType) {
+		return kindToType[k]
+	}
+	return nil
+}
+
+func simpleValueCoercion(object interface{}) (interface{}, error) {
+	switch original := object.(type) {
+	case string, bool, int64, uint64, float64, time.Time:
+		return original, nil
+	case int:
+		return int64(original), nil
+	case int8:
+		return int64(original), nil
+	case int16:
+		return int64(original), nil
+	case int32:
+		return int64(original), nil
+	case uint:
+		return uint64(original), nil
+	case uint8:
+		return uint64(original), nil
+	case uint16:
+		return uint64(original), nil
+	case uint32:
+		return uint64(original), nil
+	case float32:
+		return float64(original), nil
+	case fmt.Stringer:
+		return original.String(), nil
+	default:
+		return nil, fmt.Errorf("cannot convert type %T to Tree", object)
+	}
+}
+
+func sliceToTree(object interface{}) (interface{}, error) {
+	// arrays are a bit tricky, since they can represent either a
+	// collection of simple values, which is represented by one
+	// *tomlValue, or an array of tables, which is represented by an
+	// array of *Tree.
+
+	// holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice
+	value := reflect.ValueOf(object)
+	insideType := value.Type().Elem()
+	length := value.Len()
+	if length > 0 {
+		insideType = reflect.ValueOf(value.Index(0).Interface()).Type()
+	}
+	if insideType.Kind() == reflect.Map {
+		// this is considered as an array of tables
+		tablesArray := make([]*Tree, 0, length)
+		for i := 0; i < length; i++ {
+			table := value.Index(i)
+			tree, err := toTree(table.Interface())
+			if err != nil {
+				return nil, err
+			}
+			tablesArray = append(tablesArray, tree.(*Tree))
+		}
+		return tablesArray, nil
+	}
+
+	sliceType := typeFor(insideType.Kind())
+	if sliceType == nil {
+		sliceType = insideType
+	}
+
+	arrayValue := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, length)
+
+	for i := 0; i < length; i++ {
+		val := value.Index(i).Interface()
+		simpleValue, err := simpleValueCoercion(val)
+		if err != nil {
+			return nil, err
+		}
+		arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue))
+	}
+	return &tomlValue{arrayValue.Interface(), Position{}}, nil
+}
+
+func toTree(object interface{}) (interface{}, error) {
+	value := reflect.ValueOf(object)
+
+	if value.Kind() == reflect.Map {
+		values := map[string]interface{}{}
+		keys := value.MapKeys()
+		for _, key := range keys {
+			if key.Kind() != reflect.String {
+				if _, ok := key.Interface().(string); !ok {
+					return nil, fmt.Errorf("map key needs to be a string, not %T (%v)", key.Interface(), key.Kind())
+				}
+			}
+
+			v := value.MapIndex(key)
+			newValue, err := toTree(v.Interface())
+			if err != nil {
+				return nil, err
+			}
+			values[key.String()] = newValue
+		}
+		return &Tree{values, Position{}}, nil
+	}
+
+	if value.Kind() == reflect.Array || value.Kind() == reflect.Slice {
+		return sliceToTree(object)
+	}
+
+	simpleValue, err := simpleValueCoercion(object)
+	if err != nil {
+		return nil, err
+	}
+	return &tomlValue{simpleValue, Position{}}, nil
+}

tomltree_create_test.go

@@ -0,0 +1,126 @@
+package toml
+
+import (
+	"strconv"
+	"testing"
+	"time"
+)
+
+type customString string
+
+type stringer struct{}
+
+func (s stringer) String() string {
+	return "stringer"
+}
+
+func validate(t *testing.T, path string, object interface{}) {
+	switch o := object.(type) {
+	case *Tree:
+		for key, tree := range o.values {
+			validate(t, path+"."+key, tree)
+		}
+	case []*Tree:
+		for index, tree := range o {
+			validate(t, path+"."+strconv.Itoa(index), tree)
+		}
+	case *tomlValue:
+		switch o.value.(type) {
+		case int64, uint64, bool, string, float64, time.Time,
+			[]int64, []uint64, []bool, []string, []float64, []time.Time:
+		default:
+			t.Fatalf("tomlValue at key %s containing incorrect type %T", path, o.value)
+		}
+	default:
+		t.Fatalf("value at key %s is of incorrect type %T", path, object)
+	}
+	t.Logf("validation ok %s as %T", path, object)
+}
+
+func validateTree(t *testing.T, tree *Tree) {
+	validate(t, "", tree)
+}
+
+func TestTreeCreateToTree(t *testing.T) {
+	data := map[string]interface{}{
+		"a_string": "bar",
+		"an_int":   42,
+		"time":     time.Now(),
+		"int8":     int8(2),
+		"int16":    int16(2),
+		"int32":    int32(2),
+		"uint8":    uint8(2),
+		"uint16":   uint16(2),
+		"uint32":   uint32(2),
+		"float32":  float32(2),
+		"a_bool":   false,
+		"stringer": stringer{},
+		"nested": map[string]interface{}{
+			"foo": "bar",
+		},
+		"array":                 []string{"a", "b", "c"},
+		"array_uint":            []uint{uint(1), uint(2)},
+		"array_table":           []map[string]interface{}{map[string]interface{}{"sub_map": 52}},
+		"array_times":           []time.Time{time.Now(), time.Now()},
+		"map_times":             map[string]time.Time{"now": time.Now()},
+		"custom_string_map_key": map[customString]interface{}{customString("custom"): "custom"},
+	}
+	tree, err := TreeFromMap(data)
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	validateTree(t, tree)
+}
+
+func TestTreeCreateToTreeInvalidLeafType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": t})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidMapKeyType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": map[int]interface{}{2: 1}})
+	expected := "map key needs to be a string, not int (int)"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidArrayMemberType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": []*testing.T{t}})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidTableGroupType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"hello": t}}})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestRoundTripArrayOfTables(t *testing.T) {
+	orig := "\n[[stuff]]\n  name = \"foo\"\n  things = [\"a\",\"b\"]\n"
+	tree, err := Load(orig)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	m := tree.ToMap()
+
+	tree, err = TreeFromMap(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	want := orig
+	got := tree.String()
+
+	if got != want {
+		t.Errorf("want:\n%s\ngot:\n%s", want, got)
+	}
+}

tomltree_write.go

@@ -0,0 +1,233 @@
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	var b bytes.Buffer
+
+	for _, rr := range value {
+		switch rr {
+		case '\b':
+			b.WriteString(`\b`)
+		case '\t':
+			b.WriteString(`\t`)
+		case '\n':
+			b.WriteString(`\n`)
+		case '\f':
+			b.WriteString(`\f`)
+		case '\r':
+			b.WriteString(`\r`)
+		case '"':
+			b.WriteString(`\"`)
+		case '\\':
+			b.WriteString(`\\`)
+		default:
+			intRr := uint16(rr)
+			if intRr < 0x001F {
+				b.WriteString(fmt.Sprintf("\\u%0.4X", intRr))
+			} else {
+				b.WriteRune(rr)
+			}
+		}
+	}
+	return b.String()
+}
+
+func tomlValueStringRepresentation(v interface{}) (string, error) {
+	switch value := v.(type) {
+	case uint64:
+		return strconv.FormatUint(value, 10), nil
+	case int64:
+		return strconv.FormatInt(value, 10), nil
+	case float64:
+		// Ensure a round float does contain a decimal point. Otherwise feeding
+		// the output back to the parser would convert to an integer.
+		if math.Trunc(value) == value {
+			return strconv.FormatFloat(value, 'f', 1, 32), nil
+		}
+		return strconv.FormatFloat(value, 'f', -1, 32), nil
+	case string:
+		return "\"" + encodeTomlString(value) + "\"", nil
+	case []byte:
+		b, _ := v.([]byte)
+		return tomlValueStringRepresentation(string(b))
+	case bool:
+		if value {
+			return "true", nil
+		}
+		return "false", nil
+	case time.Time:
+		return value.Format(time.RFC3339), nil
+	case nil:
+		return "", nil
+	}
+
+	rv := reflect.ValueOf(v)
+
+	if rv.Kind() == reflect.Slice {
+		values := []string{}
+		for i := 0; i < rv.Len(); i++ {
+			item := rv.Index(i).Interface()
+			itemRepr, err := tomlValueStringRepresentation(item)
+			if err != nil {
+				return "", err
+			}
+			values = append(values, itemRepr)
+		}
+		return "[" + strings.Join(values, ",") + "]", nil
+	}
+	return "", fmt.Errorf("unsupported value type %T: %v", v, v)
+}
+
+func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) {
+	simpleValuesKeys := make([]string, 0)
+	complexValuesKeys := make([]string, 0)
+
+	for k := range t.values {
+		v := t.values[k]
+		switch v.(type) {
+		case *Tree, []*Tree:
+			complexValuesKeys = append(complexValuesKeys, k)
+		default:
+			simpleValuesKeys = append(simpleValuesKeys, k)
+		}
+	}
+
+	sort.Strings(simpleValuesKeys)
+	sort.Strings(complexValuesKeys)
+
+	for _, k := range simpleValuesKeys {
+		v, ok := t.values[k].(*tomlValue)
+		if !ok {
+			return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k])
+		}
+
+		repr, err := tomlValueStringRepresentation(v.value)
+		if err != nil {
+			return bytesCount, err
+		}
+
+		writtenBytesCount, err := writeStrings(w, indent, k, " = ", repr, "\n")
+		bytesCount += int64(writtenBytesCount)
+		if err != nil {
+			return bytesCount, err
+		}
+	}
+
+	for _, k := range complexValuesKeys {
+		v := t.values[k]
+
+		combinedKey := k
+		if keyspace != "" {
+			combinedKey = keyspace + "." + combinedKey
+		}
+
+		switch node := v.(type) {
+		// node has to be of those two types given how keys are sorted above
+		case *Tree:
+			writtenBytesCount, err := writeStrings(w, "\n", indent, "[", combinedKey, "]\n")
+			bytesCount += int64(writtenBytesCount)
+			if err != nil {
+				return bytesCount, err
+			}
+			bytesCount, err = node.writeTo(w, indent+"  ", combinedKey, bytesCount)
+			if err != nil {
+				return bytesCount, err
+			}
+		case []*Tree:
+			for _, subTree := range node {
+				writtenBytesCount, err := writeStrings(w, "\n", indent, "[[", combinedKey, "]]\n")
+				bytesCount += int64(writtenBytesCount)
+				if err != nil {
+					return bytesCount, err
+				}
+
+				bytesCount, err = subTree.writeTo(w, indent+"  ", combinedKey, bytesCount)
+				if err != nil {
+					return bytesCount, err
+				}
+			}
+		}
+	}
+
+	return bytesCount, nil
+}
+
+func writeStrings(w io.Writer, s ...string) (int, error) {
+	var n int
+	for i := range s {
+		b, err := io.WriteString(w, s[i])
+		n += b
+		if err != nil {
+			return n, err
+		}
+	}
+	return n, nil
+}
+
+// WriteTo encode the Tree as Toml and writes it to the writer w.
+// Returns the number of bytes written in case of success, or an error if anything happened.
+func (t *Tree) WriteTo(w io.Writer) (int64, error) {
+	return t.writeTo(w, "", "", 0)
+}
+
+// ToTomlString generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser.
+// If the conversion cannot be performed, ToString returns a non-nil error.
+func (t *Tree) ToTomlString() (string, error) {
+	var buf bytes.Buffer
+	_, err := t.WriteTo(&buf)
+	if err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// String generates a human-readable representation of the current tree.
+// Alias of ToString. Present to implement the fmt.Stringer interface.
+func (t *Tree) String() string {
+	result, _ := t.ToTomlString()
+	return result
+}
+
+// ToMap recursively generates a representation of the tree using Go built-in structures.
+// The following types are used:
+//
+//	* bool
+//	* float64
+//	* int64
+//	* string
+//	* uint64
+//	* time.Time
+//	* map[string]interface{} (where interface{} is any of this list)
+//	* []interface{} (where interface{} is any of this list)
+func (t *Tree) ToMap() map[string]interface{} {
+	result := map[string]interface{}{}
+
+	for k, v := range t.values {
+		switch node := v.(type) {
+		case []*Tree:
+			var array []interface{}
+			for _, item := range node {
+				array = append(array, item.ToMap())
+			}
+			result[k] = array
+		case *Tree:
+			result[k] = node.ToMap()
+		case *tomlValue:
+			result[k] = node.value
+		}
+	}
+	return result
+}

tomltree_write_test.go

@@ -0,0 +1,358 @@
+package toml
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+type failingWriter struct {
+	failAt  int
+	written int
+	buffer  bytes.Buffer
+}
+
+func (f *failingWriter) Write(p []byte) (n int, err error) {
+	count := len(p)
+	toWrite := f.failAt - (count + f.written)
+	if toWrite < 0 {
+		toWrite = 0
+	}
+	if toWrite > count {
+		f.written += count
+		f.buffer.Write(p)
+		return count, nil
+	}
+
+	f.buffer.Write(p[:toWrite])
+	f.written = f.failAt
+	return toWrite, fmt.Errorf("failingWriter failed after writting %d bytes", f.written)
+}
+
+func assertErrorString(t *testing.T, expected string, err error) {
+	expectedErr := errors.New(expected)
+	if err == nil || err.Error() != expectedErr.Error() {
+		t.Errorf("expecting error %s, but got %s instead", expected, err)
+	}
+}
+
+func TestTreeWriteToEmptyTable(t *testing.T) {
+	doc := `[[empty-tables]]
+[[empty-tables]]`
+
+	toml, err := Load(doc)
+	if err != nil {
+		t.Fatal("Unexpected Load error:", err)
+	}
+	tomlString, err := toml.ToTomlString()
+	if err != nil {
+		t.Fatal("Unexpected ToTomlString error:", err)
+	}
+
+	expected := `
+[[empty-tables]]
+
+[[empty-tables]]
+`
+
+	if tomlString != expected {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, tomlString)
+	}
+}
+
+func TestTreeWriteToTomlString(t *testing.T) {
+	toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+points = { x = 1, y = 2 }`)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	tomlString, _ := toml.ToTomlString()
+	reparsedTree, err := Load(tomlString)
+
+	assertTree(t, reparsedTree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"points": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func TestTreeWriteToTomlStringSimple(t *testing.T) {
+	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
+	if err != nil {
+		t.Errorf("Test failed to parse: %v", err)
+		return
+	}
+	result, err := tree.ToTomlString()
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err)
+	}
+	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
+	if result != expected {
+		t.Errorf("Expected got '%s', expected '%s'", result, expected)
+	}
+}
+
+func TestTreeWriteToTomlStringKeysOrders(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		tree, _ := Load(`
+		foobar = true
+		bar = "baz"
+		foo = 1
+		[qux]
+		  foo = 1
+		  bar = "baz2"`)
+
+		stringRepr, _ := tree.ToTomlString()
+
+		t.Log("Intermediate string representation:")
+		t.Log(stringRepr)
+
+		r := strings.NewReader(stringRepr)
+		toml, err := LoadReader(r)
+
+		if err != nil {
+			t.Fatal("Unexpected error:", err)
+		}
+
+		assertTree(t, toml, err, map[string]interface{}{
+			"foobar": true,
+			"bar":    "baz",
+			"foo":    1,
+			"qux": map[string]interface{}{
+				"foo": 1,
+				"bar": "baz2",
+			},
+		})
+	}
+}
+
+func testMaps(t *testing.T, actual, expected map[string]interface{}) {
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatal("trees aren't equal.\n", "Expected:\n", expected, "\nActual:\n", actual)
+	}
+}
+
+func TestTreeWriteToMapSimple(t *testing.T) {
+	tree, _ := Load("a = 42\nb = 17")
+
+	expected := map[string]interface{}{
+		"a": int64(42),
+		"b": int64(17),
+	}
+
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTreeWriteToInvalidTreeSimpleValue(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": int8(1)}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "invalid value type at foo: int8", err)
+}
+
+func TestTreeWriteToInvalidTreeTomlValue(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{int8(1), Position{}}}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "unsupported value type int8: 1", err)
+}
+
+func TestTreeWriteToInvalidTreeTomlValueArray(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{[]interface{}{int8(1)}, Position{}}}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "unsupported value type int8: 1", err)
+}
+
+func TestTreeWriteToFailingWriterInSimpleValue(t *testing.T) {
+	toml, _ := Load(`a = 2`)
+	writer := failingWriter{failAt: 0, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writting 0 bytes", err)
+}
+
+func TestTreeWriteToFailingWriterInTable(t *testing.T) {
+	toml, _ := Load(`
+[b]
+a = 2`)
+	writer := failingWriter{failAt: 2, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
+
+	writer = failingWriter{failAt: 13, written: 0}
+	_, err = toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writting 13 bytes", err)
+}
+
+func TestTreeWriteToFailingWriterInArray(t *testing.T) {
+	toml, _ := Load(`
+[[b]]
+a = 2`)
+	writer := failingWriter{failAt: 2, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
+
+	writer = failingWriter{failAt: 15, written: 0}
+	_, err = toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writting 15 bytes", err)
+}
+
+func TestTreeWriteToMapExampleFile(t *testing.T) {
+	tree, _ := LoadFile("example.toml")
+	expected := map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+			"connection_max": int64(5000),
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]interface{}{"gamma", "delta"},
+				[]interface{}{int64(1), int64(2)},
+			},
+		},
+	}
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) {
+	tree, _ := Load(`
+	[[menu.main]]
+        a = "menu 1"
+        b = "menu 2"
+        [[menu.main]]
+        c = "menu 3"
+        d = "menu 4"`)
+	expected := map[string]interface{}{
+		"menu": map[string]interface{}{
+			"main": []interface{}{
+				map[string]interface{}{"a": "menu 1", "b": "menu 2"},
+				map[string]interface{}{"c": "menu 3", "d": "menu 4"},
+			},
+		},
+	}
+	treeMap := tree.ToMap()
+
+	testMaps(t, treeMap, expected)
+}
+
+func TestTreeWriteToMapWithArrayOfInlineTables(t *testing.T) {
+	tree, _ := Load(`
+    	[params]
+	language_tabs = [
+    		{ key = "shell", name = "Shell" },
+    		{ key = "ruby", name = "Ruby" },
+    		{ key = "python", name = "Python" }
+	]`)
+
+	expected := map[string]interface{}{
+		"params": map[string]interface{}{
+			"language_tabs": []interface{}{
+				map[string]interface{}{
+					"key":  "shell",
+					"name": "Shell",
+				},
+				map[string]interface{}{
+					"key":  "ruby",
+					"name": "Ruby",
+				},
+				map[string]interface{}{
+					"key":  "python",
+					"name": "Python",
+				},
+			},
+		},
+	}
+
+	treeMap := tree.ToMap()
+	testMaps(t, treeMap, expected)
+}
+
+func TestTreeWriteToFloat(t *testing.T) {
+	tree, err := Load(`a = 3.0`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	str, err := tree.ToTomlString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `a = 3.0`
+	if strings.TrimSpace(str) != strings.TrimSpace(expected) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str)
+	}
+}
+
+func BenchmarkTreeToTomlString(b *testing.B) {
+	toml, err := Load(sampleHard)
+	if err != nil {
+		b.Fatal("Unexpected error:", err)
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, err := toml.ToTomlString()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+var sampleHard = `# Test file for TOML
+# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate
+# This part you'll really hate
+
+[the]
+test_string = "You'll hate me after this - #"          # " Annoying, isn't it?
+
+    [the.hard]
+    test_array = [ "] ", " # "]      # ] There you go, parse this!
+    test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ]
+    # You didn't think it'd as easy as chucking out the last #, did you?
+    another_test_string = " Same thing, but with a string #"
+    harder_test_string = " And when \"'s are in the string, along with # \""   # "and comments are there too"
+    # Things will get harder
+
+        [the.hard."bit#"]
+        "what?" = "You don't think some user won't do that?"
+        multi_line_array = [
+            "]",
+            # ] Oh yes I did
+            ]
+
+# Each of the following keygroups/key value pairs should produce an error. Uncomment to them to test
+
+#[error]   if you didn't catch this, your parser is broken
+#string = "Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment"   like this
+#array = [
+#         "This might most likely happen in multiline arrays",
+#         Like here,
+#         "or here,
+#         and here"
+#         ]     End of array comment, forgot the #
+#number = 3.14  pi <--again forgot the #         `