Multiline Marshal tag (#221)

The new multiline tag works just like the existing 'commented' tag (i.e.
`multiline:"true"`), and tells go-toml to marshal the value as a
multi-line string. The tag currently has no impact on any non-string
fields.

.gitignore

@@ -1 +1,2 @@
 test_program/test_program_bin
+fuzz/

.travis.yml

@@ -1,15 +1,23 @@
-language: go
-script: "./test.sh"
 sudo: false
+language: go
 go:
-    - 1.2.2
-    - 1.3.3
-    - 1.4.2
-    - 1.5.1
-    - tip
+  - 1.8.x
+  - 1.9.x
+  - 1.10.x
+  - tip
+matrix:
+  allow_failures:
+    - go: tip
+  fast_finish: true
+script:
+  - if [ -n "$(go fmt ./...)" ]; then exit 1; fi
+  - ./test.sh
+  - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git
 before_install:
   - go get github.com/axw/gocov/gocov
   - go get github.com/mattn/goveralls
   - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+branches:
+  only: [master]
 after_success:
-    - $HOME/gopath/bin/goveralls -service=travis-ci
+  - $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=coverage.out -repotoken $COVERALLS_TOKEN

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -6,73 +6,74 @@ This library supports TOML version
 [v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
 
 [![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml)
+[![license](https://img.shields.io/github/license/pelletier/go-toml.svg)](https://github.com/pelletier/go-toml/blob/master/LICENSE)
 [![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml)
-[![Coverage Status](https://coveralls.io/repos/pelletier/go-toml/badge.svg?branch=master&service=github)](https://coveralls.io/github/pelletier/go-toml?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pelletier/go-toml)](https://goreportcard.com/report/github.com/pelletier/go-toml)
 
 ## Features
 
 Go-toml provides the following features for using data parsed from TOML documents:
 
 * Load TOML documents from files and string data
-* Easily navigate TOML structure using TomlTree
+* Easily navigate TOML structure using Tree
+* Mashaling and unmarshaling to and from data structures
 * Line & column position data for all parsed elements
-* Query support similar to JSON-Path
+* [Query support similar to JSON-Path](query/)
 * Syntax errors contain line and column numbers
 
-Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing:
-
-* Semantic evaluation of parsed TOML
-* Informing a user of mistakes in the source document, after it has been parsed
-* Programatic handling of default values on a case-by-case basis
-* Using a TOML document as a flexible data-store
-
 ## Import
 
-    import "github.com/pelletier/go-toml"
-
-## Usage
-
-### Example
-
-Say you have a TOML file that looks like this:
-
-```toml
-[postgres]
-user = "pelletier"
-password = "mypassword"
+```go
+import "github.com/pelletier/go-toml"
 ```
 
-Read the username and password like this:
+## Usage example
+
+Read a TOML document:
 
 ```go
-import (
-    "fmt"
-    "github.com/pelletier/go-toml"
-)
+config, _ := toml.Load(`
+[postgres]
+user = "pelletier"
+password = "mypassword"`)
+// retrieve data directly
+user := config.Get("postgres.user").(string)
 
-config, err := toml.LoadFile("config.toml")
-if err != nil {
-    fmt.Println("Error ", err.Error())
-} else {
-    // retrieve data directly
-    user := config.Get("postgres.user").(string)
-    password := config.Get("postgres.password").(string)
+// or using an intermediate object
+postgresConfig := config.Get("postgres").(*toml.Tree)
+password := postgresConfig.Get("password").(string)
+```
 
-    // or using an intermediate object
-    configTree := config.Get("postgres").(*toml.TomlTree)
-    user = configTree.Get("user").(string)
-    password = configTree.Get("password").(string)
-    fmt.Println("User is ", user, ". Password is ", password)
+Or use Unmarshal:
 
-    // show where elements are in the file
-    fmt.Println("User position: %v", configTree.GetPosition("user"))
-    fmt.Println("Password position: %v", configTree.GetPosition("password"))
+```go
+type Postgres struct {
+    User     string
+    Password string
+}
+type Config struct {
+    Postgres Postgres
+}
 
-    // use a query to gather elements without walking the tree
-    results, _ := config.Query("$..[user,password]")
-    for ii, item := range results.Values() {
-      fmt.Println("Query result %d: %v", ii, item)
-    }
+doc := []byte(`
+[Postgres]
+User = "pelletier"
+Password = "mypassword"`)
+
+config := Config{}
+toml.Unmarshal(doc, &config)
+fmt.Println("user=", config.Postgres.User)
+```
+
+Or use a query:
+
+```go
+// use a query to gather elements without walking the tree
+q, _ := query.Compile("$..[user,password]")
+results := q.Execute(config)
+for ii, item := range results.Values() {
+    fmt.Println("Query result %d: %v", ii, item)
 }
 ```
 
@@ -81,6 +82,23 @@ if err != nil {
 The documentation and additional examples are available at
 [godoc.org](http://godoc.org/github.com/pelletier/go-toml).
 
+## Tools
+
+Go-toml provides two handy command line tools:
+
+* `tomll`: Reads TOML files and lint them.
+
+    ```
+    go install github.com/pelletier/go-toml/cmd/tomll
+    tomll --help
+    ```
+* `tomljson`: Reads a TOML file and outputs its JSON representation.
+
+    ```
+    go install github.com/pelletier/go-toml/cmd/tomljson
+    tomljson --help
+    ```
+
 ## Contribute
 
 Feel free to report bugs and patches using GitHub's pull requests system on
@@ -96,24 +114,18 @@ You have to make sure two kind of tests run:
 
 You can run both of them using `./test.sh`.
 
+### Fuzzing
+
+The script `./fuzz.sh` is available to
+run [go-fuzz](https://github.com/dvyukov/go-fuzz) on go-toml.
+
+## Versioning
+
+Go-toml follows [Semantic Versioning](http://semver.org/). The supported version
+of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of
+this document. The last two major versions of Go are supported
+(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)).
+
 ## License
 
-Copyright (c) 2013 - 2015 Thomas Pelletier, Eric Anderton
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+The MIT License (MIT). Read [LICENSE](LICENSE).

benchmark.json

@@ -0,0 +1,164 @@
+{
+    "array": {
+        "key1": [
+            1,
+            2,
+            3
+        ],
+        "key2": [
+            "red",
+            "yellow",
+            "green"
+        ],
+        "key3": [
+            [
+                1,
+                2
+            ],
+            [
+                3,
+                4,
+                5
+            ]
+        ],
+        "key4": [
+            [
+                1,
+                2
+            ],
+            [
+                "a",
+                "b",
+                "c"
+            ]
+        ],
+        "key5": [
+            1,
+            2,
+            3
+        ],
+        "key6": [
+            1,
+            2
+        ]
+    },
+    "boolean": {
+        "False": false,
+        "True": true
+    },
+    "datetime": {
+        "key1": "1979-05-27T07:32:00Z",
+        "key2": "1979-05-27T00:32:00-07:00",
+        "key3": "1979-05-27T00:32:00.999999-07:00"
+    },
+    "float": {
+        "both": {
+            "key": 6.626e-34
+        },
+        "exponent": {
+            "key1": 5e+22,
+            "key2": 1000000,
+            "key3": -0.02
+        },
+        "fractional": {
+            "key1": 1,
+            "key2": 3.1415,
+            "key3": -0.01
+        },
+        "underscores": {
+            "key1": 9224617.445991227,
+            "key2": 1e+100
+        }
+    },
+    "fruit": [{
+            "name": "apple",
+            "physical": {
+                "color": "red",
+                "shape": "round"
+            },
+            "variety": [{
+                    "name": "red delicious"
+                },
+                {
+                    "name": "granny smith"
+                }
+            ]
+        },
+        {
+            "name": "banana",
+            "variety": [{
+                "name": "plantain"
+            }]
+        }
+    ],
+    "integer": {
+        "key1": 99,
+        "key2": 42,
+        "key3": 0,
+        "key4": -17,
+        "underscores": {
+            "key1": 1000,
+            "key2": 5349221,
+            "key3": 12345
+        }
+    },
+    "products": [{
+            "name": "Hammer",
+            "sku": 738594937
+        },
+        {},
+        {
+            "color": "gray",
+            "name": "Nail",
+            "sku": 284758393
+        }
+    ],
+    "string": {
+        "basic": {
+            "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+        },
+        "literal": {
+            "multiline": {
+                "lines": "The first newline is\ntrimmed in raw strings.\n   All other whitespace\n   is preserved.\n",
+                "regex2": "I [dw]on't need \\d{2} apples"
+            },
+            "quoted": "Tom \"Dubs\" Preston-Werner",
+            "regex": "\u003c\\i\\c*\\s*\u003e",
+            "winpath": "C:\\Users\\nodejs\\templates",
+            "winpath2": "\\\\ServerX\\admin$\\system32\\"
+        },
+        "multiline": {
+            "continued": {
+                "key1": "The quick brown fox jumps over the lazy dog.",
+                "key2": "The quick brown fox jumps over the lazy dog.",
+                "key3": "The quick brown fox jumps over the lazy dog."
+            },
+            "key1": "One\nTwo",
+            "key2": "One\nTwo",
+            "key3": "One\nTwo"
+        }
+    },
+    "table": {
+        "inline": {
+            "name": {
+                "first": "Tom",
+                "last": "Preston-Werner"
+            },
+            "point": {
+                "x": 1,
+                "y": 2
+            }
+        },
+        "key": "value",
+        "subtable": {
+            "key": "another value"
+        }
+    },
+    "x": {
+        "y": {
+            "z": {
+                "w": {}
+            }
+        }
+    }
+}

benchmark.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+reference_ref=${1:-master}
+reference_git=${2:-.}
+
+if ! `hash benchstat 2>/dev/null`; then
+    echo "Installing benchstat"
+    go get golang.org/x/perf/cmd/benchstat
+    go install golang.org/x/perf/cmd/benchstat
+fi
+
+tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX`
+ref_tempdir="${tempdir}/ref"
+ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt"
+local_benchmark="`pwd`/benchmark-local.txt"
+
+echo "=== ${reference_ref} (${ref_tempdir})"
+git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null
+pushd ${ref_tempdir} >/dev/null
+git checkout ${reference_ref} >/dev/null 2>/dev/null
+go test -bench=. -benchmem | tee ${ref_benchmark}
+popd >/dev/null
+
+echo ""
+echo "=== local"
+go test -bench=. -benchmem  | tee ${local_benchmark}
+
+echo ""
+echo "=== diff"
+benchstat -delta-test=none ${ref_benchmark} ${local_benchmark}

benchmark.toml

@@ -0,0 +1,244 @@
+################################################################################
+## Comment
+
+# Speak your mind with the hash symbol. They go from the symbol to the end of
+# the line.
+
+
+################################################################################
+## Table
+
+# Tables (also known as hash tables or dictionaries) are collections of
+# key/value pairs. They appear in square brackets on a line by themselves.
+
+[table]
+
+key = "value" # Yeah, you can do this.
+
+# Nested tables are denoted by table names with dots in them. Name your tables
+# whatever crap you please, just don't use #, ., [ or ].
+
+[table.subtable]
+
+key = "another value"
+
+# You don't need to specify all the super-tables if you don't want to. TOML
+# knows how to do it for you.
+
+# [x] you
+# [x.y] don't
+# [x.y.z] need these
+[x.y.z.w] # for this to work
+
+
+################################################################################
+## Inline Table
+
+# Inline tables provide a more compact syntax for expressing tables. They are
+# especially useful for grouped data that can otherwise quickly become verbose.
+# Inline tables are enclosed in curly braces `{` and `}`. No newlines are
+# allowed between the curly braces unless they are valid within a value.
+
+[table.inline]
+
+name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }
+
+
+################################################################################
+## String
+
+# There are four ways to express strings: basic, multi-line basic, literal, and
+# multi-line literal. All strings must contain only valid UTF-8 characters.
+
+[string.basic]
+
+basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+[string.multiline]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "One\nTwo"
+key2 = """One\nTwo"""
+key3 = """
+One
+Two"""
+
+[string.multiline.continued]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "The quick brown fox jumps over the lazy dog."
+
+key2 = """
+The quick brown \
+
+
+  fox jumps over \
+    the lazy dog."""
+
+key3 = """\
+       The quick brown \
+       fox jumps over \
+       the lazy dog.\
+       """
+
+[string.literal]
+
+# What you see is what you get.
+winpath  = 'C:\Users\nodejs\templates'
+winpath2 = '\\ServerX\admin$\system32\'
+quoted   = 'Tom "Dubs" Preston-Werner'
+regex    = '<\i\c*\s*>'
+
+
+[string.literal.multiline]
+
+regex2 = '''I [dw]on't need \d{2} apples'''
+lines  = '''
+The first newline is
+trimmed in raw strings.
+   All other whitespace
+   is preserved.
+'''
+
+
+################################################################################
+## Integer
+
+# Integers are whole numbers. Positive numbers may be prefixed with a plus sign.
+# Negative numbers are prefixed with a minus sign.
+
+[integer]
+
+key1 = +99
+key2 = 42
+key3 = 0
+key4 = -17
+
+[integer.underscores]
+
+# For large numbers, you may use underscores to enhance readability. Each
+# underscore must be surrounded by at least one digit.
+key1 = 1_000
+key2 = 5_349_221
+key3 = 1_2_3_4_5     # valid but inadvisable
+
+
+################################################################################
+## Float
+
+# A float consists of an integer part (which may be prefixed with a plus or
+# minus sign) followed by a fractional part and/or an exponent part.
+
+[float.fractional]
+
+key1 = +1.0
+key2 = 3.1415
+key3 = -0.01
+
+[float.exponent]
+
+key1 = 5e+22
+key2 = 1e6
+key3 = -2E-2
+
+[float.both]
+
+key = 6.626e-34
+
+[float.underscores]
+
+key1 = 9_224_617.445_991_228_313
+key2 = 1e1_00
+
+
+################################################################################
+## Boolean
+
+# Booleans are just the tokens you're used to. Always lowercase.
+
+[boolean]
+
+True = true
+False = false
+
+
+################################################################################
+## Datetime
+
+# Datetimes are RFC 3339 dates.
+
+[datetime]
+
+key1 = 1979-05-27T07:32:00Z
+key2 = 1979-05-27T00:32:00-07:00
+key3 = 1979-05-27T00:32:00.999999-07:00
+
+
+################################################################################
+## Array
+
+# Arrays are square brackets with other primitives inside. Whitespace is
+# ignored. Elements are separated by commas. Data types may not be mixed.
+
+[array]
+
+key1 = [ 1, 2, 3 ]
+key2 = [ "red", "yellow", "green" ]
+key3 = [ [ 1, 2 ], [3, 4, 5] ]
+#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok
+
+# Arrays can also be multiline. So in addition to ignoring whitespace, arrays
+# also ignore newlines between the brackets.  Terminating commas are ok before
+# the closing bracket.
+
+key5 = [
+  1, 2, 3
+]
+key6 = [
+  1,
+  2, # this is ok
+]
+
+
+################################################################################
+## Array of Tables
+
+# These can be expressed by using a table name in double brackets. Each table
+# with the same double bracketed name will be an element in the array. The
+# tables are inserted in the order encountered.
+
+[[products]]
+
+name = "Hammer"
+sku = 738594937
+
+[[products]]
+
+[[products]]
+
+name = "Nail"
+sku = 284758393
+color = "gray"
+
+
+# You can create nested arrays of tables as well.
+
+[[fruit]]
+  name = "apple"
+
+  [fruit.physical]
+    color = "red"
+    shape = "round"
+
+  [[fruit.variety]]
+    name = "red delicious"
+
+  [[fruit.variety]]
+    name = "granny smith"
+
+[[fruit]]
+  name = "banana"
+
+  [[fruit.variety]]
+    name = "plantain"

benchmark.yml

@@ -0,0 +1,121 @@
+---
+array:
+  key1:
+  - 1
+  - 2
+  - 3
+  key2:
+  - red
+  - yellow
+  - green
+  key3:
+  - - 1
+    - 2
+  - - 3
+    - 4
+    - 5
+  key4:
+  - - 1
+    - 2
+  - - a
+    - b
+    - c
+  key5:
+  - 1
+  - 2
+  - 3
+  key6:
+  - 1
+  - 2
+boolean:
+  'False': false
+  'True': true
+datetime:
+  key1: '1979-05-27T07:32:00Z'
+  key2: '1979-05-27T00:32:00-07:00'
+  key3: '1979-05-27T00:32:00.999999-07:00'
+float:
+  both:
+    key: 6.626e-34
+  exponent:
+    key1: 5.0e+22
+    key2: 1000000
+    key3: -0.02
+  fractional:
+    key1: 1
+    key2: 3.1415
+    key3: -0.01
+  underscores:
+    key1: 9224617.445991227
+    key2: 1.0e+100
+fruit:
+- name: apple
+  physical:
+    color: red
+    shape: round
+  variety:
+  - name: red delicious
+  - name: granny smith
+- name: banana
+  variety:
+  - name: plantain
+integer:
+  key1: 99
+  key2: 42
+  key3: 0
+  key4: -17
+  underscores:
+    key1: 1000
+    key2: 5349221
+    key3: 12345
+products:
+- name: Hammer
+  sku: 738594937
+- {}
+- color: gray
+  name: Nail
+  sku: 284758393
+string:
+  basic:
+    basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+  literal:
+    multiline:
+      lines: |
+        The first newline is
+        trimmed in raw strings.
+           All other whitespace
+           is preserved.
+      regex2: I [dw]on't need \d{2} apples
+    quoted: Tom "Dubs" Preston-Werner
+    regex: "<\\i\\c*\\s*>"
+    winpath: C:\Users\nodejs\templates
+    winpath2: "\\\\ServerX\\admin$\\system32\\"
+  multiline:
+    continued:
+      key1: The quick brown fox jumps over the lazy dog.
+      key2: The quick brown fox jumps over the lazy dog.
+      key3: The quick brown fox jumps over the lazy dog.
+    key1: |-
+      One
+      Two
+    key2: |-
+      One
+      Two
+    key3: |-
+      One
+      Two
+table:
+  inline:
+    name:
+      first: Tom
+      last: Preston-Werner
+    point:
+      x: 1
+      y: 2
+  key: value
+  subtable:
+    key: another value
+x:
+  y:
+    z:
+      w: {}

benchmark_test.go

@@ -0,0 +1,192 @@
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	burntsushi "github.com/BurntSushi/toml"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type benchmarkDoc struct {
+	Table struct {
+		Key      string
+		Subtable struct {
+			Key string
+		}
+		Inline struct {
+			Name struct {
+				First string
+				Last  string
+			}
+			Point struct {
+				X int64
+				U int64
+			}
+		}
+	}
+	String struct {
+		Basic struct {
+			Basic string
+		}
+		Multiline struct {
+			Key1      string
+			Key2      string
+			Key3      string
+			Continued struct {
+				Key1 string
+				Key2 string
+				Key3 string
+			}
+		}
+		Literal struct {
+			Winpath   string
+			Winpath2  string
+			Quoted    string
+			Regex     string
+			Multiline struct {
+				Regex2 string
+				Lines  string
+			}
+		}
+	}
+	Integer struct {
+		Key1        int64
+		Key2        int64
+		Key3        int64
+		Key4        int64
+		Underscores struct {
+			Key1 int64
+			Key2 int64
+			Key3 int64
+		}
+	}
+	Float struct {
+		Fractional struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Exponent struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Both struct {
+			Key float64
+		}
+		Underscores struct {
+			Key1 float64
+			Key2 float64
+		}
+	}
+	Boolean struct {
+		True  bool
+		False bool
+	}
+	Datetime struct {
+		Key1 time.Time
+		Key2 time.Time
+		Key3 time.Time
+	}
+	Array struct {
+		Key1 []int64
+		Key2 []string
+		Key3 [][]int64
+		// TODO: Key4 not supported by go-toml's Unmarshal
+		Key5 []int64
+		Key6 []int64
+	}
+	Products []struct {
+		Name  string
+		Sku   int64
+		Color string
+	}
+	Fruit []struct {
+		Name     string
+		Physical struct {
+			Color   string
+			Shape   string
+			Variety []struct {
+				Name string
+			}
+		}
+	}
+}
+
+func BenchmarkParseToml(b *testing.B) {
+	fileBytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := LoadReader(bytes.NewReader(fileBytes))
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalBurntSushiToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := burntsushi.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalJson(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := json.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalYaml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.yml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := yaml.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

clean.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-# fail out of the script if anything here fails
-set -e
-
-# clear out stuff generated by test.sh
-rm -rf src test_program_bin toml-test

cmd/test_program.go

@@ -3,11 +3,12 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/pelletier/go-toml"
 	"io/ioutil"
 	"log"
 	"os"
 	"time"
+
+	"github.com/pelletier/go-toml"
 )
 
 func main() {
@@ -40,16 +41,16 @@ func translate(tomlData interface{}) interface{} {
 			typed[k] = translate(v)
 		}
 		return typed
-	case *toml.TomlTree:
+	case *toml.Tree:
 		return translate(*orig)
-	case toml.TomlTree:
+	case toml.Tree:
 		keys := orig.Keys()
 		typed := make(map[string]interface{}, len(keys))
 		for _, k := range keys {
 			typed[k] = translate(orig.GetPath([]string{k}))
 		}
 		return typed
-	case []*toml.TomlTree:
+	case []*toml.Tree:
 		typed := make([]map[string]interface{}, len(orig))
 		for i, v := range orig {
 			typed[i] = translate(v).(map[string]interface{})

cmd/tomljson/main.go

@@ -0,0 +1,71 @@
+// Tomljson reads TOML and converts to JSON.
+//
+// Usage:
+//   cat file.toml | tomljson > file.json
+//   tomljson file1.toml > file.json
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "tomljson can be used in two ways:")
+		fmt.Fprintln(os.Stderr, "Writing to STDIN and reading from STDOUT:")
+		fmt.Fprintln(os.Stderr, "  cat file.toml | tomljson > file.json")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "Reading from a file name:")
+		fmt.Fprintln(os.Stderr, "  tomljson file.toml")
+	}
+	flag.Parse()
+	os.Exit(processMain(flag.Args(), os.Stdin, os.Stdout, os.Stderr))
+}
+
+func processMain(files []string, defaultInput io.Reader, output io.Writer, errorOutput io.Writer) int {
+	// read from stdin and print to stdout
+	inputReader := defaultInput
+
+	if len(files) > 0 {
+		var err error
+		inputReader, err = os.Open(files[0])
+		if err != nil {
+			printError(err, errorOutput)
+			return -1
+		}
+	}
+	s, err := reader(inputReader)
+	if err != nil {
+		printError(err, errorOutput)
+		return -1
+	}
+	io.WriteString(output, s+"\n")
+	return 0
+}
+
+func printError(err error, output io.Writer) {
+	io.WriteString(output, err.Error()+"\n")
+}
+
+func reader(r io.Reader) (string, error) {
+	tree, err := toml.LoadReader(r)
+	if err != nil {
+		return "", err
+	}
+	return mapToJSON(tree)
+}
+
+func mapToJSON(tree *toml.Tree) (string, error) {
+	treeMap := tree.ToMap()
+	bytes, err := json.MarshalIndent(treeMap, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(bytes[:]), nil
+}

cmd/tomljson/main_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+func expectBufferEquality(t *testing.T, name string, buffer *bytes.Buffer, expected string) {
+	output := buffer.String()
+	if output != expected {
+		t.Errorf("incorrect %s:\n%s\n\nexpected %s:\n%s", name, output, name, expected)
+		t.Log([]rune(output))
+		t.Log([]rune(expected))
+	}
+}
+
+func expectProcessMainResults(t *testing.T, input string, args []string, exitCode int, expectedOutput string, expectedError string) {
+	inputReader := strings.NewReader(input)
+	outputBuffer := new(bytes.Buffer)
+	errorBuffer := new(bytes.Buffer)
+
+	returnCode := processMain(args, inputReader, outputBuffer, errorBuffer)
+
+	expectBufferEquality(t, "output", outputBuffer, expectedOutput)
+	expectBufferEquality(t, "error", errorBuffer, expectedError)
+
+	if returnCode != exitCode {
+		t.Error("incorrect return code:", returnCode, "expected", exitCode)
+	}
+}
+
+func TestProcessMainReadFromStdin(t *testing.T) {
+	input := `
+		[mytoml]
+		a = 42`
+	expectedOutput := `{
+  "mytoml": {
+    "a": 42
+  }
+}
+`
+	expectedError := ``
+	expectedExitCode := 0
+
+	expectProcessMainResults(t, input, []string{}, expectedExitCode, expectedOutput, expectedError)
+}
+
+func TestProcessMainReadFromFile(t *testing.T) {
+	input := `
+		[mytoml]
+		a = 42`
+
+	tmpfile, err := ioutil.TempFile("", "example.toml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tmpfile.Write([]byte(input)); err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.Remove(tmpfile.Name())
+
+	expectedOutput := `{
+  "mytoml": {
+    "a": 42
+  }
+}
+`
+	expectedError := ``
+	expectedExitCode := 0
+
+	expectProcessMainResults(t, ``, []string{tmpfile.Name()}, expectedExitCode, expectedOutput, expectedError)
+}
+
+func TestProcessMainReadFromMissingFile(t *testing.T) {
+	expectedError := `open /this/file/does/not/exist: no such file or directory
+`
+	expectProcessMainResults(t, ``, []string{"/this/file/does/not/exist"}, -1, ``, expectedError)
+}

cmd/tomll/main.go

@@ -0,0 +1,65 @@
+// Tomll is a linter for TOML
+//
+// Usage:
+//   cat file.toml | tomll > file_linted.toml
+//   tomll file1.toml file2.toml # lint the two files in place
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "tomll can be used in two ways:")
+		fmt.Fprintln(os.Stderr, "Writing to STDIN and reading from STDOUT:")
+		fmt.Fprintln(os.Stderr, "  cat file.toml | tomll > file.toml")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "Reading and updating a list of files:")
+		fmt.Fprintln(os.Stderr, "  tomll a.toml b.toml c.toml")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "When given a list of files, tomll will modify all files in place without asking.")
+	}
+	flag.Parse()
+	// read from stdin and print to stdout
+	if flag.NArg() == 0 {
+		s, err := lintReader(os.Stdin)
+		if err != nil {
+			io.WriteString(os.Stderr, err.Error())
+			os.Exit(-1)
+		}
+		io.WriteString(os.Stdout, s)
+	} else {
+		// otherwise modify a list of files
+		for _, filename := range flag.Args() {
+			s, err := lintFile(filename)
+			if err != nil {
+				io.WriteString(os.Stderr, err.Error())
+				os.Exit(-1)
+			}
+			ioutil.WriteFile(filename, []byte(s), 0644)
+		}
+	}
+}
+
+func lintFile(filename string) (string, error) {
+	tree, err := toml.LoadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	return tree.String(), nil
+}
+
+func lintReader(r io.Reader) (string, error) {
+	tree, err := toml.LoadReader(r)
+	if err != nil {
+		return "", err
+	}
+	return tree.String(), nil
+}

doc.go

@@ -1,245 +1,23 @@
-// Package toml is a TOML markup language parser.
+// Package toml is a TOML parser and manipulation library.
 //
 // This version supports the specification as described in
-// https://github.com/toml-lang/toml/blob/master/versions/toml-v0.2.0.md
+// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md
 //
-// TOML Parsing
+// Marshaling
 //
-// TOML data may be parsed in two ways: by file, or by string.
+// Go-toml can marshal and unmarshal TOML documents from and to data
+// structures.
 //
-//   // load TOML data by filename
-//   tree, err := toml.LoadFile("filename.toml")
+// TOML document as a tree
 //
-//   // load TOML data stored in a string
-//   tree, err := toml.Load(stringContainingTomlData)
+// Go-toml can operate on a TOML document as a tree. Use one of the Load*
+// functions to parse TOML data and obtain a Tree instance, then one of its
+// methods to manipulate the tree.
 //
-// Either way, the result is a TomlTree object that can be used to navigate the
-// structure and data within the original document.
+// JSONPath-like queries
 //
-//
-// Getting data from the TomlTree
-//
-// After parsing TOML data with Load() or LoadFile(), use the Has() and Get()
-// methods on the returned TomlTree, to find your way through the document data.
-//
-//   if tree.Has('foo') {
-//     fmt.Prinln("foo is: %v", tree.Get('foo'))
-//   }
-//
-// Working with Paths
-//
-// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set()
-// and GetDefault() methods.  These are the same kind of key paths used within the
-// TOML specification for struct tames.
-//
-//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
-//   tree.Has("foo.bar.baz")
-//
-//   // returns the key at this path, if it is there
-//   tree.Get("foo.bar.baz")
-//
-// TOML allows keys to contain '.', which can cause this syntax to be problematic
-// for some documents.  In such cases, use the GetPath(), HasPath(), and SetPath(),
-// methods to explicitly define the path.  This form is also faster, since
-// it avoids having to parse the passed key for '.' delimiters.
-//
-//   // looks for a key named 'baz', within struct 'bar', within struct 'foo'
-//   tree.HasPath(string{}{"foo","bar","baz"})
-//
-//   // returns the key at this path, if it is there
-//   tree.GetPath(string{}{"foo","bar","baz"})
-//
-// Note that this is distinct from the heavyweight query syntax supported by
-// TomlTree.Query() and the Query() struct (see below).
-//
-// Position Support
-//
-// Each element within the TomlTree is stored with position metadata, which is
-// invaluable for providing semantic feedback to a user.  This helps in
-// situations where the TOML file parses correctly, but contains data that is
-// not correct for the application.  In such cases, an error message can be
-// generated that indicates the problem line and column number in the source
-// TOML document.
-//
-//   // load TOML data
-//   tree, _ := toml.Load("filename.toml")
-//
-//   // get an entry and report an error if it's the wrong type
-//   element := tree.Get("foo")
-//   if value, ok := element.(int64); !ok {
-//       return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo"))
-//   }
-//
-//   // report an error if an expected element is missing
-//   if !tree.Has("bar") {
-//      return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition(""))
-//   }
-//
-// Query Support
-//
-// The TOML query path implementation is based loosely on the JSONPath specification:
-// http://goessner.net/articles/JsonPath/
-//
-// The idea behind a query path is to allow quick access to any element, or set
-// of elements within TOML document, with a single expression.
-//
-//   result := tree.Query("$.foo.bar.baz")  // result is 'nil' if the path is not present
-//
-// This is equivalent to:
-//
-//   next := tree.Get("foo")
-//   if next != nil {
-//     next = next.Get("bar")
-//     if next != nil {
-//       next = next.Get("baz")
-//     }
-//   }
-//   result := next
-//
-// As illustrated above, the query path is much more efficient, especially since
-// the structure of the TOML file can vary.  Rather than making assumptions about
-// a document's structure, a query allows the programmer to make structured
-// requests into the document, and get zero or more values as a result.
-//
-// The syntax of a query begins with a root token, followed by any number
-// sub-expressions:
-//
-//   $
-//                    Root of the TOML tree.  This must always come first.
-//   .name
-//                    Selects child of this node, where 'name' is a TOML key
-//                    name.
-//   ['name']
-//                    Selects child of this node, where 'name' is a string
-//                    containing a TOML key name.
-//   [index]
-//                    Selcts child array element at 'index'.
-//   ..expr
-//                    Recursively selects all children, filtered by an a union,
-//                    index, or slice expression.
-//   ..*
-//                    Recursive selection of all nodes at this point in the
-//                    tree.
-//   .*
-//                    Selects all children of the current node.
-//   [expr,expr]
-//                    Union operator - a logical 'or' grouping of two or more
-//                    sub-expressions: index, key name, or filter.
-//   [start:end:step]
-//                    Slice operator - selects array elements from start to
-//                    end-1, at the given step.  All three arguments are
-//                    optional.
-//   [?(filter)]
-//                    Named filter expression - the function 'filter' is
-//                    used to filter children at this node.
-//
-// Query Indexes And Slices
-//
-// Index expressions perform no bounds checking, and will contribute no
-// values to the result set if the provided index or index range is invalid.
-// Negative indexes represent values from the end of the array, counting backwards.
-//
-//   // select the last index of the array named 'foo'
-//   tree.Query("$.foo[-1]")
-//
-// Slice expressions are supported, by using ':' to separate a start/end index pair.
-//
-//   // select up to the first five elements in the array
-//   tree.Query("$.foo[0:5]")
-//
-// Slice expressions also allow negative indexes for the start and stop
-// arguments.
-//
-//   // select all array elements.
-//   tree.Query("$.foo[0:-1]")
-//
-// Slice expressions may have an optional stride/step parameter:
-//
-//   // select every other element
-//   tree.Query("$.foo[0:-1:2]")
-//
-// Slice start and end parameters are also optional:
-//
-//   // these are all equivalent and select all the values in the array
-//   tree.Query("$.foo[:]")
-//   tree.Query("$.foo[0:]")
-//   tree.Query("$.foo[:-1]")
-//   tree.Query("$.foo[0:-1:]")
-//   tree.Query("$.foo[::1]")
-//   tree.Query("$.foo[0::1]")
-//   tree.Query("$.foo[:-1:1]")
-//   tree.Query("$.foo[0:-1:1]")
-//
-// Query Filters
-//
-// Query filters are used within a Union [,] or single Filter [] expression.
-// A filter only allows nodes that qualify through to the next expression,
-// and/or into the result set.
-//
-//   // returns children of foo that are permitted by the 'bar' filter.
-//   tree.Query("$.foo[?(bar)]")
-//
-// There are several filters provided with the library:
-//
-//   tree
-//          Allows nodes of type TomlTree.
-//   int
-//          Allows nodes of type int64.
-//   float
-//          Allows nodes of type float64.
-//   string
-//          Allows nodes of type string.
-//   time
-//          Allows nodes of type time.Time.
-//   bool
-//          Allows nodes of type bool.
-//
-// Query Results
-//
-// An executed query returns a QueryResult object.  This contains the nodes
-// in the TOML tree that qualify the query expression.  Position information
-// is also available for each value in the set.
-//
-//   // display the results of a query
-//   results := tree.Query("$.foo.bar.baz")
-//   for idx, value := results.Values() {
-//       fmt.Println("%v: %v", results.Positions()[idx], value)
-//   }
-//
-// Compiled Queries
-//
-// Queries may be executed directly on a TomlTree object, or compiled ahead
-// of time and executed discretely.  The former is more convienent, but has the
-// penalty of having to recompile the query expression each time.
-//
-//   // basic query
-//   results := tree.Query("$.foo.bar.baz")
-//
-//   // compiled query
-//   query := toml.CompileQuery("$.foo.bar.baz")
-//   results := query.Execute(tree)
-//
-//   // run the compiled query again on a different tree
-//   moreResults := query.Execute(anotherTree)
-//
-// User Defined Query Filters
-//
-// Filter expressions may also be user defined by using the SetFilter()
-// function on the Query object.  The function must return true/false, which
-// signifies if the passed node is kept or discarded, respectively.
-//
-//   // create a query that references a user-defined filter
-//   query, _ := CompileQuery("$[?(bazOnly)]")
-//
-//   // define the filter, and assign it to the query
-//   query.SetFilter("bazOnly", func(node interface{}) bool{
-//       if tree, ok := node.(*TomlTree); ok {
-//           return tree.Has("baz")
-//       }
-//       return false  // reject all other node types
-//   })
-//
-//   // run the query
-//   query.Execute(tree)
+// The package github.com/pelletier/go-toml/query implements a system
+// similar to JSONPath to quickly retrieve elements of a TOML document using a
+// single expression. See the package documentation for more information.
 //
 package toml

doc_test.go

@@ -1,59 +1,16 @@
 // code examples for godoc
 
-package toml
+package toml_test
 
 import (
 	"fmt"
+	"log"
+
+	toml "github.com/pelletier/go-toml"
 )
 
-func ExampleNodeFilterFn_filterExample() {
-	tree, _ := Load(`
-      [struct_one]
-      foo = "foo"
-      bar = "bar"
-
-      [struct_two]
-      baz = "baz"
-      gorf = "gorf"
-    `)
-
-	// create a query that references a user-defined-filter
-	query, _ := CompileQuery("$[?(bazOnly)]")
-
-	// define the filter, and assign it to the query
-	query.SetFilter("bazOnly", func(node interface{}) bool {
-		if tree, ok := node.(*TomlTree); ok {
-			return tree.Has("baz")
-		}
-		return false // reject all other node types
-	})
-
-	// results contain only the 'struct_two' TomlTree
-	query.Execute(tree)
-}
-
-func ExampleQuery_queryExample() {
-	config, _ := Load(`
-      [[book]]
-      title = "The Stand"
-      author = "Stephen King"
-      [[book]]
-      title = "For Whom the Bell Tolls"
-      author = "Ernest Hemmingway"
-      [[book]]
-      title = "Neuromancer"
-      author = "William Gibson"
-    `)
-
-	// find and print all the authors in the document
-	authors, _ := config.Query("$.book.author")
-	for _, name := range authors.Values() {
-		fmt.Println(name)
-	}
-}
-
-func Example_comprehensiveExample() {
-	config, err := LoadFile("config.toml")
+func Example_tree() {
+	config, err := toml.LoadFile("config.toml")
 
 	if err != nil {
 		fmt.Println("Error ", err.Error())
@@ -63,19 +20,86 @@ func Example_comprehensiveExample() {
 		password := config.Get("postgres.password").(string)
 
 		// or using an intermediate object
-		configTree := config.Get("postgres").(*TomlTree)
+		configTree := config.Get("postgres").(*toml.Tree)
 		user = configTree.Get("user").(string)
 		password = configTree.Get("password").(string)
-		fmt.Println("User is ", user, ". Password is ", password)
+		fmt.Println("User is", user, " and password is", password)
 
 		// show where elements are in the file
-		fmt.Println("User position: %v", configTree.GetPosition("user"))
-		fmt.Println("Password position: %v", configTree.GetPosition("password"))
-
-		// use a query to gather elements without walking the tree
-		results, _ := config.Query("$..[user,password]")
-		for ii, item := range results.Values() {
-			fmt.Println("Query result %d: %v", ii, item)
-		}
+		fmt.Printf("User position: %v\n", configTree.GetPosition("user"))
+		fmt.Printf("Password position: %v\n", configTree.GetPosition("password"))
 	}
 }
+
+func Example_unmarshal() {
+	type Employer struct {
+		Name  string
+		Phone string
+	}
+	type Person struct {
+		Name     string
+		Age      int64
+		Employer Employer
+	}
+
+	document := []byte(`
+	name = "John"
+	age = 30
+	[employer]
+		name = "Company Inc."
+		phone = "+1 234 567 89012"
+	`)
+
+	person := Person{}
+	toml.Unmarshal(document, &person)
+	fmt.Println(person.Name, "is", person.Age, "and works at", person.Employer.Name)
+	// Output:
+	// John is 30 and works at Company Inc.
+}
+
+func ExampleMarshal() {
+	type Postgres struct {
+		User     string `toml:"user"`
+		Password string `toml:"password"`
+		Database string `toml:"db" commented:"true" comment:"not used anymore"`
+	}
+	type Config struct {
+		Postgres Postgres `toml:"postgres" comment:"Postgres configuration"`
+	}
+
+	config := Config{Postgres{User: "pelletier", Password: "mypassword", Database: "old_database"}}
+	b, err := toml.Marshal(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b))
+	// Output:
+	// # Postgres configuration
+	// [postgres]
+	//
+	//   # not used anymore
+	//   # db = "old_database"
+	//   password = "mypassword"
+	//   user = "pelletier"
+}
+
+func ExampleUnmarshal() {
+	type Postgres struct {
+		User     string
+		Password string
+	}
+	type Config struct {
+		Postgres Postgres
+	}
+
+	doc := []byte(`
+	[postgres]
+	user = "pelletier"
+	password = "mypassword"`)
+
+	config := Config{}
+	toml.Unmarshal(doc, &config)
+	fmt.Println("user=", config.Postgres.User)
+	// Output:
+	// user= pelletier
+}

example-crlf.toml

@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it

fuzz.go

@@ -0,0 +1,31 @@
+// +build gofuzz
+
+package toml
+
+func Fuzz(data []byte) int {
+	tree, err := LoadBytes(data)
+	if err != nil {
+		if tree != nil {
+			panic("tree must be nil if there is an error")
+		}
+		return 0
+	}
+
+	str, err := tree.ToTomlString()
+	if err != nil {
+		if str != "" {
+			panic(`str must be "" if there is an error`)
+		}
+		panic(err)
+	}
+
+	tree, err = Load(str)
+	if err != nil {
+		if tree != nil {
+			panic("tree must be nil if there is an error")
+		}
+		return 0
+	}
+
+	return 1
+}

fuzz.sh

@@ -0,0 +1,15 @@
+#! /bin/sh
+set -eu
+
+go get github.com/dvyukov/go-fuzz/go-fuzz
+go get github.com/dvyukov/go-fuzz/go-fuzz-build
+
+if [ ! -e toml-fuzz.zip ]; then
+    go-fuzz-build github.com/pelletier/go-toml
+fi
+
+rm -fr fuzz
+mkdir -p fuzz/corpus
+cp *.toml fuzz/corpus
+
+go-fuzz -bin=toml-fuzz.zip -workdir=fuzz

keysparsing.go

@@ -4,15 +4,19 @@ package toml
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"unicode"
 )
 
+// Convert the bare key group string to an array.
+// The input supports double quotation to allow "." inside the key name,
+// but escape sequences are not supported. Lexers must unescape them beforehand.
 func parseKey(key string) ([]string, error) {
 	groups := []string{}
 	var buffer bytes.Buffer
 	inQuotes := false
-	escapeNext := false
+	wasInQuotes := false
 	ignoreSpace := true
 	expectDot := false
 
@@ -23,26 +27,29 @@ func parseKey(key string) ([]string, error) {
 			}
 			ignoreSpace = false
 		}
-		if escapeNext {
-			buffer.WriteRune(char)
-			escapeNext = false
-			continue
-		}
 		switch char {
-		case '\\':
-			escapeNext = true
-			continue
 		case '"':
+			if inQuotes {
+				groups = append(groups, buffer.String())
+				buffer.Reset()
+				wasInQuotes = true
+			}
 			inQuotes = !inQuotes
 			expectDot = false
 		case '.':
 			if inQuotes {
 				buffer.WriteRune(char)
 			} else {
-				groups = append(groups, buffer.String())
-				buffer.Reset()
+				if !wasInQuotes {
+					if buffer.Len() == 0 {
+						return nil, errors.New("empty table key")
+					}
+					groups = append(groups, buffer.String())
+					buffer.Reset()
+				}
 				ignoreSpace = true
 				expectDot = false
+				wasInQuotes = false
 			}
 		case ' ':
 			if inQuotes {
@@ -55,21 +62,21 @@ func parseKey(key string) ([]string, error) {
 				return nil, fmt.Errorf("invalid bare character: %c", char)
 			}
 			if !inQuotes && expectDot {
-				return nil, fmt.Errorf("what?")
+				return nil, errors.New("what?")
 			}
 			buffer.WriteRune(char)
 			expectDot = false
 		}
 	}
 	if inQuotes {
-		return nil, fmt.Errorf("mismatched quotes")
-	}
-	if escapeNext {
-		return nil, fmt.Errorf("unfinished escape sequence")
+		return nil, errors.New("mismatched quotes")
 	}
 	if buffer.Len() > 0 {
 		groups = append(groups, buffer.String())
 	}
+	if len(groups) == 0 {
+		return nil, errors.New("empty key")
+	}
 	return groups, nil
 }
 

keysparsing_test.go

@@ -7,6 +7,7 @@ import (
 
 func testResult(t *testing.T, key string, expected []string) {
 	parsed, err := parseKey(key)
+	t.Logf("key=%s expected=%s parsed=%s", key, expected, parsed)
 	if err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
@@ -21,7 +22,10 @@ func testResult(t *testing.T, key string, expected []string) {
 }
 
 func testError(t *testing.T, key string, expectedError string) {
-	_, err := parseKey(key)
+	res, err := parseKey(key)
+	if err == nil {
+		t.Fatalf("Expected error, but succesfully parsed key %s", res)
+	}
 	if fmt.Sprintf("%s", err) != expectedError {
 		t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
 	}
@@ -42,3 +46,18 @@ func TestDottedKeyBasic(t *testing.T) {
 func TestBaseKeyPound(t *testing.T) {
 	testError(t, "hello#world", "invalid bare character: #")
 }
+
+func TestQuotedKeys(t *testing.T) {
+	testResult(t, `hello."foo".bar`, []string{"hello", "foo", "bar"})
+	testResult(t, `"hello!"`, []string{"hello!"})
+	testResult(t, `foo."ba.r".baz`, []string{"foo", "ba.r", "baz"})
+
+	// escape sequences must not be converted
+	testResult(t, `"hello\tworld"`, []string{`hello\tworld`})
+}
+
+func TestEmptyKey(t *testing.T) {
+	testError(t, "", "empty key")
+	testError(t, " ", "empty key")
+	testResult(t, `""`, []string{""})
+}

lexer.go

@@ -1,16 +1,17 @@
 // TOML lexer.
 //
-// Written using the principles developped by Rob Pike in
+// Written using the principles developed by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
 package toml
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode/utf8"
 )
 
 var dateRegexp *regexp.Regexp
@@ -20,118 +21,123 @@ type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
 type tomlLexer struct {
-	input  string
-	start  int
-	pos    int
-	width  int
-	tokens chan token
-	depth  int
-	line   int
-	col    int
+	inputIdx          int
+	input             []rune // Textual source
+	currentTokenStart int
+	currentTokenStop  int
+	tokens            []token
+	depth             int
+	line              int
+	col               int
+	endbufferLine     int
+	endbufferCol      int
 }
 
-func (l *tomlLexer) run() {
-	for state := l.lexVoid; state != nil; {
-		state = state()
-	}
-	close(l.tokens)
-}
+// Basic read operations on input
 
-func (l *tomlLexer) nextStart() {
-	// iterate by runes (utf8 characters)
-	// search for newlines and advance line/col counts
-	for i := l.start; i < l.pos; {
-		r, width := utf8.DecodeRuneInString(l.input[i:])
-		if r == '\n' {
-			l.line++
-			l.col = 1
-		} else {
-			l.col++
-		}
-		i += width
+func (l *tomlLexer) read() rune {
+	r := l.peek()
+	if r == '\n' {
+		l.endbufferLine++
+		l.endbufferCol = 1
+	} else {
+		l.endbufferCol++
 	}
-	// advance start position to next token
-	l.start = l.pos
-}
-
-func (l *tomlLexer) emit(t tokenType) {
-	l.tokens <- token{
-		Position: Position{l.line, l.col},
-		typ:      t,
-		val:      l.input[l.start:l.pos],
-	}
-	l.nextStart()
-}
-
-func (l *tomlLexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{
-		Position: Position{l.line, l.col},
-		typ:      t,
-		val:      value,
-	}
-	l.nextStart()
+	l.inputIdx++
+	return r
 }
 
 func (l *tomlLexer) next() rune {
-	if l.pos >= len(l.input) {
-		l.width = 0
-		return eof
+	r := l.read()
+
+	if r != eof {
+		l.currentTokenStop++
 	}
-	var r rune
-	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
-	l.pos += l.width
 	return r
 }
 
 func (l *tomlLexer) ignore() {
-	l.nextStart()
+	l.currentTokenStart = l.currentTokenStop
+	l.line = l.endbufferLine
+	l.col = l.endbufferCol
 }
 
-func (l *tomlLexer) backup() {
-	l.pos -= l.width
+func (l *tomlLexer) skip() {
+	l.next()
+	l.ignore()
 }
 
-func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
-	l.tokens <- token{
-		Position: Position{l.line, l.col},
-		typ:      tokenError,
-		val:      fmt.Sprintf(format, args...),
+func (l *tomlLexer) fastForward(n int) {
+	for i := 0; i < n; i++ {
+		l.next()
 	}
-	return nil
+}
+
+func (l *tomlLexer) emitWithValue(t tokenType, value string) {
+	l.tokens = append(l.tokens, token{
+		Position: Position{l.line, l.col},
+		typ:      t,
+		val:      value,
+	})
+	l.ignore()
+}
+
+func (l *tomlLexer) emit(t tokenType) {
+	l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop]))
 }
 
 func (l *tomlLexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
+	if l.inputIdx >= len(l.input) {
+		return eof
+	}
+	return l.input[l.inputIdx]
 }
 
-func (l *tomlLexer) accept(valid string) bool {
-	if strings.IndexRune(valid, l.next()) >= 0 {
-		return true
+func (l *tomlLexer) peekString(size int) string {
+	maxIdx := len(l.input)
+	upperIdx := l.inputIdx + size // FIXME: potential overflow
+	if upperIdx > maxIdx {
+		upperIdx = maxIdx
 	}
-	l.backup()
-	return false
+	return string(l.input[l.inputIdx:upperIdx])
 }
 
 func (l *tomlLexer) follow(next string) bool {
-	return strings.HasPrefix(l.input[l.pos:], next)
+	return next == l.peekString(len(next))
 }
 
+// Error management
+
+func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
+	l.tokens = append(l.tokens, token{
+		Position: Position{l.line, l.col},
+		typ:      tokenError,
+		val:      fmt.Sprintf(format, args...),
+	})
+	return nil
+}
+
+// State functions
+
 func (l *tomlLexer) lexVoid() tomlLexStateFn {
 	for {
 		next := l.peek()
 		switch next {
 		case '[':
-			return l.lexKeyGroup
+			return l.lexTableKey
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexVoid)
 		case '=':
 			return l.lexEqual
+		case '\r':
+			fallthrough
+		case '\n':
+			l.skip()
+			continue
 		}
 
 		if isSpace(next) {
-			l.ignore()
+			l.skip()
 		}
 
 		if l.depth > 0 {
@@ -142,7 +148,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			return l.lexKey
 		}
 
-		if l.next() == eof {
+		if next == eof {
+			l.next()
 			break
 		}
 	}
@@ -170,20 +177,23 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 		case '}':
 			return l.lexRightCurlyBrace
 		case '#':
-			return l.lexComment
+			return l.lexComment(l.lexRvalue)
 		case '"':
 			return l.lexString
 		case '\'':
 			return l.lexLiteralString
 		case ',':
 			return l.lexComma
+		case '\r':
+			fallthrough
 		case '\n':
-			l.ignore()
-			l.pos++
+			l.skip()
 			if l.depth == 0 {
 				return l.lexVoid
 			}
 			return l.lexRvalue
+		case '_':
+			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
@@ -194,14 +204,28 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexFalse
 		}
 
-		if isAlphanumeric(next) {
-			return l.lexKey
+		if l.follow("inf") {
+			return l.lexInf
 		}
 
-		dateMatch := dateRegexp.FindString(l.input[l.pos:])
+		if l.follow("nan") {
+			return l.lexNan
+		}
+
+		if isSpace(next) {
+			l.skip()
+			continue
+		}
+
+		if next == eof {
+			l.next()
+			break
+		}
+
+		possibleDate := l.peekString(35)
+		dateMatch := dateRegexp.FindString(possibleDate)
 		if dateMatch != "" {
-			l.ignore()
-			l.pos += len(dateMatch)
+			l.fastForward(len(dateMatch))
 			return l.lexDate
 		}
 
@@ -209,13 +233,11 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexNumber
 		}
 
-		if isSpace(next) {
-			l.ignore()
+		if isAlphanumeric(next) {
+			return l.lexKey
 		}
 
-		if l.next() == eof {
-			break
-		}
+		return l.errorf("no value can start with %c", next)
 	}
 
 	l.emit(tokenEOF)
@@ -223,15 +245,13 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenLeftCurlyBrace)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenRightCurlyBrace)
 	return l.lexRvalue
 }
@@ -242,137 +262,172 @@ func (l *tomlLexer) lexDate() tomlLexStateFn {
 }
 
 func (l *tomlLexer) lexTrue() tomlLexStateFn {
-	l.ignore()
-	l.pos += 4
+	l.fastForward(4)
 	l.emit(tokenTrue)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexFalse() tomlLexStateFn {
-	l.ignore()
-	l.pos += 5
+	l.fastForward(5)
 	l.emit(tokenFalse)
 	return l.lexRvalue
 }
 
+func (l *tomlLexer) lexInf() tomlLexStateFn {
+	l.fastForward(3)
+	l.emit(tokenInf)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexNan() tomlLexStateFn {
+	l.fastForward(3)
+	l.emit(tokenNan)
+	return l.lexRvalue
+}
+
 func (l *tomlLexer) lexEqual() tomlLexStateFn {
-	l.ignore()
-	l.accept("=")
+	l.next()
 	l.emit(tokenEqual)
 	return l.lexRvalue
 }
 
 func (l *tomlLexer) lexComma() tomlLexStateFn {
-	l.ignore()
-	l.accept(",")
+	l.next()
 	l.emit(tokenComma)
 	return l.lexRvalue
 }
 
+// Parse the key and emits its value without escape sequences.
+// bare keys, basic string keys and literal string keys are supported.
 func (l *tomlLexer) lexKey() tomlLexStateFn {
-	l.ignore()
-	inQuotes := false
-	for r := l.next(); isKeyChar(r) || r == '\n'; r = l.next() {
+	growingString := ""
+
+	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
-			inQuotes = !inQuotes
+			l.next()
+			str, err := l.lexStringAsString(`"`, false, true)
+			if err != nil {
+				return l.errorf(err.Error())
+			}
+			growingString += str
+			l.next()
+			continue
+		} else if r == '\'' {
+			l.next()
+			str, err := l.lexLiteralStringAsString(`'`, false)
+			if err != nil {
+				return l.errorf(err.Error())
+			}
+			growingString += str
+			l.next()
+			continue
 		} else if r == '\n' {
 			return l.errorf("keys cannot contain new lines")
-		} else if isSpace(r) && !inQuotes {
+		} else if isSpace(r) {
 			break
-		} else if !isValidBareChar(r) && !inQuotes {
+		} else if !isValidBareChar(r) {
 			return l.errorf("keys cannot contain %c character", r)
 		}
+		growingString += string(r)
+		l.next()
 	}
-	l.backup()
-	l.emit(tokenKey)
+	l.emitWithValue(tokenKey, growingString)
 	return l.lexVoid
 }
 
-func (l *tomlLexer) lexComment() tomlLexStateFn {
-	for {
-		next := l.next()
-		if next == '\n' || next == eof {
-			break
+func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn {
+	return func() tomlLexStateFn {
+		for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+			if next == '\r' && l.follow("\r\n") {
+				break
+			}
+			l.next()
 		}
+		l.ignore()
+		return previousState
 	}
-	l.ignore()
-	return l.lexVoid
 }
 
 func (l *tomlLexer) lexLeftBracket() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenLeftBracket)
 	return l.lexRvalue
 }
 
-func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
-	l.pos++
-	l.ignore()
+func (l *tomlLexer) lexLiteralStringAsString(terminator string, discardLeadingNewLine bool) (string, error) {
 	growingString := ""
 
-	// handle special case for triple-quote
-	terminator := "'"
-	if l.follow("''") {
-		l.pos += 2
-		l.ignore()
-		terminator = "'''"
-
-		// special case: discard leading newline
-		if l.peek() == '\n' {
-			l.pos++
-			l.ignore()
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
 		}
 	}
 
 	// find end of string
 	for {
 		if l.follow(terminator) {
-			l.emitWithValue(tokenString, growingString)
-			l.pos += len(terminator)
-			l.ignore()
-			return l.lexRvalue
+			return growingString, nil
 		}
 
-		growingString += string(l.peek())
-
-		if l.next() == eof {
+		next := l.peek()
+		if next == eof {
 			break
 		}
+		growingString += string(l.next())
 	}
 
-	return l.errorf("unclosed string")
+	return "", errors.New("unclosed string")
 }
 
-func (l *tomlLexer) lexString() tomlLexStateFn {
-	l.pos++
-	l.ignore()
-	growingString := ""
+func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
+	l.skip()
 
 	// handle special case for triple-quote
-	terminator := "\""
-	if l.follow("\"\"") {
-		l.pos += 2
-		l.ignore()
-		terminator = "\"\"\""
+	terminator := "'"
+	discardLeadingNewLine := false
+	if l.follow("''") {
+		l.skip()
+		l.skip()
+		terminator = "'''"
+		discardLeadingNewLine = true
+	}
 
-		// special case: discard leading newline
-		if l.peek() == '\n' {
-			l.pos++
-			l.ignore()
+	str, err := l.lexLiteralStringAsString(terminator, discardLeadingNewLine)
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
+	l.ignore()
+	return l.lexRvalue
+}
+
+// Lex a string and return the results as a string.
+// Terminator is the substring indicating the end of the token.
+// The resulting string does not include the terminator.
+func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) {
+	growingString := ""
+
+	if discardLeadingNewLine {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
+			l.skip()
 		}
 	}
 
 	for {
 		if l.follow(terminator) {
-			l.emitWithValue(tokenString, growingString)
-			l.pos += len(terminator)
-			l.ignore()
-			return l.lexRvalue
+			return growingString, nil
 		}
 
 		if l.follow("\\") {
-			l.pos++
+			l.next()
 			switch l.peek() {
 			case '\r':
 				fallthrough
@@ -382,177 +437,275 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 				fallthrough
 			case ' ':
 				// skip all whitespace chars following backslash
-				l.pos++
 				for strings.ContainsRune("\r\n\t ", l.peek()) {
-					l.pos++
+					l.next()
 				}
-				l.pos--
 			case '"':
 				growingString += "\""
+				l.next()
 			case 'n':
 				growingString += "\n"
+				l.next()
 			case 'b':
 				growingString += "\b"
+				l.next()
 			case 'f':
 				growingString += "\f"
+				l.next()
 			case '/':
 				growingString += "/"
+				l.next()
 			case 't':
 				growingString += "\t"
+				l.next()
 			case 'r':
 				growingString += "\r"
+				l.next()
 			case '\\':
 				growingString += "\\"
+				l.next()
 			case 'u':
-				l.pos++
+				l.next()
 				code := ""
 				for i := 0; i < 4; i++ {
 					c := l.peek()
-					l.pos++
 					if !isHexDigit(c) {
-						return l.errorf("unfinished unicode escape")
+						return "", errors.New("unfinished unicode escape")
 					}
+					l.next()
 					code = code + string(c)
 				}
-				l.pos--
 				intcode, err := strconv.ParseInt(code, 16, 32)
 				if err != nil {
-					return l.errorf("invalid unicode escape: \\u" + code)
+					return "", errors.New("invalid unicode escape: \\u" + code)
 				}
 				growingString += string(rune(intcode))
 			case 'U':
-				l.pos++
+				l.next()
 				code := ""
 				for i := 0; i < 8; i++ {
 					c := l.peek()
-					l.pos++
 					if !isHexDigit(c) {
-						return l.errorf("unfinished unicode escape")
+						return "", errors.New("unfinished unicode escape")
 					}
+					l.next()
 					code = code + string(c)
 				}
-				l.pos--
 				intcode, err := strconv.ParseInt(code, 16, 64)
 				if err != nil {
-					return l.errorf("invalid unicode escape: \\U" + code)
+					return "", errors.New("invalid unicode escape: \\U" + code)
 				}
 				growingString += string(rune(intcode))
 			default:
-				return l.errorf("invalid escape sequence: \\" + string(l.peek()))
+				return "", errors.New("invalid escape sequence: \\" + string(l.peek()))
 			}
 		} else {
 			r := l.peek()
-			if 0x00 <= r && r <= 0x1F {
-				return l.errorf("unescaped control character %U", r)
+
+			if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) {
+				return "", fmt.Errorf("unescaped control character %U", r)
 			}
+			l.next()
 			growingString += string(r)
 		}
 
-		if l.next() == eof {
+		if l.peek() == eof {
 			break
 		}
 	}
 
-	return l.errorf("unclosed string")
+	return "", errors.New("unclosed string")
 }
 
-func (l *tomlLexer) lexKeyGroup() tomlLexStateFn {
+func (l *tomlLexer) lexString() tomlLexStateFn {
+	l.skip()
+
+	// handle special case for triple-quote
+	terminator := `"`
+	discardLeadingNewLine := false
+	acceptNewLines := false
+	if l.follow(`""`) {
+		l.skip()
+		l.skip()
+		terminator = `"""`
+		discardLeadingNewLine = true
+		acceptNewLines = true
+	}
+
+	str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines)
+
+	if err != nil {
+		return l.errorf(err.Error())
+	}
+
+	l.emitWithValue(tokenString, str)
+	l.fastForward(len(terminator))
 	l.ignore()
-	l.pos++
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexTableKey() tomlLexStateFn {
+	l.next()
 
 	if l.peek() == '[' {
-		// token '[[' signifies an array of anonymous key groups
-		l.pos++
+		// token '[[' signifies an array of tables
+		l.next()
 		l.emit(tokenDoubleLeftBracket)
-		return l.lexInsideKeyGroupArray
+		return l.lexInsideTableArrayKey
 	}
-	// vanilla key group
+	// vanilla table key
 	l.emit(tokenLeftBracket)
-	return l.lexInsideKeyGroup
+	return l.lexInsideTableKey
 }
 
-func (l *tomlLexer) lexInsideKeyGroupArray() tomlLexStateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+// Parse the key till "]]", but only bare keys are supported
+func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroupArray)
 			}
-			l.ignore()
-			l.pos++
+			l.next()
 			if l.peek() != ']' {
-				break // error
+				break
 			}
-			l.pos++
+			l.next()
 			l.emit(tokenDoubleRightBracket)
 			return l.lexVoid
-		} else if l.peek() == '[' {
-			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		case '[':
+			return l.errorf("table array key cannot contain ']'")
+		default:
+			l.next()
 		}
 	}
-	return l.errorf("unclosed key group array")
+	return l.errorf("unclosed table array key")
 }
 
-func (l *tomlLexer) lexInsideKeyGroup() tomlLexStateFn {
-	for {
-		if l.peek() == ']' {
-			if l.pos > l.start {
+// Parse the key till "]" but only bare keys are supported
+func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn {
+	for r := l.peek(); r != eof; r = l.peek() {
+		switch r {
+		case ']':
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroup)
 			}
-			l.ignore()
-			l.pos++
+			l.next()
 			l.emit(tokenRightBracket)
 			return l.lexVoid
-		} else if l.peek() == '[' {
-			return l.errorf("group name cannot contain ']'")
-		}
-
-		if l.next() == eof {
-			break
+		case '[':
+			return l.errorf("table key cannot contain ']'")
+		default:
+			l.next()
 		}
 	}
-	return l.errorf("unclosed key group")
+	return l.errorf("unclosed table key")
 }
 
 func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
-	l.ignore()
-	l.pos++
+	l.next()
 	l.emit(tokenRightBracket)
 	return l.lexRvalue
 }
 
+type validRuneFn func(r rune) bool
+
+func isValidHexRune(r rune) bool {
+	return r >= 'a' && r <= 'f' ||
+		r >= 'A' && r <= 'F' ||
+		r >= '0' && r <= '9' ||
+		r == '_'
+}
+
+func isValidOctalRune(r rune) bool {
+	return r >= '0' && r <= '7' || r == '_'
+}
+
+func isValidBinaryRune(r rune) bool {
+	return r == '0' || r == '1' || r == '_'
+}
+
 func (l *tomlLexer) lexNumber() tomlLexStateFn {
-	l.ignore()
-	if !l.accept("+") {
-		l.accept("-")
+	r := l.peek()
+
+	if r == '0' {
+		follow := l.peekString(2)
+		if len(follow) == 2 {
+			var isValidRune validRuneFn
+			switch follow[1] {
+			case 'x':
+				isValidRune = isValidHexRune
+			case 'o':
+				isValidRune = isValidOctalRune
+			case 'b':
+				isValidRune = isValidBinaryRune
+			default:
+				if follow[1] >= 'a' && follow[1] <= 'z' || follow[1] >= 'A' && follow[1] <= 'Z' {
+					return l.errorf("unknown number base: %s. possible options are x (hex) o (octal) b (binary)", string(follow[1]))
+				}
+			}
+
+			if isValidRune != nil {
+				l.next()
+				l.next()
+				digitSeen := false
+				for {
+					next := l.peek()
+					if !isValidRune(next) {
+						break
+					}
+					digitSeen = true
+					l.next()
+				}
+
+				if !digitSeen {
+					return l.errorf("number needs at least one digit")
+				}
+
+				l.emit(tokenInteger)
+
+				return l.lexRvalue
+			}
+		}
 	}
+
+	if r == '+' || r == '-' {
+		l.next()
+		if l.follow("inf") {
+			return l.lexInf
+		}
+		if l.follow("nan") {
+			return l.lexNan
+		}
+	}
+
 	pointSeen := false
 	expSeen := false
 	digitSeen := false
 	for {
-		next := l.next()
+		next := l.peek()
 		if next == '.' {
 			if pointSeen {
 				return l.errorf("cannot have two dots in one float")
 			}
+			l.next()
 			if !isDigit(l.peek()) {
 				return l.errorf("float cannot end with a dot")
 			}
 			pointSeen = true
 		} else if next == 'e' || next == 'E' {
 			expSeen = true
-			if !l.accept("+") {
-				l.accept("-")
+			l.next()
+			r := l.peek()
+			if r == '+' || r == '-' {
+				l.next()
 			}
 		} else if isDigit(next) {
 			digitSeen = true
+			l.next()
 		} else if next == '_' {
-			l.pos++
+			l.next()
 		} else {
-			l.backup()
 			break
 		}
 		if pointSeen && !digitSeen {
@@ -571,18 +724,27 @@ func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+func (l *tomlLexer) run() {
+	for state := l.lexVoid; state != nil; {
+		state = state()
+	}
+}
+
 func init() {
-	dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})")
+	dateRegexp = regexp.MustCompile(`^\d{1,4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})`)
 }
 
 // Entry point
-func lexToml(input string) chan token {
+func lexToml(inputBytes []byte) []token {
+	runes := bytes.Runes(inputBytes)
 	l := &tomlLexer{
-		input:  input,
-		tokens: make(chan token),
-		line:   1,
-		col:    1,
+		input:         runes,
+		tokens:        make([]token, 0, 256),
+		line:          1,
+		col:           1,
+		endbufferLine: 1,
+		endbufferCol:  1,
 	}
-	go l.run()
+	l.run()
 	return l.tokens
 }

marshal.go

@@ -0,0 +1,609 @@
+package toml
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const tagKeyMultiline = "multiline"
+
+type tomlOpts struct {
+	name      string
+	comment   string
+	commented bool
+	multiline bool
+	include   bool
+	omitempty bool
+}
+
+type encOpts struct {
+	quoteMapKeys            bool
+	arraysOneElementPerLine bool
+}
+
+var encOptsDefaults = encOpts{
+	quoteMapKeys: false,
+}
+
+var timeType = reflect.TypeOf(time.Time{})
+var marshalerType = reflect.TypeOf(new(Marshaler)).Elem()
+
+// Check if the given marshall type maps to a Tree primitive
+func isPrimitive(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Ptr:
+		return isPrimitive(mtype.Elem())
+	case reflect.Bool:
+		return true
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return true
+	case reflect.Float32, reflect.Float64:
+		return true
+	case reflect.String:
+		return true
+	case reflect.Struct:
+		return mtype == timeType || isCustomMarshaler(mtype)
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a Tree slice
+func isTreeSlice(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Slice:
+		return !isOtherSlice(mtype)
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a non-Tree slice
+func isOtherSlice(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Ptr:
+		return isOtherSlice(mtype.Elem())
+	case reflect.Slice:
+		return isPrimitive(mtype.Elem()) || isOtherSlice(mtype.Elem())
+	default:
+		return false
+	}
+}
+
+// Check if the given marshall type maps to a Tree
+func isTree(mtype reflect.Type) bool {
+	switch mtype.Kind() {
+	case reflect.Map:
+		return true
+	case reflect.Struct:
+		return !isPrimitive(mtype)
+	default:
+		return false
+	}
+}
+
+func isCustomMarshaler(mtype reflect.Type) bool {
+	return mtype.Implements(marshalerType)
+}
+
+func callCustomMarshaler(mval reflect.Value) ([]byte, error) {
+	return mval.Interface().(Marshaler).MarshalTOML()
+}
+
+// Marshaler is the interface implemented by types that
+// can marshal themselves into valid TOML.
+type Marshaler interface {
+	MarshalTOML() ([]byte, error)
+}
+
+/*
+Marshal returns the TOML encoding of v.  Behavior is similar to the Go json
+encoder, except that there is no concept of a Marshaler interface or MarshalTOML
+function for sub-structs, and currently only definite types can be marshaled
+(i.e. no `interface{}`).
+
+The following struct annotations are supported:
+
+  toml:"Field"      Overrides the field's name to output.
+  omitempty         When set, empty values and groups are not emitted.
+  comment:"comment" Emits a # comment on the same line. This supports new lines.
+  commented:"true"  Emits the value as commented.
+
+Note that pointers are automatically assigned the "omitempty" option, as TOML
+explicitly does not handle null values (saying instead the label should be
+dropped).
+
+Tree structural types and corresponding marshal types:
+
+  *Tree                            (*)struct, (*)map[string]interface{}
+  []*Tree                          (*)[](*)struct, (*)[](*)map[string]interface{}
+  []interface{} (as interface{})   (*)[]primitive, (*)[]([]interface{})
+  interface{}                      (*)primitive
+
+Tree primitive types and corresponding marshal types:
+
+  uint64     uint, uint8-uint64, pointers to same
+  int64      int, int8-uint64, pointers to same
+  float64    float32, float64, pointers to same
+  string     string, pointers to same
+  bool       bool, pointers to same
+  time.Time  time.Time{}, pointers to same
+*/
+func Marshal(v interface{}) ([]byte, error) {
+	return NewEncoder(nil).marshal(v)
+}
+
+// Encoder writes TOML values to an output stream.
+type Encoder struct {
+	w io.Writer
+	encOpts
+}
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		w:       w,
+		encOpts: encOptsDefaults,
+	}
+}
+
+// Encode writes the TOML encoding of v to the stream.
+//
+// See the documentation for Marshal for details.
+func (e *Encoder) Encode(v interface{}) error {
+	b, err := e.marshal(v)
+	if err != nil {
+		return err
+	}
+	if _, err := e.w.Write(b); err != nil {
+		return err
+	}
+	return nil
+}
+
+// QuoteMapKeys sets up the encoder to encode
+// maps with string type keys with quoted TOML keys.
+//
+// This relieves the character limitations on map keys.
+func (e *Encoder) QuoteMapKeys(v bool) *Encoder {
+	e.quoteMapKeys = v
+	return e
+}
+
+// ArraysWithOneElementPerLine sets up the encoder to encode arrays
+// with more than one element on multiple lines instead of one.
+//
+// For example:
+//
+//   A = [1,2,3]
+//
+// Becomes
+//
+//   A = [
+//     1,
+//     2,
+//     3,
+//   ]
+func (e *Encoder) ArraysWithOneElementPerLine(v bool) *Encoder {
+	e.arraysOneElementPerLine = v
+	return e
+}
+
+func (e *Encoder) marshal(v interface{}) ([]byte, error) {
+	mtype := reflect.TypeOf(v)
+	if mtype.Kind() != reflect.Struct {
+		return []byte{}, errors.New("Only a struct can be marshaled to TOML")
+	}
+	sval := reflect.ValueOf(v)
+	if isCustomMarshaler(mtype) {
+		return callCustomMarshaler(sval)
+	}
+	t, err := e.valueToTree(mtype, sval)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	var buf bytes.Buffer
+	_, err = t.writeTo(&buf, "", "", 0, e.arraysOneElementPerLine)
+
+	return buf.Bytes(), err
+}
+
+// Convert given marshal struct or map value to toml tree
+func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return e.valueToTree(mtype.Elem(), mval.Elem())
+	}
+	tval := newTree()
+	switch mtype.Kind() {
+	case reflect.Struct:
+		for i := 0; i < mtype.NumField(); i++ {
+			mtypef, mvalf := mtype.Field(i), mval.Field(i)
+			opts := tomlOptions(mtypef)
+			if opts.include && (!opts.omitempty || !isZero(mvalf)) {
+				val, err := e.valueToToml(mtypef.Type, mvalf)
+				if err != nil {
+					return nil, err
+				}
+
+				tval.SetWithOptions(opts.name, SetOptions{
+					Comment:   opts.comment,
+					Commented: opts.commented,
+					Multiline: opts.multiline,
+				}, val)
+			}
+		}
+	case reflect.Map:
+		for _, key := range mval.MapKeys() {
+			mvalf := mval.MapIndex(key)
+			val, err := e.valueToToml(mtype.Elem(), mvalf)
+			if err != nil {
+				return nil, err
+			}
+			if e.quoteMapKeys {
+				keyStr, err := tomlValueStringRepresentation(key.String(), "", e.arraysOneElementPerLine)
+				if err != nil {
+					return nil, err
+				}
+				tval.SetPath([]string{keyStr}, val)
+			} else {
+				tval.Set(key.String(), val)
+			}
+		}
+	}
+	return tval, nil
+}
+
+// Convert given marshal slice to slice of Toml trees
+func (e *Encoder) valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) {
+	tval := make([]*Tree, mval.Len(), mval.Len())
+	for i := 0; i < mval.Len(); i++ {
+		val, err := e.valueToTree(mtype.Elem(), mval.Index(i))
+		if err != nil {
+			return nil, err
+		}
+		tval[i] = val
+	}
+	return tval, nil
+}
+
+// Convert given marshal slice to slice of toml values
+func (e *Encoder) valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+	tval := make([]interface{}, mval.Len(), mval.Len())
+	for i := 0; i < mval.Len(); i++ {
+		val, err := e.valueToToml(mtype.Elem(), mval.Index(i))
+		if err != nil {
+			return nil, err
+		}
+		tval[i] = val
+	}
+	return tval, nil
+}
+
+// Convert given marshal value to toml value
+func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return e.valueToToml(mtype.Elem(), mval.Elem())
+	}
+	switch {
+	case isCustomMarshaler(mtype):
+		return callCustomMarshaler(mval)
+	case isTree(mtype):
+		return e.valueToTree(mtype, mval)
+	case isTreeSlice(mtype):
+		return e.valueToTreeSlice(mtype, mval)
+	case isOtherSlice(mtype):
+		return e.valueToOtherSlice(mtype, mval)
+	default:
+		switch mtype.Kind() {
+		case reflect.Bool:
+			return mval.Bool(), nil
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			return mval.Int(), nil
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			return mval.Uint(), nil
+		case reflect.Float32, reflect.Float64:
+			return mval.Float(), nil
+		case reflect.String:
+			return mval.String(), nil
+		case reflect.Struct:
+			return mval.Interface().(time.Time), nil
+		default:
+			return nil, fmt.Errorf("Marshal can't handle %v(%v)", mtype, mtype.Kind())
+		}
+	}
+}
+
+// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v.
+// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for
+// sub-structs, and only definite types can be unmarshaled.
+func (t *Tree) Unmarshal(v interface{}) error {
+	d := Decoder{tval: t}
+	return d.unmarshal(v)
+}
+
+// Marshal returns the TOML encoding of Tree.
+// See Marshal() documentation for types mapping table.
+func (t *Tree) Marshal() ([]byte, error) {
+	var buf bytes.Buffer
+	err := NewEncoder(&buf).Encode(t)
+	return buf.Bytes(), err
+}
+
+// Unmarshal parses the TOML-encoded data and stores the result in the value
+// pointed to by v. Behavior is similar to the Go json encoder, except that there
+// is no concept of an Unmarshaler interface or UnmarshalTOML function for
+// sub-structs, and currently only definite types can be unmarshaled to (i.e. no
+// `interface{}`).
+//
+// The following struct annotations are supported:
+//
+//   toml:"Field" Overrides the field's name to map to.
+//
+// See Marshal() documentation for types mapping table.
+func Unmarshal(data []byte, v interface{}) error {
+	t, err := LoadReader(bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	return t.Unmarshal(v)
+}
+
+// Decoder reads and decodes TOML values from an input stream.
+type Decoder struct {
+	r    io.Reader
+	tval *Tree
+	encOpts
+}
+
+// NewDecoder returns a new decoder that reads from r.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{
+		r:       r,
+		encOpts: encOptsDefaults,
+	}
+}
+
+// Decode reads a TOML-encoded value from it's input
+// and unmarshals it in the value pointed at by v.
+//
+// See the documentation for Marshal for details.
+func (d *Decoder) Decode(v interface{}) error {
+	var err error
+	d.tval, err = LoadReader(d.r)
+	if err != nil {
+		return err
+	}
+	return d.unmarshal(v)
+}
+
+func (d *Decoder) unmarshal(v interface{}) error {
+	mtype := reflect.TypeOf(v)
+	if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct {
+		return errors.New("Only a pointer to struct can be unmarshaled from TOML")
+	}
+
+	sval, err := d.valueFromTree(mtype.Elem(), d.tval)
+	if err != nil {
+		return err
+	}
+	reflect.ValueOf(v).Elem().Set(sval)
+	return nil
+}
+
+// Convert toml tree to marshal struct or map, using marshal type
+func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return d.unwrapPointer(mtype, tval)
+	}
+	var mval reflect.Value
+	switch mtype.Kind() {
+	case reflect.Struct:
+		mval = reflect.New(mtype).Elem()
+		for i := 0; i < mtype.NumField(); i++ {
+			mtypef := mtype.Field(i)
+			opts := tomlOptions(mtypef)
+			if opts.include {
+				baseKey := opts.name
+				keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)}
+				for _, key := range keysToTry {
+					exists := tval.Has(key)
+					if !exists {
+						continue
+					}
+					val := tval.Get(key)
+					mvalf, err := d.valueFromToml(mtypef.Type, val)
+					if err != nil {
+						return mval, formatError(err, tval.GetPosition(key))
+					}
+					mval.Field(i).Set(mvalf)
+					break
+				}
+			}
+		}
+	case reflect.Map:
+		mval = reflect.MakeMap(mtype)
+		for _, key := range tval.Keys() {
+			// TODO: path splits key
+			val := tval.GetPath([]string{key})
+			mvalf, err := d.valueFromToml(mtype.Elem(), val)
+			if err != nil {
+				return mval, formatError(err, tval.GetPosition(key))
+			}
+			mval.SetMapIndex(reflect.ValueOf(key), mvalf)
+		}
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal struct/map slice, using marshal type
+func (d *Decoder) valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) {
+	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
+	for i := 0; i < len(tval); i++ {
+		val, err := d.valueFromTree(mtype.Elem(), tval[i])
+		if err != nil {
+			return mval, err
+		}
+		mval.Index(i).Set(val)
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal primitive slice, using marshal type
+func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) {
+	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
+	for i := 0; i < len(tval); i++ {
+		val, err := d.valueFromToml(mtype.Elem(), tval[i])
+		if err != nil {
+			return mval, err
+		}
+		mval.Index(i).Set(val)
+	}
+	return mval, nil
+}
+
+// Convert toml value to marshal value, using marshal type
+func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+	if mtype.Kind() == reflect.Ptr {
+		return d.unwrapPointer(mtype, tval)
+	}
+
+	switch tval.(type) {
+	case *Tree:
+		if isTree(mtype) {
+			return d.valueFromTree(mtype, tval.(*Tree))
+		}
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a tree", tval, tval)
+	case []*Tree:
+		if isTreeSlice(mtype) {
+			return d.valueFromTreeSlice(mtype, tval.([]*Tree))
+		}
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to trees", tval, tval)
+	case []interface{}:
+		if isOtherSlice(mtype) {
+			return d.valueFromOtherSlice(mtype, tval.([]interface{}))
+		}
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a slice", tval, tval)
+	default:
+		switch mtype.Kind() {
+		case reflect.Bool, reflect.Struct:
+			val := reflect.ValueOf(tval)
+			// if this passes for when mtype is reflect.Struct, tval is a time.Time
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.String:
+			val := reflect.ValueOf(tval)
+			// stupidly, int64 is convertible to string. So special case this.
+			if !val.Type().ConvertibleTo(mtype) || val.Kind() == reflect.Int64 {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			val := reflect.ValueOf(tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+			if reflect.Indirect(reflect.New(mtype)).OverflowInt(val.Int()) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+			val := reflect.ValueOf(tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+			if val.Int() < 0 {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) is negative so does not fit in %v", tval, tval, mtype.String())
+			}
+			if reflect.Indirect(reflect.New(mtype)).OverflowUint(uint64(val.Int())) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.Float32, reflect.Float64:
+			val := reflect.ValueOf(tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+			if reflect.Indirect(reflect.New(mtype)).OverflowFloat(val.Float()) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		default:
+			return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
+		}
+	}
+}
+
+func (d *Decoder) unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+	val, err := d.valueFromToml(mtype.Elem(), tval)
+	if err != nil {
+		return reflect.ValueOf(nil), err
+	}
+	mval := reflect.New(mtype.Elem())
+	mval.Elem().Set(val)
+	return mval, nil
+}
+
+func tomlOptions(vf reflect.StructField) tomlOpts {
+	tag := vf.Tag.Get("toml")
+	parse := strings.Split(tag, ",")
+	var comment string
+	if c := vf.Tag.Get("comment"); c != "" {
+		comment = c
+	}
+	commented, _ := strconv.ParseBool(vf.Tag.Get("commented"))
+	multiline, _ := strconv.ParseBool(vf.Tag.Get(tagKeyMultiline))
+	result := tomlOpts{name: vf.Name, comment: comment, commented: commented, multiline: multiline, include: true, omitempty: false}
+	if parse[0] != "" {
+		if parse[0] == "-" && len(parse) == 1 {
+			result.include = false
+		} else {
+			result.name = strings.Trim(parse[0], " ")
+		}
+	}
+	if vf.PkgPath != "" {
+		result.include = false
+	}
+	if len(parse) > 1 && strings.Trim(parse[1], " ") == "omitempty" {
+		result.omitempty = true
+	}
+	if vf.Type.Kind() == reflect.Ptr {
+		result.omitempty = true
+	}
+	return result
+}
+
+func isZero(val reflect.Value) bool {
+	switch val.Type().Kind() {
+	case reflect.Map:
+		fallthrough
+	case reflect.Array:
+		fallthrough
+	case reflect.Slice:
+		return val.Len() == 0
+	default:
+		return reflect.DeepEqual(val.Interface(), reflect.Zero(val.Type()).Interface())
+	}
+}
+
+func formatError(err error, pos Position) error {
+	if err.Error()[0] == '(' { // Error already contains position information
+		return err
+	}
+	return fmt.Errorf("%s: %s", pos, err)
+}

marshal_test.go

@@ -0,0 +1,806 @@
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+type basicMarshalTestStruct struct {
+	String     string                      `toml:"string"`
+	StringList []string                    `toml:"strlist"`
+	Sub        basicMarshalTestSubStruct   `toml:"subdoc"`
+	SubList    []basicMarshalTestSubStruct `toml:"sublist"`
+}
+
+type basicMarshalTestSubStruct struct {
+	String2 string
+}
+
+var basicTestData = basicMarshalTestStruct{
+	String:     "Hello",
+	StringList: []string{"Howdy", "Hey There"},
+	Sub:        basicMarshalTestSubStruct{"One"},
+	SubList:    []basicMarshalTestSubStruct{{"Two"}, {"Three"}},
+}
+
+var basicTestToml = []byte(`string = "Hello"
+strlist = ["Howdy","Hey There"]
+
+[subdoc]
+  String2 = "One"
+
+[[sublist]]
+  String2 = "Two"
+
+[[sublist]]
+  String2 = "Three"
+`)
+
+func TestBasicMarshal(t *testing.T) {
+	result, err := Marshal(basicTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := basicTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestBasicUnmarshal(t *testing.T) {
+	result := basicMarshalTestStruct{}
+	err := Unmarshal(basicTestToml, &result)
+	expected := basicTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type testDoc struct {
+	Title       string            `toml:"title"`
+	Basics      testDocBasics     `toml:"basic"`
+	BasicLists  testDocBasicLists `toml:"basic_lists"`
+	BasicMap    map[string]string `toml:"basic_map"`
+	Subdocs     testDocSubs       `toml:"subdoc"`
+	SubDocList  []testSubDoc      `toml:"subdoclist"`
+	SubDocPtrs  []*testSubDoc     `toml:"subdocptrs"`
+	err         int               `toml:"shouldntBeHere"`
+	unexported  int               `toml:"shouldntBeHere"`
+	Unexported2 int               `toml:"-"`
+}
+
+type testDocBasics struct {
+	Bool       bool      `toml:"bool"`
+	Date       time.Time `toml:"date"`
+	Float      float32   `toml:"float"`
+	Int        int       `toml:"int"`
+	Uint       uint      `toml:"uint"`
+	String     *string   `toml:"string"`
+	unexported int       `toml:"shouldntBeHere"`
+}
+
+type testDocBasicLists struct {
+	Bools   []bool      `toml:"bools"`
+	Dates   []time.Time `toml:"dates"`
+	Floats  []*float32  `toml:"floats"`
+	Ints    []int       `toml:"ints"`
+	Strings []string    `toml:"strings"`
+	UInts   []uint      `toml:"uints"`
+}
+
+type testDocSubs struct {
+	First  testSubDoc  `toml:"first"`
+	Second *testSubDoc `toml:"second"`
+}
+
+type testSubDoc struct {
+	Name       string `toml:"name"`
+	unexported int    `toml:"shouldntBeHere"`
+}
+
+var biteMe = "Bite me"
+var float1 float32 = 12.3
+var float2 float32 = 45.6
+var float3 float32 = 78.9
+var subdoc = testSubDoc{"Second", 0}
+
+var docData = testDoc{
+	Title:       "TOML Marshal Testing",
+	unexported:  0,
+	Unexported2: 0,
+	Basics: testDocBasics{
+		Bool:       true,
+		Date:       time.Date(1979, 5, 27, 7, 32, 0, 0, time.UTC),
+		Float:      123.4,
+		Int:        5000,
+		Uint:       5001,
+		String:     &biteMe,
+		unexported: 0,
+	},
+	BasicLists: testDocBasicLists{
+		Bools: []bool{true, false, true},
+		Dates: []time.Time{
+			time.Date(1979, 5, 27, 7, 32, 0, 0, time.UTC),
+			time.Date(1980, 5, 27, 7, 32, 0, 0, time.UTC),
+		},
+		Floats:  []*float32{&float1, &float2, &float3},
+		Ints:    []int{8001, 8001, 8002},
+		Strings: []string{"One", "Two", "Three"},
+		UInts:   []uint{5002, 5003},
+	},
+	BasicMap: map[string]string{
+		"one": "one",
+		"two": "two",
+	},
+	Subdocs: testDocSubs{
+		First:  testSubDoc{"First", 0},
+		Second: &subdoc,
+	},
+	SubDocList: []testSubDoc{
+		{"List.First", 0},
+		{"List.Second", 0},
+	},
+	SubDocPtrs: []*testSubDoc{&subdoc},
+}
+
+func TestDocMarshal(t *testing.T) {
+	result, err := Marshal(docData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected, _ := ioutil.ReadFile("marshal_test.toml")
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestDocUnmarshal(t *testing.T) {
+	result := testDoc{}
+	tomlData, _ := ioutil.ReadFile("marshal_test.toml")
+	err := Unmarshal(tomlData, &result)
+	expected := docData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		resStr, _ := json.MarshalIndent(result, "", "  ")
+		expStr, _ := json.MarshalIndent(expected, "", "  ")
+		t.Errorf("Bad unmarshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expStr, resStr)
+	}
+}
+
+func TestDocPartialUnmarshal(t *testing.T) {
+	result := testDocSubs{}
+
+	tree, _ := LoadFile("marshal_test.toml")
+	subTree := tree.Get("subdoc").(*Tree)
+	err := subTree.Unmarshal(&result)
+	expected := docData.Subdocs
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		resStr, _ := json.MarshalIndent(result, "", "  ")
+		expStr, _ := json.MarshalIndent(expected, "", "  ")
+		t.Errorf("Bad partial unmartial: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expStr, resStr)
+	}
+}
+
+type tomlTypeCheckTest struct {
+	name string
+	item interface{}
+	typ  int //0=primitive, 1=otherslice, 2=treeslice, 3=tree
+}
+
+func TestTypeChecks(t *testing.T) {
+	tests := []tomlTypeCheckTest{
+		{"integer", 2, 0},
+		{"time", time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC), 0},
+		{"stringlist", []string{"hello", "hi"}, 1},
+		{"timelist", []time.Time{time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)}, 1},
+		{"objectlist", []tomlTypeCheckTest{}, 2},
+		{"object", tomlTypeCheckTest{}, 3},
+	}
+
+	for _, test := range tests {
+		expected := []bool{false, false, false, false}
+		expected[test.typ] = true
+		result := []bool{
+			isPrimitive(reflect.TypeOf(test.item)),
+			isOtherSlice(reflect.TypeOf(test.item)),
+			isTreeSlice(reflect.TypeOf(test.item)),
+			isTree(reflect.TypeOf(test.item)),
+		}
+		if !reflect.DeepEqual(expected, result) {
+			t.Errorf("Bad type check on %q: expected %v, got %v", test.name, expected, result)
+		}
+	}
+}
+
+type unexportedMarshalTestStruct struct {
+	String      string                      `toml:"string"`
+	StringList  []string                    `toml:"strlist"`
+	Sub         basicMarshalTestSubStruct   `toml:"subdoc"`
+	SubList     []basicMarshalTestSubStruct `toml:"sublist"`
+	unexported  int                         `toml:"shouldntBeHere"`
+	Unexported2 int                         `toml:"-"`
+}
+
+var unexportedTestData = unexportedMarshalTestStruct{
+	String:      "Hello",
+	StringList:  []string{"Howdy", "Hey There"},
+	Sub:         basicMarshalTestSubStruct{"One"},
+	SubList:     []basicMarshalTestSubStruct{{"Two"}, {"Three"}},
+	unexported:  0,
+	Unexported2: 0,
+}
+
+var unexportedTestToml = []byte(`string = "Hello"
+strlist = ["Howdy","Hey There"]
+unexported = 1
+shouldntBeHere = 2
+
+[subdoc]
+  String2 = "One"
+
+[[sublist]]
+  String2 = "Two"
+
+[[sublist]]
+  String2 = "Three"
+`)
+
+func TestUnexportedUnmarshal(t *testing.T) {
+	result := unexportedMarshalTestStruct{}
+	err := Unmarshal(unexportedTestToml, &result)
+	expected := unexportedTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad unexported unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type errStruct struct {
+	Bool   bool      `toml:"bool"`
+	Date   time.Time `toml:"date"`
+	Float  float64   `toml:"float"`
+	Int    int16     `toml:"int"`
+	String *string   `toml:"string"`
+}
+
+var errTomls = []string{
+	"bool = truly\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:3200Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123a4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = j000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = Bite me",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = Bite me",
+	"bool = 1\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1\nfloat = 123.4\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\n\"sorry\"\nint = 5000\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = \"sorry\"\nstring = \"Bite me\"",
+	"bool = true\ndate = 1979-05-27T07:32:00Z\nfloat = 123.4\nint = 5000\nstring = 1",
+}
+
+type mapErr struct {
+	Vals map[string]float64
+}
+
+type intErr struct {
+	Int1  int
+	Int2  int8
+	Int3  int16
+	Int4  int32
+	Int5  int64
+	UInt1 uint
+	UInt2 uint8
+	UInt3 uint16
+	UInt4 uint32
+	UInt5 uint64
+	Flt1  float32
+	Flt2  float64
+}
+
+var intErrTomls = []string{
+	"Int1 = []\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = []\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = []\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = []\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = []\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = []\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = []\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = []\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = []\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = []\nFlt1 = 1.0\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = []\nFlt2 = 2.0",
+	"Int1 = 1\nInt2 = 2\nInt3 = 3\nInt4 = 4\nInt5 = 5\nUInt1 = 1\nUInt2 = 2\nUInt3 = 3\nUInt4 = 4\nUInt5 = 5\nFlt1 = 1.0\nFlt2 = []",
+}
+
+func TestErrUnmarshal(t *testing.T) {
+	for ind, toml := range errTomls {
+		result := errStruct{}
+		err := Unmarshal([]byte(toml), &result)
+		if err == nil {
+			t.Errorf("Expected err from case %d\n", ind)
+		}
+	}
+	result2 := mapErr{}
+	err := Unmarshal([]byte("[Vals]\nfred=\"1.2\""), &result2)
+	if err == nil {
+		t.Errorf("Expected err from map")
+	}
+	for ind, toml := range intErrTomls {
+		result3 := intErr{}
+		err := Unmarshal([]byte(toml), &result3)
+		if err == nil {
+			t.Errorf("Expected int err from case %d\n", ind)
+		}
+	}
+}
+
+type emptyMarshalTestStruct struct {
+	Title      string                  `toml:"title"`
+	Bool       bool                    `toml:"bool"`
+	Int        int                     `toml:"int"`
+	String     string                  `toml:"string"`
+	StringList []string                `toml:"stringlist"`
+	Ptr        *basicMarshalTestStruct `toml:"ptr"`
+	Map        map[string]string       `toml:"map"`
+}
+
+var emptyTestData = emptyMarshalTestStruct{
+	Title:      "Placeholder",
+	Bool:       false,
+	Int:        0,
+	String:     "",
+	StringList: []string{},
+	Ptr:        nil,
+	Map:        map[string]string{},
+}
+
+var emptyTestToml = []byte(`bool = false
+int = 0
+string = ""
+stringlist = []
+title = "Placeholder"
+
+[map]
+`)
+
+type emptyMarshalTestStruct2 struct {
+	Title      string                  `toml:"title"`
+	Bool       bool                    `toml:"bool,omitempty"`
+	Int        int                     `toml:"int, omitempty"`
+	String     string                  `toml:"string,omitempty "`
+	StringList []string                `toml:"stringlist,omitempty"`
+	Ptr        *basicMarshalTestStruct `toml:"ptr,omitempty"`
+	Map        map[string]string       `toml:"map,omitempty"`
+}
+
+var emptyTestData2 = emptyMarshalTestStruct2{
+	Title:      "Placeholder",
+	Bool:       false,
+	Int:        0,
+	String:     "",
+	StringList: []string{},
+	Ptr:        nil,
+	Map:        map[string]string{},
+}
+
+var emptyTestToml2 = []byte(`title = "Placeholder"
+`)
+
+func TestEmptyMarshal(t *testing.T) {
+	result, err := Marshal(emptyTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := emptyTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad empty marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestEmptyMarshalOmit(t *testing.T) {
+	result, err := Marshal(emptyTestData2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := emptyTestToml2
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad empty omit marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestEmptyUnmarshal(t *testing.T) {
+	result := emptyMarshalTestStruct{}
+	err := Unmarshal(emptyTestToml, &result)
+	expected := emptyTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad empty unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+func TestEmptyUnmarshalOmit(t *testing.T) {
+	result := emptyMarshalTestStruct2{}
+	err := Unmarshal(emptyTestToml, &result)
+	expected := emptyTestData2
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad empty omit unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type pointerMarshalTestStruct struct {
+	Str       *string
+	List      *[]string
+	ListPtr   *[]*string
+	Map       *map[string]string
+	MapPtr    *map[string]*string
+	EmptyStr  *string
+	EmptyList *[]string
+	EmptyMap  *map[string]string
+	DblPtr    *[]*[]*string
+}
+
+var pointerStr = "Hello"
+var pointerList = []string{"Hello back"}
+var pointerListPtr = []*string{&pointerStr}
+var pointerMap = map[string]string{"response": "Goodbye"}
+var pointerMapPtr = map[string]*string{"alternate": &pointerStr}
+var pointerTestData = pointerMarshalTestStruct{
+	Str:       &pointerStr,
+	List:      &pointerList,
+	ListPtr:   &pointerListPtr,
+	Map:       &pointerMap,
+	MapPtr:    &pointerMapPtr,
+	EmptyStr:  nil,
+	EmptyList: nil,
+	EmptyMap:  nil,
+}
+
+var pointerTestToml = []byte(`List = ["Hello back"]
+ListPtr = ["Hello"]
+Str = "Hello"
+
+[Map]
+  response = "Goodbye"
+
+[MapPtr]
+  alternate = "Hello"
+`)
+
+func TestPointerMarshal(t *testing.T) {
+	result, err := Marshal(pointerTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := pointerTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad pointer marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestPointerUnmarshal(t *testing.T) {
+	result := pointerMarshalTestStruct{}
+	err := Unmarshal(pointerTestToml, &result)
+	expected := pointerTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad pointer unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+func TestUnmarshalTypeMismatch(t *testing.T) {
+	result := pointerMarshalTestStruct{}
+	err := Unmarshal([]byte("List = 123"), &result)
+	if !strings.HasPrefix(err.Error(), "(1, 1): Can't convert 123(int64) to []string(slice)") {
+		t.Errorf("Type mismatch must be reported: got %v", err.Error())
+	}
+}
+
+type nestedMarshalTestStruct struct {
+	String [][]string
+	//Struct [][]basicMarshalTestSubStruct
+	StringPtr *[]*[]*string
+	// StructPtr *[]*[]*basicMarshalTestSubStruct
+}
+
+var str1 = "Three"
+var str2 = "Four"
+var strPtr = []*string{&str1, &str2}
+var strPtr2 = []*[]*string{&strPtr}
+
+var nestedTestData = nestedMarshalTestStruct{
+	String:    [][]string{{"Five", "Six"}, {"One", "Two"}},
+	StringPtr: &strPtr2,
+}
+
+var nestedTestToml = []byte(`String = [["Five","Six"],["One","Two"]]
+StringPtr = [["Three","Four"]]
+`)
+
+func TestNestedMarshal(t *testing.T) {
+	result, err := Marshal(nestedTestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := nestedTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad nested marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestNestedUnmarshal(t *testing.T) {
+	result := nestedMarshalTestStruct{}
+	err := Unmarshal(nestedTestToml, &result)
+	expected := nestedTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad nested unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type customMarshalerParent struct {
+	Self    customMarshaler   `toml:"me"`
+	Friends []customMarshaler `toml:"friends"`
+}
+
+type customMarshaler struct {
+	FirsName string
+	LastName string
+}
+
+func (c customMarshaler) MarshalTOML() ([]byte, error) {
+	fullName := fmt.Sprintf("%s %s", c.FirsName, c.LastName)
+	return []byte(fullName), nil
+}
+
+var customMarshalerData = customMarshaler{FirsName: "Sally", LastName: "Fields"}
+var customMarshalerToml = []byte(`Sally Fields`)
+var nestedCustomMarshalerData = customMarshalerParent{
+	Self:    customMarshaler{FirsName: "Maiku", LastName: "Suteda"},
+	Friends: []customMarshaler{customMarshalerData},
+}
+var nestedCustomMarshalerToml = []byte(`friends = ["Sally Fields"]
+me = "Maiku Suteda"
+`)
+
+func TestCustomMarshaler(t *testing.T) {
+	result, err := Marshal(customMarshalerData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := customMarshalerToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestNestedCustomMarshaler(t *testing.T) {
+	result, err := Marshal(nestedCustomMarshalerData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := nestedCustomMarshalerToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad nested custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+var commentTestToml = []byte(`
+# it's a comment on type
+[postgres]
+  # isCommented = "dvalue"
+  noComment = "cvalue"
+
+  # A comment on AttrB with a
+  # break line
+  password = "bvalue"
+
+  # A comment on AttrA
+  user = "avalue"
+
+  [[postgres.My]]
+
+    # a comment on my on typeC
+    My = "Foo"
+
+  [[postgres.My]]
+
+    # a comment on my on typeC
+    My = "Baar"
+`)
+
+func TestMarshalComment(t *testing.T) {
+	type TypeC struct {
+		My string `comment:"a comment on my on typeC"`
+	}
+	type TypeB struct {
+		AttrA string `toml:"user" comment:"A comment on AttrA"`
+		AttrB string `toml:"password" comment:"A comment on AttrB with a\n break line"`
+		AttrC string `toml:"noComment"`
+		AttrD string `toml:"isCommented" commented:"true"`
+		My    []TypeC
+	}
+	type TypeA struct {
+		TypeB TypeB `toml:"postgres" comment:"it's a comment on type"`
+	}
+
+	ta := []TypeC{{My: "Foo"}, {My: "Baar"}}
+	config := TypeA{TypeB{AttrA: "avalue", AttrB: "bvalue", AttrC: "cvalue", AttrD: "dvalue", My: ta}}
+	result, err := Marshal(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := commentTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+type mapsTestStruct struct {
+	Simple map[string]string
+	Paths  map[string]string
+	Other  map[string]float64
+	X      struct {
+		Y struct {
+			Z map[string]bool
+		}
+	}
+}
+
+var mapsTestData = mapsTestStruct{
+	Simple: map[string]string{
+		"one plus one": "two",
+		"next":         "three",
+	},
+	Paths: map[string]string{
+		"/this/is/a/path": "/this/is/also/a/path",
+		"/heloo.txt":      "/tmp/lololo.txt",
+	},
+	Other: map[string]float64{
+		"testing": 3.9999,
+	},
+	X: struct{ Y struct{ Z map[string]bool } }{
+		Y: struct{ Z map[string]bool }{
+			Z: map[string]bool{
+				"is.Nested": true,
+			},
+		},
+	},
+}
+var mapsTestToml = []byte(`
+[Other]
+  "testing" = 3.9999
+
+[Paths]
+  "/heloo.txt" = "/tmp/lololo.txt"
+  "/this/is/a/path" = "/this/is/also/a/path"
+
+[Simple]
+  "next" = "three"
+  "one plus one" = "two"
+
+[X]
+
+  [X.Y]
+
+    [X.Y.Z]
+      "is.Nested" = true
+`)
+
+func TestEncodeQuotedMapKeys(t *testing.T) {
+	var buf bytes.Buffer
+	if err := NewEncoder(&buf).QuoteMapKeys(true).Encode(mapsTestData); err != nil {
+		t.Fatal(err)
+	}
+	result := buf.Bytes()
+	expected := mapsTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad maps marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestDecodeQuotedMapKeys(t *testing.T) {
+	result := mapsTestStruct{}
+	err := NewDecoder(bytes.NewBuffer(mapsTestToml)).Decode(&result)
+	expected := mapsTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad maps unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type structArrayNoTag struct {
+	A struct {
+		B []int64
+		C []int64
+	}
+}
+
+func TestMarshalArray(t *testing.T) {
+	expected := []byte(`
+[A]
+  B = [1,2,3]
+  C = [1]
+`)
+
+	m := structArrayNoTag{
+		A: struct {
+			B []int64
+			C []int64
+		}{
+			B: []int64{1, 2, 3},
+			C: []int64{1},
+		},
+	}
+
+	b, err := Marshal(m)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(b, expected) {
+		t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b)
+	}
+}
+
+func TestMarshalArrayOnePerLine(t *testing.T) {
+	expected := []byte(`
+[A]
+  B = [
+    1,
+    2,
+    3,
+  ]
+  C = [1]
+`)
+
+	m := structArrayNoTag{
+		A: struct {
+			B []int64
+			C []int64
+		}{
+			B: []int64{1, 2, 3},
+			C: []int64{1},
+		},
+	}
+
+	var buf bytes.Buffer
+	encoder := NewEncoder(&buf).ArraysWithOneElementPerLine(true)
+	err := encoder.Encode(m)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+
+	if !bytes.Equal(b, expected) {
+		t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b)
+	}
+}

marshal_test.toml

@@ -0,0 +1,38 @@
+title = "TOML Marshal Testing"
+
+[basic]
+  bool = true
+  date = 1979-05-27T07:32:00Z
+  float = 123.4
+  int = 5000
+  string = "Bite me"
+  uint = 5001
+
+[basic_lists]
+  bools = [true,false,true]
+  dates = [1979-05-27T07:32:00Z,1980-05-27T07:32:00Z]
+  floats = [12.3,45.6,78.9]
+  ints = [8001,8001,8002]
+  strings = ["One","Two","Three"]
+  uints = [5002,5003]
+
+[basic_map]
+  one = "one"
+  two = "two"
+
+[subdoc]
+
+  [subdoc.first]
+    name = "First"
+
+  [subdoc.second]
+    name = "Second"
+
+[[subdoclist]]
+  name = "List.First"
+
+[[subdoclist]]
+  name = "List.Second"
+
+[[subdocptrs]]
+  name = "Second"

parser.go

@@ -3,19 +3,22 @@
 package toml
 
 import (
+	"errors"
 	"fmt"
+	"math"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 )
 
 type tomlParser struct {
-	flow          chan token
-	tree          *TomlTree
-	tokensBuffer  []token
-	currentGroup  []string
-	seenGroupKeys []string
+	flowIdx       int
+	flow          []token
+	tree          *Tree
+	currentTable  []string
+	seenTableKeys []string
 }
 
 type tomlParserStateFn func() tomlParserStateFn
@@ -32,16 +35,10 @@ func (p *tomlParser) run() {
 }
 
 func (p *tomlParser) peek() *token {
-	if len(p.tokensBuffer) != 0 {
-		return &(p.tokensBuffer[0])
-	}
-
-	tok, ok := <-p.flow
-	if !ok {
+	if p.flowIdx >= len(p.flow) {
 		return nil
 	}
-	p.tokensBuffer = append(p.tokensBuffer, tok)
-	return &tok
+	return &p.flow[p.flowIdx]
 }
 
 func (p *tomlParser) assume(typ tokenType) {
@@ -55,16 +52,12 @@ func (p *tomlParser) assume(typ tokenType) {
 }
 
 func (p *tomlParser) getToken() *token {
-	if len(p.tokensBuffer) != 0 {
-		tok := p.tokensBuffer[0]
-		p.tokensBuffer = p.tokensBuffer[1:]
-		return &tok
-	}
-	tok, ok := <-p.flow
-	if !ok {
+	tok := p.peek()
+	if tok == nil {
 		return nil
 	}
-	return &tok
+	p.flowIdx++
+	return tok
 }
 
 func (p *tomlParser) parseStart() tomlParserStateFn {
@@ -94,48 +87,48 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn {
 	startToken := p.getToken() // discard the [[
 	key := p.getToken()
 	if key.typ != tokenKeyGroupArray {
-		p.raiseError(key, "unexpected token %s, was expecting a key group array", key)
+		p.raiseError(key, "unexpected token %s, was expecting a table array key", key)
 	}
 
-	// get or create group array element at the indicated part in the path
+	// get or create table array element at the indicated part in the path
 	keys, err := parseKey(key.val)
 	if err != nil {
-		p.raiseError(key, "invalid group array key: %s", err)
+		p.raiseError(key, "invalid table array key: %s", err)
 	}
 	p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
 	destTree := p.tree.GetPath(keys)
-	var array []*TomlTree
+	var array []*Tree
 	if destTree == nil {
-		array = make([]*TomlTree, 0)
-	} else if destTree.([]*TomlTree) != nil {
-		array = destTree.([]*TomlTree)
+		array = make([]*Tree, 0)
+	} else if target, ok := destTree.([]*Tree); ok && target != nil {
+		array = destTree.([]*Tree)
 	} else {
-		p.raiseError(key, "key %s is already assigned and not of type group array", key)
+		p.raiseError(key, "key %s is already assigned and not of type table array", key)
 	}
-	p.currentGroup = keys
+	p.currentTable = keys
 
-	// add a new tree to the end of the group array
-	newTree := newTomlTree()
+	// add a new tree to the end of the table array
+	newTree := newTree()
 	newTree.position = startToken.Position
 	array = append(array, newTree)
-	p.tree.SetPath(p.currentGroup, array)
+	p.tree.SetPath(p.currentTable, array)
 
-	// remove all keys that were children of this group array
+	// remove all keys that were children of this table array
 	prefix := key.val + "."
 	found := false
-	for ii := 0; ii < len(p.seenGroupKeys); {
-		groupKey := p.seenGroupKeys[ii]
-		if strings.HasPrefix(groupKey, prefix) {
-			p.seenGroupKeys = append(p.seenGroupKeys[:ii], p.seenGroupKeys[ii+1:]...)
+	for ii := 0; ii < len(p.seenTableKeys); {
+		tableKey := p.seenTableKeys[ii]
+		if strings.HasPrefix(tableKey, prefix) {
+			p.seenTableKeys = append(p.seenTableKeys[:ii], p.seenTableKeys[ii+1:]...)
 		} else {
-			found = (groupKey == key.val)
+			found = (tableKey == key.val)
 			ii++
 		}
 	}
 
 	// keep this key name from use by other kinds of assignments
 	if !found {
-		p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+		p.seenTableKeys = append(p.seenTableKeys, key.val)
 	}
 
 	// move to next parser state
@@ -147,24 +140,24 @@ func (p *tomlParser) parseGroup() tomlParserStateFn {
 	startToken := p.getToken() // discard the [
 	key := p.getToken()
 	if key.typ != tokenKeyGroup {
-		p.raiseError(key, "unexpected token %s, was expecting a key group", key)
+		p.raiseError(key, "unexpected token %s, was expecting a table key", key)
 	}
-	for _, item := range p.seenGroupKeys {
+	for _, item := range p.seenTableKeys {
 		if item == key.val {
 			p.raiseError(key, "duplicated tables")
 		}
 	}
 
-	p.seenGroupKeys = append(p.seenGroupKeys, key.val)
+	p.seenTableKeys = append(p.seenTableKeys, key.val)
 	keys, err := parseKey(key.val)
 	if err != nil {
-		p.raiseError(key, "invalid group array key: %s", err)
+		p.raiseError(key, "invalid table array key: %s", err)
 	}
 	if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
 		p.raiseError(key, "%s", err)
 	}
 	p.assume(tokenRightBracket)
-	p.currentGroup = keys
+	p.currentTable = keys
 	return p.parseStart
 }
 
@@ -173,44 +166,71 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	p.assume(tokenEqual)
 
 	value := p.parseRvalue()
-	var groupKey []string
-	if len(p.currentGroup) > 0 {
-		groupKey = p.currentGroup
+	var tableKey []string
+	if len(p.currentTable) > 0 {
+		tableKey = p.currentTable
 	} else {
-		groupKey = []string{}
+		tableKey = []string{}
 	}
 
-	// find the group to assign, looking out for arrays of groups
-	var targetNode *TomlTree
-	switch node := p.tree.GetPath(groupKey).(type) {
-	case []*TomlTree:
+	// find the table to assign, looking out for arrays of tables
+	var targetNode *Tree
+	switch node := p.tree.GetPath(tableKey).(type) {
+	case []*Tree:
 		targetNode = node[len(node)-1]
-	case *TomlTree:
+	case *Tree:
 		targetNode = node
 	default:
-		p.raiseError(key, "Unknown group type for path: %s",
-			strings.Join(groupKey, "."))
+		p.raiseError(key, "Unknown table type for path: %s",
+			strings.Join(tableKey, "."))
 	}
 
-	// assign value to the found group
-	keyVals, err := parseKey(key.val)
-	if err != nil {
-		p.raiseError(key, "%s", err)
-	}
+	// assign value to the found table
+	keyVals := []string{key.val}
 	if len(keyVals) != 1 {
 		p.raiseError(key, "Invalid key")
 	}
 	keyVal := keyVals[0]
 	localKey := []string{keyVal}
-	finalKey := append(groupKey, keyVal)
+	finalKey := append(tableKey, keyVal)
 	if targetNode.GetPath(localKey) != nil {
 		p.raiseError(key, "The following key was defined twice: %s",
 			strings.Join(finalKey, "."))
 	}
-	targetNode.values[keyVal] = &tomlValue{value, key.Position}
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *Tree, []*Tree:
+		toInsert = value
+	default:
+		toInsert = &tomlValue{value: value, position: key.Position}
+	}
+	targetNode.values[keyVal] = toInsert
 	return p.parseStart
 }
 
+var numberUnderscoreInvalidRegexp *regexp.Regexp
+var hexNumberUnderscoreInvalidRegexp *regexp.Regexp
+
+func numberContainsInvalidUnderscore(value string) error {
+	if numberUnderscoreInvalidRegexp.MatchString(value) {
+		return errors.New("invalid use of _ in number")
+	}
+	return nil
+}
+
+func hexNumberContainsInvalidUnderscore(value string) error {
+	if hexNumberUnderscoreInvalidRegexp.MatchString(value) {
+		return errors.New("invalid use of _ in hex number")
+	}
+	return nil
+}
+
+func cleanupNumberToken(value string) string {
+	cleanedVal := strings.Replace(value, "_", "", -1)
+	return cleanedVal
+}
+
 func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
@@ -224,15 +244,57 @@ func (p *tomlParser) parseRvalue() interface{} {
 		return true
 	case tokenFalse:
 		return false
+	case tokenInf:
+		if tok.val[0] == '-' {
+			return math.Inf(-1)
+		}
+		return math.Inf(1)
+	case tokenNan:
+		return math.NaN()
 	case tokenInteger:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
-		val, err := strconv.ParseInt(cleanedVal, 10, 64)
+		cleanedVal := cleanupNumberToken(tok.val)
+		var err error
+		var val int64
+		if len(cleanedVal) >= 3 && cleanedVal[0] == '0' {
+			switch cleanedVal[1] {
+			case 'x':
+				err = hexNumberContainsInvalidUnderscore(tok.val)
+				if err != nil {
+					p.raiseError(tok, "%s", err)
+				}
+				val, err = strconv.ParseInt(cleanedVal[2:], 16, 64)
+			case 'o':
+				err = numberContainsInvalidUnderscore(tok.val)
+				if err != nil {
+					p.raiseError(tok, "%s", err)
+				}
+				val, err = strconv.ParseInt(cleanedVal[2:], 8, 64)
+			case 'b':
+				err = numberContainsInvalidUnderscore(tok.val)
+				if err != nil {
+					p.raiseError(tok, "%s", err)
+				}
+				val, err = strconv.ParseInt(cleanedVal[2:], 2, 64)
+			default:
+				panic("invalid base") // the lexer should catch this first
+			}
+		} else {
+			err = numberContainsInvalidUnderscore(tok.val)
+			if err != nil {
+				p.raiseError(tok, "%s", err)
+			}
+			val, err = strconv.ParseInt(cleanedVal, 10, 64)
+		}
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
+		err := numberContainsInvalidUnderscore(tok.val)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
+		}
+		cleanedVal := cleanupNumberToken(tok.val)
 		val, err := strconv.ParseFloat(cleanedVal, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
@@ -263,8 +325,8 @@ func tokenIsComma(t *token) bool {
 	return t != nil && t.typ == tokenComma
 }
 
-func (p *tomlParser) parseInlineTable() *TomlTree {
-	tree := newTomlTree()
+func (p *tomlParser) parseInlineTable() *Tree {
+	tree := newTree()
 	var previous *token
 Loop:
 	for {
@@ -293,7 +355,7 @@ Loop:
 			}
 			p.getToken()
 		default:
-			p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String())
+			p.raiseError(follow, "unexpected token type in inline table: %s", follow.String())
 		}
 		previous = follow
 	}
@@ -334,30 +396,35 @@ func (p *tomlParser) parseArray() interface{} {
 			p.getToken()
 		}
 	}
-	// An array of TomlTrees is actually an array of inline
+	// An array of Trees is actually an array of inline
 	// tables, which is a shorthand for a table array. If the
-	// array was not converted from []interface{} to []*TomlTree,
+	// array was not converted from []interface{} to []*Tree,
 	// the two notations would not be equivalent.
-	if arrayType == reflect.TypeOf(newTomlTree()) {
-		tomlArray := make([]*TomlTree, len(array))
+	if arrayType == reflect.TypeOf(newTree()) {
+		tomlArray := make([]*Tree, len(array))
 		for i, v := range array {
-			tomlArray[i] = v.(*TomlTree)
+			tomlArray[i] = v.(*Tree)
 		}
 		return tomlArray
 	}
 	return array
 }
 
-func parseToml(flow chan token) *TomlTree {
-	result := newTomlTree()
+func parseToml(flow []token) *Tree {
+	result := newTree()
 	result.position = Position{1, 1}
 	parser := &tomlParser{
+		flowIdx:       0,
 		flow:          flow,
 		tree:          result,
-		tokensBuffer:  make([]token, 0),
-		currentGroup:  make([]string, 0),
-		seenGroupKeys: make([]string, 0),
+		currentTable:  make([]string, 0),
+		seenTableKeys: make([]string, 0),
 	}
 	parser.run()
 	return result
 }
+
+func init() {
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d])|_$|^_`)
+	hexNumberUnderscoreInvalidRegexp = regexp.MustCompile(`(^0x_)|([^\da-f]_|_[^\da-f])|_$|^_`)
+}

parser_test.go

@@ -2,26 +2,35 @@ package toml
 
 import (
 	"fmt"
+	"math"
+	"reflect"
 	"testing"
 	"time"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
-func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) {
+func assertSubTree(t *testing.T, path []string, tree *Tree, err error, ref map[string]interface{}) {
 	if err != nil {
 		t.Error("Non-nil error:", err.Error())
 		return
 	}
 	for k, v := range ref {
+		nextPath := append(path, k)
+		t.Log("asserting path", nextPath)
 		// NOTE: directly access key instead of resolve by path
 		// NOTE: see TestSpecialKV
 		switch node := tree.GetPath([]string{k}).(type) {
-		case []*TomlTree:
+		case []*Tree:
+			t.Log("\tcomparing key", nextPath, "by array iteration")
 			for idx, item := range node {
-				assertTree(t, item, err, v.([]map[string]interface{})[idx])
+				assertSubTree(t, nextPath, item, err, v.([]map[string]interface{})[idx])
 			}
-		case *TomlTree:
-			assertTree(t, node, err, v.(map[string]interface{}))
+		case *Tree:
+			t.Log("\tcomparing key", nextPath, "by subtree assestion")
+			assertSubTree(t, nextPath, node, err, v.(map[string]interface{}))
 		default:
+			t.Log("\tcomparing key", nextPath, "by string representation because it's of type", reflect.TypeOf(node))
 			if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
 				t.Errorf("was expecting %v at %v but got %v", v, k, node)
 			}
@@ -29,8 +38,14 @@ func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interfac
 	}
 }
 
+func assertTree(t *testing.T, tree *Tree, err error, ref map[string]interface{}) {
+	t.Log("Asserting tree:\n", spew.Sdump(tree))
+	assertSubTree(t, []string{}, tree, err, ref)
+	t.Log("Finished tree assertion.")
+}
+
 func TestCreateSubTree(t *testing.T) {
-	tree := newTomlTree()
+	tree := newTree()
 	tree.createSubTree([]string{"a", "b", "c"}, Position{})
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
@@ -58,6 +73,17 @@ func TestNumberInKey(t *testing.T) {
 	})
 }
 
+func TestIncorrectKeyExtraSquareBracket(t *testing.T) {
+	_, err := Load(`[a]b]
+zyx = 42`)
+	if err == nil {
+		t.Error("Error should have been returned.")
+	}
+	if err.Error() != "(1, 4): unexpected token" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestSimpleNumbers(t *testing.T) {
 	tree, err := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -68,6 +94,78 @@ func TestSimpleNumbers(t *testing.T) {
 	})
 }
 
+func TestSpecialFloats(t *testing.T) {
+	tree, err := Load(`
+normalinf = inf
+plusinf = +inf
+minusinf = -inf
+normalnan = nan
+plusnan = +nan
+minusnan = -nan
+`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"normalinf": math.Inf(1),
+		"plusinf":   math.Inf(1),
+		"minusinf":  math.Inf(-1),
+		"normalnan": math.NaN(),
+		"plusnan":   math.NaN(),
+		"minusnan":  math.NaN(),
+	})
+}
+
+func TestHexIntegers(t *testing.T) {
+	tree, err := Load(`a = 0xDEADBEEF`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	tree, err = Load(`a = 0xdeadbeef`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	tree, err = Load(`a = 0xdead_beef`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	_, err = Load(`a = 0x_1`)
+	if err.Error() != "(1, 5): invalid use of _ in hex number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestOctIntegers(t *testing.T) {
+	tree, err := Load(`a = 0o01234567`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(342391)})
+
+	tree, err = Load(`a = 0o755`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(493)})
+
+	_, err = Load(`a = 0o_1`)
+	if err.Error() != "(1, 5): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestBinIntegers(t *testing.T) {
+	tree, err := Load(`a = 0b11010110`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(214)})
+
+	_, err = Load(`a = 0b_1`)
+	if err.Error() != "(1, 5): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestBadIntegerBase(t *testing.T) {
+	_, err := Load(`a = 0k1`)
+	if err.Error() != "(1, 5): unknown number base: k. possible options are x (hex) o (octal) b (binary)" {
+		t.Error("Error should have been returned.")
+	}
+}
+
+func TestIntegerNoDigit(t *testing.T) {
+	_, err := Load(`a = 0b`)
+	if err.Error() != "(1, 5): number needs at least one digit" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestNumbersWithUnderscores(t *testing.T) {
 	tree, err := Load("a = 1_000")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -141,6 +239,36 @@ func TestSpaceKey(t *testing.T) {
 	})
 }
 
+func TestDoubleQuotedKey(t *testing.T) {
+	tree, err := Load(`
+	"key"        = "a"
+	"\t"         = "b"
+	"\U0001F914" = "c"
+	"\u2764"     = "d"
+	`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"key":        "a",
+		"\t":         "b",
+		"\U0001F914": "c",
+		"\u2764":     "d",
+	})
+}
+
+func TestSingleQuotedKey(t *testing.T) {
+	tree, err := Load(`
+	'key'        = "a"
+	'\t'         = "b"
+	'\U0001F914' = "c"
+	'\u2764'     = "d"
+	`)
+	assertTree(t, tree, err, map[string]interface{}{
+		`key`:        "a",
+		`\t`:         "b",
+		`\U0001F914`: "c",
+		`\u2764`:     "d",
+	})
+}
+
 func TestStringEscapables(t *testing.T) {
 	tree, err := Load("a = \"a \\n b\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -163,6 +291,16 @@ func TestStringEscapables(t *testing.T) {
 	})
 }
 
+func TestEmptyQuotedString(t *testing.T) {
+	tree, err := Load(`[""]
+"" = 1`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"": map[string]interface{}{
+			"": int64(1),
+		},
+	})
+}
+
 func TestBools(t *testing.T) {
 	tree, err := Load("a = true\nb = false")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -255,14 +393,25 @@ func TestArrayMultiline(t *testing.T) {
 func TestArrayNested(t *testing.T) {
 	tree, err := Load("a = [[42, 21], [10]]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": [][]int64{[]int64{int64(42), int64(21)}, []int64{int64(10)}},
+		"a": [][]int64{{int64(42), int64(21)}, {int64(10)}},
+	})
+}
+
+func TestNestedArrayComment(t *testing.T) {
+	tree, err := Load(`
+someArray = [
+# does not work
+["entry1"]
+]`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"someArray": [][]string{{"entry1"}},
 	})
 }
 
 func TestNestedEmptyArrays(t *testing.T) {
 	tree, err := Load("a = [[[]]]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"a": [][][]interface{}{[][]interface{}{[]interface{}{}}},
+		"a": [][][]interface{}{{{}}},
 	})
 }
 
@@ -281,13 +430,25 @@ func TestArrayMixedTypes(t *testing.T) {
 func TestArrayNestedStrings(t *testing.T) {
 	tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]")
 	assertTree(t, tree, err, map[string]interface{}{
-		"data": [][]string{[]string{"gamma", "delta"}, []string{"Foo"}},
+		"data": [][]string{{"gamma", "delta"}, {"Foo"}},
 	})
 }
 
+func TestParseUnknownRvalue(t *testing.T) {
+	_, err := Load("a = !bssss")
+	if err == nil {
+		t.Error("Expecting a parse error")
+	}
+
+	_, err = Load("a = /b")
+	if err == nil {
+		t.Error("Expecting a parse error")
+	}
+}
+
 func TestMissingValue(t *testing.T) {
 	_, err := Load("a = ")
-	if err.Error() != "(1, 4): expecting a value" {
+	if err.Error() != "(1, 5): expecting a value" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -368,7 +529,7 @@ func TestExampleInlineGroupInArray(t *testing.T) {
 	tree, err := Load(`points = [{ x = 1, y = 2 }]`)
 	assertTree(t, tree, err, map[string]interface{}{
 		"points": []map[string]interface{}{
-			map[string]interface{}{
+			{
 				"x": int64(1),
 				"y": int64(2),
 			},
@@ -420,7 +581,7 @@ func TestDuplicateKeys(t *testing.T) {
 
 func TestEmptyIntermediateTable(t *testing.T) {
 	_, err := Load("[foo..bar]")
-	if err.Error() != "(1, 2): empty intermediate table" {
+	if err.Error() != "(1, 2): invalid table array key: empty table key" {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -441,7 +602,7 @@ func TestImplicitDeclarationBefore(t *testing.T) {
 
 func TestFloatsWithoutLeadingZeros(t *testing.T) {
 	_, err := Load("a = .42")
-	if err.Error() != "(1, 4): cannot start float with a dot" {
+	if err.Error() != "(1, 5): cannot start float with a dot" {
 		t.Error("Bad error message:", err.Error())
 	}
 
@@ -453,7 +614,8 @@ func TestFloatsWithoutLeadingZeros(t *testing.T) {
 
 func TestMissingFile(t *testing.T) {
 	_, err := LoadFile("foo.toml")
-	if err.Error() != "open foo.toml: no such file or directory" {
+	if err.Error() != "open foo.toml: no such file or directory" &&
+		err.Error() != "open foo.toml: The system cannot find the file specified." {
 		t.Error("Bad error message:", err.Error())
 	}
 }
@@ -494,6 +656,42 @@ func TestParseFile(t *testing.T) {
 	})
 }
 
+func TestParseFileCRLF(t *testing.T) {
+	tree, err := LoadFile("example-crlf.toml")
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
+	})
+}
+
 func TestParseKeyGroupArray(t *testing.T) {
 	tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -506,6 +704,40 @@ func TestParseKeyGroupArray(t *testing.T) {
 	})
 }
 
+func TestParseKeyGroupArrayUnfinished(t *testing.T) {
+	_, err := Load("[[foo.bar]\na = 42")
+	if err.Error() != "(1, 10): was expecting token [[, but got unclosed table array key instead" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("[[foo.[bar]\na = 42")
+	if err.Error() != "(1, 3): unexpected token table array key cannot contain ']', was expecting a table array key" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestParseKeyGroupArrayQueryExample(t *testing.T) {
+	tree, err := Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"book": []map[string]interface{}{
+			{"title": "The Stand", "author": "Stephen King"},
+			{"title": "For Whom the Bell Tolls", "author": "Ernest Hemmingway"},
+			{"title": "Neuromancer", "author": "William Gibson"},
+		},
+	})
+}
+
 func TestParseKeyGroupArraySpec(t *testing.T) {
 	tree, err := Load("[[fruit]]\n name=\"apple\"\n [fruit.physical]\n color=\"red\"\n shape=\"round\"\n [[fruit]]\n name=\"banana\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -516,39 +748,44 @@ func TestParseKeyGroupArraySpec(t *testing.T) {
 	})
 }
 
-func TestToTomlValue(t *testing.T) {
+func TestTomlValueStringRepresentation(t *testing.T) {
 	for idx, item := range []struct {
 		Value  interface{}
 		Expect string
 	}{
 		{int64(12345), "12345"},
+		{uint64(50), "50"},
 		{float64(123.45), "123.45"},
-		{bool(true), "true"},
+		{true, "true"},
 		{"hello world", "\"hello world\""},
 		{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
 		{"\x05", "\"\\u0005\""},
 		{time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
 			"1979-05-27T07:32:00Z"},
 		{[]interface{}{"gamma", "delta"},
-			"[\n  \"gamma\",\n  \"delta\",\n]"},
+			"[\"gamma\",\"delta\"]"},
+		{nil, ""},
 	} {
-		result := toTomlValue(item.Value, 0)
+		result, err := tomlValueStringRepresentation(item.Value, "", false)
+		if err != nil {
+			t.Errorf("Test %d - unexpected error: %s", idx, err)
+		}
 		if result != item.Expect {
 			t.Errorf("Test %d - got '%s', expected '%s'", idx, result, item.Expect)
 		}
 	}
 }
 
-func TestToString(t *testing.T) {
-	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
+func TestToStringMapStringString(t *testing.T) {
+	tree, err := TreeFromMap(map[string]interface{}{"m": map[string]interface{}{"v": "abc"}})
 	if err != nil {
-		t.Errorf("Test failed to parse: %v", err)
-		return
+		t.Fatalf("unexpected error: %s", err)
 	}
-	result := tree.ToString()
-	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
-	if result != expected {
-		t.Errorf("Expected got '%s', expected '%s'", result, expected)
+	want := "\n[m]\n  v = \"abc\"\n"
+	got := tree.String()
+
+	if got != want {
+		t.Errorf("want:\n%q\ngot:\n%q", want, got)
 	}
 }
 
@@ -572,10 +809,10 @@ func TestDocumentPositions(t *testing.T) {
 	assertPosition(t,
 		"[foo]\nbar=42\nbaz=69",
 		map[string]Position{
-			"":        Position{1, 1},
-			"foo":     Position{1, 1},
-			"foo.bar": Position{2, 1},
-			"foo.baz": Position{3, 1},
+			"":        {1, 1},
+			"foo":     {1, 1},
+			"foo.bar": {2, 1},
+			"foo.baz": {3, 1},
 		})
 }
 
@@ -583,10 +820,10 @@ func TestDocumentPositionsWithSpaces(t *testing.T) {
 	assertPosition(t,
 		"  [foo]\n  bar=42\n  baz=69",
 		map[string]Position{
-			"":        Position{1, 1},
-			"foo":     Position{1, 3},
-			"foo.bar": Position{2, 3},
-			"foo.baz": Position{3, 3},
+			"":        {1, 1},
+			"foo":     {1, 3},
+			"foo.bar": {2, 3},
+			"foo.baz": {3, 3},
 		})
 }
 
@@ -594,10 +831,10 @@ func TestDocumentPositionsWithGroupArray(t *testing.T) {
 	assertPosition(t,
 		"[[foo]]\nbar=42\nbaz=69",
 		map[string]Position{
-			"":        Position{1, 1},
-			"foo":     Position{1, 1},
-			"foo.bar": Position{2, 1},
-			"foo.baz": Position{3, 1},
+			"":        {1, 1},
+			"foo":     {1, 1},
+			"foo.bar": {2, 1},
+			"foo.baz": {3, 1},
 		})
 }
 
@@ -605,19 +842,24 @@ func TestNestedTreePosition(t *testing.T) {
 	assertPosition(t,
 		"[foo.bar]\na=42\nb=69",
 		map[string]Position{
-			"":          Position{1, 1},
-			"foo":       Position{1, 1},
-			"foo.bar":   Position{1, 1},
-			"foo.bar.a": Position{2, 1},
-			"foo.bar.b": Position{3, 1},
+			"":          {1, 1},
+			"foo":       {1, 1},
+			"foo.bar":   {1, 1},
+			"foo.bar.a": {2, 1},
+			"foo.bar.b": {3, 1},
 		})
 }
 
 func TestInvalidGroupArray(t *testing.T) {
-	_, err := Load("[key#group]\nanswer = 42")
+	_, err := Load("[table#key]\nanswer = 42")
 	if err == nil {
 		t.Error("Should error")
 	}
+
+	_, err = Load("[foo.[bar]\na = 42")
+	if err.Error() != "(1, 2): unexpected token table key cannot contain ']', was expecting a table key" {
+		t.Error("Bad error message:", err.Error())
+	}
 }
 
 func TestDoubleEqual(t *testing.T) {
@@ -626,3 +868,32 @@ func TestDoubleEqual(t *testing.T) {
 		t.Error("Bad error message:", err.Error())
 	}
 }
+
+func TestGroupArrayReassign(t *testing.T) {
+	_, err := Load("[hello]\n[[hello]]")
+	if err.Error() != "(2, 3): key \"hello\" is already assigned and not of type table array" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestInvalidFloatParsing(t *testing.T) {
+	_, err := Load("a=1e_2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1e2_")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1__2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=_1_2")
+	if err.Error() != "(1, 3): cannot start number with underscore" {
+		t.Error("Bad error message:", err.Error())
+	}
+}

position.go

@@ -6,13 +6,11 @@ import (
 	"fmt"
 )
 
-/*
-  Position of a document element within a TOML document.
-
-  Line and Col are both 1-indexed positions for the element's line number and
-  column number, respectively.  Values of zero or less will cause Invalid(),
-  to return true.
-*/
+// Position of a document element within a TOML document.
+//
+// Line and Col are both 1-indexed positions for the element's line number and
+// column number, respectively.  Values of zero or less will cause Invalid(),
+// to return true.
 type Position struct {
 	Line int // line within the document
 	Col  int // column within the line
@@ -20,12 +18,12 @@ type Position struct {
 
 // String representation of the position.
 // Displays 1-indexed line and column numbers.
-func (p *Position) String() string {
+func (p Position) String() string {
 	return fmt.Sprintf("(%d, %d)", p.Line, p.Col)
 }
 
-// Returns whether or not the position is valid (i.e. with negative or
+// Invalid returns whether or not the position is valid (i.e. with negative or
 // null values)
-func (p *Position) Invalid() bool {
+func (p Position) Invalid() bool {
 	return p.Line <= 0 || p.Col <= 0
 }

position_test.go

@@ -18,9 +18,9 @@ func TestPositionString(t *testing.T) {
 
 func TestInvalid(t *testing.T) {
 	for i, v := range []Position{
-		Position{0, 1234},
-		Position{1234, 0},
-		Position{0, 0},
+		{0, 1234},
+		{1234, 0},
+		{0, 0},
 	} {
 		if !v.Invalid() {
 			t.Errorf("Position at %v is valid: %v", i, v)

query.go

@@ -1,142 +0,0 @@
-package toml
-
-import (
-	"time"
-)
-
-//  Type of a user-defined filter function, for use with Query.SetFilter().
-//
-//  The return value of the function must indicate if 'node' is to be included
-//  at this stage of the TOML path.  Returning true will include the node, and
-//  returning false will exclude it.
-//
-//  NOTE: Care should be taken to write script callbacks such that they are safe
-//  to use from multiple goroutines.
-type NodeFilterFn func(node interface{}) bool
-
-// The result of Executing a Query
-type QueryResult struct {
-	items     []interface{}
-	positions []Position
-}
-
-// appends a value/position pair to the result set
-func (r *QueryResult) appendResult(node interface{}, pos Position) {
-	r.items = append(r.items, node)
-	r.positions = append(r.positions, pos)
-}
-
-// Set of values within a QueryResult.  The order of values is not guaranteed
-// to be in document order, and may be different each time a query is executed.
-func (r *QueryResult) Values() []interface{} {
-	return r.items
-}
-
-// Set of positions for values within a QueryResult.  Each index in Positions()
-// corresponds to the entry in Value() of the same index.
-func (r *QueryResult) Positions() []Position {
-	return r.positions
-}
-
-// runtime context for executing query paths
-type queryContext struct {
-	result       *QueryResult
-	filters      *map[string]NodeFilterFn
-	lastPosition Position
-}
-
-// generic path functor interface
-type pathFn interface {
-	setNext(next pathFn)
-	call(node interface{}, ctx *queryContext)
-}
-
-// A Query is the representation of a compiled TOML path.  A Query is safe
-// for concurrent use by multiple goroutines.
-type Query struct {
-	root    pathFn
-	tail    pathFn
-	filters *map[string]NodeFilterFn
-}
-
-func newQuery() *Query {
-	return &Query{
-		root:    nil,
-		tail:    nil,
-		filters: &defaultFilterFunctions,
-	}
-}
-
-func (q *Query) appendPath(next pathFn) {
-	if q.root == nil {
-		q.root = next
-	} else {
-		q.tail.setNext(next)
-	}
-	q.tail = next
-	next.setNext(newTerminatingFn()) // init the next functor
-}
-
-// Compiles a TOML path expression.  The returned Query can be used to match
-// elements within a TomlTree and its descendants.
-func CompileQuery(path string) (*Query, error) {
-	return parseQuery(lexQuery(path))
-}
-
-// Executes a query against a TomlTree, and returns the result of the query.
-func (q *Query) Execute(tree *TomlTree) *QueryResult {
-	result := &QueryResult{
-		items:     []interface{}{},
-		positions: []Position{},
-	}
-	if q.root == nil {
-		result.appendResult(tree, tree.GetPosition(""))
-	} else {
-		ctx := &queryContext{
-			result:  result,
-			filters: q.filters,
-		}
-		q.root.call(tree, ctx)
-	}
-	return result
-}
-
-// Sets a user-defined filter function.  These may be used inside "?(..)" query
-// expressions to filter TOML document elements within a query.
-func (q *Query) SetFilter(name string, fn NodeFilterFn) {
-	if q.filters == &defaultFilterFunctions {
-		// clone the static table
-		q.filters = &map[string]NodeFilterFn{}
-		for k, v := range defaultFilterFunctions {
-			(*q.filters)[k] = v
-		}
-	}
-	(*q.filters)[name] = fn
-}
-
-var defaultFilterFunctions = map[string]NodeFilterFn{
-	"tree": func(node interface{}) bool {
-		_, ok := node.(*TomlTree)
-		return ok
-	},
-	"int": func(node interface{}) bool {
-		_, ok := node.(int64)
-		return ok
-	},
-	"float": func(node interface{}) bool {
-		_, ok := node.(float64)
-		return ok
-	},
-	"string": func(node interface{}) bool {
-		_, ok := node.(string)
-		return ok
-	},
-	"time": func(node interface{}) bool {
-		_, ok := node.(time.Time)
-		return ok
-	},
-	"bool": func(node interface{}) bool {
-		_, ok := node.(bool)
-		return ok
-	},
-}

query/doc.go

@@ -0,0 +1,175 @@
+// Package query performs JSONPath-like queries on a TOML document.
+//
+// The query path implementation is based loosely on the JSONPath specification:
+// http://goessner.net/articles/JsonPath/.
+//
+// The idea behind a query path is to allow quick access to any element, or set
+// of elements within TOML document, with a single expression.
+//
+//   result, err := query.CompileAndExecute("$.foo.bar.baz", tree)
+//
+// This is roughly equivalent to:
+//
+//   next := tree.Get("foo")
+//   if next != nil {
+//     next = next.Get("bar")
+//     if next != nil {
+//       next = next.Get("baz")
+//     }
+//   }
+//   result := next
+//
+// err is nil if any parsing exception occurs.
+//
+// If no node in the tree matches the query, result will simply contain an empty list of
+// items.
+//
+// As illustrated above, the query path is much more efficient, especially since
+// the structure of the TOML file can vary.  Rather than making assumptions about
+// a document's structure, a query allows the programmer to make structured
+// requests into the document, and get zero or more values as a result.
+//
+// Query syntax
+//
+// The syntax of a query begins with a root token, followed by any number
+// sub-expressions:
+//
+//   $
+//                    Root of the TOML tree.  This must always come first.
+//   .name
+//                    Selects child of this node, where 'name' is a TOML key
+//                    name.
+//   ['name']
+//                    Selects child of this node, where 'name' is a string
+//                    containing a TOML key name.
+//   [index]
+//                    Selcts child array element at 'index'.
+//   ..expr
+//                    Recursively selects all children, filtered by an a union,
+//                    index, or slice expression.
+//   ..*
+//                    Recursive selection of all nodes at this point in the
+//                    tree.
+//   .*
+//                    Selects all children of the current node.
+//   [expr,expr]
+//                    Union operator - a logical 'or' grouping of two or more
+//                    sub-expressions: index, key name, or filter.
+//   [start:end:step]
+//                    Slice operator - selects array elements from start to
+//                    end-1, at the given step.  All three arguments are
+//                    optional.
+//   [?(filter)]
+//                    Named filter expression - the function 'filter' is
+//                    used to filter children at this node.
+//
+// Query Indexes And Slices
+//
+// Index expressions perform no bounds checking, and will contribute no
+// values to the result set if the provided index or index range is invalid.
+// Negative indexes represent values from the end of the array, counting backwards.
+//
+//   // select the last index of the array named 'foo'
+//   query.CompileAndExecute("$.foo[-1]", tree)
+//
+// Slice expressions are supported, by using ':' to separate a start/end index pair.
+//
+//   // select up to the first five elements in the array
+//   query.CompileAndExecute("$.foo[0:5]", tree)
+//
+// Slice expressions also allow negative indexes for the start and stop
+// arguments.
+//
+//   // select all array elements.
+//   query.CompileAndExecute("$.foo[0:-1]", tree)
+//
+// Slice expressions may have an optional stride/step parameter:
+//
+//   // select every other element
+//   query.CompileAndExecute("$.foo[0:-1:2]", tree)
+//
+// Slice start and end parameters are also optional:
+//
+//   // these are all equivalent and select all the values in the array
+//   query.CompileAndExecute("$.foo[:]", tree)
+//   query.CompileAndExecute("$.foo[0:]", tree)
+//   query.CompileAndExecute("$.foo[:-1]", tree)
+//   query.CompileAndExecute("$.foo[0:-1:]", tree)
+//   query.CompileAndExecute("$.foo[::1]", tree)
+//   query.CompileAndExecute("$.foo[0::1]", tree)
+//   query.CompileAndExecute("$.foo[:-1:1]", tree)
+//   query.CompileAndExecute("$.foo[0:-1:1]", tree)
+//
+// Query Filters
+//
+// Query filters are used within a Union [,] or single Filter [] expression.
+// A filter only allows nodes that qualify through to the next expression,
+// and/or into the result set.
+//
+//   // returns children of foo that are permitted by the 'bar' filter.
+//   query.CompileAndExecute("$.foo[?(bar)]", tree)
+//
+// There are several filters provided with the library:
+//
+//   tree
+//          Allows nodes of type Tree.
+//   int
+//          Allows nodes of type int64.
+//   float
+//          Allows nodes of type float64.
+//   string
+//          Allows nodes of type string.
+//   time
+//          Allows nodes of type time.Time.
+//   bool
+//          Allows nodes of type bool.
+//
+// Query Results
+//
+// An executed query returns a Result object.  This contains the nodes
+// in the TOML tree that qualify the query expression.  Position information
+// is also available for each value in the set.
+//
+//   // display the results of a query
+//   results := query.CompileAndExecute("$.foo.bar.baz", tree)
+//   for idx, value := results.Values() {
+//       fmt.Println("%v: %v", results.Positions()[idx], value)
+//   }
+//
+// Compiled Queries
+//
+// Queries may be executed directly on a Tree object, or compiled ahead
+// of time and executed discretely.  The former is more convenient, but has the
+// penalty of having to recompile the query expression each time.
+//
+//   // basic query
+//   results := query.CompileAndExecute("$.foo.bar.baz", tree)
+//
+//   // compiled query
+//   query, err := toml.Compile("$.foo.bar.baz")
+//   results := query.Execute(tree)
+//
+//   // run the compiled query again on a different tree
+//   moreResults := query.Execute(anotherTree)
+//
+// User Defined Query Filters
+//
+// Filter expressions may also be user defined by using the SetFilter()
+// function on the Query object.  The function must return true/false, which
+// signifies if the passed node is kept or discarded, respectively.
+//
+//   // create a query that references a user-defined filter
+//   query, _ := query.Compile("$[?(bazOnly)]")
+//
+//   // define the filter, and assign it to the query
+//   query.SetFilter("bazOnly", func(node interface{}) bool{
+//       if tree, ok := node.(*Tree); ok {
+//           return tree.Has("baz")
+//       }
+//       return false  // reject all other node types
+//   })
+//
+//   // run the query
+//   query.Execute(tree)
+//
+package query

query/lexer.go

@@ -3,10 +3,11 @@
 // Written using the principles developed by Rob Pike in
 // http://www.youtube.com/watch?v=HxaD_trXwRE
 
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -54,7 +55,7 @@ func (l *queryLexer) nextStart() {
 
 func (l *queryLexer) emit(t tokenType) {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      l.input[l.start:l.pos],
 	}
@@ -63,7 +64,7 @@ func (l *queryLexer) emit(t tokenType) {
 
 func (l *queryLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      value,
 	}
@@ -91,7 +92,7 @@ func (l *queryLexer) backup() {
 
 func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
 	l.tokens <- token{
-		Position: Position{l.line, l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      tokenError,
 		val:      fmt.Sprintf(format, args...),
 	}
@@ -105,7 +106,7 @@ func (l *queryLexer) peek() rune {
 }
 
 func (l *queryLexer) accept(valid string) bool {
-	if strings.IndexRune(valid, l.next()) >= 0 {
+	if strings.ContainsRune(valid, l.next()) {
 		return true
 	}
 	l.backup()
@@ -272,6 +273,23 @@ func (l *queryLexer) lexString() queryLexStateFn {
 				return l.errorf("invalid unicode escape: \\u" + code)
 			}
 			growingString += string(rune(intcode))
+		} else if l.follow("\\U") {
+			l.pos += 2
+			code := ""
+			for i := 0; i < 8; i++ {
+				c := l.peek()
+				l.pos++
+				if !isHexDigit(c) {
+					return l.errorf("unfinished unicode escape")
+				}
+				code = code + string(c)
+			}
+			l.pos--
+			intcode, err := strconv.ParseInt(code, 16, 32)
+			if err != nil {
+				return l.errorf("invalid unicode escape: \\u" + code)
+			}
+			growingString += string(rune(intcode))
 		} else if l.follow("\\") {
 			l.pos++
 			return l.errorf("invalid escape sequence: \\" + string(l.peek()))

query/lexer_test.go

@@ -0,0 +1,179 @@
+package query
+
+import (
+	"github.com/pelletier/go-toml"
+	"testing"
+)
+
+func testQLFlow(t *testing.T, input string, expectedFlow []token) {
+	ch := lexQuery(input)
+	for idx, expected := range expectedFlow {
+		token := <-ch
+		if token != expected {
+			t.Log("While testing #", idx, ":", input)
+			t.Log("compared (got)", token, "to (expected)", expected)
+			t.Log("\tvalue:", token.val, "<->", expected.val)
+			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
+			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
+			t.Log("\tline:", token.Line, "<->", expected.Line)
+			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
+			t.Log("compared", token, "to", expected)
+			t.FailNow()
+		}
+	}
+
+	tok, ok := <-ch
+	if ok {
+		t.Log("channel is not closed!")
+		t.Log(len(ch)+1, "tokens remaining:")
+
+		t.Log("token ->", tok)
+		for token := range ch {
+			t.Log("token ->", token)
+		}
+		t.FailNow()
+	}
+}
+
+func TestLexSpecialChars(t *testing.T) {
+	testQLFlow(t, " .$[]..()?*", []token{
+		{toml.Position{1, 2}, tokenDot, "."},
+		{toml.Position{1, 3}, tokenDollar, "$"},
+		{toml.Position{1, 4}, tokenLeftBracket, "["},
+		{toml.Position{1, 5}, tokenRightBracket, "]"},
+		{toml.Position{1, 6}, tokenDotDot, ".."},
+		{toml.Position{1, 8}, tokenLeftParen, "("},
+		{toml.Position{1, 9}, tokenRightParen, ")"},
+		{toml.Position{1, 10}, tokenQuestion, "?"},
+		{toml.Position{1, 11}, tokenStar, "*"},
+		{toml.Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexString(t *testing.T) {
+	testQLFlow(t, "'foo\n'", []token{
+		{toml.Position{1, 2}, tokenString, "foo\n"},
+		{toml.Position{2, 2}, tokenEOF, ""},
+	})
+}
+
+func TestLexDoubleString(t *testing.T) {
+	testQLFlow(t, `"bar"`, []token{
+		{toml.Position{1, 2}, tokenString, "bar"},
+		{toml.Position{1, 6}, tokenEOF, ""},
+	})
+}
+
+func TestLexStringEscapes(t *testing.T) {
+	testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{
+		{toml.Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"},
+		{toml.Position{1, 55}, tokenEOF, ""},
+	})
+}
+
+func TestLexStringUnfinishedUnicode4(t *testing.T) {
+	testQLFlow(t, `"\u000"`, []token{
+		{toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
+	})
+}
+
+func TestLexStringUnfinishedUnicode8(t *testing.T) {
+	testQLFlow(t, `"\U0000"`, []token{
+		{toml.Position{1, 2}, tokenError, "unfinished unicode escape"},
+	})
+}
+
+func TestLexStringInvalidEscape(t *testing.T) {
+	testQLFlow(t, `"\x"`, []token{
+		{toml.Position{1, 2}, tokenError, "invalid escape sequence: \\x"},
+	})
+}
+
+func TestLexStringUnfinished(t *testing.T) {
+	testQLFlow(t, `"bar`, []token{
+		{toml.Position{1, 2}, tokenError, "unclosed string"},
+	})
+}
+
+func TestLexKey(t *testing.T) {
+	testQLFlow(t, "foo", []token{
+		{toml.Position{1, 1}, tokenKey, "foo"},
+		{toml.Position{1, 4}, tokenEOF, ""},
+	})
+}
+
+func TestLexRecurse(t *testing.T) {
+	testQLFlow(t, "$..*", []token{
+		{toml.Position{1, 1}, tokenDollar, "$"},
+		{toml.Position{1, 2}, tokenDotDot, ".."},
+		{toml.Position{1, 4}, tokenStar, "*"},
+		{toml.Position{1, 5}, tokenEOF, ""},
+	})
+}
+
+func TestLexBracketKey(t *testing.T) {
+	testQLFlow(t, "$[foo]", []token{
+		{toml.Position{1, 1}, tokenDollar, "$"},
+		{toml.Position{1, 2}, tokenLeftBracket, "["},
+		{toml.Position{1, 3}, tokenKey, "foo"},
+		{toml.Position{1, 6}, tokenRightBracket, "]"},
+		{toml.Position{1, 7}, tokenEOF, ""},
+	})
+}
+
+func TestLexSpace(t *testing.T) {
+	testQLFlow(t, "foo bar baz", []token{
+		{toml.Position{1, 1}, tokenKey, "foo"},
+		{toml.Position{1, 5}, tokenKey, "bar"},
+		{toml.Position{1, 9}, tokenKey, "baz"},
+		{toml.Position{1, 12}, tokenEOF, ""},
+	})
+}
+
+func TestLexInteger(t *testing.T) {
+	testQLFlow(t, "100 +200 -300", []token{
+		{toml.Position{1, 1}, tokenInteger, "100"},
+		{toml.Position{1, 5}, tokenInteger, "+200"},
+		{toml.Position{1, 10}, tokenInteger, "-300"},
+		{toml.Position{1, 14}, tokenEOF, ""},
+	})
+}
+
+func TestLexFloat(t *testing.T) {
+	testQLFlow(t, "100.0 +200.0 -300.0", []token{
+		{toml.Position{1, 1}, tokenFloat, "100.0"},
+		{toml.Position{1, 7}, tokenFloat, "+200.0"},
+		{toml.Position{1, 14}, tokenFloat, "-300.0"},
+		{toml.Position{1, 20}, tokenEOF, ""},
+	})
+}
+
+func TestLexFloatWithMultipleDots(t *testing.T) {
+	testQLFlow(t, "4.2.", []token{
+		{toml.Position{1, 1}, tokenError, "cannot have two dots in one float"},
+	})
+}
+
+func TestLexFloatLeadingDot(t *testing.T) {
+	testQLFlow(t, "+.1", []token{
+		{toml.Position{1, 1}, tokenError, "cannot start float with a dot"},
+	})
+}
+
+func TestLexFloatWithTrailingDot(t *testing.T) {
+	testQLFlow(t, "42.", []token{
+		{toml.Position{1, 1}, tokenError, "float cannot end with a dot"},
+	})
+}
+
+func TestLexNumberWithoutDigit(t *testing.T) {
+	testQLFlow(t, "+", []token{
+		{toml.Position{1, 1}, tokenError, "no digit in that number"},
+	})
+}
+
+func TestLexUnknown(t *testing.T) {
+	testQLFlow(t, "^", []token{
+		{toml.Position{1, 1}, tokenError, "unexpected char: '94'"},
+	})
+}

query/match.go

@@ -1,27 +1,10 @@
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 )
 
-// support function to set positions for tomlValues
-// NOTE: this is done to allow ctx.lastPosition to indicate the start of any
-// values returned by the query engines
-func tomlValueCheck(node interface{}, ctx *queryContext) interface{} {
-	switch castNode := node.(type) {
-	case *tomlValue:
-		ctx.lastPosition = castNode.position
-		return castNode.value
-	case []*TomlTree:
-		if len(castNode) > 0 {
-			ctx.lastPosition = castNode[0].position
-		}
-		return node
-	default:
-		return node
-	}
-}
-
 // base match
 type matchBase struct {
 	next pathFn
@@ -45,15 +28,7 @@ func (f *terminatingFn) setNext(next pathFn) {
 }
 
 func (f *terminatingFn) call(node interface{}, ctx *queryContext) {
-	switch castNode := node.(type) {
-	case *TomlTree:
-		ctx.result.appendResult(node, castNode.position)
-	case *tomlValue:
-		ctx.result.appendResult(node, castNode.position)
-	default:
-		// use last position for scalars
-		ctx.result.appendResult(node, ctx.lastPosition)
-	}
+	ctx.result.appendResult(node, ctx.lastPosition)
 }
 
 // match single key
@@ -67,9 +42,18 @@ func newMatchKeyFn(name string) *matchKeyFn {
 }
 
 func (f *matchKeyFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
-		item := tree.values[f.Name]
+	if array, ok := node.([]*toml.Tree); ok {
+		for _, tree := range array {
+			item := tree.Get(f.Name)
+			if item != nil {
+				ctx.lastPosition = tree.GetPosition(f.Name)
+				f.next.call(item, ctx)
+			}
+		}
+	} else if tree, ok := node.(*toml.Tree); ok {
+		item := tree.Get(f.Name)
 		if item != nil {
+			ctx.lastPosition = tree.GetPosition(f.Name)
 			f.next.call(item, ctx)
 		}
 	}
@@ -86,8 +70,13 @@ func newMatchIndexFn(idx int) *matchIndexFn {
 }
 
 func (f *matchIndexFn) call(node interface{}, ctx *queryContext) {
-	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+	if arr, ok := node.([]interface{}); ok {
 		if f.Idx < len(arr) && f.Idx >= 0 {
+			if treesArray, ok := node.([]*toml.Tree); ok {
+				if len(treesArray) > 0 {
+					ctx.lastPosition = treesArray[0].Position()
+				}
+			}
 			f.next.call(arr[f.Idx], ctx)
 		}
 	}
@@ -104,7 +93,7 @@ func newMatchSliceFn(start, end, step int) *matchSliceFn {
 }
 
 func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
-	if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok {
+	if arr, ok := node.([]interface{}); ok {
 		// adjust indexes for negative values, reverse ordering
 		realStart, realEnd := f.Start, f.End
 		if realStart < 0 {
@@ -118,6 +107,11 @@ func (f *matchSliceFn) call(node interface{}, ctx *queryContext) {
 		}
 		// loop and gather
 		for idx := realStart; idx < realEnd; idx += f.Step {
+			if treesArray, ok := node.([]*toml.Tree); ok {
+				if len(treesArray) > 0 {
+					ctx.lastPosition = treesArray[0].Position()
+				}
+			}
 			f.next.call(arr[idx], ctx)
 		}
 	}
@@ -133,8 +127,10 @@ func newMatchAnyFn() *matchAnyFn {
 }
 
 func (f *matchAnyFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
-		for _, v := range tree.values {
+	if tree, ok := node.(*toml.Tree); ok {
+		for _, k := range tree.Keys() {
+			v := tree.Get(k)
+			ctx.lastPosition = tree.GetPosition(k)
 			f.next.call(v, ctx)
 		}
 	}
@@ -167,21 +163,25 @@ func newMatchRecursiveFn() *matchRecursiveFn {
 }
 
 func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
-	if tree, ok := node.(*TomlTree); ok {
-		var visit func(tree *TomlTree)
-		visit = func(tree *TomlTree) {
-			for _, v := range tree.values {
+	originalPosition := ctx.lastPosition
+	if tree, ok := node.(*toml.Tree); ok {
+		var visit func(tree *toml.Tree)
+		visit = func(tree *toml.Tree) {
+			for _, k := range tree.Keys() {
+				v := tree.Get(k)
+				ctx.lastPosition = tree.GetPosition(k)
 				f.next.call(v, ctx)
 				switch node := v.(type) {
-				case *TomlTree:
+				case *toml.Tree:
 					visit(node)
-				case []*TomlTree:
+				case []*toml.Tree:
 					for _, subtree := range node {
 						visit(subtree)
 					}
 				}
 			}
 		}
+		ctx.lastPosition = originalPosition
 		f.next.call(tree, ctx)
 		visit(tree)
 	}
@@ -190,11 +190,11 @@ func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) {
 // match based on an externally provided functional filter
 type matchFilterFn struct {
 	matchBase
-	Pos  Position
+	Pos  toml.Position
 	Name string
 }
 
-func newMatchFilterFn(name string, pos Position) *matchFilterFn {
+func newMatchFilterFn(name string, pos toml.Position) *matchFilterFn {
 	return &matchFilterFn{Name: name, Pos: pos}
 }
 
@@ -202,19 +202,24 @@ func (f *matchFilterFn) call(node interface{}, ctx *queryContext) {
 	fn, ok := (*ctx.filters)[f.Name]
 	if !ok {
 		panic(fmt.Sprintf("%s: query context does not have filter '%s'",
-			f.Pos, f.Name))
+			f.Pos.String(), f.Name))
 	}
-	switch castNode := tomlValueCheck(node, ctx).(type) {
-	case *TomlTree:
-		for _, v := range castNode.values {
-			if tv, ok := v.(*tomlValue); ok {
-				if fn(tv.value) {
-					f.next.call(v, ctx)
-				}
-			} else {
-				if fn(v) {
-					f.next.call(v, ctx)
+	switch castNode := node.(type) {
+	case *toml.Tree:
+		for _, k := range castNode.Keys() {
+			v := castNode.Get(k)
+			if fn(v) {
+				ctx.lastPosition = castNode.GetPosition(k)
+				f.next.call(v, ctx)
+			}
+		}
+	case []*toml.Tree:
+		for _, v := range castNode {
+			if fn(v) {
+				if len(castNode) > 0 {
+					ctx.lastPosition = castNode[0].Position()
 				}
+				f.next.call(v, ctx)
 			}
 		}
 	case []interface{}:

query/match_test.go

@@ -1,7 +1,8 @@
-package toml
+package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"testing"
 )
 
@@ -109,7 +110,7 @@ func TestPathSliceStart(t *testing.T) {
 	assertPath(t,
 		"$[123:]",
 		buildPath(
-			newMatchSliceFn(123, MaxInt, 1),
+			newMatchSliceFn(123, maxInt, 1),
 		))
 }
 
@@ -133,7 +134,7 @@ func TestPathSliceStartStep(t *testing.T) {
 	assertPath(t,
 		"$[123::7]",
 		buildPath(
-			newMatchSliceFn(123, MaxInt, 7),
+			newMatchSliceFn(123, maxInt, 7),
 		))
 }
 
@@ -149,7 +150,7 @@ func TestPathSliceStep(t *testing.T) {
 	assertPath(t,
 		"$[::7]",
 		buildPath(
-			newMatchSliceFn(0, MaxInt, 7),
+			newMatchSliceFn(0, maxInt, 7),
 		))
 }
 
@@ -194,8 +195,8 @@ func TestPathFilterExpr(t *testing.T) {
 		"$[?('foo'),?(bar)]",
 		buildPath(
 			&matchUnionFn{[]pathFn{
-				newMatchFilterFn("foo", Position{}),
-				newMatchFilterFn("bar", Position{}),
+				newMatchFilterFn("foo", toml.Position{}),
+				newMatchFilterFn("bar", toml.Position{}),
 			}},
 		))
 }

query/parser.go

@@ -5,13 +5,13 @@
   https://code.google.com/p/json-path/
 */
 
-package toml
+package query
 
 import (
 	"fmt"
 )
 
-const MaxInt = int(^uint(0) >> 1)
+const maxInt = int(^uint(0) >> 1)
 
 type queryParser struct {
 	flow         chan token
@@ -138,7 +138,6 @@ func (p *queryParser) parseMatchExpr() queryParserStateFn {
 		return nil // allow EOF at this stage
 	}
 	return p.parseError(tok, "expected match expression")
-	return nil
 }
 
 func (p *queryParser) parseBracketExpr() queryParserStateFn {
@@ -204,7 +203,7 @@ loop: // labeled loop for easy breaking
 
 func (p *queryParser) parseSliceExpr() queryParserStateFn {
 	// init slice to grab all elements
-	start, end, step := 0, MaxInt, 1
+	start, end, step := 0, maxInt, 1
 
 	// parse optional start
 	tok := p.getToken()
@@ -254,7 +253,7 @@ func (p *queryParser) parseFilterExpr() queryParserStateFn {
 	}
 	tok = p.getToken()
 	if tok.typ != tokenKey && tok.typ != tokenString {
-		return p.parseError(tok, "expected key or string for filter funciton name")
+		return p.parseError(tok, "expected key or string for filter function name")
 	}
 	name := tok.val
 	tok = p.getToken()

query/parser_test.go

@@ -1,4 +1,4 @@
-package toml
+package query
 
 import (
 	"fmt"
@@ -7,19 +7,19 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/pelletier/go-toml"
 )
 
 type queryTestNode struct {
 	value    interface{}
-	position Position
+	position toml.Position
 }
 
 func valueString(root interface{}) string {
 	result := "" //fmt.Sprintf("%T:", root)
 	switch node := root.(type) {
-	case *tomlValue:
-		return valueString(node.value)
-	case *QueryResult:
+	case *Result:
 		items := []string{}
 		for i, v := range node.Values() {
 			items = append(items, fmt.Sprintf("%s:%s",
@@ -37,7 +37,7 @@ func valueString(root interface{}) string {
 		}
 		sort.Strings(items)
 		result = "[" + strings.Join(items, ", ") + "]"
-	case *TomlTree:
+	case *toml.Tree:
 		// workaround for unreliable map key ordering
 		items := []string{}
 		for _, k := range node.Keys() {
@@ -78,13 +78,13 @@ func assertValue(t *testing.T, result, ref interface{}) {
 	}
 }
 
-func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) {
-	tree, err := Load(toml)
+func assertQueryPositions(t *testing.T, tomlDoc string, query string, ref []interface{}) {
+	tree, err := toml.Load(tomlDoc)
 	if err != nil {
 		t.Errorf("Non-nil toml parse error: %v", err)
 		return
 	}
-	q, err := CompileQuery(query)
+	q, err := Compile(query)
 	if err != nil {
 		t.Error(err)
 		return
@@ -101,7 +101,7 @@ func TestQueryRoot(t *testing.T) {
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(42),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 		})
 }
@@ -112,7 +112,7 @@ func TestQueryKey(t *testing.T) {
 		"$.foo.a",
 		[]interface{}{
 			queryTestNode{
-				int64(42), Position{2, 1},
+				int64(42), toml.Position{2, 1},
 			},
 		})
 }
@@ -123,7 +123,7 @@ func TestQueryKeyString(t *testing.T) {
 		"$.foo['a']",
 		[]interface{}{
 			queryTestNode{
-				int64(42), Position{2, 1},
+				int64(42), toml.Position{2, 1},
 			},
 		})
 }
@@ -134,7 +134,7 @@ func TestQueryIndex(t *testing.T) {
 		"$.foo.a[5]",
 		[]interface{}{
 			queryTestNode{
-				int64(6), Position{2, 1},
+				int64(6), toml.Position{2, 1},
 			},
 		})
 }
@@ -145,19 +145,19 @@ func TestQuerySliceRange(t *testing.T) {
 		"$.foo.a[0:5]",
 		[]interface{}{
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(2), Position{2, 1},
+				int64(2), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(3), Position{2, 1},
+				int64(3), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(4), Position{2, 1},
+				int64(4), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(5), Position{2, 1},
+				int64(5), toml.Position{2, 1},
 			},
 		})
 }
@@ -168,13 +168,13 @@ func TestQuerySliceStep(t *testing.T) {
 		"$.foo.a[0:5:2]",
 		[]interface{}{
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(3), Position{2, 1},
+				int64(3), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(5), Position{2, 1},
+				int64(5), toml.Position{2, 1},
 			},
 		})
 }
@@ -188,13 +188,13 @@ func TestQueryAny(t *testing.T) {
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 		})
 }
@@ -207,19 +207,19 @@ func TestQueryUnionSimple(t *testing.T) {
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 		})
 }
@@ -249,7 +249,7 @@ func TestQueryRecursionAll(t *testing.T) {
 							"b": int64(6),
 						},
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -257,19 +257,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(1),
 						"b": int64(2),
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
-				int64(1), Position{2, 1},
+				int64(1), toml.Position{2, 1},
 			},
 			queryTestNode{
-				int64(2), Position{3, 1},
+				int64(2), toml.Position{3, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -277,19 +277,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(3),
 						"b": int64(4),
 					},
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
-				int64(3), Position{5, 1},
+				int64(3), toml.Position{5, 1},
 			},
 			queryTestNode{
-				int64(4), Position{6, 1},
+				int64(4), toml.Position{6, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -297,19 +297,19 @@ func TestQueryRecursionAll(t *testing.T) {
 						"a": int64(5),
 						"b": int64(6),
 					},
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 			queryTestNode{
-				int64(5), Position{8, 1},
+				int64(5), toml.Position{8, 1},
 			},
 			queryTestNode{
-				int64(6), Position{9, 1},
+				int64(6), toml.Position{9, 1},
 			},
 		})
 }
@@ -325,31 +325,31 @@ func TestQueryRecursionUnionSimple(t *testing.T) {
 						"a": int64(1),
 						"b": int64(2),
 					},
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(3),
 					"b": int64(4),
-				}, Position{4, 1},
+				}, toml.Position{4, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(1),
 					"b": int64(2),
-				}, Position{1, 1},
+				}, toml.Position{1, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"a": int64(5),
 					"b": int64(6),
-				}, Position{7, 1},
+				}, toml.Position{7, 1},
 			},
 		})
 }
 
 func TestQueryFilterFn(t *testing.T) {
-	buff, err := ioutil.ReadFile("example.toml")
+	buff, err := ioutil.ReadFile("../example.toml")
 	if err != nil {
 		t.Error(err)
 		return
@@ -359,16 +359,16 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(int)]",
 		[]interface{}{
 			queryTestNode{
-				int64(8001), Position{13, 1},
+				int64(8001), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(8001), Position{13, 1},
+				int64(8001), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(8002), Position{13, 1},
+				int64(8002), toml.Position{13, 1},
 			},
 			queryTestNode{
-				int64(5000), Position{14, 1},
+				int64(5000), toml.Position{14, 1},
 			},
 		})
 
@@ -376,39 +376,38 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(string)]",
 		[]interface{}{
 			queryTestNode{
-				"TOML Example", Position{3, 1},
+				"TOML Example", toml.Position{3, 1},
 			},
 			queryTestNode{
-				"Tom Preston-Werner", Position{6, 1},
+				"Tom Preston-Werner", toml.Position{6, 1},
 			},
 			queryTestNode{
-				"GitHub", Position{7, 1},
+				"GitHub", toml.Position{7, 1},
 			},
 			queryTestNode{
 				"GitHub Cofounder & CEO\nLikes tater tots and beer.",
-				Position{8, 1},
+				toml.Position{8, 1},
 			},
 			queryTestNode{
-				"192.168.1.1", Position{12, 1},
+				"192.168.1.1", toml.Position{12, 1},
 			},
 			queryTestNode{
-				"10.0.0.1", Position{21, 3},
+				"10.0.0.1", toml.Position{21, 3},
 			},
 			queryTestNode{
-				"eqdc10", Position{22, 3},
+				"eqdc10", toml.Position{22, 3},
 			},
 			queryTestNode{
-				"10.0.0.2", Position{25, 3},
+				"10.0.0.2", toml.Position{25, 3},
 			},
 			queryTestNode{
-				"eqdc10", Position{26, 3},
+				"eqdc10", toml.Position{26, 3},
 			},
 		})
 
 	assertQueryPositions(t, string(buff),
 		"$..[?(float)]",
-		[]interface{}{
-		// no float values in document
+		[]interface{}{ // no float values in document
 		})
 
 	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
@@ -421,7 +420,7 @@ func TestQueryFilterFn(t *testing.T) {
 					"organization": "GitHub",
 					"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
 					"dob":          tv,
-				}, Position{5, 1},
+				}, toml.Position{5, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -429,7 +428,7 @@ func TestQueryFilterFn(t *testing.T) {
 					"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
 					"connection_max": int64(5000),
 					"enabled":        true,
-				}, Position{11, 1},
+				}, toml.Position{11, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -441,19 +440,19 @@ func TestQueryFilterFn(t *testing.T) {
 						"ip": "10.0.0.2",
 						"dc": "eqdc10",
 					},
-				}, Position{17, 1},
+				}, toml.Position{17, 1},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"ip": "10.0.0.1",
 					"dc": "eqdc10",
-				}, Position{20, 3},
+				}, toml.Position{20, 3},
 			},
 			queryTestNode{
 				map[string]interface{}{
 					"ip": "10.0.0.2",
 					"dc": "eqdc10",
-				}, Position{24, 3},
+				}, toml.Position{24, 3},
 			},
 			queryTestNode{
 				map[string]interface{}{
@@ -461,7 +460,7 @@ func TestQueryFilterFn(t *testing.T) {
 						[]interface{}{"gamma", "delta"},
 						[]interface{}{int64(1), int64(2)},
 					},
-				}, Position{28, 1},
+				}, toml.Position{28, 1},
 			},
 		})
 
@@ -469,7 +468,7 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(time)]",
 		[]interface{}{
 			queryTestNode{
-				tv, Position{9, 1},
+				tv, toml.Position{9, 1},
 			},
 		})
 
@@ -477,7 +476,7 @@ func TestQueryFilterFn(t *testing.T) {
 		"$..[?(bool)]",
 		[]interface{}{
 			queryTestNode{
-				true, Position{15, 1},
+				true, toml.Position{15, 1},
 			},
 		})
 }

query/query.go

@@ -0,0 +1,158 @@
+package query
+
+import (
+	"time"
+
+	"github.com/pelletier/go-toml"
+)
+
+// NodeFilterFn represents a user-defined filter function, for use with
+// Query.SetFilter().
+//
+// The return value of the function must indicate if 'node' is to be included
+// at this stage of the TOML path.  Returning true will include the node, and
+// returning false will exclude it.
+//
+// NOTE: Care should be taken to write script callbacks such that they are safe
+// to use from multiple goroutines.
+type NodeFilterFn func(node interface{}) bool
+
+// Result is the result of Executing a Query.
+type Result struct {
+	items     []interface{}
+	positions []toml.Position
+}
+
+// appends a value/position pair to the result set.
+func (r *Result) appendResult(node interface{}, pos toml.Position) {
+	r.items = append(r.items, node)
+	r.positions = append(r.positions, pos)
+}
+
+// Values is a set of values within a Result.  The order of values is not
+// guaranteed to be in document order, and may be different each time a query is
+// executed.
+func (r Result) Values() []interface{} {
+	return r.items
+}
+
+// Positions is a set of positions for values within a Result.  Each index
+// in Positions() corresponds to the entry in Value() of the same index.
+func (r Result) Positions() []toml.Position {
+	return r.positions
+}
+
+// runtime context for executing query paths
+type queryContext struct {
+	result       *Result
+	filters      *map[string]NodeFilterFn
+	lastPosition toml.Position
+}
+
+// generic path functor interface
+type pathFn interface {
+	setNext(next pathFn)
+	// it is the caller's responsibility to set the ctx.lastPosition before invoking call()
+	// node can be one of: *toml.Tree, []*toml.Tree, or a scalar
+	call(node interface{}, ctx *queryContext)
+}
+
+// A Query is the representation of a compiled TOML path.  A Query is safe
+// for concurrent use by multiple goroutines.
+type Query struct {
+	root    pathFn
+	tail    pathFn
+	filters *map[string]NodeFilterFn
+}
+
+func newQuery() *Query {
+	return &Query{
+		root:    nil,
+		tail:    nil,
+		filters: &defaultFilterFunctions,
+	}
+}
+
+func (q *Query) appendPath(next pathFn) {
+	if q.root == nil {
+		q.root = next
+	} else {
+		q.tail.setNext(next)
+	}
+	q.tail = next
+	next.setNext(newTerminatingFn()) // init the next functor
+}
+
+// Compile compiles a TOML path expression. The returned Query can be used
+// to match elements within a Tree and its descendants. See Execute.
+func Compile(path string) (*Query, error) {
+	return parseQuery(lexQuery(path))
+}
+
+// Execute executes a query against a Tree, and returns the result of the query.
+func (q *Query) Execute(tree *toml.Tree) *Result {
+	result := &Result{
+		items:     []interface{}{},
+		positions: []toml.Position{},
+	}
+	if q.root == nil {
+		result.appendResult(tree, tree.GetPosition(""))
+	} else {
+		ctx := &queryContext{
+			result:  result,
+			filters: q.filters,
+		}
+		ctx.lastPosition = tree.Position()
+		q.root.call(tree, ctx)
+	}
+	return result
+}
+
+// CompileAndExecute is a shorthand for Compile(path) followed by Execute(tree).
+func CompileAndExecute(path string, tree *toml.Tree) (*Result, error) {
+	query, err := Compile(path)
+	if err != nil {
+		return nil, err
+	}
+	return query.Execute(tree), nil
+}
+
+// SetFilter sets a user-defined filter function.  These may be used inside
+// "?(..)" query expressions to filter TOML document elements within a query.
+func (q *Query) SetFilter(name string, fn NodeFilterFn) {
+	if q.filters == &defaultFilterFunctions {
+		// clone the static table
+		q.filters = &map[string]NodeFilterFn{}
+		for k, v := range defaultFilterFunctions {
+			(*q.filters)[k] = v
+		}
+	}
+	(*q.filters)[name] = fn
+}
+
+var defaultFilterFunctions = map[string]NodeFilterFn{
+	"tree": func(node interface{}) bool {
+		_, ok := node.(*toml.Tree)
+		return ok
+	},
+	"int": func(node interface{}) bool {
+		_, ok := node.(int64)
+		return ok
+	},
+	"float": func(node interface{}) bool {
+		_, ok := node.(float64)
+		return ok
+	},
+	"string": func(node interface{}) bool {
+		_, ok := node.(string)
+		return ok
+	},
+	"time": func(node interface{}) bool {
+		_, ok := node.(time.Time)
+		return ok
+	},
+	"bool": func(node interface{}) bool {
+		_, ok := node.(bool)
+		return ok
+	},
+}

query/query_test.go

@@ -0,0 +1,157 @@
+package query
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pelletier/go-toml"
+)
+
+func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) {
+	if len(array) != len(objects) {
+		t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects))
+	}
+
+	for _, o := range objects {
+		found := false
+		for _, a := range array {
+			if a == o {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal(o, "not found in array", array)
+		}
+	}
+}
+
+func TestQueryExample(t *testing.T) {
+	config, _ := toml.Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+	authors, err := CompileAndExecute("$.book.author", config)
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	names := authors.Values()
+	if len(names) != 3 {
+		t.Fatalf("query should return 3 names but returned %d", len(names))
+	}
+	assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson")
+}
+
+func TestQueryReadmeExample(t *testing.T) {
+	config, _ := toml.Load(`
+[postgres]
+user = "pelletier"
+password = "mypassword"
+`)
+
+	query, err := Compile("$..[user,password]")
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	results := query.Execute(config)
+	values := results.Values()
+	if len(values) != 2 {
+		t.Fatalf("query should return 2 values but returned %d", len(values))
+	}
+	assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword")
+}
+
+func TestQueryPathNotPresent(t *testing.T) {
+	config, _ := toml.Load(`a = "hello"`)
+	query, err := Compile("$.foo.bar")
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	results := query.Execute(config)
+	if err != nil {
+		t.Fatalf("err should be nil. got %s instead", err)
+	}
+	if len(results.items) != 0 {
+		t.Fatalf("no items should be matched. %d matched instead", len(results.items))
+	}
+}
+
+func ExampleNodeFilterFn_filterExample() {
+	tree, _ := toml.Load(`
+      [struct_one]
+      foo = "foo"
+      bar = "bar"
+
+      [struct_two]
+      baz = "baz"
+      gorf = "gorf"
+    `)
+
+	// create a query that references a user-defined-filter
+	query, _ := Compile("$[?(bazOnly)]")
+
+	// define the filter, and assign it to the query
+	query.SetFilter("bazOnly", func(node interface{}) bool {
+		if tree, ok := node.(*toml.Tree); ok {
+			return tree.Has("baz")
+		}
+		return false // reject all other node types
+	})
+
+	// results contain only the 'struct_two' Tree
+	query.Execute(tree)
+}
+
+func ExampleQuery_queryExample() {
+	config, _ := toml.Load(`
+      [[book]]
+      title = "The Stand"
+      author = "Stephen King"
+      [[book]]
+      title = "For Whom the Bell Tolls"
+      author = "Ernest Hemmingway"
+      [[book]]
+      title = "Neuromancer"
+      author = "William Gibson"
+    `)
+
+	// find and print all the authors in the document
+	query, _ := Compile("$.book.author")
+	authors := query.Execute(config)
+	for _, name := range authors.Values() {
+		fmt.Println(name)
+	}
+}
+
+func TestTomlQuery(t *testing.T) {
+	tree, err := toml.Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	query, err := Compile("$.foo.bar")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	result := query.Execute(tree)
+	values := result.Values()
+	if len(values) != 1 {
+		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
+	}
+
+	if tt, ok := values[0].(*toml.Tree); !ok {
+		t.Errorf("Expected type of Tree: %T", values[0])
+	} else if tt.Get("a") != int64(1) {
+		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
+	} else if tt.Get("b") != int64(2) {
+		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	}
+}

query/tokens.go

@@ -0,0 +1,106 @@
+package query
+
+import (
+	"fmt"
+	"github.com/pelletier/go-toml"
+	"strconv"
+	"unicode"
+)
+
+// Define tokens
+type tokenType int
+
+const (
+	eof = -(iota + 1)
+)
+
+const (
+	tokenError tokenType = iota
+	tokenEOF
+	tokenKey
+	tokenString
+	tokenInteger
+	tokenFloat
+	tokenLeftBracket
+	tokenRightBracket
+	tokenLeftParen
+	tokenRightParen
+	tokenComma
+	tokenColon
+	tokenDollar
+	tokenStar
+	tokenQuestion
+	tokenDot
+	tokenDotDot
+)
+
+var tokenTypeNames = []string{
+	"Error",
+	"EOF",
+	"Key",
+	"String",
+	"Integer",
+	"Float",
+	"[",
+	"]",
+	"(",
+	")",
+	",",
+	":",
+	"$",
+	"*",
+	"?",
+	".",
+	"..",
+}
+
+type token struct {
+	toml.Position
+	typ tokenType
+	val string
+}
+
+func (tt tokenType) String() string {
+	idx := int(tt)
+	if idx < len(tokenTypeNames) {
+		return tokenTypeNames[idx]
+	}
+	return "Unknown"
+}
+
+func (t token) Int() int {
+	if result, err := strconv.Atoi(t.val); err != nil {
+		panic(err)
+	} else {
+		return result
+	}
+}
+
+func (t token) String() string {
+	switch t.typ {
+	case tokenEOF:
+		return "EOF"
+	case tokenError:
+		return t.val
+	}
+
+	return fmt.Sprintf("%q", t.val)
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t'
+}
+
+func isAlphanumeric(r rune) bool {
+	return unicode.IsLetter(r) || r == '_'
+}
+
+func isDigit(r rune) bool {
+	return unicode.IsNumber(r)
+}
+
+func isHexDigit(r rune) bool {
+	return isDigit(r) ||
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
+}

querylexer_test.go

@@ -1,97 +0,0 @@
-package toml
-
-import (
-	"testing"
-)
-
-func testQLFlow(t *testing.T, input string, expectedFlow []token) {
-	ch := lexQuery(input)
-	for idx, expected := range expectedFlow {
-		token := <-ch
-		if token != expected {
-			t.Log("While testing #", idx, ":", input)
-			t.Log("compared", token, "to", expected)
-			t.Log(token.val, "<->", expected.val)
-			t.Log(token.typ, "<->", expected.typ)
-			t.Log(token.Line, "<->", expected.Line)
-			t.Log(token.Col, "<->", expected.Col)
-			t.FailNow()
-		}
-	}
-
-	tok, ok := <-ch
-	if ok {
-		t.Log("channel is not closed!")
-		t.Log(len(ch)+1, "tokens remaining:")
-
-		t.Log("token ->", tok)
-		for token := range ch {
-			t.Log("token ->", token)
-		}
-		t.FailNow()
-	}
-}
-
-func TestLexSpecialChars(t *testing.T) {
-	testQLFlow(t, " .$[]..()?*", []token{
-		token{Position{1, 2}, tokenDot, "."},
-		token{Position{1, 3}, tokenDollar, "$"},
-		token{Position{1, 4}, tokenLeftBracket, "["},
-		token{Position{1, 5}, tokenRightBracket, "]"},
-		token{Position{1, 6}, tokenDotDot, ".."},
-		token{Position{1, 8}, tokenLeftParen, "("},
-		token{Position{1, 9}, tokenRightParen, ")"},
-		token{Position{1, 10}, tokenQuestion, "?"},
-		token{Position{1, 11}, tokenStar, "*"},
-		token{Position{1, 12}, tokenEOF, ""},
-	})
-}
-
-func TestLexString(t *testing.T) {
-	testQLFlow(t, "'foo'", []token{
-		token{Position{1, 2}, tokenString, "foo"},
-		token{Position{1, 6}, tokenEOF, ""},
-	})
-}
-
-func TestLexDoubleString(t *testing.T) {
-	testQLFlow(t, `"bar"`, []token{
-		token{Position{1, 2}, tokenString, "bar"},
-		token{Position{1, 6}, tokenEOF, ""},
-	})
-}
-
-func TestLexKey(t *testing.T) {
-	testQLFlow(t, "foo", []token{
-		token{Position{1, 1}, tokenKey, "foo"},
-		token{Position{1, 4}, tokenEOF, ""},
-	})
-}
-
-func TestLexRecurse(t *testing.T) {
-	testQLFlow(t, "$..*", []token{
-		token{Position{1, 1}, tokenDollar, "$"},
-		token{Position{1, 2}, tokenDotDot, ".."},
-		token{Position{1, 4}, tokenStar, "*"},
-		token{Position{1, 5}, tokenEOF, ""},
-	})
-}
-
-func TestLexBracketKey(t *testing.T) {
-	testQLFlow(t, "$[foo]", []token{
-		token{Position{1, 1}, tokenDollar, "$"},
-		token{Position{1, 2}, tokenLeftBracket, "["},
-		token{Position{1, 3}, tokenKey, "foo"},
-		token{Position{1, 6}, tokenRightBracket, "]"},
-		token{Position{1, 7}, tokenEOF, ""},
-	})
-}
-
-func TestLexSpace(t *testing.T) {
-	testQLFlow(t, "foo bar baz", []token{
-		token{Position{1, 1}, tokenKey, "foo"},
-		token{Position{1, 5}, tokenKey, "bar"},
-		token{Position{1, 9}, tokenKey, "baz"},
-		token{Position{1, 12}, tokenEOF, ""},
-	})
-}

test.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # fail out of the script if anything here fails
 set -e
+set -o pipefail
 
 # set the path to the present working directory
 export GOPATH=`pwd`
@@ -19,6 +20,14 @@ function git_clone() {
   popd
 }
 
+# Remove potential previous runs
+rm -rf src test_program_bin toml-test
+
+go get github.com/pelletier/go-buffruneio
+go get github.com/davecgh/go-spew/spew
+go get gopkg.in/yaml.v2
+go get github.com/BurntSushi/toml
+
 # get code for BurntSushi TOML validation
 # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)
 git_clone github.com/BurntSushi/toml master HEAD
@@ -30,12 +39,16 @@ go build -o toml-test github.com/BurntSushi/toml-test
 # vendorize the current lib for testing
 # NOTE: this basically mocks an install without having to go back out to github for code
 mkdir -p src/github.com/pelletier/go-toml/cmd
+mkdir -p src/github.com/pelletier/go-toml/query
 cp *.go *.toml src/github.com/pelletier/go-toml
-cp cmd/*.go src/github.com/pelletier/go-toml/cmd
+cp -R cmd/* src/github.com/pelletier/go-toml/cmd
+cp -R query/* src/github.com/pelletier/go-toml/query
 go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
 
 # Run basic unit tests
-go test -v github.com/pelletier/go-toml
+go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out
+go test github.com/pelletier/go-toml/cmd/tomljson
+go test github.com/pelletier/go-toml/query
 
 # run the entire BurntSushi test suite
 if [[ $# -eq 0 ]] ; then
@@ -66,7 +79,7 @@ else
     echo "Invalid Test TOML for $test:"
     echo "===="
     cat "$invalid_test.toml"
-    
+
     echo "Go-TOML Output for $test:"
     echo "===="
     echo "go-toml Output:"

token.go

@@ -23,6 +23,8 @@ const (
 	tokenTrue
 	tokenFalse
 	tokenFloat
+	tokenInf
+	tokenNan
 	tokenEqual
 	tokenLeftBracket
 	tokenRightBracket
@@ -55,6 +57,8 @@ var tokenTypeNames = []string{
 	"True",
 	"False",
 	"Float",
+	"Inf",
+	"NaN",
 	"=",
 	"[",
 	"]",
@@ -107,9 +111,6 @@ func (t token) String() string {
 		return t.val
 	}
 
-	if len(t.val) > 10 {
-		return fmt.Sprintf("%.10q...", t.val)
-	}
 	return fmt.Sprintf("%q", t.val)
 }
 
@@ -138,5 +139,6 @@ func isDigit(r rune) bool {
 
 func isHexDigit(r rune) bool {
 	return isDigit(r) ||
-		r == 'A' || r == 'B' || r == 'C' || r == 'D' || r == 'E' || r == 'F'
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
 }

token_test.go

@@ -0,0 +1,67 @@
+package toml
+
+import "testing"
+
+func TestTokenStringer(t *testing.T) {
+	var tests = []struct {
+		tt     tokenType
+		expect string
+	}{
+		{tokenError, "Error"},
+		{tokenEOF, "EOF"},
+		{tokenComment, "Comment"},
+		{tokenKey, "Key"},
+		{tokenString, "String"},
+		{tokenInteger, "Integer"},
+		{tokenTrue, "True"},
+		{tokenFalse, "False"},
+		{tokenFloat, "Float"},
+		{tokenEqual, "="},
+		{tokenLeftBracket, "["},
+		{tokenRightBracket, "]"},
+		{tokenLeftCurlyBrace, "{"},
+		{tokenRightCurlyBrace, "}"},
+		{tokenLeftParen, "("},
+		{tokenRightParen, ")"},
+		{tokenDoubleLeftBracket, "]]"},
+		{tokenDoubleRightBracket, "[["},
+		{tokenDate, "Date"},
+		{tokenKeyGroup, "KeyGroup"},
+		{tokenKeyGroupArray, "KeyGroupArray"},
+		{tokenComma, ","},
+		{tokenColon, ":"},
+		{tokenDollar, "$"},
+		{tokenStar, "*"},
+		{tokenQuestion, "?"},
+		{tokenDot, "."},
+		{tokenDotDot, ".."},
+		{tokenEOL, "EOL"},
+		{tokenEOL + 1, "Unknown"},
+	}
+
+	for i, test := range tests {
+		got := test.tt.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid string of token type; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}
+
+func TestTokenString(t *testing.T) {
+	var tests = []struct {
+		tok    token
+		expect string
+	}{
+		{token{Position{1, 1}, tokenEOF, ""}, "EOF"},
+		{token{Position{1, 1}, tokenError, "Δt"}, "Δt"},
+		{token{Position{1, 1}, tokenString, "bar"}, `"bar"`},
+		{token{Position{1, 1}, tokenString, "123456789012345"}, `"123456789012345"`},
+	}
+
+	for i, test := range tests {
+		got := test.tok.String()
+		if got != test.expect {
+			t.Errorf("[%d] invalid of string token; got %q, expected %q", i, got, test.expect)
+		}
+	}
+}

toml.go

@@ -3,39 +3,52 @@ package toml
 import (
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"os"
 	"runtime"
-	"strconv"
 	"strings"
-	"time"
 )
 
 type tomlValue struct {
-	value    interface{}
-	position Position
+	value     interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list
+	comment   string
+	commented bool
+	multiline bool
+	position  Position
 }
 
-// TomlTree is the result of the parsing of a TOML file.
-type TomlTree struct {
-	values   map[string]interface{}
-	position Position
+// Tree is the result of the parsing of a TOML file.
+type Tree struct {
+	values    map[string]interface{} // string -> *tomlValue, *Tree, []*Tree
+	comment   string
+	commented bool
+	position  Position
 }
 
-func newTomlTree() *TomlTree {
-	return &TomlTree{
+func newTree() *Tree {
+	return &Tree{
 		values:   make(map[string]interface{}),
 		position: Position{},
 	}
 }
 
-func TreeFromMap(m map[string]interface{}) *TomlTree {
-	return &TomlTree{
-		values: m,
+// TreeFromMap initializes a new Tree object using the given map.
+func TreeFromMap(m map[string]interface{}) (*Tree, error) {
+	result, err := toTree(m)
+	if err != nil {
+		return nil, err
 	}
+	return result.(*Tree), nil
+}
+
+// Position returns the position of the tree.
+func (t *Tree) Position() Position {
+	return t.position
 }
 
 // Has returns a boolean indicating if the given key exists.
-func (t *TomlTree) Has(key string) bool {
+func (t *Tree) Has(key string) bool {
 	if key == "" {
 		return false
 	}
@@ -43,38 +56,36 @@ func (t *TomlTree) Has(key string) bool {
 }
 
 // HasPath returns true if the given path of keys exists, false otherwise.
-func (t *TomlTree) HasPath(keys []string) bool {
+func (t *Tree) HasPath(keys []string) bool {
 	return t.GetPath(keys) != nil
 }
 
-// Keys returns the keys of the toplevel tree.
-// Warning: this is a costly operation.
-func (t *TomlTree) Keys() []string {
-	var keys []string
+// Keys returns the keys of the toplevel tree (does not recurse).
+func (t *Tree) Keys() []string {
+	keys := make([]string, len(t.values))
+	i := 0
 	for k := range t.values {
-		keys = append(keys, k)
+		keys[i] = k
+		i++
 	}
 	return keys
 }
 
-// Get the value at key in the TomlTree.
-// Key is a dot-separated path (e.g. a.b.c).
+// Get the value at key in the Tree.
+// Key is a dot-separated path (e.g. a.b.c) without single/double quoted strings.
+// If you need to retrieve non-bare keys, use GetPath.
 // Returns nil if the path does not exist in the tree.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) Get(key string) interface{} {
+func (t *Tree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
-	comps, err := parseKey(key)
-	if err != nil {
-		return nil
-	}
-	return t.GetPath(comps)
+	return t.GetPath(strings.Split(key, "."))
 }
 
 // GetPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) GetPath(keys []string) interface{} {
+func (t *Tree) GetPath(keys []string) interface{} {
 	if len(keys) == 0 {
 		return t
 	}
@@ -85,16 +96,16 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 			return nil
 		}
 		switch node := value.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				return nil
 			}
 			subtree = node[len(node)-1]
 		default:
-			return nil // cannot naigate through other node types
+			return nil // cannot navigate through other node types
 		}
 	}
 	// branch based on final node type
@@ -107,7 +118,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} {
 }
 
 // GetPosition returns the position of the given key.
-func (t *TomlTree) GetPosition(key string) Position {
+func (t *Tree) GetPosition(key string) Position {
 	if key == "" {
 		return t.position
 	}
@@ -116,7 +127,7 @@ func (t *TomlTree) GetPosition(key string) Position {
 
 // GetPositionPath returns the element in the tree indicated by 'keys'.
 // If keys is of length zero, the current tree is returned.
-func (t *TomlTree) GetPositionPath(keys []string) Position {
+func (t *Tree) GetPositionPath(keys []string) Position {
 	if len(keys) == 0 {
 		return t.position
 	}
@@ -127,9 +138,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 			return Position{0, 0}
 		}
 		switch node := value.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				return Position{0, 0}
@@ -143,9 +154,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 	switch node := subtree.values[keys[len(keys)-1]].(type) {
 	case *tomlValue:
 		return node.position
-	case *TomlTree:
+	case *Tree:
 		return node.position
-	case []*TomlTree:
+	case []*Tree:
 		// go to most recent element
 		if len(node) == 0 {
 			return Position{0, 0}
@@ -157,7 +168,7 @@ func (t *TomlTree) GetPositionPath(keys []string) Position {
 }
 
 // GetDefault works like Get but with a default value
-func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
+func (t *Tree) GetDefault(key string, def interface{}) interface{} {
 	val := t.Get(key)
 	if val == nil {
 		return def
@@ -165,32 +176,38 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} {
 	return val
 }
 
-// Set an element in the tree.
-// Key is a dot-separated path (e.g. a.b.c).
-// Creates all necessary intermediates trees, if needed.
-func (t *TomlTree) Set(key string, value interface{}) {
-	t.SetPath(strings.Split(key, "."), value)
+// SetOptions arguments are supplied to the SetWithOptions and SetPathWithOptions functions to modify marshalling behaviour.
+// The default values within the struct are valid default options.
+type SetOptions struct {
+	Comment   string
+	Commented bool
+	Multiline bool
 }
 
-// SetPath sets an element in the tree.
-// Keys is an array of path elements (e.g. {"a","b","c"}).
-// Creates all necessary intermediates trees, if needed.
-func (t *TomlTree) SetPath(keys []string, value interface{}) {
+// SetWithOptions is the same as Set, but allows you to provide formatting
+// instructions to the key, that will be used by Marshal().
+func (t *Tree) SetWithOptions(key string, opts SetOptions, value interface{}) {
+	t.SetPathWithOptions(strings.Split(key, "."), opts, value)
+}
+
+// SetPathWithOptions is the same as SetPath, but allows you to provide
+// formatting instructions to the key, that will be reused by Marshal().
+func (t *Tree) SetPathWithOptions(keys []string, opts SetOptions, value interface{}) {
 	subtree := t
 	for _, intermediateKey := range keys[:len(keys)-1] {
 		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			nextTree = newTomlTree()
+			nextTree = newTree()
 			subtree.values[intermediateKey] = nextTree // add new element here
 		}
 		switch node := nextTree.(type) {
-		case *TomlTree:
+		case *Tree:
 			subtree = node
-		case []*TomlTree:
+		case []*Tree:
 			// go to most recent element
 			if len(node) == 0 {
 				// create element if it does not exist
-				subtree.values[intermediateKey] = append(node, newTomlTree())
+				subtree.values[intermediateKey] = append(node, newTree())
 			}
 			subtree = node[len(node)-1]
 		}
@@ -199,14 +216,81 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 	var toInsert interface{}
 
 	switch value.(type) {
-	case *TomlTree:
+	case *Tree:
+		tt := value.(*Tree)
+		tt.comment = opts.Comment
 		toInsert = value
-	case []*TomlTree:
+	case []*Tree:
 		toInsert = value
 	case *tomlValue:
-		toInsert = value
+		tt := value.(*tomlValue)
+		tt.comment = opts.Comment
+		toInsert = tt
 	default:
-		toInsert = &tomlValue{value: value}
+		toInsert = &tomlValue{value: value, comment: opts.Comment, commented: opts.Commented, multiline: opts.Multiline}
+	}
+
+	subtree.values[keys[len(keys)-1]] = toInsert
+}
+
+// Set an element in the tree.
+// Key is a dot-separated path (e.g. a.b.c).
+// Creates all necessary intermediate trees, if needed.
+func (t *Tree) Set(key string, value interface{}) {
+	t.SetWithComment(key, "", false, value)
+}
+
+// SetWithComment is the same as Set, but allows you to provide comment
+// information to the key, that will be reused by Marshal().
+func (t *Tree) SetWithComment(key string, comment string, commented bool, value interface{}) {
+	t.SetPathWithComment(strings.Split(key, "."), comment, commented, value)
+}
+
+// SetPath sets an element in the tree.
+// Keys is an array of path elements (e.g. {"a","b","c"}).
+// Creates all necessary intermediate trees, if needed.
+func (t *Tree) SetPath(keys []string, value interface{}) {
+	t.SetPathWithComment(keys, "", false, value)
+}
+
+// SetPathWithComment is the same as SetPath, but allows you to provide comment
+// information to the key, that will be reused by Marshal().
+func (t *Tree) SetPathWithComment(keys []string, comment string, commented bool, value interface{}) {
+	subtree := t
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		nextTree, exists := subtree.values[intermediateKey]
+		if !exists {
+			nextTree = newTree()
+			subtree.values[intermediateKey] = nextTree // add new element here
+		}
+		switch node := nextTree.(type) {
+		case *Tree:
+			subtree = node
+		case []*Tree:
+			// go to most recent element
+			if len(node) == 0 {
+				// create element if it does not exist
+				subtree.values[intermediateKey] = append(node, newTree())
+			}
+			subtree = node[len(node)-1]
+		}
+	}
+
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *Tree:
+		tt := value.(*Tree)
+		tt.comment = comment
+		toInsert = value
+	case []*Tree:
+		toInsert = value
+	case *tomlValue:
+		tt := value.(*tomlValue)
+		tt.comment = comment
+		toInsert = tt
+	default:
+		toInsert = &tomlValue{value: value, comment: comment, commented: commented}
 	}
 
 	subtree.values[keys[len(keys)-1]] = toInsert
@@ -219,24 +303,21 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) {
 // and tree[a][b][c]
 //
 // Returns nil on success, error object on failure
-func (t *TomlTree) createSubTree(keys []string, pos Position) error {
+func (t *Tree) createSubTree(keys []string, pos Position) error {
 	subtree := t
 	for _, intermediateKey := range keys {
-		if intermediateKey == "" {
-			return fmt.Errorf("empty intermediate table")
-		}
 		nextTree, exists := subtree.values[intermediateKey]
 		if !exists {
-			tree := newTomlTree()
+			tree := newTree()
 			tree.position = pos
 			subtree.values[intermediateKey] = tree
 			nextTree = tree
 		}
 
 		switch node := nextTree.(type) {
-		case []*TomlTree:
+		case []*Tree:
 			subtree = node[len(node)-1]
-		case *TomlTree:
+		case *Tree:
 			subtree = node
 		default:
 			return fmt.Errorf("unknown type for path %s (%s): %T (%#v)",
@@ -246,122 +327,8 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error {
 	return nil
 }
 
-// encodes a string to a TOML-compliant string value
-func encodeTomlString(value string) string {
-	result := ""
-	for _, rr := range value {
-		intRr := uint16(rr)
-		switch rr {
-		case '\b':
-			result += "\\b"
-		case '\t':
-			result += "\\t"
-		case '\n':
-			result += "\\n"
-		case '\f':
-			result += "\\f"
-		case '\r':
-			result += "\\r"
-		case '"':
-			result += "\\\""
-		case '\\':
-			result += "\\\\"
-		default:
-			if intRr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", intRr)
-			} else {
-				result += string(rr)
-			}
-		}
-	}
-	return result
-}
-
-// Value print support function for ToString()
-// Outputs the TOML compliant string representation of a value
-func toTomlValue(item interface{}, indent int) string {
-	tab := strings.Repeat(" ", indent)
-	switch value := item.(type) {
-	case int64:
-		return tab + strconv.FormatInt(value, 10)
-	case float64:
-		return tab + strconv.FormatFloat(value, 'f', -1, 64)
-	case string:
-		return tab + "\"" + encodeTomlString(value) + "\""
-	case bool:
-		if value {
-			return "true"
-		}
-		return "false"
-	case time.Time:
-		return tab + value.Format(time.RFC3339)
-	case []interface{}:
-		result := tab + "[\n"
-		for _, item := range value {
-			result += toTomlValue(item, indent+2) + ",\n"
-		}
-		return result + tab + "]"
-	default:
-		panic(fmt.Sprintf("unsupported value type: %v", value))
-	}
-}
-
-// Recursive support function for ToString()
-// Outputs a tree, using the provided keyspace to prefix group names
-func (t *TomlTree) toToml(indent, keyspace string) string {
-	result := ""
-	for k, v := range t.values {
-		// figure out the keyspace
-		combinedKey := k
-		if keyspace != "" {
-			combinedKey = keyspace + "." + combinedKey
-		}
-		// output based on type
-		switch node := v.(type) {
-		case []*TomlTree:
-			for _, item := range node {
-				if len(item.Keys()) > 0 {
-					result += fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
-				}
-				result += item.toToml(indent+"  ", combinedKey)
-			}
-		case *TomlTree:
-			if len(node.Keys()) > 0 {
-				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			result += node.toToml(indent+"  ", combinedKey)
-		case map[string]interface{}:
-			sub := TreeFromMap(node)
-
-			if len(sub.Keys()) > 0 {
-				result += fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
-			}
-			result += sub.toToml(indent+"  ", combinedKey)
-		case *tomlValue:
-			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(node.value, 0))
-		default:
-			result += fmt.Sprintf("%s%s = %s\n", indent, k, toTomlValue(v, 0))
-		}
-	}
-	return result
-}
-
-func (t *TomlTree) Query(query string) (*QueryResult, error) {
-	if q, err := CompileQuery(query); err != nil {
-		return nil, err
-	} else {
-		return q.Execute(t), nil
-	}
-}
-
-// ToString generates a human-readable representation of the current tree.
-// Output spans multiple lines, and is suitable for ingest by a TOML parser
-func (t *TomlTree) ToString() string {
-	return t.toToml("", "")
-}
-
-// Load creates a TomlTree from a string.
-func Load(content string) (tree *TomlTree, err error) {
+// LoadBytes creates a Tree from a []byte.
+func LoadBytes(b []byte) (tree *Tree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -370,18 +337,31 @@ func Load(content string) (tree *TomlTree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	tree = parseToml(lexToml(content))
+	tree = parseToml(lexToml(b))
 	return
 }
 
-// LoadFile creates a TomlTree from a file.
-func LoadFile(path string) (tree *TomlTree, err error) {
-	buff, ferr := ioutil.ReadFile(path)
-	if ferr != nil {
-		err = ferr
-	} else {
-		s := string(buff)
-		tree, err = Load(s)
+// LoadReader creates a Tree from any io.Reader.
+func LoadReader(reader io.Reader) (tree *Tree, err error) {
+	inputBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return
 	}
+	tree, err = LoadBytes(inputBytes)
 	return
 }
+
+// Load creates a Tree from a string.
+func Load(content string) (tree *Tree, err error) {
+	return LoadBytes([]byte(content))
+}
+
+// LoadFile creates a Tree from a file.
+func LoadFile(path string) (tree *Tree, err error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	return LoadReader(file)
+}

toml_test.go

@@ -15,6 +15,47 @@ func TestTomlHas(t *testing.T) {
 	if !tree.Has("test.key") {
 		t.Errorf("Has - expected test.key to exists")
 	}
+
+	if tree.Has("") {
+		t.Errorf("Should return false if the key is not provided")
+	}
+}
+
+func TestTomlGet(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.Get("") != tree {
+		t.Errorf("Get should return the tree itself when given an empty path")
+	}
+
+	if tree.Get("test.key") != "value" {
+		t.Errorf("Get should return the value")
+	}
+	if tree.Get(`\`) != nil {
+		t.Errorf("should return nil when the key is malformed")
+	}
+}
+
+func TestTomlGetDefault(t *testing.T) {
+	tree, _ := Load(`
+		[test]
+		key = "value"
+	`)
+
+	if tree.GetDefault("", "hello") != tree {
+		t.Error("GetDefault should return the tree itself when given an empty path")
+	}
+
+	if tree.GetDefault("test.key", "hello") != "value" {
+		t.Error("Get should return the value")
+	}
+
+	if tree.GetDefault("whatever", "hello") != "hello" {
+		t.Error("GetDefault should return the default value if the key does not exist")
+	}
 }
 
 func TestTomlHasPath(t *testing.T) {
@@ -29,12 +70,12 @@ func TestTomlHasPath(t *testing.T) {
 }
 
 func TestTomlGetPath(t *testing.T) {
-	node := newTomlTree()
+	node := newTree()
 	//TODO: set other node data
 
 	for idx, item := range []struct {
 		Path     []string
-		Expected *TomlTree
+		Expected *Tree
 	}{
 		{ // empty path test
 			[]string{},
@@ -46,29 +87,20 @@ func TestTomlGetPath(t *testing.T) {
 			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
 		}
 	}
+
+	tree, _ := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+	if tree.GetPath([]string{"whatever"}) != nil {
+		t.Error("GetPath should return nil when the key does not exist")
+	}
 }
 
-func TestTomlQuery(t *testing.T) {
-	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
+func TestTomlFromMap(t *testing.T) {
+	simpleMap := map[string]interface{}{"hello": 42}
+	tree, err := TreeFromMap(simpleMap)
 	if err != nil {
-		t.Error(err)
-		return
+		t.Fatal("unexpected error:", err)
 	}
-	result, err := tree.Query("$.foo.bar")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	values := result.Values()
-	if len(values) != 1 {
-		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
-	}
-
-	if tt, ok := values[0].(*TomlTree); !ok {
-		t.Errorf("Expected type of TomlTree: %T Tv", values[0], values[0])
-	} else if tt.Get("a") != int64(1) {
-		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
-	} else if tt.Get("b") != int64(2) {
-		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
+	if tree.Get("hello") != int64(42) {
+		t.Fatal("hello should be 42, not", tree.Get("hello"))
 	}
 }

tomltree_create.go

@@ -0,0 +1,142 @@
+package toml
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+)
+
+var kindToType = [reflect.String + 1]reflect.Type{
+	reflect.Bool:    reflect.TypeOf(true),
+	reflect.String:  reflect.TypeOf(""),
+	reflect.Float32: reflect.TypeOf(float64(1)),
+	reflect.Float64: reflect.TypeOf(float64(1)),
+	reflect.Int:     reflect.TypeOf(int64(1)),
+	reflect.Int8:    reflect.TypeOf(int64(1)),
+	reflect.Int16:   reflect.TypeOf(int64(1)),
+	reflect.Int32:   reflect.TypeOf(int64(1)),
+	reflect.Int64:   reflect.TypeOf(int64(1)),
+	reflect.Uint:    reflect.TypeOf(uint64(1)),
+	reflect.Uint8:   reflect.TypeOf(uint64(1)),
+	reflect.Uint16:  reflect.TypeOf(uint64(1)),
+	reflect.Uint32:  reflect.TypeOf(uint64(1)),
+	reflect.Uint64:  reflect.TypeOf(uint64(1)),
+}
+
+// typeFor returns a reflect.Type for a reflect.Kind, or nil if none is found.
+// supported values:
+// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32
+func typeFor(k reflect.Kind) reflect.Type {
+	if k > 0 && int(k) < len(kindToType) {
+		return kindToType[k]
+	}
+	return nil
+}
+
+func simpleValueCoercion(object interface{}) (interface{}, error) {
+	switch original := object.(type) {
+	case string, bool, int64, uint64, float64, time.Time:
+		return original, nil
+	case int:
+		return int64(original), nil
+	case int8:
+		return int64(original), nil
+	case int16:
+		return int64(original), nil
+	case int32:
+		return int64(original), nil
+	case uint:
+		return uint64(original), nil
+	case uint8:
+		return uint64(original), nil
+	case uint16:
+		return uint64(original), nil
+	case uint32:
+		return uint64(original), nil
+	case float32:
+		return float64(original), nil
+	case fmt.Stringer:
+		return original.String(), nil
+	default:
+		return nil, fmt.Errorf("cannot convert type %T to Tree", object)
+	}
+}
+
+func sliceToTree(object interface{}) (interface{}, error) {
+	// arrays are a bit tricky, since they can represent either a
+	// collection of simple values, which is represented by one
+	// *tomlValue, or an array of tables, which is represented by an
+	// array of *Tree.
+
+	// holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice
+	value := reflect.ValueOf(object)
+	insideType := value.Type().Elem()
+	length := value.Len()
+	if length > 0 {
+		insideType = reflect.ValueOf(value.Index(0).Interface()).Type()
+	}
+	if insideType.Kind() == reflect.Map {
+		// this is considered as an array of tables
+		tablesArray := make([]*Tree, 0, length)
+		for i := 0; i < length; i++ {
+			table := value.Index(i)
+			tree, err := toTree(table.Interface())
+			if err != nil {
+				return nil, err
+			}
+			tablesArray = append(tablesArray, tree.(*Tree))
+		}
+		return tablesArray, nil
+	}
+
+	sliceType := typeFor(insideType.Kind())
+	if sliceType == nil {
+		sliceType = insideType
+	}
+
+	arrayValue := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, length)
+
+	for i := 0; i < length; i++ {
+		val := value.Index(i).Interface()
+		simpleValue, err := simpleValueCoercion(val)
+		if err != nil {
+			return nil, err
+		}
+		arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue))
+	}
+	return &tomlValue{value: arrayValue.Interface(), position: Position{}}, nil
+}
+
+func toTree(object interface{}) (interface{}, error) {
+	value := reflect.ValueOf(object)
+
+	if value.Kind() == reflect.Map {
+		values := map[string]interface{}{}
+		keys := value.MapKeys()
+		for _, key := range keys {
+			if key.Kind() != reflect.String {
+				if _, ok := key.Interface().(string); !ok {
+					return nil, fmt.Errorf("map key needs to be a string, not %T (%v)", key.Interface(), key.Kind())
+				}
+			}
+
+			v := value.MapIndex(key)
+			newValue, err := toTree(v.Interface())
+			if err != nil {
+				return nil, err
+			}
+			values[key.String()] = newValue
+		}
+		return &Tree{values: values, position: Position{}}, nil
+	}
+
+	if value.Kind() == reflect.Array || value.Kind() == reflect.Slice {
+		return sliceToTree(object)
+	}
+
+	simpleValue, err := simpleValueCoercion(object)
+	if err != nil {
+		return nil, err
+	}
+	return &tomlValue{value: simpleValue, position: Position{}}, nil
+}

tomltree_create_test.go

@@ -0,0 +1,126 @@
+package toml
+
+import (
+	"strconv"
+	"testing"
+	"time"
+)
+
+type customString string
+
+type stringer struct{}
+
+func (s stringer) String() string {
+	return "stringer"
+}
+
+func validate(t *testing.T, path string, object interface{}) {
+	switch o := object.(type) {
+	case *Tree:
+		for key, tree := range o.values {
+			validate(t, path+"."+key, tree)
+		}
+	case []*Tree:
+		for index, tree := range o {
+			validate(t, path+"."+strconv.Itoa(index), tree)
+		}
+	case *tomlValue:
+		switch o.value.(type) {
+		case int64, uint64, bool, string, float64, time.Time,
+			[]int64, []uint64, []bool, []string, []float64, []time.Time:
+		default:
+			t.Fatalf("tomlValue at key %s containing incorrect type %T", path, o.value)
+		}
+	default:
+		t.Fatalf("value at key %s is of incorrect type %T", path, object)
+	}
+	t.Logf("validation ok %s as %T", path, object)
+}
+
+func validateTree(t *testing.T, tree *Tree) {
+	validate(t, "", tree)
+}
+
+func TestTreeCreateToTree(t *testing.T) {
+	data := map[string]interface{}{
+		"a_string": "bar",
+		"an_int":   42,
+		"time":     time.Now(),
+		"int8":     int8(2),
+		"int16":    int16(2),
+		"int32":    int32(2),
+		"uint8":    uint8(2),
+		"uint16":   uint16(2),
+		"uint32":   uint32(2),
+		"float32":  float32(2),
+		"a_bool":   false,
+		"stringer": stringer{},
+		"nested": map[string]interface{}{
+			"foo": "bar",
+		},
+		"array":                 []string{"a", "b", "c"},
+		"array_uint":            []uint{uint(1), uint(2)},
+		"array_table":           []map[string]interface{}{{"sub_map": 52}},
+		"array_times":           []time.Time{time.Now(), time.Now()},
+		"map_times":             map[string]time.Time{"now": time.Now()},
+		"custom_string_map_key": map[customString]interface{}{customString("custom"): "custom"},
+	}
+	tree, err := TreeFromMap(data)
+	if err != nil {
+		t.Fatal("unexpected error:", err)
+	}
+	validateTree(t, tree)
+}
+
+func TestTreeCreateToTreeInvalidLeafType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": t})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidMapKeyType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": map[int]interface{}{2: 1}})
+	expected := "map key needs to be a string, not int (int)"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidArrayMemberType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": []*testing.T{t}})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestTreeCreateToTreeInvalidTableGroupType(t *testing.T) {
+	_, err := TreeFromMap(map[string]interface{}{"foo": []map[string]interface{}{{"hello": t}}})
+	expected := "cannot convert type *testing.T to Tree"
+	if err.Error() != expected {
+		t.Fatalf("expected error %s, got %s", expected, err.Error())
+	}
+}
+
+func TestRoundTripArrayOfTables(t *testing.T) {
+	orig := "\n[[stuff]]\n  name = \"foo\"\n  things = [\"a\",\"b\"]\n"
+	tree, err := Load(orig)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	m := tree.ToMap()
+
+	tree, err = TreeFromMap(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	want := orig
+	got := tree.String()
+
+	if got != want {
+		t.Errorf("want:\n%s\ngot:\n%s", want, got)
+	}
+}

tomltree_write.go

@@ -0,0 +1,333 @@
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Encodes a string to a TOML-compliant multi-line string value
+// This function is a clone of the existing encodeTomlString function, except that whitespace characters
+// are preserved. Quotation marks and backslashes are also not escaped.
+func encodeMultilineTomlString(value string) string {
+	var b bytes.Buffer
+
+	for _, rr := range value {
+		switch rr {
+		case '\b':
+			b.WriteString(`\b`)
+		case '\t':
+			b.WriteString("\t")
+		case '\n':
+			b.WriteString("\n")
+		case '\f':
+			b.WriteString(`\f`)
+		case '\r':
+			b.WriteString("\r")
+		case '"':
+			b.WriteString(`"`)
+		case '\\':
+			b.WriteString(`\`)
+		default:
+			intRr := uint16(rr)
+			if intRr < 0x001F {
+				b.WriteString(fmt.Sprintf("\\u%0.4X", intRr))
+			} else {
+				b.WriteRune(rr)
+			}
+		}
+	}
+	return b.String()
+}
+
+// Encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	var b bytes.Buffer
+
+	for _, rr := range value {
+		switch rr {
+		case '\b':
+			b.WriteString(`\b`)
+		case '\t':
+			b.WriteString(`\t`)
+		case '\n':
+			b.WriteString(`\n`)
+		case '\f':
+			b.WriteString(`\f`)
+		case '\r':
+			b.WriteString(`\r`)
+		case '"':
+			b.WriteString(`\"`)
+		case '\\':
+			b.WriteString(`\\`)
+		default:
+			intRr := uint16(rr)
+			if intRr < 0x001F {
+				b.WriteString(fmt.Sprintf("\\u%0.4X", intRr))
+			} else {
+				b.WriteRune(rr)
+			}
+		}
+	}
+	return b.String()
+}
+
+func tomlValueStringRepresentation(v interface{}, indent string, arraysOneElementPerLine bool) (string, error) {
+	// this interface check is added to dereference the change made in the writeTo function.
+	// That change was made to allow this function to see formatting options.
+	tv, ok := v.(*tomlValue)
+	if ok {
+		v = tv.value
+	} else {
+		tv = &tomlValue{}
+	}
+
+	switch value := v.(type) {
+	case uint64:
+		return strconv.FormatUint(value, 10), nil
+	case int64:
+		return strconv.FormatInt(value, 10), nil
+	case float64:
+		// Ensure a round float does contain a decimal point. Otherwise feeding
+		// the output back to the parser would convert to an integer.
+		if math.Trunc(value) == value {
+			return strings.ToLower(strconv.FormatFloat(value, 'f', 1, 32)), nil
+		}
+		return strings.ToLower(strconv.FormatFloat(value, 'f', -1, 32)), nil
+	case string:
+		if tv.multiline {
+			return "\"\"\"\n" + encodeMultilineTomlString(value) + "\"\"\"", nil
+		}
+		return "\"" + encodeTomlString(value) + "\"", nil
+	case []byte:
+		b, _ := v.([]byte)
+		return tomlValueStringRepresentation(string(b), indent, arraysOneElementPerLine)
+	case bool:
+		if value {
+			return "true", nil
+		}
+		return "false", nil
+	case time.Time:
+		return value.Format(time.RFC3339), nil
+	case nil:
+		return "", nil
+	}
+
+	rv := reflect.ValueOf(v)
+
+	if rv.Kind() == reflect.Slice {
+		var values []string
+		for i := 0; i < rv.Len(); i++ {
+			item := rv.Index(i).Interface()
+			itemRepr, err := tomlValueStringRepresentation(item, indent, arraysOneElementPerLine)
+			if err != nil {
+				return "", err
+			}
+			values = append(values, itemRepr)
+		}
+		if arraysOneElementPerLine && len(values) > 1 {
+			stringBuffer := bytes.Buffer{}
+			valueIndent := indent + `  ` // TODO: move that to a shared encoder state
+
+			stringBuffer.WriteString("[\n")
+
+			for _, value := range values {
+				stringBuffer.WriteString(valueIndent)
+				stringBuffer.WriteString(value)
+				stringBuffer.WriteString(`,`)
+				stringBuffer.WriteString("\n")
+			}
+
+			stringBuffer.WriteString(indent + "]")
+
+			return stringBuffer.String(), nil
+		}
+		return "[" + strings.Join(values, ",") + "]", nil
+	}
+	return "", fmt.Errorf("unsupported value type %T: %v", v, v)
+}
+
+func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64, arraysOneElementPerLine bool) (int64, error) {
+	simpleValuesKeys := make([]string, 0)
+	complexValuesKeys := make([]string, 0)
+
+	for k := range t.values {
+		v := t.values[k]
+		switch v.(type) {
+		case *Tree, []*Tree:
+			complexValuesKeys = append(complexValuesKeys, k)
+		default:
+			simpleValuesKeys = append(simpleValuesKeys, k)
+		}
+	}
+
+	sort.Strings(simpleValuesKeys)
+	sort.Strings(complexValuesKeys)
+
+	for _, k := range simpleValuesKeys {
+		v, ok := t.values[k].(*tomlValue)
+		if !ok {
+			return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k])
+		}
+
+		repr, err := tomlValueStringRepresentation(v, indent, arraysOneElementPerLine)
+		if err != nil {
+			return bytesCount, err
+		}
+
+		if v.comment != "" {
+			comment := strings.Replace(v.comment, "\n", "\n"+indent+"#", -1)
+			start := "# "
+			if strings.HasPrefix(comment, "#") {
+				start = ""
+			}
+			writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment, "\n")
+			bytesCount += int64(writtenBytesCountComment)
+			if errc != nil {
+				return bytesCount, errc
+			}
+		}
+
+		var commented string
+		if v.commented {
+			commented = "# "
+		}
+		writtenBytesCount, err := writeStrings(w, indent, commented, k, " = ", repr, "\n")
+		bytesCount += int64(writtenBytesCount)
+		if err != nil {
+			return bytesCount, err
+		}
+	}
+
+	for _, k := range complexValuesKeys {
+		v := t.values[k]
+
+		combinedKey := k
+		if keyspace != "" {
+			combinedKey = keyspace + "." + combinedKey
+		}
+		var commented string
+		if t.commented {
+			commented = "# "
+		}
+
+		switch node := v.(type) {
+		// node has to be of those two types given how keys are sorted above
+		case *Tree:
+			tv, ok := t.values[k].(*Tree)
+			if !ok {
+				return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k])
+			}
+			if tv.comment != "" {
+				comment := strings.Replace(tv.comment, "\n", "\n"+indent+"#", -1)
+				start := "# "
+				if strings.HasPrefix(comment, "#") {
+					start = ""
+				}
+				writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment)
+				bytesCount += int64(writtenBytesCountComment)
+				if errc != nil {
+					return bytesCount, errc
+				}
+			}
+			writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[", combinedKey, "]\n")
+			bytesCount += int64(writtenBytesCount)
+			if err != nil {
+				return bytesCount, err
+			}
+			bytesCount, err = node.writeTo(w, indent+"  ", combinedKey, bytesCount, arraysOneElementPerLine)
+			if err != nil {
+				return bytesCount, err
+			}
+		case []*Tree:
+			for _, subTree := range node {
+				writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[[", combinedKey, "]]\n")
+				bytesCount += int64(writtenBytesCount)
+				if err != nil {
+					return bytesCount, err
+				}
+
+				bytesCount, err = subTree.writeTo(w, indent+"  ", combinedKey, bytesCount, arraysOneElementPerLine)
+				if err != nil {
+					return bytesCount, err
+				}
+			}
+		}
+	}
+
+	return bytesCount, nil
+}
+
+func writeStrings(w io.Writer, s ...string) (int, error) {
+	var n int
+	for i := range s {
+		b, err := io.WriteString(w, s[i])
+		n += b
+		if err != nil {
+			return n, err
+		}
+	}
+	return n, nil
+}
+
+// WriteTo encode the Tree as Toml and writes it to the writer w.
+// Returns the number of bytes written in case of success, or an error if anything happened.
+func (t *Tree) WriteTo(w io.Writer) (int64, error) {
+	return t.writeTo(w, "", "", 0, false)
+}
+
+// ToTomlString generates a human-readable representation of the current tree.
+// Output spans multiple lines, and is suitable for ingest by a TOML parser.
+// If the conversion cannot be performed, ToString returns a non-nil error.
+func (t *Tree) ToTomlString() (string, error) {
+	var buf bytes.Buffer
+	_, err := t.WriteTo(&buf)
+	if err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// String generates a human-readable representation of the current tree.
+// Alias of ToString. Present to implement the fmt.Stringer interface.
+func (t *Tree) String() string {
+	result, _ := t.ToTomlString()
+	return result
+}
+
+// ToMap recursively generates a representation of the tree using Go built-in structures.
+// The following types are used:
+//
+//	* bool
+//	* float64
+//	* int64
+//	* string
+//	* uint64
+//	* time.Time
+//	* map[string]interface{} (where interface{} is any of this list)
+//	* []interface{} (where interface{} is any of this list)
+func (t *Tree) ToMap() map[string]interface{} {
+	result := map[string]interface{}{}
+
+	for k, v := range t.values {
+		switch node := v.(type) {
+		case []*Tree:
+			var array []interface{}
+			for _, item := range node {
+				array = append(array, item.ToMap())
+			}
+			result[k] = array
+		case *Tree:
+			result[k] = node.ToMap()
+		case *tomlValue:
+			result[k] = node.value
+		}
+	}
+	return result
+}

tomltree_write_test.go

@@ -0,0 +1,376 @@
+package toml
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+type failingWriter struct {
+	failAt  int
+	written int
+	buffer  bytes.Buffer
+}
+
+func (f *failingWriter) Write(p []byte) (n int, err error) {
+	count := len(p)
+	toWrite := f.failAt - (count + f.written)
+	if toWrite < 0 {
+		toWrite = 0
+	}
+	if toWrite > count {
+		f.written += count
+		f.buffer.Write(p)
+		return count, nil
+	}
+
+	f.buffer.Write(p[:toWrite])
+	f.written = f.failAt
+	return toWrite, fmt.Errorf("failingWriter failed after writing %d bytes", f.written)
+}
+
+func assertErrorString(t *testing.T, expected string, err error) {
+	expectedErr := errors.New(expected)
+	if err == nil || err.Error() != expectedErr.Error() {
+		t.Errorf("expecting error %s, but got %s instead", expected, err)
+	}
+}
+
+func TestTreeWriteToEmptyTable(t *testing.T) {
+	doc := `[[empty-tables]]
+[[empty-tables]]`
+
+	toml, err := Load(doc)
+	if err != nil {
+		t.Fatal("Unexpected Load error:", err)
+	}
+	tomlString, err := toml.ToTomlString()
+	if err != nil {
+		t.Fatal("Unexpected ToTomlString error:", err)
+	}
+
+	expected := `
+[[empty-tables]]
+
+[[empty-tables]]
+`
+
+	if tomlString != expected {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, tomlString)
+	}
+}
+
+func TestTreeWriteToTomlString(t *testing.T) {
+	toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" }
+points = { x = 1, y = 2 }`)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	tomlString, _ := toml.ToTomlString()
+	reparsedTree, err := Load(tomlString)
+
+	assertTree(t, reparsedTree, err, map[string]interface{}{
+		"name": map[string]interface{}{
+			"first": "Tom",
+			"last":  "Preston-Werner",
+		},
+		"points": map[string]interface{}{
+			"x": int64(1),
+			"y": int64(2),
+		},
+	})
+}
+
+func TestTreeWriteToTomlStringSimple(t *testing.T) {
+	tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n")
+	if err != nil {
+		t.Errorf("Test failed to parse: %v", err)
+		return
+	}
+	result, err := tree.ToTomlString()
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err)
+	}
+	expected := "\n[foo]\n\n  [[foo.bar]]\n    a = 42\n\n  [[foo.bar]]\n    a = 69\n"
+	if result != expected {
+		t.Errorf("Expected got '%s', expected '%s'", result, expected)
+	}
+}
+
+func TestTreeWriteToTomlStringKeysOrders(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		tree, _ := Load(`
+		foobar = true
+		bar = "baz"
+		foo = 1
+		[qux]
+		  foo = 1
+		  bar = "baz2"`)
+
+		stringRepr, _ := tree.ToTomlString()
+
+		t.Log("Intermediate string representation:")
+		t.Log(stringRepr)
+
+		r := strings.NewReader(stringRepr)
+		toml, err := LoadReader(r)
+
+		if err != nil {
+			t.Fatal("Unexpected error:", err)
+		}
+
+		assertTree(t, toml, err, map[string]interface{}{
+			"foobar": true,
+			"bar":    "baz",
+			"foo":    1,
+			"qux": map[string]interface{}{
+				"foo": 1,
+				"bar": "baz2",
+			},
+		})
+	}
+}
+
+func testMaps(t *testing.T, actual, expected map[string]interface{}) {
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatal("trees aren't equal.\n", "Expected:\n", expected, "\nActual:\n", actual)
+	}
+}
+
+func TestTreeWriteToMapSimple(t *testing.T) {
+	tree, _ := Load("a = 42\nb = 17")
+
+	expected := map[string]interface{}{
+		"a": int64(42),
+		"b": int64(17),
+	}
+
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTreeWriteToInvalidTreeSimpleValue(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": int8(1)}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "invalid value type at foo: int8", err)
+}
+
+func TestTreeWriteToInvalidTreeTomlValue(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "unsupported value type int8: 1", err)
+}
+
+func TestTreeWriteToInvalidTreeTomlValueArray(t *testing.T) {
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}}
+	_, err := tree.ToTomlString()
+	assertErrorString(t, "unsupported value type int8: 1", err)
+}
+
+func TestTreeWriteToFailingWriterInSimpleValue(t *testing.T) {
+	toml, _ := Load(`a = 2`)
+	writer := failingWriter{failAt: 0, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writing 0 bytes", err)
+}
+
+func TestTreeWriteToFailingWriterInTable(t *testing.T) {
+	toml, _ := Load(`
+[b]
+a = 2`)
+	writer := failingWriter{failAt: 2, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writing 2 bytes", err)
+
+	writer = failingWriter{failAt: 13, written: 0}
+	_, err = toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writing 13 bytes", err)
+}
+
+func TestTreeWriteToFailingWriterInArray(t *testing.T) {
+	toml, _ := Load(`
+[[b]]
+a = 2`)
+	writer := failingWriter{failAt: 2, written: 0}
+	_, err := toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writing 2 bytes", err)
+
+	writer = failingWriter{failAt: 15, written: 0}
+	_, err = toml.WriteTo(&writer)
+	assertErrorString(t, "failingWriter failed after writing 15 bytes", err)
+}
+
+func TestTreeWriteToMapExampleFile(t *testing.T) {
+	tree, _ := LoadFile("example.toml")
+	expected := map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []interface{}{int64(8001), int64(8001), int64(8002)},
+			"connection_max": int64(5000),
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]interface{}{"gamma", "delta"},
+				[]interface{}{int64(1), int64(2)},
+			},
+		},
+	}
+	testMaps(t, tree.ToMap(), expected)
+}
+
+func TestTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) {
+	tree, _ := Load(`
+	[[menu.main]]
+        a = "menu 1"
+        b = "menu 2"
+        [[menu.main]]
+        c = "menu 3"
+        d = "menu 4"`)
+	expected := map[string]interface{}{
+		"menu": map[string]interface{}{
+			"main": []interface{}{
+				map[string]interface{}{"a": "menu 1", "b": "menu 2"},
+				map[string]interface{}{"c": "menu 3", "d": "menu 4"},
+			},
+		},
+	}
+	treeMap := tree.ToMap()
+
+	testMaps(t, treeMap, expected)
+}
+
+func TestTreeWriteToMapWithArrayOfInlineTables(t *testing.T) {
+	tree, _ := Load(`
+    	[params]
+	language_tabs = [
+    		{ key = "shell", name = "Shell" },
+    		{ key = "ruby", name = "Ruby" },
+    		{ key = "python", name = "Python" }
+	]`)
+
+	expected := map[string]interface{}{
+		"params": map[string]interface{}{
+			"language_tabs": []interface{}{
+				map[string]interface{}{
+					"key":  "shell",
+					"name": "Shell",
+				},
+				map[string]interface{}{
+					"key":  "ruby",
+					"name": "Ruby",
+				},
+				map[string]interface{}{
+					"key":  "python",
+					"name": "Python",
+				},
+			},
+		},
+	}
+
+	treeMap := tree.ToMap()
+	testMaps(t, treeMap, expected)
+}
+
+func TestTreeWriteToFloat(t *testing.T) {
+	tree, err := Load(`a = 3.0`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	str, err := tree.ToTomlString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `a = 3.0`
+	if strings.TrimSpace(str) != strings.TrimSpace(expected) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str)
+	}
+}
+
+func TestTreeWriteToSpecialFloat(t *testing.T) {
+	expected := `a = +inf
+b = -inf
+c = nan`
+
+	tree, err := Load(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	str, err := tree.ToTomlString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.TrimSpace(str) != strings.TrimSpace(expected) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str)
+	}
+}
+
+func BenchmarkTreeToTomlString(b *testing.B) {
+	toml, err := Load(sampleHard)
+	if err != nil {
+		b.Fatal("Unexpected error:", err)
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, err := toml.ToTomlString()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+var sampleHard = `# Test file for TOML
+# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate
+# This part you'll really hate
+
+[the]
+test_string = "You'll hate me after this - #"          # " Annoying, isn't it?
+
+    [the.hard]
+    test_array = [ "] ", " # "]      # ] There you go, parse this!
+    test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ]
+    # You didn't think it'd as easy as chucking out the last #, did you?
+    another_test_string = " Same thing, but with a string #"
+    harder_test_string = " And when \"'s are in the string, along with # \""   # "and comments are there too"
+    # Things will get harder
+
+        [the.hard."bit#"]
+        "what?" = "You don't think some user won't do that?"
+        multi_line_array = [
+            "]",
+            # ] Oh yes I did
+            ]
+
+# Each of the following keygroups/key value pairs should produce an error. Uncomment to them to test
+
+#[error]   if you didn't catch this, your parser is broken
+#string = "Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment"   like this
+#array = [
+#         "This might most likely happen in multiline arrays",
+#         Like here,
+#         "or here,
+#         and here"
+#         ]     End of array comment, forgot the #
+#number = 3.14  pi <--again forgot the #         `