Multiline Marshal tag (#221)

The new multiline tag works just like the existing 'commented' tag (i.e.
`multiline:"true"`), and tells go-toml to marshal the value as a
multi-line string. The tag currently has no impact on any non-string
fields.

.gitignore

@@ -1 +1,2 @@
 test_program/test_program_bin
+fuzz/

.travis.yml

@@ -1,16 +1,18 @@
 sudo: false
 language: go
 go:
-  - 1.6.4
+  - 1.8.x
-  - 1.7.5
+  - 1.9.x
-  - 1.8
+  - 1.10.x
   - tip
 matrix:
   allow_failures:
     - go: tip
   fast_finish: true
 script:
+  - if [ -n "$(go fmt ./...)" ]; then exit 1; fi
   - ./test.sh
+  - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git
 before_install:
   - go get github.com/axw/gocov/gocov
   - go get github.com/mattn/goveralls

README.md

@@ -33,7 +33,7 @@ import "github.com/pelletier/go-toml"
 Read a TOML document:
 
 ```go
-config, _ := toml.LoadString(`
+config, _ := toml.Load(`
 [postgres]
 user = "pelletier"
 password = "mypassword"`)
@@ -42,7 +42,7 @@ user := config.Get("postgres.user").(string)
 
 // or using an intermediate object
 postgresConfig := config.Get("postgres").(*toml.Tree)
-password = postgresConfig.Get("password").(string)
+password := postgresConfig.Get("password").(string)
 ```
 
 Or use Unmarshal:
@@ -57,12 +57,12 @@ type Config struct {
 }
 
 doc := []byte(`
-[postgres]
+[Postgres]
-user = "pelletier"
+User = "pelletier"
-password = "mypassword"`)
+Password = "mypassword"`)
 
 config := Config{}
-Unmarshal(doc, &config)
+toml.Unmarshal(doc, &config)
 fmt.Println("user=", config.Postgres.User)
 ```
 
@@ -70,7 +70,8 @@ Or use a query:
 
 ```go
 // use a query to gather elements without walking the tree
-results, _ := config.Query("$..[user,password]")
+q, _ := query.Compile("$..[user,password]")
+results := q.Execute(config)
 for ii, item := range results.Values() {
     fmt.Println("Query result %d: %v", ii, item)
 }
@@ -113,6 +114,18 @@ You have to make sure two kind of tests run:
 
 You can run both of them using `./test.sh`.
 
+### Fuzzing
+
+The script `./fuzz.sh` is available to
+run [go-fuzz](https://github.com/dvyukov/go-fuzz) on go-toml.
+
+## Versioning
+
+Go-toml follows [Semantic Versioning](http://semver.org/). The supported version
+of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of
+this document. The last two major versions of Go are supported
+(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)).
+
 ## License
 
 The MIT License (MIT). Read [LICENSE](LICENSE).

benchmark.json

@@ -0,0 +1,164 @@
+{
+    "array": {
+        "key1": [
+            1,
+            2,
+            3
+        ],
+        "key2": [
+            "red",
+            "yellow",
+            "green"
+        ],
+        "key3": [
+            [
+                1,
+                2
+            ],
+            [
+                3,
+                4,
+                5
+            ]
+        ],
+        "key4": [
+            [
+                1,
+                2
+            ],
+            [
+                "a",
+                "b",
+                "c"
+            ]
+        ],
+        "key5": [
+            1,
+            2,
+            3
+        ],
+        "key6": [
+            1,
+            2
+        ]
+    },
+    "boolean": {
+        "False": false,
+        "True": true
+    },
+    "datetime": {
+        "key1": "1979-05-27T07:32:00Z",
+        "key2": "1979-05-27T00:32:00-07:00",
+        "key3": "1979-05-27T00:32:00.999999-07:00"
+    },
+    "float": {
+        "both": {
+            "key": 6.626e-34
+        },
+        "exponent": {
+            "key1": 5e+22,
+            "key2": 1000000,
+            "key3": -0.02
+        },
+        "fractional": {
+            "key1": 1,
+            "key2": 3.1415,
+            "key3": -0.01
+        },
+        "underscores": {
+            "key1": 9224617.445991227,
+            "key2": 1e+100
+        }
+    },
+    "fruit": [{
+            "name": "apple",
+            "physical": {
+                "color": "red",
+                "shape": "round"
+            },
+            "variety": [{
+                    "name": "red delicious"
+                },
+                {
+                    "name": "granny smith"
+                }
+            ]
+        },
+        {
+            "name": "banana",
+            "variety": [{
+                "name": "plantain"
+            }]
+        }
+    ],
+    "integer": {
+        "key1": 99,
+        "key2": 42,
+        "key3": 0,
+        "key4": -17,
+        "underscores": {
+            "key1": 1000,
+            "key2": 5349221,
+            "key3": 12345
+        }
+    },
+    "products": [{
+            "name": "Hammer",
+            "sku": 738594937
+        },
+        {},
+        {
+            "color": "gray",
+            "name": "Nail",
+            "sku": 284758393
+        }
+    ],
+    "string": {
+        "basic": {
+            "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+        },
+        "literal": {
+            "multiline": {
+                "lines": "The first newline is\ntrimmed in raw strings.\n   All other whitespace\n   is preserved.\n",
+                "regex2": "I [dw]on't need \\d{2} apples"
+            },
+            "quoted": "Tom \"Dubs\" Preston-Werner",
+            "regex": "\u003c\\i\\c*\\s*\u003e",
+            "winpath": "C:\\Users\\nodejs\\templates",
+            "winpath2": "\\\\ServerX\\admin$\\system32\\"
+        },
+        "multiline": {
+            "continued": {
+                "key1": "The quick brown fox jumps over the lazy dog.",
+                "key2": "The quick brown fox jumps over the lazy dog.",
+                "key3": "The quick brown fox jumps over the lazy dog."
+            },
+            "key1": "One\nTwo",
+            "key2": "One\nTwo",
+            "key3": "One\nTwo"
+        }
+    },
+    "table": {
+        "inline": {
+            "name": {
+                "first": "Tom",
+                "last": "Preston-Werner"
+            },
+            "point": {
+                "x": 1,
+                "y": 2
+            }
+        },
+        "key": "value",
+        "subtable": {
+            "key": "another value"
+        }
+    },
+    "x": {
+        "y": {
+            "z": {
+                "w": {}
+            }
+        }
+    }
+}

benchmark.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+reference_ref=${1:-master}
+reference_git=${2:-.}
+
+if ! `hash benchstat 2>/dev/null`; then
+    echo "Installing benchstat"
+    go get golang.org/x/perf/cmd/benchstat
+    go install golang.org/x/perf/cmd/benchstat
+fi
+
+tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX`
+ref_tempdir="${tempdir}/ref"
+ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt"
+local_benchmark="`pwd`/benchmark-local.txt"
+
+echo "=== ${reference_ref} (${ref_tempdir})"
+git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null
+pushd ${ref_tempdir} >/dev/null
+git checkout ${reference_ref} >/dev/null 2>/dev/null
+go test -bench=. -benchmem | tee ${ref_benchmark}
+popd >/dev/null
+
+echo ""
+echo "=== local"
+go test -bench=. -benchmem  | tee ${local_benchmark}
+
+echo ""
+echo "=== diff"
+benchstat -delta-test=none ${ref_benchmark} ${local_benchmark}

benchmark.toml

@@ -0,0 +1,244 @@
+################################################################################
+## Comment
+
+# Speak your mind with the hash symbol. They go from the symbol to the end of
+# the line.
+
+
+################################################################################
+## Table
+
+# Tables (also known as hash tables or dictionaries) are collections of
+# key/value pairs. They appear in square brackets on a line by themselves.
+
+[table]
+
+key = "value" # Yeah, you can do this.
+
+# Nested tables are denoted by table names with dots in them. Name your tables
+# whatever crap you please, just don't use #, ., [ or ].
+
+[table.subtable]
+
+key = "another value"
+
+# You don't need to specify all the super-tables if you don't want to. TOML
+# knows how to do it for you.
+
+# [x] you
+# [x.y] don't
+# [x.y.z] need these
+[x.y.z.w] # for this to work
+
+
+################################################################################
+## Inline Table
+
+# Inline tables provide a more compact syntax for expressing tables. They are
+# especially useful for grouped data that can otherwise quickly become verbose.
+# Inline tables are enclosed in curly braces `{` and `}`. No newlines are
+# allowed between the curly braces unless they are valid within a value.
+
+[table.inline]
+
+name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }
+
+
+################################################################################
+## String
+
+# There are four ways to express strings: basic, multi-line basic, literal, and
+# multi-line literal. All strings must contain only valid UTF-8 characters.
+
+[string.basic]
+
+basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+[string.multiline]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "One\nTwo"
+key2 = """One\nTwo"""
+key3 = """
+One
+Two"""
+
+[string.multiline.continued]
+
+# The following strings are byte-for-byte equivalent:
+key1 = "The quick brown fox jumps over the lazy dog."
+
+key2 = """
+The quick brown \
+
+
+  fox jumps over \
+    the lazy dog."""
+
+key3 = """\
+       The quick brown \
+       fox jumps over \
+       the lazy dog.\
+       """
+
+[string.literal]
+
+# What you see is what you get.
+winpath  = 'C:\Users\nodejs\templates'
+winpath2 = '\\ServerX\admin$\system32\'
+quoted   = 'Tom "Dubs" Preston-Werner'
+regex    = '<\i\c*\s*>'
+
+
+[string.literal.multiline]
+
+regex2 = '''I [dw]on't need \d{2} apples'''
+lines  = '''
+The first newline is
+trimmed in raw strings.
+   All other whitespace
+   is preserved.
+'''
+
+
+################################################################################
+## Integer
+
+# Integers are whole numbers. Positive numbers may be prefixed with a plus sign.
+# Negative numbers are prefixed with a minus sign.
+
+[integer]
+
+key1 = +99
+key2 = 42
+key3 = 0
+key4 = -17
+
+[integer.underscores]
+
+# For large numbers, you may use underscores to enhance readability. Each
+# underscore must be surrounded by at least one digit.
+key1 = 1_000
+key2 = 5_349_221
+key3 = 1_2_3_4_5     # valid but inadvisable
+
+
+################################################################################
+## Float
+
+# A float consists of an integer part (which may be prefixed with a plus or
+# minus sign) followed by a fractional part and/or an exponent part.
+
+[float.fractional]
+
+key1 = +1.0
+key2 = 3.1415
+key3 = -0.01
+
+[float.exponent]
+
+key1 = 5e+22
+key2 = 1e6
+key3 = -2E-2
+
+[float.both]
+
+key = 6.626e-34
+
+[float.underscores]
+
+key1 = 9_224_617.445_991_228_313
+key2 = 1e1_00
+
+
+################################################################################
+## Boolean
+
+# Booleans are just the tokens you're used to. Always lowercase.
+
+[boolean]
+
+True = true
+False = false
+
+
+################################################################################
+## Datetime
+
+# Datetimes are RFC 3339 dates.
+
+[datetime]
+
+key1 = 1979-05-27T07:32:00Z
+key2 = 1979-05-27T00:32:00-07:00
+key3 = 1979-05-27T00:32:00.999999-07:00
+
+
+################################################################################
+## Array
+
+# Arrays are square brackets with other primitives inside. Whitespace is
+# ignored. Elements are separated by commas. Data types may not be mixed.
+
+[array]
+
+key1 = [ 1, 2, 3 ]
+key2 = [ "red", "yellow", "green" ]
+key3 = [ [ 1, 2 ], [3, 4, 5] ]
+#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok
+
+# Arrays can also be multiline. So in addition to ignoring whitespace, arrays
+# also ignore newlines between the brackets.  Terminating commas are ok before
+# the closing bracket.
+
+key5 = [
+  1, 2, 3
+]
+key6 = [
+  1,
+  2, # this is ok
+]
+
+
+################################################################################
+## Array of Tables
+
+# These can be expressed by using a table name in double brackets. Each table
+# with the same double bracketed name will be an element in the array. The
+# tables are inserted in the order encountered.
+
+[[products]]
+
+name = "Hammer"
+sku = 738594937
+
+[[products]]
+
+[[products]]
+
+name = "Nail"
+sku = 284758393
+color = "gray"
+
+
+# You can create nested arrays of tables as well.
+
+[[fruit]]
+  name = "apple"
+
+  [fruit.physical]
+    color = "red"
+    shape = "round"
+
+  [[fruit.variety]]
+    name = "red delicious"
+
+  [[fruit.variety]]
+    name = "granny smith"
+
+[[fruit]]
+  name = "banana"
+
+  [[fruit.variety]]
+    name = "plantain"

benchmark.yml

@@ -0,0 +1,121 @@
+---
+array:
+  key1:
+  - 1
+  - 2
+  - 3
+  key2:
+  - red
+  - yellow
+  - green
+  key3:
+  - - 1
+    - 2
+  - - 3
+    - 4
+    - 5
+  key4:
+  - - 1
+    - 2
+  - - a
+    - b
+    - c
+  key5:
+  - 1
+  - 2
+  - 3
+  key6:
+  - 1
+  - 2
+boolean:
+  'False': false
+  'True': true
+datetime:
+  key1: '1979-05-27T07:32:00Z'
+  key2: '1979-05-27T00:32:00-07:00'
+  key3: '1979-05-27T00:32:00.999999-07:00'
+float:
+  both:
+    key: 6.626e-34
+  exponent:
+    key1: 5.0e+22
+    key2: 1000000
+    key3: -0.02
+  fractional:
+    key1: 1
+    key2: 3.1415
+    key3: -0.01
+  underscores:
+    key1: 9224617.445991227
+    key2: 1.0e+100
+fruit:
+- name: apple
+  physical:
+    color: red
+    shape: round
+  variety:
+  - name: red delicious
+  - name: granny smith
+- name: banana
+  variety:
+  - name: plantain
+integer:
+  key1: 99
+  key2: 42
+  key3: 0
+  key4: -17
+  underscores:
+    key1: 1000
+    key2: 5349221
+    key3: 12345
+products:
+- name: Hammer
+  sku: 738594937
+- {}
+- color: gray
+  name: Nail
+  sku: 284758393
+string:
+  basic:
+    basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."
+  literal:
+    multiline:
+      lines: |
+        The first newline is
+        trimmed in raw strings.
+           All other whitespace
+           is preserved.
+      regex2: I [dw]on't need \d{2} apples
+    quoted: Tom "Dubs" Preston-Werner
+    regex: "<\\i\\c*\\s*>"
+    winpath: C:\Users\nodejs\templates
+    winpath2: "\\\\ServerX\\admin$\\system32\\"
+  multiline:
+    continued:
+      key1: The quick brown fox jumps over the lazy dog.
+      key2: The quick brown fox jumps over the lazy dog.
+      key3: The quick brown fox jumps over the lazy dog.
+    key1: |-
+      One
+      Two
+    key2: |-
+      One
+      Two
+    key3: |-
+      One
+      Two
+table:
+  inline:
+    name:
+      first: Tom
+      last: Preston-Werner
+    point:
+      x: 1
+      y: 2
+  key: value
+  subtable:
+    key: another value
+x:
+  y:
+    z:
+      w: {}

benchmark_test.go

@@ -0,0 +1,192 @@
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	burntsushi "github.com/BurntSushi/toml"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type benchmarkDoc struct {
+	Table struct {
+		Key      string
+		Subtable struct {
+			Key string
+		}
+		Inline struct {
+			Name struct {
+				First string
+				Last  string
+			}
+			Point struct {
+				X int64
+				U int64
+			}
+		}
+	}
+	String struct {
+		Basic struct {
+			Basic string
+		}
+		Multiline struct {
+			Key1      string
+			Key2      string
+			Key3      string
+			Continued struct {
+				Key1 string
+				Key2 string
+				Key3 string
+			}
+		}
+		Literal struct {
+			Winpath   string
+			Winpath2  string
+			Quoted    string
+			Regex     string
+			Multiline struct {
+				Regex2 string
+				Lines  string
+			}
+		}
+	}
+	Integer struct {
+		Key1        int64
+		Key2        int64
+		Key3        int64
+		Key4        int64
+		Underscores struct {
+			Key1 int64
+			Key2 int64
+			Key3 int64
+		}
+	}
+	Float struct {
+		Fractional struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Exponent struct {
+			Key1 float64
+			Key2 float64
+			Key3 float64
+		}
+		Both struct {
+			Key float64
+		}
+		Underscores struct {
+			Key1 float64
+			Key2 float64
+		}
+	}
+	Boolean struct {
+		True  bool
+		False bool
+	}
+	Datetime struct {
+		Key1 time.Time
+		Key2 time.Time
+		Key3 time.Time
+	}
+	Array struct {
+		Key1 []int64
+		Key2 []string
+		Key3 [][]int64
+		// TODO: Key4 not supported by go-toml's Unmarshal
+		Key5 []int64
+		Key6 []int64
+	}
+	Products []struct {
+		Name  string
+		Sku   int64
+		Color string
+	}
+	Fruit []struct {
+		Name     string
+		Physical struct {
+			Color   string
+			Shape   string
+			Variety []struct {
+				Name string
+			}
+		}
+	}
+}
+
+func BenchmarkParseToml(b *testing.B) {
+	fileBytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := LoadReader(bytes.NewReader(fileBytes))
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalBurntSushiToml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := burntsushi.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalJson(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := json.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshalYaml(b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.yml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		target := benchmarkDoc{}
+		err := yaml.Unmarshal(bytes, &target)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/tomljson/main.go

@@ -17,13 +17,12 @@ import (
 
 func main() {
 	flag.Usage = func() {
-		fmt.Fprintln(os.Stderr, `tomljson can be used in two ways:
+		fmt.Fprintln(os.Stderr, "tomljson can be used in two ways:")
-Writing to STDIN and reading from STDOUT:
+		fmt.Fprintln(os.Stderr, "Writing to STDIN and reading from STDOUT:")
-  cat file.toml | tomljson > file.json
+		fmt.Fprintln(os.Stderr, "  cat file.toml | tomljson > file.json")
-
+		fmt.Fprintln(os.Stderr, "")
-Reading from a file name:
+		fmt.Fprintln(os.Stderr, "Reading from a file name:")
-  tomljson file.toml
+		fmt.Fprintln(os.Stderr, "  tomljson file.toml")
-`)
 	}
 	flag.Parse()
 	os.Exit(processMain(flag.Args(), os.Stdin, os.Stdout, os.Stderr))

cmd/tomll/main.go

@@ -17,15 +17,14 @@ import (
 
 func main() {
 	flag.Usage = func() {
-		fmt.Fprintln(os.Stderr, `tomll can be used in two ways:
+		fmt.Fprintln(os.Stderr, "tomll can be used in two ways:")
-Writing to STDIN and reading from STDOUT:
+		fmt.Fprintln(os.Stderr, "Writing to STDIN and reading from STDOUT:")
-  cat file.toml | tomll > file.toml
+		fmt.Fprintln(os.Stderr, "  cat file.toml | tomll > file.toml")
-
+		fmt.Fprintln(os.Stderr, "")
-Reading and updating a list of files:
+		fmt.Fprintln(os.Stderr, "Reading and updating a list of files:")
-  tomll a.toml b.toml c.toml
+		fmt.Fprintln(os.Stderr, "  tomll a.toml b.toml c.toml")
-
+		fmt.Fprintln(os.Stderr, "")
-When given a list of files, tomll will modify all files in place without asking.
+		fmt.Fprintln(os.Stderr, "When given a list of files, tomll will modify all files in place without asking.")
-`)
 	}
 	flag.Parse()
 	// read from stdin and print to stdout

doc.go

@@ -17,7 +17,7 @@
 // JSONPath-like queries
 //
 // The package github.com/pelletier/go-toml/query implements a system
-// similar to JSONPath to quickly retrive elements of a TOML document using a
+// similar to JSONPath to quickly retrieve elements of a TOML document using a
 // single expression. See the package documentation for more information.
 //
 package toml

doc_test.go

@@ -1,13 +1,16 @@
 // code examples for godoc
 
-package toml
+package toml_test
 
 import (
 	"fmt"
+	"log"
+
+	toml "github.com/pelletier/go-toml"
 )
 
 func Example_tree() {
-	config, err := LoadFile("config.toml")
+	config, err := toml.LoadFile("config.toml")
 
 	if err != nil {
 		fmt.Println("Error ", err.Error())
@@ -17,7 +20,7 @@ func Example_tree() {
 		password := config.Get("postgres.password").(string)
 
 		// or using an intermediate object
-		configTree := config.Get("postgres").(*Tree)
+		configTree := config.Get("postgres").(*toml.Tree)
 		user = configTree.Get("user").(string)
 		password = configTree.Get("password").(string)
 		fmt.Println("User is", user, " and password is", password)
@@ -48,6 +51,55 @@ func Example_unmarshal() {
 	`)
 
 	person := Person{}
-	Unmarshal(document, &person)
+	toml.Unmarshal(document, &person)
 	fmt.Println(person.Name, "is", person.Age, "and works at", person.Employer.Name)
+	// Output:
+	// John is 30 and works at Company Inc.
+}
+
+func ExampleMarshal() {
+	type Postgres struct {
+		User     string `toml:"user"`
+		Password string `toml:"password"`
+		Database string `toml:"db" commented:"true" comment:"not used anymore"`
+	}
+	type Config struct {
+		Postgres Postgres `toml:"postgres" comment:"Postgres configuration"`
+	}
+
+	config := Config{Postgres{User: "pelletier", Password: "mypassword", Database: "old_database"}}
+	b, err := toml.Marshal(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b))
+	// Output:
+	// # Postgres configuration
+	// [postgres]
+	//
+	//   # not used anymore
+	//   # db = "old_database"
+	//   password = "mypassword"
+	//   user = "pelletier"
+}
+
+func ExampleUnmarshal() {
+	type Postgres struct {
+		User     string
+		Password string
+	}
+	type Config struct {
+		Postgres Postgres
+	}
+
+	doc := []byte(`
+	[postgres]
+	user = "pelletier"
+	password = "mypassword"`)
+
+	config := Config{}
+	toml.Unmarshal(doc, &config)
+	fmt.Println("user=", config.Postgres.User)
+	// Output:
+	// user= pelletier
 }

fuzz.go

@@ -0,0 +1,31 @@
+// +build gofuzz
+
+package toml
+
+func Fuzz(data []byte) int {
+	tree, err := LoadBytes(data)
+	if err != nil {
+		if tree != nil {
+			panic("tree must be nil if there is an error")
+		}
+		return 0
+	}
+
+	str, err := tree.ToTomlString()
+	if err != nil {
+		if str != "" {
+			panic(`str must be "" if there is an error`)
+		}
+		panic(err)
+	}
+
+	tree, err = Load(str)
+	if err != nil {
+		if tree != nil {
+			panic("tree must be nil if there is an error")
+		}
+		return 0
+	}
+
+	return 1
+}

fuzz.sh

@@ -0,0 +1,15 @@
+#! /bin/sh
+set -eu
+
+go get github.com/dvyukov/go-fuzz/go-fuzz
+go get github.com/dvyukov/go-fuzz/go-fuzz-build
+
+if [ ! -e toml-fuzz.zip ]; then
+    go-fuzz-build github.com/pelletier/go-toml
+fi
+
+rm -fr fuzz
+mkdir -p fuzz/corpus
+cp *.toml fuzz/corpus
+
+go-fuzz -bin=toml-fuzz.zip -workdir=fuzz

keysparsing.go

@@ -9,12 +9,14 @@ import (
 	"unicode"
 )
 
+// Convert the bare key group string to an array.
+// The input supports double quotation to allow "." inside the key name,
+// but escape sequences are not supported. Lexers must unescape them beforehand.
 func parseKey(key string) ([]string, error) {
 	groups := []string{}
 	var buffer bytes.Buffer
 	inQuotes := false
 	wasInQuotes := false
-	escapeNext := false
 	ignoreSpace := true
 	expectDot := false
 
@@ -25,15 +27,7 @@ func parseKey(key string) ([]string, error) {
 			}
 			ignoreSpace = false
 		}
-		if escapeNext {
-			buffer.WriteRune(char)
-			escapeNext = false
-			continue
-		}
 		switch char {
-		case '\\':
-			escapeNext = true
-			continue
 		case '"':
 			if inQuotes {
 				groups = append(groups, buffer.String())
@@ -77,9 +71,6 @@ func parseKey(key string) ([]string, error) {
 	if inQuotes {
 		return nil, errors.New("mismatched quotes")
 	}
-	if escapeNext {
-		return nil, errors.New("unfinished escape sequence")
-	}
 	if buffer.Len() > 0 {
 		groups = append(groups, buffer.String())
 	}

keysparsing_test.go

@@ -22,7 +22,10 @@ func testResult(t *testing.T, key string, expected []string) {
 }
 
 func testError(t *testing.T, key string, expectedError string) {
-	_, err := parseKey(key)
+	res, err := parseKey(key)
+	if err == nil {
+		t.Fatalf("Expected error, but succesfully parsed key %s", res)
+	}
 	if fmt.Sprintf("%s", err) != expectedError {
 		t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
 	}
@@ -47,6 +50,10 @@ func TestBaseKeyPound(t *testing.T) {
 func TestQuotedKeys(t *testing.T) {
 	testResult(t, `hello."foo".bar`, []string{"hello", "foo", "bar"})
 	testResult(t, `"hello!"`, []string{"hello!"})
+	testResult(t, `foo."ba.r".baz`, []string{"foo", "ba.r", "baz"})
+
+	// escape sequences must not be converted
+	testResult(t, `"hello\tworld"`, []string{`hello\tworld`})
 }
 
 func TestEmptyKey(t *testing.T) {

lexer.go

@@ -9,12 +9,9 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"regexp"
 	"strconv"
 	"strings"
-
-	"github.com/pelletier/go-buffruneio"
 )
 
 var dateRegexp *regexp.Regexp
@@ -24,9 +21,11 @@ type tomlLexStateFn func() tomlLexStateFn
 
 // Define lexer
 type tomlLexer struct {
-	input         *buffruneio.Reader // Textual source
+	inputIdx          int
-	buffer        bytes.Buffer       // Runes composing the current token
+	input             []rune // Textual source
-	tokens        chan token
+	currentTokenStart int
+	currentTokenStop  int
+	tokens            []token
 	depth             int
 	line              int
 	col               int
@@ -37,16 +36,14 @@ type tomlLexer struct {
 // Basic read operations on input
 
 func (l *tomlLexer) read() rune {
-	r, _, err := l.input.ReadRune()
+	r := l.peek()
-	if err != nil {
-		panic(err)
-	}
 	if r == '\n' {
 		l.endbufferLine++
 		l.endbufferCol = 1
 	} else {
 		l.endbufferCol++
 	}
+	l.inputIdx++
 	return r
 }
 
@@ -54,13 +51,13 @@ func (l *tomlLexer) next() rune {
 	r := l.read()
 
 	if r != eof {
-		l.buffer.WriteRune(r)
+		l.currentTokenStop++
 	}
 	return r
 }
 
 func (l *tomlLexer) ignore() {
-	l.buffer.Reset()
+	l.currentTokenStart = l.currentTokenStop
 	l.line = l.endbufferLine
 	l.col = l.endbufferCol
 }
@@ -77,49 +74,46 @@ func (l *tomlLexer) fastForward(n int) {
 }
 
 func (l *tomlLexer) emitWithValue(t tokenType, value string) {
-	l.tokens <- token{
+	l.tokens = append(l.tokens, token{
 		Position: Position{l.line, l.col},
 		typ:      t,
 		val:      value,
-	}
+	})
 	l.ignore()
 }
 
 func (l *tomlLexer) emit(t tokenType) {
-	l.emitWithValue(t, l.buffer.String())
+	l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop]))
 }
 
 func (l *tomlLexer) peek() rune {
-	r, _, err := l.input.ReadRune()
+	if l.inputIdx >= len(l.input) {
-	if err != nil {
+		return eof
-		panic(err)
 	}
-	l.input.UnreadRune()
+	return l.input[l.inputIdx]
-	return r
+}
+
+func (l *tomlLexer) peekString(size int) string {
+	maxIdx := len(l.input)
+	upperIdx := l.inputIdx + size // FIXME: potential overflow
+	if upperIdx > maxIdx {
+		upperIdx = maxIdx
+	}
+	return string(l.input[l.inputIdx:upperIdx])
 }
 
 func (l *tomlLexer) follow(next string) bool {
-	for _, expectedRune := range next {
+	return next == l.peekString(len(next))
-		r, _, err := l.input.ReadRune()
-		defer l.input.UnreadRune()
-		if err != nil {
-			panic(err)
-		}
-		if expectedRune != r {
-			return false
-		}
-	}
-	return true
 }
 
 // Error management
 
 func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn {
-	l.tokens <- token{
+	l.tokens = append(l.tokens, token{
 		Position: Position{l.line, l.col},
 		typ:      tokenError,
 		val:      fmt.Sprintf(format, args...),
-	}
+	})
 	return nil
 }
 
@@ -210,6 +204,14 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexFalse
 		}
 
+		if l.follow("inf") {
+			return l.lexInf
+		}
+
+		if l.follow("nan") {
+			return l.lexNan
+		}
+
 		if isSpace(next) {
 			l.skip()
 			continue
@@ -220,7 +222,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			break
 		}
 
-		possibleDate := string(l.input.PeekRunes(35))
+		possibleDate := l.peekString(35)
 		dateMatch := dateRegexp.FindString(possibleDate)
 		if dateMatch != "" {
 			l.fastForward(len(dateMatch))
@@ -271,6 +273,18 @@ func (l *tomlLexer) lexFalse() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+func (l *tomlLexer) lexInf() tomlLexStateFn {
+	l.fastForward(3)
+	l.emit(tokenInf)
+	return l.lexRvalue
+}
+
+func (l *tomlLexer) lexNan() tomlLexStateFn {
+	l.fastForward(3)
+	l.emit(tokenNan)
+	return l.lexRvalue
+}
+
 func (l *tomlLexer) lexEqual() tomlLexStateFn {
 	l.next()
 	l.emit(tokenEqual)
@@ -283,6 +297,8 @@ func (l *tomlLexer) lexComma() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+// Parse the key and emits its value without escape sequences.
+// bare keys, basic string keys and literal string keys are supported.
 func (l *tomlLexer) lexKey() tomlLexStateFn {
 	growingString := ""
 
@@ -293,7 +309,16 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 			if err != nil {
 				return l.errorf(err.Error())
 			}
-			growingString += `"` + str + `"`
+			growingString += str
+			l.next()
+			continue
+		} else if r == '\'' {
+			l.next()
+			str, err := l.lexLiteralStringAsString(`'`, false)
+			if err != nil {
+				return l.errorf(err.Error())
+			}
+			growingString += str
 			l.next()
 			continue
 		} else if r == '\n' {
@@ -533,11 +558,12 @@ func (l *tomlLexer) lexTableKey() tomlLexStateFn {
 	return l.lexInsideTableKey
 }
 
+// Parse the key till "]]", but only bare keys are supported
 func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn {
 	for r := l.peek(); r != eof; r = l.peek() {
 		switch r {
 		case ']':
-			if l.buffer.Len() > 0 {
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroupArray)
 			}
 			l.next()
@@ -556,11 +582,12 @@ func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn {
 	return l.errorf("unclosed table array key")
 }
 
+// Parse the key till "]" but only bare keys are supported
 func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn {
 	for r := l.peek(); r != eof; r = l.peek() {
 		switch r {
 		case ']':
-			if l.buffer.Len() > 0 {
+			if l.currentTokenStop > l.currentTokenStart {
 				l.emit(tokenKeyGroup)
 			}
 			l.next()
@@ -581,11 +608,77 @@ func (l *tomlLexer) lexRightBracket() tomlLexStateFn {
 	return l.lexRvalue
 }
 
+type validRuneFn func(r rune) bool
+
+func isValidHexRune(r rune) bool {
+	return r >= 'a' && r <= 'f' ||
+		r >= 'A' && r <= 'F' ||
+		r >= '0' && r <= '9' ||
+		r == '_'
+}
+
+func isValidOctalRune(r rune) bool {
+	return r >= '0' && r <= '7' || r == '_'
+}
+
+func isValidBinaryRune(r rune) bool {
+	return r == '0' || r == '1' || r == '_'
+}
+
 func (l *tomlLexer) lexNumber() tomlLexStateFn {
 	r := l.peek()
-	if r == '+' || r == '-' {
+
+	if r == '0' {
+		follow := l.peekString(2)
+		if len(follow) == 2 {
+			var isValidRune validRuneFn
+			switch follow[1] {
+			case 'x':
+				isValidRune = isValidHexRune
+			case 'o':
+				isValidRune = isValidOctalRune
+			case 'b':
+				isValidRune = isValidBinaryRune
+			default:
+				if follow[1] >= 'a' && follow[1] <= 'z' || follow[1] >= 'A' && follow[1] <= 'Z' {
+					return l.errorf("unknown number base: %s. possible options are x (hex) o (octal) b (binary)", string(follow[1]))
+				}
+			}
+
+			if isValidRune != nil {
+				l.next()
+				l.next()
+				digitSeen := false
+				for {
+					next := l.peek()
+					if !isValidRune(next) {
+						break
+					}
+					digitSeen = true
 					l.next()
 				}
+
+				if !digitSeen {
+					return l.errorf("number needs at least one digit")
+				}
+
+				l.emit(tokenInteger)
+
+				return l.lexRvalue
+			}
+		}
+	}
+
+	if r == '+' || r == '-' {
+		l.next()
+		if l.follow("inf") {
+			return l.lexInf
+		}
+		if l.follow("nan") {
+			return l.lexNan
+		}
+	}
+
 	pointSeen := false
 	expSeen := false
 	digitSeen := false
@@ -635,7 +728,6 @@ func (l *tomlLexer) run() {
 	for state := l.lexVoid; state != nil; {
 		state = state()
 	}
-	close(l.tokens)
 }
 
 func init() {
@@ -643,16 +735,16 @@ func init() {
 }
 
 // Entry point
-func lexToml(input io.Reader) chan token {
+func lexToml(inputBytes []byte) []token {
-	bufferedInput := buffruneio.NewReader(input)
+	runes := bytes.Runes(inputBytes)
 	l := &tomlLexer{
-		input:         bufferedInput,
+		input:         runes,
-		tokens:        make(chan token),
+		tokens:        make([]token, 0, 256),
 		line:          1,
 		col:           1,
 		endbufferLine: 1,
 		endbufferCol:  1,
 	}
-	go l.run()
+	l.run()
 	return l.tokens
 }

lexer_test.go

@@ -1,38 +1,14 @@
 package toml
 
 import (
-	"os"
+	"reflect"
-	"strings"
 	"testing"
 )
 
 func testFlow(t *testing.T, input string, expectedFlow []token) {
-	ch := lexToml(strings.NewReader(input))
+	tokens := lexToml([]byte(input))
-	for _, expected := range expectedFlow {
+	if !reflect.DeepEqual(tokens, expectedFlow) {
-		token := <-ch
+		t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens)
-		if token != expected {
-			t.Log("While testing: ", input)
-			t.Log("compared (got)", token, "to (expected)", expected)
-			t.Log("\tvalue:", token.val, "<->", expected.val)
-			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
-			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
-			t.Log("\tline:", token.Line, "<->", expected.Line)
-			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
-			t.Log("compared", token, "to", expected)
-			t.FailNow()
-		}
-	}
-
-	tok, ok := <-ch
-	if ok {
-		t.Log("channel is not closed!")
-		t.Log(len(ch)+1, "tokens remaining:")
-
-		t.Log("token ->", tok)
-		for token := range ch {
-			t.Log("token ->", token)
-		}
-		t.FailNow()
 	}
 }
 
@@ -714,7 +690,7 @@ func TestKeyGroupArray(t *testing.T) {
 
 func TestQuotedKey(t *testing.T) {
 	testFlow(t, "\"a b\" = 42", []token{
-		{Position{1, 1}, tokenKey, "\"a b\""},
+		{Position{1, 1}, tokenKey, "a b"},
 		{Position{1, 7}, tokenEqual, "="},
 		{Position{1, 9}, tokenInteger, "42"},
 		{Position{1, 11}, tokenEOF, ""},
@@ -767,13 +743,8 @@ pluralizeListTitles = false
 	url = "https://github.com/spf13/hugo/releases"
 	weight = -200
 `
-	rd := strings.NewReader(sample)
-
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		rd.Seek(0, os.SEEK_SET)
+		lexToml([]byte(sample))
-		ch := lexToml(rd)
-		for _ = range ch {
-		}
 	}
 }

marshal.go

@@ -4,17 +4,33 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 )
 
+const tagKeyMultiline = "multiline"
+
 type tomlOpts struct {
 	name      string
+	comment   string
+	commented bool
+	multiline bool
 	include   bool
 	omitempty bool
 }
 
+type encOpts struct {
+	quoteMapKeys            bool
+	arraysOneElementPerLine bool
+}
+
+var encOptsDefaults = encOpts{
+	quoteMapKeys: false,
+}
+
 var timeType = reflect.TypeOf(time.Time{})
 var marshalerType = reflect.TypeOf(new(Marshaler)).Elem()
 
@@ -94,8 +110,15 @@ encoder, except that there is no concept of a Marshaler interface or MarshalTOML
 function for sub-structs, and currently only definite types can be marshaled
 (i.e. no `interface{}`).
 
+The following struct annotations are supported:
+
+  toml:"Field"      Overrides the field's name to output.
+  omitempty         When set, empty values and groups are not emitted.
+  comment:"comment" Emits a # comment on the same line. This supports new lines.
+  commented:"true"  Emits the value as commented.
+
 Note that pointers are automatically assigned the "omitempty" option, as TOML
-explicity does not handle null values (saying instead the label should be
+explicitly does not handle null values (saying instead the label should be
 dropped).
 
 Tree structural types and corresponding marshal types:
@@ -115,6 +138,66 @@ Tree primitive types and corresponding marshal types:
   time.Time  time.Time{}, pointers to same
 */
 func Marshal(v interface{}) ([]byte, error) {
+	return NewEncoder(nil).marshal(v)
+}
+
+// Encoder writes TOML values to an output stream.
+type Encoder struct {
+	w io.Writer
+	encOpts
+}
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		w:       w,
+		encOpts: encOptsDefaults,
+	}
+}
+
+// Encode writes the TOML encoding of v to the stream.
+//
+// See the documentation for Marshal for details.
+func (e *Encoder) Encode(v interface{}) error {
+	b, err := e.marshal(v)
+	if err != nil {
+		return err
+	}
+	if _, err := e.w.Write(b); err != nil {
+		return err
+	}
+	return nil
+}
+
+// QuoteMapKeys sets up the encoder to encode
+// maps with string type keys with quoted TOML keys.
+//
+// This relieves the character limitations on map keys.
+func (e *Encoder) QuoteMapKeys(v bool) *Encoder {
+	e.quoteMapKeys = v
+	return e
+}
+
+// ArraysWithOneElementPerLine sets up the encoder to encode arrays
+// with more than one element on multiple lines instead of one.
+//
+// For example:
+//
+//   A = [1,2,3]
+//
+// Becomes
+//
+//   A = [
+//     1,
+//     2,
+//     3,
+//   ]
+func (e *Encoder) ArraysWithOneElementPerLine(v bool) *Encoder {
+	e.arraysOneElementPerLine = v
+	return e
+}
+
+func (e *Encoder) marshal(v interface{}) ([]byte, error) {
 	mtype := reflect.TypeOf(v)
 	if mtype.Kind() != reflect.Struct {
 		return []byte{}, errors.New("Only a struct can be marshaled to TOML")
@@ -123,18 +206,21 @@ func Marshal(v interface{}) ([]byte, error) {
 	if isCustomMarshaler(mtype) {
 		return callCustomMarshaler(sval)
 	}
-	t, err := valueToTree(mtype, sval)
+	t, err := e.valueToTree(mtype, sval)
 	if err != nil {
 		return []byte{}, err
 	}
-	s, err := t.ToTomlString()
+
-	return []byte(s), err
+	var buf bytes.Buffer
+	_, err = t.writeTo(&buf, "", "", 0, e.arraysOneElementPerLine)
+
+	return buf.Bytes(), err
 }
 
 // Convert given marshal struct or map value to toml tree
-func valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) {
+func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) {
 	if mtype.Kind() == reflect.Ptr {
-		return valueToTree(mtype.Elem(), mval.Elem())
+		return e.valueToTree(mtype.Elem(), mval.Elem())
 	}
 	tval := newTree()
 	switch mtype.Kind() {
@@ -143,31 +229,44 @@ func valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) {
 			mtypef, mvalf := mtype.Field(i), mval.Field(i)
 			opts := tomlOptions(mtypef)
 			if opts.include && (!opts.omitempty || !isZero(mvalf)) {
-				val, err := valueToToml(mtypef.Type, mvalf)
+				val, err := e.valueToToml(mtypef.Type, mvalf)
 				if err != nil {
 					return nil, err
 				}
-				tval.Set(opts.name, val)
+
+				tval.SetWithOptions(opts.name, SetOptions{
+					Comment:   opts.comment,
+					Commented: opts.commented,
+					Multiline: opts.multiline,
+				}, val)
 			}
 		}
 	case reflect.Map:
 		for _, key := range mval.MapKeys() {
 			mvalf := mval.MapIndex(key)
-			val, err := valueToToml(mtype.Elem(), mvalf)
+			val, err := e.valueToToml(mtype.Elem(), mvalf)
 			if err != nil {
 				return nil, err
 			}
+			if e.quoteMapKeys {
+				keyStr, err := tomlValueStringRepresentation(key.String(), "", e.arraysOneElementPerLine)
+				if err != nil {
+					return nil, err
+				}
+				tval.SetPath([]string{keyStr}, val)
+			} else {
 				tval.Set(key.String(), val)
 			}
 		}
+	}
 	return tval, nil
 }
 
 // Convert given marshal slice to slice of Toml trees
-func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) {
+func (e *Encoder) valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) {
 	tval := make([]*Tree, mval.Len(), mval.Len())
 	for i := 0; i < mval.Len(); i++ {
-		val, err := valueToTree(mtype.Elem(), mval.Index(i))
+		val, err := e.valueToTree(mtype.Elem(), mval.Index(i))
 		if err != nil {
 			return nil, err
 		}
@@ -177,10 +276,10 @@ func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) {
 }
 
 // Convert given marshal slice to slice of toml values
-func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+func (e *Encoder) valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
 	tval := make([]interface{}, mval.Len(), mval.Len())
 	for i := 0; i < mval.Len(); i++ {
-		val, err := valueToToml(mtype.Elem(), mval.Index(i))
+		val, err := e.valueToToml(mtype.Elem(), mval.Index(i))
 		if err != nil {
 			return nil, err
 		}
@@ -190,19 +289,19 @@ func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, err
 }
 
 // Convert given marshal value to toml value
-func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
+func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
 	if mtype.Kind() == reflect.Ptr {
-		return valueToToml(mtype.Elem(), mval.Elem())
+		return e.valueToToml(mtype.Elem(), mval.Elem())
 	}
 	switch {
 	case isCustomMarshaler(mtype):
 		return callCustomMarshaler(mval)
 	case isTree(mtype):
-		return valueToTree(mtype, mval)
+		return e.valueToTree(mtype, mval)
 	case isTreeSlice(mtype):
-		return valueToTreeSlice(mtype, mval)
+		return e.valueToTreeSlice(mtype, mval)
 	case isOtherSlice(mtype):
-		return valueToOtherSlice(mtype, mval)
+		return e.valueToOtherSlice(mtype, mval)
 	default:
 		switch mtype.Kind() {
 		case reflect.Bool:
@@ -227,17 +326,16 @@ func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) {
 // Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for
 // sub-structs, and only definite types can be unmarshaled.
 func (t *Tree) Unmarshal(v interface{}) error {
-	mtype := reflect.TypeOf(v)
+	d := Decoder{tval: t}
-	if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct {
+	return d.unmarshal(v)
-		return errors.New("Only a pointer to struct can be unmarshaled from TOML")
+}
-	}
 
-	sval, err := valueFromTree(mtype.Elem(), t)
+// Marshal returns the TOML encoding of Tree.
-	if err != nil {
+// See Marshal() documentation for types mapping table.
-		return err
+func (t *Tree) Marshal() ([]byte, error) {
-	}
+	var buf bytes.Buffer
-	reflect.ValueOf(v).Elem().Set(sval)
+	err := NewEncoder(&buf).Encode(t)
-	return nil
+	return buf.Bytes(), err
 }
 
 // Unmarshal parses the TOML-encoded data and stores the result in the value
@@ -246,6 +344,10 @@ func (t *Tree) Unmarshal(v interface{}) error {
 // sub-structs, and currently only definite types can be unmarshaled to (i.e. no
 // `interface{}`).
 //
+// The following struct annotations are supported:
+//
+//   toml:"Field" Overrides the field's name to map to.
+//
 // See Marshal() documentation for types mapping table.
 func Unmarshal(data []byte, v interface{}) error {
 	t, err := LoadReader(bytes.NewReader(data))
@@ -255,10 +357,52 @@ func Unmarshal(data []byte, v interface{}) error {
 	return t.Unmarshal(v)
 }
 
+// Decoder reads and decodes TOML values from an input stream.
+type Decoder struct {
+	r    io.Reader
+	tval *Tree
+	encOpts
+}
+
+// NewDecoder returns a new decoder that reads from r.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{
+		r:       r,
+		encOpts: encOptsDefaults,
+	}
+}
+
+// Decode reads a TOML-encoded value from it's input
+// and unmarshals it in the value pointed at by v.
+//
+// See the documentation for Marshal for details.
+func (d *Decoder) Decode(v interface{}) error {
+	var err error
+	d.tval, err = LoadReader(d.r)
+	if err != nil {
+		return err
+	}
+	return d.unmarshal(v)
+}
+
+func (d *Decoder) unmarshal(v interface{}) error {
+	mtype := reflect.TypeOf(v)
+	if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct {
+		return errors.New("Only a pointer to struct can be unmarshaled from TOML")
+	}
+
+	sval, err := d.valueFromTree(mtype.Elem(), d.tval)
+	if err != nil {
+		return err
+	}
+	reflect.ValueOf(v).Elem().Set(sval)
+	return nil
+}
+
 // Convert toml tree to marshal struct or map, using marshal type
-func valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
+func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
 	if mtype.Kind() == reflect.Ptr {
-		return unwrapPointer(mtype, tval)
+		return d.unwrapPointer(mtype, tval)
 	}
 	var mval reflect.Value
 	switch mtype.Kind() {
@@ -268,23 +412,29 @@ func valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
 			mtypef := mtype.Field(i)
 			opts := tomlOptions(mtypef)
 			if opts.include {
-				key := opts.name
+				baseKey := opts.name
+				keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)}
+				for _, key := range keysToTry {
 					exists := tval.Has(key)
-				if exists {
+					if !exists {
+						continue
+					}
 					val := tval.Get(key)
-					mvalf, err := valueFromToml(mtypef.Type, val)
+					mvalf, err := d.valueFromToml(mtypef.Type, val)
 					if err != nil {
 						return mval, formatError(err, tval.GetPosition(key))
 					}
 					mval.Field(i).Set(mvalf)
+					break
 				}
 			}
 		}
 	case reflect.Map:
 		mval = reflect.MakeMap(mtype)
 		for _, key := range tval.Keys() {
-			val := tval.Get(key)
+			// TODO: path splits key
-			mvalf, err := valueFromToml(mtype.Elem(), val)
+			val := tval.GetPath([]string{key})
+			mvalf, err := d.valueFromToml(mtype.Elem(), val)
 			if err != nil {
 				return mval, formatError(err, tval.GetPosition(key))
 			}
@@ -295,10 +445,10 @@ func valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) {
 }
 
 // Convert toml value to marshal struct/map slice, using marshal type
-func valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) {
+func (d *Decoder) valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) {
 	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
 	for i := 0; i < len(tval); i++ {
-		val, err := valueFromTree(mtype.Elem(), tval[i])
+		val, err := d.valueFromTree(mtype.Elem(), tval[i])
 		if err != nil {
 			return mval, err
 		}
@@ -308,10 +458,10 @@ func valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error)
 }
 
 // Convert toml value to marshal primitive slice, using marshal type
-func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) {
+func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) {
 	mval := reflect.MakeSlice(mtype, len(tval), len(tval))
 	for i := 0; i < len(tval); i++ {
-		val, err := valueFromToml(mtype.Elem(), tval[i])
+		val, err := d.valueFromToml(mtype.Elem(), tval[i])
 		if err != nil {
 			return mval, err
 		}
@@ -321,117 +471,86 @@ func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value,
 }
 
 // Convert toml value to marshal value, using marshal type
-func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
 	if mtype.Kind() == reflect.Ptr {
-		return unwrapPointer(mtype, tval)
+		return d.unwrapPointer(mtype, tval)
 	}
-	switch {
+
-	case isTree(mtype):
+	switch tval.(type) {
-		return valueFromTree(mtype, tval.(*Tree))
+	case *Tree:
-	case isTreeSlice(mtype):
+		if isTree(mtype) {
-		return valueFromTreeSlice(mtype, tval.([]*Tree))
+			return d.valueFromTree(mtype, tval.(*Tree))
-	case isOtherSlice(mtype):
+		}
-		return valueFromOtherSlice(mtype, tval.([]interface{}))
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a tree", tval, tval)
+	case []*Tree:
+		if isTreeSlice(mtype) {
+			return d.valueFromTreeSlice(mtype, tval.([]*Tree))
+		}
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to trees", tval, tval)
+	case []interface{}:
+		if isOtherSlice(mtype) {
+			return d.valueFromOtherSlice(mtype, tval.([]interface{}))
+		}
+		return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a slice", tval, tval)
 	default:
 		switch mtype.Kind() {
-		case reflect.Bool:
+		case reflect.Bool, reflect.Struct:
-			val, ok := tval.(bool)
+			val := reflect.ValueOf(tval)
-			if !ok {
+			// if this passes for when mtype is reflect.Struct, tval is a time.Time
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to bool", tval, tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
 			}
-			return reflect.ValueOf(val), nil
+
-		case reflect.Int:
+			return val.Convert(mtype), nil
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
-			}
-			return reflect.ValueOf(int(val)), nil
-		case reflect.Int8:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
-			}
-			return reflect.ValueOf(int8(val)), nil
-		case reflect.Int16:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
-			}
-			return reflect.ValueOf(int16(val)), nil
-		case reflect.Int32:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
-			}
-			return reflect.ValueOf(int32(val)), nil
-		case reflect.Int64:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval)
-			}
-			return reflect.ValueOf(val), nil
-		case reflect.Uint:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
-			}
-			return reflect.ValueOf(uint(val)), nil
-		case reflect.Uint8:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
-			}
-			return reflect.ValueOf(uint8(val)), nil
-		case reflect.Uint16:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
-			}
-			return reflect.ValueOf(uint16(val)), nil
-		case reflect.Uint32:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
-			}
-			return reflect.ValueOf(uint32(val)), nil
-		case reflect.Uint64:
-			val, ok := tval.(int64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval)
-			}
-			return reflect.ValueOf(uint64(val)), nil
-		case reflect.Float32:
-			val, ok := tval.(float64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval)
-			}
-			return reflect.ValueOf(float32(val)), nil
-		case reflect.Float64:
-			val, ok := tval.(float64)
-			if !ok {
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval)
-			}
-			return reflect.ValueOf(val), nil
 		case reflect.String:
-			val, ok := tval.(string)
+			val := reflect.ValueOf(tval)
-			if !ok {
+			// stupidly, int64 is convertible to string. So special case this.
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to string", tval, tval)
+			if !val.Type().ConvertibleTo(mtype) || val.Kind() == reflect.Int64 {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
 			}
-			return reflect.ValueOf(val), nil
+
-		case reflect.Struct:
+			return val.Convert(mtype), nil
-			val, ok := tval.(time.Time)
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-			if !ok {
+			val := reflect.ValueOf(tval)
-				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to time", tval, tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
 			}
-			return reflect.ValueOf(val), nil
+			if reflect.Indirect(reflect.New(mtype)).OverflowInt(val.Int()) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+			val := reflect.ValueOf(tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+			if val.Int() < 0 {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) is negative so does not fit in %v", tval, tval, mtype.String())
+			}
+			if reflect.Indirect(reflect.New(mtype)).OverflowUint(uint64(val.Int())) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
+		case reflect.Float32, reflect.Float64:
+			val := reflect.ValueOf(tval)
+			if !val.Type().ConvertibleTo(mtype) {
+				return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String())
+			}
+			if reflect.Indirect(reflect.New(mtype)).OverflowFloat(val.Float()) {
+				return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String())
+			}
+
+			return val.Convert(mtype), nil
 		default:
-			return reflect.ValueOf(nil), fmt.Errorf("Unmarshal can't handle %v(%v)", mtype, mtype.Kind())
+			return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind())
 		}
 	}
 }
 
-func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
+func (d *Decoder) unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) {
-	val, err := valueFromToml(mtype.Elem(), tval)
+	val, err := d.valueFromToml(mtype.Elem(), tval)
 	if err != nil {
 		return reflect.ValueOf(nil), err
 	}
@@ -443,7 +562,13 @@ func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error)
 func tomlOptions(vf reflect.StructField) tomlOpts {
 	tag := vf.Tag.Get("toml")
 	parse := strings.Split(tag, ",")
-	result := tomlOpts{vf.Name, true, false}
+	var comment string
+	if c := vf.Tag.Get("comment"); c != "" {
+		comment = c
+	}
+	commented, _ := strconv.ParseBool(vf.Tag.Get("commented"))
+	multiline, _ := strconv.ParseBool(vf.Tag.Get(tagKeyMultiline))
+	result := tomlOpts{name: vf.Name, comment: comment, commented: commented, multiline: multiline, include: true, omitempty: false}
 	if parse[0] != "" {
 		if parse[0] == "-" && len(parse) == 1 {
 			result.include = false

marshal_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 )
@@ -145,8 +146,8 @@ var docData = testDoc{
 		Second: &subdoc,
 	},
 	SubDocList: []testSubDoc{
-		testSubDoc{"List.First", 0},
+		{"List.First", 0},
-		testSubDoc{"List.Second", 0},
+		{"List.Second", 0},
 	},
 	SubDocPtrs: []*testSubDoc{&subdoc},
 }
@@ -177,25 +178,6 @@ func TestDocUnmarshal(t *testing.T) {
 	}
 }
 
-func ExampleUnmarshal() {
-	type Postgres struct {
-		User     string
-		Password string
-	}
-	type Config struct {
-		Postgres Postgres
-	}
-
-	doc := []byte(`
-	[postgres]
-	user = "pelletier"
-	password = "mypassword"`)
-
-	config := Config{}
-	Unmarshal(doc, &config)
-	fmt.Println("user=", config.Postgres.User)
-}
-
 func TestDocPartialUnmarshal(t *testing.T) {
 	result := testDocSubs{}
 
@@ -527,6 +509,14 @@ func TestPointerUnmarshal(t *testing.T) {
 	}
 }
 
+func TestUnmarshalTypeMismatch(t *testing.T) {
+	result := pointerMarshalTestStruct{}
+	err := Unmarshal([]byte("List = 123"), &result)
+	if !strings.HasPrefix(err.Error(), "(1, 1): Can't convert 123(int64) to []string(slice)") {
+		t.Errorf("Type mismatch must be reported: got %v", err.Error())
+	}
+}
+
 type nestedMarshalTestStruct struct {
 	String [][]string
 	//Struct [][]basicMarshalTestSubStruct
@@ -540,7 +530,7 @@ var strPtr = []*string{&str1, &str2}
 var strPtr2 = []*[]*string{&strPtr}
 
 var nestedTestData = nestedMarshalTestStruct{
-	String:    [][]string{[]string{"Five", "Six"}, []string{"One", "Two"}},
+	String:    [][]string{{"Five", "Six"}, {"One", "Two"}},
 	StringPtr: &strPtr2,
 }
 
@@ -617,3 +607,200 @@ func TestNestedCustomMarshaler(t *testing.T) {
 		t.Errorf("Bad nested custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
 	}
 }
+
+var commentTestToml = []byte(`
+# it's a comment on type
+[postgres]
+  # isCommented = "dvalue"
+  noComment = "cvalue"
+
+  # A comment on AttrB with a
+  # break line
+  password = "bvalue"
+
+  # A comment on AttrA
+  user = "avalue"
+
+  [[postgres.My]]
+
+    # a comment on my on typeC
+    My = "Foo"
+
+  [[postgres.My]]
+
+    # a comment on my on typeC
+    My = "Baar"
+`)
+
+func TestMarshalComment(t *testing.T) {
+	type TypeC struct {
+		My string `comment:"a comment on my on typeC"`
+	}
+	type TypeB struct {
+		AttrA string `toml:"user" comment:"A comment on AttrA"`
+		AttrB string `toml:"password" comment:"A comment on AttrB with a\n break line"`
+		AttrC string `toml:"noComment"`
+		AttrD string `toml:"isCommented" commented:"true"`
+		My    []TypeC
+	}
+	type TypeA struct {
+		TypeB TypeB `toml:"postgres" comment:"it's a comment on type"`
+	}
+
+	ta := []TypeC{{My: "Foo"}, {My: "Baar"}}
+	config := TypeA{TypeB{AttrA: "avalue", AttrB: "bvalue", AttrC: "cvalue", AttrD: "dvalue", My: ta}}
+	result, err := Marshal(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := commentTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+type mapsTestStruct struct {
+	Simple map[string]string
+	Paths  map[string]string
+	Other  map[string]float64
+	X      struct {
+		Y struct {
+			Z map[string]bool
+		}
+	}
+}
+
+var mapsTestData = mapsTestStruct{
+	Simple: map[string]string{
+		"one plus one": "two",
+		"next":         "three",
+	},
+	Paths: map[string]string{
+		"/this/is/a/path": "/this/is/also/a/path",
+		"/heloo.txt":      "/tmp/lololo.txt",
+	},
+	Other: map[string]float64{
+		"testing": 3.9999,
+	},
+	X: struct{ Y struct{ Z map[string]bool } }{
+		Y: struct{ Z map[string]bool }{
+			Z: map[string]bool{
+				"is.Nested": true,
+			},
+		},
+	},
+}
+var mapsTestToml = []byte(`
+[Other]
+  "testing" = 3.9999
+
+[Paths]
+  "/heloo.txt" = "/tmp/lololo.txt"
+  "/this/is/a/path" = "/this/is/also/a/path"
+
+[Simple]
+  "next" = "three"
+  "one plus one" = "two"
+
+[X]
+
+  [X.Y]
+
+    [X.Y.Z]
+      "is.Nested" = true
+`)
+
+func TestEncodeQuotedMapKeys(t *testing.T) {
+	var buf bytes.Buffer
+	if err := NewEncoder(&buf).QuoteMapKeys(true).Encode(mapsTestData); err != nil {
+		t.Fatal(err)
+	}
+	result := buf.Bytes()
+	expected := mapsTestToml
+	if !bytes.Equal(result, expected) {
+		t.Errorf("Bad maps marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result)
+	}
+}
+
+func TestDecodeQuotedMapKeys(t *testing.T) {
+	result := mapsTestStruct{}
+	err := NewDecoder(bytes.NewBuffer(mapsTestToml)).Decode(&result)
+	expected := mapsTestData
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Bad maps unmarshal: expected %v, got %v", expected, result)
+	}
+}
+
+type structArrayNoTag struct {
+	A struct {
+		B []int64
+		C []int64
+	}
+}
+
+func TestMarshalArray(t *testing.T) {
+	expected := []byte(`
+[A]
+  B = [1,2,3]
+  C = [1]
+`)
+
+	m := structArrayNoTag{
+		A: struct {
+			B []int64
+			C []int64
+		}{
+			B: []int64{1, 2, 3},
+			C: []int64{1},
+		},
+	}
+
+	b, err := Marshal(m)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(b, expected) {
+		t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b)
+	}
+}
+
+func TestMarshalArrayOnePerLine(t *testing.T) {
+	expected := []byte(`
+[A]
+  B = [
+    1,
+    2,
+    3,
+  ]
+  C = [1]
+`)
+
+	m := structArrayNoTag{
+		A: struct {
+			B []int64
+			C []int64
+		}{
+			B: []int64{1, 2, 3},
+			C: []int64{1},
+		},
+	}
+
+	var buf bytes.Buffer
+	encoder := NewEncoder(&buf).ArraysWithOneElementPerLine(true)
+	err := encoder.Encode(m)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+
+	if !bytes.Equal(b, expected) {
+		t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b)
+	}
+}

parser.go

@@ -5,6 +5,7 @@ package toml
 import (
 	"errors"
 	"fmt"
+	"math"
 	"reflect"
 	"regexp"
 	"strconv"
@@ -13,9 +14,9 @@ import (
 )
 
 type tomlParser struct {
-	flow          chan token
+	flowIdx       int
+	flow          []token
 	tree          *Tree
-	tokensBuffer  []token
 	currentTable  []string
 	seenTableKeys []string
 }
@@ -34,16 +35,10 @@ func (p *tomlParser) run() {
 }
 
 func (p *tomlParser) peek() *token {
-	if len(p.tokensBuffer) != 0 {
+	if p.flowIdx >= len(p.flow) {
-		return &(p.tokensBuffer[0])
-	}
-
-	tok, ok := <-p.flow
-	if !ok {
 		return nil
 	}
-	p.tokensBuffer = append(p.tokensBuffer, tok)
+	return &p.flow[p.flowIdx]
-	return &tok
 }
 
 func (p *tomlParser) assume(typ tokenType) {
@@ -57,16 +52,12 @@ func (p *tomlParser) assume(typ tokenType) {
 }
 
 func (p *tomlParser) getToken() *token {
-	if len(p.tokensBuffer) != 0 {
+	tok := p.peek()
-		tok := p.tokensBuffer[0]
+	if tok == nil {
-		p.tokensBuffer = p.tokensBuffer[1:]
-		return &tok
-	}
-	tok, ok := <-p.flow
-	if !ok {
 		return nil
 	}
-	return &tok
+	p.flowIdx++
+	return tok
 }
 
 func (p *tomlParser) parseStart() tomlParserStateFn {
@@ -195,10 +186,7 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	}
 
 	// assign value to the found table
-	keyVals, err := parseKey(key.val)
+	keyVals := []string{key.val}
-	if err != nil {
-		p.raiseError(key, "%s", err)
-	}
 	if len(keyVals) != 1 {
 		p.raiseError(key, "Invalid key")
 	}
@@ -215,20 +203,32 @@ func (p *tomlParser) parseAssign() tomlParserStateFn {
 	case *Tree, []*Tree:
 		toInsert = value
 	default:
-		toInsert = &tomlValue{value, key.Position}
+		toInsert = &tomlValue{value: value, position: key.Position}
 	}
 	targetNode.values[keyVal] = toInsert
 	return p.parseStart
 }
 
 var numberUnderscoreInvalidRegexp *regexp.Regexp
+var hexNumberUnderscoreInvalidRegexp *regexp.Regexp
 
-func cleanupNumberToken(value string) (string, error) {
+func numberContainsInvalidUnderscore(value string) error {
 	if numberUnderscoreInvalidRegexp.MatchString(value) {
-		return "", errors.New("invalid use of _ in number")
+		return errors.New("invalid use of _ in number")
 	}
+	return nil
+}
+
+func hexNumberContainsInvalidUnderscore(value string) error {
+	if hexNumberUnderscoreInvalidRegexp.MatchString(value) {
+		return errors.New("invalid use of _ in hex number")
+	}
+	return nil
+}
+
+func cleanupNumberToken(value string) string {
 	cleanedVal := strings.Replace(value, "_", "", -1)
-	return cleanedVal, nil
+	return cleanedVal
 }
 
 func (p *tomlParser) parseRvalue() interface{} {
@@ -244,21 +244,57 @@ func (p *tomlParser) parseRvalue() interface{} {
 		return true
 	case tokenFalse:
 		return false
+	case tokenInf:
+		if tok.val[0] == '-' {
+			return math.Inf(-1)
+		}
+		return math.Inf(1)
+	case tokenNan:
+		return math.NaN()
 	case tokenInteger:
-		cleanedVal, err := cleanupNumberToken(tok.val)
+		cleanedVal := cleanupNumberToken(tok.val)
+		var err error
+		var val int64
+		if len(cleanedVal) >= 3 && cleanedVal[0] == '0' {
+			switch cleanedVal[1] {
+			case 'x':
+				err = hexNumberContainsInvalidUnderscore(tok.val)
 				if err != nil {
 					p.raiseError(tok, "%s", err)
 				}
-		val, err := strconv.ParseInt(cleanedVal, 10, 64)
+				val, err = strconv.ParseInt(cleanedVal[2:], 16, 64)
+			case 'o':
+				err = numberContainsInvalidUnderscore(tok.val)
+				if err != nil {
+					p.raiseError(tok, "%s", err)
+				}
+				val, err = strconv.ParseInt(cleanedVal[2:], 8, 64)
+			case 'b':
+				err = numberContainsInvalidUnderscore(tok.val)
+				if err != nil {
+					p.raiseError(tok, "%s", err)
+				}
+				val, err = strconv.ParseInt(cleanedVal[2:], 2, 64)
+			default:
+				panic("invalid base") // the lexer should catch this first
+			}
+		} else {
+			err = numberContainsInvalidUnderscore(tok.val)
+			if err != nil {
+				p.raiseError(tok, "%s", err)
+			}
+			val, err = strconv.ParseInt(cleanedVal, 10, 64)
+		}
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		cleanedVal, err := cleanupNumberToken(tok.val)
+		err := numberContainsInvalidUnderscore(tok.val)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}
+		cleanedVal := cleanupNumberToken(tok.val)
 		val, err := strconv.ParseFloat(cleanedVal, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
@@ -319,7 +355,7 @@ Loop:
 			}
 			p.getToken()
 		default:
-			p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String())
+			p.raiseError(follow, "unexpected token type in inline table: %s", follow.String())
 		}
 		previous = follow
 	}
@@ -374,13 +410,13 @@ func (p *tomlParser) parseArray() interface{} {
 	return array
 }
 
-func parseToml(flow chan token) *Tree {
+func parseToml(flow []token) *Tree {
 	result := newTree()
 	result.position = Position{1, 1}
 	parser := &tomlParser{
+		flowIdx:       0,
 		flow:          flow,
 		tree:          result,
-		tokensBuffer:  make([]token, 0),
 		currentTable:  make([]string, 0),
 		seenTableKeys: make([]string, 0),
 	}
@@ -389,5 +425,6 @@ func parseToml(flow chan token) *Tree {
 }
 
 func init() {
-	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`)
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d])|_$|^_`)
+	hexNumberUnderscoreInvalidRegexp = regexp.MustCompile(`(^0x_)|([^\da-f]_|_[^\da-f])|_$|^_`)
 }

parser_test.go

@@ -2,6 +2,7 @@ package toml
 
 import (
 	"fmt"
+	"math"
 	"reflect"
 	"testing"
 	"time"
@@ -72,6 +73,17 @@ func TestNumberInKey(t *testing.T) {
 	})
 }
 
+func TestIncorrectKeyExtraSquareBracket(t *testing.T) {
+	_, err := Load(`[a]b]
+zyx = 42`)
+	if err == nil {
+		t.Error("Error should have been returned.")
+	}
+	if err.Error() != "(1, 4): unexpected token" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestSimpleNumbers(t *testing.T) {
 	tree, err := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -82,6 +94,78 @@ func TestSimpleNumbers(t *testing.T) {
 	})
 }
 
+func TestSpecialFloats(t *testing.T) {
+	tree, err := Load(`
+normalinf = inf
+plusinf = +inf
+minusinf = -inf
+normalnan = nan
+plusnan = +nan
+minusnan = -nan
+`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"normalinf": math.Inf(1),
+		"plusinf":   math.Inf(1),
+		"minusinf":  math.Inf(-1),
+		"normalnan": math.NaN(),
+		"plusnan":   math.NaN(),
+		"minusnan":  math.NaN(),
+	})
+}
+
+func TestHexIntegers(t *testing.T) {
+	tree, err := Load(`a = 0xDEADBEEF`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	tree, err = Load(`a = 0xdeadbeef`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	tree, err = Load(`a = 0xdead_beef`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(3735928559)})
+
+	_, err = Load(`a = 0x_1`)
+	if err.Error() != "(1, 5): invalid use of _ in hex number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestOctIntegers(t *testing.T) {
+	tree, err := Load(`a = 0o01234567`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(342391)})
+
+	tree, err = Load(`a = 0o755`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(493)})
+
+	_, err = Load(`a = 0o_1`)
+	if err.Error() != "(1, 5): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestBinIntegers(t *testing.T) {
+	tree, err := Load(`a = 0b11010110`)
+	assertTree(t, tree, err, map[string]interface{}{"a": int64(214)})
+
+	_, err = Load(`a = 0b_1`)
+	if err.Error() != "(1, 5): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
+func TestBadIntegerBase(t *testing.T) {
+	_, err := Load(`a = 0k1`)
+	if err.Error() != "(1, 5): unknown number base: k. possible options are x (hex) o (octal) b (binary)" {
+		t.Error("Error should have been returned.")
+	}
+}
+
+func TestIntegerNoDigit(t *testing.T) {
+	_, err := Load(`a = 0b`)
+	if err.Error() != "(1, 5): number needs at least one digit" {
+		t.Error("Bad error message:", err.Error())
+	}
+}
+
 func TestNumbersWithUnderscores(t *testing.T) {
 	tree, err := Load("a = 1_000")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -155,6 +239,36 @@ func TestSpaceKey(t *testing.T) {
 	})
 }
 
+func TestDoubleQuotedKey(t *testing.T) {
+	tree, err := Load(`
+	"key"        = "a"
+	"\t"         = "b"
+	"\U0001F914" = "c"
+	"\u2764"     = "d"
+	`)
+	assertTree(t, tree, err, map[string]interface{}{
+		"key":        "a",
+		"\t":         "b",
+		"\U0001F914": "c",
+		"\u2764":     "d",
+	})
+}
+
+func TestSingleQuotedKey(t *testing.T) {
+	tree, err := Load(`
+	'key'        = "a"
+	'\t'         = "b"
+	'\U0001F914' = "c"
+	'\u2764'     = "d"
+	`)
+	assertTree(t, tree, err, map[string]interface{}{
+		`key`:        "a",
+		`\t`:         "b",
+		`\U0001F914`: "c",
+		`\u2764`:     "d",
+	})
+}
+
 func TestStringEscapables(t *testing.T) {
 	tree, err := Load("a = \"a \\n b\"")
 	assertTree(t, tree, err, map[string]interface{}{
@@ -642,7 +756,7 @@ func TestTomlValueStringRepresentation(t *testing.T) {
 		{int64(12345), "12345"},
 		{uint64(50), "50"},
 		{float64(123.45), "123.45"},
-		{bool(true), "true"},
+		{true, "true"},
 		{"hello world", "\"hello world\""},
 		{"\b\t\n\f\r\"\\", "\"\\b\\t\\n\\f\\r\\\"\\\\\""},
 		{"\x05", "\"\\u0005\""},
@@ -652,7 +766,7 @@ func TestTomlValueStringRepresentation(t *testing.T) {
 			"[\"gamma\",\"delta\"]"},
 		{nil, ""},
 	} {
-		result, err := tomlValueStringRepresentation(item.Value)
+		result, err := tomlValueStringRepresentation(item.Value, "", false)
 		if err != nil {
 			t.Errorf("Test %d - unexpected error: %s", idx, err)
 		}

query/doc.go

@@ -139,7 +139,7 @@
 // Compiled Queries
 //
 // Queries may be executed directly on a Tree object, or compiled ahead
-// of time and executed discretely.  The former is more convienent, but has the
+// of time and executed discretely.  The former is more convenient, but has the
 // penalty of having to recompile the query expression each time.
 //
 //   // basic query

query/lexer.go

@@ -7,10 +7,10 @@ package query
 
 import (
 	"fmt"
+	"github.com/pelletier/go-toml"
 	"strconv"
 	"strings"
 	"unicode/utf8"
-	"github.com/pelletier/go-toml"
 )
 
 // Lexer state function
@@ -55,7 +55,7 @@ func (l *queryLexer) nextStart() {
 
 func (l *queryLexer) emit(t tokenType) {
 	l.tokens <- token{
-		Position: toml.Position{Line:l.line, Col:l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      l.input[l.start:l.pos],
 	}
@@ -64,7 +64,7 @@ func (l *queryLexer) emit(t tokenType) {
 
 func (l *queryLexer) emitWithValue(t tokenType, value string) {
 	l.tokens <- token{
-		Position: toml.Position{Line:l.line, Col:l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      t,
 		val:      value,
 	}
@@ -92,7 +92,7 @@ func (l *queryLexer) backup() {
 
 func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn {
 	l.tokens <- token{
-		Position: toml.Position{Line:l.line, Col:l.col},
+		Position: toml.Position{Line: l.line, Col: l.col},
 		typ:      tokenError,
 		val:      fmt.Sprintf(format, args...),
 	}

query/lexer_test.go

@@ -1,8 +1,8 @@
 package query
 
 import (
-	"testing"
 	"github.com/pelletier/go-toml"
+	"testing"
 )
 
 func testQLFlow(t *testing.T, input string, expectedFlow []token) {

query/match_test.go

@@ -2,8 +2,8 @@ package query
 
 import (
 	"fmt"
-	"testing"
 	"github.com/pelletier/go-toml"
+	"testing"
 )
 
 // dump path tree to a string

query/parser.go

@@ -253,7 +253,7 @@ func (p *queryParser) parseFilterExpr() queryParserStateFn {
 	}
 	tok = p.getToken()
 	if tok.typ != tokenKey && tok.typ != tokenString {
-		return p.parseError(tok, "expected key or string for filter funciton name")
+		return p.parseError(tok, "expected key or string for filter function name")
 	}
 	name := tok.val
 	tok = p.getToken()

query/parser_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+
 	"github.com/pelletier/go-toml"
 )
 
@@ -406,8 +407,7 @@ func TestQueryFilterFn(t *testing.T) {
 
 	assertQueryPositions(t, string(buff),
 		"$..[?(float)]",
-		[]interface{}{
+		[]interface{}{ // no float values in document
-		// no float values in document
 		})
 
 	tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")

query/tokens.go

@@ -1,10 +1,10 @@
 package query
 
 import (
-"fmt"
+	"fmt"
-"strconv"
-"unicode"
 	"github.com/pelletier/go-toml"
+	"strconv"
+	"unicode"
 )
 
 // Define tokens
@@ -104,4 +104,3 @@ func isHexDigit(r rune) bool {
 		(r >= 'a' && r <= 'f') ||
 		(r >= 'A' && r <= 'F')
 }
-

test.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # fail out of the script if anything here fails
 set -e
+set -o pipefail
 
 # set the path to the present working directory
 export GOPATH=`pwd`
@@ -22,11 +23,10 @@ function git_clone() {
 # Remove potential previous runs
 rm -rf src test_program_bin toml-test
 
-# Run go vet
-go vet ./...
-
 go get github.com/pelletier/go-buffruneio
 go get github.com/davecgh/go-spew/spew
+go get gopkg.in/yaml.v2
+go get github.com/BurntSushi/toml
 
 # get code for BurntSushi TOML validation
 # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize)

token.go

@@ -23,6 +23,8 @@ const (
 	tokenTrue
 	tokenFalse
 	tokenFloat
+	tokenInf
+	tokenNan
 	tokenEqual
 	tokenLeftBracket
 	tokenRightBracket
@@ -55,6 +57,8 @@ var tokenTypeNames = []string{
 	"True",
 	"False",
 	"Float",
+	"Inf",
+	"NaN",
 	"=",
 	"[",
 	"]",

toml.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
@@ -11,12 +12,17 @@ import (
 
 type tomlValue struct {
 	value     interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list
+	comment   string
+	commented bool
+	multiline bool
 	position  Position
 }
 
 // Tree is the result of the parsing of a TOML file.
 type Tree struct {
 	values    map[string]interface{} // string -> *tomlValue, *Tree, []*Tree
+	comment   string
+	commented bool
 	position  Position
 }
 
@@ -66,18 +72,15 @@ func (t *Tree) Keys() []string {
 }
 
 // Get the value at key in the Tree.
-// Key is a dot-separated path (e.g. a.b.c).
+// Key is a dot-separated path (e.g. a.b.c) without single/double quoted strings.
+// If you need to retrieve non-bare keys, use GetPath.
 // Returns nil if the path does not exist in the tree.
 // If keys is of length zero, the current tree is returned.
 func (t *Tree) Get(key string) interface{} {
 	if key == "" {
 		return t
 	}
-	comps, err := parseKey(key)
+	return t.GetPath(strings.Split(key, "."))
-	if err != nil {
-		return nil
-	}
-	return t.GetPath(comps)
 }
 
 // GetPath returns the element in the tree indicated by 'keys'.
@@ -173,17 +176,23 @@ func (t *Tree) GetDefault(key string, def interface{}) interface{} {
 	return val
 }
 
-// Set an element in the tree.
+// SetOptions arguments are supplied to the SetWithOptions and SetPathWithOptions functions to modify marshalling behaviour.
-// Key is a dot-separated path (e.g. a.b.c).
+// The default values within the struct are valid default options.
-// Creates all necessary intermediate trees, if needed.
+type SetOptions struct {
-func (t *Tree) Set(key string, value interface{}) {
+	Comment   string
-	t.SetPath(strings.Split(key, "."), value)
+	Commented bool
+	Multiline bool
 }
 
-// SetPath sets an element in the tree.
+// SetWithOptions is the same as Set, but allows you to provide formatting
-// Keys is an array of path elements (e.g. {"a","b","c"}).
+// instructions to the key, that will be used by Marshal().
-// Creates all necessary intermediate trees, if needed.
+func (t *Tree) SetWithOptions(key string, opts SetOptions, value interface{}) {
-func (t *Tree) SetPath(keys []string, value interface{}) {
+	t.SetPathWithOptions(strings.Split(key, "."), opts, value)
+}
+
+// SetPathWithOptions is the same as SetPath, but allows you to provide
+// formatting instructions to the key, that will be reused by Marshal().
+func (t *Tree) SetPathWithOptions(keys []string, opts SetOptions, value interface{}) {
 	subtree := t
 	for _, intermediateKey := range keys[:len(keys)-1] {
 		nextTree, exists := subtree.values[intermediateKey]
@@ -208,13 +217,80 @@ func (t *Tree) SetPath(keys []string, value interface{}) {
 
 	switch value.(type) {
 	case *Tree:
+		tt := value.(*Tree)
+		tt.comment = opts.Comment
 		toInsert = value
 	case []*Tree:
 		toInsert = value
 	case *tomlValue:
-		toInsert = value
+		tt := value.(*tomlValue)
+		tt.comment = opts.Comment
+		toInsert = tt
 	default:
-		toInsert = &tomlValue{value: value}
+		toInsert = &tomlValue{value: value, comment: opts.Comment, commented: opts.Commented, multiline: opts.Multiline}
+	}
+
+	subtree.values[keys[len(keys)-1]] = toInsert
+}
+
+// Set an element in the tree.
+// Key is a dot-separated path (e.g. a.b.c).
+// Creates all necessary intermediate trees, if needed.
+func (t *Tree) Set(key string, value interface{}) {
+	t.SetWithComment(key, "", false, value)
+}
+
+// SetWithComment is the same as Set, but allows you to provide comment
+// information to the key, that will be reused by Marshal().
+func (t *Tree) SetWithComment(key string, comment string, commented bool, value interface{}) {
+	t.SetPathWithComment(strings.Split(key, "."), comment, commented, value)
+}
+
+// SetPath sets an element in the tree.
+// Keys is an array of path elements (e.g. {"a","b","c"}).
+// Creates all necessary intermediate trees, if needed.
+func (t *Tree) SetPath(keys []string, value interface{}) {
+	t.SetPathWithComment(keys, "", false, value)
+}
+
+// SetPathWithComment is the same as SetPath, but allows you to provide comment
+// information to the key, that will be reused by Marshal().
+func (t *Tree) SetPathWithComment(keys []string, comment string, commented bool, value interface{}) {
+	subtree := t
+	for _, intermediateKey := range keys[:len(keys)-1] {
+		nextTree, exists := subtree.values[intermediateKey]
+		if !exists {
+			nextTree = newTree()
+			subtree.values[intermediateKey] = nextTree // add new element here
+		}
+		switch node := nextTree.(type) {
+		case *Tree:
+			subtree = node
+		case []*Tree:
+			// go to most recent element
+			if len(node) == 0 {
+				// create element if it does not exist
+				subtree.values[intermediateKey] = append(node, newTree())
+			}
+			subtree = node[len(node)-1]
+		}
+	}
+
+	var toInsert interface{}
+
+	switch value.(type) {
+	case *Tree:
+		tt := value.(*Tree)
+		tt.comment = comment
+		toInsert = value
+	case []*Tree:
+		toInsert = value
+	case *tomlValue:
+		tt := value.(*tomlValue)
+		tt.comment = comment
+		toInsert = tt
+	default:
+		toInsert = &tomlValue{value: value, comment: comment, commented: commented}
 	}
 
 	subtree.values[keys[len(keys)-1]] = toInsert
@@ -251,8 +327,8 @@ func (t *Tree) createSubTree(keys []string, pos Position) error {
 	return nil
 }
 
-// LoadReader creates a Tree from any io.Reader.
+// LoadBytes creates a Tree from a []byte.
-func LoadReader(reader io.Reader) (tree *Tree, err error) {
+func LoadBytes(b []byte) (tree *Tree, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(runtime.Error); ok {
@@ -261,13 +337,23 @@ func LoadReader(reader io.Reader) (tree *Tree, err error) {
 			err = errors.New(r.(string))
 		}
 	}()
-	tree = parseToml(lexToml(reader))
+	tree = parseToml(lexToml(b))
+	return
+}
+
+// LoadReader creates a Tree from any io.Reader.
+func LoadReader(reader io.Reader) (tree *Tree, err error) {
+	inputBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return
+	}
+	tree, err = LoadBytes(inputBytes)
 	return
 }
 
 // Load creates a Tree from a string.
 func Load(content string) (tree *Tree, err error) {
-	return LoadReader(strings.NewReader(content))
+	return LoadBytes([]byte(content))
 }
 
 // LoadFile creates a Tree from a file.

tomltree_create.go

@@ -104,7 +104,7 @@ func sliceToTree(object interface{}) (interface{}, error) {
 		}
 		arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue))
 	}
-	return &tomlValue{arrayValue.Interface(), Position{}}, nil
+	return &tomlValue{value: arrayValue.Interface(), position: Position{}}, nil
 }
 
 func toTree(object interface{}) (interface{}, error) {
@@ -127,7 +127,7 @@ func toTree(object interface{}) (interface{}, error) {
 			}
 			values[key.String()] = newValue
 		}
-		return &Tree{values, Position{}}, nil
+		return &Tree{values: values, position: Position{}}, nil
 	}
 
 	if value.Kind() == reflect.Array || value.Kind() == reflect.Slice {
@@ -138,5 +138,5 @@ func toTree(object interface{}) (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &tomlValue{simpleValue, Position{}}, nil
+	return &tomlValue{value: simpleValue, position: Position{}}, nil
 }

tomltree_create_test.go

@@ -60,7 +60,7 @@ func TestTreeCreateToTree(t *testing.T) {
 		},
 		"array":                 []string{"a", "b", "c"},
 		"array_uint":            []uint{uint(1), uint(2)},
-		"array_table":           []map[string]interface{}{map[string]interface{}{"sub_map": 52}},
+		"array_table":           []map[string]interface{}{{"sub_map": 52}},
 		"array_times":           []time.Time{time.Now(), time.Now()},
 		"map_times":             map[string]time.Time{"now": time.Now()},
 		"custom_string_map_key": map[customString]interface{}{customString("custom"): "custom"},
@@ -97,7 +97,7 @@ func TestTreeCreateToTreeInvalidArrayMemberType(t *testing.T) {
 }
 
 func TestTreeCreateToTreeInvalidTableGroupType(t *testing.T) {
-	_, err := TreeFromMap(map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"hello": t}}})
+	_, err := TreeFromMap(map[string]interface{}{"foo": []map[string]interface{}{{"hello": t}}})
 	expected := "cannot convert type *testing.T to Tree"
 	if err.Error() != expected {
 		t.Fatalf("expected error %s, got %s", expected, err.Error())

tomltree_write.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math"
 	"reflect"
 	"sort"
 	"strconv"
@@ -11,50 +12,102 @@ import (
 	"time"
 )
 
-// encodes a string to a TOML-compliant string value
+// Encodes a string to a TOML-compliant multi-line string value
-func encodeTomlString(value string) string {
+// This function is a clone of the existing encodeTomlString function, except that whitespace characters
-	result := ""
+// are preserved. Quotation marks and backslashes are also not escaped.
+func encodeMultilineTomlString(value string) string {
+	var b bytes.Buffer
+
 	for _, rr := range value {
 		switch rr {
 		case '\b':
-			result += "\\b"
+			b.WriteString(`\b`)
 		case '\t':
-			result += "\\t"
+			b.WriteString("\t")
 		case '\n':
-			result += "\\n"
+			b.WriteString("\n")
 		case '\f':
-			result += "\\f"
+			b.WriteString(`\f`)
 		case '\r':
-			result += "\\r"
+			b.WriteString("\r")
 		case '"':
-			result += "\\\""
+			b.WriteString(`"`)
 		case '\\':
-			result += "\\\\"
+			b.WriteString(`\`)
 		default:
 			intRr := uint16(rr)
 			if intRr < 0x001F {
-				result += fmt.Sprintf("\\u%0.4X", intRr)
+				b.WriteString(fmt.Sprintf("\\u%0.4X", intRr))
 			} else {
-				result += string(rr)
+				b.WriteRune(rr)
 			}
 		}
 	}
-	return result
+	return b.String()
 }
 
-func tomlValueStringRepresentation(v interface{}) (string, error) {
+// Encodes a string to a TOML-compliant string value
+func encodeTomlString(value string) string {
+	var b bytes.Buffer
+
+	for _, rr := range value {
+		switch rr {
+		case '\b':
+			b.WriteString(`\b`)
+		case '\t':
+			b.WriteString(`\t`)
+		case '\n':
+			b.WriteString(`\n`)
+		case '\f':
+			b.WriteString(`\f`)
+		case '\r':
+			b.WriteString(`\r`)
+		case '"':
+			b.WriteString(`\"`)
+		case '\\':
+			b.WriteString(`\\`)
+		default:
+			intRr := uint16(rr)
+			if intRr < 0x001F {
+				b.WriteString(fmt.Sprintf("\\u%0.4X", intRr))
+			} else {
+				b.WriteRune(rr)
+			}
+		}
+	}
+	return b.String()
+}
+
+func tomlValueStringRepresentation(v interface{}, indent string, arraysOneElementPerLine bool) (string, error) {
+	// this interface check is added to dereference the change made in the writeTo function.
+	// That change was made to allow this function to see formatting options.
+	tv, ok := v.(*tomlValue)
+	if ok {
+		v = tv.value
+	} else {
+		tv = &tomlValue{}
+	}
+
 	switch value := v.(type) {
 	case uint64:
 		return strconv.FormatUint(value, 10), nil
 	case int64:
 		return strconv.FormatInt(value, 10), nil
 	case float64:
-		return strconv.FormatFloat(value, 'f', -1, 32), nil
+		// Ensure a round float does contain a decimal point. Otherwise feeding
+		// the output back to the parser would convert to an integer.
+		if math.Trunc(value) == value {
+			return strings.ToLower(strconv.FormatFloat(value, 'f', 1, 32)), nil
+		}
+		return strings.ToLower(strconv.FormatFloat(value, 'f', -1, 32)), nil
 	case string:
+		if tv.multiline {
+			return "\"\"\"\n" + encodeMultilineTomlString(value) + "\"\"\"", nil
+		}
 		return "\"" + encodeTomlString(value) + "\"", nil
 	case []byte:
 		b, _ := v.([]byte)
-		return tomlValueStringRepresentation(string(b))
+		return tomlValueStringRepresentation(string(b), indent, arraysOneElementPerLine)
 	case bool:
 		if value {
 			return "true", nil
@@ -69,21 +122,38 @@ func tomlValueStringRepresentation(v interface{}) (string, error) {
 	rv := reflect.ValueOf(v)
 
 	if rv.Kind() == reflect.Slice {
-		values := []string{}
+		var values []string
 		for i := 0; i < rv.Len(); i++ {
 			item := rv.Index(i).Interface()
-			itemRepr, err := tomlValueStringRepresentation(item)
+			itemRepr, err := tomlValueStringRepresentation(item, indent, arraysOneElementPerLine)
 			if err != nil {
 				return "", err
 			}
 			values = append(values, itemRepr)
 		}
+		if arraysOneElementPerLine && len(values) > 1 {
+			stringBuffer := bytes.Buffer{}
+			valueIndent := indent + `  ` // TODO: move that to a shared encoder state
+
+			stringBuffer.WriteString("[\n")
+
+			for _, value := range values {
+				stringBuffer.WriteString(valueIndent)
+				stringBuffer.WriteString(value)
+				stringBuffer.WriteString(`,`)
+				stringBuffer.WriteString("\n")
+			}
+
+			stringBuffer.WriteString(indent + "]")
+
+			return stringBuffer.String(), nil
+		}
 		return "[" + strings.Join(values, ",") + "]", nil
 	}
 	return "", fmt.Errorf("unsupported value type %T: %v", v, v)
 }
 
-func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) {
+func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64, arraysOneElementPerLine bool) (int64, error) {
 	simpleValuesKeys := make([]string, 0)
 	complexValuesKeys := make([]string, 0)
 
@@ -106,13 +176,29 @@ func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (
 			return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k])
 		}
 
-		repr, err := tomlValueStringRepresentation(v.value)
+		repr, err := tomlValueStringRepresentation(v, indent, arraysOneElementPerLine)
 		if err != nil {
 			return bytesCount, err
 		}
 
-		kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr)
+		if v.comment != "" {
-		writtenBytesCount, err := w.Write([]byte(kvRepr))
+			comment := strings.Replace(v.comment, "\n", "\n"+indent+"#", -1)
+			start := "# "
+			if strings.HasPrefix(comment, "#") {
+				start = ""
+			}
+			writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment, "\n")
+			bytesCount += int64(writtenBytesCountComment)
+			if errc != nil {
+				return bytesCount, errc
+			}
+		}
+
+		var commented string
+		if v.commented {
+			commented = "# "
+		}
+		writtenBytesCount, err := writeStrings(w, indent, commented, k, " = ", repr, "\n")
 		bytesCount += int64(writtenBytesCount)
 		if err != nil {
 			return bytesCount, err
@@ -126,30 +212,48 @@ func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (
 		if keyspace != "" {
 			combinedKey = keyspace + "." + combinedKey
 		}
+		var commented string
+		if t.commented {
+			commented = "# "
+		}
 
 		switch node := v.(type) {
 		// node has to be of those two types given how keys are sorted above
 		case *Tree:
-			tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey)
+			tv, ok := t.values[k].(*Tree)
-			writtenBytesCount, err := w.Write([]byte(tableName))
+			if !ok {
+				return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k])
+			}
+			if tv.comment != "" {
+				comment := strings.Replace(tv.comment, "\n", "\n"+indent+"#", -1)
+				start := "# "
+				if strings.HasPrefix(comment, "#") {
+					start = ""
+				}
+				writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment)
+				bytesCount += int64(writtenBytesCountComment)
+				if errc != nil {
+					return bytesCount, errc
+				}
+			}
+			writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[", combinedKey, "]\n")
 			bytesCount += int64(writtenBytesCount)
 			if err != nil {
 				return bytesCount, err
 			}
-			bytesCount, err = node.writeTo(w, indent+"  ", combinedKey, bytesCount)
+			bytesCount, err = node.writeTo(w, indent+"  ", combinedKey, bytesCount, arraysOneElementPerLine)
 			if err != nil {
 				return bytesCount, err
 			}
 		case []*Tree:
 			for _, subTree := range node {
-				tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey)
+				writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[[", combinedKey, "]]\n")
-				writtenBytesCount, err := w.Write([]byte(tableArrayName))
 				bytesCount += int64(writtenBytesCount)
 				if err != nil {
 					return bytesCount, err
 				}
 
-				bytesCount, err = subTree.writeTo(w, indent+"  ", combinedKey, bytesCount)
+				bytesCount, err = subTree.writeTo(w, indent+"  ", combinedKey, bytesCount, arraysOneElementPerLine)
 				if err != nil {
 					return bytesCount, err
 				}
@@ -160,10 +264,22 @@ func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (
 	return bytesCount, nil
 }
 
+func writeStrings(w io.Writer, s ...string) (int, error) {
+	var n int
+	for i := range s {
+		b, err := io.WriteString(w, s[i])
+		n += b
+		if err != nil {
+			return n, err
+		}
+	}
+	return n, nil
+}
+
 // WriteTo encode the Tree as Toml and writes it to the writer w.
 // Returns the number of bytes written in case of success, or an error if anything happened.
 func (t *Tree) WriteTo(w io.Writer) (int64, error) {
-	return t.writeTo(w, "", "", 0)
+	return t.writeTo(w, "", "", 0, false)
 }
 
 // ToTomlString generates a human-readable representation of the current tree.

tomltree_write_test.go

@@ -16,26 +16,26 @@ type failingWriter struct {
 	buffer  bytes.Buffer
 }
 
-func (f failingWriter) Write(p []byte) (n int, err error) {
+func (f *failingWriter) Write(p []byte) (n int, err error) {
 	count := len(p)
-	toWrite := f.failAt - count + f.written
+	toWrite := f.failAt - (count + f.written)
 	if toWrite < 0 {
 		toWrite = 0
 	}
 	if toWrite > count {
 		f.written += count
-		f.buffer.WriteString(string(p))
+		f.buffer.Write(p)
 		return count, nil
 	}
 
-	f.buffer.WriteString(string(p[:toWrite]))
+	f.buffer.Write(p[:toWrite])
 	f.written = f.failAt
-	return f.written, fmt.Errorf("failingWriter failed after writting %d bytes", f.written)
+	return toWrite, fmt.Errorf("failingWriter failed after writing %d bytes", f.written)
 }
 
 func assertErrorString(t *testing.T, expected string, err error) {
 	expectedErr := errors.New(expected)
-	if err.Error() != expectedErr.Error() {
+	if err == nil || err.Error() != expectedErr.Error() {
 		t.Errorf("expecting error %s, but got %s instead", expected, err)
 	}
 }
@@ -161,13 +161,13 @@ func TestTreeWriteToInvalidTreeSimpleValue(t *testing.T) {
 }
 
 func TestTreeWriteToInvalidTreeTomlValue(t *testing.T) {
-	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{int8(1), Position{}}}}
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}}
 	_, err := tree.ToTomlString()
 	assertErrorString(t, "unsupported value type int8: 1", err)
 }
 
 func TestTreeWriteToInvalidTreeTomlValueArray(t *testing.T) {
-	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{[]interface{}{int8(1)}, Position{}}}}
+	tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}}
 	_, err := tree.ToTomlString()
 	assertErrorString(t, "unsupported value type int8: 1", err)
 }
@@ -175,8 +175,8 @@ func TestTreeWriteToInvalidTreeTomlValueArray(t *testing.T) {
 func TestTreeWriteToFailingWriterInSimpleValue(t *testing.T) {
 	toml, _ := Load(`a = 2`)
 	writer := failingWriter{failAt: 0, written: 0}
-	_, err := toml.WriteTo(writer)
+	_, err := toml.WriteTo(&writer)
-	assertErrorString(t, "failingWriter failed after writting 0 bytes", err)
+	assertErrorString(t, "failingWriter failed after writing 0 bytes", err)
 }
 
 func TestTreeWriteToFailingWriterInTable(t *testing.T) {
@@ -184,12 +184,12 @@ func TestTreeWriteToFailingWriterInTable(t *testing.T) {
 [b]
 a = 2`)
 	writer := failingWriter{failAt: 2, written: 0}
-	_, err := toml.WriteTo(writer)
+	_, err := toml.WriteTo(&writer)
-	assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
+	assertErrorString(t, "failingWriter failed after writing 2 bytes", err)
 
 	writer = failingWriter{failAt: 13, written: 0}
-	_, err = toml.WriteTo(writer)
+	_, err = toml.WriteTo(&writer)
-	assertErrorString(t, "failingWriter failed after writting 13 bytes", err)
+	assertErrorString(t, "failingWriter failed after writing 13 bytes", err)
 }
 
 func TestTreeWriteToFailingWriterInArray(t *testing.T) {
@@ -197,12 +197,12 @@ func TestTreeWriteToFailingWriterInArray(t *testing.T) {
 [[b]]
 a = 2`)
 	writer := failingWriter{failAt: 2, written: 0}
-	_, err := toml.WriteTo(writer)
+	_, err := toml.WriteTo(&writer)
-	assertErrorString(t, "failingWriter failed after writting 2 bytes", err)
+	assertErrorString(t, "failingWriter failed after writing 2 bytes", err)
 
 	writer = failingWriter{failAt: 15, written: 0}
-	_, err = toml.WriteTo(writer)
+	_, err = toml.WriteTo(&writer)
-	assertErrorString(t, "failingWriter failed after writting 15 bytes", err)
+	assertErrorString(t, "failingWriter failed after writing 15 bytes", err)
 }
 
 func TestTreeWriteToMapExampleFile(t *testing.T) {
@@ -293,3 +293,84 @@ func TestTreeWriteToMapWithArrayOfInlineTables(t *testing.T) {
 	treeMap := tree.ToMap()
 	testMaps(t, treeMap, expected)
 }
+
+func TestTreeWriteToFloat(t *testing.T) {
+	tree, err := Load(`a = 3.0`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	str, err := tree.ToTomlString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `a = 3.0`
+	if strings.TrimSpace(str) != strings.TrimSpace(expected) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str)
+	}
+}
+
+func TestTreeWriteToSpecialFloat(t *testing.T) {
+	expected := `a = +inf
+b = -inf
+c = nan`
+
+	tree, err := Load(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	str, err := tree.ToTomlString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.TrimSpace(str) != strings.TrimSpace(expected) {
+		t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str)
+	}
+}
+
+func BenchmarkTreeToTomlString(b *testing.B) {
+	toml, err := Load(sampleHard)
+	if err != nil {
+		b.Fatal("Unexpected error:", err)
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, err := toml.ToTomlString()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+var sampleHard = `# Test file for TOML
+# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate
+# This part you'll really hate
+
+[the]
+test_string = "You'll hate me after this - #"          # " Annoying, isn't it?
+
+    [the.hard]
+    test_array = [ "] ", " # "]      # ] There you go, parse this!
+    test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ]
+    # You didn't think it'd as easy as chucking out the last #, did you?
+    another_test_string = " Same thing, but with a string #"
+    harder_test_string = " And when \"'s are in the string, along with # \""   # "and comments are there too"
+    # Things will get harder
+
+        [the.hard."bit#"]
+        "what?" = "You don't think some user won't do that?"
+        multi_line_array = [
+            "]",
+            # ] Oh yes I did
+            ]
+
+# Each of the following keygroups/key value pairs should produce an error. Uncomment to them to test
+
+#[error]   if you didn't catch this, your parser is broken
+#string = "Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment"   like this
+#array = [
+#         "This might most likely happen in multiline arrays",
+#         Like here,
+#         "or here,
+#         and here"
+#         ]     End of array comment, forgot the #
+#number = 3.14  pi <--again forgot the #         `