Port toml-test to pure Go (#264)

* Port toml-test to pure Go

This change basically ports the toml-test examples test suite to pure
Go. This removes the snowflake test.sh required to run such tests, and
allows us to the example tests on any platform (which includes Windows
as part of the pull-request testing).

* Allow CircleCI failure for go tip

.circleci/config.yml

@@ -27,6 +27,9 @@ commands:
             coverage:
                 default: false
                 type: boolean
+            allow_fail:
+                type: boolean
+                default: false
         steps:
             - run:
                 name: "Run tests for <<parameters.test_name>>"
@@ -36,13 +39,16 @@ commands:
                     trap "go-junit-report </tmp/test-results/go-test.out > ${TEST_DIR}/go-test-report.xml" EXIT
                     go test <<parameters.module>> -race -v \
                         <<# parameters.coverage >>-coverprofile=/tmp/workspace/coverage.txt -covermode=atomic<</ parameters.coverage >> \
-                        | tee /tmp/test-results/go-test.out
+                        | tee /tmp/test-results/go-test.out <<# parameters.allow_fail >>|| true<</ parameters.allow_fail >>
 
 jobs:
     go:
         parameters:
             version:
                 type: string
+            allow_fail:
+                type: boolean
+                default: false
         executor:
             name: golang
             version: "<<parameters.version>>"
@@ -50,21 +56,25 @@ jobs:
         steps:
             - checkout
             - run: mkdir -p /tmp/workspace
-            - run: go fmt ./...
+            - run: go fmt ./... <<# parameters.allow_fail >>|| true<</ parameters.allow_fail >>
             - get_deps
             - run_test:
                 test_name: "go-toml"
                 module: "github.com/pelletier/go-toml"
                 coverage: true
+                allow_fail: <<parameters.allow_fail>>
             - run_test:
                 test_name: "tomljson"
                 module: "github.com/pelletier/go-toml/cmd/tomljson"
+                allow_fail: <<parameters.allow_fail>>
             - run_test:
                 test_name: "tomll"
                 module: "github.com/pelletier/go-toml/cmd/tomll"
+                allow_fail: <<parameters.allow_fail>>
             - run_test:
                 test_name: "query"
                 module: "github.com/pelletier/go-toml/query"
+                allow_fail: <<parameters.allow_fail>>
             - store_test_results:
                 path: /tmp/test-results
     codecov:
@@ -103,6 +113,7 @@ workflows:
             - go:
                 name: "gotip"
                 version: "1.12" # use as base
+                allow_fail: true
                 pre-steps:
                     - restore_cache:
                         keys:

.travis.yml

@@ -10,7 +10,13 @@ matrix:
   fast_finish: true
 script:
   - if [ -n "$(go fmt ./...)" ]; then exit 1; fi
-  - ./test.sh
+  - go get github.com/davecgh/go-spew/spew
+  - go get gopkg.in/yaml.v2
+  - go get github.com/BurntSushi/toml
+  - go test github.com/pelletier/go-toml -race -coverprofile=coverage.txt -covermode=atomic
+  - go test github.com/pelletier/go-toml/cmd/tomljson
+  - go test github.com/pelletier/go-toml/cmd/tomll
+  - go test github.com/pelletier/go-toml/query
   - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git
 
 after_success:

CONTRIBUTING.md

@@ -83,9 +83,7 @@ some early feedback!
 
 #### Run the tests
 
-go-toml uses two kind of tests: unit tests and TOML example tests.
+You can run tests for go-toml using Go's test tool: `go test ./...`.
-You can run both of them using ./test.sh.
-
 When creating a pull requests, all tests will be ran on Linux on a few Go
 versions (Travis CI), and on Windows using the latest Go version
 (AppVeyor).

README.md

@@ -109,12 +109,7 @@ much appreciated!
 
 ### Run tests
 
-You have to make sure two kind of tests run:
+`go test ./...`
-
-1. The Go unit tests
-2. The TOML examples base
-
-You can run both of them using `./test.sh`.
 
 ### Fuzzing
 

appveyor.yml

@@ -30,7 +30,7 @@ test_script:
   - go get github.com/davecgh/go-spew/spew
   - go get gopkg.in/yaml.v2
   - go get github.com/BurntSushi/toml
-  - go build github.com/pelletier/go-toml
   - go test github.com/pelletier/go-toml
   - go test github.com/pelletier/go-toml/cmd/tomljson
+  - go test github.com/pelletier/go-toml/cmd/tomll
   - go test github.com/pelletier/go-toml/query

cmd/test_program.go

@@ -1,91 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"time"
-
-	"github.com/pelletier/go-toml"
-)
-
-func main() {
-	bytes, err := ioutil.ReadAll(os.Stdin)
-	if err != nil {
-		log.Fatalf("Error during TOML read: %s", err)
-		os.Exit(2)
-	}
-	tree, err := toml.Load(string(bytes))
-	if err != nil {
-		log.Fatalf("Error during TOML load: %s", err)
-		os.Exit(1)
-	}
-
-	typedTree := translate(*tree)
-
-	if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
-		log.Fatalf("Error encoding JSON: %s", err)
-		os.Exit(3)
-	}
-
-	os.Exit(0)
-}
-
-func translate(tomlData interface{}) interface{} {
-	switch orig := tomlData.(type) {
-	case map[string]interface{}:
-		typed := make(map[string]interface{}, len(orig))
-		for k, v := range orig {
-			typed[k] = translate(v)
-		}
-		return typed
-	case *toml.Tree:
-		return translate(*orig)
-	case toml.Tree:
-		keys := orig.Keys()
-		typed := make(map[string]interface{}, len(keys))
-		for _, k := range keys {
-			typed[k] = translate(orig.GetPath([]string{k}))
-		}
-		return typed
-	case []*toml.Tree:
-		typed := make([]map[string]interface{}, len(orig))
-		for i, v := range orig {
-			typed[i] = translate(v).(map[string]interface{})
-		}
-		return typed
-	case []map[string]interface{}:
-		typed := make([]map[string]interface{}, len(orig))
-		for i, v := range orig {
-			typed[i] = translate(v).(map[string]interface{})
-		}
-		return typed
-	case []interface{}:
-		typed := make([]interface{}, len(orig))
-		for i, v := range orig {
-			typed[i] = translate(v)
-		}
-		return tag("array", typed)
-	case time.Time:
-		return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
-	case bool:
-		return tag("bool", fmt.Sprintf("%v", orig))
-	case int64:
-		return tag("integer", fmt.Sprintf("%d", orig))
-	case float64:
-		return tag("float", fmt.Sprintf("%v", orig))
-	case string:
-		return tag("string", orig)
-	}
-
-	panic(fmt.Sprintf("Unknown type: %T", tomlData))
-}
-
-func tag(typeName string, data interface{}) map[string]interface{} {
-	return map[string]interface{}{
-		"type":  typeName,
-		"value": data,
-	}
-}

cmd/tomltestgen/main.go

@@ -0,0 +1,219 @@
+// Tomltestgen is a program that retrieves a given version of
+// https://github.com/BurntSushi/toml-test and generates go code for go-toml's unit tests
+// based on the test files.
+//
+// Usage: go run github.com/pelletier/go-toml/cmd/tomltestgen > toml_testgen_test.go
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+)
+
+type invalid struct {
+	Name  string
+	Input string
+}
+
+type valid struct {
+	Name    string
+	Input   string
+	JsonRef string
+}
+
+type testsCollection struct {
+	Ref       string
+	Timestamp string
+	Invalid   []invalid
+	Valid     []valid
+	Count     int
+}
+
+const srcTemplate = "// Generated by tomltestgen for toml-test ref {{.Ref}} on {{.Timestamp}}\n" +
+	"package toml\n" +
+	" import (\n" +
+	"	\"testing\"\n" +
+	")\n" +
+
+	"{{range .Invalid}}\n" +
+	"func TestInvalid{{.Name}}(t *testing.T) {\n" +
+	"	input := {{.Input|gostr}}\n" +
+	"	testgenInvalid(t, input)\n" +
+	"}\n" +
+	"{{end}}\n" +
+	"\n" +
+	"{{range .Valid}}\n" +
+	"func TestValid{{.Name}}(t *testing.T) {\n" +
+	"   input := {{.Input|gostr}}\n" +
+	"   jsonRef := {{.JsonRef|gostr}}\n" +
+	"   testgenValid(t, input, jsonRef)\n" +
+	"}\n" +
+	"{{end}}\n"
+
+func downloadTmpFile(url string) string {
+	log.Println("starting to download file from", url)
+	resp, err := http.Get(url)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	tmpfile, err := ioutil.TempFile("", "toml-test-*.zip")
+	if err != nil {
+		panic(err)
+	}
+	defer tmpfile.Close()
+
+	copiedLen, err := io.Copy(tmpfile, resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	if resp.ContentLength > 0 && copiedLen != resp.ContentLength {
+		panic(fmt.Errorf("copied %d bytes, request body had %d", copiedLen, resp.ContentLength))
+	}
+	return tmpfile.Name()
+}
+
+func kebabToCamel(kebab string) string {
+	camel := ""
+	nextUpper := true
+	for _, c := range kebab {
+		if nextUpper {
+			camel += strings.ToUpper(string(c))
+			nextUpper = false
+		} else if c == '-' {
+			nextUpper = true
+		} else {
+			camel += string(c)
+		}
+	}
+	return camel
+}
+
+func readFileFromZip(f *zip.File) string {
+	reader, err := f.Open()
+	if err != nil {
+		panic(err)
+	}
+	defer reader.Close()
+	bytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func templateGoStr(input string) string {
+	if len(input) > 0 && input[len(input)-1] == '\n' {
+		input = input[0 : len(input)-1]
+	}
+	if strings.Contains(input, "`") {
+		lines := strings.Split(input, "\n")
+		for idx, line := range lines {
+			lines[idx] = strconv.Quote(line + "\n")
+		}
+		return strings.Join(lines, " + \n")
+	}
+	return "`" + input + "`"
+}
+
+var (
+	ref = flag.String("r", "master", "git reference")
+)
+
+func usage() {
+	_, _ = fmt.Fprintf(os.Stderr, "usage: tomltestgen [flags]\n")
+	flag.PrintDefaults()
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+
+	url := "https://codeload.github.com/BurntSushi/toml-test/zip/" + *ref
+	resultFile := downloadTmpFile(url)
+	defer os.Remove(resultFile)
+	log.Println("file written to", resultFile)
+
+	zipReader, err := zip.OpenReader(resultFile)
+	if err != nil {
+		panic(err)
+	}
+	defer zipReader.Close()
+
+	collection := testsCollection{
+		Ref:       *ref,
+		Timestamp: time.Now().Format(time.RFC3339),
+	}
+
+	zipFilesMap := map[string]*zip.File{}
+
+	for _, f := range zipReader.File {
+		zipFilesMap[f.Name] = f
+	}
+
+	testFileRegexp := regexp.MustCompile(`([^/]+/tests/(valid|invalid)/(.+))\.(toml)`)
+	for _, f := range zipReader.File {
+		groups := testFileRegexp.FindStringSubmatch(f.Name)
+		if len(groups) > 0 {
+			name := kebabToCamel(groups[3])
+			testType := groups[2]
+
+			log.Printf("> [%s] %s\n", testType, name)
+
+			tomlContent := readFileFromZip(f)
+
+			switch testType {
+			case "invalid":
+				collection.Invalid = append(collection.Invalid, invalid{
+					Name:  name,
+					Input: tomlContent,
+				})
+				collection.Count++
+			case "valid":
+				baseFilePath := groups[1]
+				jsonFilePath := baseFilePath + ".json"
+				jsonContent := readFileFromZip(zipFilesMap[jsonFilePath])
+
+				collection.Valid = append(collection.Valid, valid{
+					Name:    name,
+					Input:   tomlContent,
+					JsonRef: jsonContent,
+				})
+				collection.Count++
+			default:
+				panic(fmt.Sprintf("unknown test type: %s", testType))
+			}
+		}
+	}
+
+	log.Printf("Collected %d tests from toml-test\n", collection.Count)
+
+	funcMap := template.FuncMap{
+		"gostr": templateGoStr,
+	}
+	t := template.Must(template.New("src").Funcs(funcMap).Parse(srcTemplate))
+	buf := new(bytes.Buffer)
+	err = t.Execute(buf, collection)
+	if err != nil {
+		panic(err)
+	}
+	outputBytes, err := format.Source(buf.Bytes())
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(string(outputBytes))
+}

test.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-# fail out of the script if anything here fails
-set -e
-set -o pipefail
-
-# set the path to the present working directory
-export GOPATH=`pwd`
-
-function git_clone() {
-  path=$1
-  branch=$2
-  version=$3
-  if [ ! -d "src/$path" ]; then
-    mkdir -p src/$path
-    git clone https://$path.git src/$path
-  fi
-  pushd src/$path
-  git checkout "$branch"
-  git reset --hard "$version"
-  popd
-}
-
-# Remove potential previous runs
-rm -rf src test_program_bin toml-test
-
-go get github.com/davecgh/go-spew/spew
-go get gopkg.in/yaml.v2
-go get github.com/BurntSushi/toml
-
-# get code for BurntSushi TOML validation
-git_clone github.com/BurntSushi/toml master a368813
-git_clone github.com/BurntSushi/toml-test master 39e37e6
-
-# build the BurntSushi test application
-go build -o toml-test github.com/BurntSushi/toml-test
-
-# vendorize the current lib for testing
-# NOTE: this basically mocks an install without having to go back out to github for code
-mkdir -p src/github.com/pelletier/go-toml/cmd
-mkdir -p src/github.com/pelletier/go-toml/query
-cp *.go *.toml src/github.com/pelletier/go-toml
-cp -R cmd/* src/github.com/pelletier/go-toml/cmd
-cp -R query/* src/github.com/pelletier/go-toml/query
-go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go
-
-# Run basic unit tests
-go test github.com/pelletier/go-toml -race -coverprofile=coverage.txt -covermode=atomic
-go test github.com/pelletier/go-toml/cmd/tomljson
-go test github.com/pelletier/go-toml/query
-
-# run the entire BurntSushi test suite
-if [[ $# -eq 0 ]] ; then
-  echo "Running all BurntSushi tests"
-  ./toml-test ./test_program_bin | tee test_out
-else
-  # run a specific test
-  test=$1
-  test_path='src/github.com/BurntSushi/toml-test/tests'
-  valid_test="$test_path/valid/$test"
-  invalid_test="$test_path/invalid/$test"
-
-  if [ -e "$valid_test.toml" ]; then
-    echo "Valid Test TOML for $test:"
-    echo "===="
-    cat "$valid_test.toml"
-
-    echo "Valid Test JSON for $test:"
-    echo "===="
-    cat "$valid_test.json"
-
-    echo "Go-TOML Output for $test:"
-    echo "===="
-    cat "$valid_test.toml" | ./test_program_bin
-  fi
-
-  if [ -e "$invalid_test.toml" ]; then
-    echo "Invalid Test TOML for $test:"
-    echo "===="
-    cat "$invalid_test.toml"
-
-    echo "Go-TOML Output for $test:"
-    echo "===="
-    echo "go-toml Output:"
-    cat "$invalid_test.toml" | ./test_program_bin
-  fi
-fi

toml_testgen_support_test.go

@@ -0,0 +1,119 @@
+// This is a support file for toml_testgen_test.go
+package toml
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+func testgenInvalid(t *testing.T, input string) {
+	t.Logf("Input TOML:\n%s", input)
+	tree, err := Load(input)
+	if err != nil {
+		return
+	}
+
+	typedTree := testgenTranslate(*tree)
+
+	buf := new(bytes.Buffer)
+	if err := json.NewEncoder(buf).Encode(typedTree); err != nil {
+		return
+	}
+
+	t.Fatalf("test did not fail. resulting tree:\n%s", buf.String())
+}
+
+func testgenValid(t *testing.T, input string, jsonRef string) {
+	t.Logf("Input TOML:\n%s", input)
+	tree, err := Load(input)
+	if err != nil {
+		t.Fatalf("failed parsing toml: %s", err)
+	}
+
+	typedTree := testgenTranslate(*tree)
+
+	buf := new(bytes.Buffer)
+	if err := json.NewEncoder(buf).Encode(typedTree); err != nil {
+		t.Fatalf("failed translating to JSON: %s", err)
+	}
+
+	var jsonTest interface{}
+	if err := json.NewDecoder(buf).Decode(&jsonTest); err != nil {
+		t.Logf("translated JSON:\n%s", buf.String())
+		t.Fatalf("failed decoding translated JSON: %s", err)
+	}
+
+	var jsonExpected interface{}
+	if err := json.NewDecoder(bytes.NewBufferString(jsonRef)).Decode(&jsonExpected); err != nil {
+		t.Logf("reference JSON:\n%s", jsonRef)
+		t.Fatalf("failed decoding reference JSON: %s", err)
+	}
+
+	if !reflect.DeepEqual(jsonExpected, jsonTest) {
+		t.Logf("Diff:\n%s", spew.Sdump(jsonExpected, jsonTest))
+		t.Fatal("parsed TOML tree is different than expected structure")
+	}
+}
+
+func testgenTranslate(tomlData interface{}) interface{} {
+	switch orig := tomlData.(type) {
+	case map[string]interface{}:
+		typed := make(map[string]interface{}, len(orig))
+		for k, v := range orig {
+			typed[k] = testgenTranslate(v)
+		}
+		return typed
+	case *Tree:
+		return testgenTranslate(*orig)
+	case Tree:
+		keys := orig.Keys()
+		typed := make(map[string]interface{}, len(keys))
+		for _, k := range keys {
+			typed[k] = testgenTranslate(orig.GetPath([]string{k}))
+		}
+		return typed
+	case []*Tree:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = testgenTranslate(v).(map[string]interface{})
+		}
+		return typed
+	case []map[string]interface{}:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = testgenTranslate(v).(map[string]interface{})
+		}
+		return typed
+	case []interface{}:
+		typed := make([]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = testgenTranslate(v)
+		}
+		return testgenTag("array", typed)
+	case time.Time:
+		return testgenTag("datetime", orig.Format("2006-01-02T15:04:05Z"))
+	case bool:
+		return testgenTag("bool", fmt.Sprintf("%v", orig))
+	case int64:
+		return testgenTag("integer", fmt.Sprintf("%d", orig))
+	case float64:
+		return testgenTag("float", fmt.Sprintf("%v", orig))
+	case string:
+		return testgenTag("string", orig)
+	}
+
+	panic(fmt.Sprintf("Unknown type: %T", tomlData))
+}
+
+func testgenTag(typeName string, data interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"type":  typeName,
+		"value": data,
+	}
+}

toml_testgen_test.go

@@ -0,0 +1,943 @@
+// Generated by tomltestgen for toml-test ref 39e37e6 on 2019-03-19T23:58:45-07:00
+package toml
+
+import (
+	"testing"
+)
+
+func TestInvalidArrayMixedTypesArraysAndInts(t *testing.T) {
+	input := `arrays-and-ints =  [1, ["Arrays are not integers."]]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidArrayMixedTypesIntsAndFloats(t *testing.T) {
+	input := `ints-and-floats = [1, 1.1]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidArrayMixedTypesStringsAndInts(t *testing.T) {
+	input := `strings-and-ints = ["hi", 42]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDatetimeMalformedNoLeads(t *testing.T) {
+	input := `no-leads = 1987-7-05T17:45:00Z`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDatetimeMalformedNoSecs(t *testing.T) {
+	input := `no-secs = 1987-07-05T17:45Z`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDatetimeMalformedNoT(t *testing.T) {
+	input := `no-t = 1987-07-0517:45:00Z`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDatetimeMalformedWithMilli(t *testing.T) {
+	input := `with-milli = 1987-07-5T17:45:00.12Z`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDuplicateKeyTable(t *testing.T) {
+	input := `[fruit]
+type = "apple"
+
+[fruit.type]
+apple = "yes"`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDuplicateKeys(t *testing.T) {
+	input := `dupe = false
+dupe = true`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidDuplicateTables(t *testing.T) {
+	input := `[a]
+[a]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidEmptyImplicitTable(t *testing.T) {
+	input := `[naughty..naughty]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidEmptyTable(t *testing.T) {
+	input := `[]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidFloatNoLeadingZero(t *testing.T) {
+	input := `answer = .12345
+neganswer = -.12345`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidFloatNoTrailingDigits(t *testing.T) {
+	input := `answer = 1.
+neganswer = -1.`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyEmpty(t *testing.T) {
+	input := ` = 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyHash(t *testing.T) {
+	input := `a# = 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyNewline(t *testing.T) {
+	input := `a
+= 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyOpenBracket(t *testing.T) {
+	input := `[abc = 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeySingleOpenBracket(t *testing.T) {
+	input := `[`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeySpace(t *testing.T) {
+	input := `a b = 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyStartBracket(t *testing.T) {
+	input := `[a]
+[xyz = 5
+[b]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidKeyTwoEquals(t *testing.T) {
+	input := `key= = 1`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidStringBadByteEscape(t *testing.T) {
+	input := `naughty = "\xAg"`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidStringBadEscape(t *testing.T) {
+	input := `invalid-escape = "This string has a bad \a escape character."`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidStringByteEscapes(t *testing.T) {
+	input := `answer = "\x33"`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidStringNoClose(t *testing.T) {
+	input := `no-ending-quote = "One time, at band camp`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableArrayImplicit(t *testing.T) {
+	input := "# This test is a bit tricky. It should fail because the first use of\n" +
+		"# `[[albums.songs]]` without first declaring `albums` implies that `albums`\n" +
+		"# must be a table. The alternative would be quite weird. Namely, it wouldn't\n" +
+		"# comply with the TOML spec: \"Each double-bracketed sub-table will belong to \n" +
+		"# the most *recently* defined table element *above* it.\"\n" +
+		"#\n" +
+		"# This is in contrast to the *valid* test, table-array-implicit where\n" +
+		"# `[[albums.songs]]` works by itself, so long as `[[albums]]` isn't declared\n" +
+		"# later. (Although, `[albums]` could be.)\n" +
+		"[[albums.songs]]\n" +
+		"name = \"Glory Days\"\n" +
+		"\n" +
+		"[[albums]]\n" +
+		"name = \"Born in the USA\"\n"
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableArrayMalformedBracket(t *testing.T) {
+	input := `[[albums]
+name = "Born to Run"`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableArrayMalformedEmpty(t *testing.T) {
+	input := `[[]]
+name = "Born to Run"`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableEmpty(t *testing.T) {
+	input := `[]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableNestedBracketsClose(t *testing.T) {
+	input := `[a]b]
+zyx = 42`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableNestedBracketsOpen(t *testing.T) {
+	input := `[a[b]
+zyx = 42`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableWhitespace(t *testing.T) {
+	input := `[invalid key]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTableWithPound(t *testing.T) {
+	input := `[key#group]
+answer = 42`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextAfterArrayEntries(t *testing.T) {
+	input := `array = [
+  "Is there life after an array separator?", No
+  "Entry"
+]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextAfterInteger(t *testing.T) {
+	input := `answer = 42 the ultimate answer?`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextAfterString(t *testing.T) {
+	input := `string = "Is there life after strings?" No.`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextAfterTable(t *testing.T) {
+	input := `[error] this shouldn't be here`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextBeforeArraySeparator(t *testing.T) {
+	input := `array = [
+  "Is there life before an array separator?" No,
+  "Entry"
+]`
+	testgenInvalid(t, input)
+}
+
+func TestInvalidTextInArray(t *testing.T) {
+	input := `array = [
+  "Entry 1",
+  I don't belong,
+  "Entry 2",
+]`
+	testgenInvalid(t, input)
+}
+
+func TestValidArrayEmpty(t *testing.T) {
+	input := `thevoid = [[[[[]]]]]`
+	jsonRef := `{
+    "thevoid": { "type": "array", "value": [
+        {"type": "array", "value": [
+            {"type": "array", "value": [
+                {"type": "array", "value": [
+                    {"type": "array", "value": []}
+                ]}
+            ]}
+        ]}
+    ]}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidArrayNospaces(t *testing.T) {
+	input := `ints = [1,2,3]`
+	jsonRef := `{
+    "ints": {
+        "type": "array",
+        "value": [
+            {"type": "integer", "value": "1"},
+            {"type": "integer", "value": "2"},
+            {"type": "integer", "value": "3"}
+        ]
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidArraysHetergeneous(t *testing.T) {
+	input := `mixed = [[1, 2], ["a", "b"], [1.1, 2.1]]`
+	jsonRef := `{
+    "mixed": {
+        "type": "array",
+        "value": [
+            {"type": "array", "value": [
+                {"type": "integer", "value": "1"},
+                {"type": "integer", "value": "2"}
+            ]},
+            {"type": "array", "value": [
+                {"type": "string", "value": "a"},
+                {"type": "string", "value": "b"}
+            ]},
+            {"type": "array", "value": [
+                {"type": "float", "value": "1.1"},
+                {"type": "float", "value": "2.1"}
+            ]}
+        ]
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidArraysNested(t *testing.T) {
+	input := `nest = [["a"], ["b"]]`
+	jsonRef := `{
+    "nest": {
+        "type": "array",
+        "value": [
+            {"type": "array", "value": [
+                {"type": "string", "value": "a"}
+            ]},
+            {"type": "array", "value": [
+                {"type": "string", "value": "b"}
+            ]}
+        ]
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidArrays(t *testing.T) {
+	input := `ints = [1, 2, 3]
+floats = [1.1, 2.1, 3.1]
+strings = ["a", "b", "c"]
+dates = [
+  1987-07-05T17:45:00Z,
+  1979-05-27T07:32:00Z,
+  2006-06-01T11:00:00Z,
+]`
+	jsonRef := `{
+    "ints": {
+        "type": "array",
+        "value": [
+            {"type": "integer", "value": "1"},
+            {"type": "integer", "value": "2"},
+            {"type": "integer", "value": "3"}
+        ]
+    },
+    "floats": {
+        "type": "array",
+        "value": [
+            {"type": "float", "value": "1.1"},
+            {"type": "float", "value": "2.1"},
+            {"type": "float", "value": "3.1"}
+        ]
+    },
+    "strings": {
+        "type": "array",
+        "value": [
+            {"type": "string", "value": "a"},
+            {"type": "string", "value": "b"},
+            {"type": "string", "value": "c"}
+        ]
+    },
+    "dates": {
+        "type": "array",
+        "value": [
+            {"type": "datetime", "value": "1987-07-05T17:45:00Z"},
+            {"type": "datetime", "value": "1979-05-27T07:32:00Z"},
+            {"type": "datetime", "value": "2006-06-01T11:00:00Z"}
+        ]
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidBool(t *testing.T) {
+	input := `t = true
+f = false`
+	jsonRef := `{
+    "f": {"type": "bool", "value": "false"},
+    "t": {"type": "bool", "value": "true"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidCommentsEverywhere(t *testing.T) {
+	input := `# Top comment.
+  # Top comment.
+# Top comment.
+
+# [no-extraneous-groups-please]
+
+[group] # Comment
+answer = 42 # Comment
+# no-extraneous-keys-please = 999
+# Inbetween comment.
+more = [ # Comment
+  # What about multiple # comments?
+  # Can you handle it?
+  #
+          # Evil.
+# Evil.
+  42, 42, # Comments within arrays are fun.
+  # What about multiple # comments?
+  # Can you handle it?
+  #
+          # Evil.
+# Evil.
+# ] Did I fool you?
+] # Hopefully not.`
+	jsonRef := `{
+    "group": {
+        "answer": {"type": "integer", "value": "42"},
+        "more": {
+            "type": "array",
+            "value": [
+                {"type": "integer", "value": "42"},
+                {"type": "integer", "value": "42"}
+            ]
+        }
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidDatetime(t *testing.T) {
+	input := `bestdayever = 1987-07-05T17:45:00Z`
+	jsonRef := `{
+    "bestdayever": {"type": "datetime", "value": "1987-07-05T17:45:00Z"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidEmpty(t *testing.T) {
+	input := ``
+	jsonRef := `{}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidExample(t *testing.T) {
+	input := `best-day-ever = 1987-07-05T17:45:00Z
+
+[numtheory]
+boring = false
+perfection = [6, 28, 496]`
+	jsonRef := `{
+  "best-day-ever": {"type": "datetime", "value": "1987-07-05T17:45:00Z"},
+  "numtheory": {
+    "boring": {"type": "bool", "value": "false"},
+    "perfection": {
+      "type": "array",
+      "value": [
+        {"type": "integer", "value": "6"},
+        {"type": "integer", "value": "28"},
+        {"type": "integer", "value": "496"}
+      ]
+    }
+  }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidFloat(t *testing.T) {
+	input := `pi = 3.14
+negpi = -3.14`
+	jsonRef := `{
+    "pi": {"type": "float", "value": "3.14"},
+    "negpi": {"type": "float", "value": "-3.14"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidImplicitAndExplicitAfter(t *testing.T) {
+	input := `[a.b.c]
+answer = 42
+
+[a]
+better = 43`
+	jsonRef := `{
+    "a": {
+        "better": {"type": "integer", "value": "43"},
+        "b": {
+            "c": {
+                "answer": {"type": "integer", "value": "42"}
+            }
+        }
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidImplicitAndExplicitBefore(t *testing.T) {
+	input := `[a]
+better = 43
+
+[a.b.c]
+answer = 42`
+	jsonRef := `{
+    "a": {
+        "better": {"type": "integer", "value": "43"},
+        "b": {
+            "c": {
+                "answer": {"type": "integer", "value": "42"}
+            }
+        }
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidImplicitGroups(t *testing.T) {
+	input := `[a.b.c]
+answer = 42`
+	jsonRef := `{
+    "a": {
+        "b": {
+            "c": {
+                "answer": {"type": "integer", "value": "42"}
+            }
+        }
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidInteger(t *testing.T) {
+	input := `answer = 42
+neganswer = -42`
+	jsonRef := `{
+    "answer": {"type": "integer", "value": "42"},
+    "neganswer": {"type": "integer", "value": "-42"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidKeyEqualsNospace(t *testing.T) {
+	input := `answer=42`
+	jsonRef := `{
+    "answer": {"type": "integer", "value": "42"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidKeySpace(t *testing.T) {
+	input := `"a b" = 1`
+	jsonRef := `{
+    "a b": {"type": "integer", "value": "1"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidKeySpecialChars(t *testing.T) {
+	input := "\"~!@$^&*()_+-`1234567890[]|/?><.,;:'\" = 1\n"
+	jsonRef := "{\n" +
+		"    \"~!@$^&*()_+-`1234567890[]|/?><.,;:'\": {\n" +
+		"        \"type\": \"integer\", \"value\": \"1\"\n" +
+		"    }\n" +
+		"}\n"
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidLongFloat(t *testing.T) {
+	input := `longpi = 3.141592653589793
+neglongpi = -3.141592653589793`
+	jsonRef := `{
+    "longpi": {"type": "float", "value": "3.141592653589793"},
+    "neglongpi": {"type": "float", "value": "-3.141592653589793"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidLongInteger(t *testing.T) {
+	input := `answer = 9223372036854775807
+neganswer = -9223372036854775808`
+	jsonRef := `{
+    "answer": {"type": "integer", "value": "9223372036854775807"},
+    "neganswer": {"type": "integer", "value": "-9223372036854775808"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidMultilineString(t *testing.T) {
+	input := `multiline_empty_one = """"""
+multiline_empty_two = """
+"""
+multiline_empty_three = """\
+    """
+multiline_empty_four = """\
+   \
+   \
+   """
+
+equivalent_one = "The quick brown fox jumps over the lazy dog."
+equivalent_two = """
+The quick brown \
+
+
+  fox jumps over \
+    the lazy dog."""
+
+equivalent_three = """\
+       The quick brown \
+       fox jumps over \
+       the lazy dog.\
+       """`
+	jsonRef := `{
+    "multiline_empty_one": {
+        "type": "string",
+        "value": ""
+    },
+    "multiline_empty_two": {
+        "type": "string",
+        "value": ""
+    },
+    "multiline_empty_three": {
+        "type": "string",
+        "value": ""
+    },
+    "multiline_empty_four": {
+        "type": "string",
+        "value": ""
+    },
+    "equivalent_one": {
+        "type": "string",
+        "value": "The quick brown fox jumps over the lazy dog."
+    },
+    "equivalent_two": {
+        "type": "string",
+        "value": "The quick brown fox jumps over the lazy dog."
+    },
+    "equivalent_three": {
+        "type": "string",
+        "value": "The quick brown fox jumps over the lazy dog."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidRawMultilineString(t *testing.T) {
+	input := `oneline = '''This string has a ' quote character.'''
+firstnl = '''
+This string has a ' quote character.'''
+multiline = '''
+This string
+has ' a quote character
+and more than
+one newline
+in it.'''`
+	jsonRef := `{
+    "oneline": {
+        "type": "string",
+        "value": "This string has a ' quote character."
+    },
+    "firstnl": {
+        "type": "string",
+        "value": "This string has a ' quote character."
+    },
+    "multiline": {
+        "type": "string",
+        "value": "This string\nhas ' a quote character\nand more than\none newline\nin it."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidRawString(t *testing.T) {
+	input := `backspace = 'This string has a \b backspace character.'
+tab = 'This string has a \t tab character.'
+newline = 'This string has a \n new line character.'
+formfeed = 'This string has a \f form feed character.'
+carriage = 'This string has a \r carriage return character.'
+slash = 'This string has a \/ slash character.'
+backslash = 'This string has a \\ backslash character.'`
+	jsonRef := `{
+    "backspace": {
+        "type": "string",
+        "value": "This string has a \\b backspace character."
+    },
+    "tab": {
+        "type": "string",
+        "value": "This string has a \\t tab character."
+    },
+    "newline": {
+        "type": "string",
+        "value": "This string has a \\n new line character."
+    },
+    "formfeed": {
+        "type": "string",
+        "value": "This string has a \\f form feed character."
+    },
+    "carriage": {
+        "type": "string",
+        "value": "This string has a \\r carriage return character."
+    },
+    "slash": {
+        "type": "string",
+        "value": "This string has a \\/ slash character."
+    },
+    "backslash": {
+        "type": "string",
+        "value": "This string has a \\\\ backslash character."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidStringEmpty(t *testing.T) {
+	input := `answer = ""`
+	jsonRef := `{
+    "answer": {
+        "type": "string",
+        "value": ""
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidStringEscapes(t *testing.T) {
+	input := `backspace = "This string has a \b backspace character."
+tab = "This string has a \t tab character."
+newline = "This string has a \n new line character."
+formfeed = "This string has a \f form feed character."
+carriage = "This string has a \r carriage return character."
+quote = "This string has a \" quote character."
+backslash = "This string has a \\ backslash character."
+notunicode1 = "This string does not have a unicode \\u escape."
+notunicode2 = "This string does not have a unicode \u005Cu escape."
+notunicode3 = "This string does not have a unicode \\u0075 escape."
+notunicode4 = "This string does not have a unicode \\\u0075 escape."`
+	jsonRef := `{
+    "backspace": {
+        "type": "string",
+        "value": "This string has a \u0008 backspace character."
+    },
+    "tab": {
+        "type": "string",
+        "value": "This string has a \u0009 tab character."
+    },
+    "newline": {
+        "type": "string",
+        "value": "This string has a \u000A new line character."
+    },
+    "formfeed": {
+        "type": "string",
+        "value": "This string has a \u000C form feed character."
+    },
+    "carriage": {
+        "type": "string",
+        "value": "This string has a \u000D carriage return character."
+    },
+    "quote": {
+        "type": "string",
+        "value": "This string has a \u0022 quote character."
+    },
+    "backslash": {
+        "type": "string",
+        "value": "This string has a \u005C backslash character."
+    },
+    "notunicode1": {
+        "type": "string",
+        "value": "This string does not have a unicode \\u escape."
+    },
+    "notunicode2": {
+        "type": "string",
+        "value": "This string does not have a unicode \u005Cu escape."
+    },
+    "notunicode3": {
+        "type": "string",
+        "value": "This string does not have a unicode \\u0075 escape."
+    },
+    "notunicode4": {
+        "type": "string",
+        "value": "This string does not have a unicode \\\u0075 escape."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidStringSimple(t *testing.T) {
+	input := `answer = "You are not drinking enough whisky."`
+	jsonRef := `{
+    "answer": {
+        "type": "string",
+        "value": "You are not drinking enough whisky."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidStringWithPound(t *testing.T) {
+	input := `pound = "We see no # comments here."
+poundcomment = "But there are # some comments here." # Did I # mess you up?`
+	jsonRef := `{
+    "pound": {"type": "string", "value": "We see no # comments here."},
+    "poundcomment": {
+        "type": "string",
+        "value": "But there are # some comments here."
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableArrayImplicit(t *testing.T) {
+	input := `[[albums.songs]]
+name = "Glory Days"`
+	jsonRef := `{
+    "albums": {
+       "songs": [
+           {"name": {"type": "string", "value": "Glory Days"}}
+       ]
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableArrayMany(t *testing.T) {
+	input := `[[people]]
+first_name = "Bruce"
+last_name = "Springsteen"
+
+[[people]]
+first_name = "Eric"
+last_name = "Clapton"
+
+[[people]]
+first_name = "Bob"
+last_name = "Seger"`
+	jsonRef := `{
+    "people": [
+        {
+            "first_name": {"type": "string", "value": "Bruce"},
+            "last_name": {"type": "string", "value": "Springsteen"}
+        },
+        {
+            "first_name": {"type": "string", "value": "Eric"},
+            "last_name": {"type": "string", "value": "Clapton"}
+        },
+        {
+            "first_name": {"type": "string", "value": "Bob"},
+            "last_name": {"type": "string", "value": "Seger"}
+        }
+    ]
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableArrayNest(t *testing.T) {
+	input := `[[albums]]
+name = "Born to Run"
+
+  [[albums.songs]]
+  name = "Jungleland"
+
+  [[albums.songs]]
+  name = "Meeting Across the River"
+
+[[albums]]
+name = "Born in the USA"
+  
+  [[albums.songs]]
+  name = "Glory Days"
+
+  [[albums.songs]]
+  name = "Dancing in the Dark"`
+	jsonRef := `{
+    "albums": [
+        {
+            "name": {"type": "string", "value": "Born to Run"},
+            "songs": [
+                {"name": {"type": "string", "value": "Jungleland"}},
+                {"name": {"type": "string", "value": "Meeting Across the River"}}
+            ]
+        },
+        {
+            "name": {"type": "string", "value": "Born in the USA"},
+            "songs": [
+                {"name": {"type": "string", "value": "Glory Days"}},
+                {"name": {"type": "string", "value": "Dancing in the Dark"}}
+            ]
+        }
+    ]
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableArrayOne(t *testing.T) {
+	input := `[[people]]
+first_name = "Bruce"
+last_name = "Springsteen"`
+	jsonRef := `{
+    "people": [
+        {
+            "first_name": {"type": "string", "value": "Bruce"},
+            "last_name": {"type": "string", "value": "Springsteen"}
+        }
+    ]
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableEmpty(t *testing.T) {
+	input := `[a]`
+	jsonRef := `{
+    "a": {}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableSubEmpty(t *testing.T) {
+	input := `[a]
+[a.b]`
+	jsonRef := `{
+    "a": { "b": {} }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableWhitespace(t *testing.T) {
+	input := `["valid key"]`
+	jsonRef := `{
+    "valid key": {}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidTableWithPound(t *testing.T) {
+	input := `["key#group"]
+answer = 42`
+	jsonRef := `{
+    "key#group": {
+        "answer": {"type": "integer", "value": "42"}
+    }
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidUnicodeEscape(t *testing.T) {
+	input := `answer4 = "\u03B4"
+answer8 = "\U000003B4"`
+	jsonRef := `{
+    "answer4": {"type": "string", "value": "\u03B4"},
+    "answer8": {"type": "string", "value": "\u03B4"}
+}`
+	testgenValid(t, input, jsonRef)
+}
+
+func TestValidUnicodeLiteral(t *testing.T) {
+	input := `answer = "δ"`
+	jsonRef := `{
+    "answer": {"type": "string", "value": "δ"}
+}`
+	testgenValid(t, input, jsonRef)
+}