v2: benchdiff (#524)

.github/workflows/benchmark.yml

@@ -0,0 +1,38 @@
+name: benchmark
+on:
+  push:
+    branches:
+      - v2
+  pull_request:
+    branches:
+      - v2
+
+jobs:
+  base:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+      - name: Setup go
+        uses: actions/setup-go@master
+        with:
+          go-version: "~1.16"
+      - uses: WillAbides/benchdiff-action@main
+        with:
+          benchdiff_version: 0.7.0
+          status_sha: ${{ github.sha }}
+          status_name: bench-result
+          status_on_degraded: neutral
+          benchdiff_args: |
+            --packages ./...
+            --cpu=1,2
+            --count=10
+            --warmup-count=1
+            --warmup-time=10ms
+            --tolerance=10
+            --base-ref origin/v2
+            --debug
+            --benchmem
+            --geomean
+            --sort=name

.github/workflows/workflow.yml

@@ -23,6 +23,3 @@ jobs:
           go-version: ${{ matrix.go }}
       - name: Run unit tests
         run: go test -race ./...
-      - name: Run benchmark tests
-        run: go test -race ./...
-        working-directory: benchmark

benchmark/bench_datasets_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/require"
 )
 
@@ -32,20 +33,12 @@ func TestUnmarshalDatasetCode(t *testing.T) {
 	for _, tc := range bench_inputs {
 		buf := fixture(t, tc.name)
 		t.Run(tc.name, func(t *testing.T) {
-			for _, r := range runners {
+			var v interface{}
-				if r.name == "bs" && tc.name == "canada" {
+			check(t, toml.Unmarshal(buf, &v))
-					t.Skip("skipping: burntsushi can't handle mixed arrays")
-				}
 
-				t.Run(r.name, func(t *testing.T) {
+			b, err := json.Marshal(v)
-					var v interface{}
+			check(t, err)
-					check(t, r.unmarshal(buf, &v))
+			require.Equal(t, len(b), tc.jsonLen)
-
-					b, err := json.Marshal(v)
-					check(t, err)
-					require.Equal(t, len(b), tc.jsonLen)
-				})
-			}
 		})
 	}
 }
@@ -54,19 +47,13 @@ func BenchmarkUnmarshalDataset(b *testing.B) {
 	for _, tc := range bench_inputs {
 		buf := fixture(b, tc.name)
 		b.Run(tc.name, func(b *testing.B) {
-			bench(b, func(r runner, b *testing.B) {
+			b.SetBytes(int64(len(buf)))
-				if r.name == "bs" && tc.name == "canada" {
+			b.ReportAllocs()
-					b.Skip("skipping: burntsushi can't handle mixed arrays")
+			b.ResetTimer()
-				}
+			for i := 0; i < b.N; i++ {
-
+				var v interface{}
-				b.SetBytes(int64(len(buf)))
+				check(b, toml.Unmarshal(buf, &v))
-				b.ReportAllocs()
+			}
-				b.ResetTimer()
-				for i := 0; i < b.N; i++ {
-					var v interface{}
-					check(b, r.unmarshal(buf, &v))
-				}
-			})
 		})
 	}
 }

benchmark/benchmark_test.go

@@ -5,44 +5,21 @@ import (
 	"testing"
 	"time"
 
-	tomlbs "github.com/BurntSushi/toml"
-	tomlv1 "github.com/pelletier/go-toml-v1"
 	"github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/require"
 )
 
-type runner struct {
-	name      string
-	unmarshal func([]byte, interface{}) error
-}
-
-var runners = []runner{
-	{"v2", toml.Unmarshal},
-	{"v1", tomlv1.Unmarshal},
-	{"bs", tomlbs.Unmarshal},
-}
-
-func bench(b *testing.B, f func(r runner, b *testing.B)) {
-	for _, r := range runners {
-		b.Run(r.name, func(b *testing.B) {
-			f(r, b)
-		})
-	}
-}
-
 func BenchmarkUnmarshalSimple(b *testing.B) {
-	bench(b, func(r runner, b *testing.B) {
+	d := struct {
-		d := struct {
+		A string
-			A string
+	}{}
-		}{}
+	doc := []byte(`A = "hello"`)
-		doc := []byte(`A = "hello"`)
+	for i := 0; i < b.N; i++ {
-		for i := 0; i < b.N; i++ {
+		err := toml.Unmarshal(doc, &d)
-			err := r.unmarshal(doc, &d)
+		if err != nil {
-			if err != nil {
+			panic(err)
-				panic(err)
-			}
 		}
-	})
+	}
 }
 
 type benchmarkDoc struct {
@@ -152,22 +129,20 @@ type benchmarkDoc struct {
 }
 
 func BenchmarkReferenceFile(b *testing.B) {
-	bench(b, func(r runner, b *testing.B) {
+	bytes, err := ioutil.ReadFile("benchmark.toml")
-		bytes, err := ioutil.ReadFile("benchmark.toml")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.SetBytes(int64(len(bytes)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		d := benchmarkDoc{}
+		err := toml.Unmarshal(bytes, &d)
 		if err != nil {
-			b.Fatal(err)
+			panic(err)
 		}
-		b.SetBytes(int64(len(bytes)))
+	}
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			d := benchmarkDoc{}
-			err := r.unmarshal(bytes, &d)
-			if err != nil {
-				panic(err)
-			}
-		}
-	})
 }
 
 func TestReferenceFile(t *testing.T) {

benchmark/go.mod

@@ -1,14 +0,0 @@
-module github.com/pelletier/go-toml/v2/benchmark
-
-go 1.16
-
-replace github.com/pelletier/go-toml/v2 => ../
-
-replace github.com/pelletier/go-toml-v1 => github.com/pelletier/go-toml v1.8.1
-
-require (
-	github.com/BurntSushi/toml v0.3.1
-	github.com/pelletier/go-toml-v1 v0.0.0-00010101000000-000000000000
-	github.com/pelletier/go-toml/v2 v2.0.0-00010101000000-000000000000
-	github.com/stretchr/testify v1.7.0
-)

benchmark/go.sum

@@ -1,16 +0,0 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=