From a533331aeeda0ec42b063ca0dccf5d38efb0f69b Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Fri, 23 Apr 2021 15:21:41 -0400 Subject: [PATCH] v2: benchdiff (#524) --- .github/workflows/benchmark.yml | 38 ++++++++++++++++++ .github/workflows/workflow.yml | 3 -- benchmark/bench_datasets_test.go | 39 +++++++------------ benchmark/benchmark_test.go | 67 ++++++++++---------------------- benchmark/go.mod | 14 ------- benchmark/go.sum | 16 -------- 6 files changed, 72 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/benchmark.yml delete mode 100644 benchmark/go.mod delete mode 100644 benchmark/go.sum diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..a39d2b6 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,38 @@ +name: benchmark +on: + push: + branches: + - v2 + pull_request: + branches: + - v2 + +jobs: + base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: Setup go + uses: actions/setup-go@master + with: + go-version: "~1.16" + - uses: WillAbides/benchdiff-action@main + with: + benchdiff_version: 0.7.0 + status_sha: ${{ github.sha }} + status_name: bench-result + status_on_degraded: neutral + benchdiff_args: | + --packages ./... + --cpu=1,2 + --count=10 + --warmup-count=1 + --warmup-time=10ms + --tolerance=10 + --base-ref origin/v2 + --debug + --benchmem + --geomean + --sort=name diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 7d358e7..f9e073e 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -23,6 +23,3 @@ jobs: go-version: ${{ matrix.go }} - name: Run unit tests run: go test -race ./... - - name: Run benchmark tests - run: go test -race ./... - working-directory: benchmark diff --git a/benchmark/bench_datasets_test.go b/benchmark/bench_datasets_test.go index 7c0165a..1d668d9 100644 --- a/benchmark/bench_datasets_test.go +++ b/benchmark/bench_datasets_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" ) @@ -32,20 +33,12 @@ func TestUnmarshalDatasetCode(t *testing.T) { for _, tc := range bench_inputs { buf := fixture(t, tc.name) t.Run(tc.name, func(t *testing.T) { - for _, r := range runners { - if r.name == "bs" && tc.name == "canada" { - t.Skip("skipping: burntsushi can't handle mixed arrays") - } + var v interface{} + check(t, toml.Unmarshal(buf, &v)) - t.Run(r.name, func(t *testing.T) { - var v interface{} - check(t, r.unmarshal(buf, &v)) - - b, err := json.Marshal(v) - check(t, err) - require.Equal(t, len(b), tc.jsonLen) - }) - } + b, err := json.Marshal(v) + check(t, err) + require.Equal(t, len(b), tc.jsonLen) }) } } @@ -54,19 +47,13 @@ func BenchmarkUnmarshalDataset(b *testing.B) { for _, tc := range bench_inputs { buf := fixture(b, tc.name) b.Run(tc.name, func(b *testing.B) { - bench(b, func(r runner, b *testing.B) { - if r.name == "bs" && tc.name == "canada" { - b.Skip("skipping: burntsushi can't handle mixed arrays") - } - - b.SetBytes(int64(len(buf))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - var v interface{} - check(b, r.unmarshal(buf, &v)) - } - }) + b.SetBytes(int64(len(buf))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var v interface{} + check(b, toml.Unmarshal(buf, &v)) + } }) } } diff --git a/benchmark/benchmark_test.go b/benchmark/benchmark_test.go index 65797b2..3e6c485 100644 --- a/benchmark/benchmark_test.go +++ b/benchmark/benchmark_test.go @@ -5,44 +5,21 @@ import ( "testing" "time" - tomlbs "github.com/BurntSushi/toml" - tomlv1 "github.com/pelletier/go-toml-v1" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" ) -type runner struct { - name string - unmarshal func([]byte, interface{}) error -} - -var runners = []runner{ - {"v2", toml.Unmarshal}, - {"v1", tomlv1.Unmarshal}, - {"bs", tomlbs.Unmarshal}, -} - -func bench(b *testing.B, f func(r runner, b *testing.B)) { - for _, r := range runners { - b.Run(r.name, func(b *testing.B) { - f(r, b) - }) - } -} - func BenchmarkUnmarshalSimple(b *testing.B) { - bench(b, func(r runner, b *testing.B) { - d := struct { - A string - }{} - doc := []byte(`A = "hello"`) - for i := 0; i < b.N; i++ { - err := r.unmarshal(doc, &d) - if err != nil { - panic(err) - } + d := struct { + A string + }{} + doc := []byte(`A = "hello"`) + for i := 0; i < b.N; i++ { + err := toml.Unmarshal(doc, &d) + if err != nil { + panic(err) } - }) + } } type benchmarkDoc struct { @@ -152,22 +129,20 @@ type benchmarkDoc struct { } func BenchmarkReferenceFile(b *testing.B) { - bench(b, func(r runner, b *testing.B) { - bytes, err := ioutil.ReadFile("benchmark.toml") + bytes, err := ioutil.ReadFile("benchmark.toml") + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(bytes))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + d := benchmarkDoc{} + err := toml.Unmarshal(bytes, &d) if err != nil { - b.Fatal(err) + panic(err) } - b.SetBytes(int64(len(bytes))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - d := benchmarkDoc{} - err := r.unmarshal(bytes, &d) - if err != nil { - panic(err) - } - } - }) + } } func TestReferenceFile(t *testing.T) { diff --git a/benchmark/go.mod b/benchmark/go.mod deleted file mode 100644 index c1bf154..0000000 --- a/benchmark/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module github.com/pelletier/go-toml/v2/benchmark - -go 1.16 - -replace github.com/pelletier/go-toml/v2 => ../ - -replace github.com/pelletier/go-toml-v1 => github.com/pelletier/go-toml v1.8.1 - -require ( - github.com/BurntSushi/toml v0.3.1 - github.com/pelletier/go-toml-v1 v0.0.0-00010101000000-000000000000 - github.com/pelletier/go-toml/v2 v2.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.7.0 -) diff --git a/benchmark/go.sum b/benchmark/go.sum deleted file mode 100644 index bcc2949..0000000 --- a/benchmark/go.sum +++ /dev/null @@ -1,16 +0,0 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=