From d108e104203eb77095d8574290ee76be8076fe0f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 25 Jul 2017 09:41:39 +0700 Subject: [PATCH 1/3] move benchmark test to a separate package --- .travis/script.sh | 6 +++--- benchmark/benchmark_suite_test.go | 13 +++++++++++++ benchmark_test.go => benchmark/benchmark_test.go | 9 +++++---- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 benchmark/benchmark_suite_test.go rename benchmark_test.go => benchmark/benchmark_test.go (89%) diff --git a/.travis/script.sh b/.travis/script.sh index 6c56eb25..3ddeeed0 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -4,16 +4,16 @@ set -e go get -t ./... if [ ${TESTMODE} == "unit" ]; then - ginkgo -r --cover --randomizeAllSpecs --randomizeSuites --trace --progress --skipPackage integrationtests --skipMeasurements + ginkgo -r --cover --randomizeAllSpecs --randomizeSuites --trace --progress --skipPackage integrationtests,benchmark fi if [ ${TESTMODE} == "integration" ]; then # run benchmark tests - ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress -focus "Benchmark" + ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark # run benchmark tests with the Go race detector # The Go race detector only works on amd64. if [ ${TRAVIS_GOARCH} == 'amd64' ]; then - ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress -focus "Benchmark" + ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress benchmark fi # run integration tests ginkgo -v -r --randomizeAllSpecs --randomizeSuites --trace --progress integrationtests diff --git a/benchmark/benchmark_suite_test.go b/benchmark/benchmark_suite_test.go new file mode 100644 index 00000000..7b1e122b --- /dev/null +++ b/benchmark/benchmark_suite_test.go @@ -0,0 +1,13 @@ +package benchmark + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestBenchmark(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Benchmark Suite") +} diff --git a/benchmark_test.go b/benchmark/benchmark_test.go similarity index 89% rename from benchmark_test.go rename to benchmark/benchmark_test.go index 7d935d78..d8c5c2ab 100644 --- a/benchmark_test.go +++ b/benchmark/benchmark_test.go @@ -1,4 +1,4 @@ -package quic +package benchmark import ( "bytes" @@ -8,6 +8,7 @@ import ( "math/rand" "net" + quic "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/protocol" "github.com/lucas-clemente/quic-go/testdata" . "github.com/onsi/ginkgo" @@ -25,14 +26,14 @@ var _ = Describe("Benchmarks", func() { Context(fmt.Sprintf("with version %d", version), func() { Measure("transferring a file", func(b Benchmarker) { - var ln Listener + var ln quic.Listener serverAddr := make(chan net.Addr) handshakeChan := make(chan struct{}) // start the server go func() { defer GinkgoRecover() var err error - ln, err = ListenAddr("localhost:0", testdata.GetTLSConfig(), nil) + ln, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), nil) Expect(err).ToNot(HaveOccurred()) serverAddr <- ln.Addr() sess, err := ln.Accept() @@ -50,7 +51,7 @@ var _ = Describe("Benchmarks", func() { // start the client addr := <-serverAddr - sess, err := DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil) + sess, err := quic.DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil) Expect(err).ToNot(HaveOccurred()) close(handshakeChan) str, err := sess.AcceptStream() From 30bcc48e526aa8a78f93dc0cfaab0be98e4085d8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 25 Jul 2017 11:16:58 +0700 Subject: [PATCH 2/3] read file size and number of samples for benchmark test from flags The values default to what we used previously. --- benchmark/benchmark_suite_test.go | 13 ++++ benchmark/benchmark_test.go | 108 +++++++++++++++--------------- 2 files changed, 68 insertions(+), 53 deletions(-) diff --git a/benchmark/benchmark_suite_test.go b/benchmark/benchmark_suite_test.go index 7b1e122b..d24ef6be 100644 --- a/benchmark/benchmark_suite_test.go +++ b/benchmark/benchmark_suite_test.go @@ -1,6 +1,8 @@ package benchmark import ( + "flag" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -11,3 +13,14 @@ func TestBenchmark(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Benchmark Suite") } + +var ( + size int // file size in MB, will be read from flags + samples int // number of samples for Measure, will be read from flags +) + +func init() { + flag.IntVar(&size, "size", 50, "data length (in MB)") + flag.IntVar(&samples, "samples", 6, "number of samples") + flag.Parse() +} diff --git a/benchmark/benchmark_test.go b/benchmark/benchmark_test.go index d8c5c2ab..d7a24d21 100644 --- a/benchmark/benchmark_test.go +++ b/benchmark/benchmark_test.go @@ -15,63 +15,65 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Benchmarks", func() { - dataLen := 50 /* MB */ * (1 << 20) - data := make([]byte, dataLen) - rand.Seed(GinkgoRandomSeed()) - rand.Read(data) // no need to check for an error. math.Rand.Read never errors +func init() { + var _ = Describe("Benchmarks", func() { + dataLen := size * /* MB */ (1 << 20) + data := make([]byte, dataLen) + rand.Seed(GinkgoRandomSeed()) + rand.Read(data) // no need to check for an error. math.Rand.Read never errors - for i := range protocol.SupportedVersions { - version := protocol.SupportedVersions[i] + for i := range protocol.SupportedVersions { + version := protocol.SupportedVersions[i] - Context(fmt.Sprintf("with version %d", version), func() { - Measure("transferring a file", func(b Benchmarker) { - var ln quic.Listener - serverAddr := make(chan net.Addr) - handshakeChan := make(chan struct{}) - // start the server - go func() { - defer GinkgoRecover() - var err error - ln, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), nil) + Context(fmt.Sprintf("with version %d", version), func() { + Measure(fmt.Sprintf("transferring a %d MB file", size), func(b Benchmarker) { + var ln quic.Listener + serverAddr := make(chan net.Addr) + handshakeChan := make(chan struct{}) + // start the server + go func() { + defer GinkgoRecover() + var err error + ln, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), nil) + Expect(err).ToNot(HaveOccurred()) + serverAddr <- ln.Addr() + sess, err := ln.Accept() + Expect(err).ToNot(HaveOccurred()) + // wait for the client to complete the handshake before sending the data + // this should not be necessary, but due to timing issues on the CIs, this is necessary to avoid sending too many undecryptable packets + <-handshakeChan + str, err := sess.OpenStream() + Expect(err).ToNot(HaveOccurred()) + _, err = str.Write(data) + Expect(err).ToNot(HaveOccurred()) + err = str.Close() + Expect(err).ToNot(HaveOccurred()) + }() + + // start the client + addr := <-serverAddr + sess, err := quic.DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil) Expect(err).ToNot(HaveOccurred()) - serverAddr <- ln.Addr() - sess, err := ln.Accept() + close(handshakeChan) + str, err := sess.AcceptStream() Expect(err).ToNot(HaveOccurred()) - // wait for the client to complete the handshake before sending the data - // this should not be necessary, but due to timing issues on the CIs, this is necessary to avoid sending too many undecryptable packets - <-handshakeChan - str, err := sess.OpenStream() - Expect(err).ToNot(HaveOccurred()) - _, err = str.Write(data) - Expect(err).ToNot(HaveOccurred()) - err = str.Close() - Expect(err).ToNot(HaveOccurred()) - }() - // start the client - addr := <-serverAddr - sess, err := quic.DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil) - Expect(err).ToNot(HaveOccurred()) - close(handshakeChan) - str, err := sess.AcceptStream() - Expect(err).ToNot(HaveOccurred()) + buf := &bytes.Buffer{} + // measure the time it takes to download the dataLen bytes + // note we're measuring the time for the transfer, i.e. excluding the handshake + runtime := b.Time("transfer time", func() { + _, err := io.Copy(buf, str) + Expect(err).NotTo(HaveOccurred()) + }) + // this is *a lot* faster than Expect(buf.Bytes()).To(Equal(data)) + Expect(bytes.Equal(buf.Bytes(), data)).To(BeTrue()) - buf := &bytes.Buffer{} - // measure the time it takes to download the dataLen bytes - // note we're measuring the time for the transfer, i.e. excluding the handshake - runtime := b.Time("transfer time", func() { - _, err := io.Copy(buf, str) - Expect(err).NotTo(HaveOccurred()) - }) - // this is *a lot* faster than Expect(buf.Bytes()).To(Equal(data)) - Expect(bytes.Equal(buf.Bytes(), data)).To(BeTrue()) + b.RecordValue("transfer rate [MB/s]", float64(dataLen)/1e6/runtime.Seconds()) - b.RecordValue("transfer rate [MB/s]", float64(dataLen)/1e6/runtime.Seconds()) - - ln.Close() - sess.Close(nil) - }, 6) - }) - } -}) + ln.Close() + sess.Close(nil) + }, samples) + }) + } + }) +} From bc1858c7eca1f1f502955f23a06bf97486cff81b Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 25 Jul 2017 11:21:47 +0700 Subject: [PATCH 3/3] speed up the CI builds by using the new benchmark test flags Reduce the number of samples and the amount of data transferred. --- .travis/script.sh | 4 ++-- appveyor.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis/script.sh b/.travis/script.sh index 3ddeeed0..b9d88f21 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -9,11 +9,11 @@ fi if [ ${TESTMODE} == "integration" ]; then # run benchmark tests - ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark + ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1 # run benchmark tests with the Go race detector # The Go race detector only works on amd64. if [ ${TRAVIS_GOARCH} == 'amd64' ]; then - ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress benchmark + ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1 -size=10 fi # run integration tests ginkgo -v -r --randomizeAllSpecs --randomizeSuites --trace --progress integrationtests diff --git a/appveyor.yml b/appveyor.yml index a013bb3b..8a3c9075 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,8 @@ install: build_script: - rm -r integrationtests - - ginkgo -r --randomizeAllSpecs --randomizeSuites --trace --progress + - ginkgo -r --randomizeAllSpecs --randomizeSuites --trace --progress -skipPackage benchmark + - ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1 test: off