Merge pull request #766 from lucas-clemente/benchmarks

move benchmarks to separate package, speed up CI builds

.travis/script.sh

@@ -4,16 +4,16 @@ set -e
 
 go get -t ./...
 if [ ${TESTMODE} == "unit" ]; then
-  ginkgo -r --cover --randomizeAllSpecs --randomizeSuites --trace --progress --skipPackage integrationtests --skipMeasurements
+  ginkgo -r --cover --randomizeAllSpecs --randomizeSuites --trace --progress --skipPackage integrationtests,benchmark
 fi
 
 if [ ${TESTMODE} == "integration" ]; then
   # run benchmark tests
-  ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress -focus "Benchmark"
+  ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1
   # run benchmark tests with the Go race detector
   # The Go race detector only works on amd64.
   if [ ${TRAVIS_GOARCH} == 'amd64' ]; then
-    ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress -focus "Benchmark"
+    ginkgo --race --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1 -size=10
   fi
   # run integration tests
   ginkgo -v -r --randomizeAllSpecs --randomizeSuites --trace --progress integrationtests

appveyor.yml

@@ -28,7 +28,8 @@ install:
 
 build_script:
   - rm -r integrationtests
-  - ginkgo -r --randomizeAllSpecs --randomizeSuites --trace --progress
+  - ginkgo -r --randomizeAllSpecs --randomizeSuites --trace --progress -skipPackage benchmark
+  - ginkgo --randomizeAllSpecs --randomizeSuites --trace --progress benchmark -- -samples=1
 
 test: off
 

benchmark/benchmark_suite_test.go

@@ -0,0 +1,26 @@
+package benchmark
+
+import (
+	"flag"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestBenchmark(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Benchmark Suite")
+}
+
+var (
+	size    int // file size in MB, will be read from flags
+	samples int // number of samples for Measure, will be read from flags
+)
+
+func init() {
+	flag.IntVar(&size, "size", 50, "data length (in MB)")
+	flag.IntVar(&samples, "samples", 6, "number of samples")
+	flag.Parse()
+}

benchmark/benchmark_test.go

@@ -0,0 +1,79 @@
+package benchmark
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+
+	quic "github.com/lucas-clemente/quic-go"
+	"github.com/lucas-clemente/quic-go/protocol"
+	"github.com/lucas-clemente/quic-go/testdata"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func init() {
+	var _ = Describe("Benchmarks", func() {
+		dataLen := size * /* MB */ (1 << 20)
+		data := make([]byte, dataLen)
+		rand.Seed(GinkgoRandomSeed())
+		rand.Read(data) // no need to check for an error. math.Rand.Read never errors
+
+		for i := range protocol.SupportedVersions {
+			version := protocol.SupportedVersions[i]
+
+			Context(fmt.Sprintf("with version %d", version), func() {
+				Measure(fmt.Sprintf("transferring a %d MB file", size), func(b Benchmarker) {
+					var ln quic.Listener
+					serverAddr := make(chan net.Addr)
+					handshakeChan := make(chan struct{})
+					// start the server
+					go func() {
+						defer GinkgoRecover()
+						var err error
+						ln, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), nil)
+						Expect(err).ToNot(HaveOccurred())
+						serverAddr <- ln.Addr()
+						sess, err := ln.Accept()
+						Expect(err).ToNot(HaveOccurred())
+						// wait for the client to complete the handshake before sending the data
+						// this should not be necessary, but due to timing issues on the CIs, this is necessary to avoid sending too many undecryptable packets
+						<-handshakeChan
+						str, err := sess.OpenStream()
+						Expect(err).ToNot(HaveOccurred())
+						_, err = str.Write(data)
+						Expect(err).ToNot(HaveOccurred())
+						err = str.Close()
+						Expect(err).ToNot(HaveOccurred())
+					}()
+
+					// start the client
+					addr := <-serverAddr
+					sess, err := quic.DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil)
+					Expect(err).ToNot(HaveOccurred())
+					close(handshakeChan)
+					str, err := sess.AcceptStream()
+					Expect(err).ToNot(HaveOccurred())
+
+					buf := &bytes.Buffer{}
+					// measure the time it takes to download the dataLen bytes
+					// note we're measuring the time for the transfer, i.e. excluding the handshake
+					runtime := b.Time("transfer time", func() {
+						_, err := io.Copy(buf, str)
+						Expect(err).NotTo(HaveOccurred())
+					})
+					// this is *a lot* faster than Expect(buf.Bytes()).To(Equal(data))
+					Expect(bytes.Equal(buf.Bytes(), data)).To(BeTrue())
+
+					b.RecordValue("transfer rate [MB/s]", float64(dataLen)/1e6/runtime.Seconds())
+
+					ln.Close()
+					sess.Close(nil)
+				}, samples)
+			})
+		}
+	})
+}

benchmark_test.go

@@ -1,76 +0,0 @@
-package quic
-
-import (
-	"bytes"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-
-	"github.com/lucas-clemente/quic-go/protocol"
-	"github.com/lucas-clemente/quic-go/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Benchmarks", func() {
-	dataLen := 50 /* MB */ * (1 << 20)
-	data := make([]byte, dataLen)
-	rand.Seed(GinkgoRandomSeed())
-	rand.Read(data) // no need to check for an error. math.Rand.Read never errors
-
-	for i := range protocol.SupportedVersions {
-		version := protocol.SupportedVersions[i]
-
-		Context(fmt.Sprintf("with version %d", version), func() {
-			Measure("transferring a file", func(b Benchmarker) {
-				var ln Listener
-				serverAddr := make(chan net.Addr)
-				handshakeChan := make(chan struct{})
-				// start the server
-				go func() {
-					defer GinkgoRecover()
-					var err error
-					ln, err = ListenAddr("localhost:0", testdata.GetTLSConfig(), nil)
-					Expect(err).ToNot(HaveOccurred())
-					serverAddr <- ln.Addr()
-					sess, err := ln.Accept()
-					Expect(err).ToNot(HaveOccurred())
-					// wait for the client to complete the handshake before sending the data
-					// this should not be necessary, but due to timing issues on the CIs, this is necessary to avoid sending too many undecryptable packets
-					<-handshakeChan
-					str, err := sess.OpenStream()
-					Expect(err).ToNot(HaveOccurred())
-					_, err = str.Write(data)
-					Expect(err).ToNot(HaveOccurred())
-					err = str.Close()
-					Expect(err).ToNot(HaveOccurred())
-				}()
-
-				// start the client
-				addr := <-serverAddr
-				sess, err := DialAddr(addr.String(), &tls.Config{InsecureSkipVerify: true}, nil)
-				Expect(err).ToNot(HaveOccurred())
-				close(handshakeChan)
-				str, err := sess.AcceptStream()
-				Expect(err).ToNot(HaveOccurred())
-
-				buf := &bytes.Buffer{}
-				// measure the time it takes to download the dataLen bytes
-				// note we're measuring the time for the transfer, i.e. excluding the handshake
-				runtime := b.Time("transfer time", func() {
-					_, err := io.Copy(buf, str)
-					Expect(err).NotTo(HaveOccurred())
-				})
-				// this is *a lot* faster than Expect(buf.Bytes()).To(Equal(data))
-				Expect(bytes.Equal(buf.Bytes(), data)).To(BeTrue())
-
-				b.RecordValue("transfer rate [MB/s]", float64(dataLen)/1e6/runtime.Seconds())
-
-				ln.Close()
-				sess.Close(nil)
-			}, 6)
-		})
-	}
-})