add Transport config options to limit the number of handshakes (#4248)

* add Transport config options to limit the number of handshakes

* fix accounting for failed handshakes

* increase handshake limits, improve documentation

transport.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"crypto/tls"
 	"errors"
+	"math"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -18,6 +19,18 @@ import (
 
 var errListenerAlreadySet = errors.New("listener already set")
 
+const (
+	// defaultMaxNumUnvalidatedHandshakes is the default value for Transport.MaxUnvalidatedHandshakes.
+	defaultMaxNumUnvalidatedHandshakes = 128
+	// defaultMaxNumHandshakes is the default value for Transport.MaxHandshakes.
+	// It's not clear how to choose a reasonable value that works for all use cases.
+	// In production, implementations should:
+	// 1. Choose a lower value.
+	// 2. Implement some kind of IP-address based filtering using the Config.GetConfigForClient
+	//    callback in order to prevent flooding attacks from a single / small number of IP addresses.
+	defaultMaxNumHandshakes = math.MaxInt32
+)
+
 // The Transport is the central point to manage incoming and outgoing QUIC connections.
 // QUIC demultiplexes connections based on their QUIC Connection IDs, not based on the 4-tuple.
 // This means that a single UDP socket can be used for listening for incoming connections, as well as
@@ -77,6 +90,25 @@ type Transport struct {
 	// It has no effect for clients.
 	DisableVersionNegotiationPackets bool
 
+	// MaxUnvalidatedHandshakes is the maximum number of concurrent incoming QUIC handshakes
+	// originating from unvalidated source addresses.
+	// If the number of handshakes from unvalidated addresses reaches this number, new incoming
+	// connection attempts will need to proof reachability at the respective source address using the
+	// Retry mechanism, as described in RFC 9000 section 8.1.2.
+	// Validating the source address adds one additional network roundtrip to the handshake.
+	// If unset, a default value of 128 will be used.
+	// When set to a negative value, every connection attempt will need to validate the source address.
+	// It does not make sense to set this value higher than MaxHandshakes.
+	MaxUnvalidatedHandshakes int
+	// MaxHandshakes is the maximum number of concurrent incoming handshakes, both from validated
+	// and unvalidated source addresses.
+	// If unset, the number of concurrent handshakes will not be limited.
+	// Applications should choose a reasonable value based on their thread model, and consider
+	// implementing IP-based rate limiting using Config.GetConfigForClient.
+	// If the number of handshakes reaches this number, new connection attempts will be rejected by
+	// terminating the connection attempt using a CONNECTION_REFUSED error.
+	MaxHandshakes int
+
 	// A Tracer traces events that don't belong to a single QUIC connection.
 	Tracer *logging.Tracer
 
@@ -151,6 +183,14 @@ func (t *Transport) createServer(tlsConf *tls.Config, conf *Config, allow0RTT bo
 	if err := t.init(false); err != nil {
 		return nil, err
 	}
+	maxUnvalidatedHandshakes := t.MaxUnvalidatedHandshakes
+	if maxUnvalidatedHandshakes == 0 {
+		maxUnvalidatedHandshakes = defaultMaxNumUnvalidatedHandshakes
+	}
+	maxHandshakes := t.MaxHandshakes
+	if maxHandshakes == 0 {
+		maxHandshakes = defaultMaxNumHandshakes
+	}
 	s := newServer(
 		t.conn,
 		t.handlerMap,
@@ -161,6 +201,8 @@ func (t *Transport) createServer(tlsConf *tls.Config, conf *Config, allow0RTT bo
 		t.closeServer,
 		*t.TokenGeneratorKey,
 		t.MaxTokenAge,
+		maxUnvalidatedHandshakes,
+		maxHandshakes,
 		t.DisableVersionNegotiationPackets,
 		allow0RTT,
 	)