diff --git a/packet_handler_map.go b/packet_handler_map.go
index c616c263b..0a250c7f5 100644
--- a/packet_handler_map.go
+++ b/packet_handler_map.go
@@ -11,6 +11,7 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -123,12 +124,14 @@ func newPacketHandlerMap(
 	logger utils.Logger,
 ) (packetHandlerManager, error) {
 	if err := setReceiveBuffer(c, logger); err != nil {
-		receiveBufferWarningOnce.Do(func() {
-			if disable, _ := strconv.ParseBool(os.Getenv("QUIC_GO_DISABLE_RECEIVE_BUFFER_WARNING")); disable {
-				return
-			}
-			log.Printf("%s. See https://github.com/lucas-clemente/quic-go/wiki/UDP-Receive-Buffer-Size for details.", err)
-		})
+		if !strings.Contains(err.Error(), "use of closed network connection") {
+			receiveBufferWarningOnce.Do(func() {
+				if disable, _ := strconv.ParseBool(os.Getenv("QUIC_GO_DISABLE_RECEIVE_BUFFER_WARNING")); disable {
+					return
+				}
+				log.Printf("%s. See https://github.com/lucas-clemente/quic-go/wiki/UDP-Receive-Buffer-Size for details.", err)
+			})
+		}
 	}
 	conn, err := wrapConn(c)
 	if err != nil {