diff --git a/session.go b/session.go
index 49c616511..16ee00b61 100644
--- a/session.go
+++ b/session.go
@@ -536,9 +536,13 @@ func (s *session) handlePacketImpl(p *receivedPacket) error {
 	// Only do this after decrypting, so we are sure the packet is not attacker-controlled
 	s.largestRcvdPacketNumber = utils.MaxPacketNumber(s.largestRcvdPacketNumber, hdr.PacketNumber)
 
-	isRetransmittable := ackhandler.HasRetransmittableFrames(packet.frames)
-	if err = s.receivedPacketHandler.ReceivedPacket(hdr.PacketNumber, p.rcvTime, isRetransmittable); err != nil {
-		return err
+	// If this is a Retry packet, there's no need to send an ACK.
+	// The session will be closed and recreated as soon as the crypto setup processed the HRR.
+	if hdr.Type != protocol.PacketTypeRetry {
+		isRetransmittable := ackhandler.HasRetransmittableFrames(packet.frames)
+		if err := s.receivedPacketHandler.ReceivedPacket(hdr.PacketNumber, p.rcvTime, isRetransmittable); err != nil {
+			return err
+		}
 	}
 
 	return s.handleFrames(packet.frames, packet.encryptionLevel)
diff --git a/session_test.go b/session_test.go
index 45eca72be..fb575955b 100644
--- a/session_test.go
+++ b/session_test.go
@@ -569,6 +569,17 @@ var _ = Describe("Session", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
+		It("doesn't inform the ReceivedPacketHandler about Retry packets", func() {
+			now := time.Now().Add(time.Hour)
+			rph := mockackhandler.NewMockReceivedPacketHandler(mockCtrl)
+			sess.receivedPacketHandler = rph
+			// don't EXPECT any call to ReceivedPacket
+			hdr.PacketNumber = 5
+			hdr.Type = protocol.PacketTypeRetry
+			err := sess.handlePacketImpl(&receivedPacket{header: hdr, rcvTime: now})
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("closes when handling a packet fails", func(done Done) {
 			streamManager.EXPECT().CloseWithError(gomock.Any())
 			testErr := errors.New("unpack error")