diff --git a/internal/ackhandler/sent_packet_handler.go b/internal/ackhandler/sent_packet_handler.go
index b0b565ef0..6a7ba505d 100644
--- a/internal/ackhandler/sent_packet_handler.go
+++ b/internal/ackhandler/sent_packet_handler.go
@@ -126,6 +126,11 @@ func (h *sentPacketHandler) DropPackets(encLevel protocol.EncryptionLevel) {
 }
 
 func (h *sentPacketHandler) dropPackets(encLevel protocol.EncryptionLevel) {
+	// The server won't await address validation after the handshake is confirmed.
+	// This applies even if we didn't receive an ACK for a Handshake packet.
+	if h.perspective == protocol.PerspectiveClient && encLevel == protocol.EncryptionHandshake {
+		h.peerNotAwaitingAddressValidation = true
+	}
 	// remove outstanding packets from bytes_in_flight
 	if encLevel == protocol.EncryptionInitial || encLevel == protocol.EncryptionHandshake {
 		pnSpace := h.getPacketNumberSpace(encLevel)
diff --git a/internal/ackhandler/sent_packet_handler_test.go b/internal/ackhandler/sent_packet_handler_test.go
index 3b700db0f..53a21b667 100644
--- a/internal/ackhandler/sent_packet_handler_test.go
+++ b/internal/ackhandler/sent_packet_handler_test.go
@@ -783,6 +783,16 @@ var _ = Describe("SentPacketHandler", func() {
 			handler.SentPacket(handshakePacketNonAckEliciting(&Packet{PacketNumber: 2}))
 			Expect(handler.GetLossDetectionTimeout()).To(BeZero())
 		})
+
+		It("doesn't send a packet to unblock the server after handshake confirmation, even if no Handshake ACK was received", func() {
+			handler.SentPacket(handshakePacket(&Packet{PacketNumber: 1}))
+			Expect(handler.GetLossDetectionTimeout()).ToNot(BeZero())
+			Expect(handler.OnLossDetectionTimeout()).To(Succeed())
+			Expect(handler.SendMode()).To(Equal(SendPTOHandshake))
+			// confirm the handshake
+			handler.DropPackets(protocol.EncryptionHandshake)
+			Expect(handler.GetLossDetectionTimeout()).To(BeZero())
+		})
 	})
 
 	Context("Packet-based loss detection", func() {