diff --git a/internal/ackhandler/sent_packet_handler.go b/internal/ackhandler/sent_packet_handler.go
index 3bedcd3f6..a0a7c4e21 100644
--- a/internal/ackhandler/sent_packet_handler.go
+++ b/internal/ackhandler/sent_packet_handler.go
@@ -32,6 +32,7 @@ const (
 type sentPacketHandler struct {
 	lastSentPacketNumber              protocol.PacketNumber
 	lastSentRetransmittablePacketTime time.Time
+	lastSentHandshakePacketTime       time.Time
 
 	nextPacketSendTime time.Time
 	skippedPackets     []protocol.PacketNumber
@@ -155,6 +156,9 @@ func (h *sentPacketHandler) sentPacketImpl(packet *Packet) bool /* isRetransmitt
 	isRetransmittable := len(packet.Frames) != 0
 
 	if isRetransmittable {
+		if packet.EncryptionLevel < protocol.EncryptionForwardSecure {
+			h.lastSentHandshakePacketTime = packet.SendTime
+		}
 		h.lastSentRetransmittablePacketTime = packet.SendTime
 		packet.includedInBytesInFlight = true
 		h.bytesInFlight += packet.Length
@@ -274,7 +278,7 @@ func (h *sentPacketHandler) updateLossDetectionAlarm() {
 
 	// TODO(#497): TLP
 	if !h.handshakeComplete {
-		h.alarm = h.lastSentRetransmittablePacketTime.Add(h.computeHandshakeTimeout())
+		h.alarm = h.lastSentHandshakePacketTime.Add(h.computeHandshakeTimeout())
 	} else if !h.lossTime.IsZero() {
 		// Early retransmit timer or time loss detection.
 		h.alarm = h.lossTime
diff --git a/internal/ackhandler/sent_packet_handler_test.go b/internal/ackhandler/sent_packet_handler_test.go
index 92b65f7af..5ac25192b 100644
--- a/internal/ackhandler/sent_packet_handler_test.go
+++ b/internal/ackhandler/sent_packet_handler_test.go
@@ -129,6 +129,13 @@ var _ = Describe("SentPacketHandler", func() {
 			Expect(handler.lastSentRetransmittablePacketTime).To(Equal(sendTime))
 		})
 
+		It("stores the sent time of handshake packets", func() {
+			sendTime := time.Now().Add(-time.Minute)
+			handler.SentPacket(retransmittablePacket(&Packet{PacketNumber: 1, SendTime: sendTime, EncryptionLevel: protocol.EncryptionUnencrypted}))
+			handler.SentPacket(retransmittablePacket(&Packet{PacketNumber: 2, SendTime: sendTime.Add(time.Hour), EncryptionLevel: protocol.EncryptionForwardSecure}))
+			Expect(handler.lastSentHandshakePacketTime).To(Equal(sendTime))
+		})
+
 		It("does not store non-retransmittable packets", func() {
 			handler.SentPacket(nonRetransmittablePacket(&Packet{PacketNumber: 1}))
 			Expect(handler.packetHistory.Len()).To(BeZero())
@@ -851,6 +858,7 @@ var _ = Describe("SentPacketHandler", func() {
 			handler.SentPacket(handshakePacket(&Packet{PacketNumber: 2, SendTime: sendTime}))
 			handler.SentPacket(retransmittablePacket(&Packet{PacketNumber: 3, SendTime: sendTime}))
 			handler.SentPacket(handshakePacket(&Packet{PacketNumber: 4, SendTime: lastHandshakePacketSendTime}))
+			handler.SentPacket(retransmittablePacket(&Packet{PacketNumber: 5, SendTime: now}))
 
 			err := handler.ReceivedAck(createAck([]wire.AckRange{{First: 1, Last: 1}}), 1, protocol.EncryptionForwardSecure, now)
 			// RTT is now 1 minute