diff --git a/internal/ackhandler/sent_packet_handler.go b/internal/ackhandler/sent_packet_handler.go
index c978f427c..e63b115a4 100644
--- a/internal/ackhandler/sent_packet_handler.go
+++ b/internal/ackhandler/sent_packet_handler.go
@@ -448,6 +448,12 @@ func (h *sentPacketHandler) setLossDetectionTimer() {
 
 	// PTO alarm
 	sentTime, encLevel := h.getEarliestSentTimeAndSpace()
+	if sentTime.IsZero() {
+		if h.peerCompletedAddressValidation {
+			panic("didn't expect sentTime to be zero")
+		}
+		sentTime = time.Now()
+	}
 	h.alarm = sentTime.Add(h.rttStats.PTO(encLevel == protocol.Encryption1RTT) << h.ptoCount)
 }
 
diff --git a/internal/ackhandler/sent_packet_handler_test.go b/internal/ackhandler/sent_packet_handler_test.go
index 5f4b063d1..1f692b5e9 100644
--- a/internal/ackhandler/sent_packet_handler_test.go
+++ b/internal/ackhandler/sent_packet_handler_test.go
@@ -793,6 +793,21 @@ var _ = Describe("SentPacketHandler", func() {
 			handler.DropPackets(protocol.EncryptionHandshake)
 			Expect(handler.GetLossDetectionTimeout()).To(BeZero())
 		})
+
+		It("correctly sets the timer after the Initial packet number space has been dropped", func() {
+			handler.SentPacket(initialPacket(&Packet{PacketNumber: 1, SendTime: time.Now().Add(-42 * time.Second)}))
+			Expect(handler.ReceivedAck(
+				&wire.AckFrame{AckRanges: []wire.AckRange{{Smallest: 1, Largest: 1}}},
+				protocol.EncryptionInitial,
+				time.Now(),
+			)).To(Succeed())
+			handler.SentPacket(handshakePacketNonAckEliciting(&Packet{PacketNumber: 1, SendTime: time.Now()}))
+			Expect(handler.initialPackets).To(BeNil())
+
+			pto := handler.rttStats.PTO(false)
+			Expect(pto).ToNot(BeZero())
+			Expect(handler.GetLossDetectionTimeout()).To(BeTemporally("~", time.Now().Add(pto), 10*time.Millisecond))
+		})
 	})
 
 	Context("Packet-based loss detection", func() {