diff --git a/internal/ackhandler/sent_packet_handler.go b/internal/ackhandler/sent_packet_handler.go
index f1c3fec87..3ed4a7eda 100644
--- a/internal/ackhandler/sent_packet_handler.go
+++ b/internal/ackhandler/sent_packet_handler.go
@@ -373,8 +373,20 @@ func (h *sentPacketHandler) detectLostPackets(now time.Time, priorInFlight proto
 }
 
 func (h *sentPacketHandler) OnAlarm() error {
-	now := time.Now()
+	// When all outstanding are acknowledged, the alarm is canceled in
+	// updateLossDetectionAlarm. This doesn't reset the timer in the session though.
+	// When OnAlarm is called, we therefore need to make sure that there are
+	// actually packets outstanding.
+	if h.packetHistory.HasOutstandingPackets() {
+		if err := h.onVerifiedAlarm(); err != nil {
+			return err
+		}
+	}
+	h.updateLossDetectionAlarm()
+	return nil
+}
 
+func (h *sentPacketHandler) onVerifiedAlarm() error {
 	var err error
 	if h.packetHistory.HasOutstandingHandshakePackets() {
 		if h.logger.Debug() {
@@ -387,7 +399,7 @@ func (h *sentPacketHandler) OnAlarm() error {
 			h.logger.Debugf("Loss detection alarm fired in loss timer mode. Loss time: %s", h.lossTime)
 		}
 		// Early retransmit or time loss detection
-		err = h.detectLostPackets(now, h.bytesInFlight)
+		err = h.detectLostPackets(time.Now(), h.bytesInFlight)
 	} else if h.tlpCount < maxTLPs { // TLP
 		if h.logger.Debug() {
 			h.logger.Debugf("Loss detection alarm fired in TLP mode. TLP count: %d", h.tlpCount)
@@ -405,11 +417,7 @@ func (h *sentPacketHandler) OnAlarm() error {
 		h.numRTOs += 2
 		err = h.queueRTOs()
 	}
-	if err != nil {
-		return err
-	}
-	h.updateLossDetectionAlarm()
-	return nil
+	return err
 }
 
 func (h *sentPacketHandler) GetAlarmTimeout() time.Time {
diff --git a/internal/ackhandler/sent_packet_handler_test.go b/internal/ackhandler/sent_packet_handler_test.go
index ebb5561d0..ff014db96 100644
--- a/internal/ackhandler/sent_packet_handler_test.go
+++ b/internal/ackhandler/sent_packet_handler_test.go
@@ -791,6 +791,11 @@ var _ = Describe("SentPacketHandler", func() {
 		Expect(handler.GetAlarmTimeout()).To(BeZero())
 	})
 
+	It("does nothing on OnAlarm if there are no outstanding packets", func() {
+		Expect(handler.OnAlarm()).To(Succeed())
+		Expect(handler.SendMode()).To(Equal(SendAny))
+	})
+
 	Context("TLPs", func() {
 		It("uses the RTT from RTT stats", func() {
 			rtt := 2 * time.Second