diff --git a/internal/ackhandler/sent_packet_handler.go b/internal/ackhandler/sent_packet_handler.go
index 1bb568ede..54950b0fe 100644
--- a/internal/ackhandler/sent_packet_handler.go
+++ b/internal/ackhandler/sent_packet_handler.go
@@ -372,6 +372,7 @@ func (h *sentPacketHandler) setLossDetectionTimer() {
 	if lossTime, _ := h.getEarliestLossTimeAndSpace(); !lossTime.IsZero() {
 		// Early retransmit timer or time loss detection.
 		h.alarm = lossTime
+		return
 	}
 
 	// Cancel the alarm if no packets are outstanding