diff --git a/internal/ackhandler/received_packet_history.go b/internal/ackhandler/received_packet_history.go
index 0daba413d..e75b0b0e2 100644
--- a/internal/ackhandler/received_packet_history.go
+++ b/internal/ackhandler/received_packet_history.go
@@ -13,7 +13,7 @@ import (
 type receivedPacketHistory struct {
 	ranges *utils.PacketIntervalList
 
-	lowestInReceivedPacketNumbers protocol.PacketNumber
+	deletedBelow protocol.PacketNumber
 }
 
 var errTooManyOutstandingReceivedAckRanges = qerr.Error(qerr.InternalError, "Too many outstanding received ACK ranges")
@@ -27,6 +27,10 @@ func newReceivedPacketHistory() *receivedPacketHistory {
 
 // ReceivedPacket registers a packet with PacketNumber p and updates the ranges
 func (h *receivedPacketHistory) ReceivedPacket(p protocol.PacketNumber) error {
+	// ignore delayed packets, if we already deleted the range
+	if p < h.deletedBelow {
+		return nil
+	}
 	if h.ranges.Len() >= protocol.MaxTrackedReceivedAckRanges {
 		return errTooManyOutstandingReceivedAckRanges
 	}
@@ -77,19 +81,20 @@ func (h *receivedPacketHistory) ReceivedPacket(p protocol.PacketNumber) error {
 
 // DeleteBelow deletes all entries below (but not including) p
 func (h *receivedPacketHistory) DeleteBelow(p protocol.PacketNumber) {
-	if p <= h.lowestInReceivedPacketNumbers {
+	if p < h.deletedBelow {
 		return
 	}
-	h.lowestInReceivedPacketNumbers = p
+	h.deletedBelow = p
 
 	nextEl := h.ranges.Front()
 	for el := h.ranges.Front(); nextEl != nil; el = nextEl {
 		nextEl = el.Next()
 
-		if p > el.Value.Start && p <= el.Value.End {
-			el.Value.Start = p
-		} else if el.Value.End < p { // delete a whole range
+		if el.Value.End < p { // delete a whole range
 			h.ranges.Remove(el)
+		} else if p > el.Value.Start && p <= el.Value.End {
+			el.Value.Start = p
+			return
 		} else { // no ranges affected. Nothing to do
 			return
 		}
diff --git a/internal/ackhandler/received_packet_history_test.go b/internal/ackhandler/received_packet_history_test.go
index 4a24b95c4..db3afb08d 100644
--- a/internal/ackhandler/received_packet_history_test.go
+++ b/internal/ackhandler/received_packet_history_test.go
@@ -174,6 +174,18 @@ var _ = Describe("receivedPacketHistory", func() {
 			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 4, End: 4}))
 		})
 
+		It("doesn't add delayed packets below deleted ranges", func() {
+			hist.ReceivedPacket(4)
+			hist.ReceivedPacket(5)
+			hist.ReceivedPacket(6)
+			hist.DeleteBelow(5)
+			Expect(hist.ranges.Len()).To(Equal(1))
+			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 5, End: 6}))
+			hist.ReceivedPacket(2)
+			Expect(hist.ranges.Len()).To(Equal(1))
+			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 5, End: 6}))
+		})
+
 		Context("DoS protection", func() {
 			It("doesn't create more than MaxTrackedReceivedAckRanges ranges", func() {
 				for i := protocol.PacketNumber(1); i <= protocol.MaxTrackedReceivedAckRanges; i++ {