diff --git a/internal/ackhandler/received_packet_history.go b/internal/ackhandler/received_packet_history.go
index d449bc51d..e75b0b0e2 100644
--- a/internal/ackhandler/received_packet_history.go
+++ b/internal/ackhandler/received_packet_history.go
@@ -12,6 +12,8 @@ import (
 // It does not store packet contents.
 type receivedPacketHistory struct {
 	ranges *utils.PacketIntervalList
+
+	deletedBelow protocol.PacketNumber
 }
 
 var errTooManyOutstandingReceivedAckRanges = qerr.Error(qerr.InternalError, "Too many outstanding received ACK ranges")
@@ -25,6 +27,10 @@ func newReceivedPacketHistory() *receivedPacketHistory {
 
 // ReceivedPacket registers a packet with PacketNumber p and updates the ranges
 func (h *receivedPacketHistory) ReceivedPacket(p protocol.PacketNumber) error {
+	// ignore delayed packets, if we already deleted the range
+	if p < h.deletedBelow {
+		return nil
+	}
 	if h.ranges.Len() >= protocol.MaxTrackedReceivedAckRanges {
 		return errTooManyOutstandingReceivedAckRanges
 	}
@@ -75,6 +81,11 @@ func (h *receivedPacketHistory) ReceivedPacket(p protocol.PacketNumber) error {
 
 // DeleteBelow deletes all entries below (but not including) p
 func (h *receivedPacketHistory) DeleteBelow(p protocol.PacketNumber) {
+	if p < h.deletedBelow {
+		return
+	}
+	h.deletedBelow = p
+
 	nextEl := h.ranges.Front()
 	for el := h.ranges.Front(); nextEl != nil; el = nextEl {
 		nextEl = el.Next()
diff --git a/internal/ackhandler/received_packet_history_test.go b/internal/ackhandler/received_packet_history_test.go
index 4a24b95c4..db3afb08d 100644
--- a/internal/ackhandler/received_packet_history_test.go
+++ b/internal/ackhandler/received_packet_history_test.go
@@ -174,6 +174,18 @@ var _ = Describe("receivedPacketHistory", func() {
 			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 4, End: 4}))
 		})
 
+		It("doesn't add delayed packets below deleted ranges", func() {
+			hist.ReceivedPacket(4)
+			hist.ReceivedPacket(5)
+			hist.ReceivedPacket(6)
+			hist.DeleteBelow(5)
+			Expect(hist.ranges.Len()).To(Equal(1))
+			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 5, End: 6}))
+			hist.ReceivedPacket(2)
+			Expect(hist.ranges.Len()).To(Equal(1))
+			Expect(hist.ranges.Front().Value).To(Equal(utils.PacketInterval{Start: 5, End: 6}))
+		})
+
 		Context("DoS protection", func() {
 			It("doesn't create more than MaxTrackedReceivedAckRanges ranges", func() {
 				for i := protocol.PacketNumber(1); i <= protocol.MaxTrackedReceivedAckRanges; i++ {