diff --git a/ackhandlerlegacy/received_packet_handler.go b/ackhandlerlegacy/received_packet_handler.go
index 821cea2e2..4248f3212 100644
--- a/ackhandlerlegacy/received_packet_handler.go
+++ b/ackhandlerlegacy/received_packet_handler.go
@@ -68,16 +68,21 @@ func (h *receivedPacketHandler) ReceivedPacket(packetNumber protocol.PacketNumbe
 		h.largestObserved = packetNumber
 	}
 
-	if packetNumber == h.highestInOrderObserved+1 {
-		h.highestInOrderObserved = packetNumber
-		h.highestInOrderObservedEntropy.Add(packetNumber, entropyBit)
-	}
-
 	h.packetHistory[packetNumber] = packetHistoryEntry{
 		EntropyBit:   entropyBit,
 		TimeReceived: time.Now(),
 	}
 
+	// Try to increase highestInOrderObserved as far as possible
+	for {
+		p, ok := h.packetHistory[h.highestInOrderObserved+1]
+		if !ok {
+			break
+		}
+		h.highestInOrderObserved++
+		h.highestInOrderObservedEntropy.Add(h.highestInOrderObserved, p.EntropyBit)
+	}
+
 	h.garbageCollect()
 
 	if uint32(len(h.packetHistory)) > protocol.MaxTrackedReceivedPackets {
diff --git a/ackhandlerlegacy/received_packet_handler_test.go b/ackhandlerlegacy/received_packet_handler_test.go
index 16977bd9c..d40eff259 100644
--- a/ackhandlerlegacy/received_packet_handler_test.go
+++ b/ackhandlerlegacy/received_packet_handler_test.go
@@ -31,7 +31,8 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
 			err = handler.ReceivedPacket(protocol.PacketNumber(2), false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(2)))
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(2)))
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
 		})
 
 		It("rejects packets with packet number 0", func() {
@@ -417,5 +418,14 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(2)))
 			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(4)))
 		})
+
+		It("garbage collects after receiving out-of-order packets without STOP_WAITING", func() {
+			handler.ReceivedPacket(2, true)
+			handler.ReceivedPacket(1, true)
+			handler.ReceivedPacket(3, true)
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(1)))
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(2)))
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
+		})
 	})
 })