diff --git a/ackhandlernew/sent_packet_handler.go b/ackhandlernew/sent_packet_handler.go
index a8bad032..88450eeb 100644
--- a/ackhandlernew/sent_packet_handler.go
+++ b/ackhandlernew/sent_packet_handler.go
@@ -166,12 +166,15 @@ func (h *sentPacketHandler) ReceivedAck(ackFrame *frames.AckFrameNew, withPacket
 
 	h.LargestAcked = ackFrame.LargestAcked
 
-	// Update the RTT
-	timeDelta := time.Now().Sub(h.packetHistory[h.LargestAcked].sendTime)
-	// TODO: Don't always update RTT
-	h.rttStats.UpdateRTT(timeDelta, ackFrame.DelayTime, time.Now())
-	if utils.Debug() {
-		utils.Debugf("\tEstimated RTT: %dms", h.rttStats.SmoothedRTT()/time.Millisecond)
+	packet, ok := h.packetHistory[h.LargestAcked]
+	if ok {
+		// Update the RTT
+		timeDelta := time.Now().Sub(packet.sendTime)
+		// TODO: Don't always update RTT
+		h.rttStats.UpdateRTT(timeDelta, ackFrame.DelayTime, time.Now())
+		if utils.Debug() {
+			utils.Debugf("\tEstimated RTT: %dms", h.rttStats.SmoothedRTT()/time.Millisecond)
+		}
 	}
 
 	var ackedPackets congestion.PacketVector
diff --git a/ackhandlernew/sent_packet_handler_test.go b/ackhandlernew/sent_packet_handler_test.go
index d79969c0..c88be2d3 100644
--- a/ackhandlernew/sent_packet_handler_test.go
+++ b/ackhandlernew/sent_packet_handler_test.go
@@ -339,6 +339,33 @@ var _ = Describe("SentPacketHandler", func() {
 				Expect(handler.packetHistory[8].MissingReports).To(Equal(uint8(1)))
 				Expect(handler.packetHistory[10].MissingReports).To(BeZero())
 			})
+
+			It("processes an ACK frame that would be sent after a late arrival of a packet", func() {
+				largestObserved := 6
+				ack1 := frames.AckFrameNew{
+					LargestAcked: protocol.PacketNumber(largestObserved),
+					LowestAcked:  1,
+					AckRanges: []frames.AckRange{
+						{FirstPacketNumber: 1, LastPacketNumber: 2},
+						{FirstPacketNumber: 4, LastPacketNumber: protocol.PacketNumber(largestObserved)},
+					},
+				}
+				err := handler.ReceivedAck(&ack1, 1)
+				Expect(err).ToNot(HaveOccurred())
+				// Expect(handler.BytesInFlight()).To(Equal(protocol.ByteCount(len(packets) - 5)))
+				Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
+				ack2 := frames.AckFrameNew{
+					LargestAcked: protocol.PacketNumber(largestObserved),
+					LowestAcked:  1,
+				}
+				err = handler.ReceivedAck(&ack2, 2)
+				Expect(err).ToNot(HaveOccurred())
+				// Expect(handler.BytesInFlight()).To(Equal(protocol.ByteCount(len(packets) - 6)))
+				Expect(handler.LargestInOrderAcked).To(Equal(protocol.PacketNumber(largestObserved)))
+				for i := 1; i <= largestObserved; i++ {
+					Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(i)))
+				}
+			})
 		})
 
 		Context("calculating RTT", func() {