diff --git a/session.go b/session.go
index d780bdef..f8c0448b 100644
--- a/session.go
+++ b/session.go
@@ -76,9 +76,10 @@ type Session struct {
 
 	connectionParametersManager *handshake.ConnectionParametersManager
 
+	lastRcvdPacketNumber protocol.PacketNumber
 	// Used to calculate the next packet number from the truncated wire
 	// representation, and sent back in public reset packets
-	lastRcvdPacketNumber protocol.PacketNumber
+	largestRcvdPacketNumber protocol.PacketNumber
 
 	lastNetworkActivityTime time.Time
 
@@ -239,7 +240,7 @@ func (s *Session) handlePacketImpl(remoteAddr interface{}, hdr *PublicHeader, da
 	// Calculate packet number
 	hdr.PacketNumber = protocol.InferPacketNumber(
 		hdr.PacketNumberLen,
-		s.lastRcvdPacketNumber,
+		s.largestRcvdPacketNumber,
 		hdr.PacketNumber,
 	)
 	if utils.Debug() {
@@ -254,8 +255,9 @@ func (s *Session) handlePacketImpl(remoteAddr interface{}, hdr *PublicHeader, da
 		return err
 	}
 
-	// Only do this after decrypting, so we are sure the packet is not attacker-controlled
 	s.lastRcvdPacketNumber = hdr.PacketNumber
+	// Only do this after decrypting, so we are sure the packet is not attacker-controlled
+	s.largestRcvdPacketNumber = utils.MaxPacketNumber(s.largestRcvdPacketNumber, hdr.PacketNumber)
 
 	err = s.receivedPacketHandler.ReceivedPacket(hdr.PacketNumber, packet.entropyBit)
 	// ignore duplicate packets
diff --git a/session_test.go b/session_test.go
index 09edfbfd..a456daab 100644
--- a/session_test.go
+++ b/session_test.go
@@ -464,22 +464,25 @@ var _ = Describe("Session", func() {
 					hdr = &PublicHeader{PacketNumberLen: protocol.PacketNumberLen6}
 				})
 
-				It("sets the lastRcvdPacketNumber", func() {
+				It("sets the {last,largest}RcvdPacketNumber", func() {
 					hdr.PacketNumber = 5
 					err := session.handlePacketImpl(nil, hdr, nil)
 					Expect(err).ToNot(HaveOccurred())
 					Expect(session.lastRcvdPacketNumber).To(Equal(protocol.PacketNumber(5)))
+					Expect(session.largestRcvdPacketNumber).To(Equal(protocol.PacketNumber(5)))
 				})
 
-				It("sets the lastRcvdPacketNumber, for an out-of-order packet", func() {
+				It("sets the {last,largest}RcvdPacketNumber, for an out-of-order packet", func() {
 					hdr.PacketNumber = 5
 					err := session.handlePacketImpl(nil, hdr, nil)
 					Expect(err).ToNot(HaveOccurred())
 					Expect(session.lastRcvdPacketNumber).To(Equal(protocol.PacketNumber(5)))
+					Expect(session.largestRcvdPacketNumber).To(Equal(protocol.PacketNumber(5)))
 					hdr.PacketNumber = 3
 					err = session.handlePacketImpl(nil, hdr, nil)
 					Expect(err).ToNot(HaveOccurred())
 					Expect(session.lastRcvdPacketNumber).To(Equal(protocol.PacketNumber(3)))
+					Expect(session.largestRcvdPacketNumber).To(Equal(protocol.PacketNumber(5)))
 				})
 
 				It("ignores duplicate packets", func() {