diff --git a/session.go b/session.go
index ad53499d..7401cf97 100644
--- a/session.go
+++ b/session.go
@@ -603,9 +603,6 @@ func (s *session) handlePacketImpl(p *receivedPacket) error {
 		p.rcvTime = time.Now()
 	}
 
-	s.receivedFirstPacket = true
-	s.lastNetworkActivityTime = p.rcvTime
-	s.keepAlivePingSent = false
 	hdr := p.header
 	data := p.data
 
@@ -630,6 +627,10 @@ func (s *session) handlePacketImpl(p *receivedPacket) error {
 		return err
 	}
 
+	s.receivedFirstPacket = true
+	s.lastNetworkActivityTime = p.rcvTime
+	s.keepAlivePingSent = false
+
 	// In TLS 1.3, the client considers the handshake complete as soon as
 	// it received the server's Finished message and sent its Finished.
 	// We have to wait for the first forward-secure packet from the server before