diff --git a/session.go b/session.go
index d035c1c53..97327edff 100644
--- a/session.go
+++ b/session.go
@@ -568,12 +568,12 @@ runLoop:
 				if processed := s.handlePacketImpl(p); processed {
 					processedUndecryptablePacket = true
 				}
-			}
-			// Don't set timers and send packets if the packet made us close the session.
-			select {
-			case closeErr = <-s.closeChan:
-				break runLoop
-			default:
+				// Don't set timers and send packets if the packet made us close the session.
+				select {
+				case closeErr = <-s.closeChan:
+					break runLoop
+				default:
+				}
 			}
 		} else if !processedUndecryptablePacket {
 			select {