diff --git a/server.go b/server.go
index bc494984..7e4e26b3 100644
--- a/server.go
+++ b/server.go
@@ -109,15 +109,14 @@ func (s *Server) Close() error {
 	s.sessionsMutex.Unlock()
 
 	s.connMutex.Lock()
-	defer s.connMutex.Unlock()
+	conn := s.conn
+	s.conn = nil
+	s.connMutex.Unlock()
 
-	if s.conn == nil {
+	if conn == nil {
 		return nil
 	}
-	defer func() {
-		s.conn = nil
-	}()
-	return s.conn.Close()
+	return conn.Close()
 }
 
 func (s *Server) handlePacket(conn *net.UDPConn, remoteAddr *net.UDPAddr, packet []byte) error {