diff --git a/client.go b/client.go
index 3c307998..906f55ab 100644
--- a/client.go
+++ b/client.go
@@ -151,6 +151,9 @@ func (c *client) handlePacket(remoteAddr net.Addr, packet []byte) error {
 	}
 	hdr.Raw = packet[:len(packet)-r.Len()]
 
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
 	// ignore delayed / duplicated version negotiation packets
 	if c.connState >= ConnStateVersionNegotiated && hdr.VersionFlag {
 		return nil
@@ -159,58 +162,18 @@ func (c *client) handlePacket(remoteAddr net.Addr, packet []byte) error {
 	// this is the first packet after the client sent a packet with the VersionFlag set
 	// if the server doesn't send a version negotiation packet, it supports the suggested version
 	if !hdr.VersionFlag && c.connState == ConnStateInitial {
-		c.mutex.Lock()
 		c.connState = ConnStateVersionNegotiated
 		c.connStateChangeOrErrCond.Signal()
-		c.mutex.Unlock()
 		if c.config.ConnState != nil {
 			go c.config.ConnState(c.session, ConnStateVersionNegotiated)
 		}
 	}
 
 	if hdr.VersionFlag {
-		var hasCommonVersion bool // check if we're supporting any of the offered versions
-		for _, v := range hdr.SupportedVersions {
-			// check if the server sent the offered version in supported versions
-			if v == c.version {
-				return qerr.Error(qerr.InvalidVersionNegotiationPacket, "Server already supports client's version and should have accepted the connection.")
-			}
-			if v != protocol.VersionUnsupported {
-				hasCommonVersion = true
-			}
-		}
-		if !hasCommonVersion {
-			utils.Infof("No common version found.")
-			return qerr.InvalidVersion
-		}
-
-		ok, highestSupportedVersion := protocol.HighestSupportedVersion(hdr.SupportedVersions)
-		if !ok {
-			return qerr.VersionNegotiationMismatch
-		}
-
-		// switch to negotiated version
-		c.version = highestSupportedVersion
-		c.connState = ConnStateVersionNegotiated
-		c.connectionID, err = utils.GenerateConnectionID()
+		err = c.handlePacketWithVersionFlag(hdr)
 		if err != nil {
 			return err
 		}
-		utils.Infof("Switching to QUIC version %d. New connection ID: %x", highestSupportedVersion, c.connectionID)
-
-		c.session.Close(errCloseSessionForNewVersion)
-		err = c.createNewSession(hdr.SupportedVersions)
-		if err != nil {
-			return err
-		}
-		if c.config.ConnState != nil {
-			go c.config.ConnState(c.session, ConnStateVersionNegotiated)
-		}
-		if err != nil {
-			return err
-		}
-
-		return nil // version negotiation packets have no payload
 	}
 
 	c.session.handlePacket(&receivedPacket{
@@ -222,6 +185,52 @@ func (c *client) handlePacket(remoteAddr net.Addr, packet []byte) error {
 	return nil
 }
 
+func (c *client) handlePacketWithVersionFlag(hdr *PublicHeader) error {
+	var hasCommonVersion bool // check if we're supporting any of the offered versions
+	for _, v := range hdr.SupportedVersions {
+		// check if the server sent the offered version in supported versions
+		if v == c.version {
+			return qerr.Error(qerr.InvalidVersionNegotiationPacket, "Server already supports client's version and should have accepted the connection.")
+		}
+		if v != protocol.VersionUnsupported {
+			hasCommonVersion = true
+		}
+	}
+	if !hasCommonVersion {
+		utils.Infof("No common version found.")
+		return qerr.InvalidVersion
+	}
+
+	ok, highestSupportedVersion := protocol.HighestSupportedVersion(hdr.SupportedVersions)
+	if !ok {
+		return qerr.VersionNegotiationMismatch
+	}
+
+	// switch to negotiated version
+	c.version = highestSupportedVersion
+	c.connState = ConnStateVersionNegotiated
+	var err error
+	c.connectionID, err = utils.GenerateConnectionID()
+	if err != nil {
+		return err
+	}
+	utils.Infof("Switching to QUIC version %d. New connection ID: %x", highestSupportedVersion, c.connectionID)
+
+	c.session.Close(errCloseSessionForNewVersion)
+	err = c.createNewSession(hdr.SupportedVersions)
+	if err != nil {
+		return err
+	}
+	if c.config.ConnState != nil {
+		go c.config.ConnState(c.session, ConnStateVersionNegotiated)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil // version negotiation packets have no payload
+}
+
 func (c *client) cryptoChangeCallback(_ Session, isForwardSecure bool) {
 	var state ConnState
 	if isForwardSecure {