add a function to close the packet handler map

Close will close the underlying connection and wait until listen has
returned. While not strictly necessary in production use, this will fix
a few race conditions in our tests.

server.go

@@ -32,6 +32,7 @@ type unknownPacketHandler interface {
 }
 
 type packetHandlerManager interface {
+	io.Closer
 	Add(protocol.ConnectionID, packetHandler)
 	Retire(protocol.ConnectionID)
 	Remove(protocol.ConnectionID)
@@ -300,7 +301,7 @@ func (s *server) closeWithMutex() error {
 	// If the server was started with ListenAddr, we created the packet conn.
 	// We need to close it in order to make the go routine reading from that conn return.
 	if s.createdPacketConn {
-		err = s.conn.Close()
+		err = s.sessionHandler.Close()
 	}
 	s.closed = true
 	close(s.errorChan)