diff --git a/server.go b/server.go
index fcd1cc666..b18470109 100644
--- a/server.go
+++ b/server.go
@@ -225,7 +225,7 @@ func (s *baseServer) run() {
 		case <-s.errorChan:
 			return
 		case p := <-s.receivedPackets:
-			if shouldReleaseBuffer := s.handlePacketImpl(p); !shouldReleaseBuffer {
+			if bufferStillInUse := s.handlePacketImpl(p); !bufferStillInUse {
 				p.buffer.Release()
 			}
 		}
@@ -320,7 +320,14 @@ func (s *baseServer) handlePacket(p *receivedPacket) {
 	}
 }
 
-func (s *baseServer) handlePacketImpl(p *receivedPacket) bool /* should the buffer be released */ {
+func (s *baseServer) handlePacketImpl(p *receivedPacket) bool /* is the buffer still in use? */ {
+	if wire.IsVersionNegotiationPacket(p.data) {
+		s.logger.Debugf("Dropping Version Negotiation packet.")
+		if s.config.Tracer != nil {
+			s.config.Tracer.DroppedPacket(p.remoteAddr, logging.PacketTypeVersionNegotiation, p.Size(), logging.PacketDropUnexpectedPacket)
+		}
+		return false
+	}
 	// If we're creating a new session, the packet will be passed to the session.
 	// The header will then be parsed again.
 	hdr, _, _, err := wire.ParsePacket(p.data, s.config.ConnectionIDLength)
diff --git a/server_test.go b/server_test.go
index d0c08cf75..fa2294b6c 100644
--- a/server_test.go
+++ b/server_test.go
@@ -409,6 +409,28 @@ var _ = Describe("Server", func() {
 				Eventually(done).Should(BeClosed())
 			})
 
+			It("ignores Version Negotiation packets", func() {
+				data, err := wire.ComposeVersionNegotiation(
+					protocol.ConnectionID{1, 2, 3, 4},
+					protocol.ConnectionID{4, 3, 2, 1},
+					[]protocol.VersionNumber{1, 2, 3},
+				)
+				Expect(err).ToNot(HaveOccurred())
+				raddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1337}
+				done := make(chan struct{})
+				tracer.EXPECT().DroppedPacket(raddr, logging.PacketTypeVersionNegotiation, protocol.ByteCount(len(data)), logging.PacketDropUnexpectedPacket).Do(func(net.Addr, logging.PacketType, protocol.ByteCount, logging.PacketDropReason) {
+					close(done)
+				})
+				serv.handlePacket(&receivedPacket{
+					remoteAddr: raddr,
+					data:       data,
+					buffer:     getPacketBuffer(),
+				})
+				Eventually(done).Should(BeClosed())
+				// make sure no other packet is sent
+				time.Sleep(scaleDuration(20 * time.Millisecond))
+			})
+
 			It("replies with a Retry packet, if a Token is required", func() {
 				serv.config.AcceptToken = func(_ net.Addr, _ *Token) bool { return false }
 				hdr := &wire.Header{