diff --git a/packet_handler_map.go b/packet_handler_map.go
index 1d18b5f2..7a64dbd1 100644
--- a/packet_handler_map.go
+++ b/packet_handler_map.go
@@ -258,6 +258,10 @@ func (h *packetHandlerMap) listen() {
 	defer close(h.listening)
 	for {
 		p, err := h.conn.ReadPacket()
+		if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+			h.logger.Debugf("Temporary error reading from conn: %w", err)
+			continue
+		}
 		if err != nil {
 			h.close(err)
 			return
diff --git a/packet_handler_map_test.go b/packet_handler_map_test.go
index dd2895dc..95d9737c 100644
--- a/packet_handler_map_test.go
+++ b/packet_handler_map_test.go
@@ -211,6 +211,16 @@ var _ = Describe("Packet Handler Map", func() {
 				Eventually(done).Should(BeClosed())
 			})
 
+			It("continues listening for temporary errors", func() {
+				packetHandler := NewMockPacketHandler(mockCtrl)
+				handler.Add(protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}, packetHandler)
+				err := deadlineError{}
+				Expect(err.Temporary()).To(BeTrue())
+				packetChan <- packetToRead{err: err}
+				// don't EXPECT any calls to packetHandler.destroy
+				time.Sleep(50 * time.Millisecond)
+			})
+
 			It("says if a connection ID is already taken", func() {
 				connID := protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}
 				Expect(handler.Add(connID, NewMockPacketHandler(mockCtrl))).To(BeTrue())