diff --git a/internal/ackhandler/sent_packet_handler_test.go b/internal/ackhandler/sent_packet_handler_test.go
index f2c242f4..9f3a1a76 100644
--- a/internal/ackhandler/sent_packet_handler_test.go
+++ b/internal/ackhandler/sent_packet_handler_test.go
@@ -42,7 +42,7 @@ var _ = Describe("SentPacketHandler", func() {
 	}
 
 	ackElicitingPacket := func(p *Packet) *Packet {
-		if p.EncryptionLevel == protocol.EncryptionUnspecified {
+		if p.EncryptionLevel == 0 {
 			p.EncryptionLevel = protocol.Encryption1RTT
 		}
 		if p.Length == 0 {
diff --git a/internal/protocol/encryption_level.go b/internal/protocol/encryption_level.go
index be5de8f7..32d38ab1 100644
--- a/internal/protocol/encryption_level.go
+++ b/internal/protocol/encryption_level.go
@@ -5,10 +5,8 @@ package protocol
 type EncryptionLevel uint8
 
 const (
-	// EncryptionUnspecified is a not specified encryption level
-	EncryptionUnspecified EncryptionLevel = iota
 	// EncryptionInitial is the Initial encryption level
-	EncryptionInitial
+	EncryptionInitial EncryptionLevel = 1 + iota
 	// EncryptionHandshake is the Handshake encryption level
 	EncryptionHandshake
 	// Encryption0RTT is the 0-RTT encryption level
diff --git a/internal/protocol/encryption_level_test.go b/internal/protocol/encryption_level_test.go
index 7e70d02d..9b07b08b 100644
--- a/internal/protocol/encryption_level_test.go
+++ b/internal/protocol/encryption_level_test.go
@@ -6,8 +6,12 @@ import (
 )
 
 var _ = Describe("Encryption Level", func() {
+	It("doesn't use 0 as a value", func() {
+		// 0 is used in some tests
+		Expect(EncryptionInitial * EncryptionHandshake * Encryption0RTT * Encryption1RTT).ToNot(BeZero())
+	})
+
 	It("has the correct string representation", func() {
-		Expect(EncryptionUnspecified.String()).To(Equal("unknown"))
 		Expect(EncryptionInitial.String()).To(Equal("Initial"))
 		Expect(EncryptionHandshake.String()).To(Equal("Handshake"))
 		Expect(Encryption0RTT.String()).To(Equal("0-RTT"))
diff --git a/logging/interface.go b/logging/interface.go
index d640cc7f..24398d3a 100644
--- a/logging/interface.go
+++ b/logging/interface.go
@@ -69,8 +69,6 @@ const (
 	Encryption1RTT EncryptionLevel = protocol.Encryption1RTT
 	// Encryption0RTT is the 0-RTT encryption level
 	Encryption0RTT EncryptionLevel = protocol.Encryption0RTT
-	// EncryptionNone is no encryption
-	EncryptionNone EncryptionLevel = protocol.EncryptionUnspecified
 )
 
 const (
diff --git a/packet_packer.go b/packet_packer.go
index cdb39e9a..f1cec211 100644
--- a/packet_packer.go
+++ b/packet_packer.go
@@ -68,7 +68,7 @@ func (p *packetContents) EncryptionLevel() protocol.EncryptionLevel {
 	case protocol.PacketType0RTT:
 		return protocol.Encryption0RTT
 	default:
-		return protocol.EncryptionUnspecified
+		panic("can't determine encryption level")
 	}
 }
 
diff --git a/session_test.go b/session_test.go
index 2e382991..88975940 100644
--- a/session_test.go
+++ b/session_test.go
@@ -201,7 +201,7 @@ var _ = Describe("Session", func() {
 				Expect(sess.handleFrame(&wire.ResetStreamFrame{
 					StreamID:  3,
 					ErrorCode: 42,
-				}, protocol.EncryptionUnspecified, protocol.ConnectionID{})).To(Succeed())
+				}, protocol.Encryption1RTT, protocol.ConnectionID{})).To(Succeed())
 			})
 		})
 
@@ -236,7 +236,7 @@ var _ = Describe("Session", func() {
 				Expect(sess.handleFrame(&wire.MaxStreamDataFrame{
 					StreamID:          10,
 					MaximumStreamData: 1337,
-				}, protocol.EncryptionUnspecified, protocol.ConnectionID{})).To(Succeed())
+				}, protocol.Encryption1RTT, protocol.ConnectionID{})).To(Succeed())
 			})
 		})
 
@@ -278,7 +278,7 @@ var _ = Describe("Session", func() {
 				Expect(sess.handleFrame(&wire.StopSendingFrame{
 					StreamID:  3,
 					ErrorCode: 1337,
-				}, protocol.EncryptionUnspecified, protocol.ConnectionID{})).To(Succeed())
+				}, protocol.Encryption1RTT, protocol.ConnectionID{})).To(Succeed())
 			})
 		})
 
@@ -291,18 +291,18 @@ var _ = Describe("Session", func() {
 		})
 
 		It("handles PING frames", func() {
-			err := sess.handleFrame(&wire.PingFrame{}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.PingFrame{}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("rejects PATH_RESPONSE frames", func() {
-			err := sess.handleFrame(&wire.PathResponseFrame{Data: [8]byte{1, 2, 3, 4, 5, 6, 7, 8}}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.PathResponseFrame{Data: [8]byte{1, 2, 3, 4, 5, 6, 7, 8}}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).To(MatchError("unexpected PATH_RESPONSE frame"))
 		})
 
 		It("handles PATH_CHALLENGE frames", func() {
 			data := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
-			err := sess.handleFrame(&wire.PathChallengeFrame{Data: data}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.PathChallengeFrame{Data: data}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).ToNot(HaveOccurred())
 			frames, _ := sess.framer.AppendControlFrames(nil, 1000)
 			Expect(frames).To(Equal([]ackhandler.Frame{{Frame: &wire.PathResponseFrame{Data: data}}}))
@@ -316,17 +316,17 @@ var _ = Describe("Session", func() {
 		})
 
 		It("handles BLOCKED frames", func() {
-			err := sess.handleFrame(&wire.DataBlockedFrame{}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.DataBlockedFrame{}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("handles STREAM_BLOCKED frames", func() {
-			err := sess.handleFrame(&wire.StreamDataBlockedFrame{}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.StreamDataBlockedFrame{}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("handles STREAM_ID_BLOCKED frames", func() {
-			err := sess.handleFrame(&wire.StreamsBlockedFrame{}, protocol.EncryptionUnspecified, protocol.ConnectionID{})
+			err := sess.handleFrame(&wire.StreamsBlockedFrame{}, protocol.Encryption1RTT, protocol.ConnectionID{})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -358,7 +358,7 @@ var _ = Describe("Session", func() {
 			Expect(sess.handleFrame(&wire.ConnectionCloseFrame{
 				ErrorCode:    qerr.StreamLimitError,
 				ReasonPhrase: "foobar",
-			}, protocol.EncryptionUnspecified, protocol.ConnectionID{})).To(Succeed())
+			}, protocol.Encryption1RTT, protocol.ConnectionID{})).To(Succeed())
 			Eventually(sess.Context().Done()).Should(BeClosed())
 		})
 
@@ -392,7 +392,7 @@ var _ = Describe("Session", func() {
 				ReasonPhrase:       "foobar",
 				IsApplicationError: true,
 			}
-			Expect(sess.handleFrame(ccf, protocol.EncryptionUnspecified, protocol.ConnectionID{})).To(Succeed())
+			Expect(sess.handleFrame(ccf, protocol.Encryption1RTT, protocol.ConnectionID{})).To(Succeed())
 			Eventually(sess.Context().Done()).Should(BeClosed())
 		})