diff --git a/conn.go b/conn.go
index 04b4f7d57..ccea999d9 100644
--- a/conn.go
+++ b/conn.go
@@ -625,13 +625,8 @@ func (c *Conn) readAndBuffer(ctx context.Context) error {
 			hasHandshake = true
 		}
 
-		var e *alertError
-		if errors.As(err, &e) {
-			if e.IsFatalOrCloseNotify() {
-				return e
-			}
-		} else if err != nil {
-			return e
+		if err != nil {
+			return err
 		}
 	}
 	if hasHandshake {
@@ -666,7 +661,7 @@ func (c *Conn) handleQueuedPackets(ctx context.Context) error {
 				return e
 			}
 		} else if err != nil {
-			return e
+			return err
 		}
 	}
 	return nil
diff --git a/conn_test.go b/conn_test.go
index 6083a050a..0d0a33bf9 100644
--- a/conn_test.go
+++ b/conn_test.go
@@ -3135,3 +3135,53 @@ func TestApplicationDataQueueLimited(t *testing.T) {
 	ca.Close()
 	<-done
 }
+
+func TestApplicationDataWithClientHelloRejected(t *testing.T) {
+	// Limit runtime in case of deadlocks
+	lim := test.TimeOut(time.Second * 20)
+	defer lim.Stop()
+
+	// Check for leaking routines
+	report := test.CheckRoutines(t)
+	defer report()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	ca, cb := dpipe.Pipe()
+	defer ca.Close()
+	defer cb.Close()
+
+	done := make(chan struct{})
+	go func() {
+		if _, err := testServer(ctx, cb, &Config{}, true); err == nil {
+			t.Error("expected handshake to fail")
+		}
+		close(done)
+	}()
+	extensions := []extension.Extension{}
+
+	time.Sleep(50 * time.Millisecond)
+
+	err := sendClientHello([]byte{}, ca, 0, extensions)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Send an application data packet
+	packet, err := (&recordlayer.RecordLayer{
+		Header: recordlayer.Header{
+			Version:        protocol.Version1_2,
+			SequenceNumber: uint64(3),
+			Epoch:          0,
+		},
+		Content: &protocol.ApplicationData{
+			Data: []byte{1, 2, 3, 4},
+		},
+	}).Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ca.Write(packet)
+	<-done
+}