DTLS 1.3: discard forbidden plaintext before replay checks

After the hellos, rejected plaintext records mustn't advance the epoch 0
replay window. The existing check runs after record processing updates
that window, so a rejected record can affect a later plaintext alert.

Move the check ahead of replay processing, while keeping next epoch
records eligible for buffering. Extend the existing rejection test to
verify that the bitmap stays unchanged before completing the handshake.

Assisted-by: Codex:gpt-6-astra
Reviewed-by: Tomas Mraz <[email protected]>
Reviewed-by: Matt Caswell <[email protected]>
Merge-date: Wed Oct 21 15:56:56 2026
Merged-from: https://github.com/openssl/openssl/pull/33271

2 files changed, 24 insertions, 7 deletions

diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index 4cb8c4d8271..ff5ff6986dc 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -863,6 +863,17 @@ again:
     }
 
     /*
+     * Reject forbidden epoch 0 records before they can update the replay
+     * bitmap. Next epoch records have already been buffered above, so their
+     * unified headers aren't interpreted as plaintext content types here.
+     */
+    if (!dtls13_plaintext_allows_type(rl, rr->type)) {
+        rr->length = 0;
+        rl->packet_length = 0;
+        goto again;
+    }
+
+    /*
      * dtls_record_replay_check() and dtls_process_record() read and write
      * whichever OSSL_RECORD_LAYER they are passed, not rr. When crypto_rl is
      * the retained previous epoch, seed its record slot from rr so they
@@ -919,12 +930,6 @@ again:
         goto again;
     }
 
-    if (!dtls13_plaintext_allows_type(rl, rr->type)) {
-        rr->length = 0;
-        rl->packet_length = 0;
-        goto again;
-    }
-
     if (rr->length == 0) {
         /* No payload data in this record. Dump it */
         rl->packet_length = 0;
diff --git a/test/dtlstest.c b/test/dtlstest.c
index 994f50eec1e..fc52efa689f 100644
--- a/test/dtlstest.c
+++ b/test/dtlstest.c
@@ -1623,6 +1623,8 @@ static int test_dtls13_plaintext_after_hellos(int idx)
 #endif
     SSL_CTX *sctx = NULL, *cctx = NULL;
     SSL *sssl = NULL, *cssl = NULL;
+    OSSL_RECORD_LAYER *rl;
+    DTLS_BITMAP bitmap;
     /*
      * A handshake message header. The remaining SHA384_DIGEST_LENGTH bytes
      * are not initialised explicitly, so they are zero: that is the body.
@@ -1636,7 +1638,7 @@ static int test_dtls13_plaintext_after_hellos(int idx)
     };
     unsigned char pkt[DTLS1_RT_HEADER_LENGTH + sizeof(msg)];
     size_t pktlen;
-    int testresult = 0;
+    int ret, testresult = 0;
 
     if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
             DTLS_client_method(),
@@ -1666,6 +1668,16 @@ static int test_dtls13_plaintext_after_hellos(int idx)
             || !TEST_int_le(SSL_accept(sssl), 0)
             || !inject_client_datagram(cssl, pkt, pktlen))
             goto end;
+
+        /* Discarding the plaintext record must leave replay state unchanged. */
+        rl = SSL_CONNECTION_FROM_SSL_ONLY(sssl)->rlayer.rrl;
+        bitmap = rl->bitmap;
+        ret = SSL_accept(sssl);
+        if (!TEST_int_eq(SSL_get_error(sssl, ret), SSL_ERROR_WANT_READ)
+            || !TEST_ptr_eq(SSL_CONNECTION_FROM_SSL_ONLY(sssl)->rlayer.rrl, rl)
+            || !TEST_uint64_t_eq(rl->bitmap.max_seq_num, bitmap.max_seq_num)
+            || !TEST_uint64_t_eq(rl->bitmap.map, bitmap.map))
+            goto end;
         break;
     case 1:
         if (!inject_client_datagram(cssl, pkt, pktlen))