DTLS listener: bound the per-connection receive queue

The DTLS listener queues datagrams from a known peer into a per-connection
list (DTLS_RX::urxe_pending) with no limit on count or bytes. A reachable
peer that completes the ordinary cookie exchange can grow one parked
connection's queue without bound as long as the application keeps
the shared demux pumped, moving datagrams into userspace past the bound
the kernel socket buffer would otherwise provide.

Cap the queue at OSSL_DTLS_RX_DEFAULT_MAX_QUEUED (128) datagrams
per connection - a datagram count, not a byte cap. Every queued URXE
holds a buffer of at least the listener's maximum datagram size whatever
the datagram's length, so a full queue costs about 295 KB per connection
at the default 2000-byte size, roughly one default kernel socket receive
buffer; across a full 256-connection listener budget that is about 75 MB.
The queue is consumed FIFO, so on overflow the oldest datagram is evicted
rather than the newest: a DTLS flight can span several datagrams,
and under overload keeping the newest biases the queue toward the flight
the peer just sent instead of stale fragments of an older one.
The evicted URXE is unhooked under rx->mutex and released to the demux
after dropping it, so demux->mutex is never taken while rx->mutex
is held.

References: https://github.com/openssl/openssl/issues/33173
Assisted-by: Claude Code:moonshotai/kimi-k3
Reviewed-by: Tomas Mraz <[email protected]>
Reviewed-by: Ryan Hooper <[email protected]>
Reviewed-by: Eugene Syromiatnikov <[email protected]>
Merge-date: Wed Oct 14 10:54:54 2026
Merged-from: https://github.com/openssl/openssl/pull/33197

3 files changed, 137 insertions, 0 deletions

diff --git a/include/internal/dtls_record_rx.h b/include/internal/dtls_record_rx.h
index 44099b2477f..2be3b11a637 100644
--- a/include/internal/dtls_record_rx.h
+++ b/include/internal/dtls_record_rx.h
@@ -16,6 +16,17 @@
 
 #ifndef OPENSSL_NO_DTLS
 
+/*
+ * Maximum datagrams queued per connection in DTLS_RX::urxe_pending: a
+ * count, not a byte cap. On overflow the oldest datagram is evicted
+ * (DTLS permits datagram loss). Each queued URXE holds a buffer of the
+ * listener's maximum datagram size, so 128 entries cost about 295 KB per
+ * connection at the default 2000-byte size - roughly one default kernel
+ * socket receive buffer, the bound that queuing into userspace would
+ * otherwise bypass.
+ */
+#define OSSL_DTLS_RX_DEFAULT_MAX_QUEUED 128
+
 typedef struct dtls_rx_st {
     DGRAM_DEMUX *demux;
     DGRAM_URXE_LIST urxe_pending;
diff --git a/ssl/dtls_record_rx.c b/ssl/dtls_record_rx.c
index e216f963952..a1bf4c18d4f 100644
--- a/ssl/dtls_record_rx.c
+++ b/ssl/dtls_record_rx.c
@@ -64,12 +64,27 @@ void ossl_dtls_rx_free(DTLS_RX *rx)
 
 /*
  * Injects a received URXE into the DTLS_RX struct.
+ *
+ * On overflow the oldest queued datagram is evicted: a flight can span
+ * several datagrams, so keeping the newest biases the queue toward the
+ * flight the peer just sent. The evicted URXE is released only after
+ * rx->mutex is dropped, as release takes demux->mutex, which must not
+ * nest under rx->mutex.
  */
 void ossl_dtls_rx_inject_urxe(DTLS_RX *rx, DGRAM_URXE *e)
 {
+    DGRAM_URXE *oldest = NULL;
+
     ossl_crypto_mutex_lock(rx->mutex);
+    if (ossl_list_urxe_num(&rx->urxe_pending) >= OSSL_DTLS_RX_DEFAULT_MAX_QUEUED)
+        oldest = ossl_list_urxe_head(&rx->urxe_pending);
+    if (oldest != NULL)
+        ossl_list_urxe_remove(&rx->urxe_pending, oldest);
     ossl_list_urxe_insert_tail(&rx->urxe_pending, e);
     ossl_crypto_mutex_unlock(rx->mutex);
+
+    if (oldest != NULL)
+        ossl_dtls_rx_release_urxe(rx, oldest);
 }
 
 /*
diff --git a/test/dtlsssllistenertest.c b/test/dtlsssllistenertest.c
index 8740ce8bf84..9e09c22d493 100644
--- a/test/dtlsssllistenertest.c
+++ b/test/dtlsssllistenertest.c
@@ -28,6 +28,7 @@
 #include "internal/time.h"
 #include "internal/sockets.h"
 #include "internal/dgram_demux.h"
+#include "internal/dtls_record_rx.h"
 #include "internal/ssl_unwrap.h"
 #include "helpers/ssltestlib.h"
 #include "testutil.h"
@@ -5909,6 +5910,111 @@ end:
     return testresult;
 }
 
+#ifndef OPENSSL_NO_DTLS1_2
+/*
+ * Regression test for the receive-queue bound and its overflow policy: a
+ * connection left unread must not accumulate an unbounded datagram queue,
+ * and when the queue is full the oldest datagram is evicted, not the
+ * newest. A parked connection is flooded past the cap with datagrams
+ * carrying a sequence counter; the queue must stop at
+ * OSSL_DTLS_RX_DEFAULT_MAX_QUEUED entries, with the head carrying
+ * (flood - cap) and the tail (flood - 1). Under tail-drop the head would
+ * still carry 0.
+ */
+static int test_dtls_listener_rx_queue_drops_oldest(void)
+{
+    SSL_CTX *sctx = NULL, *cctx = NULL;
+    SSL *listener = NULL, *clientssl = NULL, *serverssl = NULL;
+    BIO_ADDR *client_addr = NULL;
+    BIO *client_bio;
+    SSL_CONNECTION *sc;
+    SSL_POLL_ITEM poll_item;
+    struct timeval poll_timeout;
+    size_t poll_result, queued, i;
+    unsigned char record[64];
+    DGRAM_URXE *head, *tail;
+    const unsigned char *head_data, *tail_data;
+    unsigned int head_seq, tail_seq;
+    int testresult = 0;
+    const size_t cap = OSSL_DTLS_RX_DEFAULT_MAX_QUEUED;
+    const size_t flood = cap + 64;
+
+    if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+            DTLS_client_method(), DTLS1_2_VERSION, DTLS1_2_VERSION,
+            &sctx, &cctx, cert, privkey)))
+        goto end;
+
+    if (!TEST_true(create_dtls_listener_and_client_mem(sctx, cctx,
+            SSL_LISTENER_FLAG_SINGLE_THREAD,
+            &listener, &clientssl, &client_addr)))
+        goto end;
+
+    if (!TEST_true(drive_until_connection_queued(listener, clientssl)))
+        goto end;
+    if (!TEST_ptr(serverssl = SSL_accept_connection(listener,
+                      SSL_ACCEPT_CONNECTION_NO_BLOCK)))
+        goto end;
+
+    client_bio = SSL_get_rbio(clientssl);
+    poll_item.desc.type = BIO_POLL_DESCRIPTOR_TYPE_SSL;
+    poll_item.desc.value.ssl = listener;
+    poll_item.events = SSL_POLL_EVENT_IC;
+    poll_timeout.tv_sec = 0;
+    poll_timeout.tv_usec = 0;
+
+    /*
+     * The client SSL is never stepped again, so no retransmit can add
+     * datagrams: every queued entry is one of ours, in order.
+     */
+    memset(record, 0xA5, sizeof(record));
+    for (i = 0; i < flood; i++) {
+        /* Stamp a big-endian sequence counter into the first four bytes. */
+        record[0] = (unsigned char)(i >> 24);
+        record[1] = (unsigned char)(i >> 16);
+        record[2] = (unsigned char)(i >> 8);
+        record[3] = (unsigned char)i;
+        if (!TEST_int_eq(BIO_write(client_bio, record, sizeof(record)),
+                (int)sizeof(record)))
+            goto end;
+        poll_item.revents = 0;
+        if (!TEST_true(SSL_poll(&poll_item, 1, sizeof(poll_item),
+                &poll_timeout, 0, &poll_result)))
+            goto end;
+    }
+
+    if (!TEST_ptr(sc = SSL_CONNECTION_FROM_SSL_ONLY(serverssl))
+        || !TEST_ptr(sc->listener_rx))
+        goto end;
+    queued = ossl_list_urxe_num(&sc->listener_rx->urxe_pending);
+    if (!TEST_size_t_eq(queued, cap))
+        goto end;
+
+    /* The oldest datagrams were evicted: only the newest cap ones remain. */
+    if (!TEST_ptr(head = ossl_list_urxe_head(&sc->listener_rx->urxe_pending))
+        || !TEST_ptr(tail = ossl_list_urxe_tail(&sc->listener_rx->urxe_pending)))
+        goto end;
+    head_data = ossl_dgram_urxe_data(head);
+    tail_data = ossl_dgram_urxe_data(tail);
+    head_seq = ((unsigned int)head_data[0] << 24) | ((unsigned int)head_data[1] << 16)
+        | ((unsigned int)head_data[2] << 8) | head_data[3];
+    tail_seq = ((unsigned int)tail_data[0] << 24) | ((unsigned int)tail_data[1] << 16)
+        | ((unsigned int)tail_data[2] << 8) | tail_data[3];
+    if (!TEST_uint_eq(head_seq, (unsigned int)(flood - cap))
+        || !TEST_uint_eq(tail_seq, (unsigned int)(flood - 1)))
+        goto end;
+
+    testresult = 1;
+end:
+    SSL_free(serverssl);
+    SSL_free(clientssl);
+    SSL_free(listener);
+    BIO_ADDR_free(client_addr);
+    SSL_CTX_free(sctx);
+    SSL_CTX_free(cctx);
+    return testresult;
+}
+#endif /* OPENSSL_NO_DTLS1_2 */
+
 OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
@@ -6041,5 +6147,10 @@ int setup_tests(void)
     ADD_TEST(test_dtls_poll_conn_honours_retransmit_timer);
     ADD_TEST(test_dtls_timer_stopped_when_retransmits_exhausted);
 
+    /* Receive-queue bound regression test */
+#ifndef OPENSSL_NO_DTLS1_2
+    ADD_TEST(test_dtls_listener_rx_queue_drops_oldest);
+#endif
+
     return 1;
 }