DTLSv1_listen rewrite

The existing implementation of DTLSv1_listen() is fundamentally flawed. This
function is used in DTLS solutions to listen for new incoming connections
from DTLS clients. A client will send an initial ClientHello. The server
will respond with a HelloVerifyRequest containing a unique cookie. The
client the responds with a second ClientHello - which this time contains the
cookie.

Once the cookie has been verified then DTLSv1_listen() returns to user code,
which is typically expected to continue the handshake with a call to (for
example) SSL_accept().

Whilst listening for incoming ClientHellos, the underlying BIO is usually in
an unconnected state. Therefore ClientHellos can come in from *any* peer.
The arrival of the first ClientHello without the cookie, and the second one
with it, could be interspersed with other intervening messages from
different clients.

The whole purpose of this mechanism is as a defence against DoS attacks. The
idea is to avoid allocating state on the server until the client has
verified that it is capable of receiving messages at the address it claims
to come from. However the existing DTLSv1_listen() implementation completely
fails to do this. It attempts to super-impose itself on the standard state
machine and reuses all of this code. However the standard state machine
expects to operate in a stateful manner with a single client, and this can
cause various problems.

A second more minor issue is that the return codes from this function are
quite confused, with no distinction made between fatal and non-fatal errors.
Most user code treats all errors as non-fatal, and simply retries the call
to DTLSv1_listen().

This commit completely rewrites the implementation of DTLSv1_listen() and
provides a stand alone implementation that does not rely on the existing
state machine. It also provides more consistent return codes.

Reviewed-by: Andy Polyakov <appro@openssl.org>

1 files changed, 6 insertions, 0 deletions

diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index 21836d831d..447bac6937 100644
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -87,6 +87,7 @@ static ERR_STRING_DATA SSL_str_functs[] = {
     {ERR_FUNC(SSL_F_DTLS1_GET_RECORD), "dtls1_get_record"},
     {ERR_FUNC(SSL_F_DTLS1_HANDLE_TIMEOUT), "dtls1_handle_timeout"},
     {ERR_FUNC(SSL_F_DTLS1_HEARTBEAT), "dtls1_heartbeat"},
+    {ERR_FUNC(SSL_F_DTLS1_LISTEN), "DTLS1_LISTEN"},
     {ERR_FUNC(SSL_F_DTLS1_OUTPUT_CERT_CHAIN), "dtls1_output_cert_chain"},
     {ERR_FUNC(SSL_F_DTLS1_PREPROCESS_FRAGMENT), "DTLS1_PREPROCESS_FRAGMENT"},
     {ERR_FUNC(SSL_F_DTLS1_PROCESS_OUT_OF_SEQ_MESSAGE),
@@ -406,6 +407,8 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "compression id not within private range"},
     {ERR_REASON(SSL_R_COMPRESSION_LIBRARY_ERROR), "compression library error"},
     {ERR_REASON(SSL_R_CONNECTION_TYPE_NOT_SET), "connection type not set"},
+    {ERR_REASON(SSL_R_COOKIE_GEN_CALLBACK_FAILURE),
+     "cookie gen callback failure"},
     {ERR_REASON(SSL_R_COOKIE_MISMATCH), "cookie mismatch"},
     {ERR_REASON(SSL_R_DATA_BETWEEN_CCS_AND_FINISHED),
      "data between ccs and finished"},
@@ -440,6 +443,7 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "error in received cipher list"},
     {ERR_REASON(SSL_R_EXCESSIVE_MESSAGE_SIZE), "excessive message size"},
     {ERR_REASON(SSL_R_EXTRA_DATA_IN_MESSAGE), "extra data in message"},
+    {ERR_REASON(SSL_R_FRAGMENTED_CLIENT_HELLO), "fragmented client hello"},
     {ERR_REASON(SSL_R_GOT_A_FIN_BEFORE_A_CCS), "got a fin before a ccs"},
     {ERR_REASON(SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS),
      "got next proto before a ccs"},
@@ -455,6 +459,7 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "invalid compression algorithm"},
     {ERR_REASON(SSL_R_INVALID_NULL_CMD_NAME), "invalid null cmd name"},
     {ERR_REASON(SSL_R_INVALID_PURPOSE), "invalid purpose"},
+    {ERR_REASON(SSL_R_INVALID_SEQUENCE_NUMBER), "invalid sequence number"},
     {ERR_REASON(SSL_R_INVALID_SERVERINFO_DATA), "invalid serverinfo data"},
     {ERR_REASON(SSL_R_INVALID_SRP_USERNAME), "invalid srp username"},
     {ERR_REASON(SSL_R_INVALID_STATUS_RESPONSE), "invalid status response"},
@@ -510,6 +515,7 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "no shared sigature algorithms"},
     {ERR_REASON(SSL_R_NO_SRTP_PROFILES), "no srtp profiles"},
     {ERR_REASON(SSL_R_NO_VERIFY_CALLBACK), "no verify callback"},
+    {ERR_REASON(SSL_R_NO_VERIFY_COOKIE_CALLBACK), "no verify cookie callback"},
     {ERR_REASON(SSL_R_NULL_SSL_CTX), "null ssl ctx"},
     {ERR_REASON(SSL_R_NULL_SSL_METHOD_PASSED), "null ssl method passed"},
     {ERR_REASON(SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED),