Further libssl size_t-ify of reading

Writing still to be done

Reviewed-by: Rich Salz <rsalz@openssl.org>

1 files changed, 46 insertions, 43 deletions

diff --git a/ssl/record/rec_layer_d1.c b/ssl/record/rec_layer_d1.c
index c9fd0669ed..7b35d590dc 100644
--- a/ssl/record/rec_layer_d1.c
+++ b/ssl/record/rec_layer_d1.c
@@ -118,8 +118,8 @@ void DTLS_RECORD_LAYER_set_write_sequence(RECORD_LAYER *rl, unsigned char *seq)
     memcpy(rl->write_sequence, seq, SEQ_NUM_SIZE);
 }
 
-static int have_handshake_fragment(SSL *s, int type, unsigned char *buf,
-                                   int len);
+static size_t have_handshake_fragment(SSL *s, int type, unsigned char *buf,
+                                      size_t len);
 
 /* copy buffered record into SSL structure */
 static int dtls1_copy_record(SSL *s, pitem *item)
@@ -336,10 +336,10 @@ int dtls1_process_buffered_records(SSL *s)
  *             none of our business
  */
 int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
-                     int len, int peek)
+                     size_t len, int peek, size_t *read)
 {
-    int al, i, j, ret;
-    unsigned int n;
+    int al, i, j, iret;
+    size_t ret, n;
     SSL3_RECORD *rr;
     void (*cb) (const SSL *ssl, int type2, int val) = NULL;
 
@@ -359,9 +359,11 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
     /*
      * check whether there's a handshake message (client hello?) waiting
      */
-    if ((ret = have_handshake_fragment(s, type, buf, len))) {
+    ret = have_handshake_fragment(s, type, buf, len);
+    if (ret > 0) {
         *recvd_type = SSL3_RT_HANDSHAKE;
-        return ret;
+        *read = ret;
+        return 1;
     }
 
     /*
@@ -385,10 +387,10 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
         /* type == SSL3_RT_APPLICATION_DATA */
         i = s->handshake_func(s);
         if (i < 0)
-            return (i);
+            return i;
         if (i == 0) {
             SSLerr(SSL_F_DTLS1_READ_BYTES, SSL_R_SSL_HANDSHAKE_FAILURE);
-            return (-1);
+            return -1;
         }
     }
 
@@ -434,12 +436,12 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
     /* get new packet if necessary */
     if ((SSL3_RECORD_get_length(rr) == 0)
         || (s->rlayer.rstate == SSL_ST_READ_BODY)) {
-        ret = dtls1_get_record(s);
-        if (ret <= 0) {
-            ret = dtls1_read_failed(s, ret);
+        iret = dtls1_get_record(s);
+        if (iret <= 0) {
+            iret = dtls1_read_failed(s, iret);
             /* anything other than a timeout is an error */
-            if (ret <= 0)
-                return (ret);
+            if (iret <= 0)
+                return iret;
             else
                 goto start;
         }
@@ -479,7 +481,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
     if (s->shutdown & SSL_RECEIVED_SHUTDOWN) {
         SSL3_RECORD_set_length(rr, 0);
         s->rwstate = SSL_NOTHING;
-        return (0);
+        return 0;
     }
 
     if (type == SSL3_RECORD_get_type(rr)
@@ -504,13 +506,13 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
         if (recvd_type != NULL)
             *recvd_type = SSL3_RECORD_get_type(rr);
 
-        if (len <= 0)
-            return (len);
+        if (len == 0)
+            return 0;
 
-        if ((unsigned int)len > SSL3_RECORD_get_length(rr))
+        if (len > SSL3_RECORD_get_length(rr))
             n = SSL3_RECORD_get_length(rr);
         else
-            n = (unsigned int)len;
+            n = len;
 
         memcpy(buf, &(SSL3_RECORD_get_data(rr)[SSL3_RECORD_get_off(rr)]), n);
         if (!peek) {
@@ -543,10 +545,11 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
             s->d1->shutdown_received
             && !BIO_dgram_sctp_msg_waiting(SSL_get_rbio(s))) {
             s->shutdown |= SSL_RECEIVED_SHUTDOWN;
-            return (0);
+            return 0;
         }
 #endif
-        return (n);
+        *read = n;
+        return 1;
     }
 
     /*
@@ -559,9 +562,9 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
      * that so that we can process the data at a fixed place.
      */
     {
-        unsigned int k, dest_maxlen = 0;
+        size_t k, dest_maxlen = 0;
         unsigned char *dest = NULL;
-        unsigned int *dest_len = NULL;
+        size_t *dest_len = NULL;
 
         if (SSL3_RECORD_get_type(rr) == SSL3_RT_HANDSHAKE) {
             dest_maxlen = sizeof s->rlayer.d->handshake_fragment;
@@ -584,7 +587,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
             s->rwstate = SSL_READING;
             BIO_clear_retry_flags(SSL_get_rbio(s));
             BIO_set_retry_read(SSL_get_rbio(s));
-            return (-1);
+            return -1;
         }
 #endif
         /* else it's a CCS message, or application data or wrong */
@@ -600,7 +603,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
                 s->rwstate = SSL_READING;
                 BIO_clear_retry_flags(bio);
                 BIO_set_retry_read(bio);
-                return (-1);
+                return -1;
             }
 
             /* Not certain if this is the right error handling */
@@ -677,10 +680,10 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
             if (ssl3_renegotiate_check(s)) {
                 i = s->handshake_func(s);
                 if (i < 0)
-                    return (i);
+                    return i;
                 if (i == 0) {
                     SSLerr(SSL_F_DTLS1_READ_BYTES, SSL_R_SSL_HANDSHAKE_FAILURE);
-                    return (-1);
+                    return -1;
                 }
 
                 if (!(s->mode & SSL_MODE_AUTO_RETRY)) {
@@ -697,7 +700,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
                         bio = SSL_get_rbio(s);
                         BIO_clear_retry_flags(bio);
                         BIO_set_retry_read(bio);
-                        return (-1);
+                        return -1;
                     }
                 }
             }
@@ -757,7 +760,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
                 }
 #endif
                 s->shutdown |= SSL_RECEIVED_SHUTDOWN;
-                return (0);
+                return 0;
             }
 #if 0
             /* XXX: this is a possible improvement in the future */
@@ -797,7 +800,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
             ERR_add_error_data(2, "SSL alert number ", tmp);
             s->shutdown |= SSL_RECEIVED_SHUTDOWN;
             SSL_CTX_remove_session(s->session_ctx, s->session);
-            return (0);
+            return 0;
         } else {
             al = SSL_AD_ILLEGAL_PARAMETER;
             SSLerr(SSL_F_DTLS1_READ_BYTES, SSL_R_UNKNOWN_ALERT_TYPE);
@@ -811,7 +814,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
                                             * shutdown */
         s->rwstate = SSL_NOTHING;
         SSL3_RECORD_set_length(rr, 0);
-        return (0);
+        return 0;
     }
 
     if (SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC) {
@@ -858,10 +861,10 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
         }
         i = s->handshake_func(s);
         if (i < 0)
-            return (i);
+            return i;
         if (i == 0) {
             SSLerr(SSL_F_DTLS1_READ_BYTES, SSL_R_SSL_HANDSHAKE_FAILURE);
-            return (-1);
+            return -1;
         }
 
         if (!(s->mode & SSL_MODE_AUTO_RETRY)) {
@@ -878,7 +881,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
                 bio = SSL_get_rbio(s);
                 BIO_clear_retry_flags(bio);
                 BIO_set_retry_read(bio);
-                return (-1);
+                return -1;
             }
         }
         goto start;
@@ -917,7 +920,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
             (s->s3->total_renegotiations != 0) &&
             ossl_statem_app_data_allowed(s)) {
             s->s3->in_read_app_data = 2;
-            return (-1);
+            return -1;
         } else {
             al = SSL_AD_UNEXPECTED_MESSAGE;
             SSLerr(SSL_F_DTLS1_READ_BYTES, SSL_R_UNEXPECTED_RECORD);
@@ -928,15 +931,15 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
 
  f_err:
     ssl3_send_alert(s, SSL3_AL_FATAL, al);
-    return (-1);
+    return -1;
 }
 
-        /*
-         * this only happens when a client hello is received and a handshake
-         * is started.
-         */
-static int have_handshake_fragment(SSL *s, int type, unsigned char *buf,
-                                   int len)
+/*
+ * this only happens when a client hello is received and a handshake
+ * is started.
+ */
+static size_t have_handshake_fragment(SSL *s, int type, unsigned char *buf,
+                                      size_t len)
 {
 
     if ((type == SSL3_RT_HANDSHAKE)
@@ -945,7 +948,7 @@ static int have_handshake_fragment(SSL *s, int type, unsigned char *buf,
     {
         unsigned char *src = s->rlayer.d->handshake_fragment;
         unsigned char *dst = buf;
-        unsigned int k, n;
+        size_t k, n;
 
         /* peek == 0 */
         n = 0;