Convert CRYPTO_LOCK_SSL_* to new multi-threading API

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>

9 files changed, 166 insertions, 122 deletions

diff --git a/include/openssl/crypto.h b/include/openssl/crypto.h
index cfda8bb214..3f9ce2b442 100644
--- a/include/openssl/crypto.h
+++ b/include/openssl/crypto.h
@@ -166,15 +166,8 @@ extern "C" {
  */
 
 # define CRYPTO_LOCK_X509_STORE          11
-# define CRYPTO_LOCK_SSL_CTX             12
-# define CRYPTO_LOCK_SSL_CERT            13
-# define CRYPTO_LOCK_SSL_SESSION         14
-# define CRYPTO_LOCK_SSL_SESS_CERT       15
-# define CRYPTO_LOCK_SSL                 16
-# define CRYPTO_LOCK_SSL_METHOD          17
 # define CRYPTO_LOCK_RAND                18
 # define CRYPTO_LOCK_RAND2               19
-# define CRYPTO_LOCK_READDIR             24
 # define CRYPTO_LOCK_RSA_BLINDING        25
 # define CRYPTO_LOCK_DYNLOCK             29
 # define CRYPTO_LOCK_ENGINE              30
diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index be2ca2ab37..adfad4c369 100644
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -1499,6 +1499,7 @@ int SSL_SESSION_print_fp(FILE *fp, const SSL_SESSION *ses);
 # endif
 int SSL_SESSION_print(BIO *fp, const SSL_SESSION *ses);
 int SSL_SESSION_print_keylog(BIO *bp, const SSL_SESSION *x);
+int SSL_SESSION_up_ref(SSL_SESSION *ses);
 void SSL_SESSION_free(SSL_SESSION *ses);
 __owur int i2d_SSL_SESSION(SSL_SESSION *in, unsigned char **pp);
 __owur int SSL_set_session(SSL *to, SSL_SESSION *session);
diff --git a/ssl/ssl_cert.c b/ssl/ssl_cert.c
index e8044125eb..21cfb965d2 100644
--- a/ssl/ssl_cert.c
+++ b/ssl/ssl_cert.c
@@ -129,36 +129,27 @@
 # include <openssl/dh.h>
 #endif
 #include <openssl/bn.h>
+#include "internal/threads.h"
 #include "ssl_locl.h"
 
 static int ssl_security_default_callback(SSL *s, SSL_CTX *ctx, int op,
                                          int bits, int nid, void *other,
                                          void *ex);
 
-int SSL_get_ex_data_X509_STORE_CTX_idx(void)
-{
-    static volatile int ssl_x509_store_ctx_idx = -1;
-    int got_write_lock = 0;
-
-    CRYPTO_r_lock(CRYPTO_LOCK_SSL_CTX);
+static CRYPTO_ONCE ssl_x509_store_ctx_once = CRYPTO_ONCE_STATIC_INIT;
+static volatile int ssl_x509_store_ctx_idx = -1;
 
-    if (ssl_x509_store_ctx_idx < 0) {
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL_CTX);
-        CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
-        got_write_lock = 1;
-
-        if (ssl_x509_store_ctx_idx < 0) {
-            ssl_x509_store_ctx_idx =
-                X509_STORE_CTX_get_ex_new_index(0, "SSL for verify callback",
+static void ssl_x509_store_ctx_init(void)
+{
+    ssl_x509_store_ctx_idx = X509_STORE_CTX_get_ex_new_index(0,
+                                                "SSL for verify callback",
                                                 NULL, NULL, NULL);
-        }
-    }
+}
 
-    if (got_write_lock)
-        CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
-    else
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL_CTX);
+int SSL_get_ex_data_X509_STORE_CTX_idx(void)
+{
 
+    CRYPTO_THREAD_run_once(&ssl_x509_store_ctx_once, ssl_x509_store_ctx_init);
     return ssl_x509_store_ctx_idx;
 }
 
@@ -168,7 +159,7 @@ CERT *ssl_cert_new(void)
 
     if (ret == NULL) {
         SSLerr(SSL_F_SSL_CERT_NEW, ERR_R_MALLOC_FAILURE);
-        return (NULL);
+        return NULL;
     }
 
     ret->key = &(ret->pkeys[SSL_PKEY_RSA_ENC]);
@@ -176,7 +167,14 @@ CERT *ssl_cert_new(void)
     ret->sec_cb = ssl_security_default_callback;
     ret->sec_level = OPENSSL_TLS_SECURITY_LEVEL;
     ret->sec_ex = NULL;
-    return (ret);
+    ret->lock = CRYPTO_THREAD_lock_new();
+    if (ret->lock == NULL) {
+        SSLerr(SSL_F_SSL_CERT_NEW, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(ret);
+        return NULL;
+    }
+
+    return ret;
 }
 
 CERT *ssl_cert_dup(CERT *cert)
@@ -186,11 +184,17 @@ CERT *ssl_cert_dup(CERT *cert)
 
     if (ret == NULL) {
         SSLerr(SSL_F_SSL_CERT_DUP, ERR_R_MALLOC_FAILURE);
-        return (NULL);
+        return NULL;
     }
 
     ret->references = 1;
     ret->key = &ret->pkeys[cert->key - cert->pkeys];
+    ret->lock = CRYPTO_THREAD_lock_new();
+    if (ret == NULL) {
+        SSLerr(SSL_F_SSL_CERT_DUP, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(ret);
+        return NULL;
+    }
 
 #ifndef OPENSSL_NO_DH
     if (cert->dh_tmp != NULL) {
@@ -297,7 +301,7 @@ CERT *ssl_cert_dup(CERT *cert)
             goto err;
     }
 #endif
-    return (ret);
+    return ret;
 
  err:
     ssl_cert_free(ret);
@@ -333,7 +337,7 @@ void ssl_cert_free(CERT *c)
     if (c == NULL)
         return;
 
-    i = CRYPTO_add(&c->references, -1, CRYPTO_LOCK_SSL_CERT);
+    CRYPTO_atomic_add(&c->references, -1, &i, c->lock);
     REF_PRINT_COUNT("CERT", c);
     if (i > 0)
         return;
@@ -355,6 +359,7 @@ void ssl_cert_free(CERT *c)
 #ifndef OPENSSL_NO_PSK
     OPENSSL_free(c->psk_identity_hint);
 #endif
+    CRYPTO_THREAD_lock_free(c->lock);
     OPENSSL_free(c);
 }
 
@@ -784,8 +789,6 @@ int SSL_add_dir_cert_subjects_to_stack(STACK_OF(X509_NAME) *stack,
     const char *filename;
     int ret = 0;
 
-    CRYPTO_w_lock(CRYPTO_LOCK_READDIR);
-
     /* Note that a side effect is that the CAs will be sorted by name */
 
     while ((filename = OPENSSL_DIR_read(&d, dir))) {
@@ -820,7 +823,7 @@ int SSL_add_dir_cert_subjects_to_stack(STACK_OF(X509_NAME) *stack,
  err:
     if (d)
         OPENSSL_DIR_end(&d);
-    CRYPTO_w_unlock(CRYPTO_LOCK_READDIR);
+
     return ret;
 }
 
diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index b26e972697..352bab9b0e 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -147,6 +147,7 @@
 #ifndef OPENSSL_NO_ENGINE
 # include <openssl/engine.h>
 #endif
+#include "internal/threads.h"
 #include "ssl_locl.h"
 
 #define SSL_ENC_DES_IDX         0
@@ -213,6 +214,8 @@ static const EVP_CIPHER *ssl_cipher_methods[SSL_ENC_NUM_IDX] = {
 
 static STACK_OF(SSL_COMP) *ssl_comp_methods = NULL;
 
+static CRYPTO_ONCE ssl_load_builtin_comp_once = CRYPTO_ONCE_STATIC_INIT;
+
 /*
  * Constant SSL_MAX_DIGEST equal to size of digests array should be defined
  * in the ssl_locl.h
@@ -575,41 +578,31 @@ static int sk_comp_cmp(const SSL_COMP *const *a, const SSL_COMP *const *b)
     return ((*a)->id - (*b)->id);
 }
 
-static void load_builtin_compressions(void)
+static void do_load_builtin_compressions(void)
 {
-    int got_write_lock = 0;
-
-    CRYPTO_r_lock(CRYPTO_LOCK_SSL);
-    if (ssl_comp_methods == NULL) {
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL);
-        CRYPTO_w_lock(CRYPTO_LOCK_SSL);
-        got_write_lock = 1;
-
-        if (ssl_comp_methods == NULL) {
-            SSL_COMP *comp = NULL;
-            COMP_METHOD *method = COMP_zlib();
-
-            CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_DISABLE);
-            ssl_comp_methods = sk_SSL_COMP_new(sk_comp_cmp);
-            if (COMP_get_type(method) != NID_undef
-                && ssl_comp_methods != NULL) {
-                comp = OPENSSL_malloc(sizeof(*comp));
-                if (comp != NULL) {
-                    comp->method = method;
-                    comp->id = SSL_COMP_ZLIB_IDX;
-                    comp->name = COMP_get_name(method);
-                    sk_SSL_COMP_push(ssl_comp_methods, comp);
-                    sk_SSL_COMP_sort(ssl_comp_methods);
-                }
-            }
-            CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+    SSL_COMP *comp = NULL;
+    COMP_METHOD *method = COMP_zlib();
+
+    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_DISABLE);
+    ssl_comp_methods = sk_SSL_COMP_new(sk_comp_cmp);
+
+    if (COMP_get_type(method) != NID_undef && ssl_comp_methods != NULL) {
+        comp = OPENSSL_malloc(sizeof(*comp));
+        if (comp != NULL) {
+            comp->method = method;
+            comp->id = SSL_COMP_ZLIB_IDX;
+            comp->name = COMP_get_name(method);
+            sk_SSL_COMP_push(ssl_comp_methods, comp);
+            sk_SSL_COMP_sort(ssl_comp_methods);
         }
     }
+    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+}
 
-    if (got_write_lock)
-        CRYPTO_w_unlock(CRYPTO_LOCK_SSL);
-    else
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL);
+static void load_builtin_compressions(void)
+{
+    CRYPTO_THREAD_run_once(&ssl_load_builtin_comp_once,
+                           do_load_builtin_compressions);
 }
 #endif
 
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index a1c8da8890..75e81252b3 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -631,6 +631,13 @@ SSL *SSL_new(SSL_CTX *ctx)
     if (s == NULL)
         goto err;
 
+    s->lock = CRYPTO_THREAD_lock_new();
+    if (s->lock == NULL) {
+        SSLerr(SSL_F_SSL_NEW, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(s);
+        return NULL;
+    }
+
     RECORD_LAYER_init(&s->rlayer, s);
 
     s->options = ctx->options;
@@ -677,7 +684,7 @@ SSL *SSL_new(SSL_CTX *ctx)
     if (ctx->default_read_buf_len > 0)
         SSL_set_default_read_buffer_len(s, ctx->default_read_buf_len);
 
-    CRYPTO_add(&ctx->references, 1, CRYPTO_LOCK_SSL_CTX);
+    SSL_CTX_up_ref(ctx);
     s->ctx = ctx;
     s->tlsext_debug_cb = 0;
     s->tlsext_debug_arg = NULL;
@@ -688,7 +695,7 @@ SSL *SSL_new(SSL_CTX *ctx)
     s->tlsext_ocsp_exts = NULL;
     s->tlsext_ocsp_resp = NULL;
     s->tlsext_ocsp_resplen = -1;
-    CRYPTO_add(&ctx->references, 1, CRYPTO_LOCK_SSL_CTX);
+    SSL_CTX_up_ref(ctx);
     s->initial_ctx = ctx;
 # ifndef OPENSSL_NO_EC
     if (ctx->tlsext_ecpointformatlist) {
@@ -755,16 +762,17 @@ SSL *SSL_new(SSL_CTX *ctx)
         goto err;
 #endif
 
-    return (s);
+    return s;
  err:
     SSL_free(s);
     SSLerr(SSL_F_SSL_NEW, ERR_R_MALLOC_FAILURE);
-    return (NULL);
+    return NULL;
 }
 
 void SSL_up_ref(SSL *s)
 {
-    CRYPTO_add(&s->references, 1, CRYPTO_LOCK_SSL);
+    int i;
+    CRYPTO_atomic_add(&s->references, 1, &i, s->lock);
 }
 
 int SSL_CTX_set_session_id_context(SSL_CTX *ctx, const unsigned char *sid_ctx,
@@ -797,17 +805,17 @@ int SSL_set_session_id_context(SSL *ssl, const unsigned char *sid_ctx,
 
 int SSL_CTX_set_generate_session_id(SSL_CTX *ctx, GEN_SESSION_CB cb)
 {
-    CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_write_lock(ctx->lock);
     ctx->generate_session_id = cb;
-    CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_unlock(ctx->lock);
     return 1;
 }
 
 int SSL_set_generate_session_id(SSL *ssl, GEN_SESSION_CB cb)
 {
-    CRYPTO_w_lock(CRYPTO_LOCK_SSL);
+    CRYPTO_THREAD_write_lock(ssl->lock);
     ssl->generate_session_id = cb;
-    CRYPTO_w_unlock(CRYPTO_LOCK_SSL);
+    CRYPTO_THREAD_unlock(ssl->lock);
     return 1;
 }
 
@@ -830,9 +838,9 @@ int SSL_has_matching_session_id(const SSL *ssl, const unsigned char *id,
     r.session_id_length = id_len;
     memcpy(r.session_id, id, id_len);
 
-    CRYPTO_r_lock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_read_lock(ssl->ctx->lock);
     p = lh_SSL_SESSION_retrieve(ssl->ctx->sessions, &r);
-    CRYPTO_r_unlock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_unlock(ssl->ctx->lock);
     return (p != NULL);
 }
 
@@ -1009,7 +1017,7 @@ void SSL_free(SSL *s)
     if (s == NULL)
         return;
 
-    i = CRYPTO_add(&s->references, -1, CRYPTO_LOCK_SSL);
+    CRYPTO_atomic_add(&s->references, -1, &i, s->lock);
     REF_PRINT_COUNT("SSL", s);
     if (i > 0)
         return;
@@ -1084,6 +1092,8 @@ void SSL_free(SSL *s)
     sk_SRTP_PROTECTION_PROFILE_free(s->srtp_profiles);
 #endif
 
+    CRYPTO_THREAD_lock_free(s->lock);
+
     OPENSSL_free(s);
 }
 
@@ -1366,6 +1376,7 @@ STACK_OF(X509) *SSL_get_peer_cert_chain(const SSL *s)
  */
 int SSL_copy_session_id(SSL *t, const SSL *f)
 {
+    int i;
     /* Do we need to to SSL locking? */
     if (!SSL_set_session(t, SSL_get_session(f))) {
         return 0;
@@ -1381,7 +1392,7 @@ int SSL_copy_session_id(SSL *t, const SSL *f)
             return 0;
     }
 
-    CRYPTO_add(&f->cert->references, 1, CRYPTO_LOCK_SSL_CERT);
+    CRYPTO_atomic_add(&f->cert->references, 1, &i, f->cert->lock);
     ssl_cert_free(t->cert);
     t->cert = f->cert;
     if (!SSL_set_session_id_context(t, f->sid_ctx, f->sid_ctx_length)) {
@@ -2369,6 +2380,12 @@ SSL_CTX *SSL_CTX_new(const SSL_METHOD *meth)
     /* We take the system default. */
     ret->session_timeout = meth->get_timeout();
     ret->references = 1;
+    ret->lock = CRYPTO_THREAD_lock_new();
+    if (ret->lock == NULL) {
+        SSLerr(SSL_F_SSL_CTX_NEW, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(ret);
+        return NULL;
+    }
     ret->max_cert_list = SSL_MAX_CERT_LIST_DEFAULT;
     ret->verify_mode = SSL_VERIFY_NONE;
     if ((ret->cert = ssl_cert_new()) == NULL)
@@ -2459,17 +2476,18 @@ SSL_CTX *SSL_CTX_new(const SSL_METHOD *meth)
      */
     ret->options |= SSL_OP_NO_COMPRESSION;
 
-    return (ret);
+    return ret;
  err:
     SSLerr(SSL_F_SSL_CTX_NEW, ERR_R_MALLOC_FAILURE);
  err2:
     SSL_CTX_free(ret);
-    return (NULL);
+    return NULL;
 }
 
 void SSL_CTX_up_ref(SSL_CTX *ctx)
 {
-    CRYPTO_add(&ctx->references, 1, CRYPTO_LOCK_SSL_CTX);
+    int i;
+    CRYPTO_atomic_add(&ctx->references, 1, &i, ctx->lock);
 }
 
 void SSL_CTX_free(SSL_CTX *a)
@@ -2479,7 +2497,7 @@ void SSL_CTX_free(SSL_CTX *a)
     if (a == NULL)
         return;
 
-    i = CRYPTO_add(&a->references, -1, CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_atomic_add(&a->references, -1, &i, a->lock);
     REF_PRINT_COUNT("SSL_CTX", a);
     if (i > 0)
         return;
@@ -2528,6 +2546,8 @@ void SSL_CTX_free(SSL_CTX *a)
 #endif
     OPENSSL_free(a->alpn_client_proto_list);
 
+    CRYPTO_THREAD_lock_free(a->lock);
+
     OPENSSL_free(a);
 }
 
@@ -2833,7 +2853,7 @@ void ssl_update_cache(SSL *s, int mode)
         && ((i & SSL_SESS_CACHE_NO_INTERNAL_STORE)
             || SSL_CTX_add_session(s->session_ctx, s->session))
         && (s->session_ctx->new_session_cb != NULL)) {
-        CRYPTO_add(&s->session->references, 1, CRYPTO_LOCK_SSL_SESSION);
+        SSL_SESSION_up_ref(s->session);
         if (!s->session_ctx->new_session_cb(s, s->session))
             SSL_SESSION_free(s->session);
     }
@@ -3069,7 +3089,7 @@ SSL *SSL_dup(SSL *s)
 
     /* If we're not quiescent, just up_ref! */
     if (!SSL_in_init(s) || !SSL_in_before(s)) {
-        CRYPTO_add(&s->references, 1, CRYPTO_LOCK_SSL);
+        CRYPTO_atomic_add(&s->references, 1, &i, s->lock);
         return s;
     }
 
@@ -3382,11 +3402,11 @@ SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx)
         memcpy(&ssl->sid_ctx, &ctx->sid_ctx, sizeof(ssl->sid_ctx));
     }
 
-    CRYPTO_add(&ctx->references, 1, CRYPTO_LOCK_SSL_CTX);
+    SSL_CTX_up_ref(ctx);
     SSL_CTX_free(ssl->ctx); /* decrement reference count */
     ssl->ctx = ctx;
 
-    return (ssl->ctx);
+    return ssl->ctx;
 }
 
 int SSL_CTX_set_default_verify_paths(SSL_CTX *ctx)
diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index 4d816de18d..b9beca536b 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -649,6 +649,7 @@ struct ssl_session_st {
     char *srp_username;
 # endif
     uint32_t flags;
+    CRYPTO_RWLOCK *lock;
 };
 
 /* Extended master secret support */
@@ -953,6 +954,7 @@ struct ssl_ctx_st {
     size_t tlsext_ellipticcurvelist_length;
     unsigned char *tlsext_ellipticcurvelist;
 #  endif                        /* OPENSSL_NO_EC */
+    CRYPTO_RWLOCK *lock;
 };
 
 
@@ -1234,6 +1236,8 @@ struct ssl_st {
     /* Async Job info */
     ASYNC_JOB *job;
     ASYNC_WAIT_CTX *waitctx;
+
+    CRYPTO_RWLOCK *lock;
 };
 
 
@@ -1642,6 +1646,7 @@ typedef struct cert_st {
     char *psk_identity_hint;
 #endif
     int references;             /* >1 only if SSL_copy_session_id is used */
+    CRYPTO_RWLOCK *lock;
 } CERT;
 
 /* Structure containing decoded values of signature algorithms extension */
diff --git a/ssl/ssl_sess.c b/ssl/ssl_sess.c
index 97f3ce2480..ebeeba011c 100644
--- a/ssl/ssl_sess.c
+++ b/ssl/ssl_sess.c
@@ -161,12 +161,12 @@ SSL_SESSION *SSL_get1_session(SSL *ssl)
      * somebody doesn't free ssl->session between when we check it's non-null
      * and when we up the reference count.
      */
-    CRYPTO_w_lock(CRYPTO_LOCK_SSL_SESSION);
+    CRYPTO_THREAD_read_lock(ssl->lock);
     sess = ssl->session;
     if (sess)
-        sess->references++;
-    CRYPTO_w_unlock(CRYPTO_LOCK_SSL_SESSION);
-    return (sess);
+        SSL_SESSION_up_ref(sess);
+    CRYPTO_THREAD_unlock(ssl->lock);
+    return sess;
 }
 
 int SSL_SESSION_set_ex_data(SSL_SESSION *s, int idx, void *arg)
@@ -186,15 +186,23 @@ SSL_SESSION *SSL_SESSION_new(void)
     ss = OPENSSL_zalloc(sizeof(*ss));
     if (ss == NULL) {
         SSLerr(SSL_F_SSL_SESSION_NEW, ERR_R_MALLOC_FAILURE);
-        return (NULL);
+        return NULL;
     }
 
     ss->verify_result = 1;      /* avoid 0 (= X509_V_OK) just in case */
     ss->references = 1;
     ss->timeout = 60 * 5 + 4;   /* 5 minute timeout by default */
     ss->time = (unsigned long)time(NULL);
+    ss->lock = CRYPTO_THREAD_lock_new();
+    if (ss->lock == NULL) {
+        SSLerr(SSL_F_SSL_SESSION_NEW, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(ss);
+        return NULL;
+    }
+
     CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, ss, &ss->ex_data);
-    return (ss);
+
+    return ss;
 }
 
 /*
@@ -237,6 +245,10 @@ SSL_SESSION *ssl_session_dup(SSL_SESSION *src, int ticket)
 
     dest->references = 1;
 
+    dest->lock = CRYPTO_THREAD_lock_new();
+    if (dest->lock == NULL)
+        goto err;
+
     if (src->peer != NULL)
         X509_up_ref(src->peer);
 
@@ -437,12 +449,14 @@ int ssl_get_new_session(SSL *s, int session)
         }
 
         /* Choose which callback will set the session ID */
-        CRYPTO_r_lock(CRYPTO_LOCK_SSL_CTX);
+        CRYPTO_THREAD_read_lock(s->lock);
+        CRYPTO_THREAD_read_lock(s->session_ctx->lock);
         if (s->generate_session_id)
             cb = s->generate_session_id;
         else if (s->session_ctx->generate_session_id)
             cb = s->session_ctx->generate_session_id;
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL_CTX);
+        CRYPTO_THREAD_unlock(s->session_ctx->lock);
+        CRYPTO_THREAD_unlock(s->lock);
         /* Choose a session ID */
         tmp = ss->session_id_length;
         if (!cb(s, ss->session_id, &tmp)) {
@@ -562,13 +576,13 @@ int ssl_get_prev_session(SSL *s, const PACKET *ext, const PACKET *session_id)
             goto err;
         }
         data.session_id_length = local_len;
-        CRYPTO_r_lock(CRYPTO_LOCK_SSL_CTX);
+        CRYPTO_THREAD_read_lock(s->session_ctx->lock);
         ret = lh_SSL_SESSION_retrieve(s->session_ctx->sessions, &data);
         if (ret != NULL) {
             /* don't allow other threads to steal it: */
-            CRYPTO_add(&ret->references, 1, CRYPTO_LOCK_SSL_SESSION);
+            SSL_SESSION_up_ref(ret);
         }
-        CRYPTO_r_unlock(CRYPTO_LOCK_SSL_CTX);
+        CRYPTO_THREAD_unlock(s->session_ctx->lock);
         if (ret == NULL)
             s->session_ctx->stats.sess_miss++;
     }
@@ -591,7 +605,7 @@ int ssl_get_prev_session(SSL *s, const PACKET *ext, const PACKET *session_id)
              * thread-safe).
              */
             if (copy)
-                CRYPTO_add(&ret->references, 1, CRYPTO_LOCK_SSL_SESSION);
+                SSL_SESSION_up_ref(ret);
 
             /*
              * Add the externally cached session to the internal cache as
@@ -714,12 +728,12 @@ int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *c)
      * it has two ways of access: each session is in a doubly linked list and
      * an lhash
      */
-    CRYPTO_add(&c->references, 1, CRYPTO_LOCK_SSL_SESSION);
+    SSL_SESSION_up_ref(c);
     /*
      * if session c is in already in cache, we take back the increment later
      */
 
-    CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_write_lock(ctx->lock);
     s = lh_SSL_SESSION_insert(ctx->sessions, c);
 
     /*
@@ -769,8 +783,8 @@ int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *c)
             }
         }
     }
-    CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
-    return (ret);
+    CRYPTO_THREAD_unlock(ctx->lock);
+    return ret;
 }
 
 int SSL_CTX_remove_session(SSL_CTX *ctx, SSL_SESSION *c)
@@ -785,7 +799,7 @@ static int remove_session_lock(SSL_CTX *ctx, SSL_SESSION *c, int lck)
 
     if ((c != NULL) && (c->session_id_length != 0)) {
         if (lck)
-            CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+            CRYPTO_THREAD_write_lock(ctx->lock);
         if ((r = lh_SSL_SESSION_retrieve(ctx->sessions, c)) == c) {
             ret = 1;
             r = lh_SSL_SESSION_delete(ctx->sessions, c);
@@ -793,7 +807,7 @@ static int remove_session_lock(SSL_CTX *ctx, SSL_SESSION *c, int lck)
         }
 
         if (lck)
-            CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
+            CRYPTO_THREAD_unlock(ctx->lock);
 
         if (ret) {
             r->not_resumable = 1;
@@ -813,7 +827,7 @@ void SSL_SESSION_free(SSL_SESSION *ss)
     if (ss == NULL)
         return;
 
-    i = CRYPTO_add(&ss->references, -1, CRYPTO_LOCK_SSL_SESSION);
+    CRYPTO_atomic_add(&ss->references, -1, &i, ss->lock);
     REF_PRINT_COUNT("SSL_SESSION", ss);
     if (i > 0)
         return;
@@ -841,9 +855,22 @@ void SSL_SESSION_free(SSL_SESSION *ss)
 #ifndef OPENSSL_NO_SRP
     OPENSSL_free(ss->srp_username);
 #endif
+    CRYPTO_THREAD_lock_free(ss->lock);
     OPENSSL_clear_free(ss, sizeof(*ss));
 }
 
+int SSL_SESSION_up_ref(SSL_SESSION *ss)
+{
+    int i;
+
+    if (CRYPTO_atomic_add(&ss->references, 1, &i, ss->lock) <= 0)
+        return 0;
+
+    REF_PRINT_COUNT("SSL_SESSION", ss);
+    REF_ASSERT_ISNT(i < 2);
+    return ((i > 1) ? 1 : 0);
+}
+
 int SSL_set_session(SSL *s, SSL_SESSION *session)
 {
     int ret = 0;
@@ -863,12 +890,10 @@ int SSL_set_session(SSL *s, SSL_SESSION *session)
                 return (0);
         }
 
-        /* CRYPTO_w_lock(CRYPTO_LOCK_SSL); */
-        CRYPTO_add(&session->references, 1, CRYPTO_LOCK_SSL_SESSION);
+        SSL_SESSION_up_ref(session);
         SSL_SESSION_free(s->session);
         s->session = session;
         s->verify_result = s->session->verify_result;
-        /* CRYPTO_w_unlock(CRYPTO_LOCK_SSL); */
         ret = 1;
     } else {
         SSL_SESSION_free(s->session);
@@ -1056,12 +1081,12 @@ void SSL_CTX_flush_sessions(SSL_CTX *s, long t)
     if (tp.cache == NULL)
         return;
     tp.time = t;
-    CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_write_lock(s->lock);
     i = CHECKED_LHASH_OF(SSL_SESSION, tp.cache)->down_load;
     CHECKED_LHASH_OF(SSL_SESSION, tp.cache)->down_load = 0;
     lh_SSL_SESSION_doall_TIMEOUT_PARAM(tp.cache, timeout_cb, &tp);
     CHECKED_LHASH_OF(SSL_SESSION, tp.cache)->down_load = i;
-    CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
+    CRYPTO_THREAD_unlock(s->lock);
 }
 
 int ssl_clear_bad_session(SSL *s)
diff --git a/test/ssltest.c b/test/ssltest.c
index 2ca4d69558..94aeebe9c0 100644
--- a/test/ssltest.c
+++ b/test/ssltest.c
@@ -191,6 +191,7 @@
 # include <openssl/ct.h>
 #endif
 
+#include "internal/threads.h"
 #include "../ssl/ssl_locl.h"
 
 /*
@@ -2949,19 +2950,21 @@ int doit(SSL *s_ssl, SSL *c_ssl, long count)
     return (ret);
 }
 
+static CRYPTO_ONCE proxy_auth_ex_data_once = CRYPTO_ONCE_STATIC_INIT;
+static volatile int proxy_auth_ex_data_idx = -1;
+
+static void do_get_proxy_auth_ex_data_idx(void)
+{
+    proxy_auth_ex_data_idx = X509_STORE_CTX_get_ex_new_index(0,
+                                                "SSLtest for verify callback",
+                                                NULL, NULL, NULL);
+}
+
 static int get_proxy_auth_ex_data_idx(void)
 {
-    static volatile int idx = -1;
-    if (idx < 0) {
-        CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
-        if (idx < 0) {
-            idx = X509_STORE_CTX_get_ex_new_index(0,
-                                                  "SSLtest for verify callback",
-                                                  NULL, NULL, NULL);
-        }
-        CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
-    }
-    return idx;
+    CRYPTO_THREAD_run_once(&proxy_auth_ex_data_once,
+                           do_get_proxy_auth_ex_data_idx);
+    return proxy_auth_ex_data_idx;
 }
 
 static int verify_callback(int ok, X509_STORE_CTX *ctx)
diff --git a/util/libssl.num b/util/libssl.num
index 7ec5525aa6..49c9c35484 100644
--- a/util/libssl.num
+++ b/util/libssl.num
@@ -387,3 +387,4 @@ SSL_has_pending                         386	1_1_0	EXIST::FUNCTION:
 SSL_CIPHER_get_auth_nid                 387	1_1_0	EXIST::FUNCTION:
 SSL_CIPHER_get_kx_nid                   388	1_1_0	EXIST::FUNCTION:
 SSL_CIPHER_is_aead                      389	1_1_0	EXIST::FUNCTION:
+SSL_SESSION_up_ref                      390	1_1_0	EXIST::FUNCTION: