diff options
author | Rich Salz <rsalz@akamai.com> | 2015-09-04 08:13:19 -0400 |
---|---|---|
committer | Rich Salz <rsalz@openssl.org> | 2015-09-05 17:40:48 -0400 |
commit | 8e704858f21983383be2b77e986f475b51719a1e (patch) | |
tree | 77661553cd59016b1b6b7d2e3857a6f3fc97a7bb | |
parent | ecdaa1aefd30a3624624a28139a1e78e17993725 (diff) | |
download | openssl-8e704858f21983383be2b77e986f475b51719a1e.tar.gz |
RT3955: Reduce some stack usage
Use malloc/free instead of big onstack buffers.
Reviewed-by: Tim Hudson <tjh@openssl.org>
-rw-r--r-- | crypto/bn/bn_prime.c | 14 | ||||
-rw-r--r-- | crypto/pkcs7/pk7_smime.c | 54 |
2 files changed, 43 insertions, 25 deletions
diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c index c83820cc25..42d574bb2e 100644 --- a/crypto/bn/bn_prime.c +++ b/crypto/bn/bn_prime.c @@ -131,7 +131,7 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1, const BIGNUM *a1_odd, int k, BN_CTX *ctx, BN_MONT_CTX *mont); -static int probable_prime(BIGNUM *rnd, int bits); +static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods); static int probable_prime_dh_safe(BIGNUM *rnd, int bits, const BIGNUM *add, const BIGNUM *rem, BN_CTX *ctx); @@ -211,9 +211,13 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, BIGNUM *t; int found = 0; int i, j, c1 = 0; - BN_CTX *ctx; + BN_CTX *ctx = NULL; + prime_t *mods = NULL; int checks = BN_prime_checks_for_size(bits); + mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES); + if (mods == NULL) + goto err; if (bits < 2) { /* There are no prime numbers this small. */ BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL); @@ -234,7 +238,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, loop: /* make a random number and set the top and bottom bits */ if (add == NULL) { - if (!probable_prime(ret, bits)) + if (!probable_prime(ret, bits, mods)) goto err; } else { if (safe) { @@ -285,6 +289,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, /* we have a prime :-) */ found = 1; err: + OPENSSL_free(mods); if (ctx != NULL) BN_CTX_end(ctx); BN_CTX_free(ctx); @@ -497,10 +502,9 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1, return 1; } -static int probable_prime(BIGNUM *rnd, int bits) +static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods) { int i; - prime_t mods[NUMPRIMES]; BN_ULONG delta; BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1]; char is_single_word = bits <= BN_BITS2; diff --git a/crypto/pkcs7/pk7_smime.c b/crypto/pkcs7/pk7_smime.c index 91557af5c8..6522a51650 100644 --- a/crypto/pkcs7/pk7_smime.c +++ b/crypto/pkcs7/pk7_smime.c @@ -64,6 +64,9 @@ #include <openssl/x509.h> #include <openssl/x509v3.h> + +#define BUFFERSIZE 4096 + static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si); PKCS7 *PKCS7_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, @@ -113,6 +116,7 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags) { BIO *p7bio; int ret = 0; + if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) { PKCS7err(PKCS7_F_PKCS7_FINAL, ERR_R_MALLOC_FAILURE); return 0; @@ -253,7 +257,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, STACK_OF(PKCS7_SIGNER_INFO) *sinfos; PKCS7_SIGNER_INFO *si; X509_STORE_CTX cert_ctx; - char buf[4096]; + char *buf = NULL; int i, j = 0, k, ret = 0; BIO *p7bio = NULL; BIO *tmpin = NULL, *tmpout = NULL; @@ -355,8 +359,12 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, tmpout = out; /* We now have to 'read' from p7bio to calculate digests etc. */ + if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) { + PKCS7err(PKCS7_F_PKCS7_VERIFY, ERR_R_MALLOC_FAILURE); + goto err; + } for (;;) { - i = BIO_read(p7bio, buf, sizeof(buf)); + i = BIO_read(p7bio, buf, BUFFERSIZE); if (i <= 0) break; if (tmpout) @@ -387,6 +395,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, ret = 1; err: + OPENSSL_free(buf); if (tmpin == indata) { if (indata) BIO_pop(p7bio); @@ -505,7 +514,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) { BIO *tmpmem; int ret, i; - char buf[4096]; + char *buf = NULL; if (!p7) { PKCS7err(PKCS7_F_PKCS7_DECRYPT, PKCS7_R_INVALID_NULL_POINTER); @@ -549,24 +558,29 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) } BIO_free_all(bread); return ret; - } else { - for (;;) { - i = BIO_read(tmpmem, buf, sizeof(buf)); - if (i <= 0) { - ret = 1; - if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) { - if (!BIO_get_cipher_status(tmpmem)) - ret = 0; - } - - break; - } - if (BIO_write(data, buf, i) != i) { - ret = 0; - break; + } + if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) { + PKCS7err(PKCS7_F_PKCS7_DECRYPT, ERR_R_MALLOC_FAILURE); + goto err; + } + for (;;) { + i = BIO_read(tmpmem, buf, BUFFERSIZE); + if (i <= 0) { + ret = 1; + if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) { + if (!BIO_get_cipher_status(tmpmem)) + ret = 0; } + + break; + } + if (BIO_write(data, buf, i) != i) { + ret = 0; + break; } - BIO_free_all(tmpmem); - return ret; } +err: + OPENSSL_free(buf); + BIO_free_all(tmpmem); + return ret; } |