Allow RSA key-generation to specify an arbitrary public exponent. Jelte

proposed the change and submitted the patch, I jiggled it slightly and
adjusted the other parts of openssl that were affected.

PR: 867
Submitted by: Jelte Jansen
Reviewed by: Geoff Thorpe

3 files changed, 20 insertions, 10 deletions

diff --git a/apps/genrsa.c b/apps/genrsa.c
index 85da98d45d..f0bb30c56b 100644
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -92,7 +92,6 @@ int MAIN(int argc, char **argv)
 	ENGINE *e = NULL;
 #endif
 	int ret=1;
-	RSA *rsa=NULL;
 	int i,num=DEFBITS;
 	long l;
 	const EVP_CIPHER *enc=NULL;
@@ -104,6 +103,10 @@ int MAIN(int argc, char **argv)
 #endif
 	char *inrand=NULL;
 	BIO *out=NULL;
+	BIGNUM *bn = BN_new();
+	RSA *rsa = RSA_new();
+
+	if(!bn || !rsa) goto err;
 
 	apps_startup();
 	BN_GENCB_set(&cb, genrsa_cb, bio_err);
@@ -242,13 +245,11 @@ bad:
 	BIO_printf(bio_err,"Generating RSA private key, %d bit long modulus\n",
 		num);
 
-	if(((rsa = RSA_new()) == NULL) || !RSA_generate_key_ex(rsa, num, f4, &cb))
+	if(!BN_set_word(bn, f4) || !RSA_generate_key_ex(rsa, num, bn, &cb))
 		goto err;
 		
 	app_RAND_write_file(NULL, bio_err);
 
-	if (rsa == NULL) goto err;
-	
 	/* We need to do the following for when the base number size is <
 	 * long, esp windows 3.1 :-(. */
 	l=0L;
@@ -272,8 +273,9 @@ bad:
 
 	ret=0;
 err:
-	if (rsa != NULL) RSA_free(rsa);
-	if (out != NULL) BIO_free_all(out);
+	if (bn) BN_free(bn);
+	if (rsa) RSA_free(rsa);
+	if (out) BIO_free_all(out);
 	if(passout) OPENSSL_free(passout);
 	if (ret != 0)
 		ERR_print_errors(bio_err);
diff --git a/apps/req.c b/apps/req.c
index 16e27d1b38..0f3d496d47 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -748,12 +748,16 @@ bad:
 		if (pkey_type == TYPE_RSA)
 			{
 			RSA *rsa = RSA_new();
-			if(!rsa || !RSA_generate_key_ex(rsa, newkey, 0x10001, &cb) ||
+			BIGNUM *bn = BN_new();
+			if(!bn || !rsa || !BN_set_word(bn, 0x10001) ||
+					!RSA_generate_key_ex(rsa, newkey, bn, &cb) ||
 					!EVP_PKEY_assign_RSA(pkey, rsa))
 				{
+				if(bn) BN_free(bn);
 				if(rsa) RSA_free(rsa);
 				goto end;
 				}
+			BN_free(bn);
 			}
 		else
 #endif
diff --git a/apps/s_server.c b/apps/s_server.c
index c342a2ba66..cc2c10d10c 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -1782,17 +1782,20 @@ err:
 #ifndef OPENSSL_NO_RSA
 static RSA MS_CALLBACK *tmp_rsa_cb(SSL *s, int is_export, int keylength)
 	{
+	BIGNUM *bn = NULL;
 	static RSA *rsa_tmp=NULL;
 
-	if (rsa_tmp == NULL)
+	if (!rsa_tmp && ((bn = BN_new()) == NULL))
+		BIO_printf(bio_err,"Allocation error in generating RSA key\n");
+	if (!rsa_tmp && bn)
 		{
 		if (!s_quiet)
 			{
 			BIO_printf(bio_err,"Generating temp (%d bit) RSA key...",keylength);
 			(void)BIO_flush(bio_err);
 			}
-		if(((rsa_tmp = RSA_new()) == NULL) || !RSA_generate_key_ex(
-					rsa_tmp, keylength,RSA_F4,NULL))
+		if(!BN_set_word(bn, RSA_F4) || ((rsa_tmp = RSA_new()) == NULL) ||
+				!RSA_generate_key_ex(rsa_tmp, keylength, bn, NULL))
 			{
 			if(rsa_tmp) RSA_free(rsa_tmp);
 			rsa_tmp = NULL;
@@ -1802,6 +1805,7 @@ static RSA MS_CALLBACK *tmp_rsa_cb(SSL *s, int is_export, int keylength)
 			BIO_printf(bio_err,"\n");
 			(void)BIO_flush(bio_err);
 			}
+		BN_free(bn);
 		}
 	return(rsa_tmp);
 	}