Allow RSA key-generation to specify an arbitrary public exponent. Jelte

proposed the change and submitted the patch, I jiggled it slightly and
adjusted the other parts of openssl that were affected.

PR: 867
Submitted by: Jelte Jansen
Reviewed by: Geoff Thorpe

3 files changed, 28 insertions, 23 deletions

diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index f82d493f95..5bf862cd68 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -120,7 +120,7 @@ struct rsa_meth_st
  * is for behavioural compatibility whilst the code gets rewired, but one day
  * it would be nice to assume there are no such things as "builtin software"
  * implementations. */
-	int (*rsa_keygen)(RSA *rsa, int bits, unsigned long e, BN_GENCB *cb);
+	int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
 	};
 
 struct rsa_st
@@ -203,7 +203,7 @@ RSA *	RSA_generate_key(int bits, unsigned long e,void
 #endif /* !defined(OPENSSL_NO_DEPRECATED) */
 
 /* New version */
-int	RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e, BN_GENCB *cb);
+int	RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
 
 int	RSA_check_key(const RSA *);
 	/* next 4 return -1 on error */
diff --git a/crypto/rsa/rsa_depr.c b/crypto/rsa/rsa_depr.c
index 2d87cd39f3..73fa6eea90 100644
--- a/crypto/rsa/rsa_depr.c
+++ b/crypto/rsa/rsa_depr.c
@@ -62,23 +62,39 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
+#ifdef OPENSSL_NO_DEPRECATED
+
 static void *dummy=&dummy;
 
-#ifndef OPENSSL_NO_DEPRECATED
+#else
+
 RSA *RSA_generate_key(int bits, unsigned long e_value,
 	     void (*callback)(int,int,void *), void *cb_arg)
 	{
 	BN_GENCB cb;
-	RSA *rsa;
+	int i;
+	RSA *rsa = RSA_new();
+	BIGNUM *e = BN_new();
 
-	if((rsa=RSA_new()) == NULL)
-		return 0;
+	if(!rsa || !e) goto err;
+
+	/* The problem is when building with 8, 16, or 32 BN_ULONG,
+	 * unsigned long can be larger */
+	for (i=0; i<sizeof(unsigned long)*8; i++)
+		{
+		if (e_value & (1UL<<i))
+			BN_set_bit(e,i);
+		}
 
 	BN_GENCB_set_old(&cb, callback, cb_arg);
 
-	if(RSA_generate_key_ex(rsa, bits, e_value, &cb))
+	if(RSA_generate_key_ex(rsa, bits, e, &cb)) {
+		BN_free(e);
 		return rsa;
-	RSA_free(rsa);
+	}
+err:
+	if(e) BN_free(e);
+	if(rsa) RSA_free(rsa);
 	return 0;
 	}
 #endif
diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 024e11b8e7..68a2661796 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -68,25 +68,24 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb);
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb);
 
 /* NB: this wrapper would normally be placed in rsa_lib.c and the static
  * implementation would probably be in rsa_eay.c. Nonetheless, is kept here so
  * that we don't introduce a new linker dependency. Eg. any application that
  * wasn't previously linking object code related to key-generation won't have to
  * now just because key-generation is part of RSA_METHOD. */
-int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
+int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 	{
 	if(rsa->meth->rsa_keygen)
 		return rsa->meth->rsa_keygen(rsa, bits, e_value, cb);
 	return rsa_builtin_keygen(rsa, bits, e_value, cb);
 	}
 
-static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 	{
 	BIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;
 	int bitsp,bitsq,ok= -1,n=0;
-	unsigned int i;
 	BN_CTX *ctx=NULL,*ctx2=NULL;
 
 	ctx=BN_CTX_new();
@@ -113,17 +112,7 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENC
 	if(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;
 	if(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;
 
-#if 1
-	/* The problem is when building with 8, 16, or 32 BN_ULONG,
-	 * unsigned long can be larger */
-	for (i=0; i<sizeof(unsigned long)*8; i++)
-		{
-		if (e_value & (1UL<<i))
-			BN_set_bit(rsa->e,i);
-		}
-#else
-	if (!BN_set_word(rsa->e,e_value)) goto err;
-#endif
+	BN_copy(rsa->e, e_value);
 
 	/* generate p and q */
 	for (;;)