Fix some things that look like bugs.

One problem that looked like a problem in bn_recp.c at first turned
out to be a BN_mul bug.  An example is given in bn_recp.c; finding
the bug responsible for this is left as an exercise.

1 files changed, 25 insertions, 19 deletions

diff --git a/crypto/bn/bn_mont.c b/crypto/bn/bn_mont.c
index 7f8296c893..f1765c03ac 100644
--- a/crypto/bn/bn_mont.c
+++ b/crypto/bn/bn_mont.c
@@ -226,7 +226,7 @@ int BN_from_montgomery(BIGNUM *ret, const BIGNUM *a, BN_MONT_CTX *mont,
 
 	if (BN_ucmp(ret, &(mont->N)) >= 0)
 		{
-		BN_usub(ret,ret,&(mont->N));
+		if (!BN_usub(ret,ret,&(mont->N))) goto err;
 		}
 	retn=1;
  err:
@@ -274,6 +274,7 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
 	BN_init(&Ri);
 	R= &(mont->RR);					/* grab RR as a temp */
 	BN_copy(&(mont->N),mod);			/* Set N */
+	mont->N.neg = 0;
 
 #ifdef MONT_WORD
 		{
@@ -289,40 +290,45 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
 		tmod.d=buf;
 		tmod.top=1;
 		tmod.dmax=2;
-		tmod.neg=mod->neg;
+		tmod.neg=0;
 							/* Ri = R^-1 mod N*/
 		if ((BN_mod_inverse(&Ri,R,&tmod,ctx)) == NULL)
 			goto err;
-		BN_lshift(&Ri,&Ri,BN_BITS2);		/* R*Ri */
+		if (!BN_lshift(&Ri,&Ri,BN_BITS2)) goto err; /* R*Ri */
 		if (!BN_is_zero(&Ri))
-			BN_sub_word(&Ri,1);
+			{
+			if (!BN_sub_word(&Ri,1)) goto err;
+			}
 		else /* if N mod word size == 1 */
-			BN_set_word(&Ri,BN_MASK2);  /* Ri-- (mod word size) */
-		BN_div(&Ri,NULL,&Ri,&tmod,ctx);	/* Ni = (R*Ri-1)/N,
-		                                 * keep only least significant word: */
-		mont->n0=Ri.d[0];
+			{
+			if (!BN_set_word(&Ri,BN_MASK2)) goto err;  /* Ri-- (mod word size) */
+			}
+		if (!BN_div(&Ri,NULL,&Ri,&tmod,ctx)) goto err;
+		/* Ni = (R*Ri-1)/N,
+		 * keep only least significant word: */
+		mont->n0 = (Ri.top > 0) ? Ri.d[0] : 0;
 		BN_free(&Ri);
 		}
 #else /* !MONT_WORD */
 		{ /* bignum version */
-		mont->ri=BN_num_bits(mod);
-		BN_zero(R);
-		BN_set_bit(R,mont->ri);			/* R = 2^ri */
-							/* Ri = R^-1 mod N*/
-		if ((BN_mod_inverse(&Ri,R,mod,ctx)) == NULL)
+		mont->ri=BN_num_bits(&mont->N);
+		if (!BN_zero(R)) goto err;
+		if (!BN_set_bit(R,mont->ri)) goto err;  /* R = 2^ri */
+		                                        /* Ri = R^-1 mod N*/
+		if ((BN_mod_inverse(&Ri,R,&mont->N,ctx)) == NULL)
 			goto err;
-		BN_lshift(&Ri,&Ri,mont->ri);		/* R*Ri */
-		BN_sub_word(&Ri,1);
+		if (!BN_lshift(&Ri,&Ri,mont->ri)) goto err; /* R*Ri */
+		if (!BN_sub_word(&Ri,1)) goto err;
 							/* Ni = (R*Ri-1) / N */
-		BN_div(&(mont->Ni),NULL,&Ri,mod,ctx);
+		if (!BN_div(&(mont->Ni),NULL,&Ri,&mont->N,ctx)) goto err;
 		BN_free(&Ri);
 		}
 #endif
 
 	/* setup RR for conversions */
-	BN_zero(&(mont->RR));
-	BN_set_bit(&(mont->RR),mont->ri*2);
-	BN_mod(&(mont->RR),&(mont->RR),&(mont->N),ctx);
+	if (!BN_zero(&(mont->RR))) goto err;
+	if (!BN_set_bit(&(mont->RR),mont->ri*2)) goto err;
+	if (!BN_mod(&(mont->RR),&(mont->RR),&(mont->N),ctx)) goto err;
 
 	return(1);
 err: