bn/bn_exp.c: harmonize BN_mod_exp_mont_consttime with negative input.

All exponentiation subroutines but BN_mod_exp_mont_consttime produce
non-negative result for negative input, which is confusing for fuzzer.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
(Merged from https://github.com/openssl/openssl/pull/4676)
This commit is contained in:
Andy Polyakov 2017-11-05 20:35:47 +01:00
parent 899e62d186
commit c1ec4db33d

View file

@ -651,31 +651,33 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
}
#ifdef RSAZ_ENABLED
/*
* If the size of the operands allow it, perform the optimized
* RSAZ exponentiation. For further information see
* crypto/bn/rsaz_exp.c and accompanying assembly modules.
*/
if ((16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024)
&& rsaz_avx2_eligible()) {
if (NULL == bn_wexpand(rr, 16))
if (!a->neg) {
/*
* If the size of the operands allow it, perform the optimized
* RSAZ exponentiation. For further information see
* crypto/bn/rsaz_exp.c and accompanying assembly modules.
*/
if ((16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024)
&& rsaz_avx2_eligible()) {
if (NULL == bn_wexpand(rr, 16))
goto err;
RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d,
mont->n0[0]);
rr->top = 16;
rr->neg = 0;
bn_correct_top(rr);
ret = 1;
goto err;
RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d,
mont->n0[0]);
rr->top = 16;
rr->neg = 0;
bn_correct_top(rr);
ret = 1;
goto err;
} else if ((8 == a->top) && (8 == p->top) && (BN_num_bits(m) == 512)) {
if (NULL == bn_wexpand(rr, 8))
} else if ((8 == a->top) && (8 == p->top) && (BN_num_bits(m) == 512)) {
if (NULL == bn_wexpand(rr, 8))
goto err;
RSAZ_512_mod_exp(rr->d, a->d, p->d, m->d, mont->n0[0], mont->RR.d);
rr->top = 8;
rr->neg = 0;
bn_correct_top(rr);
ret = 1;
goto err;
RSAZ_512_mod_exp(rr->d, a->d, p->d, m->d, mont->n0[0], mont->RR.d);
rr->top = 8;
rr->neg = 0;
bn_correct_top(rr);
ret = 1;
goto err;
}
}
#endif
@ -748,7 +750,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
/* prepare a^1 in Montgomery domain */
if (a->neg || BN_ucmp(a, m) >= 0) {
if (!BN_mod(&am, a, m, ctx))
if (!BN_nnmod(&am, a, m, ctx))
goto err;
if (!BN_to_montgomery(&am, &am, mont, ctx))
goto err;
@ -1258,7 +1260,7 @@ int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
}
bits = BN_num_bits(p);
if (bits == 0) {
if (bits == 0) {
/* x**0 mod 1 is still zero. */
if (BN_is_one(m)) {
ret = 1;