Misc BN fixes

Never output -0; make "negative zero" an impossibility.
Do better checking on BN_rand top/bottom requirements and #bits.
Update doc.
Ignoring trailing garbage in BN_asc2bn.

Port this commit from boringSSL: https://boringssl.googlesource.com/boringssl/+/899b9b19a4cd3fe526aaf5047ab9234cdca19f7d%5E!/
        Ensure |BN_div| never gives negative zero in the no_branch code.

        Have |bn_correct_top| fix |bn->neg| if the input is zero so that we
        don't have negative zeros lying around.

        Thanks to Brian Smith for noticing.

Reviewed-by: Richard Levitte <levitte@openssl.org>

crypto/bn/bn_lib.c

@@ -1031,5 +1031,7 @@ void bn_correct_top(BIGNUM *a)
         }
         a->top = tmp_top;
     }
+    if (a->top == 0)
+        a->neg = 0;
     bn_pollute(a);
 }

crypto/bn/bn_print.c

@@ -23,12 +23,9 @@ char *BN_bn2hex(const BIGNUM *a)
     char *buf;
     char *p;
 
-    if (a->neg && BN_is_zero(a)) {
-        /* "-0" == 3 bytes including NULL terminator */
-        buf = OPENSSL_malloc(3);
-    } else {
-        buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
-    }
+    if (BN_is_zero(a))
+        return OPENSSL_strdup("0");
+    buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
     if (buf == NULL) {
         BNerr(BN_F_BN_BN2HEX, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -186,10 +183,12 @@ int BN_hex2bn(BIGNUM **bn, const char *a)
     }
     ret->top = h;
     bn_correct_top(ret);
-    ret->neg = neg;
 
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -241,7 +240,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
     if (j == BN_DEC_NUM)
         j = 0;
     l = 0;
-    while (*a) {
+    while (--i >= 0) {
         l *= 10;
         l += *a - '0';
         a++;
@@ -253,11 +252,13 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
             j = 0;
         }
     }
-    ret->neg = neg;
 
     bn_correct_top(ret);
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -268,6 +269,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
 int BN_asc2bn(BIGNUM **bn, const char *a)
 {
     const char *p = a;
+
     if (*p == '-')
         p++;
 
@@ -278,7 +280,8 @@ int BN_asc2bn(BIGNUM **bn, const char *a)
         if (!BN_dec2bn(bn, p))
             return 0;
     }
-    if (*a == '-')
+    /* Don't set the negative flag if it's zero. */
+    if (*a == '-' && (*bn)->top != 0)
         (*bn)->neg = 1;
     return 1;
 }

crypto/bn/bn_rand.c

@@ -20,15 +20,14 @@ static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
     int ret = 0, bit, bytes, mask;
     time_t tim;
 
-    if (bits < 0 || (bits == 1 && top > 0)) {
-        BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
-        return 0;
-    }
-
     if (bits == 0) {
+        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
+            goto toosmall;
         BN_zero(rnd);
         return 1;
     }
+    if (bits < 0 || (bits == 1 && top > 0))
+        goto toosmall;
 
     bytes = (bits + 7) / 8;
     bit = (bits - 1) % 8;
@@ -88,6 +87,10 @@ static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
     OPENSSL_clear_free(buf, bytes);
     bn_check_top(rnd);
     return (ret);
+
+toosmall:
+    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
+    return 0;
 }
 
 int BN_rand(BIGNUM *rnd, int bits, int top, int bottom)

doc/crypto/BN_bn2bin.pod

@@ -57,6 +57,7 @@ including the leading character '-' which means negative, to form a valid
 hexadecimal number representation and converts them to a B<BIGNUM> and
 stores it in **B<bn>. If *B<bn> is NULL, a new B<BIGNUM> is created. If
 B<bn> is NULL, it only computes the length of valid representation.
+A "negative zero" is converted to zero.
 BN_dec2bn() is the same using the decimal system.
 
 BN_print() and BN_print_fp() write the hexadecimal encoding of B<a>,

doc/crypto/BN_rand.pod

@@ -20,7 +20,9 @@ BN_rand, BN_pseudo_rand, BN_rand_range, BN_pseudo_rand_range - generate pseudo-r
 
 BN_rand() generates a cryptographically strong pseudo-random number of
 B<bits> in length and stores it in B<rnd>.
-The value of B<bits> must be zero or greater.
+If B<bits> is less than zero, or too small to
+accomodate the requirements specified by the B<top> and B<bottom>
+parameters, an error is returned.
 The B<top> parameters specifies
 requirements on the most significant bit of the generated number.
 If it is B<BN_RAND_TOP_ANY>, there is no constraint.