wbrawner/openssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

RT3955: Reduce some stack usage

Browse source 
Use malloc/free instead of big onstack buffers.

Reviewed-by: Tim Hudson <tjh@openssl.org>
This commit is contained in:
Rich Salz 2015-09-04 08:13:19 -04:00 committed by Rich Salz
parent ecdaa1aefd
commit 8e704858f2
2 changed files with 45 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
crypto/bn/bn_prime.c
View file

@ -131,7 +131,7 @@
static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,
const BIGNUM *a1_odd, int k, BN_CTX *ctx,
BN_MONT_CTX *mont);
static int probable_prime(BIGNUM *rnd, int bits);
static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods);
static int probable_prime_dh_safe(BIGNUM *rnd, int bits,
const BIGNUM *add, const BIGNUM *rem,
BN_CTX *ctx);
@ -211,9 +211,13 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
BIGNUM *t;
int found = 0;
int i, j, c1 = 0;
BN_CTX *ctx;
BN_CTX *ctx = NULL;
prime_t *mods = NULL;
int checks = BN_prime_checks_for_size(bits);
mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);
if (mods == NULL)
goto err;
if (bits < 2) {
/* There are no prime numbers this small. */
BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
@ -234,7 +238,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
loop:
/* make a random number and set the top and bottom bits */
if (add == NULL) {
if (!probable_prime(ret, bits))
if (!probable_prime(ret, bits, mods))
goto err;
} else {
if (safe) {
@ -285,6 +289,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
/* we have a prime :-) */
found = 1;
err:
OPENSSL_free(mods);
if (ctx != NULL)
BN_CTX_end(ctx);
BN_CTX_free(ctx);
@ -497,10 +502,9 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,
return 1;
}
static int probable_prime(BIGNUM *rnd, int bits)
static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)
{
int i;
prime_t mods[NUMPRIMES];
BN_ULONG delta;
BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];
char is_single_word = bits <= BN_BITS2;

58
crypto/pkcs7/pk7_smime.c
View file

@ -64,6 +64,9 @@
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#define BUFFERSIZE 4096
static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si);
PKCS7 *PKCS7_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,
@ -113,6 +116,7 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags)
{
BIO *p7bio;
int ret = 0;
if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) {
PKCS7err(PKCS7_F_PKCS7_FINAL, ERR_R_MALLOC_FAILURE);
return 0;
@ -253,7 +257,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
STACK_OF(PKCS7_SIGNER_INFO) *sinfos;
PKCS7_SIGNER_INFO *si;
X509_STORE_CTX cert_ctx;
char buf[4096];
char *buf = NULL;
int i, j = 0, k, ret = 0;
BIO *p7bio = NULL;
BIO *tmpin = NULL, *tmpout = NULL;
@ -355,8 +359,12 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
tmpout = out;
/* We now have to 'read' from p7bio to calculate digests etc. */
if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
PKCS7err(PKCS7_F_PKCS7_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
for (;;) {
i = BIO_read(p7bio, buf, sizeof(buf));
i = BIO_read(p7bio, buf, BUFFERSIZE);
if (i <= 0)
break;
if (tmpout)
@ -387,6 +395,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
ret = 1;
err:
OPENSSL_free(buf);
if (tmpin == indata) {
if (indata)
BIO_pop(p7bio);
@ -505,7 +514,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
{
BIO *tmpmem;
int ret, i;
char buf[4096];
char *buf = NULL;
if (!p7) {
PKCS7err(PKCS7_F_PKCS7_DECRYPT, PKCS7_R_INVALID_NULL_POINTER);
@ -549,24 +558,29 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
}
BIO_free_all(bread);
return ret;
} else {
for (;;) {
i = BIO_read(tmpmem, buf, sizeof(buf));
if (i <= 0) {
ret = 1;
if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {
if (!BIO_get_cipher_status(tmpmem))
ret = 0;
}
break;
}
if (BIO_write(data, buf, i) != i) {
ret = 0;
break;
}
}
BIO_free_all(tmpmem);
return ret;
}
if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
PKCS7err(PKCS7_F_PKCS7_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
for (;;) {
i = BIO_read(tmpmem, buf, BUFFERSIZE);
if (i <= 0) {
ret = 1;
if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {
if (!BIO_get_cipher_status(tmpmem))
ret = 0;
}
break;
}
if (BIO_write(data, buf, i) != i) {
ret = 0;
break;
}
}
err:
OPENSSL_free(buf);
BIO_free_all(tmpmem);
return ret;
}