Coverity fix in some crypto/asn1 code

Call to i2d method returns an int value.

Fix:
CID 1338183 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371691 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371692 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)

[extended tests]

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7359)
This commit is contained in:
FdaSilvaYY 2018-09-30 22:39:38 +02:00 committed by Dr. Matthias St. Pierre
parent 39fc4c17c4
commit da84249be6
3 changed files with 43 additions and 19 deletions

View file

@ -23,18 +23,22 @@
int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data,
unsigned char *md, unsigned int *len)
{
int i;
int inl;
unsigned char *str, *p;
i = i2d(data, NULL);
if ((str = OPENSSL_malloc(i)) == NULL) {
inl = i2d(data, NULL);
if (inl <= 0) {
ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_INTERNAL_ERROR);
return 0;
}
if ((str = OPENSSL_malloc(inl)) == NULL) {
ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_MALLOC_FAILURE);
return 0;
}
p = str;
i2d(data, &p);
if (!EVP_Digest(str, i, md, len, type, NULL)) {
if (!EVP_Digest(str, inl, md, len, type, NULL)) {
OPENSSL_free(str);
return 0;
}

View file

@ -29,7 +29,8 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
{
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
unsigned char *p, *buf_in = NULL, *buf_out = NULL;
int i, inl = 0, outl = 0, outll = 0;
int i, inl = 0, outl = 0;
size_t inll = 0, outll = 0;
X509_ALGOR *a;
if (ctx == NULL) {
@ -70,10 +71,15 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
}
}
inl = i2d(data, NULL);
buf_in = OPENSSL_malloc((unsigned int)inl);
if (inl <= 0) {
ASN1err(ASN1_F_ASN1_SIGN, ERR_R_INTERNAL_ERROR);
goto err;
}
inll = (size_t)inl;
buf_in = OPENSSL_malloc(inll);
outll = outl = EVP_PKEY_size(pkey);
buf_out = OPENSSL_malloc((unsigned int)outl);
if ((buf_in == NULL) || (buf_out == NULL)) {
buf_out = OPENSSL_malloc(outll);
if (buf_in == NULL || buf_out == NULL) {
outl = 0;
ASN1err(ASN1_F_ASN1_SIGN, ERR_R_MALLOC_FAILURE);
goto err;
@ -101,7 +107,7 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
err:
EVP_MD_CTX_free(ctx);
OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
OPENSSL_clear_free((char *)buf_in, inll);
OPENSSL_clear_free((char *)buf_out, outll);
return outl;
}
@ -138,7 +144,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
EVP_PKEY *pkey;
unsigned char *buf_in = NULL, *buf_out = NULL;
size_t inl = 0, outl = 0, outll = 0;
int signid, paramtype;
int signid, paramtype, buf_len = 0;
int rv;
type = EVP_MD_CTX_md(ctx);
@ -198,10 +204,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
}
inl = ASN1_item_i2d(asn, &buf_in, it);
buf_len = ASN1_item_i2d(asn, &buf_in, it);
if (buf_len <= 0) {
outl = 0;
ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_INTERNAL_ERROR);
goto err;
}
inl = buf_len;
outll = outl = EVP_PKEY_size(pkey);
buf_out = OPENSSL_malloc((unsigned int)outl);
if ((buf_in == NULL) || (buf_out == NULL)) {
buf_out = OPENSSL_malloc(outll);
if (buf_in == NULL || buf_out == NULL) {
outl = 0;
ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_MALLOC_FAILURE);
goto err;
@ -223,7 +235,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
err:
OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
OPENSSL_clear_free((char *)buf_in, inl);
OPENSSL_clear_free((char *)buf_out, outll);
return outl;
}

View file

@ -48,6 +48,10 @@ int ASN1_verify(i2d_of_void *i2d, X509_ALGOR *a, ASN1_BIT_STRING *signature,
}
inl = i2d(data, NULL);
if (inl <= 0) {
ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_INTERNAL_ERROR);
goto err;
}
buf_in = OPENSSL_malloc((unsigned int)inl);
if (buf_in == NULL) {
ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_MALLOC_FAILURE);
@ -87,8 +91,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
EVP_MD_CTX *ctx = NULL;
unsigned char *buf_in = NULL;
int ret = -1, inl = 0;
int mdnid, pknid;
size_t inll = 0;
if (!pkey) {
ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_PASSED_NULL_PARAMETER);
@ -127,8 +131,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
goto err;
ret = -1;
} else {
const EVP_MD *type;
type = EVP_get_digestbynid(mdnid);
const EVP_MD *type = EVP_get_digestbynid(mdnid);
if (type == NULL) {
ASN1err(ASN1_F_ASN1_ITEM_VERIFY,
ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
@ -150,11 +154,15 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
}
inl = ASN1_item_i2d(asn, &buf_in, it);
if (inl <= 0) {
ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_INTERNAL_ERROR);
goto err;
}
if (buf_in == NULL) {
ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
inll = inl;
ret = EVP_DigestVerify(ctx, signature->data, (size_t)signature->length,
buf_in, inl);
@ -164,7 +172,7 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
}
ret = 1;
err:
OPENSSL_clear_free(buf_in, (unsigned int)inl);
OPENSSL_clear_free(buf_in, inll);
EVP_MD_CTX_free(ctx);
return ret;
}