Add main() test methods to reduce test boilerplate.

Simple tests only need to implement register_tests().
Tests that need a custom main() should implement test_main(). This will
be wrapped in a main() that performs common setup/teardown (currently
crypto-mdebug).

Note that for normal development, enable-asan is usually
sufficient for detecting leaks, and more versatile.

enable-crypto-mdebug is stricter as it will also
insist that all static variables be freed. This is useful for debugging
library init/deinit; however, it also means that test_main() must free
everything it allocates.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
This commit is contained in:
Emilia Kasper 2016-11-07 16:53:15 +01:00
parent 7380737d77
commit e364c3b24e
20 changed files with 223 additions and 130 deletions

View file

@ -15,6 +15,7 @@
#include <openssl/asn1.h>
#include <openssl/evp.h>
#include <openssl/objects.h>
#include "test_main.h"
#include "testutil.h"
#include "e_os.h"
@ -90,10 +91,8 @@ static int test_standard_methods()
return 0;
}
int main(int argc, char **argv)
void register_tests(void)
{
ADD_TEST(test_tbl_standard);
ADD_TEST(test_standard_methods);
return run_tests(argv[0]);
}

View file

@ -226,7 +226,7 @@ IF[{- !$disabled{tests} -}]
INCLUDE[dtlsv1listentest]=.. ../include
DEPEND[dtlsv1listentest]=../libssl
SOURCE[ct_test]=ct_test.c testutil.c
SOURCE[ct_test]=ct_test.c testutil.c test_main_custom.c
INCLUDE[ct_test]=../crypto/include ../include
DEPEND[ct_test]=../libcrypto
@ -238,19 +238,19 @@ IF[{- !$disabled{tests} -}]
INCLUDE[afalgtest]=.. ../include
DEPEND[afalgtest]=../libcrypto
SOURCE[d2i_test]=d2i_test.c testutil.c
SOURCE[d2i_test]=d2i_test.c testutil.c test_main_custom.c
INCLUDE[d2i_test]=.. ../include
DEPEND[d2i_test]=../libcrypto
SOURCE[ssl_test_ctx_test]=ssl_test_ctx_test.c ssl_test_ctx.c testutil.c
SOURCE[ssl_test_ctx_test]=ssl_test_ctx_test.c ssl_test_ctx.c testutil.c test_main_custom.c
INCLUDE[ssl_test_ctx_test]=.. ../include
DEPEND[ssl_test_ctx_test]=../libcrypto
SOURCE[ssl_test]=ssl_test.c ssl_test_ctx.c testutil.c handshake_helper.c
SOURCE[ssl_test]=ssl_test.c ssl_test_ctx.c testutil.c handshake_helper.c test_main_custom.c
INCLUDE[ssl_test]=.. ../include
DEPEND[ssl_test]=../libcrypto ../libssl
SOURCE[cipherlist_test]=cipherlist_test.c testutil.c
SOURCE[cipherlist_test]=cipherlist_test.c testutil.c test_main.c
INCLUDE[cipherlist_test]=.. ../include
DEPEND[cipherlist_test]=../libcrypto ../libssl
@ -271,11 +271,11 @@ IF[{- !$disabled{tests} -}]
INCLUDE[bioprinttest]=../include
DEPEND[bioprinttest]=../libcrypto
SOURCE[sslapitest]=sslapitest.c ssltestlib.c testutil.c
SOURCE[sslapitest]=sslapitest.c ssltestlib.c testutil.c test_main_custom.c
INCLUDE[sslapitest]=../include
DEPEND[sslapitest]=../libcrypto ../libssl
SOURCE[dtlstest]=dtlstest.c ssltestlib.c testutil.c
SOURCE[dtlstest]=dtlstest.c ssltestlib.c testutil.c test_main_custom.c
INCLUDE[dtlstest]=../include .
DEPEND[dtlstest]=../libcrypto ../libssl
@ -298,7 +298,7 @@ IF[{- !$disabled{tests} -}]
IF[{- $disabled{shared} -}]
PROGRAMS_NO_INST=wpackettest cipher_overhead_test
SOURCE[wpackettest]=wpackettest.c testutil.c
SOURCE[wpackettest]=wpackettest.c testutil.c test_main_custom.c
INCLUDE[wpackettest]=../include
DEPEND[wpackettest]=../libcrypto ../libssl
@ -330,7 +330,7 @@ IF[{- !$disabled{tests} -}]
PROGRAMS_NO_INST=poly1305_internal_test
ENDIF
SOURCE[poly1305_internal_test]=poly1305_internal_test.c testutil.c
SOURCE[poly1305_internal_test]=poly1305_internal_test.c testutil.c test_main_custom.c
IF[{- !$disabled{shared} -}]
SOURCE[poly1305_internal_test]= ../crypto/poly1305/poly1305.c \
{- rebase_files("../crypto/poly1305", $target{poly1305_asm_src}) -} \
@ -340,7 +340,7 @@ IF[{- !$disabled{tests} -}]
INCLUDE[poly1305_internal_test]=.. ../include ../crypto/include
DEPEND[poly1305_internal_test]=../libcrypto
SOURCE[asn1_internal_test]=asn1_internal_test.c testutil.c
SOURCE[asn1_internal_test]=asn1_internal_test.c testutil.c test_main.c
IF[{- !$disabled{shared} -}]
SOURCE[asn1_internal_test]= ../crypto/asn1/a_strnid.c \
../crypto/rsa/rsa_ameth.c ../crypto/dsa/dsa_ameth.c \
@ -351,7 +351,7 @@ IF[{- !$disabled{tests} -}]
INCLUDE[asn1_internal_test]=.. ../include ../crypto/include
DEPEND[asn1_internal_test]=../libcrypto
SOURCE[modes_internal_test]=modes_internal_test.c testutil.c
SOURCE[modes_internal_test]=modes_internal_test.c testutil.c test_main_custom.c
IF[{- !$disabled{shared} -}]
SOURCE[modes_internal_test]= \
{- rebase_files("../crypto", $target{cpuid_asm_src}); -} \
@ -363,7 +363,7 @@ IF[{- !$disabled{tests} -}]
# The reason for the huge amount of directly included x509v3 files
# is that a table that is checked by x509_internal_test refers to
# structures that are spread all over those files.
SOURCE[x509_internal_test]=x509_internal_test.c testutil.c
SOURCE[x509_internal_test]=x509_internal_test.c testutil.c test_main.c
IF[{- !$disabled{shared} -}]
SOURCE[x509_internal_test]= ../crypto/x509v3/v3_bitst.c \
../crypto/x509v3/v3_ia5.c ../crypto/x509v3/v3_skey.c \
@ -382,7 +382,7 @@ IF[{- !$disabled{tests} -}]
INCLUDE[x509_internal_test]=.. ../include
DEPEND[x509_internal_test]=../libcrypto
SOURCE[mdc2_internal_test]=mdc2_internal_test.c testutil.c
SOURCE[mdc2_internal_test]=mdc2_internal_test.c testutil.c test_main.c
INCLUDE[mdc2_internal_test]=.. ../include
DEPEND[mdc2_internal_test]=../libcrypto

View file

@ -18,6 +18,7 @@
#include <openssl/tls1.h>
#include "e_os.h"
#include "test_main.h"
#include "testutil.h"
typedef struct cipherlist_test_fixture {
@ -189,14 +190,8 @@ static int test_default_cipherlist_explicit()
EXECUTE_CIPHERLIST_TEST();
}
int main(int argc, char **argv)
void register_tests()
{
int result = 0;
ADD_TEST(test_default_cipherlist_implicit);
ADD_TEST(test_default_cipherlist_explicit);
result = run_tests(argv[0]);
return result;
}

View file

@ -18,9 +18,9 @@
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#include "testutil.h"
#include "test_main_custom.h"
#ifndef OPENSSL_NO_CT
/* Used when declaring buffers to read text files into */
#define CT_TEST_MAX_FILE_SIZE 8096
@ -530,15 +530,10 @@ static int test_encode_tls_sct()
EXECUTE_CT_TEST();
}
int main(int argc, char *argv[])
int test_main(int argc, char *argv[])
{
int result = 0;
char *tmp_env = NULL;
tmp_env = getenv("OPENSSL_DEBUG_MEMORY");
if (tmp_env != NULL && strcmp(tmp_env, "on") == 0)
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
char *tmp_env;
tmp_env = getenv("CT_DIR");
ct_dir = OPENSSL_strdup(tmp_env != NULL ? tmp_env : "ct");
@ -554,24 +549,10 @@ int main(int argc, char *argv[])
ADD_TEST(test_encode_tls_sct);
result = run_tests(argv[0]);
ERR_print_errors_fp(stderr);
OPENSSL_free(ct_dir);
OPENSSL_free(certs_dir);
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (CRYPTO_mem_leaks_fp(stderr) <= 0)
result = 1;
#endif
return result;
}
#else /* OPENSSL_NO_CT */
int main(int argc, char* argv[])
{
return EXIT_SUCCESS;
}
#endif /* OPENSSL_NO_CT */
#endif

View file

@ -13,6 +13,7 @@
#include <string.h>
#include "testutil.h"
#include "test_main_custom.h"
#include <openssl/asn1.h>
#include <openssl/asn1t.h>
@ -109,12 +110,10 @@ static int test_bad_asn1()
* Usage: d2i_test <type> <file>, e.g.
* d2i_test generalname bad_generalname.der
*/
int main(int argc, char **argv)
int test_main(int argc, char *argv[])
{
int result = 0;
const char *test_type_name;
const char *expected_error_string;
const char *p = getenv("OPENSSL_DEBUG_MEMORY");
size_t i;
@ -126,10 +125,6 @@ int main(int argc, char **argv)
{"compare", ASN1_COMPARE}
};
if (p != NULL && strcmp(p, "on") == 0)
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
if (argc != 4) {
fprintf(stderr,
"Usage: d2i_test item_name expected_error file.der\n");
@ -169,12 +164,5 @@ int main(int argc, char **argv)
ADD_TEST(test_bad_asn1);
result = run_tests(argv[0]);
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (CRYPTO_mem_leaks_fp(stderr) <= 0)
result = 1;
#endif
return result;
return run_tests(argv[0]);
}

View file

@ -14,6 +14,7 @@
#include "ssltestlib.h"
#include "testutil.h"
#include "test_main_custom.h"
static char *cert = NULL;
static char *privkey = NULL;
@ -104,9 +105,8 @@ static int test_dtls_unprocessed(int testidx)
return testresult;
}
int main(int argc, char *argv[])
int test_main(int argc, char *argv[])
{
BIO *err = NULL;
int testresult = 1;
if (argc != 3) {
@ -117,11 +117,6 @@ int main(int argc, char *argv[])
cert = argv[1];
privkey = argv[2];
err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
ADD_ALL_TESTS(test_dtls_unprocessed, NUM_TESTS);
testresult = run_tests(argv[0]);
@ -129,14 +124,5 @@ int main(int argc, char *argv[])
bio_f_tls_dump_filter_free();
bio_s_mempacket_test_free();
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (CRYPTO_mem_leaks(err) <= 0)
testresult = 1;
#endif
BIO_free(err);
if (!testresult)
printf("PASS\n");
return testresult;
}

View file

@ -14,6 +14,7 @@
#include <openssl/mdc2.h>
#include "testutil.h"
#include "test_main.h"
#include "e_os.h"
typedef struct {
@ -63,9 +64,7 @@ static int test_mdc2(int idx)
return 1;
}
int main(int argc, char **argv)
void register_tests()
{
ADD_ALL_TESTS(test_mdc2, OSSL_NELEM(tests));
return run_tests(argv[0]);
}

View file

@ -16,6 +16,7 @@
#include <openssl/modes.h>
#include "../crypto/modes/modes_lcl.h"
#include "testutil.h"
#include "test_main_custom.h"
#include "e_os.h"
typedef struct {
@ -964,7 +965,7 @@ static void benchmark_gcm128(const unsigned char *K, size_t Klen,
#endif
}
int main(int argc, char **argv)
int test_main(int argc, char **argv)
{
int result = 0;
int iter_argv;

View file

@ -13,6 +13,7 @@
#include <string.h>
#include "testutil.h"
#include "test_main_custom.h"
#include "internal/poly1305.h"
#include "../crypto/poly1305/poly1305_local.h"
#include "e_os.h"
@ -1638,7 +1639,7 @@ static int test_poly1305(int idx)
return 1;
}
int main(int argc, char **argv)
int test_main(int argc, char **argv)
{
int result = 0;
int iter_argv;

View file

@ -17,6 +17,7 @@
#include "handshake_helper.h"
#include "ssl_test_ctx.h"
#include "testutil.h"
#include "test_main_custom.h"
static CONF *conf = NULL;
@ -296,7 +297,7 @@ err:
return ret;
}
int main(int argc, char **argv)
int test_main(int argc, char **argv)
{
int result = 0;
long num_tests;
@ -315,5 +316,6 @@ int main(int argc, char **argv)
ADD_ALL_TESTS(test_handshake, (int)(num_tests));
result = run_tests(argv[0]);
NCONF_free(conf);
return result;
}

View file

@ -18,6 +18,7 @@
#include "e_os.h"
#include "ssl_test_ctx.h"
#include "testutil.h"
#include "test_main_custom.h"
#include <openssl/e_os2.h>
#include <openssl/err.h>
#include <openssl/conf.h>
@ -303,7 +304,7 @@ static int test_bad_configuration(int idx)
return 1;
}
int main(int argc, char **argv)
int test_main(int argc, char **argv)
{
int result = 0;

View file

@ -17,6 +17,7 @@
#include "ssltestlib.h"
#include "testutil.h"
#include "test_main_custom.h"
static char *cert = NULL;
static char *privkey = NULL;
@ -863,9 +864,8 @@ static int test_ssl_bio_change_wbio(void)
EXECUTE_TEST(execute_test_ssl_bio, ssl_bio_tear_down);
}
int main(int argc, char *argv[])
int test_main(int argc, char *argv[])
{
BIO *err = NULL;
int testresult = 1;
if (argc != 3) {
@ -876,11 +876,6 @@ int main(int argc, char *argv[])
cert = argv[1];
privkey = argv[2];
err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
ADD_TEST(test_large_message_tls);
ADD_TEST(test_large_message_tls_read_ahead);
#ifndef OPENSSL_NO_DTLS
@ -900,14 +895,5 @@ int main(int argc, char *argv[])
bio_s_mempacket_test_free();
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (CRYPTO_mem_leaks(err) <= 0)
testresult = 1;
#endif
BIO_free(err);
if (!testresult)
printf("PASS\n");
return testresult;
}

26
test/test_main.c Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "test_main.h"
#include "testutil.h"
#include <stdio.h>
int main(int argc, char *argv[])
{
int ret;
if (argc > 1)
printf("Warning: ignoring extra command-line arguments.\n");
setup_test();
register_tests();
ret = run_tests(argv[0]);
return finish_test(ret);
}

18
test/test_main.h Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef HEADER_TEST_MAIN_H
# define HEADER_TEST_MAIN_H
/*
* Simple unit tests should implement register_tests() and link to test_main.c.
*/
extern void register_tests(void);
#endif /* HEADER_TEST_MAIN_H */

24
test/test_main_custom.c Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "test_main_custom.h"
#include "testutil.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[])
{
int ret;
setup_test();
ret = test_main(argc, argv);
return finish_test(ret);
}

20
test/test_main_custom.h Normal file
View file

@ -0,0 +1,20 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef HEADER_TEST_MAIN_CUSTOM_H
# define HEADER_TEST_MAIN_CUSTOM_H
/*
* Unit tests that need a custom main() should implement test_main and link to
* test_main_custom.c
* test_main() should return the result of run_tests().
*/
extern int test_main(int argc, char *argv[]);
#endif /* HEADER_TEST_MAIN_CUSTOM_H */

View file

@ -15,6 +15,8 @@
#include <string.h>
#include "e_os.h"
#include <openssl/opensslconf.h>
#include <openssl/crypto.h>
#include <openssl/err.h>
/*
@ -57,6 +59,42 @@ void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
num_test_cases += num;
}
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
static int should_report_leaks()
{
/*
* When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
* can be used to disable leak checking at runtime.
* Note this only works when running the test binary manually;
* the test harness always enables OPENSSL_DEBUG_MEMORY.
*/
char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
return mem_debug_env == NULL
|| (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
}
#endif
void setup_test()
{
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks()) {
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
}
#endif
}
int finish_test(int ret)
{
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks() && CRYPTO_mem_leaks_fp(stderr) <= 0)
return EXIT_FAILURE;
#endif
return ret;
}
static void finalize(int success)
{
if (success)

View file

@ -11,13 +11,51 @@
# define HEADER_TESTUTIL_H
#include <openssl/err.h>
#include <openssl/e_os2.h>
/*-
* Simple unit tests should implement register_tests() from test_main.h
* and link against test_main.c.
* To register tests, call ADD_TEST or ADD_ALL_TESTS:
*
* #include "test_main.h"
*
* void register_tests(void)
* {
* ADD_TEST(test_foo);
* ADD_ALL_TESTS(test_bar, num_test_bar);
* }
*
* Tests that need to perform custom setup or read command-line arguments should
* implement test_main() from test_main_custom.h and link against
* test_main_custom.c:
*
* int test_main(int argc, char *argv[])
* {
* int ret;
*
* // Custom setup ...
*
* ADD_TEST(test_foo);
* ADD_ALL_TESTS(test_bar, num_test_bar);
* // Add more tests ...
*
* ret = run_tests(argv[0]);
*
* // Custom teardown ...
*
* return ret;
* }
*/
/* Adds a simple test case. */
# define ADD_TEST(test_function) add_test(#test_function, test_function)
/*
* In main(), call ADD_TEST to register each test case function, then call
* run_tests() to execute all tests and report the results. The result
* returned from run_tests() should be used as the return value for main().
* Simple parameterized tests. Calls test_function(idx) for each 0 <= idx < num.
*/
# define ADD_TEST(test_function) add_test(#test_function, test_function)
# define ADD_ALL_TESTS(test_function, num) \
add_all_tests(#test_function, test_function, num)
/*-
* Test cases that share common setup should use the helper
@ -82,15 +120,21 @@
# endif /* __STDC_VERSION__ */
/*
* Simple parameterized tests. Adds a test_function(idx) test for each
* 0 <= idx < num.
* Internal helpers. Test programs shouldn't use these directly, but should
* rather link to one of the helper main() methods.
*/
# define ADD_ALL_TESTS(test_function, num) \
add_all_tests(#test_function, test_function, num)
/* setup_test() should be called as the first thing in a test main(). */
void setup_test(void);
/*
* finish_test() should be called as the last thing in a test main().
* The result of run_tests() should be the input to finish_test().
*/
__owur int finish_test(int ret);
void add_test(const char *test_case_name, int (*test_fn) ());
void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);
int run_tests(const char *test_prog_name);
__owur int run_tests(const char *test_prog_name);
/*
* Test assumption verification helpers.
@ -101,7 +145,6 @@ int run_tests(const char *test_prog_name);
* Otherwise, returns 0 and pretty-prints diagnostics using |desc|.
*/
int strings_equal(const char *desc, const char *s1, const char *s2);
#endif /* HEADER_TESTUTIL_H */
/*
* For "impossible" conditions such as malloc failures or bugs in test code,
@ -115,3 +158,4 @@ int strings_equal(const char *desc, const char *s1, const char *s2);
OPENSSL_assert(!#condition); \
} \
} while (0)
#endif /* HEADER_TESTUTIL_H */

View file

@ -11,6 +11,7 @@
#include <openssl/buffer.h>
#include "../ssl/packet_locl.h"
#include "testutil.h"
#include "test_main_custom.h"
const static unsigned char simple1 = 0xff;
const static unsigned char simple2[] = { 0x01, 0xff };
@ -401,16 +402,10 @@ static int test_WPACKET_memcpy(void)
return 1;
}
int main(int argc, char *argv[])
int test_main(int argc, char *argv[])
{
BIO *err = NULL;
int testresult = 0;
err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
buf = BUF_MEM_new();
if (buf != NULL) {
ADD_TEST(test_WPACKET_init);
@ -425,15 +420,5 @@ int main(int argc, char *argv[])
BUF_MEM_free(buf);
}
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (CRYPTO_mem_leaks(err) <= 0)
testresult = 1;
#endif
BIO_free(err);
if (!testresult)
printf("PASS\n");
return testresult;
}

View file

@ -15,6 +15,7 @@
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#include "testutil.h"
#include "test_main.h"
#include "e_os.h"
/**********************************************************************
@ -52,9 +53,7 @@ static int test_standard_exts()
return good;
}
int main(int argc, char **argv)
void register_tests()
{
ADD_TEST(test_standard_exts);
return run_tests(argv[0]);
}