From 4db40c94c38ee94fbef820a9816ac2d68f65506e Mon Sep 17 00:00:00 2001 From: Richard Levitte Date: Tue, 18 Apr 2017 16:27:27 +0200 Subject: [PATCH] Refactor the test framework testutil It's now built as a static library, and greatly simplified for test programs, which no longer need to include test_main_custom.h or test_main.h and link with the corresponding object files. Reviewed-by: Rich Salz (Merged from https://github.com/openssl/openssl/pull/3243) --- test/README | 5 +- test/build.info | 9 +- test/test_main.h | 18 -- test/test_main_custom.h | 20 --- test/testutil.h | 46 ++++- test/testutil/basic_output.c | 80 +++++++++ test/testutil/driver.c | 167 +++++++++++++++++ test/{test_main_custom.c => testutil/main.c} | 8 +- test/{ => testutil}/test_main.c | 15 +- test/{testutil.c => testutil/tests.c} | 177 +++---------------- 10 files changed, 325 insertions(+), 220 deletions(-) delete mode 100644 test/test_main.h delete mode 100644 test/test_main_custom.h create mode 100644 test/testutil/basic_output.c create mode 100644 test/testutil/driver.c rename test/{test_main_custom.c => testutil/main.c} (71%) rename test/{ => testutil}/test_main.c (51%) rename test/{testutil.c => testutil/tests.c} (71%) diff --git a/test/README b/test/README index 7831c860f5..fc9f7d03b1 100644 --- a/test/README +++ b/test/README @@ -99,14 +99,13 @@ test): to modify the include paths and source files if you don't want to use the basic test framework: - SOURCE[{name}]={name}.c testutil.c test_main.c + SOURCE[{name}]={name}.c INCLUDE[{name}]=.. ../include - DEPEND[{name}]=../libcrypto + DEPEND[{name}]=../libcrypto libtestutil.a Generic form of C test executables ================================== - #include "test_main.h" #include "testutil.h" static int my_test(void) diff --git a/test/build.info b/test/build.info index 5a2e03e607..f75b85a641 100644 --- a/test/build.info +++ b/test/build.info @@ -5,11 +5,16 @@ my ($base, $files) = @_; return join(" ", map { "$base/$_" } split(/\s+/, $files)); } - our $apps_extra = - $config{target} =~ /^vms-/ ? "../apps/vms_term_sock.c" : ""; "" -} IF[{- !$disabled{tests} -}] + LIBS_NO_INST=libtestutil.a + SOURCE[libtestutil.a]=testutil/basic_output.c testutil/driver.c \ + testutil/tests.c testutil/test_main.c testutil/main.c \ + {- rebase_files("../apps", $target{apps_aux_src}) -} + INCLUDE[libtestutil.a]=../include + DEPEND[libtestutil.a]=../libcrypto + PROGRAMS_NO_INST=\ aborttest test_test \ sanitytest exdatatest bntest \ diff --git a/test/test_main.h b/test/test_main.h deleted file mode 100644 index ce0d06a407..0000000000 --- a/test/test_main.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef HEADER_TEST_MAIN_H -# define HEADER_TEST_MAIN_H - -/* - * Simple unit tests should implement register_tests() and link to test_main.c. - */ -extern void register_tests(void); - -#endif /* HEADER_TEST_MAIN_H */ diff --git a/test/test_main_custom.h b/test/test_main_custom.h deleted file mode 100644 index 22bc8f51a6..0000000000 --- a/test/test_main_custom.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef HEADER_TEST_MAIN_CUSTOM_H -# define HEADER_TEST_MAIN_CUSTOM_H - -/* - * Unit tests that need a custom main() should implement test_main and link to - * test_main_custom.c - * test_main() should return the result of run_tests(). - */ -extern int test_main(int argc, char *argv[]); - -#endif /* HEADER_TEST_MAIN_CUSTOM_H */ diff --git a/test/testutil.h b/test/testutil.h index e7478a8ade..d266fd5288 100644 --- a/test/testutil.h +++ b/test/testutil.h @@ -16,12 +16,9 @@ #include /*- - * Simple unit tests should implement register_tests() from test_main.h - * and link against test_main.c. + * Simple unit tests should implement register_tests(). * To register tests, call ADD_TEST or ADD_ALL_TESTS: * - * #include "test_main.h" - * * void register_tests(void) * { * ADD_TEST(test_foo); @@ -29,8 +26,7 @@ * } * * Tests that need to perform custom setup or read command-line arguments should - * implement test_main() from test_main_custom.h and link against - * test_main_custom.c: + * implement test_main(): * * int test_main(int argc, char *argv[]) * { @@ -138,14 +134,27 @@ void add_test(const char *test_case_name, int (*test_fn) ()); void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num); __owur int run_tests(const char *test_prog_name); +/* + * Declarations for user defined functions + */ +void register_tests(void); +int test_main(int argc, char *argv[]); + + /* * Test assumption verification helpers. */ -# if defined(__GNUC__) -#define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b))) -# else #define PRINTF_FORMAT(a, b) +#if defined(__GNUC__) && defined(__STDC_VERSION__) + /* + * Because we support the 'z' modifier, which made its appearance in C99, + * we can't use __attribute__ with pre C99 dialects. + */ +# if __STDC_VERSION__ >= 199901L +# undef PRINTF_FORMAT +# define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b))) +# endif #endif # define DECLARE_COMPARISON(type, name, opname) \ @@ -335,3 +344,22 @@ void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2); } \ } while (0) #endif /* HEADER_TESTUTIL_H */ + + +/* + * The basic I/O functions used by the test framework. These can be + * overriden when needed. Note that if one is, then all must be. + */ +void test_open_streams(void); +void test_close_streams(void); +/* The following ALL return the number of characters written */ +int test_puts_stdout(const char *str); +int test_puts_stderr(const char *str); +int test_vprintf_stdout(const char *fmt, va_list ap); +int test_vprintf_stderr(const char *fmt, va_list ap); +/* These return failure or success */ +int test_flush_stdout(void); +int test_flush_stderr(void); + +extern BIO *bio_out; +extern BIO *bio_err; diff --git a/test/testutil/basic_output.c b/test/testutil/basic_output.c new file mode 100644 index 0000000000..ac413a6a4a --- /dev/null +++ b/test/testutil/basic_output.c @@ -0,0 +1,80 @@ +/* + * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include +#include + +BIO *bio_out = NULL; +BIO *bio_err = NULL; + +#ifdef OPENSSL_USE_APPLINK +/* + * Using BIO_new_fd() obligates the use of applinks on platforms where it's + * relevant. Because it becomes a module of the libtestutil library and would + * be disregarded if not actively referred to, we have this dummy that does + * exactly this. For any module that uses the rest of the routines here, + * OPENSSL_Applink should tag along for sure. + */ +void Applink_dummy(void); +void Applink_dummy(void) +{ + OPENSSL_EXTERN void OPENSSL_Applink(void); + + OPENSSL_Applink(); +} +/* Generate an error for anyone who tries to actually use this dummy */ +# define Applink_dummy "DON'T USE THIS" +#endif + +void test_open_streams(void) +{ + bio_out = BIO_new_fd(1, 0); + bio_err = BIO_new_fd(2, 0); + + OPENSSL_assert(bio_out != NULL); + OPENSSL_assert(bio_err != NULL); +} + +void test_close_streams(void) +{ + BIO_free(bio_out); + BIO_free(bio_err); +} + +int test_puts_stdout(const char *str) +{ + return BIO_puts(bio_out, str); +} + +int test_puts_stderr(const char *str) +{ + return BIO_puts(bio_err, str); +} + +int test_vprintf_stdout(const char *fmt, va_list ap) +{ + return BIO_vprintf(bio_out, fmt, ap); +} + +int test_vprintf_stderr(const char *fmt, va_list ap) +{ + return BIO_vprintf(bio_err, fmt, ap); +} + +int test_flush_stdout(void) +{ + return BIO_flush(bio_out); +} + +int test_flush_stderr(void) +{ + return BIO_flush(bio_err); +} diff --git a/test/testutil/driver.c b/test/testutil/driver.c new file mode 100644 index 0000000000..e70fd21615 --- /dev/null +++ b/test/testutil/driver.c @@ -0,0 +1,167 @@ +/* + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include +#include + +#include "../../e_os.h" +#include + +/* + * Declares the structures needed to register each test case function. + */ +typedef struct test_info { + const char *test_case_name; + int (*test_fn) (); + int (*param_test_fn)(int idx); + int num; +} TEST_INFO; + +static TEST_INFO all_tests[1024]; +static int num_tests = 0; +/* + * A parameterised tests runs a loop of test cases. + * |num_test_cases| counts the total number of test cases + * across all tests. + */ +static int num_test_cases = 0; + +void add_test(const char *test_case_name, int (*test_fn) ()) +{ + assert(num_tests != OSSL_NELEM(all_tests)); + all_tests[num_tests].test_case_name = test_case_name; + all_tests[num_tests].test_fn = test_fn; + all_tests[num_tests].num = -1; + ++num_tests; + ++num_test_cases; +} + +void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), + int num) +{ + assert(num_tests != OSSL_NELEM(all_tests)); + all_tests[num_tests].test_case_name = test_case_name; + all_tests[num_tests].param_test_fn = test_fn; + all_tests[num_tests].num = num; + ++num_tests; + num_test_cases += num; +} + +#ifndef OPENSSL_NO_CRYPTO_MDEBUG +static int should_report_leaks() +{ + /* + * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0 + * can be used to disable leak checking at runtime. + * Note this only works when running the test binary manually; + * the test harness always enables OPENSSL_DEBUG_MEMORY. + */ + char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY"); + + return mem_debug_env == NULL + || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, "")); +} +#endif + + +static int err_cb(const char *str, size_t len, void *u) +{ + return test_puts_stderr(str); +} + +void setup_test() +{ + test_open_streams(); + +#ifndef OPENSSL_NO_CRYPTO_MDEBUG + if (should_report_leaks()) { + CRYPTO_set_mem_debug(1); + CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON); + } +#endif +} + +int finish_test(int ret) +{ +#ifndef OPENSSL_NO_CRYPTO_MDEBUG + if (should_report_leaks() && CRYPTO_mem_leaks_cb(err_cb, NULL) <= 0) + return EXIT_FAILURE; +#endif + + test_close_streams(); + + return ret; +} + +static void finalize(int success) +{ + if (success) + ERR_clear_error(); + else + ERR_print_errors_cb(err_cb, NULL); +} + +static void helper_printf_stdout(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + test_vprintf_stdout(fmt, ap); + va_end(ap); +} + +int run_tests(const char *test_prog_name) +{ + int num_failed = 0; + int i, j; + + helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases, + num_test_cases == 1 ? "" : "s"); + test_flush_stdout(); + + for (i = 0; i != num_tests; ++i) { + if (all_tests[i].num == -1) { + int ret = all_tests[i].test_fn(); + + if (!ret) { + helper_printf_stdout("** %s failed **\n--------\n", + all_tests[i].test_case_name); + test_flush_stdout(); + ++num_failed; + } + finalize(ret); + } else { + for (j = 0; j < all_tests[i].num; j++) { + int ret = all_tests[i].param_test_fn(j); + + if (!ret) { + helper_printf_stdout("** %s failed test %d\n--------\n", + all_tests[i].test_case_name, j); + test_flush_stdout(); + ++num_failed; + } + finalize(ret); + } + } + } + + if (num_failed != 0) { + helper_printf_stdout("%s: %d test%s failed (out of %d)\n", + test_prog_name, num_failed, + num_failed != 1 ? "s" : "", num_test_cases); + test_flush_stdout(); + return EXIT_FAILURE; + } + helper_printf_stdout(" All tests passed.\n"); + test_flush_stdout(); + return EXIT_SUCCESS; +} + diff --git a/test/test_main_custom.c b/test/testutil/main.c similarity index 71% rename from test/test_main_custom.c rename to test/testutil/main.c index e10f2795df..435a3585ce 100644 --- a/test/test_main_custom.c +++ b/test/testutil/main.c @@ -1,5 +1,5 @@ /* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,11 +7,7 @@ * https://www.openssl.org/source/license.html */ -#include "test_main_custom.h" -#include "testutil.h" - -#include -#include +#include "../testutil.h" int main(int argc, char *argv[]) { diff --git a/test/test_main.c b/test/testutil/test_main.c similarity index 51% rename from test/test_main.c rename to test/testutil/test_main.c index 99b21acc65..0152421fe1 100644 --- a/test/test_main.c +++ b/test/testutil/test_main.c @@ -1,5 +1,5 @@ /* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,20 +7,15 @@ * https://www.openssl.org/source/license.html */ -#include "test_main.h" -#include "testutil.h" +#include "../testutil.h" #include -int main(int argc, char *argv[]) +int test_main(int argc, char *argv[]) { - int ret; if (argc > 1) - printf("Warning: ignoring extra command-line arguments.\n"); + test_puts_stderr("Warning: ignoring extra command-line arguments.\n"); - setup_test(); register_tests(); - ret = run_tests(argv[0]); - - return finish_test(ret); + return run_tests(argv[0]); } diff --git a/test/testutil.c b/test/testutil/tests.c similarity index 71% rename from test/testutil.c rename to test/testutil/tests.c index ae98e1096d..f00ec6c19e 100644 --- a/test/testutil.c +++ b/test/testutil/tests.c @@ -1,5 +1,5 @@ /* - * Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,152 +7,14 @@ * https://www.openssl.org/source/license.html */ -#include "testutil.h" +#include "../testutil.h" -#include -#include -#include #include -#include "e_os.h" - -#include -#include -#include +#include "../../e_os.h" /* The size of memory buffers to display on failure */ #define MEM_BUFFER_SIZE (21) -/* - * Declares the structures needed to register each test case function. - */ -typedef struct test_info { - const char *test_case_name; - int (*test_fn) (); - int (*param_test_fn)(int idx); - int num; -} TEST_INFO; - -static TEST_INFO all_tests[1024]; -static int num_tests = 0; -/* - * A parameterised tests runs a loop of test cases. - * |num_test_cases| counts the total number of test cases - * across all tests. - */ -static int num_test_cases = 0; - -void add_test(const char *test_case_name, int (*test_fn) ()) -{ - assert(num_tests != OSSL_NELEM(all_tests)); - all_tests[num_tests].test_case_name = test_case_name; - all_tests[num_tests].test_fn = test_fn; - all_tests[num_tests].num = -1; - ++num_test_cases; - ++num_tests; -} - -void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), - int num) -{ - assert(num_tests != OSSL_NELEM(all_tests)); - all_tests[num_tests].test_case_name = test_case_name; - all_tests[num_tests].param_test_fn = test_fn; - all_tests[num_tests].num = num; - ++num_tests; - num_test_cases += num; -} - -#ifndef OPENSSL_NO_CRYPTO_MDEBUG -static int should_report_leaks() -{ - /* - * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0 - * can be used to disable leak checking at runtime. - * Note this only works when running the test binary manually; - * the test harness always enables OPENSSL_DEBUG_MEMORY. - */ - char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY"); - - return mem_debug_env == NULL - || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, "")); -} -#endif - - -void setup_test() -{ -#ifndef OPENSSL_NO_CRYPTO_MDEBUG - if (should_report_leaks()) { - CRYPTO_set_mem_debug(1); - CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON); - } -#endif -} - -int finish_test(int ret) -{ -#ifndef OPENSSL_NO_CRYPTO_MDEBUG - if (should_report_leaks() && CRYPTO_mem_leaks_fp(stderr) <= 0) - return EXIT_FAILURE; -#endif - return ret; -} - -static void finalize(int success) -{ - if (success) - ERR_clear_error(); - else - ERR_print_errors_fp(stderr); -} - -int run_tests(const char *test_prog_name) -{ - int num_failed = 0; - - int i, j; - - printf("%s: %d test case%s\n", test_prog_name, num_test_cases, - num_test_cases == 1 ? "" : "s"); - fflush(stdout); - - for (i = 0; i != num_tests; ++i) { - if (all_tests[i].num == -1) { - int ret = all_tests[i].test_fn(); - - if (!ret) { - printf("** %s failed **\n--------\n", - all_tests[i].test_case_name); - fflush(stdout); - ++num_failed; - } - finalize(ret); - } else { - for (j = 0; j < all_tests[i].num; j++) { - int ret = all_tests[i].param_test_fn(j); - - if (!ret) { - printf("** %s failed test %d\n--------\n", - all_tests[i].test_case_name, j); - fflush(stdout); - ++num_failed; - } - finalize(ret); - } - } - } - - if (num_failed != 0) { - printf("%s: %d test%s failed (out of %d)\n", test_prog_name, - num_failed, num_failed != 1 ? "s" : "", num_test_cases); - fflush(stdout); - return EXIT_FAILURE; - } - printf(" All tests passed.\n"); - fflush(stdout); - return EXIT_SUCCESS; -} - /* * A common routine to output test failure messages. Generally this should not * be called directly, rather it should be called by the following functions. @@ -182,27 +44,38 @@ static void test_fail_message(const char *prefix, const char *file, int line, const char *type, const char *fmt, ...) PRINTF_FORMAT(5, 6); +static void helper_printf_stderr(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + test_vprintf_stderr(fmt, ap); + va_end(ap); +} + static void test_fail_message_va(const char *prefix, const char *file, int line, const char *type, const char *fmt, va_list ap) { - fputs(prefix != NULL ? prefix : "ERROR", stderr); - fputs(":", stderr); + test_puts_stderr(prefix != NULL ? prefix : "ERROR"); + test_puts_stderr(":"); if (type) - fprintf(stderr, " (%s)", type); + helper_printf_stderr(" (%s)", type); if (fmt != NULL) { - fputc(' ', stderr); - vfprintf(stderr, fmt, ap); + test_puts_stderr(" "); + test_vprintf_stderr(fmt, ap); } if (file != NULL) { - fprintf(stderr, " @ %s:%d", file, line); + helper_printf_stderr(" @ %s:%d", file, line); } - fputc('\n', stderr); + test_puts_stderr("\n"); + test_flush_stderr(); } static void test_fail_message(const char *prefix, const char *file, int line, const char *type, const char *fmt, ...) { va_list ap; + va_start(ap, fmt); test_fail_message_va(prefix, file, line, type, fmt, ap); va_end(ap); @@ -293,7 +166,7 @@ DEFINE_COMPARISONS(char, char, "%c") DEFINE_COMPARISONS(unsigned char, uchar, "%u") DEFINE_COMPARISONS(long, long, "%ld") DEFINE_COMPARISONS(unsigned long, ulong, "%lu") -DEFINE_COMPARISONS(size_t, size_t, "%" OSSLzu) +DEFINE_COMPARISONS(size_t, size_t, "%zu") DEFINE_COMPARISON(void *, ptr, eq, ==, "%p") DEFINE_COMPARISON(void *, ptr, ne, !=, "%p") @@ -434,14 +307,14 @@ int test_mem_eq(const char *file, int line, const char *st1, const char *st2, return 1; if (n1 != n2) { test_fail_message(NULL, file, line, "memory", - "size mismatch %s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", + "size mismatch %s %s [%zu] != %s %s [%zu]", st1, print_mem_maybe_null(s1, n1, b1), n1, st2, print_mem_maybe_null(s2, n2, b2), n2); return 0; } if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) { test_fail_message(NULL, file, line, "memory", - "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", + "%s %s [%zu] != %s %s [%zu]", st1, print_mem_maybe_null(s1, n1, b1), n1, st2, print_mem_maybe_null(s2, n2, b2), n2); return 0; @@ -460,7 +333,7 @@ int test_mem_ne(const char *file, int line, const char *st1, const char *st2, return 1; if (s1 == NULL || memcmp(s1, s2, n1) == 0) { test_fail_message(NULL, file, line, "memory", - "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", + "%s %s [%zu] != %s %s [%zu]", st1, print_mem_maybe_null(s1, n1, b1), n1, st2, print_mem_maybe_null(s2, n2, b2), n2); return 0;