wbrawner/openssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor the test framework testutil

Browse source 
It's now built as a static library, and greatly simplified for test
programs, which no longer need to include test_main_custom.h or
test_main.h and link with the corresponding object files.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3243)
This commit is contained in:
Richard Levitte 2017-04-18 16:27:27 +02:00
parent 20626cfd58
commit 4db40c94c3
10 changed files with 325 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
test/README
View file

@ -99,14 +99,13 @@ test):
to modify the include paths and source files if you don't want to use the
basic test framework:
SOURCE[{name}]={name}.c testutil.c test_main.c
SOURCE[{name}]={name}.c
INCLUDE[{name}]=.. ../include
DEPEND[{name}]=../libcrypto
DEPEND[{name}]=../libcrypto libtestutil.a
Generic form of C test executables
==================================
#include "test_main.h"
#include "testutil.h"
static int my_test(void)

9
test/build.info
View file

@ -5,11 +5,16 @@
my ($base, $files) = @_;
return join(" ", map { "$base/$_" } split(/\s+/, $files));
}
our $apps_extra =
$config{target} =~ /^vms-/ ? "../apps/vms_term_sock.c" : "";
""
-}
IF[{- !$disabled{tests} -}]
LIBS_NO_INST=libtestutil.a
SOURCE[libtestutil.a]=testutil/basic_output.c testutil/driver.c \
testutil/tests.c testutil/test_main.c testutil/main.c \
{- rebase_files("../apps", $target{apps_aux_src}) -}
INCLUDE[libtestutil.a]=../include
DEPEND[libtestutil.a]=../libcrypto
PROGRAMS_NO_INST=\
aborttest test_test \
sanitytest exdatatest bntest \

18
test/test_main.h
View file

@ -1,18 +0,0 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef HEADER_TEST_MAIN_H
# define HEADER_TEST_MAIN_H
/*
* Simple unit tests should implement register_tests() and link to test_main.c.
*/
extern void register_tests(void);
#endif /* HEADER_TEST_MAIN_H */

20
test/test_main_custom.h
View file

@ -1,20 +0,0 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef HEADER_TEST_MAIN_CUSTOM_H
# define HEADER_TEST_MAIN_CUSTOM_H
/*
* Unit tests that need a custom main() should implement test_main and link to
* test_main_custom.c
* test_main() should return the result of run_tests().
*/
extern int test_main(int argc, char *argv[]);
#endif /* HEADER_TEST_MAIN_CUSTOM_H */

46
test/testutil.h
View file

@ -16,12 +16,9 @@
#include <openssl/e_os2.h>
/*-
* Simple unit tests should implement register_tests() from test_main.h
* and link against test_main.c.
* Simple unit tests should implement register_tests().
* To register tests, call ADD_TEST or ADD_ALL_TESTS:
*
* #include "test_main.h"
*
* void register_tests(void)
* {
* ADD_TEST(test_foo);
@ -29,8 +26,7 @@
* }
*
* Tests that need to perform custom setup or read command-line arguments should
* implement test_main() from test_main_custom.h and link against
* test_main_custom.c:
* implement test_main():
*
* int test_main(int argc, char *argv[])
* {
@ -138,14 +134,27 @@ void add_test(const char *test_case_name, int (*test_fn) ());
void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);
__owur int run_tests(const char *test_prog_name);
/*
* Declarations for user defined functions
*/
void register_tests(void);
int test_main(int argc, char *argv[]);
/*
* Test assumption verification helpers.
*/
# if defined(__GNUC__)
#define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b)))
# else
#define PRINTF_FORMAT(a, b)
#if defined(__GNUC__) && defined(__STDC_VERSION__)
/*
* Because we support the 'z' modifier, which made its appearance in C99,
* we can't use __attribute__ with pre C99 dialects.
*/
# if __STDC_VERSION__ >= 199901L
# undef PRINTF_FORMAT
# define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b)))
# endif
#endif
# define DECLARE_COMPARISON(type, name, opname) \
@ -335,3 +344,22 @@ void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);
} \
} while (0)
#endif /* HEADER_TESTUTIL_H */
/*
* The basic I/O functions used by the test framework. These can be
* overriden when needed. Note that if one is, then all must be.
*/
void test_open_streams(void);
void test_close_streams(void);
/* The following ALL return the number of characters written */
int test_puts_stdout(const char *str);
int test_puts_stderr(const char *str);
int test_vprintf_stdout(const char *fmt, va_list ap);
int test_vprintf_stderr(const char *fmt, va_list ap);
/* These return failure or success */
int test_flush_stdout(void);
int test_flush_stderr(void);
extern BIO *bio_out;
extern BIO *bio_err;

80
test/testutil/basic_output.c Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "../testutil.h"
#include <openssl/crypto.h>
#include <openssl/bio.h>
BIO *bio_out = NULL;
BIO *bio_err = NULL;
#ifdef OPENSSL_USE_APPLINK
/*
* Using BIO_new_fd() obligates the use of applinks on platforms where it's
* relevant. Because it becomes a module of the libtestutil library and would
* be disregarded if not actively referred to, we have this dummy that does
* exactly this. For any module that uses the rest of the routines here,
* OPENSSL_Applink should tag along for sure.
*/
void Applink_dummy(void);
void Applink_dummy(void)
{
OPENSSL_EXTERN void OPENSSL_Applink(void);
OPENSSL_Applink();
}
/* Generate an error for anyone who tries to actually use this dummy */
# define Applink_dummy "DON'T USE THIS"
#endif
void test_open_streams(void)
{
bio_out = BIO_new_fd(1, 0);
bio_err = BIO_new_fd(2, 0);
OPENSSL_assert(bio_out != NULL);
OPENSSL_assert(bio_err != NULL);
}
void test_close_streams(void)
{
BIO_free(bio_out);
BIO_free(bio_err);
}
int test_puts_stdout(const char *str)
{
return BIO_puts(bio_out, str);
}
int test_puts_stderr(const char *str)
{
return BIO_puts(bio_err, str);
}
int test_vprintf_stdout(const char *fmt, va_list ap)
{
return BIO_vprintf(bio_out, fmt, ap);
}
int test_vprintf_stderr(const char *fmt, va_list ap)
{
return BIO_vprintf(bio_err, fmt, ap);
}
int test_flush_stdout(void)
{
return BIO_flush(bio_out);
}
int test_flush_stderr(void)
{
return BIO_flush(bio_err);
}

167
test/testutil/driver.c Normal file
View file

@ -0,0 +1,167 @@
/*
* Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "../testutil.h"
#include <string.h>
#include <assert.h>
#include "../../e_os.h"
#include <openssl/bio.h>
/*
* Declares the structures needed to register each test case function.
*/
typedef struct test_info {
const char *test_case_name;
int (*test_fn) ();
int (*param_test_fn)(int idx);
int num;
} TEST_INFO;
static TEST_INFO all_tests[1024];
static int num_tests = 0;
/*
* A parameterised tests runs a loop of test cases.
* |num_test_cases| counts the total number of test cases
* across all tests.
*/
static int num_test_cases = 0;
void add_test(const char *test_case_name, int (*test_fn) ())
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
all_tests[num_tests].test_fn = test_fn;
all_tests[num_tests].num = -1;
++num_tests;
++num_test_cases;
}
void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
int num)
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
all_tests[num_tests].param_test_fn = test_fn;
all_tests[num_tests].num = num;
++num_tests;
num_test_cases += num;
}
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
static int should_report_leaks()
{
/*
* When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
* can be used to disable leak checking at runtime.
* Note this only works when running the test binary manually;
* the test harness always enables OPENSSL_DEBUG_MEMORY.
*/
char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
return mem_debug_env == NULL
|| (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
}
#endif
static int err_cb(const char *str, size_t len, void *u)
{
return test_puts_stderr(str);
}
void setup_test()
{
test_open_streams();
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks()) {
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
}
#endif
}
int finish_test(int ret)
{
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks() && CRYPTO_mem_leaks_cb(err_cb, NULL) <= 0)
return EXIT_FAILURE;
#endif
test_close_streams();
return ret;
}
static void finalize(int success)
{
if (success)
ERR_clear_error();
else
ERR_print_errors_cb(err_cb, NULL);
}
static void helper_printf_stdout(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
test_vprintf_stdout(fmt, ap);
va_end(ap);
}
int run_tests(const char *test_prog_name)
{
int num_failed = 0;
int i, j;
helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases,
num_test_cases == 1 ? "" : "s");
test_flush_stdout();
for (i = 0; i != num_tests; ++i) {
if (all_tests[i].num == -1) {
int ret = all_tests[i].test_fn();
if (!ret) {
helper_printf_stdout("** %s failed **\n--------\n",
all_tests[i].test_case_name);
test_flush_stdout();
++num_failed;
}
finalize(ret);
} else {
for (j = 0; j < all_tests[i].num; j++) {
int ret = all_tests[i].param_test_fn(j);
if (!ret) {
helper_printf_stdout("** %s failed test %d\n--------\n",
all_tests[i].test_case_name, j);
test_flush_stdout();
++num_failed;
}
finalize(ret);
}
}
}
if (num_failed != 0) {
helper_printf_stdout("%s: %d test%s failed (out of %d)\n",
test_prog_name, num_failed,
num_failed != 1 ? "s" : "", num_test_cases);
test_flush_stdout();
return EXIT_FAILURE;
}
helper_printf_stdout(" All tests passed.\n");
test_flush_stdout();
return EXIT_SUCCESS;
}

8
test/test_main_custom.c → test/testutil/main.c
View file

@ -1,5 +1,5 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
* Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
@ -7,11 +7,7 @@
* https://www.openssl.org/source/license.html
*/
#include "test_main_custom.h"
#include "testutil.h"
#include <stdio.h>
#include <stdlib.h>
#include "../testutil.h"
int main(int argc, char *argv[])
{

15
test/test_main.c → test/testutil/test_main.c
View file

@ -1,5 +1,5 @@
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
* Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
@ -7,20 +7,15 @@
* https://www.openssl.org/source/license.html
*/
#include "test_main.h"
#include "testutil.h"
#include "../testutil.h"
#include <stdio.h>
int main(int argc, char *argv[])
int test_main(int argc, char *argv[])
{
int ret;
if (argc > 1)
printf("Warning: ignoring extra command-line arguments.\n");
test_puts_stderr("Warning: ignoring extra command-line arguments.\n");
setup_test();
register_tests();
ret = run_tests(argv[0]);
return finish_test(ret);
return run_tests(argv[0]);
}

177
test/testutil.c → test/testutil/tests.c
View file

@ -1,5 +1,5 @@
/*
* Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
* Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
@ -7,152 +7,14 @@
* https://www.openssl.org/source/license.html
*/
#include "testutil.h"
#include "../testutil.h"
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "e_os.h"
#include <openssl/opensslconf.h>
#include <openssl/crypto.h>
#include <openssl/err.h>
#include "../../e_os.h"
/* The size of memory buffers to display on failure */
#define MEM_BUFFER_SIZE (21)
/*
* Declares the structures needed to register each test case function.
*/
typedef struct test_info {
const char *test_case_name;
int (*test_fn) ();
int (*param_test_fn)(int idx);
int num;
} TEST_INFO;
static TEST_INFO all_tests[1024];
static int num_tests = 0;
/*
* A parameterised tests runs a loop of test cases.
* |num_test_cases| counts the total number of test cases
* across all tests.
*/
static int num_test_cases = 0;
void add_test(const char *test_case_name, int (*test_fn) ())
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
all_tests[num_tests].test_fn = test_fn;
all_tests[num_tests].num = -1;
++num_test_cases;
++num_tests;
}
void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
int num)
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
all_tests[num_tests].param_test_fn = test_fn;
all_tests[num_tests].num = num;
++num_tests;
num_test_cases += num;
}
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
static int should_report_leaks()
{
/*
* When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
* can be used to disable leak checking at runtime.
* Note this only works when running the test binary manually;
* the test harness always enables OPENSSL_DEBUG_MEMORY.
*/
char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
return mem_debug_env == NULL
|| (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
}
#endif
void setup_test()
{
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks()) {
CRYPTO_set_mem_debug(1);
CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
}
#endif
}
int finish_test(int ret)
{
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks() && CRYPTO_mem_leaks_fp(stderr) <= 0)
return EXIT_FAILURE;
#endif
return ret;
}
static void finalize(int success)
{
if (success)
ERR_clear_error();
else
ERR_print_errors_fp(stderr);
}
int run_tests(const char *test_prog_name)
{
int num_failed = 0;
int i, j;
printf("%s: %d test case%s\n", test_prog_name, num_test_cases,
num_test_cases == 1 ? "" : "s");
fflush(stdout);
for (i = 0; i != num_tests; ++i) {
if (all_tests[i].num == -1) {
int ret = all_tests[i].test_fn();
if (!ret) {
printf("** %s failed **\n--------\n",
all_tests[i].test_case_name);
fflush(stdout);
++num_failed;
}
finalize(ret);
} else {
for (j = 0; j < all_tests[i].num; j++) {
int ret = all_tests[i].param_test_fn(j);
if (!ret) {
printf("** %s failed test %d\n--------\n",
all_tests[i].test_case_name, j);
fflush(stdout);
++num_failed;
}
finalize(ret);
}
}
}
if (num_failed != 0) {
printf("%s: %d test%s failed (out of %d)\n", test_prog_name,
num_failed, num_failed != 1 ? "s" : "", num_test_cases);
fflush(stdout);
return EXIT_FAILURE;
}
printf(" All tests passed.\n");
fflush(stdout);
return EXIT_SUCCESS;
}
/*
* A common routine to output test failure messages. Generally this should not
* be called directly, rather it should be called by the following functions.
@ -182,27 +44,38 @@ static void test_fail_message(const char *prefix, const char *file, int line,
const char *type, const char *fmt, ...)
PRINTF_FORMAT(5, 6);
static void helper_printf_stderr(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
test_vprintf_stderr(fmt, ap);
va_end(ap);
}
static void test_fail_message_va(const char *prefix, const char *file, int line,
const char *type, const char *fmt, va_list ap)
{
fputs(prefix != NULL ? prefix : "ERROR", stderr);
fputs(":", stderr);
test_puts_stderr(prefix != NULL ? prefix : "ERROR");
test_puts_stderr(":");
if (type)
fprintf(stderr, " (%s)", type);
helper_printf_stderr(" (%s)", type);
if (fmt != NULL) {
fputc(' ', stderr);
vfprintf(stderr, fmt, ap);
test_puts_stderr(" ");
test_vprintf_stderr(fmt, ap);
}
if (file != NULL) {
fprintf(stderr, " @ %s:%d", file, line);
helper_printf_stderr(" @ %s:%d", file, line);
}
fputc('\n', stderr);
test_puts_stderr("\n");
test_flush_stderr();
}
static void test_fail_message(const char *prefix, const char *file, int line,
const char *type, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
test_fail_message_va(prefix, file, line, type, fmt, ap);
va_end(ap);
@ -293,7 +166,7 @@ DEFINE_COMPARISONS(char, char, "%c")
DEFINE_COMPARISONS(unsigned char, uchar, "%u")
DEFINE_COMPARISONS(long, long, "%ld")
DEFINE_COMPARISONS(unsigned long, ulong, "%lu")
DEFINE_COMPARISONS(size_t, size_t, "%" OSSLzu)
DEFINE_COMPARISONS(size_t, size_t, "%zu")
DEFINE_COMPARISON(void *, ptr, eq, ==, "%p")
DEFINE_COMPARISON(void *, ptr, ne, !=, "%p")
@ -434,14 +307,14 @@ int test_mem_eq(const char *file, int line, const char *st1, const char *st2,
return 1;
if (n1 != n2) {
test_fail_message(NULL, file, line, "memory",
"size mismatch %s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
"size mismatch %s %s [%zu] != %s %s [%zu]",
st1, print_mem_maybe_null(s1, n1, b1), n1,
st2, print_mem_maybe_null(s2, n2, b2), n2);
return 0;
}
if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) {
test_fail_message(NULL, file, line, "memory",
"%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
"%s %s [%zu] != %s %s [%zu]",
st1, print_mem_maybe_null(s1, n1, b1), n1,
st2, print_mem_maybe_null(s2, n2, b2), n2);
return 0;
@ -460,7 +333,7 @@ int test_mem_ne(const char *file, int line, const char *st1, const char *st2,
return 1;
if (s1 == NULL || memcmp(s1, s2, n1) == 0) {
test_fail_message(NULL, file, line, "memory",
"%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
"%s %s [%zu] != %s %s [%zu]",
st1, print_mem_maybe_null(s1, n1, b1), n1,
st2, print_mem_maybe_null(s2, n2, b2), n2);
return 0;