Document the tracing functionality

Co-authored-by: Dr. Matthias St. Pierre <Matthias.St.Pierre@ncp-e.com>

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8198)

doc/man1/openssl.pod

@@ -531,6 +531,74 @@ Read the password from standard input.
 
 =back
 
+=head1 ENVIRONMENT
+
+=over 4
+
+=item B<OPENSSL_TRACE=>I<name,...>
+
+Enable tracing output of OpenSSL library, by name.
+This output will only make sense if you know OpenSSL internals well.
+Also, it might not give you any output at all, depending on how
+OpenSSL was built.
+
+The value is a comma separated list of names, with the following
+available:
+
+=over 4
+
+=item B<TRACE>
+
+The tracing functionality.
+
+=item B<TLS>
+
+General SSL/TLS.
+
+=item B<TLS_CIPHER>
+
+SSL/TLS cipher.
+
+=item B<ENGINE_CONF>
+
+ENGINE configuration.
+
+=item B<ENGINE_TABLE>
+
+The function that is used by RSA, DSA (etc) code to select registered
+ENGINEs, cache defaults and functional references (etc), will generate
+debugging summaries.
+
+=item B<ENGINE_REF_COUNT>
+
+Reference counts in the ENGINE structure will be monitored with a line
+of generated for each change.
+
+=item B<PKCS5V2>
+
+PKCS#5 v2 keygen.
+
+=item B<PKCS12_KEYGEN>
+
+PKCS#12 key generation.
+
+=item B<PKCS12_DECRYPT>
+
+PKCS#12 decryption.
+
+=item B<X509V3_POLICY>
+
+Generates the complete policy tree at various point during X.509 v3
+policy evaluation.
+
+=item B<BN_CTX>
+
+BIGNUM context.
+
+=back
+
+=back
+
 =head1 SEE ALSO
 
 L<asn1parse(1)>, L<ca(1)>, L<ciphers(1)>, L<cms(1)>, L<config(5)>,

doc/man3/OSSL_trace_enabled.pod

@@ -0,0 +1,183 @@
+=pod
+
+=head1 NAME
+
+OSSL_trace_enabled, OSSL_trace_begin, OSSL_trace_end
+- OpenSSL Tracing API
+
+=head1 SYNOPSIS
+
+ #include <openssl/trace.h>
+
+ int OSSL_trace_enabled(int category);
+
+ BIO *OSSL_trace_begin(int category);
+ void OSSL_trace_end(int category, BIO *channel);
+
+=head1 DESCRIPTION
+
+The functions described here are mainly interesting for those who provide
+OpenSSL functionality, either in OpenSSL itself or in engine modules
+or similar.
+
+If operational (see L</NOTES> below), these functions are used to
+generate free text tracing output.
+
+The tracing output is divided into types which are enabled
+individually by the application.
+The tracing types are described in detail in
+L<OSSL_trace_set_callback(3)/Trace types>.
+The fallback type C<OSSL_TRACE_CATEGORY_ANY> should I<not> be used
+with the functions described here.
+
+=head2 Functions
+
+OSSL_trace_enabled() can be used to check if tracing for the given
+C<category> is enabled.
+
+OSSL_trace_begin() is used to starts a tracing section, and get the
+channel for the given C<category> in form of a BIO.
+This BIO can only be used for output.
+
+OSSL_trace_end() is used to end a tracing section.
+
+Using OSSL_trace_begin() and OSSL_trace_end() to wrap tracing sections
+is I<mandatory>.
+The result of trying to produce tracing output outside of such
+sections is undefined.
+
+=head2 Convenience Macros
+
+There are a number of convenience macros defined, to make tracing
+easy and consistent.
+
+C<OSSL_TRACE_BEGIN(category)> and C<OSSL_TRACE_END(category)> reserve
+the B<BIO> C<trc_out> and are used as follows to wrap a trace section:
+
+ OSSL_TRACE_BEGIN(TLS) {
+
+     BIO_fprintf(trc_out, ... );
+
+ } OSSL_TRACE_END(TLS);
+
+This will normally expands to:
+
+ do {
+     BIO *trc_out = OSSL_trace_begin(OSSL_TRACE_CATEGORY_TLS);
+     if (trc_out != NULL) {
+         ...
+         BIO_fprintf(trc_out, ...);
+     }
+     OSSL_trace_end(OSSL_TRACE_CATEGORY_TLS, trc_out);
+ } while (0);
+
+C<OSSL_TRACE_CANCEL(category)> must be used before returning from or
+jumping out of a trace section:
+
+ OSSL_TRACE_BEGIN(TLS) {
+
+     if (condition) {
+         OSSL_TRACE_CANCEL(TLS);
+         goto err;
+     }
+     BIO_fprintf(trc_out, ... );
+
+ } OSSL_TRACE_END(TLS);
+
+This will normally expand to:
+
+ do {
+     BIO *trc_out = OSSL_trace_begin(OSSL_TRACE_CATEGORY_TLS);
+     if (trc_out != NULL) {
+         if (condition) {
+             OSSL_trace_end(OSSL_TRACE_CATEGORY_TLS, trc_out);
+             goto err;
+         }
+         BIO_fprintf(trc_out, ... );
+     }
+     OSSL_trace_end(OSSL_TRACE_CATEGORY_TLS, trc_out);
+ } while (0);
+
+=head1 NOTES
+
+It is advisable to always check that a trace type is enabled with
+OSSL_trace_enabled() before generating any output, for example:
+
+    if (OSSL_trace_enabled(OSSL_TRACE_CATEGORY_TLS)) {
+        BIO *trace = OSSL_trace_begin(OSSL_TRACE_CATEGORY_TLS);
+        BIO_printf(trace, "FOO %d\n", somevalue);
+        BIO_dump(trace, somememory, somememory_l);
+        OSSL_trace_end(OSSL_TRACE_CATEGORY_TLS, trace);
+    }
+
+=head2 Tracing disabled
+
+The OpenSSL library may be built with tracing disabled, which makes
+everything documented here inoperational.
+
+When the library is built with tracing disabled:
+
+=over 4
+
+=item *
+
+The macro C<OPENSSL_NO_TRACE> is defined in C<openssl/opensslconf.h>.
+
+=item *
+
+all functions are still present, bu OSSL_trace_enabled() will always
+report the categories as disabled, and all other functions will do
+nothing.
+
+=item *
+
+the convenience macros are defined to produce dead code.
+For example, take this example from L</Convenience Macros> above:
+
+ OSSL_TRACE_BEGIN(TLS) {
+
+     if (condition) {
+         OSSL_TRACE_CANCEL(TLS);
+         goto err;
+     }
+     BIO_fprintf(trc_out, ... );
+
+ } OSSL_TRACE_END(TLS);
+
+When the tracing API isn't operational, that will expand to:
+
+ do {
+     BIO *trc_out = NULL;
+     if (0) {
+         if (condition) {
+             ((void)0);
+             goto err;
+         }
+         BIO_fprintf(trc_out, ... );
+     }
+ } while (0);
+
+=back
+
+=head1 RETURN VALUES
+
+OSSL_trace_enabled() returns 1 if tracing for the given B<type> is
+operational and enabled, otherwise 0.
+
+OSSL_trace_begin() returns a C<BIO *> if the given B<type> is enabled,
+otherwise C<NULL>.
+
+=head1 HISTORY
+
+The OpenSSL Tracing API was added ino OpenSSL 3.0.0.
+
+=head1 COPYRIGHT
+
+Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut

doc/man3/OSSL_trace_get_category_num.pod

@@ -0,0 +1,44 @@
+=pod
+
+=head1 NAME
+
+OSSL_trace_get_category_num, OSSL_trace_get_category_name
+- OpenSSL tracing information functions
+
+=head1 SYNOPSIS
+
+ #include <openssl/trace.h>
+
+ int OSSL_trace_get_category_num(const char *name);
+ const char *OSSL_trace_get_category_name(int num);
+
+=head1 DESCRIPTION
+
+OSSL_trace_get_category_num() gives the category number corresponding
+to the given C<name>.
+
+OSSL_trace_get_category_name() gives the category name corresponding
+to the given C<num>.
+
+=head1 RETURN VALUES
+
+OSSL_trace_get_category_num() returns the category number if the given
+C<name> is a recognised category name, otherwise -1.
+
+OSSL_trace_get_category_name() returns the category name if the given
+C<num> is a recognised category number, otherwise NULL.
+
+=head1 HISTORY
+
+The OpenSSL Tracing API was added ino OpenSSL 3.0.0.
+
+=head1 COPYRIGHT
+
+Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut

doc/man3/OSSL_trace_set_channel.pod

@@ -0,0 +1,288 @@
+=pod
+
+=head1 NAME
+
+OSSL_trace_set_channel, OSSL_trace_set_prefix, OSSL_trace_set_suffix,
+OSSL_trace_set_callback, OSSL_trace_cb - Enabling trace output
+
+=head1 SYNOPSIS
+
+ #include <openssl/trace.h>
+
+ typedef size_t (*OSSL_trace_cb)(const char *buf, size_t cnt,
+                                 int category, int cmd, void *data);
+
+ void OSSL_trace_set_channel(int category, BIO *bio);
+ void OSSL_trace_set_prefix(int category, const char *prefix);
+ void OSSL_trace_set_suffix(int category, const char *suffix);
+ void OSSL_trace_set_callback(int category, OSSL_trace_cb cb, void  *data);
+
+=head1 DESCRIPTION
+
+If available (see L</NOTES> below), the application can request
+internal trace output.
+This output comes in form of free text for humans to read.
+
+The trace output is divided into categories which can be
+enabled individually.
+They are enabled by giving them a channel in form of a BIO, or a
+tracer callback, which is responsible for performing the actual
+output.
+
+=head2 Functions
+
+OSSL_trace_set_channel() is used to enable the given trace C<category>
+by giving it the B<BIO> C<bio>.
+
+OSSL_trace_set_prefix() and OSSL_trace_set_suffix() can be used to add
+an extra line for each channel, to be output before and after group of
+tracing output.
+What constitues an output group is decided by the code that produces
+the output.
+The lines given here are considered immutable; for more dynamic
+tracing prefixes, consider setting a callback with
+OSSL_trace_set_callback() instead.
+
+OSSL_trace_set_callback() is used to enable the given trace
+C<category> by giving it the tracer callback C<cb> with the associated
+data C<data>, which will simply be passed through to C<cb> whenever
+it's called.
+This should be used when it's desirable to do form the trace output to
+something suitable for application needs where a prefix and suffix
+line aren't enough.
+
+OSSL_trace_set_channel() and OSSL_trace_set_callback() are mutually
+exclusive, calling one of them will clear whatever was set by the
+previous call.
+
+Calling OSSL_trace_set_channel() with C<NULL> for C<channel> or
+OSSL_trace_set_callback() with C<NULL> for C<cb> disables tracing for
+the given C<category>
+
+=head2 Trace callback
+
+The tracer callback must return a C<size_t>, which must be zero on
+error and otherwise return the number of bytes that were output.
+It receives a text buffer C<buf> with C<cnt> bytes of text, as well as
+the C<category>, a control number C<cmd>, and the C<data> that was
+passed to OSSL_trace_set_callback().
+
+The possible control numbers are:
+
+=over 4
+
+=item C<OSSL_TRACE_CTRL_BEGIN>
+
+The callback is called from OSSL_trace_begin(), which gives the
+callback the possibility to output a dynamic starting line, or set a
+prefix that should be output at the beginning of each line, or
+something other.
+
+=item C<OSSL_TRACE_CTRL_DURING>
+
+The callback is called from any regular BIO output routine.
+
+=item C<OSSL_TRACE_CTRL_END>
+
+The callback is called from OSSL_trace_end(), which gives the callback
+the possibility to output a dynamic ending line, or reset the line
+prefix that was set with OSSL_TRACE_CTRL_BEGIN, or something other.
+
+=back
+
+=head2 Trace categories
+
+The trace categories are simple numbers available through macros.
+
+=over 4
+
+=item C<OSSL_TRACE_CATEGORY_TRACE>
+
+Traces the OpenSSL trace API itself.
+
+More precisely, this will generate trace output any time a new
+trace hook is set.
+
+=item C<OSSL_TRACE_CATEGORY_INIT>
+
+Traces OpenSSL library initialization and cleanup.
+
+This needs special care, as OpenSSL will do automatic cleanup after
+exit from C<main()>, and any tracing output done during this cleanup
+will be lost if the tracing channel or callback were cleaned away
+prematurely.
+A suggestion is to make such cleanup part of a function that's
+registered very early with L<atexit(3)>.
+
+=item C<OSSL_TRACE_CATEGORY_TLS>
+
+Traces the TLS/SSL protocoll.
+
+=item C<OSSL_TRACE_CATEGORY_TLS_CIPHER>
+
+Traces the ciphers used by the TLS/SSL protocoll.
+
+=item C<OSSL_TRACE_CATEGORY_ENGINE_CONF>
+
+Traces the ENGINE configuration.
+
+=item C<OSSL_TRACE_CATEGORY_ENGINE_TABLE>
+
+Traces the ENGINE algorithm table selection.
+
+More precisely, engine_table_select(), the function that is used by
+RSA, DSA (etc) code to select registered ENGINEs, cache defaults and
+functional references (etc), will generate trace summaries.
+
+=item C<OSSL_TRACE_CATEGORY_ENGINE_REF_COUNT>
+
+Tracds the ENGINE reference counting.
+
+More precisely, both reference counts in the ENGINE structure will be
+monitored with a line of trace output generated for each change.
+
+=item C<OSSL_TRACE_CATEGORY_PKCS5V2>
+
+Traces PKCS#5 v2 key generation.
+
+=item C<OSSL_TRACE_CATEGORY_PKCS12_KEYGEN>
+
+Traces PKCS#12 key generation.
+
+=item C<OSSL_TRACE_CATEGORY_PKCS12_DECRYPT>
+
+Traces PKCS#12 decryption.
+
+=item C<OSSL_TRACE_CATEGORY_X509V3_POLICY>
+
+Traces X509v3 policy processing.
+
+More precisely, this generates the complete policy tree at various
+point during evaluation.
+
+=item C<OSSL_TRACE_CATEGORY_BN_CTX>
+
+Traces BIGNUM context operations.
+
+=back
+
+There is also C<OSSL_TRACE_CATEGORY_ANY>, which works as a fallback
+and can be used to get I<all> trace output.
+
+=head1 RETURN VALUES
+
+OSSL_trace_set_channel(), OSSL_trace_set_prefix(),
+OSSL_trace_set_suffix(), and OSSL_trace_set_callback() return 1 on
+success, or 0 on failure.
+
+=head1 EXAMPLES
+
+In all examples below, we assume that the trace producing code is
+this:
+
+ int foo = 42;
+ const char bar[] = { 0,  1,  2,  3,  4,  5,  6,  7,
+                      8,  9, 10, 11, 12, 13, 14, 15 };
+
+ OSSL_TRACE_BEGIN(TLS) {
+     BIO_puts(trc_out, "foo: ");
+     BIO_printf(trc_out, "%d\n", foo);
+     BIO_dump(trc_out, bar, sizeof(bar));
+ } OSSL_TRACE_END(TLS);
+
+=head1 Simple example
+
+An example with just a channel and constant prefix / suffix.
+
+ int main(int argc, char *argv[])
+ {
+     BIO *err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
+     OSSL_trace_set_channel(OSSL_TRACE_CATEGORY_SSL, err);
+     OSSL_trace_set_prefix(OSSL_TRACE_CATEGORY_SSL, "BEGIN TRACE[TLS]");
+     OSSL_trace_set_suffix(OSSL_TRACE_CATEGORY_SSL, "END TRACE[TLS]");
+
+     /* ... work ... */
+ }
+
+When the trace producing code above is performed, this will be output
+on standard error:
+
+ BEGIN TRACE[TLS]
+ foo: 42
+ 0000 - 00 01 02 03 04 05 06 07-08 09 0a 0b 0c 0d 0e 0f   ................
+ END TRACE[TLS]
+
+=head2 Advanced example
+
+This example uses the callback, and depends on pthreads functionality.
+
+ static size_t cb(const char *buf, size_t cnt,
+                 int category, int cmd, void *vdata)
+ {
+     BIO *bio = vdata;
+     const char *label = NULL;
+
+     switch (cmd) {
+     case OSSL_TRACE_CTRL_BEGIN:
+         label = "BEGIN";
+         break;
+     case OSSL_TRACE_CTRL_END:
+         label = "END";
+         break;
+     }
+
+     if (label != NULL) {
+         union {
+             pthread_t tid;
+             unsigned long ltid;
+         } tid;
+
+         tid.tid = pthread_self();
+         BIO_printf(bio, "%s TRACE[%s]:%lx\n",
+                    label, OSSL_trace_get_category_name(category), tid.ltid);
+     }
+     return (size_t)BIO_puts(bio, buf);
+ }
+
+ int main(int argc, char *argv[])
+ {
+     BIO *err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
+     OSSL_trace_set_callback(OSSL_TRACE_CATEGORY_SSL, cb, err);
+
+     /* ... work ... */
+ }
+
+The output is almost the same as for the simple example above.
+
+ BEGIN TRACE[TLS]:7f9eb0193b80
+ foo: 42
+ 0000 - 00 01 02 03 04 05 06 07-08 09 0a 0b 0c 0d 0e 0f   ................
+ END TRACE[TLS]:7f9eb0193b80
+
+=head1 NOTES
+
+=head2 Tracing disabled
+
+The OpenSSL library may be built with tracing disabled, which makes
+everything documented here inoperational.
+
+When the library is built with tracing disabled, the macro
+C<OPENSSL_NO_TRACE> is defined in C<openssl/opensslconf.h> and all
+functions described here are inoperational, i.e. will do nothing.
+
+=head1 HISTORY
+
+OSSL_trace_set_channel(), OSSL_trace_set_prefix(),
+OSSL_trace_set_suffix(), and OSSL_trace_set_callback() were all added
+in OpenSSL 3.0.0.
+
+=head1 COPYRIGHT
+
+Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut

util/private.num

@@ -47,6 +47,7 @@ OSSL_STORE_error_fn                     datatype
 OSSL_STORE_load_fn                      datatype
 OSSL_STORE_open_fn                      datatype
 OSSL_STORE_post_process_info_fn         datatype
+OSSL_trace_cb                           datatype
 PROFESSION_INFO                         datatype
 PROFESSION_INFOS                        datatype
 RAND_DRBG_cleanup_entropy_fn            datatype