Introduce a no-pinshared option

This option prevents OpenSSL from pinning itself in memory.

Fixes #7598

[extended tests]

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7983)

Configurations/10-main.conf

@@ -651,7 +651,7 @@ my %targets = (
         dso_scheme       => "dlfcn",
         shared_target    => "linux-shared",
         shared_cflag     => "-fPIC",
-        shared_ldflag    => "-Wl,-znodelete",
+        shared_ldflag    => sub { $disabled{pinshared} ? () : "-Wl,-znodelete" },
         shared_extension => ".so.\$(SHLIB_VERSION_NUMBER)",
         enable           => [ "afalgeng" ],
     },

Configure

@@ -374,6 +374,7 @@ my @disablables = (
     "msan",
     "multiblock",
     "nextprotoneg",
+    "pinshared",
     "ocb",
     "ocsp",
     "pic",

INSTALL

@@ -407,6 +407,24 @@
   no-pic
                    Don't build with support for Position Independent Code.
 
+  no-pinshared     By default OpenSSL will attempt to stay in memory until the
+                   process exits. This is so that libcrypto and libssl can be
+                   properly cleaned up automatically via an "atexit()" handler.
+                   The handler is registered by libcrypto and cleans up both
+                   libraries. On some platforms the atexit() handler will run on
+                   unload of libcrypto (if it has been dynamically loaded)
+                   rather than at process exit. This option can be used to stop
+                   OpenSSL from attempting to stay in memory until the process
+                   exits. This could lead to crashes if either libcrypto or
+                   libssl have already been unloaded at the point
+                   that the atexit handler is invoked, e.g. on a platform which
+                   calls atexit() on unload of the library, and libssl is
+                   unloaded before libcrypto then a crash is likely to happen.
+                   Applications can suppress running of the atexit() handler at
+                   run time by using the OPENSSL_INIT_NO_ATEXIT option to
+                   OPENSSL_init_crypto(). See the man page for it for further
+                   details.
+
   no-posix-io
                    Don't use POSIX IO capabilities.
 

crypto/init.c

@@ -147,7 +147,9 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_nodelete)
 #ifdef OPENSSL_INIT_DEBUG
     fprintf(stderr, "OPENSSL_INIT: ossl_init_load_crypto_nodelete()\n");
 #endif
-#if !defined(OPENSSL_NO_DSO) && !defined(OPENSSL_USE_NODELETE)
+#if !defined(OPENSSL_NO_DSO) \
+    && !defined(OPENSSL_USE_NODELETE) \
+    && !defined(OPENSSL_NO_PINSHARED)
 # ifdef DSO_WIN32
     {
         HMODULE handle = NULL;
@@ -735,7 +737,9 @@ int OPENSSL_atexit(void (*handler)(void))
 {
     OPENSSL_INIT_STOP *newhand;
 
-#if !defined(OPENSSL_NO_DSO) && !defined(OPENSSL_USE_NODELETE)
+#if !defined(OPENSSL_NO_DSO) \
+    && !defined(OPENSSL_USE_NODELETE)\
+    && !defined(OPENSSL_NO_PINSHARED)
     {
         union {
             void *sym;

test/shlibloadtest.c

@@ -102,6 +102,8 @@ static int shlib_close(SHLIB lib)
 
 #if defined(DSO_DLFCN) || defined(DSO_WIN32)
 
+static int atexit_handler_done = 0;
+
 static void atexit_handler(void)
 {
     FILE *atexit_file = fopen(path_atexit, "w");
@@ -111,6 +113,7 @@ static void atexit_handler(void)
 
     fprintf(atexit_file, "atexit() run\n");
     fclose(atexit_file);
+    atexit_handler_done++;
 }
 
 static int test_lib(void)
@@ -250,34 +253,35 @@ static int test_lib(void)
 # endif /* DSO_DLFCN */
     }
 
-    switch (test_type) {
-    case JUST_CRYPTO:
-    case DSO_REFTEST:
-    case NO_ATEXIT:
-    case CRYPTO_FIRST:
-        if (!shlib_close(cryptolib)) {
-            fprintf(stderr, "Failed to close libcrypto\n");
-            goto end;
-        }
-        if (test_type != CRYPTO_FIRST)
-            break;
-        /* Fall through */
+    if (!shlib_close(cryptolib)) {
+        fprintf(stderr, "Failed to close libcrypto\n");
+        goto end;
+    }
 
-    case SSL_FIRST:
-        if (test_type == CRYPTO_FIRST && !shlib_close(ssllib)) {
+    if (test_type == CRYPTO_FIRST || test_type == SSL_FIRST) {
+        if (!shlib_close(ssllib)) {
             fprintf(stderr, "Failed to close libssl\n");
             goto end;
         }
-        if (test_type != SSL_FIRST)
-            break;
-
-        if (!shlib_close(cryptolib)) {
-            fprintf(stderr, "Failed to close libcrypto\n");
-            goto end;
-        }
-        break;
     }
 
+# if defined(OPENSSL_NO_PINSHARED) \
+    && defined(__GLIBC__) \
+    && defined(__GLIBC_PREREQ) \
+    && defined(OPENSSL_SYS_LINUX)
+#  if __GLIBC_PREREQ(2, 3)
+    /*
+     * If we didn't pin the so then we are hopefully on a platform that supports
+     * running atexit() on so unload. If not we might crash. We know this is
+     * true on linux since glibc 2.2.3
+     */
+    if (test_type != NO_ATEXIT && atexit_handler_done != 1) {
+        fprintf(stderr, "atexit() handler did not run\n");
+        goto end;
+    }
+#  endif
+# endif
+
     result = 1;
 end:
     return result;