diff --git a/crypto/err/err.c b/crypto/err/err.c
index 196f782b17..9eb477ccda 100644
--- a/crypto/err/err.c
+++ b/crypto/err/err.c
@@ -850,32 +850,41 @@ void ERR_add_error_data(int num, ...)
 
 void ERR_add_error_vdata(int num, va_list args)
 {
-    int i, n, s;
-    char *str, *p, *a;
+    int i, len, size;
+    char *str, *p, *arg;
+    ERR_STATE *es;
 
-    s = 80;
-    if ((str = OPENSSL_malloc(s + 1)) == NULL) {
+    /* Get the current error data; if an allocated string get it. */
+    es = ERR_get_state();
+    if (es == NULL)
+        return;
+    i = es->top;
+    p = es->err_data_flags[i] == (ERR_TXT_MALLOCED | ERR_TXT_STRING)
+            ? es->err_data[i] : "";
+
+    /* Start with initial (or empty) string and allocate a new buffer */
+    size = 80 + strlen(p);
+    if ((str = OPENSSL_malloc(size + 1)) == NULL) {
         /* ERRerr(ERR_F_ERR_ADD_ERROR_VDATA, ERR_R_MALLOC_FAILURE); */
         return;
     }
-    str[0] = '\0';
+    strcpy(str, p);
 
-    n = 0;
-    for (i = 0; i < num; i++) {
-        a = va_arg(args, char *);
-        if (a == NULL)
-            a = "<NULL>";
-        n += strlen(a);
-        if (n > s) {
-            s = n + 20;
-            p = OPENSSL_realloc(str, s + 1);
+    for (len = 0; --num >= 0; ) {
+        arg = va_arg(args, char *);
+        if (arg == NULL)
+            arg = "<NULL>";
+        len += strlen(arg);
+        if (len > size) {
+            size = len + 20;
+            p = OPENSSL_realloc(str, size + 1);
             if (p == NULL) {
                 OPENSSL_free(str);
                 return;
             }
             str = p;
         }
-        OPENSSL_strlcat(str, a, (size_t)s + 1);
+        OPENSSL_strlcat(str, arg, (size_t)size + 1);
     }
     if (!err_set_error_data_int(str, ERR_TXT_MALLOCED | ERR_TXT_STRING))
         OPENSSL_free(str);
diff --git a/doc/man3/ERR_put_error.pod b/doc/man3/ERR_put_error.pod
index abc9805e6a..31a0e76f38 100644
--- a/doc/man3/ERR_put_error.pod
+++ b/doc/man3/ERR_put_error.pod
@@ -23,6 +23,7 @@ This function is usually called by a macro.
 ERR_add_error_data() associates the concatenation of its B<num> string
 arguments with the error code added last.
 ERR_add_error_vdata() is similar except the argument is a B<va_list>.
+Multiple calls to these functions append to the current top of the error queue.
 
 L<ERR_load_strings(3)> can be used to register
 error strings so that the application can a generate human-readable
diff --git a/providers/fips/fipsprov.c b/providers/fips/fipsprov.c
index 9f9b4289ac..a30ece8e27 100644
--- a/providers/fips/fipsprov.c
+++ b/providers/fips/fipsprov.c
@@ -326,11 +326,13 @@ void ERR_put_error(int lib, int func, int reason, const char *file, int line)
      * so we'll need to come up with something else for them.
      */
     c_put_error(lib, func, reason, file, line);
+    ERR_add_error_data(1, "(in the FIPS module)");
 }
 
 void ERR_add_error_data(int num, ...)
 {
     va_list args;
+
     va_start(args, num);
     ERR_add_error_vdata(num, args);
     va_end(args);
diff --git a/test/errtest.c b/test/errtest.c
index 39017da8e4..df222da6f6 100644
--- a/test/errtest.c
+++ b/test/errtest.c
@@ -32,8 +32,21 @@ static int preserves_system_error(void)
 #endif
 }
 
+/* Test that calls to ERR_add_error_[v]data append */
+static int vdata_appends(void)
+{
+    const char *data;
+
+    CRYPTOerr(0, ERR_R_MALLOC_FAILURE);
+    ERR_add_error_data(1, "hello ");
+    ERR_add_error_data(1, "world");
+    ERR_get_error_line_data(NULL, NULL, &data, NULL);
+    return TEST_str_eq(data, "hello world");
+}
+
 int setup_tests(void)
 {
     ADD_TEST(preserves_system_error);
+    ADD_TEST(vdata_appends);
     return 1;
 }