build: Detect openssl library/header mismatch.

This catches the case where openssl-1.0 headers are used for compiling
but openssl-1.1 libraries for linking. Without the check that is added
by this commit the compilation succeeds in this case but the executable
segfaults on the attempt to modify the RSA structure directly. With
the new check, configure fails gracefully.

This happened on a FreeBSD system where the openssl-1.0 headers were
installed in /usr/local and  openssl-1.1 headers in /usr.

diff --git a/configure.ac b/configure.ac
index 053996d1606b13a1ff3da085d59a0f1451f065af..54f9fe1cfec169fa544d49a673860e2b9c207ca3 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -96,8 +96,14 @@ AC_CHECK_HEADER(openssl/ssl.h, [], [HAVE_OPENSSL=no])
 AC_CHECK_LIB([crypto], [RAND_bytes], [], [HAVE_OPENSSL=no])
 LIB_SUBST_FLAGS(openssl)
 if test $HAVE_OPENSSL = yes; then
-       AC_CHECK_LIB([crypto], [RSA_set0_key],
-               AC_DEFINE([HAVE_RSA_SET0_KEY], [1], [openssl-1.1]))
+       HAVE_RSA_SET0_KEY=yes
+       AC_CHECK_DECL([RSA_set0_key], [], [], [#include <openssl/rsa.h>])
+       AC_CHECK_LIB([crypto], [RSA_set0_key], [], [])
+       if test "$ac_cv_have_decl_RSA_set0_key" != "$ac_cv_lib_crypto_RSA_set0_key"; then
+               AC_MSG_ERROR([openssl header/library mismatch])
+       fi
+       test "$ac_cv_have_decl_RSA_set0_key" = yes &&
+               AC_DEFINE([HAVE_RSA_SET0_KEY], [1], [openssl >= 1.1])
 fi
 UNSTASH_FLAGS
 ######################################################################### gcrypt