From: Andre Noll <maan@tuebingen.mpg.de>
Date: Fri, 21 Dec 2018 15:43:54 +0000 (+0100)
Subject: build: Detect openssl library/header mismatch.
X-Git-Tag: v0.6.3~35^2~1
X-Git-Url: http://git.tuebingen.mpg.de/?p=paraslash.git;a=commitdiff_plain;h=ddecac81e92694005c8577ea7246caeeefe8361b

build: Detect openssl library/header mismatch.

This catches the case where openssl-1.0 headers are used for compiling
but openssl-1.1 libraries for linking. Without the check that is added
by this commit the compilation succeeds in this case but the executable
segfaults on the attempt to modify the RSA structure directly. With
the new check, configure fails gracefully.

This happened on a FreeBSD system where the openssl-1.0 headers were
installed in /usr/local and  openssl-1.1 headers in /usr.
---

diff --git a/configure.ac b/configure.ac
index 053996d1..54f9fe1c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -96,8 +96,14 @@ AC_CHECK_HEADER(openssl/ssl.h, [], [HAVE_OPENSSL=no])
 AC_CHECK_LIB([crypto], [RAND_bytes], [], [HAVE_OPENSSL=no])
 LIB_SUBST_FLAGS(openssl)
 if test $HAVE_OPENSSL = yes; then
-	AC_CHECK_LIB([crypto], [RSA_set0_key],
-		AC_DEFINE([HAVE_RSA_SET0_KEY], [1], [openssl-1.1]))
+	HAVE_RSA_SET0_KEY=yes
+	AC_CHECK_DECL([RSA_set0_key], [], [], [#include <openssl/rsa.h>])
+	AC_CHECK_LIB([crypto], [RSA_set0_key], [], [])
+	if test "$ac_cv_have_decl_RSA_set0_key" != "$ac_cv_lib_crypto_RSA_set0_key"; then
+		AC_MSG_ERROR([openssl header/library mismatch])
+	fi
+	test "$ac_cv_have_decl_RSA_set0_key" = yes &&
+		AC_DEFINE([HAVE_RSA_SET0_KEY], [1], [openssl >= 1.1])
 fi
 UNSTASH_FLAGS
 ######################################################################### gcrypt