Move strict gcc compilation flags to configure.ac to avoid breaking incompatible environments

- Move gcc hardening/warning/advanced warnings flags to configure.ac to avoid
  breaking incompatible environments
- Use -Warith-conversion only with gcc 10 and later
- Keep -Wundef -Wpedantic enabled globally

Signed-off-by: Loïc Gomez <bip@animanova.fr>

configure.ac

@@ -94,6 +94,57 @@ if test "$ap_cv_cc_pie" = "yes"; then
 	enable_pie=yes
 fi
 
+
+AC_CACHE_CHECK([whether $CC accepts hardening flags], [ap_cv_cc_hardening], [
+	save_CFLAGS=$CFLAGS
+	save_LDFLAGS=$LDFLAGS
+	CFLAGS="$CFLAGS -O2 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wl,-z,separate-code"
+	AC_RUN_IFELSE([AC_LANG_SOURCE([[static int foo[30000]; int main () { return 0; }]])],
+		[ap_cv_cc_hardening=yes],
+		[ap_cv_cc_hardening=no],
+		[ap_cv_cc_hardening=yes]
+	)
+	CFLAGS=$save_CFLAGS
+])
+if test "$ap_cv_cc_hardening" = "yes"; then
+	CFLAGS="$CFLAGS -O2 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wl,-z,separate-code"
+	enable_cc_hardening=yes
+fi
+
+AC_CACHE_CHECK([whether $CC accepts some warning flags], [ap_cv_cc_warnings], [
+	save_CFLAGS=$CFLAGS
+	save_LDFLAGS=$LDFLAGS
+	CFLAGS="$CFLAGS -Wformat-overflow=2 -Wformat-truncation=2 -Wtrampolines -Warray-bounds=2 -Wimplicit-fallthrough=3 -Wtraditional-conversion -Wshift-overflow=2 -Wstringop-overflow=4 -Wlogical-op -Wduplicated-cond -Wduplicated-branches -Wformat-signedness -Wstack-usage=1000000 -Wcast-align=strict"
+	AC_RUN_IFELSE([AC_LANG_SOURCE([[static int foo[30000]; int main () { return 0; }]])],
+		[ap_cv_cc_warnings=yes],
+		[ap_cv_cc_warnings=no],
+		[ap_cv_cc_warnings=yes]
+	)
+	CFLAGS=$save_CFLAGS
+])
+if test "$ap_cv_cc_warnings" = "yes"; then
+	CFLAGS="$CFLAGS -Wformat-overflow=2 -Wformat-truncation=2 -Wtrampolines -Warray-bounds=2 -Wimplicit-fallthrough=3 -Wtraditional-conversion -Wshift-overflow=2 -Wstringop-overflow=4 -Wlogical-op -Wduplicated-cond -Wduplicated-branches -Wformat-signedness -Wstack-usage=1000000 -Wcast-align=strict"
+	enable_cc_warnings=yes
+fi
+
+AC_CACHE_CHECK([whether $CC accepts some supplementary warning flags], [ap_cv_cc_warnings2], [
+	save_CFLAGS=$CFLAGS
+	save_LDFLAGS=$LDFLAGS
+	CFLAGS="$CFLAGS -Wformat=2 -Wformat-security -Wnull-dereference -Wstack-protector -Walloca -Wvla -Wcast-qual -Wconversion -Wshadow -Wstrict-overflow=4 -Wstrict-prototypes -Wswitch-default -Wswitch-enum"
+	AC_RUN_IFELSE([AC_LANG_SOURCE([[static int foo[30000]; int main () { return 0; }]])],
+		[ap_cv_cc_warnings2=yes],
+		[ap_cv_cc_warnings2=no],
+		[ap_cv_cc_warnings2=yes]
+	)
+	CFLAGS=$save_CFLAGS
+])
+if test "$ap_cv_cc_warnings2" = "yes"; then
+	CFLAGS="$CFLAGS -Wformat=2 -Wformat-security -Wnull-dereference -Wstack-protector -Walloca -Wvla -Wcast-qual -Wconversion -Wshadow -Wstrict-overflow=4 -Wstrict-prototypes -Wswitch-default -Wswitch-enum"
+	enable_cc_warnings2=yes
+fi
+
+
+
 AC_CACHE_CHECK([whether $CC accepts -Warith-conversion flag], [ap_cv_cc_warith], [
 	save_CFLAGS=$CFLAGS
 	save_LDFLAGS=$LDFLAGS

src/Makefile.am

@@ -36,9 +36,4 @@ bipmkpw_LDADD = libbip.a libbiplex.a $(OPENSSL_LIBS)
 AM_YFLAGS= -d
 BUILT_SOURCES = conf.c conf.h lex.c
 
-AM_CFLAGS=-Wall -Wextra -Werror \
+AM_CFLAGS=-Wall -Wextra -Werror -Wundef -Wpedantic
-	-O2 \
-	-D_FORTIFY_SOURCE=2 \
-	-fstack-protector-strong -fstack-clash-protection \
-	-Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wl,-z,separate-code \
-	-Wpedantic -Wformat=2 -Wformat-overflow=2 -Wformat-truncation=2 -Wformat-security -Wnull-dereference -Wstack-protector -Wtrampolines -Walloca -Wvla -Warray-bounds=2 -Wimplicit-fallthrough=3 -Wtraditional-conversion -Wshift-overflow=2 -Wcast-qual -Wstringop-overflow=4 -Wconversion -Wlogical-op -Wduplicated-cond -Wduplicated-branches -Wformat-signedness -Wshadow -Wstrict-overflow=4 -Wundef -Wstrict-prototypes -Wswitch-default -Wswitch-enum -Wstack-usage=1000000 -Wcast-align=strict