Extract hardening CFLAGS to a special $hardenflags variable

This changes the automatic detection of -fstack-protector,
-D_FORTIFY_SOURCE, and -mbranch-protection to write to $hardenflags
instead of $XCFLAGS. The definition of $cflags is changed to
"$hardenflags $orig_cflags $optflags $debugflags $warnflags" to match.

Furthermore, these flags are _prepended_ to $hardenflags, rather than
appended.

The implications of doing this are as follows:

* If a CRuby builder specifies cflags="-mbranch-protection=foobar" at
  the ./configure script, and the configure script detects that
  -mbranch-protection=pac-ret is accepted, then GCC will be invoked as
  "gcc -mbranch-protection=pac-ret -mbranch-protection=foobar". Since
  the last flags take precedence, that means that user-supplied values
  of these flags in $cflags will take priority.
* Likewise, if a CRuby builder explicitly specifies
  "hardenflags=-mbranch-protection=foobar", because we _prepend_ to
  $hardenflags in our autoconf script, we will still invoke GCC as
  "gcc -mbranch-protection=pac-ret -mbranch-protection=foobar".
* If a CRuby builder specifies CFLAGS="..." at the configure line,
  automatic detection of hardening flags is ignored as before.
* C extensions will _also_ be built with hardening flags now as well
  (this was not the case by default before because the detected flags
  went into $XCFLAGS).

Additionally, as part of this work, I changed how the detection of
PAC/BTI in Context.S works. Rather than appending the autodetected
option to ASFLAGS, we simply compile a set of test programs with the
actual CFLAGS in use to determine what PAC/BTI settings were actually
chosen by the builder. Context.S is made aware of these choices through
some custom macros.

The result of this work is that:

* Ruby will continue to choose some sensible defaults for hardening
  options for the C compiler
* Distributors are able to specify CFLAGS that are consistent with their
  distribution and override these defaults
* Context.S will react to whatever -mbranch-protection is actually in
  use, not what was autodetected
* Extensions get built with hardening flags too.

[Bug #20154]
[Bug #20520]

1 files changed, 70 insertions, 11 deletions

diff --git a/configure.ac b/configure.ac
index ac059bf862..d39ad13677 100644
--- a/configure.ac
+++ b/configure.ac
@@ -360,7 +360,7 @@ test -z "$warnflags" ||
 AS_IF([test -z "${CFLAGS+set}"], [
     cflags=`echo " $cflags " | sed "$cflagspat;s/^ *//;s/ *$//"`
     orig_cflags="$cflags"
-    cflags="$cflags "'${optflags} ${debugflags} ${warnflags}'
+    cflags='${hardenflags} '"$cflags "'${optflags} ${debugflags} ${warnflags}'
 ])
 dnl AS_IF([test -z "${CXXFLAGS+set}"], [
 dnl     cxxflags=`echo " $cxxflags " | sed "$cflagspat;s/^ *//;s/ *$//"`
@@ -813,7 +813,7 @@ AS_IF([test "$GCC" = yes], [
 		  [fortify_source=$enableval])
     AS_IF([test "x$fortify_source" != xno], [
         RUBY_TRY_CFLAGS([$optflags -D_FORTIFY_SOURCE=2],
-                        [RUBY_APPEND_OPTION(XCFLAGS, -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2)], [],
+                        [RUBY_PREPEND_OPTION(hardenflags, -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2)], [],
                         [@%:@include <stdio.h>])
     ])
 
@@ -834,20 +834,24 @@ AS_IF([test "$GCC" = yes], [
     AC_MSG_CHECKING([for -fstack-protector])
     AC_MSG_RESULT(["$stack_protector"])
     AS_CASE(["$stack_protector"], [-*], [
-	RUBY_APPEND_OPTION(XCFLAGS, $stack_protector)
-	RUBY_APPEND_OPTION(XLDFLAGS, $stack_protector)
-	RUBY_APPEND_OPTION(LDFLAGS, $stack_protector)
+        RUBY_PREPEND_OPTION(hardenflags, $stack_protector)
+        RUBY_APPEND_OPTION(XLDFLAGS, $stack_protector)
+        RUBY_APPEND_OPTION(LDFLAGS, $stack_protector)
     ])
 
     # aarch64 branch protection
     AS_CASE(["$target_cpu"], [aarch64], [
 	AS_FOR(option, opt, [-mbranch-protection=pac-ret -msign-return-address=all], [
-            RUBY_TRY_CFLAGS(option, [branch_protection=yes], [branch_protection=no])
+            # Try these flags in the _prepended_ position - i.e. we want to try building a program
+            # with CFLAGS="-mbranch-protection=pac-ret $CFLAGS". If the builder has provided different
+            # branch protection flags in CFLAGS, we don't want to overwrite those. We just want to
+            # find some branch protection flags which work if none were provided.
+            RUBY_TRY_CFLAGS_PREPEND(option, [branch_protection=yes], [branch_protection=no])
             AS_IF([test "x$branch_protection" = xyes], [
-                # C compiler and assembler must be consistent for -mbranch-protection
-                # since they both check `__ARM_FEATURE_PAC_DEFAULT` definition.
-                RUBY_APPEND_OPTION(XCFLAGS, option)
-                RUBY_APPEND_OPTION(ASFLAGS, option)
+                # _prepend_ the options to CFLAGS, so that user-provided flags will overwrite them.
+                # These CFLAGS are used during the configure script to compile further test programs;
+                # however, $harden_flags is prepended separately to CFLAGS at the end of the script.
+                RUBY_PREPEND_OPTION(hardenflags, $opt)
                 break
             ])
         ])
@@ -995,6 +999,59 @@ AC_SUBST(incflags, "$INCFLAGS")
 test -z "${ac_env_CFLAGS_set}" -a -n "${cflags+set}" && eval CFLAGS="\"$cflags $ARCH_FLAG\""
 test -z "${ac_env_CXXFLAGS_set}" -a -n "${cxxflags+set}" && eval CXXFLAGS="\"$cxxflags $ARCH_FLAG\""
 
+# The lines above expand out the $cflags/$optflags/$debugflags/$hardenflags variables into the
+# CFLAGS variable. So, at this point, we have a $CFLAGS var with the actual compiler flags we're
+# going to use.
+# That means this is the right time to check what branch protection flags are going to be in use
+# and define appropriate macros for use in Context.S based on this
+AS_CASE(["$target_cpu"], [aarch64], [
+    AC_CACHE_CHECK([whether __ARM_FEATURE_BTI_DEFAULT is defined],
+        rb_cv_aarch64_bti_enabled,
+        AC_COMPILE_IFELSE(
+            [AC_LANG_PROGRAM([[
+                @%:@ifndef __ARM_FEATURE_BTI_DEFAULT
+                @%:@error "__ARM_FEATURE_BTI_DEFAULT not defined"
+                @%:@endif
+            ]])],
+        [rb_cv_aarch64_bti_enabled=yes],
+        [rb_cv_aarch64_bti_enabled=no])
+    )
+    AS_IF([test "$rb_cv_aarch64_bti_enabled" = yes],
+          AC_DEFINE(RUBY_AARCH64_BTI_ENABLED, 1))
+    AC_CACHE_CHECK([whether __ARM_FEATURE_PAC_DEFAULT is defined],
+        rb_cv_aarch64_pac_enabled,
+        AC_COMPILE_IFELSE(
+            [AC_LANG_PROGRAM([[
+                @%:@ifndef __ARM_FEATURE_PAC_DEFAULT
+                @%:@error "__ARM_FEATURE_PAC_DEFAULT not defined"
+                @%:@endif
+            ]])],
+        [rb_cv_aarch64_pac_enabled=yes],
+        [rb_cv_aarch64_pac_enabled=no])
+    )
+    AS_IF([test "$rb_cv_aarch64_pac_enabled" = yes],
+          AC_DEFINE(RUBY_AARCH64_PAC_ENABLED, 1))
+    # Context.S will only ever sign its return address with the A-key; it doesn't support
+    # the B-key at the moment.
+    AS_IF([test "$rb_cv_aarch64_pac_enabled" = yes], [
+        AC_CACHE_CHECK([whether __ARM_FEATURE_PAC_DEFAULT specifies the b-key bit 0x02],
+            rb_cv_aarch64_pac_b_key,
+            AC_COMPILE_IFELSE(
+                [AC_LANG_PROGRAM([[
+                    @%:@ifdef __ARM_FEATURE_PAC_DEFAULT
+                    @%:@if __ARM_FEATURE_PAC_DEFAULT & 0x02
+                    @%:@error "__ARM_FEATURE_PAC_DEFAULT specifies B key"
+                    @%:@endif
+                    @%:@endif
+                ]])],
+            [rb_cv_aarch64_pac_b_key=no],
+            [rb_cv_aarch64_pac_b_key=yes])
+        )
+        AS_IF([test "$rb_cv_aarch64_pac_b_key" = yes],
+            AC_MSG_ERROR(-mbranch-protection flag specified b-key but Ruby's Context.S does not support this yet.))
+    ])
+])
+
 AC_CACHE_CHECK([whether compiler has statement and declarations in expressions],
   rb_cv_have_stmt_and_decl_in_expr,
   [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],[[ __extension__ ({ int a = 0; a; }); ]])],
@@ -4226,12 +4283,13 @@ AS_IF([test "${ARCH_FLAG}"], [
 rb_cv_warnflags=`echo "$rb_cv_warnflags" | sed 's/^ *//;s/ *$//'`
 warnflags="$rb_cv_warnflags"
 AC_SUBST(cppflags)dnl
-AC_SUBST(cflags, ["${orig_cflags:+$orig_cflags }"'${optflags} ${debugflags} ${warnflags}'])dnl
+AC_SUBST(cflags, ['${hardenflags} '"${orig_cflags:+$orig_cflags }"' ${optflags} ${debugflags} ${warnflags}'])dnl
 AC_SUBST(cxxflags)dnl
 AC_SUBST(optflags)dnl
 AC_SUBST(debugflags)dnl
 AC_SUBST(warnflags)dnl
 AC_SUBST(strict_warnflags)dnl
+AC_SUBST(hardenflags)dnl
 AC_SUBST(XCFLAGS)dnl
 AC_SUBST(XLDFLAGS)dnl
 AC_SUBST(EXTLDFLAGS)dnl
@@ -4691,6 +4749,7 @@ config_summary "DLDFLAGS"            "$DLDFLAGS"
 config_summary "optflags"            "$optflags"
 config_summary "debugflags"          "$debugflags"
 config_summary "warnflags"           "$warnflags"
+config_summary "hardenflags"         "$hardenflags"
 config_summary "strip command"       "$STRIP"
 config_summary "install doc"         "$DOCTARGETS"
 config_summary "YJIT support"        "$YJIT_SUPPORT"