[gcc(refs/users/aoliva/heads/testme)] WIP fix -fstrub={all,at-calls}

[gcc(refs/users/aoliva/heads/testme)] WIP fix -fstrub={all,at-calls}

[Alexandre Oliva]

https://gcc.gnu.org/g:4252b60901c7323900eb3e1cbfb441c6efb09d5a

commit 4252b60901c7323900eb3e1cbfb441c6efb09d5a
Author: Alexandre Oliva <oliva@adacore.com>
Date:   Wed Nov 10 00:24:30 2021 -0300

    WIP fix -fstrub={all,at-calls}

Diff:
---
 gcc/ipa-strub.cc | 134 +++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 110 insertions(+), 24 deletions(-)

diff --git a/gcc/ipa-strub.cc b/gcc/ipa-strub.cc
index 43b2dd80d5d..c8122a8b9e2 100644
--- a/gcc/ipa-strub.cc
+++ b/gcc/ipa-strub.cc
@@ -1317,6 +1317,104 @@ strub_comptypes (tree t1, tree t2)
   return 2;
 }
 
+/* Return true if CALL uses a function type different from that of the
+   callee.  ??? This cannot tell when a function has a non-explicit
+   strub mode, and it's converted to a type that does not have a strub
+   mode.  This should never happen, since such conversions necessarily
+   involve address-taking, which makes transformations that would add
+   type-changing modes non-viable.  */
+
+static bool
+call_typecast_p (gcall *call)
+{
+  tree fntype = gimple_call_fntype (call);
+  tree fndecl = gimple_call_fndecl (call);
+  tree fn_type = TREE_TYPE (fndecl
+			    ? fndecl
+			    : TREE_TYPE (gimple_call_fn (call)));
+
+  return fntype && TYPE_MAIN_VARIANT (fntype) != TYPE_MAIN_VARIANT (fn_type);
+}
+
+/* Return the effective strub mode used for CALL, and set *TYPEP to
+   the effective type used for the call.  The effective type and mode
+   are those of the callee, unless the call involves a typecast.  */
+
+static enum strub_mode
+effective_strub_mode_for_call (gcall *call, tree *typep)
+{
+  tree type;
+  enum strub_mode mode;
+
+  if (call_typecast_p (call))
+    {
+      type = gimple_call_fntype (call);
+      mode = get_strub_mode_from_type (type);
+    }
+  else
+    {
+      type = TREE_TYPE (TREE_TYPE (gimple_call_fn (call)));
+      tree decl = gimple_call_fndecl (call);
+      if (decl)
+	mode = get_strub_mode_from_fndecl (decl);
+      else
+	mode = get_strub_mode_from_type (type);
+    }
+
+  if (typep)
+    *typep = type;
+
+  return mode;
+}
+
+/* Create a distinct copy of the type of NODE's function, and change
+   the fntype of all calls to it with the same main type to the new
+   type.  */
+
+static void
+distinctify_node_type (cgraph_node *node)
+{
+  tree old_type = TREE_TYPE (node->decl);
+  tree new_type = build_distinct_type_copy (old_type);
+  tree new_ptr_type = NULL_TREE;
+
+  /* Remap any calls to node->decl that use old_type, or a variant
+     thereof, to new_type as well.  We don't look for aliases, their
+     declarations will have their types changed independently, and
+     we'll adjust their fntypes then.  */
+  for (cgraph_edge *e = node->callers; e; e = e->next_caller)
+    {
+      tree fnaddr = gimple_call_fn (e->call_stmt);
+      gcc_checking_assert (TREE_CODE (fnaddr) == ADDR_EXPR
+			   && TREE_OPERAND (fnaddr, 0) == node->decl);
+      if (!new_ptr_type)
+	new_ptr_type = build_pointer_type (new_type);
+      TREE_TYPE (fnaddr) = new_ptr_type;
+      if (call_typecast_p (e->call_stmt))
+	continue;
+      gimple_call_set_fntype (e->call_stmt, new_type);
+    }
+
+  TREE_TYPE (node->decl) = new_type;
+}
+
+/* Return TRUE if TYPE and any variants have the same strub mode,
+   and false otherwise.  */
+
+static bool
+same_strub_mode_in_variants_p (tree type)
+{
+  for (tree other = TYPE_MAIN_VARIANT (type);
+       other != NULL_TREE; other = TYPE_NEXT_VARIANT (type))
+    if (other == type)
+      continue;
+    else if (get_strub_mode_from_type (type)
+	     != get_strub_mode_from_type (TYPE_MAIN_VARIANT (type)))
+      return false;
+
+  return true;
+}
+
 /* Check that strub functions don't call non-strub functions, and that
    always_inline strub functions are only called by strub
    functions.  */
@@ -1341,9 +1439,8 @@ verify_strub ()
       {
 	gcc_checking_assert (e->indirect_unknown_callee);
 
-	tree callee_fntype = gimple_call_fntype (e->call_stmt);
 	enum strub_mode callee_mode
-	  = get_strub_mode_from_type (callee_fntype);
+	  = effective_strub_mode_for_call (e->call_stmt, NULL);
 
 	if (!strub_callable_from_p (caller_mode, callee_mode))
 	  error_at (gimple_location (e->call_stmt),
@@ -1355,17 +1452,9 @@ verify_strub ()
       {
 	gcc_checking_assert (!e->indirect_unknown_callee);
 
-	tree callee_fntype = gimple_call_fntype (e->call_stmt);
-	/* fntype is NULL for internal calls, and builtins may carry a
-	   variant type.  */
-	bool use_callee_mode_p
-	  = (!callee_fntype
-	     || (TYPE_MAIN_VARIANT (TREE_TYPE (e->callee->decl))
-		 == TYPE_MAIN_VARIANT (callee_fntype)));
-	strub_mode callee_mode
-	  = (use_callee_mode_p
-	     ? get_strub_mode (e->callee)
-	     : get_strub_mode_from_type (callee_fntype));
+	tree callee_fntype;
+	enum strub_mode callee_mode
+	  = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
 
 	if (!strub_callable_from_p (caller_mode, callee_mode))
 	  {
@@ -1379,7 +1468,7 @@ verify_strub ()
 	      /* This is ok, it will be kept in the STRUB_WRAPPER, and removed
 		 from the STRUB_WRAPPED's strub context.  */
 	      continue;
-	    else if (use_callee_mode_p)
+	    else if (!call_typecast_p (e->call_stmt))
 	      error_at (gimple_location (e->call_stmt),
 			"calling non-%<strub%> %qD in %<strub%> context %qD",
 			e->callee->decl, node->decl);
@@ -2056,6 +2145,8 @@ pass_ipa_strub::adjust_at_calls_type (tree type)
 {
   int named_args = 0;
 
+  gcc_checking_assert (same_strub_mode_in_variants_p (type));
+
   if (!TYPE_ARG_TYPES (type))
     return named_args;
 
@@ -2312,9 +2403,9 @@ pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node)
 	{
 	  gcc_checking_assert (e->indirect_unknown_callee);
 
-	  tree callee_fntype = gimple_call_fntype (e->call_stmt);
+	  tree callee_fntype;
 	  enum strub_mode callee_mode
-	    = get_strub_mode_from_type (callee_fntype);
+	    = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
 
 	  if (callee_mode != STRUB_AT_CALLS
 	      && callee_mode != STRUB_AT_CALLS_OPT)
@@ -2334,13 +2425,9 @@ pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node)
 	{
 	  gcc_checking_assert (!e->indirect_unknown_callee);
 
-	  tree callee_fntype = gimple_call_fntype (e->call_stmt);
+	  tree callee_fntype;
 	  enum strub_mode callee_mode
-	    = (!callee_fntype
-	       || (TYPE_MAIN_VARIANT (callee_fntype)
-		   != TYPE_MAIN_VARIANT (TREE_TYPE (e->callee->decl)))
-	       ? get_strub_mode (e->callee)
-	       : get_strub_mode_from_type (callee_fntype));
+	    = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
 
 	  if (callee_mode != STRUB_AT_CALLS
 	      && callee_mode != STRUB_AT_CALLS_OPT)
@@ -2401,8 +2488,7 @@ pass_ipa_strub::execute (function *)
 	/* Create a type variant if strubbing was not explicitly requested in
 	   the function type.  */
 	if (get_strub_mode_from_type (TREE_TYPE (onode->decl)) != mode)
-	  TREE_TYPE (onode->decl) = build_distinct_type_copy (TREE_TYPE
-							      (onode->decl));
+	  distinctify_node_type (onode);
 
 	int named_args = adjust_at_calls_type (TREE_TYPE (onode->decl));