[gcc(refs/vendors/redhat/heads/gcc-10-branch)] rs6000: ICE when using an MMA type as a function param or return value [PR96506]

[gcc(refs/vendors/redhat/heads/gcc-10-branch)] rs6000: ICE when using an MMA type as a function param or return value [PR96506]

[Jakub Jelinek]

https://gcc.gnu.org/g:43edad43b2cee17fb7575c40a9dafac3fe79ba1c

commit 43edad43b2cee17fb7575c40a9dafac3fe79ba1c
Author: Peter Bergner <bergner@linux.ibm.com>
Date:   Thu Aug 13 13:40:39 2020 -0500

    rs6000: ICE when using an MMA type as a function param or return value [PR96506]
    
    PR96506 shows a problem where we ICE on illegal usage, namely using MMA
    types for function arguments and return values.  The solution is to flag
    these illegal usages as errors early, before we ICE.
    
    2020-08-13  Peter Bergner  <bergner@linux.ibm.com>
    
    gcc/
            PR target/96506
            * config/rs6000/rs6000-call.c (rs6000_promote_function_mode): Disallow
            MMA types as return values.
            (rs6000_function_arg): Disallow MMA types as function arguments.
    
    gcc/testsuite/
            PR target/96506
            * gcc.target/powerpc/pr96506.c: New test.
    
    (cherry picked from commit 0ad7e730c142ef6cd0ddc1491a89a7f330caa887)

Diff:
---
 gcc/config/rs6000/rs6000-call.c            | 34 ++++++++++++++-
 gcc/testsuite/gcc.target/powerpc/pr96506.c | 66 ++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/gcc/config/rs6000/rs6000-call.c b/gcc/config/rs6000/rs6000-call.c
index 6af7e2d50c6..1b29eadd131 100644
--- a/gcc/config/rs6000/rs6000-call.c
+++ b/gcc/config/rs6000/rs6000-call.c
@@ -6076,8 +6076,30 @@ machine_mode
 rs6000_promote_function_mode (const_tree type ATTRIBUTE_UNUSED,
 			      machine_mode mode,
 			      int *punsignedp ATTRIBUTE_UNUSED,
-			      const_tree, int)
+			      const_tree, int for_return)
 {
+  /* Warning: this is a static local variable and not always NULL!
+     This function is called multiple times for the same function
+     and return value.  PREV_FUNC is used to keep track of the
+     first time we encounter a function's return value in order
+     to not report an error with that return value multiple times.  */
+  static struct function *prev_func = NULL;
+
+  /* We do not allow MMA types being used as return values.  Only report
+     the invalid return value usage the first time we encounter it.  */
+  if (for_return
+      && prev_func != cfun
+      && (mode == POImode || mode == PXImode))
+    {
+      /* Record we have now handled function CFUN, so the next time we
+	 are called, we do not re-report the same error.  */
+      prev_func = cfun;
+      if (TYPE_CANONICAL (type) != NULL_TREE)
+	type = TYPE_CANONICAL (type);
+      error ("invalid use of MMA type %qs as a function return value",
+	     IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))));
+    }
+
   PROMOTE_MODE (mode, *punsignedp, type);
 
   return mode;
@@ -7028,6 +7050,16 @@ rs6000_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
   machine_mode elt_mode;
   int n_elts;
 
+  /* We do not allow MMA types being used as function arguments.  */
+  if (mode == POImode || mode == PXImode)
+    {
+      if (TYPE_CANONICAL (type) != NULL_TREE)
+	type = TYPE_CANONICAL (type);
+      error ("invalid use of MMA operand of type %qs as a function parameter",
+	     IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))));
+      return NULL_RTX;
+    }
+
   /* Return a marker to indicate whether CR1 needs to set or clear the
      bit that V.4 uses to say fp args were passed in registers.
      Assume that we don't need the marker for software floating point,
diff --git a/gcc/testsuite/gcc.target/powerpc/pr96506.c b/gcc/testsuite/gcc.target/powerpc/pr96506.c
new file mode 100644
index 00000000000..b1b40c5a5c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr96506.c
@@ -0,0 +1,66 @@
+/* PR target/96506 */
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
+
+extern void bar0();
+extern void bar1();
+extern void bar2();
+extern void bar3();
+
+typedef __vector_pair vpair_t;
+typedef __vector_quad vquad_t;
+
+/* Verify we flag errors on the following.  */
+
+void
+foo0 (void)
+{
+  __vector_pair v;
+  bar0 (v); /* { dg-error "invalid use of MMA operand of type .__vector_pair. as a function parameter" } */
+}
+
+void
+foo1 (void)
+{
+  vpair_t v;
+  bar1 (v); /* { dg-error "invalid use of MMA operand of type .__vector_pair. as a function parameter" } */
+}
+
+void
+foo2 (void)
+{
+  __vector_quad v;
+  bar2 (v); /* { dg-error "invalid use of MMA operand of type .__vector_quad. as a function parameter" } */
+}
+
+void
+foo3 (void)
+{
+  vquad_t v;
+  bar3 (v); /* { dg-error "invalid use of MMA operand of type .__vector_quad. as a function parameter" } */
+}
+
+__vector_pair
+foo4 (__vector_pair *src) /* { dg-error "invalid use of MMA type .__vector_pair. as a function return value" } */
+{
+  return *src;
+}
+
+vpair_t
+foo5 (vpair_t *src) /* { dg-error "invalid use of MMA type .__vector_pair. as a function return value" } */
+{
+  return *src;
+}
+
+__vector_quad
+foo6 (__vector_quad *src) /* { dg-error "invalid use of MMA type .__vector_quad. as a function return value" } */
+{
+  return *src;
+}
+
+vquad_t
+foo7 (vquad_t *src) /* { dg-error "invalid use of MMA type .__vector_quad. as a function return value" } */
+{
+  return *src;
+}