[gcc r12-1386] simplify-rtx: Fix up simplify_logical_relational_operation for vector IOR [PR101008]

[Jakub Jelinek <jakub@gcc.gnu.org>]

https://gcc.gnu.org/g:4bdcdd8fa8d7659e5a19a930cf2f0332127f8a46

commit r12-1386-g4bdcdd8fa8d7659e5a19a930cf2f0332127f8a46
Author: Jakub Jelinek <jakub@redhat.com>
Date:   Fri Jun 11 12:59:43 2021 +0200

    simplify-rtx: Fix up simplify_logical_relational_operation for vector IOR [PR101008]
    
    simplify_relational_operation callees typically return just const0_rtx
    or const_true_rtx and then simplify_relational_operation attempts to fix
    that up if the comparison result has vector mode, or floating mode,
    or punt if it has scalar mode and vector mode operands (it doesn't know how
    exactly to deal with the scalar masks).
    But, simplify_logical_relational_operation has a special case, where
    it attempts to fold (x < y) | (x >= y) etc. and if it determines it is
    always true, it just returns const_true_rtx, without doing the dances that
    simplify_relational_operation does.
    That results in an ICE on the following testcase, where such folding happens
    during expansion (of debug stmts into DEBUG_INSNs) and we ICE because
    all of sudden a VOIDmode rtx appears where it expects a vector (V4SImode)
    rtx.
    
    The following patch fixes that by moving the adjustement into a separate
    helper routine and using it from both simplify_relational_operation and
    simplify_logical_relational_operation.
    
    2021-06-11  Jakub Jelinek  <jakub@redhat.com>
    
            PR rtl-optimization/101008
            * simplify-rtx.c (relational_result): New function.
            (simplify_logical_relational_operation,
            simplify_relational_operation): Use it.
    
            * gcc.dg/pr101008.c: New test.

Diff:
---
 gcc/simplify-rtx.c              | 95 +++++++++++++++++++++--------------------
 gcc/testsuite/gcc.dg/pr101008.c | 18 ++++++++
 2 files changed, 67 insertions(+), 46 deletions(-)

diff --git a/gcc/simplify-rtx.c b/gcc/simplify-rtx.c
index 04423bbd195..c82101c73a4 100644
--- a/gcc/simplify-rtx.c
+++ b/gcc/simplify-rtx.c
@@ -2294,6 +2294,53 @@ comparison_code_valid_for_mode (enum rtx_code code, enum machine_mode mode)
 	gcc_unreachable ();
     }
 }
+
+/* Canonicalize RES, a scalar const0_rtx/const_true_rtx to the right
+   false/true value of comparison with MODE where comparison operands
+   have CMP_MODE.  */
+
+static rtx
+relational_result (machine_mode mode, machine_mode cmp_mode, rtx res)
+{
+  if (SCALAR_FLOAT_MODE_P (mode))
+    {
+      if (res == const0_rtx)
+        return CONST0_RTX (mode);
+#ifdef FLOAT_STORE_FLAG_VALUE
+      REAL_VALUE_TYPE val = FLOAT_STORE_FLAG_VALUE (mode);
+      return const_double_from_real_value (val, mode);
+#else
+      return NULL_RTX;
+#endif
+    }
+  if (VECTOR_MODE_P (mode))
+    {
+      if (res == const0_rtx)
+	return CONST0_RTX (mode);
+#ifdef VECTOR_STORE_FLAG_VALUE
+      rtx val = VECTOR_STORE_FLAG_VALUE (mode);
+      if (val == NULL_RTX)
+	return NULL_RTX;
+      if (val == const1_rtx)
+	return CONST1_RTX (mode);
+
+      return gen_const_vec_duplicate (mode, val);
+#else
+      return NULL_RTX;
+#endif
+    }
+  /* For vector comparison with scalar int result, it is unknown
+     if the target means here a comparison into an integral bitmask,
+     or comparison where all comparisons true mean const_true_rtx
+     whole result, or where any comparisons true mean const_true_rtx
+     whole result.  For const0_rtx all the cases are the same.  */
+  if (VECTOR_MODE_P (cmp_mode)
+      && SCALAR_INT_MODE_P (mode)
+      && res == const_true_rtx)
+    return NULL_RTX;
+
+  return res;
+}
 				       
 /* Simplify a logical operation CODE with result mode MODE, operating on OP0
    and OP1, which should be both relational operations.  Return 0 if no such
@@ -2329,7 +2376,7 @@ simplify_context::simplify_logical_relational_operation (rtx_code code,
   int mask = mask0 | mask1;
 
   if (mask == 15)
-    return const_true_rtx;
+    return relational_result (mode, GET_MODE (op0), const_true_rtx);
 
   code = mask_to_comparison (mask);
 
@@ -5315,51 +5362,7 @@ simplify_context::simplify_relational_operation (rtx_code code,
 
   tem = simplify_const_relational_operation (code, cmp_mode, op0, op1);
   if (tem)
-    {
-      if (SCALAR_FLOAT_MODE_P (mode))
-	{
-          if (tem == const0_rtx)
-            return CONST0_RTX (mode);
-#ifdef FLOAT_STORE_FLAG_VALUE
-	  {
-	    REAL_VALUE_TYPE val;
-	    val = FLOAT_STORE_FLAG_VALUE (mode);
-	    return const_double_from_real_value (val, mode);
-	  }
-#else
-	  return NULL_RTX;
-#endif
-	}
-      if (VECTOR_MODE_P (mode))
-	{
-	  if (tem == const0_rtx)
-	    return CONST0_RTX (mode);
-#ifdef VECTOR_STORE_FLAG_VALUE
-	  {
-	    rtx val = VECTOR_STORE_FLAG_VALUE (mode);
-	    if (val == NULL_RTX)
-	      return NULL_RTX;
-	    if (val == const1_rtx)
-	      return CONST1_RTX (mode);
-
-	    return gen_const_vec_duplicate (mode, val);
-	  }
-#else
-	  return NULL_RTX;
-#endif
-	}
-      /* For vector comparison with scalar int result, it is unknown
-	 if the target means here a comparison into an integral bitmask,
-	 or comparison where all comparisons true mean const_true_rtx
-	 whole result, or where any comparisons true mean const_true_rtx
-	 whole result.  For const0_rtx all the cases are the same.  */
-      if (VECTOR_MODE_P (cmp_mode)
-	  && SCALAR_INT_MODE_P (mode)
-	  && tem == const_true_rtx)
-	return NULL_RTX;
-
-      return tem;
-    }
+    return relational_result (mode, cmp_mode, tem);
 
   /* For the following tests, ensure const0_rtx is op1.  */
   if (swap_commutative_operands_p (op0, op1)
diff --git a/gcc/testsuite/gcc.dg/pr101008.c b/gcc/testsuite/gcc.dg/pr101008.c
new file mode 100644
index 00000000000..c06208d3425
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr101008.c
@@ -0,0 +1,18 @@
+/* PR rtl-optimization/101008 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -g" } */
+
+typedef unsigned __attribute__((__vector_size__(32))) U;
+typedef unsigned __attribute__((__vector_size__(16))) V;
+
+int c, r;
+
+V v;
+
+void
+foo(void)
+{
+  U u = __builtin_shufflevector (v, (V)(v != c) | (V)(c == v),
+				 4, 3, 5, 5, 1, 2, 3, 0);
+  r = ((union { U a; int b; }) u).b;
+}