[gcc r12-2727] tree-optimization/101756 - avoid vectorizing boolean MAX reductions

[Richard Biener <rguenth@gcc.gnu.org>]

https://gcc.gnu.org/g:87a0b607e40f8122c7fc45d496ef48799fe11550

commit r12-2727-g87a0b607e40f8122c7fc45d496ef48799fe11550
Author: Richard Biener <rguenther@suse.de>
Date:   Wed Aug 4 11:42:41 2021 +0200

    tree-optimization/101756 - avoid vectorizing boolean MAX reductions
    
    The following avoids vectorizing MIN/MAX reductions on bools which,
    when ending up as vector(2) <signed-boolean:64> would need to be
    adjusted because of the sign change.  The fix instead avoids any
    reduction vectorization where the result isn't compatible
    to the original scalar type since we don't compensate for that
    either.
    
    2021-08-04  Richard Biener  <rguenther@suse.de>
    
            PR tree-optimization/101756
            * tree-vect-slp.c (vectorizable_bb_reduc_epilogue): Make sure
            the result of the reduction epilogue is compatible to the original
            scalar result.
    
            * gcc.dg/vect/bb-slp-pr101756.c: New testcase.

Diff:
---
 gcc/testsuite/gcc.dg/vect/bb-slp-pr101756.c | 15 +++++++++++++++
 gcc/tree-vect-slp.c                         |  8 +++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-pr101756.c b/gcc/testsuite/gcc.dg/vect/bb-slp-pr101756.c
new file mode 100644
index 00000000000..9420e77f64e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-pr101756.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+
+__attribute__ ((simd)) int
+tq (long int ea, int of, int kk)
+{
+  int bc;
+
+  for (bc = 0; bc < 2; ++bc)
+    {
+      ++ea;
+      of |= !!kk < !!ea;
+    }
+
+  return of;
+}
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index a554c24e0fb..d169bed8e94 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -4847,15 +4847,17 @@ static bool
 vectorizable_bb_reduc_epilogue (slp_instance instance,
 				stmt_vector_for_cost *cost_vec)
 {
-  enum tree_code reduc_code
-    = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
+  gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
+  enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
   if (reduc_code == MINUS_EXPR)
     reduc_code = PLUS_EXPR;
   internal_fn reduc_fn;
   tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
   if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
       || reduc_fn == IFN_LAST
-      || !direct_internal_fn_supported_p (reduc_fn, vectype, OPTIMIZE_FOR_BOTH))
+      || !direct_internal_fn_supported_p (reduc_fn, vectype, OPTIMIZE_FOR_BOTH)
+      || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
+				     TREE_TYPE (vectype)))
     return false;
 
   /* There's no way to cost a horizontal vector reduction via REDUC_FN so