[gcc r11-9227] tree-optimization/102788 - avoid spurious bool pattern fails

[Richard Biener <rguenth@gcc.gnu.org>]

https://gcc.gnu.org/g:34ed721929700b85f19f14fc56fb598a658b2bbc

commit r11-9227-g34ed721929700b85f19f14fc56fb598a658b2bbc
Author: Richard Biener <rguenther@suse.de>
Date:   Mon Oct 18 10:31:19 2021 +0200

    tree-optimization/102788 - avoid spurious bool pattern fails
    
    Bool pattern recog is required for correctness since vectorized
    compares otherwise produce -1 for true so any context where bool
    is used as value and not as condition or mask needs to be replaced
    with CMP ? 1 : 0.  When we fail to find a vector type for the
    result of such use we may not simply elide such transform since
    a new bool result can emerge when for example the cast_forwprop
    pattern is applied.  So the following avoids failing of the
    bool pattern recog process and instead not assign a vector type
    for the stmt.
    
    2021-10-18  Richard Biener  <rguenther@suse.de>
    
            PR tree-optimization/102788
            * tree-vect-patterns.c (vect_init_pattern_stmt): Allow
            a NULL vectype.
            (vect_pattern_recog_1): Likewise.
            (vect_recog_bool_pattern): Continue matching the pattern
            even if we do not have a vector type for a conversion
            result.
    
            * g++.dg/vect/pr102788.cc: New testcase.
    
    (cherry picked from commit eb032893675afea4b01cc6ad06a3e0dcfe9b51cd)

Diff:
---
 gcc/testsuite/g++.dg/vect/pr102788.cc | 32 ++++++++++++++++++++++++++++++++
 gcc/tree-vect-patterns.c              |  8 +++-----
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/gcc/testsuite/g++.dg/vect/pr102788.cc b/gcc/testsuite/g++.dg/vect/pr102788.cc
new file mode 100644
index 00000000000..fa9c366fe56
--- /dev/null
+++ b/gcc/testsuite/g++.dg/vect/pr102788.cc
@@ -0,0 +1,32 @@
+// { dg-do run }
+// { dg-additional-options "-O3" }
+
+unsigned long long int var_4 = 235;
+unsigned long long int var_5 = 74;
+signed char var_12 = -99;
+unsigned long long int var_349;
+unsigned char var_645;
+void test();
+
+const unsigned long long &min(const unsigned long long &a,
+			      const unsigned long long &b)
+{
+  return b < a ? b : a;
+}
+
+void __attribute__((noipa)) test()
+{
+  for (short c = var_12; c; c += 5)
+    ;
+  for (int e = 0; e < 12; e += 1) {
+      var_349 = var_4 ? 235 : 74;
+      var_645 = min((unsigned long long)true, var_5 ? var_12 : var_4);
+  }
+}
+
+int main()
+{
+  test();
+  if (var_645 != 1)
+    __builtin_abort();
+}
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index a0e5eae9540..a48b7deceb8 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -112,8 +112,9 @@ vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
     = STMT_VINFO_DEF_TYPE (orig_stmt_info);
   if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
     {
-      gcc_assert (VECTOR_BOOLEAN_TYPE_P (vectype)
-		  == vect_use_mask_type_p (orig_stmt_info));
+      gcc_assert (!vectype
+		  || (VECTOR_BOOLEAN_TYPE_P (vectype)
+		      == vect_use_mask_type_p (orig_stmt_info)));
       STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
       pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
     }
@@ -4099,8 +4100,6 @@ vect_recog_bool_pattern (vec_info *vinfo,
 	  || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
 	return NULL;
       vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
-      if (vectype == NULL_TREE)
-	return NULL;
 
       if (check_bool_pattern (var, vinfo, bool_stmts))
 	{
@@ -5508,7 +5507,6 @@ vect_pattern_recog_1 (vec_info *vinfo,
     }
 
   loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  gcc_assert (pattern_vectype);
  
   /* Found a vectorizable pattern.  */
   if (dump_enabled_p ())