[gcc r12-9241] Fix PR 105532: match.pd patterns calling tree_nonzero_bits with vector types

[Andrew Pinski <pinskia@gcc.gnu.org>]

https://gcc.gnu.org/g:67608f7a2cf3eff99654ab3f44d5183049e3b36c

commit r12-9241-g67608f7a2cf3eff99654ab3f44d5183049e3b36c
Author: Andrew Pinski <apinski@marvell.com>
Date:   Wed Nov 2 15:56:31 2022 +0000

    Fix PR 105532: match.pd patterns calling tree_nonzero_bits with vector types
    
    Even though this PR was reported with an ubsan issue, the problem is
    tree_nonzero_bits is being called with an expression which is a vector type.
    This fixes three patterns I noticed which does that.
    And adds a testcase for one of the patterns.
    
    Committed after a bootstrapped and tested on x86_64-linux-gnu with no regressions
    
    gcc/ChangeLog:
    
            PR tree-optimization/105532
            * match.pd (~(X >> Y) -> ~X >> Y): Check if it is an integral
            type before calling tree_nonzero_bits.
            (popcount(X) + popcount(Y)): Likewise.
            (popcount(X&C1)): Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.c-torture/compile/vector-shift-1.c: New test.
    
    (cherry picked from commit 193fccaa5c3525e979a989835c47c76d2c49d10c)

Diff:
---
 gcc/match.pd                                       | 25 ++++++++++++----------
 .../gcc.c-torture/compile/vector-shift-1.c         |  8 +++++++
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/gcc/match.pd b/gcc/match.pd
index ef352af1572..fc2833bbdca 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -1268,7 +1268,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    /* For logical right shifts, this is possible only if @0 doesn't
       have MSB set and the logical right shift is changed into
       arithmetic shift.  */
-   (if (!wi::neg_p (tree_nonzero_bits (@0)))
+   (if (INTEGRAL_TYPE_P (type)
+        && !wi::neg_p (tree_nonzero_bits (@0)))
     (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
      (convert (rshift (bit_not! (convert:stype @0)) @1))))))
 #endif
@@ -7169,7 +7170,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 /* popcount(X) + popcount(Y) is popcount(X|Y) when X&Y must be zero.  */
 (simplify
   (plus (POPCOUNT:s @0) (POPCOUNT:s @1))
-  (if (wi::bit_and (tree_nonzero_bits (@0), tree_nonzero_bits (@1)) == 0)
+  (if (INTEGRAL_TYPE_P (type)
+       && wi::bit_and (tree_nonzero_bits (@0), tree_nonzero_bits (@1)) == 0)
     (POPCOUNT (bit_ior @0 @1))))
 
 /* popcount(X) == 0 is X == 0, and related (in)equalities.  */
@@ -7201,15 +7203,16 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 (for pfun (POPCOUNT PARITY)
   (simplify
     (pfun @0)
-    (with { wide_int nz = tree_nonzero_bits (@0); }
-      (switch
-	(if (nz == 1)
-	  (convert @0))
-	(if (wi::popcount (nz) == 1)
-	  (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
-	    (convert (rshift:utype (convert:utype @0)
-				   { build_int_cst (integer_type_node,
-						    wi::ctz (nz)); }))))))))
+    (if (INTEGRAL_TYPE_P (type))
+     (with { wide_int nz = tree_nonzero_bits (@0); }
+       (switch
+	 (if (nz == 1)
+	   (convert @0))
+	 (if (wi::popcount (nz) == 1)
+	   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
+	     (convert (rshift:utype (convert:utype @0)
+				    { build_int_cst (integer_type_node,
+						     wi::ctz (nz)); })))))))))
 
 #if GIMPLE
 /* 64- and 32-bits branchless implementations of popcount are detected:
diff --git a/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c b/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c
new file mode 100644
index 00000000000..142ea56d5bb
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/vector-shift-1.c
@@ -0,0 +1,8 @@
+typedef unsigned char __attribute__((__vector_size__ (1))) U;
+
+U
+foo (U u)
+{
+  u = u == u;
+  return (~(u >> 255));
+}