[gcc r12-8159] analyzer: fix ICE comparing VECTOR_CSTs [PR105252]

[gcc r12-8159] analyzer: fix ICE comparing VECTOR_CSTs [PR105252]

[David Malcolm]

https://gcc.gnu.org/g:b209a349268d245f7a318861112df47bea15b89d

commit r12-8159-gb209a349268d245f7a318861112df47bea15b89d
Author: David Malcolm <dmalcolm@redhat.com>
Date:   Wed Apr 13 12:02:07 2022 -0400

    analyzer: fix ICE comparing VECTOR_CSTs [PR105252]
    
    gcc/analyzer/ChangeLog:
            PR analyzer/105252
            * svalue.cc (cmp_cst): When comparing VECTOR_CSTs, compare the
            types of the encoded elements before calling cmp_cst on them.
    
    gcc/testsuite/ChangeLog:
            PR analyzer/105252
            * gcc.dg/analyzer/pr105252.c: New test.
    
    Signed-off-by: David Malcolm <dmalcolm@redhat.com>

Diff:
---
 gcc/analyzer/svalue.cc                   | 13 ++++++++++---
 gcc/testsuite/gcc.dg/analyzer/pr105252.c | 20 ++++++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/gcc/analyzer/svalue.cc b/gcc/analyzer/svalue.cc
index 553edae7250..536bc288dbf 100644
--- a/gcc/analyzer/svalue.cc
+++ b/gcc/analyzer/svalue.cc
@@ -337,9 +337,16 @@ cmp_cst (const_tree cst1, const_tree cst2)
 	return cmp_nelts_per_pattern;
       unsigned encoded_nelts = vector_cst_encoded_nelts (cst1);
       for (unsigned i = 0; i < encoded_nelts; i++)
-	if (int el_cmp = cmp_cst (VECTOR_CST_ENCODED_ELT (cst1, i),
-				  VECTOR_CST_ENCODED_ELT (cst2, i)))
-	  return el_cmp;
+	{
+	  const_tree elt1 = VECTOR_CST_ENCODED_ELT (cst1, i);
+	  const_tree elt2 = VECTOR_CST_ENCODED_ELT (cst2, i);
+	  int t1 = TYPE_UID (TREE_TYPE (elt1));
+	  int t2 = TYPE_UID (TREE_TYPE (elt2));
+	  if (int cmp_type = t1 - t2)
+	    return cmp_type;
+	  if (int el_cmp = cmp_cst (elt1, elt2))
+	    return el_cmp;
+	}
       return 0;
     }
 }
diff --git a/gcc/testsuite/gcc.dg/analyzer/pr105252.c b/gcc/testsuite/gcc.dg/analyzer/pr105252.c
new file mode 100644
index 00000000000..a093eababc5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/pr105252.c
@@ -0,0 +1,20 @@
+/* { dg-additional-options "-fnon-call-exceptions -O" } */
+
+typedef unsigned char C;
+typedef unsigned char __attribute__((__vector_size__ (4))) V;
+
+C m;
+
+static inline void
+bar (C c, V v, V *r)
+{
+  v %= (c | v) % m;
+  *r = v;
+}
+
+void
+foo (void)
+{
+  V x;
+  bar (0, (V){2}, &x);
+}