[gcc r11-8029] vect: Don't split store groups if we have IFN_STORE_LANES [PR99873]

[gcc r11-8029] vect: Don't split store groups if we have IFN_STORE_LANES [PR99873]

[Richard Sandiford]

https://gcc.gnu.org/g:5c5b31975e62b4c52d76dc5efd9dc717a361c710

commit r11-8029-g5c5b31975e62b4c52d76dc5efd9dc717a361c710
Author: Richard Sandiford <richard.sandiford@arm.com>
Date:   Wed Apr 7 15:21:55 2021 +0100

    vect: Don't split store groups if we have IFN_STORE_LANES [PR99873]
    
    As noted in the PR, we were no longer using ST3 for the testcase and
    instead stored each lane individually.  This is because we'd split
    the store group during SLP and couldn't recover when SLP failed.
    
    However, we can also get better code with ST3 and ST4 even if SLP would
    have succeeded, such as for vect-complex-5.c.  I'm not sure exactly
    where the cut-off point is, but it seems reasonable to allow the split
    if either of the new groups would operate on full vectors *within*
    rather than across scalar loop iterations.
    
    E.g. on a Cortex-A57, pr99873_3.c performs better using ST4 while
    pr99873_2.c performs better with SLP.
    
    Another factor is that SLP can handle smaller iteration counts than
    IFN_STORE_LANES can, but we don't have the infrastructure to choose
    reliably based on that.
    
    gcc/
            PR tree-optimization/99873
            * tree-vect-slp.c (vect_slp_prefer_store_lanes_p): New function.
            (vect_build_slp_instance): Don't split store groups that could
            use IFN_STORE_LANES.
    
    gcc/testsuite/
            * gcc.dg/vect/slp-21.c: Only expect 2 of the loops to use SLP
            if IFN_STORE_LANES is available.
            * gcc.dg/vect/vect-complex-5.c: Expect no loops to use SLP if
            IFN_STORE_LANES is available.
            * gcc.target/aarch64/pr99873_1.c: New test.
            * gcc.target/aarch64/pr99873_2.c: Likewise.
            * gcc.target/aarch64/pr99873_3.c: Likewise.
            * gcc.target/aarch64/sve/pr99873_1.c: Likewise.
            * gcc.target/aarch64/sve/pr99873_2.c: Likewise.
            * gcc.target/aarch64/sve/pr99873_3.c: Likewise.

Diff:
---
 gcc/testsuite/gcc.dg/vect/slp-21.c               |  4 +--
 gcc/testsuite/gcc.dg/vect/vect-complex-5.c       |  3 ++-
 gcc/testsuite/gcc.target/aarch64/pr99873_1.c     | 17 +++++++++++++
 gcc/testsuite/gcc.target/aarch64/pr99873_2.c     | 20 +++++++++++++++
 gcc/testsuite/gcc.target/aarch64/pr99873_3.c     | 20 +++++++++++++++
 gcc/testsuite/gcc.target/aarch64/sve/pr99873_1.c | 15 ++++++++++++
 gcc/testsuite/gcc.target/aarch64/sve/pr99873_2.c | 18 ++++++++++++++
 gcc/testsuite/gcc.target/aarch64/sve/pr99873_3.c | 18 ++++++++++++++
 gcc/tree-vect-slp.c                              | 31 +++++++++++++++++++++++-
 9 files changed, 142 insertions(+), 4 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/vect/slp-21.c b/gcc/testsuite/gcc.dg/vect/slp-21.c
index bf8f434dd50..85393975b45 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-21.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-21.c
@@ -210,7 +210,7 @@ int main (void)
 
    Not all vect_perm targets support that, and it's a bit too specific to have
    its own effective-target selector, so we just test targets directly.  */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 4 "vect" { target { aarch64*-*-* arm*-*-* powerpc64*-*-* } } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target { vect_strided4 && { ! { aarch64*-*-* arm*-*-* powerpc64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 4 "vect" { target powerpc64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target { vect_strided4 && { ! powerpc64*-*-* } } } } } */
 /* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 0 "vect"  { target { ! { vect_strided4 } } } } } */
   
diff --git a/gcc/testsuite/gcc.dg/vect/vect-complex-5.c b/gcc/testsuite/gcc.dg/vect/vect-complex-5.c
index 81fdb67ce81..addcf60438c 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-complex-5.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-complex-5.c
@@ -40,4 +40,5 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { xfail { ! vect_hw_misalign } } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 0 "vect" { target vect_load_lanes } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target { ! vect_load_lanes } xfail { ! vect_hw_misalign } } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr99873_1.c b/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
new file mode 100644
index 00000000000..bc4d81e3ae5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
@@ -0,0 +1,17 @@
+/* { dg-options "-O3" } */
+
+#pragma GCC target "+nosve"
+
+void
+f (int *restrict x, int *restrict y, int *restrict z, int n)
+{
+  for (int i = 0; i < n; i += 3)
+    {
+      x[i] = y[i] + z[i];
+      x[i + 1] = y[i + 1] - z[i + 1];
+      x[i + 2] = y[i + 2] | z[i + 2];
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tld3\t} 2 } } */
+/* { dg-final { scan-assembler-times {\tst3\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr99873_2.c b/gcc/testsuite/gcc.target/aarch64/pr99873_2.c
new file mode 100644
index 00000000000..b73fbdc0a18
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr99873_2.c
@@ -0,0 +1,20 @@
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+#pragma GCC target "+nosve"
+
+void __attribute ((noipa))
+foo (uint64_t *__restrict x, uint64_t *__restrict y, int n)
+{
+  for (int i = 0; i < n; i += 4)
+    {
+      x[i] += y[i];
+      x[i + 1] += y[i + 1];
+      x[i + 2] |= y[i + 2];
+      x[i + 3] |= y[i + 3];
+    }
+}
+
+/* { dg-final { scan-assembler-not {\tld4\t} } } */
+/* { dg-final { scan-assembler-not {\tst4\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr99873_3.c b/gcc/testsuite/gcc.target/aarch64/pr99873_3.c
new file mode 100644
index 00000000000..ccbab6d74be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr99873_3.c
@@ -0,0 +1,20 @@
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+#pragma GCC target "+nosve"
+
+void __attribute ((noipa))
+foo (uint32_t *__restrict x, uint32_t *__restrict y, int n)
+{
+  for (int i = 0; i < n; i += 4)
+    {
+      x[i] += y[i];
+      x[i + 1] += y[i + 1];
+      x[i + 2] |= y[i + 2];
+      x[i + 3] |= y[i + 3];
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tld4\t} 2 } } */
+/* { dg-final { scan-assembler-times {\tst4\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr99873_1.c b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_1.c
new file mode 100644
index 00000000000..f4b95da2afa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_1.c
@@ -0,0 +1,15 @@
+/* { dg-options "-O3" } */
+
+void
+f (int *restrict x, int *restrict y, int *restrict z, int n)
+{
+  for (int i = 0; i < n; i += 3)
+    {
+      x[i] = y[i] + z[i];
+      x[i + 1] = y[i + 1] - z[i + 1];
+      x[i + 2] = y[i + 2] | z[i + 2];
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tld3w\t} 2 } } */
+/* { dg-final { scan-assembler-times {\tst3w\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr99873_2.c b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_2.c
new file mode 100644
index 00000000000..03dc4ef930d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_2.c
@@ -0,0 +1,18 @@
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+void __attribute ((noipa))
+foo (uint64_t *__restrict x, uint64_t *__restrict y, int n)
+{
+  for (int i = 0; i < n; i += 4)
+    {
+      x[i] += y[i];
+      x[i + 1] += y[i + 1];
+      x[i + 2] |= y[i + 2];
+      x[i + 3] |= y[i + 3];
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tld4d\t} 2 } } */
+/* { dg-final { scan-assembler-times {\tst4d\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pr99873_3.c b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_3.c
new file mode 100644
index 00000000000..87a0141e508
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pr99873_3.c
@@ -0,0 +1,18 @@
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+void __attribute ((noipa))
+foo (uint32_t *__restrict x, uint32_t *__restrict y, int n)
+{
+  for (int i = 0; i < n; i += 4)
+    {
+      x[i] += y[i];
+      x[i + 1] += y[i + 1];
+      x[i + 2] |= y[i + 2];
+      x[i + 3] |= y[i + 3];
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tld4w\t} 2 } } */
+/* { dg-final { scan-assembler-times {\tst4w\t} 1 } } */
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 58dedfc35b7..7d5f9d10ca4 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -2458,6 +2458,34 @@ vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
   return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, visited);
 }
 
+/* STMT_INFO is a store group of size GROUP_SIZE that we are considering
+   splitting into two, with the first split group having size NEW_GROUP_SIZE.
+   Return true if we could use IFN_STORE_LANES instead and if that appears
+   to be the better approach.  */
+
+static bool
+vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
+			       unsigned int group_size,
+			       unsigned int new_group_size)
+{
+  tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
+  tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
+  if (!vectype)
+    return false;
+  /* Allow the split if one of the two new groups would operate on full
+     vectors *within* rather than across one scalar loop iteration.
+     This is purely a heuristic, but it should work well for group
+     sizes of 3 and 4, where the possible splits are:
+
+       3->2+1:  OK if the vector has exactly two elements
+       4->2+2:  Likewise
+       4->3+1:  Less clear-cut.  */
+  if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
+      || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
+    return false;
+  return vect_store_lanes_supported (vectype, group_size, false);
+}
+
 /* Analyze an SLP instance starting from a group of grouped stores.  Call
    vect_build_slp_tree to build a tree of packed stmts if possible.
    Return FALSE if it's impossible to SLP any stmt in the loop.  */
@@ -2693,7 +2721,8 @@ vect_build_slp_instance (vec_info *vinfo,
 
       /* For loop vectorization split into arbitrary pieces of size > 1.  */
       if (is_a <loop_vec_info> (vinfo)
-	  && (i > 1 && i < group_size))
+	  && (i > 1 && i < group_size)
+	  && !vect_slp_prefer_store_lanes_p (vinfo, stmt_info, group_size, i))
 	{
 	  unsigned group1_size = i;