[gcc r13-3716] vect: Fold LEN_{LOAD, STORE} if it's for the whole vector [PR107412]

[Kewen Lin <linkw@gcc.gnu.org>]

https://gcc.gnu.org/g:8408120fecc56385b316dafec1bdfe3aac61fc05

commit r13-3716-g8408120fecc56385b316dafec1bdfe3aac61fc05
Author: Kewen Lin <linkw@linux.ibm.com>
Date:   Mon Nov 7 02:07:27 2022 -0600

    vect: Fold LEN_{LOAD,STORE} if it's for the whole vector [PR107412]
    
    As the test case in PR107412 shows, we can fold IFN .LEN_{LOAD,
    STORE} into normal vector load/store if the given length is known
    to be equal to the length of the whole vector.  It would help to
    improve overall cycles as normally the latency of vector access
    with length in bytes is bigger than normal vector access, and it
    also saves the preparation for length if constant length can not
    be encoded into instruction (such as on power).
    
            PR tree-optimization/107412
    
    gcc/ChangeLog:
    
            * gimple-fold.cc (gimple_fold_mask_load_store_mem_ref): Rename to ...
            (gimple_fold_partial_load_store_mem_ref): ... this, add one parameter
            mask_p indicating it's for mask or length, and add some handlings for
            IFN LEN_{LOAD,STORE}.
            (gimple_fold_mask_load): Rename to ...
            (gimple_fold_partial_load): ... this, add one parameter mask_p.
            (gimple_fold_mask_store): Rename to ...
            (gimple_fold_partial_store): ... this, add one parameter mask_p.
            (gimple_fold_call): Add the handlings for IFN LEN_{LOAD,STORE},
            and adjust calls on gimple_fold_mask_load_store_mem_ref to
            gimple_fold_partial_load_store_mem_ref.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/powerpc/pr107412.c: New test.
            * gcc.target/powerpc/p9-vec-length-epil-8.c: Adjust scan times for
            folded LEN_LOAD.

Diff:
---
 gcc/gimple-fold.cc                                 | 57 +++++++++++++++++-----
 .../gcc.target/powerpc/p9-vec-length-epil-8.c      |  2 +-
 gcc/testsuite/gcc.target/powerpc/pr107412.c        | 19 ++++++++
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index 9055cd8982d..410544c17bb 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -5370,19 +5370,39 @@ arith_overflowed_p (enum tree_code code, const_tree type,
   return wi::min_precision (wres, sign) > TYPE_PRECISION (type);
 }
 
-/* If IFN_MASK_LOAD/STORE call CALL is unconditional, return a MEM_REF
+/* If IFN_{MASK,LEN}_LOAD/STORE call CALL is unconditional, return a MEM_REF
    for the memory it references, otherwise return null.  VECTYPE is the
-   type of the memory vector.  */
+   type of the memory vector.  MASK_P indicates it's for MASK if true,
+   otherwise it's for LEN.  */
 
 static tree
-gimple_fold_mask_load_store_mem_ref (gcall *call, tree vectype)
+gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype, bool mask_p)
 {
   tree ptr = gimple_call_arg (call, 0);
   tree alias_align = gimple_call_arg (call, 1);
-  tree mask = gimple_call_arg (call, 2);
-  if (!tree_fits_uhwi_p (alias_align) || !integer_all_onesp (mask))
+  if (!tree_fits_uhwi_p (alias_align))
     return NULL_TREE;
 
+  if (mask_p)
+    {
+      tree mask = gimple_call_arg (call, 2);
+      if (!integer_all_onesp (mask))
+	return NULL_TREE;
+    } else {
+      tree basic_len = gimple_call_arg (call, 2);
+      if (!tree_fits_uhwi_p (basic_len))
+	return NULL_TREE;
+      unsigned int nargs = gimple_call_num_args (call);
+      tree bias = gimple_call_arg (call, nargs - 1);
+      gcc_assert (tree_fits_uhwi_p (bias));
+      tree biased_len = int_const_binop (MINUS_EXPR, basic_len, bias);
+      unsigned int len = tree_to_uhwi (biased_len);
+      unsigned int vect_len
+	= GET_MODE_SIZE (TYPE_MODE (vectype)).to_constant ();
+      if (vect_len != len)
+	return NULL_TREE;
+    }
+
   unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align);
   if (TYPE_ALIGN (vectype) != align)
     vectype = build_aligned_type (vectype, align);
@@ -5390,16 +5410,18 @@ gimple_fold_mask_load_store_mem_ref (gcall *call, tree vectype)
   return fold_build2 (MEM_REF, vectype, ptr, offset);
 }
 
-/* Try to fold IFN_MASK_LOAD call CALL.  Return true on success.  */
+/* Try to fold IFN_{MASK,LEN}_LOAD call CALL.  Return true on success.
+   MASK_P indicates it's for MASK if true, otherwise it's for LEN.  */
 
 static bool
-gimple_fold_mask_load (gimple_stmt_iterator *gsi, gcall *call)
+gimple_fold_partial_load (gimple_stmt_iterator *gsi, gcall *call, bool mask_p)
 {
   tree lhs = gimple_call_lhs (call);
   if (!lhs)
     return false;
 
-  if (tree rhs = gimple_fold_mask_load_store_mem_ref (call, TREE_TYPE (lhs)))
+  if (tree rhs
+      = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (lhs), mask_p))
     {
       gassign *new_stmt = gimple_build_assign (lhs, rhs);
       gimple_set_location (new_stmt, gimple_location (call));
@@ -5410,13 +5432,16 @@ gimple_fold_mask_load (gimple_stmt_iterator *gsi, gcall *call)
   return false;
 }
 
-/* Try to fold IFN_MASK_STORE call CALL.  Return true on success.  */
+/* Try to fold IFN_{MASK,LEN}_STORE call CALL.  Return true on success.
+   MASK_P indicates it's for MASK if true, otherwise it's for LEN.  */
 
 static bool
-gimple_fold_mask_store (gimple_stmt_iterator *gsi, gcall *call)
+gimple_fold_partial_store (gimple_stmt_iterator *gsi, gcall *call,
+			   bool mask_p)
 {
   tree rhs = gimple_call_arg (call, 3);
-  if (tree lhs = gimple_fold_mask_load_store_mem_ref (call, TREE_TYPE (rhs)))
+  if (tree lhs
+      = gimple_fold_partial_load_store_mem_ref (call, TREE_TYPE (rhs), mask_p))
     {
       gassign *new_stmt = gimple_build_assign (lhs, rhs);
       gimple_set_location (new_stmt, gimple_location (call));
@@ -5635,10 +5660,16 @@ gimple_fold_call (gimple_stmt_iterator *gsi, bool inplace)
 	  cplx_result = true;
 	  break;
 	case IFN_MASK_LOAD:
-	  changed |= gimple_fold_mask_load (gsi, stmt);
+	  changed |= gimple_fold_partial_load (gsi, stmt, true);
 	  break;
 	case IFN_MASK_STORE:
-	  changed |= gimple_fold_mask_store (gsi, stmt);
+	  changed |= gimple_fold_partial_store (gsi, stmt, true);
+	  break;
+	case IFN_LEN_LOAD:
+	  changed |= gimple_fold_partial_load (gsi, stmt, false);
+	  break;
+	case IFN_LEN_STORE:
+	  changed |= gimple_fold_partial_store (gsi, stmt, false);
 	  break;
 	default:
 	  break;
diff --git a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
index 961df0d5646..8b9c9107814 100644
--- a/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
+++ b/gcc/testsuite/gcc.target/powerpc/p9-vec-length-epil-8.c
@@ -8,5 +8,5 @@
 
 #include "p9-vec-length-8.h"
 
-/* { dg-final { scan-assembler-times {\mlxvl\M} 21 } } */
+/* { dg-final { scan-assembler-times {\mlxvl\M} 16 } } */
 /* { dg-final { scan-assembler-times {\mstxvl\M} 7 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/pr107412.c b/gcc/testsuite/gcc.target/powerpc/pr107412.c
new file mode 100644
index 00000000000..4526ea8639d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr107412.c
@@ -0,0 +1,19 @@
+/* { dg-require-effective-target powerpc_p9vector_ok } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-mdejagnu-cpu=power9 -O2 -ftree-vectorize -fno-vect-cost-model -funroll-loops -fno-tree-loop-distribute-patterns --param vect-partial-vector-usage=2 -fdump-tree-optimized" } */
+
+/* Verify there is only one IFN call LEN_LOAD and IFN_STORE separately.  */
+
+#define N 16
+int src[N];
+int dest[N];
+
+void
+foo ()
+{
+  for (int i = 0; i < (N - 1); i++)
+    dest[i] = src[i];
+}
+
+/* { dg-final { scan-tree-dump-times {\mLEN_LOAD\M} 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-times {\mLEN_STORE\M} 1 "optimized" } } */