[gcc r13-5644] Don't peel nonlinear iv(mult or shift) for epilog when vf is not constant.

[gcc r13-5644] Don't peel nonlinear iv(mult or shift) for epilog when vf is not constant.

[hongtao Liu]

https://gcc.gnu.org/g:209f02b0a9e9adc0bf0247cb5eef04e0f175d64e

commit r13-5644-g209f02b0a9e9adc0bf0247cb5eef04e0f175d64e
Author: liuhongt <hongtao.liu@intel.com>
Date:   Wed Feb 1 13:30:12 2023 +0800

    Don't peel nonlinear iv(mult or shift) for epilog when vf is not constant.
    
    Normally when vf is not constant, it will be prevented by
    vectorizable_nonlinear_inductions, but for this case, it failed going
    into
    
        if (STMT_VINFO_RELEVANT_P (stmt_info))
          {
            need_to_vectorize = true;
            if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
               && ! PURE_SLP_STMT (stmt_info))
              ok = vectorizable_induction (loop_vinfo,
                                           stmt_info, NULL, NULL,
                                           &cost_vec);
    
    since the iv is never used outside of the loop, and will be dce later, so
    vectorizer doesn't bother checking if it's vectorizable. it's
    true but hit gcc_assert in vect_can_peel_nonlinear_iv_p when vf is not
    constant. One solution is ignoring the nonlinear iv peeling if it's
    !STMT_VINFO_RELEVANT_P (stmt_info) just like the upper code, the other
    solution is returning false earlier in the
    vect_can_peel_nonlinear_iv_p when vf is not constant, the patch chooses
    the second incase there's other cases using vect_can_advance_ivs_p which
    calls vect_can_peel_nonlinear_iv_p.
    Also remove vect_peel_nonlinear_iv_p from
    vectorizable_nonlinear_inductions.
    
    gcc/ChangeLog:
    
            PR tree-optimization/108601
            * tree-vectorizer.h (vect_can_peel_nonlinear_iv_p): Removed.
            * tree-vect-loop.cc
            (vectorizable_nonlinear_induction): Remove
            vect_can_peel_nonlinear_iv_p.
            (vect_can_peel_nonlinear_iv_p): Don't peel
            nonlinear iv(mult or shift) for epilog when vf is not
            constant and moved the defination to ..
            * tree-vect-loop-manip.cc (vect_can_peel_nonlinear_iv_p):
            .. Here.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/pr108601.c: New test.

Diff:
---
 gcc/testsuite/gcc.target/aarch64/pr108601.c | 11 +++++++
 gcc/tree-vect-loop-manip.cc                 | 44 +++++++++++++++++++++++++++
 gcc/tree-vect-loop.cc                       | 46 -----------------------------
 gcc/tree-vectorizer.h                       |  3 --
 4 files changed, 55 insertions(+), 49 deletions(-)

diff --git a/gcc/testsuite/gcc.target/aarch64/pr108601.c b/gcc/testsuite/gcc.target/aarch64/pr108601.c
new file mode 100644
index 00000000000..deb8b3061d8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr108601.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fprofile-generate -mcpu=neoverse-v1" } */
+
+int
+foo() {
+  int flag = 1;
+  for (; flag <= 1 << 21; flag <<= 1)
+    ;
+  return 0;
+}
+
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index b5c5f859144..c04fcf40c44 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -1390,6 +1390,50 @@ iv_phi_p (stmt_vec_info stmt_info)
   return true;
 }
 
+/* Return true if vectorizer can peel for nonlinear iv.  */
+static bool
+vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
+			      enum vect_induction_op_type induction_type)
+{
+  tree niters_skip;
+  /* Init_expr will be update by vect_update_ivs_after_vectorizer,
+     if niters or vf is unkown:
+     For shift, when shift mount >= precision, there would be UD.
+     For mult, don't known how to generate
+     init_expr * pow (step, niters) for variable niters.
+     For neg, it should be ok, since niters of vectorized main loop
+     will always be multiple of 2.  */
+  if ((!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+       || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ())
+      && induction_type != vect_step_op_neg)
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Peeling for epilogue is not supported"
+			 " for nonlinear induction except neg"
+			 " when iteration count is unknown.\n");
+      return false;
+    }
+
+  /* Also doens't support peel for neg when niter is variable.
+     ??? generate something like niter_expr & 1 ? init_expr : -init_expr?  */
+  niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
+  if ((niters_skip != NULL_TREE
+       && TREE_CODE (niters_skip) != INTEGER_CST)
+      || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
+	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Peeling for alignement is not supported"
+			 " for nonlinear induction when niters_skip"
+			 " is not constant.\n");
+      return false;
+    }
+
+  return true;
+}
+
 /* Function vect_can_advance_ivs_p
 
    In case the number of iterations that LOOP iterates is unknown at compile
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index f03af1efd0f..becf96bb2b8 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -8812,49 +8812,6 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
 
 }
 
-/* Return true if vectorizer can peel for nonlinear iv.  */
-bool
-vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
-			      enum vect_induction_op_type induction_type)
-{
-  tree niters_skip;
-  /* Init_expr will be update by vect_update_ivs_after_vectorizer,
-     if niters is unkown:
-     For shift, when shift mount >= precision, there would be UD.
-     For mult, don't known how to generate
-     init_expr * pow (step, niters) for variable niters.
-     For neg, it should be ok, since niters of vectorized main loop
-     will always be multiple of 2.  */
-  if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
-      && induction_type != vect_step_op_neg)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "Peeling for epilogue is not supported"
-			 " for nonlinear induction except neg"
-			 " when iteration count is unknown.\n");
-      return false;
-    }
-
-  /* Also doens't support peel for neg when niter is variable.
-     ??? generate something like niter_expr & 1 ? init_expr : -init_expr?  */
-  niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
-  if ((niters_skip != NULL_TREE
-       && TREE_CODE (niters_skip) != INTEGER_CST)
-      || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
-	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "Peeling for alignement is not supported"
-			 " for nonlinear induction when niters_skip"
-			 " is not constant.\n");
-      return false;
-    }
-
-  return true;
-}
-
 /* Function vectorizable_induction
 
    Check if STMT_INFO performs an nonlinear induction computation that can be
@@ -8925,9 +8882,6 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
       return false;
     }
 
-  if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
-    return false;
-
   if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
     {
       if (dump_enabled_p ())
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index a2aa71bedc0..4ba653712e9 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2347,9 +2347,6 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
 /* Nonlinear induction.  */
 extern tree vect_peel_nonlinear_iv_init (gimple_seq*, tree, tree,
 					 tree, enum vect_induction_op_type);
-extern bool
-vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
-			      enum vect_induction_op_type induction_type);
 
 /* In tree-vect-slp.cc.  */
 extern void vect_slp_init (void);