[gcc r14-8355] Refactor exit PHI handling in vectorizer epilogue peeling

[gcc r14-8355] Refactor exit PHI handling in vectorizer epilogue peeling

[Richard Biener]

https://gcc.gnu.org/g:02e683894942da1620c07bd7e3e8b1d5950d91aa

commit r14-8355-g02e683894942da1620c07bd7e3e8b1d5950d91aa
Author: Richard Biener <rguenther@suse.de>
Date:   Mon Jan 22 09:17:35 2024 +0100

    Refactor exit PHI handling in vectorizer epilogue peeling
    
    This refactors the handling of PHIs inbetween the main and the
    epilogue loop.  Instead of trying to handle the multiple exit
    and original single exit case together the following separates
    these cases resulting in much easier to understand code.
    
            * tree-vect-loop-manip.cc (slpeel_tree_duplicate_loop_to_edge_cfg):
            Separate single and multi-exit case when creating PHIs between
            the main and epilogue.

Diff:
---
 gcc/tree-vect-loop-manip.cc | 135 ++++++++++++++++++++++++--------------------
 1 file changed, 74 insertions(+), 61 deletions(-)

diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index eacbc022549..873a28d7c56 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -1682,77 +1682,60 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
 	    }
 
 	  /* Create the merge PHI nodes in new_preheader and populate the
-	     arguments for the main exit.  */
-	  for (auto gsi_from = gsi_start_phis (loop->header),
-	       gsi_to = gsi_start_phis (new_loop->header);
-	       !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
-	       gsi_next (&gsi_from), gsi_next (&gsi_to))
+	     arguments for the exits.  */
+	  if (multiple_exits_p)
 	    {
-	      gimple *from_phi = gsi_stmt (gsi_from);
-	      gimple *to_phi = gsi_stmt (gsi_to);
-	      tree new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
-						    loop_latch_edge (loop));
-
-	      /* Check if we've already created a new phi node during edge
-		 redirection.  If we have, only propagate the value
-		 downwards in case there is no merge block.  */
-	      tree *res;
-	      if ((res = new_phi_args.get (new_arg)))
+	      for (auto gsi_from = gsi_start_phis (loop->header),
+		   gsi_to = gsi_start_phis (new_loop->header);
+		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
+		   gsi_next (&gsi_from), gsi_next (&gsi_to))
 		{
-		  if (multiple_exits_p)
-		    new_arg = *res;
+		  gimple *from_phi = gsi_stmt (gsi_from);
+		  gimple *to_phi = gsi_stmt (gsi_to);
+
+		  /* When the vector loop is peeled then we need to use the
+		     value at start of the loop, otherwise the main loop exit
+		     should use the final iter value.  */
+		  tree new_arg;
+		  if (peeled_iters)
+		    new_arg = gimple_phi_result (from_phi);
 		  else
-		    {
-		      adjust_phi_and_debug_stmts (to_phi, loop_entry, *res);
-		      continue;
-		    }
-		}
-	      /* If we have multiple exits and the vector loop is peeled then we
-		 need to use the value at start of loop.  If we're looking at
-		 virtual operands we have to keep the original link.   Virtual
-		 operands don't all become the same because we'll corrupt the
-		 vUSE chains among others.  */
-	      if (peeled_iters)
-		{
-		  tree tmp_arg = gimple_phi_result (from_phi);
-		  /* Similar to the single exit case, If we have an existing
-		     LCSSA variable thread through the original value otherwise
-		     skip it and directly use the final value.  */
-		  if ((res = new_phi_args.get (tmp_arg)))
+		    new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
+						     loop_latch_edge (loop));
+
+		  /* Check if we've already created a new phi node during edge
+		     redirection and re-use it if so.  Otherwise create a
+		     LC PHI node to feed the merge PHI.  */
+		  tree *res;
+		  if (virtual_operand_p (new_arg))
+		    /* Use the existing virtual LC SSA from exit block.  */
+		    new_arg = gimple_phi_result
+				(get_virtual_phi (main_loop_exit_block));
+		  else if ((res = new_phi_args.get (new_arg)))
 		    new_arg = *res;
-		  else if (!virtual_operand_p (new_arg))
-		    new_arg = tmp_arg;
-		}
-
-	      tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
-	      gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
-
-	      /* Otherwise, main loop exit should use the final iter value.  */
-	      if (multiple_exits_p)
-		{
-		  /* Create a LC PHI if it doesn't already exist.  */
-		  if (!virtual_operand_p (new_arg) && !res)
+		  else
 		    {
+		      /* Create the LC PHI node for the exit.  */
 		      tree new_def = copy_ssa_name (new_arg);
 		      gphi *lc_phi
-			= create_phi_node (new_def, main_loop_exit_block);
+			  = create_phi_node (new_def, main_loop_exit_block);
 		      SET_PHI_ARG_DEF (lc_phi, 0, new_arg);
 		      new_arg = new_def;
 		    }
-		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi,
-					   single_succ_edge (main_loop_exit_block),
-					   new_arg);
-		}
-	      else
-		SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, loop_exit, new_arg);
 
-	      adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
-	    }
+		  /* Create the PHI node in the merge block merging the
+		     main and early exit values.  */
+		  tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
+		  gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
+		  edge main_e = single_succ_edge (main_loop_exit_block);
+		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, main_e, new_arg);
 
-	  /* Now fill in the values for the merge PHI in new_preheader
-	     for the alternative exits.  */
-	  if (multiple_exits_p)
-	    {
+		  /* And adjust the epilog entry value.  */
+		  adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
+		}
+
+	      /* After creating the merge PHIs handle the early exits those
+		 should use the values at the start of the loop.  */
 	      for (auto gsi_from = gsi_start_phis (loop->header),
 		   gsi_to = gsi_start_phis (new_preheader);
 		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
@@ -1790,8 +1773,38 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
 			SET_PHI_ARG_DEF (lc_phi, i, alt_arg);
 		      alt_arg = alt_def;
 		    }
-		  edge main_e = single_succ_edge (alt_loop_exit_block);
-		  SET_PHI_ARG_DEF_ON_EDGE (to_phi, main_e, alt_arg);
+		  edge alt_e = single_succ_edge (alt_loop_exit_block);
+		  SET_PHI_ARG_DEF_ON_EDGE (to_phi, alt_e, alt_arg);
+		}
+	    }
+	  /* For the single exit case only create the missing LC PHI nodes
+	     for the continuation of the loop IVs that are not also already
+	     reductions and thus had LC PHI nodes on the exit already.  */
+	  else
+	    {
+	      for (auto gsi_from = gsi_start_phis (loop->header),
+		   gsi_to = gsi_start_phis (new_loop->header);
+		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
+		   gsi_next (&gsi_from), gsi_next (&gsi_to))
+		{
+		  gimple *from_phi = gsi_stmt (gsi_from);
+		  gimple *to_phi = gsi_stmt (gsi_to);
+		  tree new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
+							loop_latch_edge (loop));
+
+		  /* Check if we've already created a new phi node during edge
+		     redirection.  If we have, only propagate the value
+		     downwards.  */
+		  if (tree *res = new_phi_args.get (new_arg))
+		    {
+		      adjust_phi_and_debug_stmts (to_phi, loop_entry, *res);
+		      continue;
+		    }
+
+		  tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
+		  gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
+		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, loop_exit, new_arg);
+		  adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
 		}
 	    }
 	}