public inbox for gcc-patches@gcc.gnu.org
 help / color / mirror / Atom feed
* [PATCH] Refactor exit PHI handling in vectorizer epilogue peeling
@ 2024-01-22  9:02 Richard Biener
  0 siblings, 0 replies; only message in thread
From: Richard Biener @ 2024-01-22  9:02 UTC (permalink / raw)
  To: gcc-patches

This refactors the handling of PHIs inbetween the main and the
epilogue loop.  Instead of trying to handle the multiple exit
and original single exit case together the following separates
these cases resulting in much easier to understand code.

Bootstrap and regtest running on x86_64-unknown-linux-gnu.

Richard.

	* tree-vect-loop-manip.cc (slpeel_tree_duplicate_loop_to_edge_cfg):
	Separate single and multi-exit case when creating PHIs between
	the main and epilogue.
---
 gcc/tree-vect-loop-manip.cc | 135 ++++++++++++++++++++----------------
 1 file changed, 74 insertions(+), 61 deletions(-)

diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index eacbc022549..873a28d7c56 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -1682,77 +1682,60 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
 	    }
 
 	  /* Create the merge PHI nodes in new_preheader and populate the
-	     arguments for the main exit.  */
-	  for (auto gsi_from = gsi_start_phis (loop->header),
-	       gsi_to = gsi_start_phis (new_loop->header);
-	       !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
-	       gsi_next (&gsi_from), gsi_next (&gsi_to))
+	     arguments for the exits.  */
+	  if (multiple_exits_p)
 	    {
-	      gimple *from_phi = gsi_stmt (gsi_from);
-	      gimple *to_phi = gsi_stmt (gsi_to);
-	      tree new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
-						    loop_latch_edge (loop));
-
-	      /* Check if we've already created a new phi node during edge
-		 redirection.  If we have, only propagate the value
-		 downwards in case there is no merge block.  */
-	      tree *res;
-	      if ((res = new_phi_args.get (new_arg)))
+	      for (auto gsi_from = gsi_start_phis (loop->header),
+		   gsi_to = gsi_start_phis (new_loop->header);
+		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
+		   gsi_next (&gsi_from), gsi_next (&gsi_to))
 		{
-		  if (multiple_exits_p)
-		    new_arg = *res;
+		  gimple *from_phi = gsi_stmt (gsi_from);
+		  gimple *to_phi = gsi_stmt (gsi_to);
+
+		  /* When the vector loop is peeled then we need to use the
+		     value at start of the loop, otherwise the main loop exit
+		     should use the final iter value.  */
+		  tree new_arg;
+		  if (peeled_iters)
+		    new_arg = gimple_phi_result (from_phi);
 		  else
-		    {
-		      adjust_phi_and_debug_stmts (to_phi, loop_entry, *res);
-		      continue;
-		    }
-		}
-	      /* If we have multiple exits and the vector loop is peeled then we
-		 need to use the value at start of loop.  If we're looking at
-		 virtual operands we have to keep the original link.   Virtual
-		 operands don't all become the same because we'll corrupt the
-		 vUSE chains among others.  */
-	      if (peeled_iters)
-		{
-		  tree tmp_arg = gimple_phi_result (from_phi);
-		  /* Similar to the single exit case, If we have an existing
-		     LCSSA variable thread through the original value otherwise
-		     skip it and directly use the final value.  */
-		  if ((res = new_phi_args.get (tmp_arg)))
+		    new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
+						     loop_latch_edge (loop));
+
+		  /* Check if we've already created a new phi node during edge
+		     redirection and re-use it if so.  Otherwise create a
+		     LC PHI node to feed the merge PHI.  */
+		  tree *res;
+		  if (virtual_operand_p (new_arg))
+		    /* Use the existing virtual LC SSA from exit block.  */
+		    new_arg = gimple_phi_result
+				(get_virtual_phi (main_loop_exit_block));
+		  else if ((res = new_phi_args.get (new_arg)))
 		    new_arg = *res;
-		  else if (!virtual_operand_p (new_arg))
-		    new_arg = tmp_arg;
-		}
-
-	      tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
-	      gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
-
-	      /* Otherwise, main loop exit should use the final iter value.  */
-	      if (multiple_exits_p)
-		{
-		  /* Create a LC PHI if it doesn't already exist.  */
-		  if (!virtual_operand_p (new_arg) && !res)
+		  else
 		    {
+		      /* Create the LC PHI node for the exit.  */
 		      tree new_def = copy_ssa_name (new_arg);
 		      gphi *lc_phi
-			= create_phi_node (new_def, main_loop_exit_block);
+			  = create_phi_node (new_def, main_loop_exit_block);
 		      SET_PHI_ARG_DEF (lc_phi, 0, new_arg);
 		      new_arg = new_def;
 		    }
-		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi,
-					   single_succ_edge (main_loop_exit_block),
-					   new_arg);
-		}
-	      else
-		SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, loop_exit, new_arg);
 
-	      adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
-	    }
+		  /* Create the PHI node in the merge block merging the
+		     main and early exit values.  */
+		  tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
+		  gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
+		  edge main_e = single_succ_edge (main_loop_exit_block);
+		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, main_e, new_arg);
 
-	  /* Now fill in the values for the merge PHI in new_preheader
-	     for the alternative exits.  */
-	  if (multiple_exits_p)
-	    {
+		  /* And adjust the epilog entry value.  */
+		  adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
+		}
+
+	      /* After creating the merge PHIs handle the early exits those
+		 should use the values at the start of the loop.  */
 	      for (auto gsi_from = gsi_start_phis (loop->header),
 		   gsi_to = gsi_start_phis (new_preheader);
 		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
@@ -1790,8 +1773,38 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
 			SET_PHI_ARG_DEF (lc_phi, i, alt_arg);
 		      alt_arg = alt_def;
 		    }
-		  edge main_e = single_succ_edge (alt_loop_exit_block);
-		  SET_PHI_ARG_DEF_ON_EDGE (to_phi, main_e, alt_arg);
+		  edge alt_e = single_succ_edge (alt_loop_exit_block);
+		  SET_PHI_ARG_DEF_ON_EDGE (to_phi, alt_e, alt_arg);
+		}
+	    }
+	  /* For the single exit case only create the missing LC PHI nodes
+	     for the continuation of the loop IVs that are not also already
+	     reductions and thus had LC PHI nodes on the exit already.  */
+	  else
+	    {
+	      for (auto gsi_from = gsi_start_phis (loop->header),
+		   gsi_to = gsi_start_phis (new_loop->header);
+		   !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
+		   gsi_next (&gsi_from), gsi_next (&gsi_to))
+		{
+		  gimple *from_phi = gsi_stmt (gsi_from);
+		  gimple *to_phi = gsi_stmt (gsi_to);
+		  tree new_arg = PHI_ARG_DEF_FROM_EDGE (from_phi,
+							loop_latch_edge (loop));
+
+		  /* Check if we've already created a new phi node during edge
+		     redirection.  If we have, only propagate the value
+		     downwards.  */
+		  if (tree *res = new_phi_args.get (new_arg))
+		    {
+		      adjust_phi_and_debug_stmts (to_phi, loop_entry, *res);
+		      continue;
+		    }
+
+		  tree new_res = copy_ssa_name (gimple_phi_result (from_phi));
+		  gphi *lcssa_phi = create_phi_node (new_res, new_preheader);
+		  SET_PHI_ARG_DEF_ON_EDGE (lcssa_phi, loop_exit, new_arg);
+		  adjust_phi_and_debug_stmts (to_phi, loop_entry, new_res);
 		}
 	    }
 	}
-- 
2.35.3

^ permalink raw reply	[flat|nested] only message in thread

only message in thread, other threads:[~2024-01-22  9:03 UTC | newest]

Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-01-22  9:02 [PATCH] Refactor exit PHI handling in vectorizer epilogue peeling Richard Biener

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).