public inbox for gcc-patches@gcc.gnu.org
 help / color / mirror / Atom feed
* [PATCH] vect: Factor out the handling on scatter store having gs_info.decl
@ 2023-08-17  6:22 Kewen.Lin
  2023-08-18  9:33 ` Richard Biener
  0 siblings, 1 reply; 2+ messages in thread
From: Kewen.Lin @ 2023-08-17  6:22 UTC (permalink / raw)
  To: GCC Patches; +Cc: Richard Biener, Richard Sandiford

Hi,

Similar to the existing function vect_build_gather_load_calls,
this patch is to factor out the handling on scatter store
having gs_info.decl to vect_build_scatter_store_calls which
is a new function.  It also does some minor refactoring like
moving some variables' declarations close to their uses and
restrict the scope for some of them etc.

It's a pre-patch for upcoming vectorizable_store re-structuring
for costing.

Bootstrapped and regtested on x86_64-redhat-linux,
aarch64-linux-gnu and powerpc64{,le}-linux-gnu.

Is it ok for trunk?

BR,
Kewen
-----

gcc/ChangeLog:

	* tree-vect-stmts.cc (vect_build_scatter_store_calls): New, factor
	out from ...
	(vectorizable_store): ... here.
---
 gcc/tree-vect-stmts.cc | 411 +++++++++++++++++++++--------------------
 1 file changed, 212 insertions(+), 199 deletions(-)

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index cd8e0a76374..f8a904de503 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2989,6 +2989,216 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
   *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
 }

+/* Build a scatter store call while vectorizing STMT_INFO.  Insert new
+   instructions before GSI and add them to VEC_STMT.  GS_INFO describes
+   the scatter store operation.  If the store is conditional, MASK is the
+   unvectorized condition, otherwise MASK is null.  */
+
+static void
+vect_build_scatter_store_calls (vec_info *vinfo, stmt_vec_info stmt_info,
+				gimple_stmt_iterator *gsi, gimple **vec_stmt,
+				gather_scatter_info *gs_info, tree mask)
+{
+  loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+  int ncopies = vect_get_num_copies (loop_vinfo, vectype);
+  enum { NARROW, NONE, WIDEN } modifier;
+  poly_uint64 scatter_off_nunits
+    = TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype);
+
+  tree perm_mask = NULL_TREE, mask_halfvectype = NULL_TREE;
+  if (known_eq (nunits, scatter_off_nunits))
+    modifier = NONE;
+  else if (known_eq (nunits * 2, scatter_off_nunits))
+    {
+      modifier = WIDEN;
+
+      /* Currently gathers and scatters are only supported for
+	 fixed-length vectors.  */
+      unsigned int count = scatter_off_nunits.to_constant ();
+      vec_perm_builder sel (count, count, 1);
+      for (unsigned i = 0; i < (unsigned int) count; ++i)
+	sel.quick_push (i | (count / 2));
+
+      vec_perm_indices indices (sel, 1, count);
+      perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype, indices);
+      gcc_assert (perm_mask != NULL_TREE);
+    }
+  else if (known_eq (nunits, scatter_off_nunits * 2))
+    {
+      modifier = NARROW;
+
+      /* Currently gathers and scatters are only supported for
+	 fixed-length vectors.  */
+      unsigned int count = nunits.to_constant ();
+      vec_perm_builder sel (count, count, 1);
+      for (unsigned i = 0; i < (unsigned int) count; ++i)
+	sel.quick_push (i | (count / 2));
+
+      vec_perm_indices indices (sel, 2, count);
+      perm_mask = vect_gen_perm_mask_checked (vectype, indices);
+      gcc_assert (perm_mask != NULL_TREE);
+      ncopies *= 2;
+
+      if (mask)
+	mask_halfvectype = truth_type_for (gs_info->offset_vectype);
+    }
+  else
+    gcc_unreachable ();
+
+  tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
+  tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
+  tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+  tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+  tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+  tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
+  tree scaletype = TREE_VALUE (arglist);
+
+  gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
+		       && TREE_CODE (rettype) == VOID_TYPE);
+
+  tree ptr = fold_convert (ptrtype, gs_info->base);
+  if (!is_gimple_min_invariant (ptr))
+    {
+      gimple_seq seq;
+      ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
+      class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+      edge pe = loop_preheader_edge (loop);
+      basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
+      gcc_assert (!new_bb);
+    }
+
+  tree mask_arg = NULL_TREE;
+  if (mask == NULL_TREE)
+    {
+      mask_arg = build_int_cst (masktype, -1);
+      mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
+    }
+
+  tree scale = build_int_cst (scaletype, gs_info->scale);
+
+  auto_vec<tree> vec_oprnds0;
+  auto_vec<tree> vec_oprnds1;
+  auto_vec<tree> vec_masks;
+  if (mask)
+    {
+      tree mask_vectype = truth_type_for (vectype);
+      vect_get_vec_defs_for_operand (vinfo, stmt_info,
+				     modifier == NARROW ? ncopies / 2 : ncopies,
+				     mask, &vec_masks, mask_vectype);
+    }
+  vect_get_vec_defs_for_operand (vinfo, stmt_info,
+				 modifier == WIDEN ? ncopies / 2 : ncopies,
+				 gs_info->offset, &vec_oprnds0);
+  tree op = vect_get_store_rhs (stmt_info);
+  vect_get_vec_defs_for_operand (vinfo, stmt_info,
+				 modifier == NARROW ? ncopies / 2 : ncopies, op,
+				 &vec_oprnds1);
+
+  tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
+  tree mask_op = NULL_TREE;
+  tree src, vec_mask;
+  for (int j = 0; j < ncopies; ++j)
+    {
+      if (modifier == WIDEN)
+	{
+	  if (j & 1)
+	    op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0, perm_mask,
+				       stmt_info, gsi);
+	  else
+	    op = vec_oprnd0 = vec_oprnds0[j / 2];
+	  src = vec_oprnd1 = vec_oprnds1[j];
+	  if (mask)
+	    mask_op = vec_mask = vec_masks[j];
+	}
+      else if (modifier == NARROW)
+	{
+	  if (j & 1)
+	    src = permute_vec_elements (vinfo, vec_oprnd1, vec_oprnd1,
+					perm_mask, stmt_info, gsi);
+	  else
+	    src = vec_oprnd1 = vec_oprnds1[j / 2];
+	  op = vec_oprnd0 = vec_oprnds0[j];
+	  if (mask)
+	    mask_op = vec_mask = vec_masks[j / 2];
+	}
+      else
+	{
+	  op = vec_oprnd0 = vec_oprnds0[j];
+	  src = vec_oprnd1 = vec_oprnds1[j];
+	  if (mask)
+	    mask_op = vec_mask = vec_masks[j];
+	}
+
+      if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
+	{
+	  gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
+				TYPE_VECTOR_SUBPARTS (srctype)));
+	  tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
+	  src = build1 (VIEW_CONVERT_EXPR, srctype, src);
+	  gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
+	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	  src = var;
+	}
+
+      if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
+	{
+	  gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
+				TYPE_VECTOR_SUBPARTS (idxtype)));
+	  tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
+	  op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
+	  gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
+	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	  op = var;
+	}
+
+      if (mask)
+	{
+	  tree utype;
+	  mask_arg = mask_op;
+	  if (modifier == NARROW)
+	    {
+	      tree var
+		= vect_get_new_ssa_name (mask_halfvectype, vect_simple_var);
+	      gassign *new_stmt
+		= gimple_build_assign (var,
+				       (j & 1) ? VEC_UNPACK_HI_EXPR
+					       : VEC_UNPACK_LO_EXPR,
+				       mask_op);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      mask_arg = var;
+	    }
+	  tree optype = TREE_TYPE (mask_arg);
+	  if (TYPE_MODE (masktype) == TYPE_MODE (optype))
+	    utype = masktype;
+	  else
+	    utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
+	  tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
+	  mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
+	  gassign *new_stmt
+	    = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
+	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	  mask_arg = var;
+	  if (!useless_type_conversion_p (masktype, utype))
+	    {
+	      gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
+	      tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
+	      new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      mask_arg = var;
+	    }
+	}
+
+      gcall *new_stmt
+	= gimple_build_call (gs_info->decl, 5, ptr, mask_arg, op, src, scale);
+      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+
+      STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+    }
+  *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+}
+
 /* Prepare the base and offset in GS_INFO for vectorization.
    Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
    to the vectorized offset argument for the first copy of STMT_INFO.
@@ -8212,205 +8422,8 @@ vectorizable_store (vec_info *vinfo,

   if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
     {
-      tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, src;
-      tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
-      tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
-      tree ptr, var, scale, vec_mask;
-      tree mask_arg = NULL_TREE, mask_op = NULL_TREE, perm_mask = NULL_TREE;
-      tree mask_halfvectype = mask_vectype;
-      edge pe = loop_preheader_edge (loop);
-      gimple_seq seq;
-      basic_block new_bb;
-      enum { NARROW, NONE, WIDEN } modifier;
-      poly_uint64 scatter_off_nunits
-	= TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
-
-      if (known_eq (nunits, scatter_off_nunits))
-	modifier = NONE;
-      else if (known_eq (nunits * 2, scatter_off_nunits))
-	{
-	  modifier = WIDEN;
-
-	  /* Currently gathers and scatters are only supported for
-	     fixed-length vectors.  */
-	  unsigned int count = scatter_off_nunits.to_constant ();
-	  vec_perm_builder sel (count, count, 1);
-	  for (i = 0; i < (unsigned int) count; ++i)
-	    sel.quick_push (i | (count / 2));
-
-	  vec_perm_indices indices (sel, 1, count);
-	  perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype,
-						  indices);
-	  gcc_assert (perm_mask != NULL_TREE);
-	}
-      else if (known_eq (nunits, scatter_off_nunits * 2))
-	{
-	  modifier = NARROW;
-
-	  /* Currently gathers and scatters are only supported for
-	     fixed-length vectors.  */
-	  unsigned int count = nunits.to_constant ();
-	  vec_perm_builder sel (count, count, 1);
-	  for (i = 0; i < (unsigned int) count; ++i)
-	    sel.quick_push (i | (count / 2));
-
-	  vec_perm_indices indices (sel, 2, count);
-	  perm_mask = vect_gen_perm_mask_checked (vectype, indices);
-	  gcc_assert (perm_mask != NULL_TREE);
-	  ncopies *= 2;
-
-	  if (mask)
-	    mask_halfvectype = truth_type_for (gs_info.offset_vectype);
-	}
-      else
-	gcc_unreachable ();
-
-      rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
-      ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
-      masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
-      idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
-      srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
-      scaletype = TREE_VALUE (arglist);
-
-      gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
-			   && TREE_CODE (rettype) == VOID_TYPE);
-
-      ptr = fold_convert (ptrtype, gs_info.base);
-      if (!is_gimple_min_invariant (ptr))
-	{
-	  ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
-	  new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
-	  gcc_assert (!new_bb);
-	}
-
-      if (mask == NULL_TREE)
-	{
-	  mask_arg = build_int_cst (masktype, -1);
-	  mask_arg = vect_init_vector (vinfo, stmt_info,
-				       mask_arg, masktype, NULL);
-	}
-
-      scale = build_int_cst (scaletype, gs_info.scale);
-
-      auto_vec<tree> vec_oprnds0;
-      auto_vec<tree> vec_oprnds1;
-      auto_vec<tree> vec_masks;
-      if (mask)
-	{
-	  tree mask_vectype = truth_type_for (vectype);
-	  vect_get_vec_defs_for_operand (vinfo, stmt_info,
-					 modifier == NARROW
-					 ? ncopies / 2 : ncopies,
-					 mask, &vec_masks, mask_vectype);
-	}
-      vect_get_vec_defs_for_operand (vinfo, stmt_info,
-				     modifier == WIDEN
-				     ? ncopies / 2 : ncopies,
-				     gs_info.offset, &vec_oprnds0);
-      vect_get_vec_defs_for_operand (vinfo, stmt_info,
-				     modifier == NARROW
-				     ? ncopies / 2 : ncopies,
-				     op, &vec_oprnds1);
-      for (j = 0; j < ncopies; ++j)
-	{
-	  if (modifier == WIDEN)
-	    {
-	      if (j & 1)
-		op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
-					   perm_mask, stmt_info, gsi);
-	      else
-		op = vec_oprnd0 = vec_oprnds0[j / 2];
-	      src = vec_oprnd1 = vec_oprnds1[j];
-	      if (mask)
-		mask_op = vec_mask = vec_masks[j];
-	    }
-	  else if (modifier == NARROW)
-	    {
-	      if (j & 1)
-		src = permute_vec_elements (vinfo, vec_oprnd1, vec_oprnd1,
-					    perm_mask, stmt_info, gsi);
-	      else
-		src = vec_oprnd1 = vec_oprnds1[j / 2];
-	      op = vec_oprnd0 = vec_oprnds0[j];
-	      if (mask)
-		mask_op = vec_mask = vec_masks[j / 2];
-	    }
-	  else
-	    {
-	      op = vec_oprnd0 = vec_oprnds0[j];
-	      src = vec_oprnd1 = vec_oprnds1[j];
-	      if (mask)
-		mask_op = vec_mask = vec_masks[j];
-	    }
-
-	  if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
-	    {
-	      gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
-				    TYPE_VECTOR_SUBPARTS (srctype)));
-	      var = vect_get_new_ssa_name (srctype, vect_simple_var);
-	      src = build1 (VIEW_CONVERT_EXPR, srctype, src);
-	      gassign *new_stmt
-		= gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	      src = var;
-	    }
-
-	  if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
-	    {
-	      gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
-				    TYPE_VECTOR_SUBPARTS (idxtype)));
-	      var = vect_get_new_ssa_name (idxtype, vect_simple_var);
-	      op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
-	      gassign *new_stmt
-		= gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	      op = var;
-	    }
-
-	  if (mask)
-	    {
-	      tree utype;
-	      mask_arg = mask_op;
-	      if (modifier == NARROW)
-		{
-		  var = vect_get_new_ssa_name (mask_halfvectype,
-					       vect_simple_var);
-		  gassign *new_stmt
-		    = gimple_build_assign (var, (j & 1) ? VEC_UNPACK_HI_EXPR
-							: VEC_UNPACK_LO_EXPR,
-					   mask_op);
-		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-		  mask_arg = var;
-		}
-	      tree optype = TREE_TYPE (mask_arg);
-	      if (TYPE_MODE (masktype) == TYPE_MODE (optype))
-		utype = masktype;
-	      else
-		utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
-	      var = vect_get_new_ssa_name (utype, vect_scalar_var);
-	      mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
-	      gassign *new_stmt
-		= gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	      mask_arg = var;
-	      if (!useless_type_conversion_p (masktype, utype))
-		{
-		  gcc_assert (TYPE_PRECISION (utype)
-			      <= TYPE_PRECISION (masktype));
-		  var = vect_get_new_ssa_name (masktype, vect_scalar_var);
-		  new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
-		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-		  mask_arg = var;
-		}
-	    }
-
-	  gcall *new_stmt
-	    = gimple_build_call (gs_info.decl, 5, ptr, mask_arg, op, src, scale);
-	   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
-	}
-      *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+      vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
+				      &gs_info, mask);
       return true;
     }
   else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
--
2.31.1

^ permalink raw reply	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-08-18  9:33 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-08-17  6:22 [PATCH] vect: Factor out the handling on scatter store having gs_info.decl Kewen.Lin
2023-08-18  9:33 ` Richard Biener

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).