[gcc r14-4577] vect: Adjust vectorizable_store costing on VMAT_LOAD_STORE_LANES

[gcc r14-4577] vect: Adjust vectorizable_store costing on VMAT_LOAD_STORE_LANES

[Kewen Lin]

https://gcc.gnu.org/g:6a88202e5a43f7b9cafd9fbfca74e3c53ea0f911

commit r14-4577-g6a88202e5a43f7b9cafd9fbfca74e3c53ea0f911
Author: Kewen Lin <linkw@linux.ibm.com>
Date:   Thu Oct 12 00:04:57 2023 -0500

    vect: Adjust vectorizable_store costing on VMAT_LOAD_STORE_LANES
    
    This patch adjusts the cost handling on VMAT_LOAD_STORE_LANES
    in function vectorizable_store.  We don't call function
    vect_model_store_cost for it any more.  It's the case of
    interleaving stores, so it skips all stmts excepting for
    first_stmt_info, consider the whole group when costing
    first_stmt_info.  This patch shouldn't have any functional
    changes.
    
    gcc/ChangeLog:
    
            * tree-vect-stmts.cc (vect_model_store_cost): Assert it will never
            get VMAT_LOAD_STORE_LANES.
            (vectorizable_store): Adjust the cost handling on VMAT_LOAD_STORE_LANES
            without calling vect_model_store_cost.  Factor out new lambda function
            update_prologue_cost.

Diff:
---
 gcc/tree-vect-stmts.cc | 110 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 75 insertions(+), 35 deletions(-)

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 3d01168080a..fbd16b8a487 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -966,7 +966,8 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
 {
   gcc_assert (memory_access_type != VMAT_GATHER_SCATTER
 	      && memory_access_type != VMAT_ELEMENTWISE
-	      && memory_access_type != VMAT_STRIDED_SLP);
+	      && memory_access_type != VMAT_STRIDED_SLP
+	      && memory_access_type != VMAT_LOAD_STORE_LANES);
   unsigned int inside_cost = 0, prologue_cost = 0;
   stmt_vec_info first_stmt_info = stmt_info;
   bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
@@ -8408,7 +8409,8 @@ vectorizable_store (vec_info *vinfo,
       if (grouped_store
 	  && !slp
 	  && first_stmt_info != stmt_info
-	  && memory_access_type == VMAT_ELEMENTWISE)
+	  && (memory_access_type == VMAT_ELEMENTWISE
+	      || memory_access_type == VMAT_LOAD_STORE_LANES))
 	return true;
     }
   gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
@@ -8479,6 +8481,31 @@ vectorizable_store (vec_info *vinfo,
     dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies = %d\n",
 		     ncopies);
 
+  /* Check if we need to update prologue cost for invariant,
+     and update it accordingly if so.  If it's not for
+     interleaving store, we can just check vls_type; but if
+     it's for interleaving store, need to check the def_type
+     of the stored value since the current vls_type is just
+     for first_stmt_info.  */
+  auto update_prologue_cost = [&](unsigned *prologue_cost, tree store_rhs)
+  {
+    gcc_assert (costing_p);
+    if (slp)
+      return;
+    if (grouped_store)
+      {
+	gcc_assert (store_rhs);
+	enum vect_def_type cdt;
+	gcc_assert (vect_is_simple_use (store_rhs, vinfo, &cdt));
+	if (cdt != vect_constant_def && cdt != vect_external_def)
+	  return;
+      }
+    else if (vls_type != VLS_STORE_INVARIANT)
+      return;
+    *prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec, stmt_info,
+					0, vect_prologue);
+  };
+
   if (memory_access_type == VMAT_ELEMENTWISE
       || memory_access_type == VMAT_STRIDED_SLP)
     {
@@ -8646,14 +8673,8 @@ vectorizable_store (vec_info *vinfo,
 	  if (!costing_p)
 	    vect_get_vec_defs (vinfo, next_stmt_info, slp_node, ncopies, op,
 			       &vec_oprnds);
-	  else if (!slp)
-	    {
-	      enum vect_def_type cdt;
-	      gcc_assert (vect_is_simple_use (op, vinfo, &cdt));
-	      if (cdt == vect_constant_def || cdt == vect_external_def)
-		prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
-						   stmt_info, 0, vect_prologue);
-	    }
+	  else
+	    update_prologue_cost (&prologue_cost, op);
 	  unsigned int group_el = 0;
 	  unsigned HOST_WIDE_INT
 	    elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
@@ -8857,13 +8878,7 @@ vectorizable_store (vec_info *vinfo,
   if (memory_access_type == VMAT_LOAD_STORE_LANES)
     {
       gcc_assert (!slp && grouped_store);
-      if (costing_p)
-	{
-	  vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
-				 alignment_support_scheme, misalignment,
-				 vls_type, slp_node, cost_vec);
-	  return true;
-	}
+      unsigned inside_cost = 0, prologue_cost = 0;
       for (j = 0; j < ncopies; j++)
 	{
 	  gimple *new_stmt;
@@ -8879,29 +8894,39 @@ vectorizable_store (vec_info *vinfo,
 		     DR_GROUP_SIZE is the exact number of stmts in the
 		     chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE.  */
 		  op = vect_get_store_rhs (next_stmt_info);
-		  vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
-						 op, gvec_oprnds[i]);
-		  vec_oprnd = (*gvec_oprnds[i])[0];
-		  dr_chain.quick_push (vec_oprnd);
+		  if (costing_p)
+		    update_prologue_cost (&prologue_cost, op);
+		  else
+		    {
+		      vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
+						     ncopies, op,
+						     gvec_oprnds[i]);
+		      vec_oprnd = (*gvec_oprnds[i])[0];
+		      dr_chain.quick_push (vec_oprnd);
+		    }
 		  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
 		}
-	      if (mask)
+
+	      if (!costing_p)
 		{
-		  vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-						 mask, &vec_masks,
-						 mask_vectype);
-		  vec_mask = vec_masks[0];
-		}
+		  if (mask)
+		    {
+		      vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
+						     mask, &vec_masks,
+						     mask_vectype);
+		      vec_mask = vec_masks[0];
+		    }
 
-	      /* We should have catched mismatched types earlier.  */
-	      gcc_assert (
-		useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
-	      dataref_ptr
-		= vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
-					    NULL, offset, &dummy, gsi,
-					    &ptr_incr, false, bump);
+		  /* We should have catched mismatched types earlier.  */
+		  gcc_assert (
+		    useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
+		  dataref_ptr
+		    = vect_create_data_ref_ptr (vinfo, first_stmt_info,
+						aggr_type, NULL, offset, &dummy,
+						gsi, &ptr_incr, false, bump);
+		}
 	    }
-	  else
+	  else if (!costing_p)
 	    {
 	      gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
 	      /* DR_CHAIN is then used as an input to
@@ -8917,6 +8942,15 @@ vectorizable_store (vec_info *vinfo,
 					     stmt_info, bump);
 	    }
 
+	  if (costing_p)
+	    {
+	      for (i = 0; i < vec_num; i++)
+		vect_get_store_cost (vinfo, stmt_info, 1,
+				     alignment_support_scheme, misalignment,
+				     &inside_cost, cost_vec);
+	      continue;
+	    }
+
 	  /* Get an array into which we can store the individual vectors.  */
 	  tree vec_array = create_vector_array (vectype, vec_num);
 
@@ -9003,6 +9037,12 @@ vectorizable_store (vec_info *vinfo,
 	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
 	}
 
+      if (costing_p && dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location,
+			 "vect_model_store_cost: inside_cost = %d, "
+			 "prologue_cost = %d .\n",
+			 inside_cost, prologue_cost);
+
       return true;
     }