[PATCH 4/6] vect: Bind input vectype to lane-reducing operation

[PATCH 4/6] vect: Bind input vectype to lane-reducing operation

[Feng Xue OS]

[-- Attachment #1: Type: text/plain, Size: 3886 bytes --]

The input vectype is an attribute of lane-reducing operation, instead of
reduction PHI that it is associated to, since there might be more than one
lane-reducing operations with different type in a loop reduction chain. So
bind each lane-reducing operation with its own input type.

Thanks,
Feng
---
gcc/
	* tree-vect-loop.cc (vect_is_emulated_mixed_dot_prod): Remove parameter
    	loop_vinfo. Get input vectype from stmt_info instead of reduction PHI.
	(vect_model_reduction_cost): Remove loop_vinfo argument of call to
	vect_is_emulated_mixed_dot_prod.
	(vect_transform_reduction): Likewise.
	(vectorizable_reduction): Likewise, and bind input vectype to
	lane-reducing operation.
---
 gcc/tree-vect-loop.cc | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 51627c27f8a..20c99f11e9a 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -5270,8 +5270,7 @@ have_whole_vector_shift (machine_mode mode)
    See vect_emulate_mixed_dot_prod for the actual sequence used.  */
 
 static bool
-vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
-				 stmt_vec_info stmt_info)
+vect_is_emulated_mixed_dot_prod (stmt_vec_info stmt_info)
 {
   gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
   if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
@@ -5282,10 +5281,9 @@ vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
   if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
     return false;
 
-  stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
-  gcc_assert (reduc_info->is_reduc_info);
+  gcc_assert (STMT_VINFO_REDUC_VECTYPE_IN (stmt_info));
   return !directly_supported_p (DOT_PROD_EXPR,
-				STMT_VINFO_REDUC_VECTYPE_IN (reduc_info),
+				STMT_VINFO_REDUC_VECTYPE_IN (stmt_info),
 				optab_vector_mixed_sign);
 }
 
@@ -5324,8 +5322,8 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
   if (!gimple_extract_op (orig_stmt_info->stmt, &op))
     gcc_unreachable ();
 
-  bool emulated_mixed_dot_prod
-    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
+  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
+
   if (reduction_type == EXTRACT_LAST_REDUCTION)
     /* No extra instructions are needed in the prologue.  The loop body
        operations are costed in vectorizable_condition.  */
@@ -7840,6 +7838,11 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
     vectype_in = STMT_VINFO_VECTYPE (phi_info);
   STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
 
+  /* Each lane-reducing operation has its own input vectype, while reduction
+     PHI records the input vectype with least lanes.  */
+  if (lane_reducing)
+    STMT_VINFO_REDUC_VECTYPE_IN (stmt_info) = vectype_in;
+
   enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
   STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
   /* If we have a condition reduction, see if we can simplify it further.  */
@@ -8366,7 +8369,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
   if (single_defuse_cycle || lane_reducing)
     {
       int factor = 1;
-      if (vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info))
+      if (vect_is_emulated_mixed_dot_prod (stmt_info))
 	/* Three dot-products and a subtraction.  */
 	factor = 4;
       record_stmt_cost (cost_vec, ncopies * factor, vector_stmt,
@@ -8617,8 +8620,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 					: &vec_oprnds2));
     }
 
-  bool emulated_mixed_dot_prod
-    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
+  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
+
   FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
     {
       gimple *new_stmt;
-- 
2.17.1

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0004-vect-Bind-input-vectype-to-lane-reducing-operation.patch --]
[-- Type: text/x-patch; name="0004-vect-Bind-input-vectype-to-lane-reducing-operation.patch", Size: 4059 bytes --]

From b885de76ad7e9f5accceff18cb6c11de73a36225 Mon Sep 17 00:00:00 2001
From: Feng Xue <fxue@os.amperecomputing.com>
Date: Wed, 29 May 2024 16:41:57 +0800
Subject: [PATCH 4/6] vect: Bind input vectype to lane-reducing operation

The input vectype is an attribute of lane-reducing operation, instead of
reduction PHI that it is associated to, since there might be more than one
lane-reducing operations with different type in a loop reduction chain. So
bind each lane-reducing operation with its own input type.

2024-05-29 Feng Xue <fxue@os.amperecomputing.com>

gcc/
	* tree-vect-loop.cc (vect_is_emulated_mixed_dot_prod): Remove parameter
    	loop_vinfo. Get input vectype from stmt_info instead of reduction PHI.
	(vect_model_reduction_cost): Remove loop_vinfo argument of call to
	vect_is_emulated_mixed_dot_prod.
	(vect_transform_reduction): Likewise.
	(vectorizable_reduction): Likewise, and bind input vectype to
	lane-reducing operation.
---
 gcc/tree-vect-loop.cc | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 51627c27f8a..20c99f11e9a 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -5270,8 +5270,7 @@ have_whole_vector_shift (machine_mode mode)
    See vect_emulate_mixed_dot_prod for the actual sequence used.  */
 
 static bool
-vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
-				 stmt_vec_info stmt_info)
+vect_is_emulated_mixed_dot_prod (stmt_vec_info stmt_info)
 {
   gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
   if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
@@ -5282,10 +5281,9 @@ vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
   if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
     return false;
 
-  stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
-  gcc_assert (reduc_info->is_reduc_info);
+  gcc_assert (STMT_VINFO_REDUC_VECTYPE_IN (stmt_info));
   return !directly_supported_p (DOT_PROD_EXPR,
-				STMT_VINFO_REDUC_VECTYPE_IN (reduc_info),
+				STMT_VINFO_REDUC_VECTYPE_IN (stmt_info),
 				optab_vector_mixed_sign);
 }
 
@@ -5324,8 +5322,8 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
   if (!gimple_extract_op (orig_stmt_info->stmt, &op))
     gcc_unreachable ();
 
-  bool emulated_mixed_dot_prod
-    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
+  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
+
   if (reduction_type == EXTRACT_LAST_REDUCTION)
     /* No extra instructions are needed in the prologue.  The loop body
        operations are costed in vectorizable_condition.  */
@@ -7840,6 +7838,11 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
     vectype_in = STMT_VINFO_VECTYPE (phi_info);
   STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
 
+  /* Each lane-reducing operation has its own input vectype, while reduction
+     PHI records the input vectype with least lanes.  */
+  if (lane_reducing)
+    STMT_VINFO_REDUC_VECTYPE_IN (stmt_info) = vectype_in;
+
   enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
   STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
   /* If we have a condition reduction, see if we can simplify it further.  */
@@ -8366,7 +8369,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
   if (single_defuse_cycle || lane_reducing)
     {
       int factor = 1;
-      if (vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info))
+      if (vect_is_emulated_mixed_dot_prod (stmt_info))
 	/* Three dot-products and a subtraction.  */
 	factor = 4;
       record_stmt_cost (cost_vec, ncopies * factor, vector_stmt,
@@ -8617,8 +8620,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 					: &vec_oprnds2));
     }
 
-  bool emulated_mixed_dot_prod
-    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
+  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
+
   FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
     {
       gimple *new_stmt;
-- 
2.17.1

Re: [PATCH 4/6] vect: Bind input vectype to lane-reducing operation

[Richard Biener]

On Thu, May 30, 2024 at 4:53 PM Feng Xue OS <fxue@os.amperecomputing.com> wrote:
>
> The input vectype is an attribute of lane-reducing operation, instead of
> reduction PHI that it is associated to, since there might be more than one
> lane-reducing operations with different type in a loop reduction chain. So
> bind each lane-reducing operation with its own input type.

OK.

Thanks,
Richard.

> Thanks,
> Feng
> ---
> gcc/
>         * tree-vect-loop.cc (vect_is_emulated_mixed_dot_prod): Remove parameter
>         loop_vinfo. Get input vectype from stmt_info instead of reduction PHI.
>         (vect_model_reduction_cost): Remove loop_vinfo argument of call to
>         vect_is_emulated_mixed_dot_prod.
>         (vect_transform_reduction): Likewise.
>         (vectorizable_reduction): Likewise, and bind input vectype to
>         lane-reducing operation.
> ---
>  gcc/tree-vect-loop.cc | 23 +++++++++++++----------
>  1 file changed, 13 insertions(+), 10 deletions(-)
>
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 51627c27f8a..20c99f11e9a 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -5270,8 +5270,7 @@ have_whole_vector_shift (machine_mode mode)
>     See vect_emulate_mixed_dot_prod for the actual sequence used.  */
>
>  static bool
> -vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
> -                                stmt_vec_info stmt_info)
> +vect_is_emulated_mixed_dot_prod (stmt_vec_info stmt_info)
>  {
>    gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
>    if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
> @@ -5282,10 +5281,9 @@ vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo,
>    if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
>      return false;
>
> -  stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
> -  gcc_assert (reduc_info->is_reduc_info);
> +  gcc_assert (STMT_VINFO_REDUC_VECTYPE_IN (stmt_info));
>    return !directly_supported_p (DOT_PROD_EXPR,
> -                               STMT_VINFO_REDUC_VECTYPE_IN (reduc_info),
> +                               STMT_VINFO_REDUC_VECTYPE_IN (stmt_info),
>                                 optab_vector_mixed_sign);
>  }
>
> @@ -5324,8 +5322,8 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
>    if (!gimple_extract_op (orig_stmt_info->stmt, &op))
>      gcc_unreachable ();
>
> -  bool emulated_mixed_dot_prod
> -    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
> +  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
> +
>    if (reduction_type == EXTRACT_LAST_REDUCTION)
>      /* No extra instructions are needed in the prologue.  The loop body
>         operations are costed in vectorizable_condition.  */
> @@ -7840,6 +7838,11 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>      vectype_in = STMT_VINFO_VECTYPE (phi_info);
>    STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
>
> +  /* Each lane-reducing operation has its own input vectype, while reduction
> +     PHI records the input vectype with least lanes.  */
> +  if (lane_reducing)
> +    STMT_VINFO_REDUC_VECTYPE_IN (stmt_info) = vectype_in;
> +
>    enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
>    STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
>    /* If we have a condition reduction, see if we can simplify it further.  */
> @@ -8366,7 +8369,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
>    if (single_defuse_cycle || lane_reducing)
>      {
>        int factor = 1;
> -      if (vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info))
> +      if (vect_is_emulated_mixed_dot_prod (stmt_info))
>         /* Three dot-products and a subtraction.  */
>         factor = 4;
>        record_stmt_cost (cost_vec, ncopies * factor, vector_stmt,
> @@ -8617,8 +8620,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
>                                         : &vec_oprnds2));
>      }
>
> -  bool emulated_mixed_dot_prod
> -    = vect_is_emulated_mixed_dot_prod (loop_vinfo, stmt_info);
> +  bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (stmt_info);
> +
>    FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
>      {
>        gimple *new_stmt;
> --
> 2.17.1