public inbox for gcc-patches@gcc.gnu.org
 help / color / mirror / Atom feed
* [vect-patterns] Refactor widen_plus/widen_minus as internal_fns
@ 2021-11-11 15:13 Joel Hutton
  2021-11-12 10:48 ` Richard Biener
  0 siblings, 1 reply; 6+ messages in thread
From: Joel Hutton @ 2021-11-11 15:13 UTC (permalink / raw)
  To: gcc-patches; +Cc: Richard Sandiford, Richard Biener

[-- Attachment #1: Type: text/plain, Size: 331 bytes --]

Hi all,

This refactor allows widening vect patterns (such as widen_plus/widen_minus) to be represented as
either internal_fns or tree_codes and replaces the current widen_plus/widen_minus with internal_fn versions. This refactor is split into 3 patches.

Boostrapped and regression tested on aarch64.

Ok for stage 3?



[-- Attachment #2: 0001-vect-patterns-Refactor-to-allow-internal_fn-s.patch --]
[-- Type: application/octet-stream, Size: 25485 bytes --]

From 5959fa8ee9fe04595d70bd271e75959bdc20a9f1 Mon Sep 17 00:00:00 2001
From: Joel Hutton <joel.hutton@arm.com>
Date: Wed, 25 Aug 2021 14:31:15 +0100
Subject: [PATCH 1/3] [vect-patterns] Refactor to allow internal_fn's

Hi all,

This refactor allows widening patterns (such as widen_plus/widen_minus) to be represented as
either internal_fns or tree_codes.

[vect-patterns] Refactor as internal_fn's

Refactor vect-patterns to allow patterns to be internal_fns starting
with widening_plus/minus patterns

gcc/ChangeLog:

	* gimple-match.h (class code_helper): Move code_helper class to more
    visible function.
	* internal-fn.h (internal_fn_name): Add internal_fn range check.
	* tree-vect-patterns.c (vect_recog_widen_op_pattern): Change
    function prototype.
	* tree-vect-stmts.c (vect_gen_widened_results_half): Refactor to use
    code_helper, build internal_fns
	(vect_create_vectorized_promotion_stmts): Refactor to use
    code_helper.
	(vectorizable_conversion): Refactor to use code_helper.
	(supportable_widening_operation): Refactor to use code_helper.
	(supportable_narrowing_operation): Refactor to use code_helper.
	* tree-vectorizer.h (supportable_widening_operation): Refactor to use code_helper.
	(supportable_narrowing_operation): Refactor to use code_helper.
	* tree.h (class code_helper):  Refactor to use code_helper.
---
 gcc/gimple-match.h       |  17 ----
 gcc/internal-fn.h        |   1 +
 gcc/tree-vect-patterns.c |  21 ++--
 gcc/tree-vect-stmts.c    | 206 ++++++++++++++++++++++-----------------
 gcc/tree-vectorizer.h    |  15 +--
 gcc/tree.h               |  26 +++++
 6 files changed, 170 insertions(+), 116 deletions(-)

diff --git a/gcc/gimple-match.h b/gcc/gimple-match.h
index 2d4ea476076..40033ac7524 100644
--- a/gcc/gimple-match.h
+++ b/gcc/gimple-match.h
@@ -23,23 +23,6 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_GIMPLE_MATCH_H
 
 
-/* Helper to transparently allow tree codes and builtin function codes
-   exist in one storage entity.  */
-class code_helper
-{
-public:
-  code_helper () {}
-  code_helper (tree_code code) : rep ((int) code) {}
-  code_helper (combined_fn fn) : rep (-(int) fn) {}
-  operator tree_code () const { return (tree_code) rep; }
-  operator combined_fn () const { return (combined_fn) -rep; }
-  bool is_tree_code () const { return rep > 0; }
-  bool is_fn_code () const { return rep < 0; }
-  int get_rep () const { return rep; }
-private:
-  int rep;
-};
-
 /* Represents the condition under which an operation should happen,
    and the value to use otherwise.  The condition applies elementwise
    (as for VEC_COND_EXPR) if the values are vectors.  */
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index 19d0f849a5a..06e540ffd6a 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -108,6 +108,7 @@ extern const char *const internal_fn_name_array[];
 static inline const char *
 internal_fn_name (enum internal_fn fn)
 {
+  gcc_assert ((int) fn <= (int) IFN_NOP);
   return internal_fn_name_array[(int) fn];
 }
 
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 854cbcff390..4a8ea67e62f 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -1245,7 +1245,7 @@ vect_recog_sad_pattern (vec_info *vinfo,
 static gimple *
 vect_recog_widen_op_pattern (vec_info *vinfo,
 			     stmt_vec_info last_stmt_info, tree *type_out,
-			     tree_code orig_code, tree_code wide_code,
+			     tree_code orig_code, code_helper wide_code_or_ifn,
 			     bool shift_p, const char *name)
 {
   gimple *last_stmt = last_stmt_info->stmt;
@@ -1288,15 +1288,16 @@ vect_recog_widen_op_pattern (vec_info *vinfo,
       vecctype = get_vectype_for_scalar_type (vinfo, ctype);
     }
 
-  enum tree_code dummy_code;
+  code_helper dummy_c_or_ifn;
   int dummy_int;
   auto_vec<tree> dummy_vec;
   if (!vectype
       || !vecitype
       || !vecctype
-      || !supportable_widening_operation (vinfo, wide_code, last_stmt_info,
+      || !supportable_widening_operation (vinfo, wide_code_or_ifn,
+					  last_stmt_info,
 					  vecitype, vectype,
-					  &dummy_code, &dummy_code,
+					  &dummy_c_or_ifn, &dummy_c_or_ifn,
 					  &dummy_int, &dummy_vec))
     return NULL;
 
@@ -1309,8 +1310,16 @@ vect_recog_widen_op_pattern (vec_info *vinfo,
 		       2, oprnd, half_type, unprom, vectype);
 
   tree var = vect_recog_temp_ssa_var (itype, NULL);
-  gimple *pattern_stmt = gimple_build_assign (var, wide_code,
-					      oprnd[0], oprnd[1]);
+  gimple *pattern_stmt;
+  if (wide_code_or_ifn.is_tree_code ())
+    pattern_stmt = gimple_build_assign (var, wide_code_or_ifn,
+						oprnd[0], oprnd[1]);
+  else
+    {
+      internal_fn fn = as_internal_fn ((combined_fn) wide_code_or_ifn);
+      pattern_stmt = gimple_build_call_internal (fn, 2, oprnd[0], oprnd[1]);
+      gimple_call_set_lhs (pattern_stmt, var);
+    }
 
   if (vecctype != vecitype)
     pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 2284ad069e4..d5e1619fabc 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -4504,7 +4504,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
    STMT_INFO is the original scalar stmt that we are vectorizing.  */
 
 static gimple *
-vect_gen_widened_results_half (vec_info *vinfo, enum tree_code code,
+vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
                                tree vec_oprnd0, tree vec_oprnd1, int op_type,
 			       tree vec_dest, gimple_stmt_iterator *gsi,
 			       stmt_vec_info stmt_info)
@@ -4513,14 +4513,16 @@ vect_gen_widened_results_half (vec_info *vinfo, enum tree_code code,
   tree new_temp;
 
   /* Generate half of the widened result:  */
-  gcc_assert (op_type == TREE_CODE_LENGTH (code));
   if (op_type != binary_op)
     vec_oprnd1 = NULL;
-  new_stmt = gimple_build_assign (vec_dest, code, vec_oprnd0, vec_oprnd1);
+  if (ch.is_tree_code ())
+    new_stmt = gimple_build_assign (vec_dest, ch, vec_oprnd0, vec_oprnd1);
+  else
+    new_stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
+					   2, vec_oprnd0, vec_oprnd1);
   new_temp = make_ssa_name (vec_dest, new_stmt);
-  gimple_assign_set_lhs (new_stmt, new_temp);
+  gimple_set_lhs (new_stmt, new_temp);
   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-
   return new_stmt;
 }
 
@@ -4597,8 +4599,8 @@ vect_create_vectorized_promotion_stmts (vec_info *vinfo,
 					vec<tree> *vec_oprnds1,
 					stmt_vec_info stmt_info, tree vec_dest,
 					gimple_stmt_iterator *gsi,
-					enum tree_code code1,
-					enum tree_code code2, int op_type)
+					code_helper ch1,
+					code_helper ch2, int op_type)
 {
   int i;
   tree vop0, vop1, new_tmp1, new_tmp2;
@@ -4614,10 +4616,10 @@ vect_create_vectorized_promotion_stmts (vec_info *vinfo,
 	vop1 = NULL_TREE;
 
       /* Generate the two halves of promotion operation.  */
-      new_stmt1 = vect_gen_widened_results_half (vinfo, code1, vop0, vop1,
+      new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
 						 op_type, vec_dest, gsi,
 						 stmt_info);
-      new_stmt2 = vect_gen_widened_results_half (vinfo, code2, vop0, vop1,
+      new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
 						 op_type, vec_dest, gsi,
 						 stmt_info);
       if (is_gimple_call (new_stmt1))
@@ -4714,8 +4716,9 @@ vectorizable_conversion (vec_info *vinfo,
   tree scalar_dest;
   tree op0, op1 = NULL_TREE;
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
-  enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
+  tree_code code1;
+  code_helper code_or_ifn, code_or_ifn1, code_or_ifn2;
+  code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
   tree new_temp;
   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
   int ndts = 2;
@@ -4744,31 +4747,28 @@ vectorizable_conversion (vec_info *vinfo,
       && ! vec_stmt)
     return false;
 
-  gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
-  if (!stmt)
+  gimple* stmt = stmt_info->stmt;
+  if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
     return false;
 
-  if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
-    return false;
+  if (is_gimple_assign (stmt))
+  {
+    code_or_ifn = gimple_assign_rhs_code (stmt);
+  }
+  else
+    code_or_ifn = gimple_call_combined_fn (stmt);
 
-  code = gimple_assign_rhs_code (stmt);
-  if (!CONVERT_EXPR_CODE_P (code)
-      && code != FIX_TRUNC_EXPR
-      && code != FLOAT_EXPR
-      && code != WIDEN_PLUS_EXPR
-      && code != WIDEN_MINUS_EXPR
-      && code != WIDEN_MULT_EXPR
-      && code != WIDEN_LSHIFT_EXPR)
+  if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
     return false;
 
-  bool widen_arith = (code == WIDEN_PLUS_EXPR
-		      || code == WIDEN_MINUS_EXPR
-		      || code == WIDEN_MULT_EXPR
-		      || code == WIDEN_LSHIFT_EXPR);
-  op_type = TREE_CODE_LENGTH (code);
+  bool widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR
+		      || code_or_ifn == WIDEN_MINUS_EXPR
+		      || code_or_ifn == WIDEN_MULT_EXPR
+		      || code_or_ifn == WIDEN_LSHIFT_EXPR);
+  op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn);
 
   /* Check types of lhs and rhs.  */
-  scalar_dest = gimple_assign_lhs (stmt);
+  scalar_dest = gimple_get_lhs (stmt);
   lhs_type = TREE_TYPE (scalar_dest);
   vectype_out = STMT_VINFO_VECTYPE (stmt_info);
 
@@ -4784,7 +4784,8 @@ vectorizable_conversion (vec_info *vinfo,
     }
 
   rhs_type = TREE_TYPE (op0);
-  if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
+  if ((code_or_ifn.is_tree_code () && code_or_ifn != FIX_TRUNC_EXPR
+       && code_or_ifn != FLOAT_EXPR)
       && !((INTEGRAL_TYPE_P (lhs_type)
 	    && INTEGRAL_TYPE_P (rhs_type))
 	   || (SCALAR_FLOAT_TYPE_P (lhs_type)
@@ -4806,10 +4807,16 @@ vectorizable_conversion (vec_info *vinfo,
 
   if (op_type == binary_op)
     {
-      gcc_assert (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR
-		  || code == WIDEN_PLUS_EXPR || code == WIDEN_MINUS_EXPR);
+      gcc_assert (code_or_ifn == WIDEN_MULT_EXPR
+		  || code_or_ifn == WIDEN_LSHIFT_EXPR
+		  || code_or_ifn == WIDEN_PLUS_EXPR
+		  || code_or_ifn == WIDEN_MINUS_EXPR);
+
+      if (is_gimple_assign (stmt))
+	op1 = gimple_assign_rhs2 (stmt);
+      else
+	op1 = gimple_call_arg (stmt, 1);
 
-      op1 = gimple_assign_rhs2 (stmt);
       tree vectype1_in;
       if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
 			       &op1, &slp_op1, &dt[1], &vectype1_in))
@@ -4889,12 +4896,16 @@ vectorizable_conversion (vec_info *vinfo,
   switch (modifier)
     {
     case NONE:
-      if (code != FIX_TRUNC_EXPR
-	  && code != FLOAT_EXPR
-	  && !CONVERT_EXPR_CODE_P (code))
+      if (code_or_ifn != FIX_TRUNC_EXPR
+	  && code_or_ifn != FLOAT_EXPR
+	  && !CONVERT_EXPR_CODE_P (code_or_ifn))
 	return false;
-      if (supportable_convert_operation (code, vectype_out, vectype_in, &code1))
+      if (supportable_convert_operation (code_or_ifn, vectype_out, vectype_in,
+					 &code1))
+      {
+	code_or_ifn1 = code1;
 	break;
+      }
       /* FALLTHRU */
     unsupported:
       if (dump_enabled_p ())
@@ -4905,16 +4916,17 @@ vectorizable_conversion (vec_info *vinfo,
     case WIDEN:
       if (known_eq (nunits_in, nunits_out))
 	{
-	  if (!supportable_half_widening_operation (code, vectype_out,
-						   vectype_in, &code1))
+	  if (!supportable_half_widening_operation (code_or_ifn, vectype_out,
+						   vectype_in,
+						   (tree_code*) &code_or_ifn1))
 	    goto unsupported;
 	  gcc_assert (!(multi_step_cvt && op_type == binary_op));
 	  break;
 	}
-      if (supportable_widening_operation (vinfo, code, stmt_info,
-					       vectype_out, vectype_in, &code1,
-					       &code2, &multi_step_cvt,
-					       &interm_types))
+      if (supportable_widening_operation (vinfo, code_or_ifn, stmt_info,
+					       vectype_out, vectype_in,
+					       &code_or_ifn1, &code_or_ifn2,
+					       &multi_step_cvt, &interm_types))
 	{
 	  /* Binary widening operation can only be supported directly by the
 	     architecture.  */
@@ -4922,7 +4934,7 @@ vectorizable_conversion (vec_info *vinfo,
 	  break;
 	}
 
-      if (code != FLOAT_EXPR
+      if (code_or_ifn != FLOAT_EXPR
 	  || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
 	goto unsupported;
 
@@ -4941,14 +4953,16 @@ vectorizable_conversion (vec_info *vinfo,
 
 	  if (GET_MODE_SIZE (rhs_mode) == fltsz)
 	    {
-	      if (!supportable_convert_operation (code, vectype_out,
-						  cvt_type, &codecvt1))
+	      tree_code code;
+	      if (!supportable_convert_operation (code_or_ifn, vectype_out,
+						  cvt_type, &code))
 		goto unsupported;
+	      codecvt1 = code;
 	    }
-	  else if (!supportable_widening_operation (vinfo, code, stmt_info,
-						    vectype_out, cvt_type,
-						    &codecvt1, &codecvt2,
-						    &multi_step_cvt,
+	  else if (!supportable_widening_operation (vinfo, code_or_ifn,
+						    stmt_info, vectype_out,
+						    cvt_type, &codecvt1,
+						    &codecvt2, &multi_step_cvt,
 						    &interm_types))
 	    continue;
 	  else
@@ -4956,8 +4970,9 @@ vectorizable_conversion (vec_info *vinfo,
 
 	  if (supportable_widening_operation (vinfo, NOP_EXPR, stmt_info,
 					      cvt_type,
-					      vectype_in, &code1, &code2,
-					      &multi_step_cvt, &interm_types))
+					      vectype_in, &code_or_ifn1,
+					      &code_or_ifn2, &multi_step_cvt,
+					      &interm_types))
 	    {
 	      found_mode = true;
 	      break;
@@ -4979,12 +4994,13 @@ vectorizable_conversion (vec_info *vinfo,
 
     case NARROW:
       gcc_assert (op_type == unary_op);
-      if (supportable_narrowing_operation (code, vectype_out, vectype_in,
-					   &code1, &multi_step_cvt,
+      if (supportable_narrowing_operation (code_or_ifn, vectype_out,
+					   vectype_in,
+					   &code_or_ifn1, &multi_step_cvt,
 					   &interm_types))
 	break;
 
-      if (code != FIX_TRUNC_EXPR
+      if (code_or_ifn != FIX_TRUNC_EXPR
 	  || GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
 	goto unsupported;
 
@@ -4993,11 +5009,11 @@ vectorizable_conversion (vec_info *vinfo,
       cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
       if (cvt_type == NULL_TREE)
 	goto unsupported;
-      if (!supportable_convert_operation (code, cvt_type, vectype_in,
-					  &codecvt1))
+      if (!supportable_convert_operation (code_or_ifn, cvt_type, vectype_in,
+					  (tree_code*) &codecvt1))
 	goto unsupported;
       if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
-					   &code1, &multi_step_cvt,
+					   &code_or_ifn1, &multi_step_cvt,
 					   &interm_types))
 	break;
       goto unsupported;
@@ -5113,8 +5129,9 @@ vectorizable_conversion (vec_info *vinfo,
       FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
 	{
 	  /* Arguments are ready, create the new vector stmt.  */
-	  gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
-	  gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0);
+	  gcc_assert (TREE_CODE_LENGTH ((tree_code) code_or_ifn1) == unary_op);
+	  gassign *new_stmt = gimple_build_assign (vec_dest, code_or_ifn1,
+						   vop0);
 	  new_temp = make_ssa_name (vec_dest, new_stmt);
 	  gimple_assign_set_lhs (new_stmt, new_temp);
 	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
@@ -5133,9 +5150,9 @@ vectorizable_conversion (vec_info *vinfo,
 	 the vector stmt by a factor VF/nunits.  */
       vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies * ninputs,
 			 op0, &vec_oprnds0,
-			 code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
+			 code_or_ifn == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
 			 &vec_oprnds1);
-      if (code == WIDEN_LSHIFT_EXPR)
+      if (code_or_ifn == WIDEN_LSHIFT_EXPR)
 	{
 	  int oprnds_size = vec_oprnds0.length ();
 	  vec_oprnds1.create (oprnds_size);
@@ -5146,7 +5163,7 @@ vectorizable_conversion (vec_info *vinfo,
       for (i = multi_step_cvt; i >= 0; i--)
 	{
 	  tree this_dest = vec_dsts[i];
-	  enum tree_code c1 = code1, c2 = code2;
+	  code_helper c1 = code_or_ifn1, c2 = code_or_ifn2;
 	  if (i == 0 && codecvt2 != ERROR_MARK)
 	    {
 	      c1 = codecvt1;
@@ -5169,7 +5186,7 @@ vectorizable_conversion (vec_info *vinfo,
 	  gimple *new_stmt;
 	  if (cvt_type)
 	    {
-	      gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
+	      gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
 	      new_temp = make_ssa_name (vec_dest);
 	      new_stmt = gimple_build_assign (new_temp, codecvt1, vop0);
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
@@ -5195,7 +5212,7 @@ vectorizable_conversion (vec_info *vinfo,
       if (cvt_type)
 	FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
 	  {
-	    gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
+	    gcc_assert (TREE_CODE_LENGTH (((tree_code) codecvt1)) == unary_op);
 	    new_temp = make_ssa_name (vec_dest);
 	    gassign *new_stmt
 	      = gimple_build_assign (new_temp, codecvt1, vop0);
@@ -5206,7 +5223,7 @@ vectorizable_conversion (vec_info *vinfo,
       vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
 					     multi_step_cvt,
 					     stmt_info, vec_dsts, gsi,
-					     slp_node, code1);
+					     slp_node, code_or_ifn1);
       break;
     }
   if (!slp_node)
@@ -11721,9 +11738,11 @@ vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
 
 bool
 supportable_widening_operation (vec_info *vinfo,
-				enum tree_code code, stmt_vec_info stmt_info,
+				code_helper code_or_ifn,
+				stmt_vec_info stmt_info,
 				tree vectype_out, tree vectype_in,
-                                enum tree_code *code1, enum tree_code *code2,
+				code_helper *code_or_ifn1,
+				code_helper *code_or_ifn2,
                                 int *multi_step_cvt,
                                 vec<tree> *interm_types)
 {
@@ -11734,7 +11753,7 @@ supportable_widening_operation (vec_info *vinfo,
   optab optab1, optab2;
   tree vectype = vectype_in;
   tree wide_vectype = vectype_out;
-  enum tree_code c1, c2;
+  code_helper c1=MAX_TREE_CODES, c2=MAX_TREE_CODES;
   int i;
   tree prev_type, intermediate_type;
   machine_mode intermediate_mode, prev_mode;
@@ -11744,7 +11763,7 @@ supportable_widening_operation (vec_info *vinfo,
   if (loop_info)
     vect_loop = LOOP_VINFO_LOOP (loop_info);
 
-  switch (code)
+  switch (code_or_ifn.as_tree_code ())
     {
     case WIDEN_MULT_EXPR:
       /* The result of a vectorized widening operation usually requires
@@ -11773,7 +11792,7 @@ supportable_widening_operation (vec_info *vinfo,
 	 vectorization.  */
       /* TODO: Another case in which order doesn't *really* matter is when we
 	 widen and then contract again, e.g. (short)((int)x * y >> 8).
-	 Normally, pack_trunc performs an even/odd permute, whereas the 
+	 Normally, pack_trunc performs an even/odd permute, whereas the
 	 repack from an even/odd expansion would be an interleave, which
 	 would be significantly simpler for e.g. AVX2.  */
       /* In any case, in order to avoid duplicating the code below, recurse
@@ -11785,20 +11804,22 @@ supportable_widening_operation (vec_info *vinfo,
 	  && !nested_in_vect_loop_p (vect_loop, stmt_info)
 	  && supportable_widening_operation (vinfo, VEC_WIDEN_MULT_EVEN_EXPR,
 					     stmt_info, vectype_out,
-					     vectype_in, code1, code2,
-					     multi_step_cvt, interm_types))
-        {
-          /* Elements in a vector with vect_used_by_reduction property cannot
-             be reordered if the use chain with this property does not have the
-             same operation.  One such an example is s += a * b, where elements
-             in a and b cannot be reordered.  Here we check if the vector defined
-             by STMT is only directly used in the reduction statement.  */
+					     vectype_in, code_or_ifn1,
+					     code_or_ifn2, multi_step_cvt,
+					     interm_types))
+	{
+	  /* Elements in a vector with vect_used_by_reduction property cannot
+	     be reordered if the use chain with this property does not have
+	     the same operation.  One such an example is s += a * b, where
+	     elements in a and b cannot be reordered.  Here we check if the
+	     vector defined by STMT is only directly used in the reduction
+	     statement.  */
 	  tree lhs = gimple_assign_lhs (stmt_info->stmt);
 	  stmt_vec_info use_stmt_info = loop_info->lookup_single_use (lhs);
 	  if (use_stmt_info
 	      && STMT_VINFO_DEF_TYPE (use_stmt_info) == vect_reduction_def)
 	    return true;
-        }
+	}
       c1 = VEC_WIDEN_MULT_LO_EXPR;
       c2 = VEC_WIDEN_MULT_HI_EXPR;
       break;
@@ -11849,6 +11870,17 @@ supportable_widening_operation (vec_info *vinfo,
       c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
       break;
 
+    case MAX_TREE_CODES:
+      break;
+
+    default:
+      gcc_unreachable ();
+    }
+
+  switch (code_or_ifn.as_fn_code ())
+    {
+    case CFN_LAST:
+      break;
     default:
       gcc_unreachable ();
     }
@@ -11856,13 +11888,13 @@ supportable_widening_operation (vec_info *vinfo,
   if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
     std::swap (c1, c2);
 
-  if (code == FIX_TRUNC_EXPR)
+  if (code_or_ifn == FIX_TRUNC_EXPR)
     {
       /* The signedness is determined from output operand.  */
       optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
       optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
     }
-  else if (CONVERT_EXPR_CODE_P (code)
+  else if (CONVERT_EXPR_CODE_P ((tree_code) code_or_ifn)
 	   && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
 	   && VECTOR_BOOLEAN_TYPE_P (vectype)
 	   && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
@@ -11887,8 +11919,8 @@ supportable_widening_operation (vec_info *vinfo,
        || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
     return false;
 
-  *code1 = c1;
-  *code2 = c2;
+  *code_or_ifn1 = c1;
+  *code_or_ifn2 = c2;
 
   if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
       && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
@@ -11909,7 +11941,7 @@ supportable_widening_operation (vec_info *vinfo,
   prev_type = vectype;
   prev_mode = vec_mode;
 
-  if (!CONVERT_EXPR_CODE_P (code))
+  if (!CONVERT_EXPR_CODE_P ((tree_code) code_or_ifn))
     return false;
 
   /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
@@ -11976,7 +12008,6 @@ supportable_widening_operation (vec_info *vinfo,
   return false;
 }
 
-
 /* Function supportable_narrowing_operation
 
    Check whether an operation represented by the code CODE is a
@@ -12000,7 +12031,7 @@ supportable_widening_operation (vec_info *vinfo,
 bool
 supportable_narrowing_operation (enum tree_code code,
 				 tree vectype_out, tree vectype_in,
-				 enum tree_code *code1, int *multi_step_cvt,
+				 void* _code1, int *multi_step_cvt,
                                  vec<tree> *interm_types)
 {
   machine_mode vec_mode;
@@ -12013,6 +12044,7 @@ supportable_narrowing_operation (enum tree_code code,
   machine_mode intermediate_mode, prev_mode;
   int i;
   bool uns;
+  tree_code * code1 = (tree_code*) _code1;
 
   *multi_step_cvt = 0;
   switch (code)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index bd6f334d15f..70c06264c11 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2030,13 +2030,16 @@ extern bool vect_is_simple_use (vec_info *, stmt_vec_info, slp_tree,
 				enum vect_def_type *,
 				tree *, stmt_vec_info * = NULL);
 extern bool vect_maybe_update_slp_op_vectype (slp_tree, tree);
-extern bool supportable_widening_operation (vec_info *,
-					    enum tree_code, stmt_vec_info,
-					    tree, tree, enum tree_code *,
-					    enum tree_code *, int *,
-					    vec<tree> *);
+extern bool supportable_widening_operation (vec_info *vinfo,
+				code_helper code_or_ifn,
+				stmt_vec_info stmt_info,
+				tree vectype_out, tree vectype_in,
+				code_helper *code_or_ifn1,
+				code_helper *code_or_ifn2,
+				int *multi_step_cvt,
+				vec<tree> *interm_types);
 extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
-					     enum tree_code *, int *,
+					     void *, int *,
 					     vec<tree> *);
 
 extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
diff --git a/gcc/tree.h b/gcc/tree.h
index f62c00bc870..346565f84ce 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -6546,5 +6546,31 @@ extern unsigned fndecl_dealloc_argno (tree);
    if nonnull, set the second argument to the referenced enclosing
    object or pointer.  Otherwise return null.  */
 extern tree get_attr_nonstring_decl (tree, tree * = NULL);
+/* Helper to transparently allow tree codes and builtin function codes
+   exist in one storage entity.  */
+class code_helper
+{
+public:
+  code_helper () {}
+  code_helper (tree_code code) : rep ((int) code) {}
+  code_helper (combined_fn fn) : rep (-(int) fn) {}
+  operator tree_code () const { return is_tree_code () ?
+						       (tree_code) rep :
+						       ERROR_MARK; }
+  operator combined_fn () const { return is_fn_code () ?
+						       (combined_fn) -rep:
+						       CFN_LAST; }
+  bool is_tree_code () const { return rep > 0; }
+  bool is_fn_code () const { return rep < 0; }
+  int get_rep () const { return rep; }
+
+  enum tree_code as_tree_code () const { return is_tree_code () ?
+    (tree_code)* this : MAX_TREE_CODES; }
+  combined_fn as_fn_code () const { return is_fn_code () ? (combined_fn) *this
+    : CFN_LAST;}
+
+private:
+  int rep;
+};
 
 #endif  /* GCC_TREE_H  */
-- 
2.17.1


[-- Attachment #3: 0002-vect-patterns-Refactor-widen_plus-as-internal_fn.patch --]
[-- Type: application/octet-stream, Size: 21968 bytes --]

From 86dad266052886a93f6e8d8148e521d2475aea8a Mon Sep 17 00:00:00 2001
From: Joel Hutton <joel.hutton@arm.com>
Date: Fri, 27 Aug 2021 10:21:28 +0100
Subject: [PATCH 2/3] [vect-patterns] Refactor widen_plus as internal_fn

This patch replaces the existing tree_code widen_plus and widen_minus
patterns with internal_fn versions.

gcc/ChangeLog:

	* internal-fn.c (DEF_INTERNAL_OPTAB_MULTI_FN): Macro to define hi/lo
    widening dunctions.
	(lookup_multi_ifn_optab): Function to match hi/lo internal functions
    to optabs.
	(lookup_multi_internal_fn): Function to return the hi/lo internal
    functions.
	(first_commutative_argument): Add widening ifns.
	* internal-fn.def (DEF_INTERNAL_OPTAB_MULTI_FN): Macro to define hi/lo
    widening dunctions.
	(VEC_WIDEN_PLUS): Define widening plus function.
	(VEC_WIDEN_MINUS):Define widening minus function.
	* internal-fn.h (lookup_multi_ifn_optab): Declaration.
	(lookup_multi_internal_fn) Declaration:
	* optabs.c (commutative_optab_p): Add widening cases.
	* optabs.def (OPTAB_CD): Widening optabs.
	* tree-core.h (ECF_WIDEN): Internal function widening flag.
	(ECF_MULTI): Internal function hi/lo split flag.
	* tree-vect-patterns.c (vect_recog_widen_op_pattern): Support
    internal_fns with a hi/lo split.
	(vect_recog_widen_plus_pattern): Change to internal_fn.
	(vect_recog_widen_minus_pattern): Change to internal_fn.
	* tree-vect-stmts.c (vectorizable_conversion): Add cases for
    widening functions.
	(supportable_widening_operation): Add cases for widening functions.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/vect-widen-add.c: Test widen add patterns are
    substituted in gimple.
	* gcc.target/aarch64/vect-widen-sub.c: Test widen sub patterns are
    substituted in gimple.
---
 gcc/internal-fn.c                             | 94 +++++++++++++++++++
 gcc/internal-fn.def                           | 19 ++++
 gcc/internal-fn.h                             |  6 ++
 gcc/optabs.c                                  | 26 ++++-
 gcc/optabs.def                                |  2 +
 .../gcc.target/aarch64/vect-widen-add.c       |  4 +-
 .../gcc.target/aarch64/vect-widen-sub.c       |  4 +-
 gcc/tree-core.h                               |  6 ++
 gcc/tree-vect-patterns.c                      | 35 +++++--
 gcc/tree-vect-stmts.c                         | 79 +++++++++++-----
 10 files changed, 238 insertions(+), 37 deletions(-)

diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index e8fd16b9c21..4ab80d24b89 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -52,6 +52,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "explow.h"
 #include "rtl-iter.h"
 #include "gimple-range.h"
+#include <map>
 
 /* For lang_hooks.types.type_for_mode.  */
 #include "langhooks.h"
@@ -70,6 +71,26 @@ const int internal_fn_flags_array[] = {
   0
 };
 
+const enum internal_fn internal_fn_hilo_keys_array[] = {
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
+#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \
+  IFN_##NAME##_LO, \
+  IFN_##NAME##_HI,
+#include "internal-fn.def"
+  IFN_LAST
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
+};
+
+const optab internal_fn_hilo_values_array[] = {
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
+#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \
+  SOPTAB##_lo_optab, UOPTAB##_lo_optab, \
+  SOPTAB##_hi_optab, UOPTAB##_hi_optab,
+#include "internal-fn.def"
+  unknown_optab, unknown_optab
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
+};
+
 /* Return the internal function called NAME, or IFN_LAST if there's
    no such function.  */
 
@@ -90,6 +111,49 @@ lookup_internal_fn (const char *name)
   return entry ? *entry : IFN_LAST;
 }
 
+/* Return the optab belonging to the given internal function NAME for the given
+   SIGN or unknown_optab.  */
+
+optab
+lookup_multi_ifn_optab (enum internal_fn fn, unsigned sign)
+{
+  typedef std::pair<enum internal_fn, unsigned> ifn_pair;
+  typedef std::map<ifn_pair, optab> fn_to_optab_map_type;
+  static fn_to_optab_map_type *fn_to_optab_map;
+
+  if (!fn_to_optab_map)
+    {
+      unsigned num
+	= sizeof (internal_fn_hilo_keys_array) / sizeof (enum internal_fn);
+      fn_to_optab_map = new fn_to_optab_map_type ();
+      for (unsigned int i = 0; i < num - 1; ++i)
+	{
+	  enum internal_fn fn = internal_fn_hilo_keys_array[i];
+	  optab v1 = internal_fn_hilo_values_array[2*i];
+	  optab v2 = internal_fn_hilo_values_array[2*i + 1];
+	  ifn_pair key1 (fn, 0);
+	  fn_to_optab_map->insert ({key1, v1});
+	  ifn_pair key2 (fn, 1);
+	  fn_to_optab_map->insert ({key2, v2});
+	}
+    }
+
+  ifn_pair new_pair (fn, sign ? 1 : 0);
+  auto entry = fn_to_optab_map->find (new_pair);
+  return entry != fn_to_optab_map->end () ? entry->second : unknown_optab;
+}
+
+extern void
+lookup_multi_internal_fn (enum internal_fn ifn, enum internal_fn *lo,
+			  enum internal_fn *hi)
+{
+  int ecf_flags = internal_fn_flags (ifn);
+  gcc_assert (ecf_flags & ECF_MULTI);
+
+  *lo = internal_fn (ifn + 1);
+  *hi = internal_fn (ifn + 2);
+}
+
 /* Fnspec of each internal function, indexed by function number.  */
 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
 
@@ -3849,6 +3913,9 @@ first_commutative_argument (internal_fn fn)
     case IFN_COND_FMS:
     case IFN_COND_FNMA:
     case IFN_COND_FNMS:
+    case IFN_VEC_WIDEN_PLUS:
+    case IFN_VEC_WIDEN_PLUS_LO:
+    case IFN_VEC_WIDEN_PLUS_HI:
       return 1;
 
     default:
@@ -3868,6 +3935,32 @@ set_edom_supported_p (void)
 #endif
 }
 
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
+#define DEF_INTERNAL_OPTAB_MULTI_FN(CODE, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \
+  static void						        \
+  expand_##CODE (internal_fn, gcall *)		                \
+  {							        \
+    gcc_unreachable ();	                                        \
+  }                                                             \
+  static void						        \
+  expand_##CODE##_LO (internal_fn fn, gcall *stmt)	        \
+  {							        \
+    tree ty = TREE_TYPE (gimple_get_lhs (stmt));                \
+    if (!TYPE_UNSIGNED (ty))                                    \
+      expand_##TYPE##_optab_fn (fn, stmt, SOPTAB##_lo##_optab);	\
+    else                                                        \
+      expand_##TYPE##_optab_fn (fn, stmt, UOPTAB##_lo##_optab);	\
+  }                                                             \
+  static void						        \
+  expand_##CODE##_HI (internal_fn fn, gcall *stmt)	        \
+  {							        \
+    tree ty = TREE_TYPE (gimple_get_lhs (stmt));                \
+    if (!TYPE_UNSIGNED (ty))                                    \
+      expand_##TYPE##_optab_fn (fn, stmt, SOPTAB##_hi##_optab);	\
+    else                                                        \
+      expand_##TYPE##_optab_fn (fn, stmt, UOPTAB##_hi##_optab);	\
+  }
+
 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
   static void						\
   expand_##CODE (internal_fn fn, gcall *stmt)		\
@@ -3884,6 +3977,7 @@ set_edom_supported_p (void)
     expand_##TYPE##_optab_fn (fn, stmt, which_optab);			\
   }
 #include "internal-fn.def"
+#undef DEF_INTERNAL_OPTAB_MULTI_FN
 
 /* Routines to expand each internal function, indexed by function number.
    Each routine has the prototype:
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index bb13c6cce1b..d8aef4c7012 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -82,6 +82,12 @@ along with GCC; see the file COPYING3.  If not see
    says that the function extends the C-level BUILT_IN_<NAME>{,L,LL,IMAX}
    group of functions to any integral mode (including vector modes).
 
+   DEF_INTERNAL_OPTAB_MULTI_FN is like DEF_INTERNAL_OPTAB_FN except it
+   provides convenience wrappers for defining conversions that require a
+   hi/lo split, like widening and narrowing operations.  Each definition
+   for <NAME> will require an optab named <OPTAB> and two other optabs that
+   you specify for signed and unsigned.
+
    Each entry must have a corresponding expander of the form:
 
      void expand_NAME (gimple_call stmt)
@@ -120,6 +126,13 @@ along with GCC; see the file COPYING3.  If not see
   DEF_INTERNAL_OPTAB_FN (NAME, FLAGS, OPTAB, TYPE)
 #endif
 
+#ifndef DEF_INTERNAL_OPTAB_MULTI_FN
+#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \
+  DEF_INTERNAL_OPTAB_FN (NAME, FLAGS | ECF_MULTI, OPTAB, TYPE) \
+  DEF_INTERNAL_OPTAB_FN (NAME ## _LO, FLAGS, unknown, TYPE) \
+  DEF_INTERNAL_OPTAB_FN (NAME ## _HI, FLAGS, unknown, TYPE)
+#endif
+
 DEF_INTERNAL_OPTAB_FN (MASK_LOAD, ECF_PURE, maskload, mask_load)
 DEF_INTERNAL_OPTAB_FN (LOAD_LANES, ECF_CONST, vec_load_lanes, load_lanes)
 DEF_INTERNAL_OPTAB_FN (MASK_LOAD_LANES, ECF_PURE,
@@ -286,6 +299,12 @@ DEF_INTERNAL_OPTAB_FN (COMPLEX_ADD_ROT270, ECF_CONST, cadd270, binary)
 DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL, ECF_CONST, cmul, binary)
 DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL_CONJ, ECF_CONST, cmul_conj, binary)
 DEF_INTERNAL_OPTAB_FN (VEC_ADDSUB, ECF_CONST, vec_addsub, binary)
+DEF_INTERNAL_OPTAB_MULTI_FN (VEC_WIDEN_PLUS, ECF_CONST | ECF_WIDEN | ECF_NOTHROW,
+			     vec_widen_add, vec_widen_saddl, vec_widen_uaddl,
+			     binary)
+DEF_INTERNAL_OPTAB_MULTI_FN (VEC_WIDEN_MINUS, ECF_CONST | ECF_WIDEN | ECF_NOTHROW,
+			     vec_widen_sub, vec_widen_ssubl, vec_widen_usubl,
+			     binary)
 DEF_INTERNAL_OPTAB_FN (VEC_FMADDSUB, ECF_CONST, vec_fmaddsub, ternary)
 DEF_INTERNAL_OPTAB_FN (VEC_FMSUBADD, ECF_CONST, vec_fmsubadd, ternary)
 
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index 06e540ffd6a..406fb52bced 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -20,6 +20,9 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_INTERNAL_FN_H
 #define GCC_INTERNAL_FN_H
 
+#include "insn-codes.h"
+#include "insn-opinit.h"
+
 /* INTEGER_CST values for IFN_UNIQUE function arg-0.
 
    UNSPEC: Undifferentiated UNIQUE.
@@ -113,6 +116,9 @@ internal_fn_name (enum internal_fn fn)
 }
 
 extern internal_fn lookup_internal_fn (const char *);
+extern optab lookup_multi_ifn_optab (enum internal_fn, unsigned);
+extern void lookup_multi_internal_fn (enum internal_fn, enum internal_fn *,
+				      enum internal_fn *);
 
 /* Return the ECF_* flags for function FN.  */
 
diff --git a/gcc/optabs.c b/gcc/optabs.c
index 019bbb62882..065fa4510c3 100644
--- a/gcc/optabs.c
+++ b/gcc/optabs.c
@@ -1303,7 +1303,31 @@ commutative_optab_p (optab binoptab)
 	  || binoptab == smul_widen_optab
 	  || binoptab == umul_widen_optab
 	  || binoptab == smul_highpart_optab
-	  || binoptab == umul_highpart_optab);
+	  || binoptab == umul_highpart_optab
+	  || binoptab == and_optab
+	  || binoptab == ior_optab
+	  || binoptab == xor_optab
+	  || binoptab == smin_optab
+	  || binoptab == smax_optab
+	  || binoptab == umin_optab
+	  || binoptab == umax_optab
+	  || binoptab == add_optab
+	  || binoptab == vec_widen_add_optab
+	  || binoptab == vec_widen_sub_optab
+	  || binoptab == vec_widen_smult_hi_optab
+	  || binoptab == vec_widen_smult_lo_optab
+	  || binoptab == vec_widen_saddl_hi_optab
+	  || binoptab == vec_widen_saddl_lo_optab
+	  || binoptab == vec_widen_ssubl_hi_optab
+	  || binoptab == vec_widen_ssubl_lo_optab
+	  || binoptab == vec_widen_umult_hi_optab
+	  || binoptab == vec_widen_umult_lo_optab
+	  || binoptab == vec_widen_umult_odd_optab
+	  || binoptab == vec_widen_uaddl_hi_optab
+	  || binoptab == vec_widen_uaddl_lo_optab
+	  || binoptab == vec_widen_usubl_hi_optab
+	  || binoptab == vec_widen_usubl_lo_optab
+	  );
 }
 
 /* X is to be used in mode MODE as operand OPN to BINOPTAB.  If we're
diff --git a/gcc/optabs.def b/gcc/optabs.def
index b889ad2e5a0..433ef3d5edc 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -78,6 +78,8 @@ OPTAB_CD(smsub_widen_optab, "msub$b$a4")
 OPTAB_CD(umsub_widen_optab, "umsub$b$a4")
 OPTAB_CD(ssmsub_widen_optab, "ssmsub$b$a4")
 OPTAB_CD(usmsub_widen_optab, "usmsub$a$b4")
+OPTAB_CD(vec_widen_add_optab, "add$a$b3")
+OPTAB_CD(vec_widen_sub_optab, "sub$a$b3")
 OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b")
 OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b")
 OPTAB_CD(vec_mask_load_lanes_optab, "vec_mask_load_lanes$a$b")
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c b/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c
index 220bd9352a4..7037673d32b 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-options "-O3 -save-temps" } */
+/* { dg-options "-O3 -save-temps -fdump-tree-vect-all" } */
 #include <stdint.h>
 #include <string.h>
 
@@ -86,6 +86,8 @@ main()
     return 0;
 }
 
+/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_PLUS_LO" "vect"   } } */
+/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_PLUS_HI" "vect"   } } */
 /* { dg-final { scan-assembler-times {\tuaddl\t} 1} } */
 /* { dg-final { scan-assembler-times {\tuaddl2\t} 1} } */
 /* { dg-final { scan-assembler-times {\tsaddl\t} 1} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c b/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c
index a2bed63affb..83bc1edb610 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-options "-O3 -save-temps" } */
+/* { dg-options "-O3 -save-temps -fdump-tree-vect-all" } */
 #include <stdint.h>
 #include <string.h>
 
@@ -86,6 +86,8 @@ main()
     return 0;
 }
 
+/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_MINUS_LO" "vect"   } } */
+/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_MINUS_HI" "vect"   } } */
 /* { dg-final { scan-assembler-times {\tusubl\t} 1} } */
 /* { dg-final { scan-assembler-times {\tusubl2\t} 1} } */
 /* { dg-final { scan-assembler-times {\tssubl\t} 1} } */
diff --git a/gcc/tree-core.h b/gcc/tree-core.h
index 8ab119dc9a2..f8dd7925e91 100644
--- a/gcc/tree-core.h
+++ b/gcc/tree-core.h
@@ -96,6 +96,12 @@ struct die_struct;
 /* Nonzero if this is a cold function.  */
 #define ECF_COLD		  (1 << 15)
 
+/* Nonzero if this is a widening function.  */
+#define ECF_WIDEN		  (1 << 16)
+
+/* Nonzero if this is a function that decomposes into a lo/hi operation.  */
+#define ECF_MULTI		  (1 << 17)
+
 /* Call argument flags.  */
 
 /* Nonzero if the argument is not used by the function.  */
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 4a8ea67e62f..7e586be7f6d 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -1246,14 +1246,15 @@ static gimple *
 vect_recog_widen_op_pattern (vec_info *vinfo,
 			     stmt_vec_info last_stmt_info, tree *type_out,
 			     tree_code orig_code, code_helper wide_code_or_ifn,
-			     bool shift_p, const char *name)
+			     bool shift_p, const char *name,
+			     enum optab_subtype *subtype = NULL)
 {
   gimple *last_stmt = last_stmt_info->stmt;
 
   vect_unpromoted_value unprom[2];
   tree half_type;
   if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
-			     shift_p, 2, unprom, &half_type))
+			     shift_p, 2, unprom, &half_type, subtype))
     return NULL;
 
   /* Pattern detected.  */
@@ -1329,6 +1330,19 @@ vect_recog_widen_op_pattern (vec_info *vinfo,
 			      type, pattern_stmt, vecctype);
 }
 
+static gimple *
+vect_recog_widen_op_pattern (vec_info *vinfo,
+			     stmt_vec_info last_stmt_info, tree *type_out,
+			     tree_code orig_code, internal_fn wide_ifn,
+			     bool shift_p, const char *name,
+			     enum optab_subtype *subtype = NULL)
+{
+  combined_fn ifn = as_combined_fn (wide_ifn);
+  return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
+				      orig_code, ifn, shift_p, name,
+				      subtype);
+}
+
 /* Try to detect multiplication on widened inputs, converting MULT_EXPR
    to WIDEN_MULT_EXPR.  See vect_recog_widen_op_pattern for details.  */
 
@@ -1342,26 +1356,30 @@ vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
 }
 
 /* Try to detect addition on widened inputs, converting PLUS_EXPR
-   to WIDEN_PLUS_EXPR.  See vect_recog_widen_op_pattern for details.  */
+   to IFN_VEC_WIDEN_PLUS.  See vect_recog_widen_op_pattern for details.  */
 
 static gimple *
 vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
 			       tree *type_out)
 {
+  enum optab_subtype subtype;
   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
-				      PLUS_EXPR, WIDEN_PLUS_EXPR, false,
-				      "vect_recog_widen_plus_pattern");
+				      PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
+				      false, "vect_recog_widen_plus_pattern",
+				      &subtype);
 }
 
 /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
-   to WIDEN_MINUS_EXPR.  See vect_recog_widen_op_pattern for details.  */
+   to IFN_VEC_WIDEN_MINUS.  See vect_recog_widen_op_pattern for details.  */
 static gimple *
 vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
 			       tree *type_out)
 {
+  enum optab_subtype subtype;
   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
-				      MINUS_EXPR, WIDEN_MINUS_EXPR, false,
-				      "vect_recog_widen_minus_pattern");
+				      MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
+				      false, "vect_recog_widen_minus_pattern",
+				      &subtype);
 }
 
 /* Function vect_recog_popcount_pattern
@@ -5523,6 +5541,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
   { vect_recog_mask_conversion_pattern, "mask_conversion" },
   { vect_recog_widen_plus_pattern, "widen_plus" },
   { vect_recog_widen_minus_pattern, "widen_minus" },
+  /* These must come after the double widening ones.  */
 };
 
 const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index d5e1619fabc..3f25a5ee09a 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -4751,22 +4751,30 @@ vectorizable_conversion (vec_info *vinfo,
   if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
     return false;
 
+  bool widen_arith = false;
+
   if (is_gimple_assign (stmt))
-  {
-    code_or_ifn = gimple_assign_rhs_code (stmt);
-  }
+    {
+      code_or_ifn = gimple_assign_rhs_code (stmt);
+      op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn);
+      widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR
+		     || code_or_ifn == WIDEN_MINUS_EXPR
+		     || code_or_ifn == WIDEN_MULT_EXPR
+		     || code_or_ifn == WIDEN_LSHIFT_EXPR);
+    }
   else
-    code_or_ifn = gimple_call_combined_fn (stmt);
+    {
+      code_or_ifn = gimple_call_combined_fn (stmt);
+      op_type = gimple_call_num_args (stmt);
+      widen_arith = gimple_call_flags (stmt) & ECF_WIDEN;
+    }
 
-  if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
+  if (!widen_arith
+      && !CONVERT_EXPR_CODE_P (code_or_ifn)
+      && code_or_ifn != FIX_TRUNC_EXPR
+      && code_or_ifn != FLOAT_EXPR)
     return false;
 
-  bool widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR
-		      || code_or_ifn == WIDEN_MINUS_EXPR
-		      || code_or_ifn == WIDEN_MULT_EXPR
-		      || code_or_ifn == WIDEN_LSHIFT_EXPR);
-  op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn);
-
   /* Check types of lhs and rhs.  */
   scalar_dest = gimple_get_lhs (stmt);
   lhs_type = TREE_TYPE (scalar_dest);
@@ -4784,8 +4792,7 @@ vectorizable_conversion (vec_info *vinfo,
     }
 
   rhs_type = TREE_TYPE (op0);
-  if ((code_or_ifn.is_tree_code () && code_or_ifn != FIX_TRUNC_EXPR
-       && code_or_ifn != FLOAT_EXPR)
+  if ((code_or_ifn != FIX_TRUNC_EXPR && code_or_ifn != FLOAT_EXPR)
       && !((INTEGRAL_TYPE_P (lhs_type)
 	    && INTEGRAL_TYPE_P (rhs_type))
 	   || (SCALAR_FLOAT_TYPE_P (lhs_type)
@@ -4810,7 +4817,8 @@ vectorizable_conversion (vec_info *vinfo,
       gcc_assert (code_or_ifn == WIDEN_MULT_EXPR
 		  || code_or_ifn == WIDEN_LSHIFT_EXPR
 		  || code_or_ifn == WIDEN_PLUS_EXPR
-		  || code_or_ifn == WIDEN_MINUS_EXPR);
+		  || code_or_ifn == WIDEN_MINUS_EXPR
+		  || widen_arith);
 
       if (is_gimple_assign (stmt))
 	op1 = gimple_assign_rhs2 (stmt);
@@ -11877,16 +11885,38 @@ supportable_widening_operation (vec_info *vinfo,
       gcc_unreachable ();
     }
 
-  switch (code_or_ifn.as_fn_code ())
+  if ( code_or_ifn.is_tree_code ())
+  {
+    *code_or_ifn1 = c1;
+    *code_or_ifn2 = c2;
+    if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
+      std::swap (c1, c2);
+  }
+  else
     {
-    case CFN_LAST:
-      break;
-    default:
-      gcc_unreachable ();
-    }
+      internal_fn ifn = as_internal_fn ((combined_fn) code_or_ifn);
+      int ecf_flags = internal_fn_flags (ifn);
+      gcc_assert (ecf_flags & ECF_MULTI);
 
-  if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
-    std::swap (c1, c2);
+      switch (code_or_ifn.as_fn_code ())
+	{
+	case CFN_LAST:
+	  break;
+	case CFN_VEC_WIDEN_PLUS:
+	  break;
+	case CFN_VEC_WIDEN_MINUS:
+	  break;
+	default:
+	  gcc_unreachable ();
+	}
+
+      internal_fn lo, hi;
+      lookup_multi_internal_fn (ifn, &lo, &hi);
+      *code_or_ifn1 = as_combined_fn (lo);
+      *code_or_ifn2 = as_combined_fn (hi);
+      optab1 = lookup_multi_ifn_optab (lo, !TYPE_UNSIGNED (vectype));
+      optab2 = lookup_multi_ifn_optab (hi, !TYPE_UNSIGNED (vectype));
+    }
 
   if (code_or_ifn == FIX_TRUNC_EXPR)
     {
@@ -11905,7 +11935,7 @@ supportable_widening_operation (vec_info *vinfo,
       optab1 = vec_unpacks_sbool_lo_optab;
       optab2 = vec_unpacks_sbool_hi_optab;
     }
-  else
+  else if (code_or_ifn.is_tree_code ())
     {
       optab1 = optab_for_tree_code (c1, vectype, optab_default);
       optab2 = optab_for_tree_code (c2, vectype, optab_default);
@@ -11919,9 +11949,6 @@ supportable_widening_operation (vec_info *vinfo,
        || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
     return false;
 
-  *code_or_ifn1 = c1;
-  *code_or_ifn2 = c2;
-
   if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
       && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
     {
-- 
2.17.1


[-- Attachment #4: 0003-Remove-widen_plus-minus_expr-tree-codes.patch --]
[-- Type: application/octet-stream, Size: 18035 bytes --]

From d373cf53afb01084a34d944f3a9529bc95b803b0 Mon Sep 17 00:00:00 2001
From: Joel Hutton <joel.hutton@arm.com>
Date: Thu, 21 Oct 2021 14:18:37 +0100
Subject: [PATCH 3/3] Remove widen_plus/minus_expr tree codes

This patch removes the old widen plus/minus tree codes which have been
replaced by internal functions.

gcc/ChangeLog:

	* doc/generic.texi: Remove old tree codes.
	* expr.c (expand_expr_real_2): Remove old tree code cases.
	* gimple-pretty-print.c (dump_binary_rhs): Remove old tree code
    cases.
	* optabs-tree.c (optab_for_tree_code):  Remove old tree code
    cases.
	(supportable_half_widening_operation): Remove old tree code
    cases.
	* tree-cfg.c (verify_gimple_assign_binary): Remove old tree code
    cases.
	* tree-inline.c (estimate_operator_cost): Remove old tree code
    cases.
	* tree-pretty-print.c (dump_generic_node): Remove old tree code
    cases.
	(op_symbol_code): Remove old tree code
    cases.
	* tree-vect-data-refs.c (vect_get_smallest_scalar_type): Remove old tree code
    cases.
	* tree-vect-generic.c (expand_vector_operations_1): Remove old tree code
    cases.
	* tree-vect-patterns.c (vect_widened_op_tree): Refactor to replace
    usage in recog_sad_pattern.
	(vect_recog_sad_pattern): Replace tree code widening pattern with
    internal function.
	(vect_recog_average_pattern): Replace tree code widening pattern
    with internal function.
	* tree-vect-stmts.c (vectorizable_conversion): Remove old tree code
    cases.
	(supportable_widening_operation):
	* tree.def (WIDEN_PLUS_EXPR): Remove tree code definition.
	(WIDEN_MINUS_EXPR): Remove tree code definition.
	(VEC_WIDEN_PLUS_HI_EXPR): Remove tree code definition.
	(VEC_WIDEN_PLUS_LO_EXPR): Remove tree code definition.
	(VEC_WIDEN_MINUS_HI_EXPR): Remove tree code definition.
	(VEC_WIDEN_MINUS_LO_EXPR): Remove tree code definition.
---
 gcc/doc/generic.texi      | 31 -------------------------------
 gcc/expr.c                |  6 ------
 gcc/gimple-pretty-print.c |  4 ----
 gcc/optabs-tree.c         | 24 ------------------------
 gcc/tree-cfg.c            |  6 ------
 gcc/tree-inline.c         |  6 ------
 gcc/tree-pretty-print.c   | 12 ------------
 gcc/tree-vect-data-refs.c |  2 --
 gcc/tree-vect-generic.c   |  4 ----
 gcc/tree-vect-patterns.c  | 36 +++++++++++++++++++++++++-----------
 gcc/tree-vect-stmts.c     | 18 ++----------------
 gcc/tree.def              |  6 ------
 12 files changed, 27 insertions(+), 128 deletions(-)

diff --git a/gcc/doc/generic.texi b/gcc/doc/generic.texi
index 69f6f375181..07fdd86ff19 100644
--- a/gcc/doc/generic.texi
+++ b/gcc/doc/generic.texi
@@ -1802,10 +1802,6 @@ a value from @code{enum annot_expr_kind}, the third is an @code{INTEGER_CST}.
 @tindex VEC_RSHIFT_EXPR
 @tindex VEC_WIDEN_MULT_HI_EXPR
 @tindex VEC_WIDEN_MULT_LO_EXPR
-@tindex VEC_WIDEN_PLUS_HI_EXPR
-@tindex VEC_WIDEN_PLUS_LO_EXPR
-@tindex VEC_WIDEN_MINUS_HI_EXPR
-@tindex VEC_WIDEN_MINUS_LO_EXPR
 @tindex VEC_UNPACK_HI_EXPR
 @tindex VEC_UNPACK_LO_EXPR
 @tindex VEC_UNPACK_FLOAT_HI_EXPR
@@ -1852,33 +1848,6 @@ vector of @code{N/2} products. In the case of @code{VEC_WIDEN_MULT_LO_EXPR} the
 low @code{N/2} elements of the two vector are multiplied to produce the
 vector of @code{N/2} products.
 
-@item VEC_WIDEN_PLUS_HI_EXPR
-@itemx VEC_WIDEN_PLUS_LO_EXPR
-These nodes represent widening vector addition of the high and low parts of
-the two input vectors, respectively.  Their operands are vectors that contain
-the same number of elements (@code{N}) of the same integral type. The result
-is a vector that contains half as many elements, of an integral type whose size
-is twice as wide.  In the case of @code{VEC_WIDEN_PLUS_HI_EXPR} the high
-@code{N/2} elements of the two vectors are added to produce the vector of
-@code{N/2} products.  In the case of @code{VEC_WIDEN_PLUS_LO_EXPR} the low
-@code{N/2} elements of the two vectors are added to produce the vector of
-@code{N/2} products.
-
-@item VEC_WIDEN_MINUS_HI_EXPR
-@itemx VEC_WIDEN_MINUS_LO_EXPR
-These nodes represent widening vector subtraction of the high and low parts of
-the two input vectors, respectively.  Their operands are vectors that contain
-the same number of elements (@code{N}) of the same integral type. The high/low
-elements of the second vector are subtracted from the high/low elements of the
-first. The result is a vector that contains half as many elements, of an
-integral type whose size is twice as wide.  In the case of
-@code{VEC_WIDEN_MINUS_HI_EXPR} the high @code{N/2} elements of the second
-vector are subtracted from the high @code{N/2} of the first to produce the
-vector of @code{N/2} products.  In the case of
-@code{VEC_WIDEN_MINUS_LO_EXPR} the low @code{N/2} elements of the second
-vector are subtracted from the low @code{N/2} of the first to produce the
-vector of @code{N/2} products.
-
 @item VEC_UNPACK_HI_EXPR
 @itemx VEC_UNPACK_LO_EXPR
 These nodes represent unpacking of the high and low parts of the input vector,
diff --git a/gcc/expr.c b/gcc/expr.c
index 5673902b1fc..16c17e4caf5 100644
--- a/gcc/expr.c
+++ b/gcc/expr.c
@@ -9338,8 +9338,6 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode,
 					  target, unsignedp);
       return target;
 
-    case WIDEN_PLUS_EXPR:
-    case WIDEN_MINUS_EXPR:
     case WIDEN_MULT_EXPR:
       /* If first operand is constant, swap them.
 	 Thus the following special case checks need only
@@ -10084,10 +10082,6 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode,
 	return temp;
       }
 
-    case VEC_WIDEN_PLUS_HI_EXPR:
-    case VEC_WIDEN_PLUS_LO_EXPR:
-    case VEC_WIDEN_MINUS_HI_EXPR:
-    case VEC_WIDEN_MINUS_LO_EXPR:
     case VEC_WIDEN_MULT_HI_EXPR:
     case VEC_WIDEN_MULT_LO_EXPR:
     case VEC_WIDEN_MULT_EVEN_EXPR:
diff --git a/gcc/gimple-pretty-print.c b/gcc/gimple-pretty-print.c
index 1cd1597359e..592a9967047 100644
--- a/gcc/gimple-pretty-print.c
+++ b/gcc/gimple-pretty-print.c
@@ -459,10 +459,6 @@ dump_binary_rhs (pretty_printer *buffer, const gassign *gs, int spc,
     case VEC_PACK_FLOAT_EXPR:
     case VEC_WIDEN_LSHIFT_HI_EXPR:
     case VEC_WIDEN_LSHIFT_LO_EXPR:
-    case VEC_WIDEN_PLUS_HI_EXPR:
-    case VEC_WIDEN_PLUS_LO_EXPR:
-    case VEC_WIDEN_MINUS_HI_EXPR:
-    case VEC_WIDEN_MINUS_LO_EXPR:
     case VEC_SERIES_EXPR:
       for (p = get_tree_code_name (code); *p; p++)
 	pp_character (buffer, TOUPPER (*p));
diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c
index eeb5aeed320..06db987a207 100644
--- a/gcc/optabs-tree.c
+++ b/gcc/optabs-tree.c
@@ -175,22 +175,6 @@ optab_for_tree_code (enum tree_code code, const_tree type,
       return (TYPE_UNSIGNED (type)
 	      ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
 
-    case VEC_WIDEN_PLUS_LO_EXPR:
-      return (TYPE_UNSIGNED (type)
-	      ? vec_widen_uaddl_lo_optab : vec_widen_saddl_lo_optab);
-
-    case VEC_WIDEN_PLUS_HI_EXPR:
-      return (TYPE_UNSIGNED (type)
-	      ? vec_widen_uaddl_hi_optab : vec_widen_saddl_hi_optab);
-
-    case VEC_WIDEN_MINUS_LO_EXPR:
-      return (TYPE_UNSIGNED (type)
-	      ? vec_widen_usubl_lo_optab : vec_widen_ssubl_lo_optab);
-
-    case VEC_WIDEN_MINUS_HI_EXPR:
-      return (TYPE_UNSIGNED (type)
-	      ? vec_widen_usubl_hi_optab : vec_widen_ssubl_hi_optab);
-
     case VEC_UNPACK_HI_EXPR:
       return (TYPE_UNSIGNED (type)
 	      ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
@@ -297,8 +281,6 @@ optab_for_tree_code (enum tree_code code, const_tree type,
    'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
 
    Supported widening operations:
-    WIDEN_MINUS_EXPR
-    WIDEN_PLUS_EXPR
     WIDEN_MULT_EXPR
     WIDEN_LSHIFT_EXPR
 
@@ -330,12 +312,6 @@ supportable_half_widening_operation (enum tree_code code, tree vectype_out,
     case WIDEN_LSHIFT_EXPR:
       *code1 = LSHIFT_EXPR;
       break;
-    case WIDEN_MINUS_EXPR:
-      *code1 = MINUS_EXPR;
-      break;
-    case WIDEN_PLUS_EXPR:
-      *code1 = PLUS_EXPR;
-      break;
     case WIDEN_MULT_EXPR:
       *code1 = MULT_EXPR;
       break;
diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c
index 8ed8c69b5b1..8f2264506f9 100644
--- a/gcc/tree-cfg.c
+++ b/gcc/tree-cfg.c
@@ -3966,8 +3966,6 @@ verify_gimple_assign_binary (gassign *stmt)
         return false;
       }
 
-    case WIDEN_PLUS_EXPR:
-    case WIDEN_MINUS_EXPR:
     case PLUS_EXPR:
     case MINUS_EXPR:
       {
@@ -4088,10 +4086,6 @@ verify_gimple_assign_binary (gassign *stmt)
         return false;
       }
 
-    case VEC_WIDEN_MINUS_HI_EXPR:
-    case VEC_WIDEN_MINUS_LO_EXPR:
-    case VEC_WIDEN_PLUS_HI_EXPR:
-    case VEC_WIDEN_PLUS_LO_EXPR:
     case VEC_WIDEN_MULT_HI_EXPR:
     case VEC_WIDEN_MULT_LO_EXPR:
     case VEC_WIDEN_MULT_EVEN_EXPR:
diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 53d664ec2e4..105d45fceb9 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -4294,8 +4294,6 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights,
 
     case REALIGN_LOAD_EXPR:
 
-    case WIDEN_PLUS_EXPR:
-    case WIDEN_MINUS_EXPR:
     case WIDEN_SUM_EXPR:
     case WIDEN_MULT_EXPR:
     case DOT_PROD_EXPR:
@@ -4304,10 +4302,6 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights,
     case WIDEN_MULT_MINUS_EXPR:
     case WIDEN_LSHIFT_EXPR:
 
-    case VEC_WIDEN_PLUS_HI_EXPR:
-    case VEC_WIDEN_PLUS_LO_EXPR:
-    case VEC_WIDEN_MINUS_HI_EXPR:
-    case VEC_WIDEN_MINUS_LO_EXPR:
     case VEC_WIDEN_MULT_HI_EXPR:
     case VEC_WIDEN_MULT_LO_EXPR:
     case VEC_WIDEN_MULT_EVEN_EXPR:
diff --git a/gcc/tree-pretty-print.c b/gcc/tree-pretty-print.c
index 275dc7d8af7..4b5fbf16f7e 100644
--- a/gcc/tree-pretty-print.c
+++ b/gcc/tree-pretty-print.c
@@ -2792,8 +2792,6 @@ dump_generic_node (pretty_printer *pp, tree node, int spc, dump_flags_t flags,
       break;
 
       /* Binary arithmetic and logic expressions.  */
-    case WIDEN_PLUS_EXPR:
-    case WIDEN_MINUS_EXPR:
     case WIDEN_SUM_EXPR:
     case WIDEN_MULT_EXPR:
     case MULT_EXPR:
@@ -3757,10 +3755,6 @@ dump_generic_node (pretty_printer *pp, tree node, int spc, dump_flags_t flags,
     case VEC_SERIES_EXPR:
     case VEC_WIDEN_MULT_HI_EXPR:
     case VEC_WIDEN_MULT_LO_EXPR:
-    case VEC_WIDEN_PLUS_HI_EXPR:
-    case VEC_WIDEN_PLUS_LO_EXPR:
-    case VEC_WIDEN_MINUS_HI_EXPR:
-    case VEC_WIDEN_MINUS_LO_EXPR:
     case VEC_WIDEN_MULT_EVEN_EXPR:
     case VEC_WIDEN_MULT_ODD_EXPR:
     case VEC_WIDEN_LSHIFT_HI_EXPR:
@@ -4278,12 +4272,6 @@ op_symbol_code (enum tree_code code)
     case WIDEN_LSHIFT_EXPR:
       return "w<<";
 
-    case WIDEN_PLUS_EXPR:
-      return "w+";
-
-    case WIDEN_MINUS_EXPR:
-      return "w-";
-
     case POINTER_PLUS_EXPR:
       return "+";
 
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 2ea8e983fe6..8f86d1051c8 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -136,8 +136,6 @@ vect_get_smallest_scalar_type (stmt_vec_info stmt_info, tree scalar_type)
 	  || gimple_assign_rhs_code (assign) == WIDEN_SUM_EXPR
 	  || gimple_assign_rhs_code (assign) == WIDEN_MULT_EXPR
 	  || gimple_assign_rhs_code (assign) == WIDEN_LSHIFT_EXPR
-	  || gimple_assign_rhs_code (assign) == WIDEN_PLUS_EXPR
-	  || gimple_assign_rhs_code (assign) == WIDEN_MINUS_EXPR
 	  || gimple_assign_rhs_code (assign) == FLOAT_EXPR)
 	{
 	  tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign));
diff --git a/gcc/tree-vect-generic.c b/gcc/tree-vect-generic.c
index 0d7f04126f2..5c8e406674e 100644
--- a/gcc/tree-vect-generic.c
+++ b/gcc/tree-vect-generic.c
@@ -2192,10 +2192,6 @@ expand_vector_operations_1 (gimple_stmt_iterator *gsi,
      arguments, not the widened result.  VEC_UNPACK_FLOAT_*_EXPR is
      calculated in the same way above.  */
   if (code == WIDEN_SUM_EXPR
-      || code == VEC_WIDEN_PLUS_HI_EXPR
-      || code == VEC_WIDEN_PLUS_LO_EXPR
-      || code == VEC_WIDEN_MINUS_HI_EXPR
-      || code == VEC_WIDEN_MINUS_LO_EXPR
       || code == VEC_WIDEN_MULT_HI_EXPR
       || code == VEC_WIDEN_MULT_LO_EXPR
       || code == VEC_WIDEN_MULT_EVEN_EXPR
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 7e586be7f6d..760f26e2c27 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -551,21 +551,29 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type)
 
 static unsigned int
 vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
-		      tree_code widened_code, bool shift_p,
+		      code_helper widened_code, bool shift_p,
 		      unsigned int max_nops,
 		      vect_unpromoted_value *unprom, tree *common_type,
 		      enum optab_subtype *subtype = NULL)
 {
   /* Check for an integer operation with the right code.  */
-  gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
-  if (!assign)
+  gimple* stmt = stmt_info->stmt;
+  if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
     return 0;
 
-  tree_code rhs_code = gimple_assign_rhs_code (assign);
-  if (rhs_code != code && rhs_code != widened_code)
+  code_helper rhs_code;
+  if (is_gimple_assign (stmt))
+    rhs_code = gimple_assign_rhs_code (stmt);
+  else
+    rhs_code = gimple_call_combined_fn (stmt);
+
+  if (rhs_code.as_tree_code () != code
+      && rhs_code.get_rep () != widened_code.get_rep ())
     return 0;
 
-  tree type = TREE_TYPE (gimple_assign_lhs (assign));
+  tree lhs = is_gimple_assign (stmt) ? gimple_assign_lhs (stmt):
+				      gimple_call_lhs (stmt);
+  tree type = TREE_TYPE (lhs);
   if (!INTEGRAL_TYPE_P (type))
     return 0;
 
@@ -578,7 +586,11 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
     {
       vect_unpromoted_value *this_unprom = &unprom[next_op];
       unsigned int nops = 1;
-      tree op = gimple_op (assign, i + 1);
+      tree op;
+      if (is_gimple_assign(stmt))
+	op = gimple_op (stmt, i + 1);
+      else
+	op = gimple_call_arg(stmt, i);
       if (i == 1 && TREE_CODE (op) == INTEGER_CST)
 	{
 	  /* We already have a common type from earlier operands.
@@ -1194,8 +1206,9 @@ vect_recog_sad_pattern (vec_info *vinfo,
   /* FORNOW.  Can continue analyzing the def-use chain when this stmt in a phi
      inside the loop (in case we are analyzing an outer-loop).  */
   vect_unpromoted_value unprom[2];
-  if (!vect_widened_op_tree (vinfo, diff_stmt_vinfo, MINUS_EXPR, WIDEN_MINUS_EXPR,
-			     false, 2, unprom, &half_type))
+  if (!vect_widened_op_tree (vinfo, diff_stmt_vinfo, MINUS_EXPR,
+			     CFN_VEC_WIDEN_MINUS, false, 2, unprom,
+			     &half_type))
     return NULL;
 
   vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
@@ -2240,9 +2253,10 @@ vect_recog_average_pattern (vec_info *vinfo,
   internal_fn ifn = IFN_AVG_FLOOR;
   vect_unpromoted_value unprom[3];
   tree new_type;
+  enum optab_subtype subtype;
   unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
-					    WIDEN_PLUS_EXPR, false, 3,
-					    unprom, &new_type);
+					    CFN_VEC_WIDEN_PLUS, false, 3,
+					    unprom, &new_type, &subtype);
   if (nops == 0)
     return NULL;
   if (nops == 3)
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 3f25a5ee09a..beb1a6007ee 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -4757,9 +4757,7 @@ vectorizable_conversion (vec_info *vinfo,
     {
       code_or_ifn = gimple_assign_rhs_code (stmt);
       op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn);
-      widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR
-		     || code_or_ifn == WIDEN_MINUS_EXPR
-		     || code_or_ifn == WIDEN_MULT_EXPR
+      widen_arith = (code_or_ifn == WIDEN_MULT_EXPR
 		     || code_or_ifn == WIDEN_LSHIFT_EXPR);
     }
   else
@@ -4816,8 +4814,6 @@ vectorizable_conversion (vec_info *vinfo,
     {
       gcc_assert (code_or_ifn == WIDEN_MULT_EXPR
 		  || code_or_ifn == WIDEN_LSHIFT_EXPR
-		  || code_or_ifn == WIDEN_PLUS_EXPR
-		  || code_or_ifn == WIDEN_MINUS_EXPR
 		  || widen_arith);
 
       if (is_gimple_assign (stmt))
@@ -11758,7 +11754,7 @@ supportable_widening_operation (vec_info *vinfo,
   class loop *vect_loop = NULL;
   machine_mode vec_mode;
   enum insn_code icode1, icode2;
-  optab optab1, optab2;
+  optab optab1 = unknown_optab, optab2 = unknown_optab;
   tree vectype = vectype_in;
   tree wide_vectype = vectype_out;
   code_helper c1=MAX_TREE_CODES, c2=MAX_TREE_CODES;
@@ -11853,16 +11849,6 @@ supportable_widening_operation (vec_info *vinfo,
       c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
       break;
 
-    case WIDEN_PLUS_EXPR:
-      c1 = VEC_WIDEN_PLUS_LO_EXPR;
-      c2 = VEC_WIDEN_PLUS_HI_EXPR;
-      break;
-
-    case WIDEN_MINUS_EXPR:
-      c1 = VEC_WIDEN_MINUS_LO_EXPR;
-      c2 = VEC_WIDEN_MINUS_HI_EXPR;
-      break;
-
     CASE_CONVERT:
       c1 = VEC_UNPACK_LO_EXPR;
       c2 = VEC_UNPACK_HI_EXPR;
diff --git a/gcc/tree.def b/gcc/tree.def
index e27bc3e2b1f..93cd03bbedf 100644
--- a/gcc/tree.def
+++ b/gcc/tree.def
@@ -1375,8 +1375,6 @@ DEFTREECODE (WIDEN_MULT_MINUS_EXPR, "widen_mult_minus_expr", tcc_expression, 3)
    the first argument from type t1 to type t2, and then shifting it
    by the second argument.  */
 DEFTREECODE (WIDEN_LSHIFT_EXPR, "widen_lshift_expr", tcc_binary, 2)
-DEFTREECODE (WIDEN_PLUS_EXPR, "widen_plus_expr", tcc_binary, 2)
-DEFTREECODE (WIDEN_MINUS_EXPR, "widen_minus_expr", tcc_binary, 2)
 
 /* Widening vector multiplication.
    The two operands are vectors with N elements of size S. Multiplying the
@@ -1441,10 +1439,6 @@ DEFTREECODE (VEC_PACK_FLOAT_EXPR, "vec_pack_float_expr", tcc_binary, 2)
  */
 DEFTREECODE (VEC_WIDEN_LSHIFT_HI_EXPR, "widen_lshift_hi_expr", tcc_binary, 2)
 DEFTREECODE (VEC_WIDEN_LSHIFT_LO_EXPR, "widen_lshift_lo_expr", tcc_binary, 2)
-DEFTREECODE (VEC_WIDEN_PLUS_HI_EXPR, "widen_plus_hi_expr", tcc_binary, 2)
-DEFTREECODE (VEC_WIDEN_PLUS_LO_EXPR, "widen_plus_lo_expr", tcc_binary, 2)
-DEFTREECODE (VEC_WIDEN_MINUS_HI_EXPR, "widen_minus_hi_expr", tcc_binary, 2)
-DEFTREECODE (VEC_WIDEN_MINUS_LO_EXPR, "widen_minus_lo_expr", tcc_binary, 2)
 
 /* PREDICT_EXPR.  Specify hint for branch prediction.  The
    PREDICT_EXPR_PREDICTOR specify predictor and PREDICT_EXPR_OUTCOME the
-- 
2.17.1


^ permalink raw reply	[flat|nested] 6+ messages in thread
end of thread, other threads:[~2022-04-13 15:52 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-11-11 15:13 [vect-patterns] Refactor widen_plus/widen_minus as internal_fns Joel Hutton
2021-11-12 10:48 ` Richard Biener
2021-11-12 11:42   ` Joel Hutton
2021-11-16 10:19     ` Joel Hutton
2021-12-02 18:10       ` Richard Sandiford
2022-04-13 15:52         ` Joel Hutton

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).