[mask-vec_cond, patch 1/2] Support vectorization of VEC_COND_EXPR with no embedded comparison

[Ilya Enkovich <enkovich.gnu@gmail.com>]

Hi,

This patch allows COND_EXPR with no embedded comparison to be vectorized.  It's applied on top of vectorized comparison support series.  New optab vcond_mask_optab is introduced for such statements.  Bool patterns now avoid comparison in COND_EXPR in case vector comparison is supported by target.

Thanks,
Ilya
--
gcc/

2015-10-08  Ilya Enkovich  <enkovich.gnu@gmail.com>

	* optabs-query.h (get_vcond_mask_icode): New.
	* optabs-tree.c (expand_vec_cond_expr_p): Use
	get_vcond_mask_icode for VEC_COND_EXPR with mask.
	* optabs.c (expand_vec_cond_mask_expr): New.
	(expand_vec_cond_expr): Use get_vcond_mask_icode
	when possible.
	* optabs.def (vcond_mask_optab): New.
	* tree-vect-patterns.c (vect_recog_bool_pattern): Don't
	generate redundant comparison for COND_EXPR.
	* tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME
	as a condition.
	(vectorizable_condition): Likewise.


diff --git a/gcc/optabs-query.h b/gcc/optabs-query.h
index 162d2e9..48bcf7c 100644
--- a/gcc/optabs-query.h
+++ b/gcc/optabs-query.h
@@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns)
   return icode;
 }
 
+/* Return insn code for a conditional operator with a mask mode
+   MMODE resulting in a value of mode VMODE.  */
+
+static inline enum insn_code
+get_vcond_mask_icode (machine_mode vmode, machine_mode mmode)
+{
+  return convert_optab_handler (vcond_mask_optab, vmode, mmode);
+}
+
 /* Enumerates the possible extraction_insn operations.  */
 enum extraction_pattern { EP_insv, EP_extv, EP_extzv };
 
diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c
index aa863cf..d887619 100644
--- a/gcc/optabs-tree.c
+++ b/gcc/optabs-tree.c
@@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
 {
   machine_mode value_mode = TYPE_MODE (value_type);
   machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
+  if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type))
+    return get_vcond_mask_icode (TYPE_MODE (value_type),
+				 TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing;
   if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
       || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
       || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
diff --git a/gcc/optabs.c b/gcc/optabs.c
index ca1a6e7..d26b8f8 100644
--- a/gcc/optabs.c
+++ b/gcc/optabs.c
@@ -5346,6 +5346,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
   return tmp;
 }
 
+/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
+   three operands.  */
+
+rtx
+expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
+			   rtx target)
+{
+  struct expand_operand ops[4];
+  machine_mode mode = TYPE_MODE (vec_cond_type);
+  machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
+  enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
+  rtx mask, rtx_op1, rtx_op2;
+
+  if (icode == CODE_FOR_nothing)
+    return 0;
+
+  mask = expand_normal (op0);
+  rtx_op1 = expand_normal (op1);
+  rtx_op2 = expand_normal (op2);
+
+  mask = force_reg (GET_MODE (mask), mask);
+  rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
+
+  create_output_operand (&ops[0], target, mode);
+  create_input_operand (&ops[1], rtx_op1, mode);
+  create_input_operand (&ops[2], rtx_op2, mode);
+  create_input_operand (&ops[3], mask, mask_mode);
+  expand_insn (icode, 4, ops);
+
+  return ops[0].value;
+}
+
 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
    three operands.  */
 
@@ -5371,12 +5403,21 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
     }
   else
     {
-      /* Fake op0 < 0.  */
       gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
-      op0a = op0;
-      op0b = build_zero_cst (TREE_TYPE (op0));
-      tcode = LT_EXPR;
-      unsignedp = false;
+      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
+	  != CODE_FOR_nothing)
+	return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
+					  op2, target);
+      /* Fake op0 < 0.  */
+      else
+	{
+	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
+		      == MODE_VECTOR_INT);
+	  op0a = op0;
+	  op0b = build_zero_cst (TREE_TYPE (op0));
+	  tcode = LT_EXPR;
+	  unsignedp = false;
+	}
     }
   cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
 
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 9804378..70530a6 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b")
 OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b")
 OPTAB_CD(vcond_optab, "vcond$a$b")
 OPTAB_CD(vcondu_optab, "vcondu$a$b")
+OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
 OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
 OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
 OPTAB_CD(maskload_optab, "maskload$a$b")
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index e3be3d1..6269b69 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -3356,7 +3356,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
       else
 	{
 	  tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
-	  tree cst0, cst1, cmp, tmp;
+	  tree cst0, cst1, tmp;
 
 	  if (!type)
 	    return NULL;
@@ -3372,9 +3372,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
 	  cst0 = build_int_cst (type, 0);
 	  cst1 = build_int_cst (type, 1);
 	  tmp = vect_recog_temp_ssa_var (type, NULL);
-	  cmp = build2 (NE_EXPR, boolean_type_node,
-			var, build_int_cst (TREE_TYPE (var), 0));
-	  pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0);
+	  pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
 
 	  if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
 	    {
@@ -3418,19 +3416,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
       if (get_vectype_for_scalar_type (type) == NULL_TREE)
 	return NULL;
 
-      if (check_bool_pattern (var, loop_vinfo, bb_vinfo))
-	{
-	  rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
-	  rhs = build2 (NE_EXPR, boolean_type_node,
-			rhs, build_int_cst (type, 0));
-	}
-      else
-	rhs = build2 (NE_EXPR, boolean_type_node,
-		      var, build_int_cst (TREE_TYPE (var), 0)),
+      if (!check_bool_pattern (var, loop_vinfo, bb_vinfo))
+	return NULL;
+
+      rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
 
       lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
       pattern_stmt 
-	  = gimple_build_assign (lhs, COND_EXPR, rhs,
+	  = gimple_build_assign (lhs, COND_EXPR,
+				 build2 (NE_EXPR, boolean_type_node,
+					 rhs, build_int_cst (type, 0)),
 				 gimple_assign_rhs2 (last_stmt),
 				 gimple_assign_rhs3 (last_stmt));
       *type_out = vectype;
@@ -3457,7 +3452,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
       else
 	{
 	  tree type = search_type_for_mask (var, loop_vinfo, bb_vinfo);
-	  tree cst0, cst1, cmp, new_vectype;
+	  tree cst0, cst1, new_vectype;
 
 	  if (!type)
 	    return NULL;
@@ -3470,10 +3465,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
 	  new_vectype = get_vectype_for_scalar_type (type);
 
 	  rhs = vect_recog_temp_ssa_var (type, NULL);
-	  cmp = build2 (NE_EXPR, boolean_type_node,
-			var, build_int_cst (TREE_TYPE (var), 0));
-	  pattern_stmt = gimple_build_assign (rhs, COND_EXPR,
-					      cmp, cst1, cst0);
+	  pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
 
 	  pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo,
 						 bb_vinfo);
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index d6819ec..b139601 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -7296,6 +7296,19 @@ vect_is_simple_cond (tree cond, gimple *stmt, loop_vec_info loop_vinfo,
   enum vect_def_type dt;
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
 
+  /* Mask case.  */
+  if (TREE_CODE (cond) == SSA_NAME
+      && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE)
+    {
+      gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
+      if (!vect_is_simple_use_1 (cond, stmt, loop_vinfo, bb_vinfo,
+				 &lhs_def_stmt, &def, &dt, comp_vectype)
+	  || !*comp_vectype
+	  || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
+	return false;
+      return true;
+    }
+
   if (!COMPARISON_CLASS_P (cond))
     return false;
 
@@ -7368,6 +7381,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
   vec<tree> vec_oprnds2 = vNULL;
   vec<tree> vec_oprnds3 = vNULL;
   tree vec_cmp_type;
+  bool masked = false;
 
   if (reduc_index && STMT_SLP_TYPE (stmt_info))
     return false;
@@ -7443,7 +7457,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
 	   && TREE_CODE (else_clause) != FIXED_CST)
     return false;
 
-  vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
+  if (VECTOR_BOOLEAN_TYPE_P (comp_vectype))
+    {
+      vec_cmp_type = comp_vectype;
+      masked = true;
+    }
+  else
+    vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
   if (vec_cmp_type == NULL_TREE)
     return false;
 
@@ -7478,14 +7498,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
               auto_vec<tree, 4> ops;
 	      auto_vec<vec<tree>, 4> vec_defs;
 
-              ops.safe_push (TREE_OPERAND (cond_expr, 0));
-              ops.safe_push (TREE_OPERAND (cond_expr, 1));
+	      if (masked)
+		  ops.safe_push (cond_expr);
+	      else
+		{
+		  ops.safe_push (TREE_OPERAND (cond_expr, 0));
+		  ops.safe_push (TREE_OPERAND (cond_expr, 1));
+		}
               ops.safe_push (then_clause);
               ops.safe_push (else_clause);
               vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
 	      vec_oprnds3 = vec_defs.pop ();
 	      vec_oprnds2 = vec_defs.pop ();
-	      vec_oprnds1 = vec_defs.pop ();
+	      if (!masked)
+		vec_oprnds1 = vec_defs.pop ();
 	      vec_oprnds0 = vec_defs.pop ();
 
               ops.release ();
@@ -7494,17 +7520,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
           else
             {
 	      gimple *gtemp;
-	      vec_cond_lhs =
-	      vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
-					    stmt, NULL, comp_vectype);
-	      vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
-				  loop_vinfo, NULL, &gtemp, &def, &dts[0]);
-
-	      vec_cond_rhs =
-		vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
-					      stmt, NULL, comp_vectype);
-	      vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
-				  loop_vinfo, NULL, &gtemp, &def, &dts[1]);
+	      if (masked)
+		{
+		  vec_cond_lhs
+		    = vect_get_vec_def_for_operand (cond_expr, stmt,
+						    NULL, comp_vectype);
+		  vect_is_simple_use (cond_expr, stmt, loop_vinfo, NULL,
+				      &gtemp, &def, &dts[0]);
+		}
+	      else
+		{
+		  vec_cond_lhs =
+		    vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
+						  stmt, NULL, comp_vectype);
+		  vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
+				      loop_vinfo, NULL, &gtemp, &def, &dts[0]);
+
+		  vec_cond_rhs =
+		    vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
+						  stmt, NULL, comp_vectype);
+		  vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
+				      loop_vinfo, NULL, &gtemp, &def, &dts[1]);
+		}
 	      if (reduc_index == 1)
 		vec_then_clause = reduc_def;
 	      else
@@ -7527,10 +7564,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
 	}
       else
 	{
-	  vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0],
-							 vec_oprnds0.pop ());
-	  vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1],
-							 vec_oprnds1.pop ());
+	  vec_cond_lhs
+	    = vect_get_vec_def_for_stmt_copy (dts[0],
+					      vec_oprnds0.pop ());
+	  if (!masked)
+	    vec_cond_rhs
+	      = vect_get_vec_def_for_stmt_copy (dts[1],
+						vec_oprnds1.pop ());
+
 	  vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
 							    vec_oprnds2.pop ());
 	  vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
@@ -7540,7 +7581,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
       if (!slp_node)
         {
 	  vec_oprnds0.quick_push (vec_cond_lhs);
-	  vec_oprnds1.quick_push (vec_cond_rhs);
+	  if (!masked)
+	    vec_oprnds1.quick_push (vec_cond_rhs);
 	  vec_oprnds2.quick_push (vec_then_clause);
 	  vec_oprnds3.quick_push (vec_else_clause);
 	}
@@ -7548,12 +7590,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
       /* Arguments are ready.  Create the new vector stmt.  */
       FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
         {
-          vec_cond_rhs = vec_oprnds1[i];
           vec_then_clause = vec_oprnds2[i];
           vec_else_clause = vec_oprnds3[i];
 
-	  vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
-				vec_cond_lhs, vec_cond_rhs);
+	  if (masked)
+	    vec_compare = vec_cond_lhs;
+	  else
+	    {
+	      vec_cond_rhs = vec_oprnds1[i];
+	      vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
+				    vec_cond_lhs, vec_cond_rhs);
+	    }
           vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
  		         vec_compare, vec_then_clause, vec_else_clause);