Re: [stage1][PATCH] Lower VEC_COND_EXPR into internal functions.

["Martin Liška" <mliska@suse.cz>]

[-- Attachment #1: Type: text/plain, Size: 7750 bytes --]

On 6/15/20 1:59 PM, Richard Biener wrote:
> On Mon, Jun 15, 2020 at 1:19 PM Martin Liška <mliska@suse.cz> wrote:
>>
>> On 6/15/20 9:14 AM, Richard Biener wrote:
>>> On Fri, Jun 12, 2020 at 3:24 PM Martin Liška <mliska@suse.cz> wrote:
>>>>
>>>> On 6/12/20 11:43 AM, Richard Biener wrote:
>>>>> So ... how far are you with enforcing a split VEC_COND_EXPR?
>>>>> Thus can we avoid the above completely (even as intermediate
>>>>> state)?
>>>>
>>>> Apparently, I'm quite close. Using the attached patch I see only 2 testsuite
>>>> failures:
>>>>
>>>> FAIL: gcc.dg/tree-ssa/pr68714.c scan-tree-dump-times reassoc1 " <= " 1
>>>> FAIL: gcc.target/i386/pr78102.c scan-assembler-times pcmpeqq 3
>>>>
>>>> The first one is about teaching reassoc about the SSA_NAMEs in VEC_COND_EXPR. I haven't
>>>> analyze the second failure.
>>>>
>>>> I'm also not sure about the gimlification change, I see a superfluous assignments:
>>>>      vec_cond_cmp.5 = _1 == _2;
>>>>      vec_cond_cmp.6 = vec_cond_cmp.5;
>>>>      vec_cond_cmp.7 = vec_cond_cmp.6;
>>>>      _3 = VEC_COND_EXPR <vec_cond_cmp.7, { -1, -1, -1, -1, -1, -1, -1, -1 }, { 0, 0, 0, 0, 0, 0, 0, 0 }>;
>>>> ?
>>>>
>>>> So with the suggested patch, the EH should be gone as you suggested. Right?
>>>
>>> Right, it should be on the comparison already from the start.
>>>
>>> @@ -14221,9 +14221,13 @@ gimplify_expr (tree *expr_p, gimple_seq
>>> *pre_p, gimple_seq *post_p,
>>>           case VEC_COND_EXPR:
>>>             {
>>>               enum gimplify_status r0, r1, r2;
>>> -
>>>               r0 = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
>>>                                   post_p, is_gimple_condexpr, fb_rvalue);
>>> +           tree xop0 = TREE_OPERAND (*expr_p, 0);
>>> +           tmp = create_tmp_var_raw (TREE_TYPE (xop0), "vec_cond_cmp");
>>> +           gimple_add_tmp_var (tmp);
>>> +           gimplify_assign (tmp, xop0, pre_p);
>>> +           TREE_OPERAND (*expr_p, 0) = tmp;
>>>               r1 = gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p,
>>>                                   post_p, is_gimple_val, fb_rvalue);
>>>
>>> all of VEC_COND_EXPR can now be a simple goto expr_3;
>>
>> Works for me, thanks!
>>
>>>
>>> diff --git a/gcc/tree-ssa-forwprop.c b/gcc/tree-ssa-forwprop.c
>>> index 494c9e9c20b..090fb52a2f1 100644
>>> --- a/gcc/tree-ssa-forwprop.c
>>> +++ b/gcc/tree-ssa-forwprop.c
>>> @@ -3136,6 +3136,10 @@ pass_forwprop::execute (function *fun)
>>>                       if (code == COND_EXPR
>>>                           || code == VEC_COND_EXPR)
>>>                         {
>>> +                       /* Do not propagate into VEC_COND_EXPRs.  */
>>> +                       if (code == VEC_COND_EXPR)
>>> +                         break;
>>> +
>>>
>>> err - remove the || code == VEC_COND_EXPR instead?
>>
>> Yep.
>>
>>>
>>> @@ -2221,24 +2226,12 @@ expand_vector_operations (void)
>>>    {
>>>      gimple_stmt_iterator gsi;
>>>      basic_block bb;
>>> -  bool cfg_changed = false;
>>>
>>>      FOR_EACH_BB_FN (bb, cfun)
>>> -    {
>>> -      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
>>> -       {
>>> -         expand_vector_operations_1 (&gsi);
>>> -         /* ???  If we do not cleanup EH then we will ICE in
>>> -            verification.  But in reality we have created wrong-code
>>> -            as we did not properly transition EH info and edges to
>>> -            the piecewise computations.  */
>>> -         if (maybe_clean_eh_stmt (gsi_stmt (gsi))
>>> -             && gimple_purge_dead_eh_edges (bb))
>>> -           cfg_changed = true;
>>> -       }
>>> -    }
>>>
>>> I'm not sure about this.  Consider the C++ testcase where
>>> the ?: is replaced by a division.  If veclower needs to replace
>>> that with four scalrar division statements then the above
>>> still applies - veclower does not correctly duplicate EH info
>>> and EH edges to the individual divisions (and we do not know
>>> which component might trap).
>>>
>>> So please leave the above in.  You can try if using integer
>>> division makes it break and add such a testcase if there's
>>> no coverage for this in the testsuite.
>>
>> I'm leaving that above. Can you please explain how can a division test-case
>> be created?
> 
> typedef long v2di __attribute__((vector_size(16)));
> 
> v2di foo (v2di a, v2di b)
> {
>    try
>    {
>      v2di res = a / b;
>      return res;
>      }
>      catch (...)
>      {
>      return (v2di){};
>      }
> }
> 
> with -fnon-call-exceptions I see in t.ii.090t.ehdisp (correctly):
> 
> ;;   basic block 2, loop depth 0
> ;;    pred:       ENTRY
>    [LP 1] _6 = a_4(D) / b_5(D);
> ;;    succ:       5
> ;;                3
> 
> while after t.ii.226t.veclower we have
> 
> ;;   basic block 2, loop depth 0
> ;;    pred:       ENTRY
>    _13 = BIT_FIELD_REF <a_4(D), 64, 0>;
>    _14 = BIT_FIELD_REF <b_5(D), 64, 0>;
>    _15 = _13 / _14;
>    _16 = BIT_FIELD_REF <a_4(D), 64, 64>;
>    _17 = BIT_FIELD_REF <b_5(D), 64, 64>;
>    _18 = _16 / _17;
>    _6 = {_15, _18};
>    res_7 = _6;
>    _8 = res_7;
> ;;    succ:       3
> 
> and all EH is gone and we'd ICE if you remove the above hunk.  Hopefully.

Yes, it ICEs then:


./xg++ -B. ~/Programming/testcases/ice.c -c -fnon-call-exceptions -O3
/home/marxin/Programming/testcases/ice.c: In function ‘v2di foo(v2di, v2di)’:
/home/marxin/Programming/testcases/ice.c:3:6: error: statement marked for throw, but doesn’t
     3 | v2di foo (v2di a, v2di b)
       |      ^~~
_6 = {_12, _15};
during GIMPLE pass: veclower2
/home/marxin/Programming/testcases/ice.c:3:6: internal compiler error: verify_gimple failed
0x10e308a verify_gimple_in_cfg(function*, bool)
	/home/marxin/Programming/gcc/gcc/tree-cfg.c:5461
0xfc9caf execute_function_todo
	/home/marxin/Programming/gcc/gcc/passes.c:1985
0xfcaafc do_per_function
	/home/marxin/Programming/gcc/gcc/passes.c:1640
0xfcaafc execute_todo
	/home/marxin/Programming/gcc/gcc/passes.c:2039
Please submit a full bug report,
with preprocessed source if appropriate.
Please include the complete backtrace with any bug report.
See <https://gcc.gnu.org/bugs/> for instructions.

> 
> We still generate wrong-code obviously as we'd need to duplicate the
> EH info on each component division (and split blocks and generate
> extra EH edges).  That's a pre-existing bug of course.  I just wanted
> to avoid to create a new instance just because of the early instruction
> selection for VEC_COND_EXPR.

Fine!

> 
>>>
>>> What's missing from the patch is adjusting
>>> verify_gimple_assign_ternary from
>>>
>>>     if (((rhs_code == VEC_COND_EXPR || rhs_code == COND_EXPR)
>>>          ? !is_gimple_condexpr (rhs1) : !is_gimple_val (rhs1))
>>>         || !is_gimple_val (rhs2)
>>>         || !is_gimple_val (rhs3))
>>>       {
>>>         error ("invalid operands in ternary operation");
>>>         return true;
>>>
>>> to the same with the rhs_code == VEC_COND_EXPR case removed.
>>
>> Hmm. I'm not sure I've got this comment. Why do we want to change it
>> and is it done wright in the patch?
> 
> Ah, I missed the hunk you added.

That explains the confusion I got.

>  But the check should be an inclusive
> one, not an exclusive one and earlier accepting a is_gimple_condexpr
> is superfluous when you later reject the tcc_comparison part.  Just
> testing is_gimple_val is better.  So yes, remove your tree-cfg.c hunk
> and just adjust the above test.

I simplified that.

Patch can bootstrap on x86_64-linux-gnu and survives regression tests.

Thanks,
Martin

> 
>>>
>>> You'll likely figure the vectorizer still creates some VEC_COND_EXPRs
>>> with embedded comparisons.
>>
>> I've fixed 2 failing test-cases I mentioned in the previous email.
>>
>> Martin
>>
>>>
>>> Thanks,
>>> Richard.
>>>
>>>
>>>> Martin
>>


[-- Attachment #2: 0001-Lower-VEC_COND_EXPR-into-internal-functions.patch --]
[-- Type: text/x-patch, Size: 36630 bytes --]

From 9ce7b3f510b46e1adba1b1d7eb005beb00d21d42 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Mon, 9 Mar 2020 13:23:03 +0100
Subject: [PATCH] Lower VEC_COND_EXPR into internal functions.

gcc/ChangeLog:

	* Makefile.in: Add new file.
	* expr.c (expand_expr_real_2): Add gcc_unreachable as we should
	not meet this condition.
	(do_store_flag):
	* gimplify.c (gimplify_expr): Gimplify first argument of
	VEC_COND_EXPR to be a SSA name.
	* internal-fn.c (vec_cond_mask_direct): New.
	(vec_cond_direct): Likewise.
	(vec_condu_direct): Likewise.
	(vec_condeq_direct): Likewise.
	(expand_vect_cond_optab_fn):  New.
	(expand_vec_cond_optab_fn): Likewise.
	(expand_vec_condu_optab_fn): Likewise.
	(expand_vec_condeq_optab_fn): Likewise.
	(expand_vect_cond_mask_optab_fn): Likewise.
	(expand_vec_cond_mask_optab_fn): Likewise.
	(direct_vec_cond_mask_optab_supported_p): Likewise.
	(direct_vec_cond_optab_supported_p): Likewise.
	(direct_vec_condu_optab_supported_p): Likewise.
	(direct_vec_condeq_optab_supported_p): Likewise.
	* internal-fn.def (VCOND): New OPTAB.
	(VCONDU): Likewise.
	(VCONDEQ): Likewise.
	(VCOND_MASK): Likewise.
	* optabs.c (get_rtx_code): Make it global.
	(expand_vec_cond_mask_expr): Removed.
	(expand_vec_cond_expr): Removed.
	* optabs.h (expand_vec_cond_expr): Likewise.
	(vector_compare_rtx): Make it global.
	* passes.def: Add new pass_gimple_isel pass.
	* tree-cfg.c (verify_gimple_assign_ternary): Add check
	for VEC_COND_EXPR about first argument.
	* tree-pass.h (make_pass_gimple_isel): New.
	* tree-ssa-forwprop.c (pass_forwprop::execute): Prevent
	propagation of the first argument of a VEC_COND_EXPR.
	* tree-ssa-reassoc.c (ovce_extract_ops): Support SSA_NAME as
	first argument of a VEC_COND_EXPR.
	(optimize_vec_cond_expr): Likewise.
	* tree-vect-generic.c (expand_vector_divmod): Make SSA_NAME
	for a first argument of created VEC_COND_EXPR.
	(expand_vector_condition): Fix coding style.
	* tree-vect-stmts.c (vectorizable_condition): Gimplify
	first argument.
	* tree-vect-isel.c: New file.

gcc/testsuite/ChangeLog:

	* g++.dg/vect/vec-cond-expr-eh.C: New test.
---
 gcc/Makefile.in                              |   2 +
 gcc/expr.c                                   |  25 +-
 gcc/gimplify.c                               |  15 +-
 gcc/internal-fn.c                            |  89 +++++++
 gcc/internal-fn.def                          |   5 +
 gcc/optabs.c                                 | 124 +---------
 gcc/optabs.h                                 |   7 +-
 gcc/passes.def                               |   1 +
 gcc/testsuite/g++.dg/vect/vec-cond-expr-eh.C |  17 ++
 gcc/tree-cfg.c                               |   2 +-
 gcc/tree-pass.h                              |   1 +
 gcc/tree-ssa-forwprop.c                      |   3 +-
 gcc/tree-ssa-reassoc.c                       |  69 ++++--
 gcc/tree-vect-generic.c                      |  45 ++--
 gcc/tree-vect-isel.c                         | 244 +++++++++++++++++++
 gcc/tree-vect-stmts.c                        |   4 +-
 16 files changed, 442 insertions(+), 211 deletions(-)
 create mode 100644 gcc/testsuite/g++.dg/vect/vec-cond-expr-eh.C
 create mode 100644 gcc/tree-vect-isel.c

diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 4f70c189b9d..4cbb9d23606 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1631,6 +1631,7 @@ OBJS = \
 	tree-streamer-out.o \
 	tree-tailcall.o \
 	tree-vect-generic.o \
+	tree-vect-isel.o \
 	tree-vect-patterns.o \
 	tree-vect-data-refs.o \
 	tree-vect-stmts.o \
@@ -2600,6 +2601,7 @@ GTFILES = $(CPPLIB_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(srcdir)/dwarf2cfi.c \
   $(srcdir)/dwarf2out.c \
   $(srcdir)/tree-vect-generic.c \
+  $(srcdir)/tree-vect-isel.c \
   $(srcdir)/dojump.c $(srcdir)/emit-rtl.h \
   $(srcdir)/emit-rtl.c $(srcdir)/except.h $(srcdir)/explow.c $(srcdir)/expr.c \
   $(srcdir)/expr.h \
diff --git a/gcc/expr.c b/gcc/expr.c
index ca6b1c1291e..3c68b0d754c 100644
--- a/gcc/expr.c
+++ b/gcc/expr.c
@@ -9316,17 +9316,8 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode,
       if (temp != 0)
 	return temp;
 
-      /* For vector MIN <x, y>, expand it a VEC_COND_EXPR <x <= y, x, y>
-	 and similarly for MAX <x, y>.  */
       if (VECTOR_TYPE_P (type))
-	{
-	  tree t0 = make_tree (type, op0);
-	  tree t1 = make_tree (type, op1);
-	  tree comparison = build2 (code == MIN_EXPR ? LE_EXPR : GE_EXPR,
-				    type, t0, t1);
-	  return expand_vec_cond_expr (type, comparison, t0, t1,
-				       original_target);
-	}
+	gcc_unreachable ();
 
       /* At this point, a MEM target is no longer useful; we will get better
 	 code without it.  */
@@ -9915,10 +9906,6 @@ expand_expr_real_2 (sepops ops, rtx target, machine_mode tmode,
 	return temp;
       }
 
-    case VEC_COND_EXPR:
-      target = expand_vec_cond_expr (type, treeop0, treeop1, treeop2, target);
-      return target;
-
     case VEC_DUPLICATE_EXPR:
       op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
       target = expand_vector_broadcast (mode, op0);
@@ -12249,8 +12236,7 @@ do_store_flag (sepops ops, rtx target, machine_mode mode)
   STRIP_NOPS (arg1);
 
   /* For vector typed comparisons emit code to generate the desired
-     all-ones or all-zeros mask.  Conveniently use the VEC_COND_EXPR
-     expander for this.  */
+     all-ones or all-zeros mask.  */
   if (TREE_CODE (ops->type) == VECTOR_TYPE)
     {
       tree ifexp = build2 (ops->code, ops->type, arg0, arg1);
@@ -12258,12 +12244,7 @@ do_store_flag (sepops ops, rtx target, machine_mode mode)
 	  && expand_vec_cmp_expr_p (TREE_TYPE (arg0), ops->type, ops->code))
 	return expand_vec_cmp_expr (ops->type, ifexp, target);
       else
-	{
-	  tree if_true = constant_boolean_node (true, ops->type);
-	  tree if_false = constant_boolean_node (false, ops->type);
-	  return expand_vec_cond_expr (ops->type, ifexp, if_true,
-				       if_false, target);
-	}
+	gcc_unreachable ();
     }
 
   /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
diff --git a/gcc/gimplify.c b/gcc/gimplify.c
index 416fb609b94..339e8bc5089 100644
--- a/gcc/gimplify.c
+++ b/gcc/gimplify.c
@@ -14219,20 +14219,7 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
 	  }
 
 	case VEC_COND_EXPR:
-	  {
-	    enum gimplify_status r0, r1, r2;
-
-	    r0 = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
-				post_p, is_gimple_condexpr, fb_rvalue);
-	    r1 = gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p,
-				post_p, is_gimple_val, fb_rvalue);
-	    r2 = gimplify_expr (&TREE_OPERAND (*expr_p, 2), pre_p,
-				post_p, is_gimple_val, fb_rvalue);
-
-	    ret = MIN (MIN (r0, r1), r2);
-	    recalculate_side_effects (*expr_p);
-	  }
-	  break;
+	  goto expr_3;
 
 	case VEC_PERM_EXPR:
 	  /* Classified as tcc_expression.  */
diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 5e9aa60721e..644f234e087 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -49,6 +49,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-ssa.h"
 #include "tree-phinodes.h"
 #include "ssa-iterators.h"
+#include "explow.h"
 
 /* The names of each internal function, indexed by function number.  */
 const char *const internal_fn_name_array[] = {
@@ -107,6 +108,10 @@ init_internal_fns ()
 #define mask_store_direct { 3, 2, false }
 #define store_lanes_direct { 0, 0, false }
 #define mask_store_lanes_direct { 0, 0, false }
+#define vec_cond_mask_direct { 0, 0, false }
+#define vec_cond_direct { 0, 0, false }
+#define vec_condu_direct { 0, 0, false }
+#define vec_condeq_direct { 0, 0, false }
 #define scatter_store_direct { 3, 1, false }
 #define unary_direct { 0, 0, true }
 #define binary_direct { 0, 0, true }
@@ -2548,6 +2553,86 @@ expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
 
 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
 
+/* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
+   The expansion of STMT happens based on OPTAB table associated.  */
+
+static void
+expand_vect_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
+{
+  class expand_operand ops[6];
+  insn_code icode;
+  tree lhs = gimple_call_lhs (stmt);
+  tree op0a = gimple_call_arg (stmt, 0);
+  tree op0b = gimple_call_arg (stmt, 1);
+  tree op1 = gimple_call_arg (stmt, 2);
+  tree op2 = gimple_call_arg (stmt, 3);
+  enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (stmt, 4));
+
+  tree vec_cond_type = TREE_TYPE (lhs);
+  tree op_mode = TREE_TYPE (op0a);
+  bool unsignedp = TYPE_UNSIGNED (op_mode);
+
+  machine_mode mode = TYPE_MODE (vec_cond_type);
+  machine_mode cmp_op_mode = TYPE_MODE (op_mode);
+
+  icode = convert_optab_handler (optab, mode, cmp_op_mode);
+  rtx comparison
+    = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, icode, 4);
+  rtx rtx_op1 = expand_normal (op1);
+  rtx rtx_op2 = expand_normal (op2);
+
+  rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  create_output_operand (&ops[0], target, mode);
+  create_input_operand (&ops[1], rtx_op1, mode);
+  create_input_operand (&ops[2], rtx_op2, mode);
+  create_fixed_operand (&ops[3], comparison);
+  create_fixed_operand (&ops[4], XEXP (comparison, 0));
+  create_fixed_operand (&ops[5], XEXP (comparison, 1));
+  expand_insn (icode, 6, ops);
+}
+
+#define expand_vec_cond_optab_fn expand_vect_cond_optab_fn
+#define expand_vec_condu_optab_fn expand_vect_cond_optab_fn
+#define expand_vec_condeq_optab_fn expand_vect_cond_optab_fn
+
+/* Expand VCOND_MASK optab internal function.
+   The expansion of STMT happens based on OPTAB table associated.  */
+
+static void
+expand_vect_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
+{
+  class expand_operand ops[4];
+
+  tree lhs = gimple_call_lhs (stmt);
+  tree op0 = gimple_call_arg (stmt, 0);
+  tree op1 = gimple_call_arg (stmt, 1);
+  tree op2 = gimple_call_arg (stmt, 2);
+  tree vec_cond_type = TREE_TYPE (lhs);
+
+  machine_mode mode = TYPE_MODE (vec_cond_type);
+  machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
+  enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
+  rtx mask, rtx_op1, rtx_op2;
+
+  gcc_assert (icode != CODE_FOR_nothing);
+
+  mask = expand_normal (op0);
+  rtx_op1 = expand_normal (op1);
+  rtx_op2 = expand_normal (op2);
+
+  mask = force_reg (mask_mode, mask);
+  rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
+
+  rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  create_output_operand (&ops[0], target, mode);
+  create_input_operand (&ops[1], rtx_op1, mode);
+  create_input_operand (&ops[2], rtx_op2, mode);
+  create_input_operand (&ops[3], mask, mask_mode);
+  expand_insn (icode, 4, ops);
+}
+
+#define expand_vec_cond_mask_optab_fn expand_vect_cond_mask_optab_fn
+
 static void
 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
 {
@@ -3131,6 +3216,10 @@ multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
 #define direct_mask_store_optab_supported_p direct_optab_supported_p
 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
+#define direct_vec_cond_mask_optab_supported_p multi_vector_optab_supported_p
+#define direct_vec_cond_optab_supported_p multi_vector_optab_supported_p
+#define direct_vec_condu_optab_supported_p multi_vector_optab_supported_p
+#define direct_vec_condeq_optab_supported_p multi_vector_optab_supported_p
 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
 #define direct_while_optab_supported_p convert_optab_supported_p
 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 1d190d492ff..0c6fc371190 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -136,6 +136,11 @@ DEF_INTERNAL_OPTAB_FN (STORE_LANES, ECF_CONST, vec_store_lanes, store_lanes)
 DEF_INTERNAL_OPTAB_FN (MASK_STORE_LANES, 0,
 		       vec_mask_store_lanes, mask_store_lanes)
 
+DEF_INTERNAL_OPTAB_FN (VCOND, 0, vcond, vec_cond)
+DEF_INTERNAL_OPTAB_FN (VCONDU, 0, vcondu, vec_condu)
+DEF_INTERNAL_OPTAB_FN (VCONDEQ, 0, vcondeq, vec_condeq)
+DEF_INTERNAL_OPTAB_FN (VCOND_MASK, 0, vcond_mask, vec_cond_mask)
+
 DEF_INTERNAL_OPTAB_FN (WHILE_ULT, ECF_CONST | ECF_NOTHROW, while_ult, while)
 DEF_INTERNAL_OPTAB_FN (CHECK_RAW_PTRS, ECF_CONST | ECF_NOTHROW,
 		       check_raw_ptrs, check_ptrs)
diff --git a/gcc/optabs.c b/gcc/optabs.c
index 6d0b76c13ba..184827fdf4e 100644
--- a/gcc/optabs.c
+++ b/gcc/optabs.c
@@ -5442,7 +5442,7 @@ get_rtx_code (enum tree_code tcode, bool unsignedp)
    first comparison operand for insn ICODE.  Do not generate the
    compare instruction itself.  */
 
-static rtx
+rtx
 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
 		    tree t_op0, tree t_op1, bool unsignedp,
 		    enum insn_code icode, unsigned int opno)
@@ -5809,128 +5809,6 @@ expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
   return tmp;
 }
 
-/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
-   three operands.  */
-
-rtx
-expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
-			   rtx target)
-{
-  class expand_operand ops[4];
-  machine_mode mode = TYPE_MODE (vec_cond_type);
-  machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
-  enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
-  rtx mask, rtx_op1, rtx_op2;
-
-  if (icode == CODE_FOR_nothing)
-    return 0;
-
-  mask = expand_normal (op0);
-  rtx_op1 = expand_normal (op1);
-  rtx_op2 = expand_normal (op2);
-
-  mask = force_reg (mask_mode, mask);
-  rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
-
-  create_output_operand (&ops[0], target, mode);
-  create_input_operand (&ops[1], rtx_op1, mode);
-  create_input_operand (&ops[2], rtx_op2, mode);
-  create_input_operand (&ops[3], mask, mask_mode);
-  expand_insn (icode, 4, ops);
-
-  return ops[0].value;
-}
-
-/* Generate insns for a VEC_COND_EXPR, given its TYPE and its
-   three operands.  */
-
-rtx
-expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
-		      rtx target)
-{
-  class expand_operand ops[6];
-  enum insn_code icode;
-  rtx comparison, rtx_op1, rtx_op2;
-  machine_mode mode = TYPE_MODE (vec_cond_type);
-  machine_mode cmp_op_mode;
-  bool unsignedp;
-  tree op0a, op0b;
-  enum tree_code tcode;
-
-  if (COMPARISON_CLASS_P (op0))
-    {
-      op0a = TREE_OPERAND (op0, 0);
-      op0b = TREE_OPERAND (op0, 1);
-      tcode = TREE_CODE (op0);
-    }
-  else
-    {
-      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
-      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
-	  != CODE_FOR_nothing)
-	return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
-					  op2, target);
-      /* Fake op0 < 0.  */
-      else
-	{
-	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
-		      == MODE_VECTOR_INT);
-	  op0a = op0;
-	  op0b = build_zero_cst (TREE_TYPE (op0));
-	  tcode = LT_EXPR;
-	}
-    }
-  cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
-  unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
-
-
-  gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
-	      && known_eq (GET_MODE_NUNITS (mode),
-			   GET_MODE_NUNITS (cmp_op_mode)));
-
-  icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
-  if (icode == CODE_FOR_nothing)
-    {
-      if (tcode == LT_EXPR
-	  && op0a == op0
-	  && TREE_CODE (op0) == VECTOR_CST)
-	{
-	  /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
-	     into a constant when only get_vcond_eq_icode is supported.
-	     Verify < 0 and != 0 behave the same and change it to NE_EXPR.  */
-	  unsigned HOST_WIDE_INT nelts;
-	  if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
-	    {
-	      if (VECTOR_CST_STEPPED_P (op0))
-		return 0;
-	      nelts = vector_cst_encoded_nelts (op0);
-	    }
-	  for (unsigned int i = 0; i < nelts; ++i)
-	    if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
-	      return 0;
-	  tcode = NE_EXPR;
-	}
-      if (tcode == EQ_EXPR || tcode == NE_EXPR)
-	icode = get_vcond_eq_icode (mode, cmp_op_mode);
-      if (icode == CODE_FOR_nothing)
-	return 0;
-    }
-
-  comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
-				   icode, 4);
-  rtx_op1 = expand_normal (op1);
-  rtx_op2 = expand_normal (op2);
-
-  create_output_operand (&ops[0], target, mode);
-  create_input_operand (&ops[1], rtx_op1, mode);
-  create_input_operand (&ops[2], rtx_op2, mode);
-  create_fixed_operand (&ops[3], comparison);
-  create_fixed_operand (&ops[4], XEXP (comparison, 0));
-  create_fixed_operand (&ops[5], XEXP (comparison, 1));
-  expand_insn (icode, 6, ops);
-  return ops[0].value;
-}
-
 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
    Use TARGET for the result if nonnull and convenient.  */
 
diff --git a/gcc/optabs.h b/gcc/optabs.h
index 5bd19503a0a..7c2ec257cb0 100644
--- a/gcc/optabs.h
+++ b/gcc/optabs.h
@@ -321,9 +321,6 @@ extern rtx expand_vec_perm_const (machine_mode, rtx, rtx,
 /* Generate code for vector comparison.  */
 extern rtx expand_vec_cmp_expr (tree, tree, rtx);
 
-/* Generate code for VEC_COND_EXPR.  */
-extern rtx expand_vec_cond_expr (tree, tree, tree, tree, rtx);
-
 /* Generate code for VEC_SERIES_EXPR.  */
 extern rtx expand_vec_series_expr (machine_mode, rtx, rtx, rtx);
 
@@ -364,5 +361,9 @@ extern void expand_jump_insn (enum insn_code icode, unsigned int nops,
 			      class expand_operand *ops);
 
 extern enum rtx_code get_rtx_code (enum tree_code tcode, bool unsignedp);
+extern rtx vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
+			       tree t_op0, tree t_op1, bool unsignedp,
+			       enum insn_code icode, unsigned int opno);
+
 
 #endif /* GCC_OPTABS_H */
diff --git a/gcc/passes.def b/gcc/passes.def
index 56322025226..2b1e09fdda3 100644
--- a/gcc/passes.def
+++ b/gcc/passes.def
@@ -399,6 +399,7 @@ along with GCC; see the file COPYING3.  If not see
   NEXT_PASS (pass_cleanup_eh);
   NEXT_PASS (pass_lower_resx);
   NEXT_PASS (pass_nrv);
+  NEXT_PASS (pass_gimple_isel);
   NEXT_PASS (pass_cleanup_cfg_post_optimizing);
   NEXT_PASS (pass_warn_function_noreturn);
   NEXT_PASS (pass_gen_hsail);
diff --git a/gcc/testsuite/g++.dg/vect/vec-cond-expr-eh.C b/gcc/testsuite/g++.dg/vect/vec-cond-expr-eh.C
new file mode 100644
index 00000000000..00fe2422444
--- /dev/null
+++ b/gcc/testsuite/g++.dg/vect/vec-cond-expr-eh.C
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-fnon-call-exceptions" } */
+
+typedef double v2df __attribute__((vector_size(16)));
+
+v2df foo (v2df a, v2df b, v2df c, v2df d)
+{
+  try
+  {
+    v2df res = a < b ? c : d;
+    return res;
+    }
+    catch (...)
+    {
+    return (v2df){};
+    }
+}
diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c
index d06a479e570..7a1ac80c2ac 100644
--- a/gcc/tree-cfg.c
+++ b/gcc/tree-cfg.c
@@ -4155,7 +4155,7 @@ verify_gimple_assign_ternary (gassign *stmt)
       return true;
     }
 
-  if (((rhs_code == VEC_COND_EXPR || rhs_code == COND_EXPR)
+  if ((rhs_code == COND_EXPR
        ? !is_gimple_condexpr (rhs1) : !is_gimple_val (rhs1))
       || !is_gimple_val (rhs2)
       || !is_gimple_val (rhs3))
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index 396428f167f..215c8f2a337 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -627,6 +627,7 @@ extern gimple_opt_pass *make_pass_local_fn_summary (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_update_address_taken (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_convert_switch (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_lower_vaarg (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_gimple_isel (gcc::context *ctxt);
 
 /* Current optimization pass.  */
 extern opt_pass *current_pass;
diff --git a/gcc/tree-ssa-forwprop.c b/gcc/tree-ssa-forwprop.c
index 494c9e9c20b..0ab8267fdf9 100644
--- a/gcc/tree-ssa-forwprop.c
+++ b/gcc/tree-ssa-forwprop.c
@@ -3133,8 +3133,7 @@ pass_forwprop::execute (function *fun)
 		    tree rhs1 = gimple_assign_rhs1 (stmt);
 		    enum tree_code code = gimple_assign_rhs_code (stmt);
 
-		    if (code == COND_EXPR
-			|| code == VEC_COND_EXPR)
+		    if (code == COND_EXPR)
 		      {
 			/* In this case the entire COND_EXPR is in rhs1. */
 			if (forward_propagate_into_cond (&gsi))
diff --git a/gcc/tree-ssa-reassoc.c b/gcc/tree-ssa-reassoc.c
index af8faf2e6ea..8d80a2c768c 100644
--- a/gcc/tree-ssa-reassoc.c
+++ b/gcc/tree-ssa-reassoc.c
@@ -3831,7 +3831,8 @@ optimize_range_tests (enum tree_code opcode,
    to type of comparison.  */
 
 static tree_code
-ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type)
+ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type,
+		  tree *lhs, tree *rhs, gassign **vcond)
 {
   if (TREE_CODE (var) != SSA_NAME)
     return ERROR_MARK;
@@ -3839,6 +3840,8 @@ ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type)
   gassign *stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (var));
   if (stmt == NULL)
     return ERROR_MARK;
+  if (*vcond)
+    *vcond = stmt;
 
   /* ??? If we start creating more COND_EXPR, we could perform
      this same optimization with them.	For now, simplify.  */
@@ -3847,9 +3850,20 @@ ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type)
 
   tree cond = gimple_assign_rhs1 (stmt);
   tree_code cmp = TREE_CODE (cond);
-  if (TREE_CODE_CLASS (cmp) != tcc_comparison)
+  if (cmp != SSA_NAME)
     return ERROR_MARK;
 
+  gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cond));
+  if (stmt != NULL
+      && TREE_CODE_CLASS (gimple_assign_rhs_code (assign)) != tcc_comparison)
+    return ERROR_MARK;
+
+  cmp = gimple_assign_rhs_code (assign);
+  if (lhs)
+    *lhs = gimple_assign_rhs1 (assign);
+  if (rhs)
+    *rhs = gimple_assign_rhs2 (assign);
+
   /* ??? For now, allow only canonical true and false result vectors.
      We could expand this to other constants should the need arise,
      but at the moment we don't create them.  */
@@ -3870,7 +3884,7 @@ ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type)
 
   /* Success!  */
   if (rets)
-    *rets = stmt;
+    *rets = assign;
   if (reti)
     *reti = inv;
   if (type)
@@ -3894,10 +3908,11 @@ optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
     {
       tree elt0 = (*ops)[i]->op;
 
-      gassign *stmt0;
+      gassign *stmt0, *vcond0;
       bool invert;
-      tree type;
-      tree_code cmp0 = ovce_extract_ops (elt0, &stmt0, &invert, &type);
+      tree type, lhs0, rhs0;
+      tree_code cmp0 = ovce_extract_ops (elt0, &stmt0, &invert, &type, &lhs0,
+					 &rhs0, &vcond0);
       if (cmp0 == ERROR_MARK)
 	continue;
 
@@ -3905,26 +3920,20 @@ optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
 	{
 	  tree &elt1 = (*ops)[j]->op;
 
-	  gassign *stmt1;
-	  tree_code cmp1 = ovce_extract_ops (elt1, &stmt1, NULL, NULL);
+	  gassign *stmt1, *vcond1;
+	  tree lhs1, rhs1;
+	  tree_code cmp1 = ovce_extract_ops (elt1, &stmt1, NULL, NULL, &lhs1,
+					     &rhs1, &vcond1);
 	  if (cmp1 == ERROR_MARK)
 	    continue;
 
-	  tree cond0 = gimple_assign_rhs1 (stmt0);
-	  tree x0 = TREE_OPERAND (cond0, 0);
-	  tree y0 = TREE_OPERAND (cond0, 1);
-
-	  tree cond1 = gimple_assign_rhs1 (stmt1);
-	  tree x1 = TREE_OPERAND (cond1, 0);
-	  tree y1 = TREE_OPERAND (cond1, 1);
-
 	  tree comb;
 	  if (opcode == BIT_AND_EXPR)
-	    comb = maybe_fold_and_comparisons (type, cmp0, x0, y0, cmp1, x1,
-					       y1);
+	    comb = maybe_fold_and_comparisons (type, cmp0, lhs0, rhs0,
+					       cmp1, lhs1, rhs1);
 	  else if (opcode == BIT_IOR_EXPR)
-	    comb = maybe_fold_or_comparisons (type, cmp0, x0, y0, cmp1, x1,
-					      y1);
+	    comb = maybe_fold_or_comparisons (type, cmp0, lhs0, rhs0,
+					      cmp1, lhs1, rhs1);
 	  else
 	    gcc_unreachable ();
 	  if (comb == NULL)
@@ -3934,19 +3943,27 @@ optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
 	  if (dump_file && (dump_flags & TDF_DETAILS))
 	    {
 	      fprintf (dump_file, "Transforming ");
-	      print_generic_expr (dump_file, cond0);
+	      print_generic_expr (dump_file, gimple_assign_lhs (stmt0));
 	      fprintf (dump_file, " %c ", opcode == BIT_AND_EXPR ? '&' : '|');
-	      print_generic_expr (dump_file, cond1);
+	      print_generic_expr (dump_file, gimple_assign_lhs (stmt1));
 	      fprintf (dump_file, " into ");
 	      print_generic_expr (dump_file, comb);
 	      fputc ('\n', dump_file);
 	    }
 
-	  gimple_assign_set_rhs1 (stmt0, comb);
+	  gimple_seq seq;
+	  tree exp = force_gimple_operand (comb, &seq, true, NULL_TREE);
+	  if (seq)
+	    {
+	      gimple_stmt_iterator gsi = gsi_for_stmt (vcond0);
+	      gsi_insert_before (&gsi, seq, GSI_SAME_STMT);
+	    }
 	  if (invert)
-	    std::swap (*gimple_assign_rhs2_ptr (stmt0),
-		       *gimple_assign_rhs3_ptr (stmt0));
-	  update_stmt (stmt0);
+	    std::swap (*gimple_assign_rhs2_ptr (vcond0),
+		       *gimple_assign_rhs3_ptr (vcond0));
+
+	  gimple_assign_set_rhs1 (vcond0, exp);
+	  update_stmt (vcond0);
 
 	  elt1 = error_mark_node;
 	  any_changes = true;
diff --git a/gcc/tree-vect-generic.c b/gcc/tree-vect-generic.c
index a7fe83da0e3..fb955bbf3d2 100644
--- a/gcc/tree-vect-generic.c
+++ b/gcc/tree-vect-generic.c
@@ -694,12 +694,14 @@ expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
 	  if (addend == NULL_TREE
 	      && expand_vec_cond_expr_p (type, type, LT_EXPR))
 	    {
-	      tree zero, cst, cond, mask_type;
-	      gimple *stmt;
+	      tree zero, cst, mask_type, mask;
+	      gimple *stmt, *cond;
 
 	      mask_type = truth_type_for (type);
 	      zero = build_zero_cst (type);
-	      cond = build2 (LT_EXPR, mask_type, op0, zero);
+	      mask = make_ssa_name (mask_type);
+	      cond = gimple_build_assign (mask, LT_EXPR, op0, zero);
+	      gsi_insert_before (gsi, cond, GSI_SAME_STMT);
 	      tree_vector_builder vec (type, nunits, 1);
 	      for (i = 0; i < nunits; i++)
 		vec.quick_push (build_int_cst (TREE_TYPE (type),
@@ -707,8 +709,8 @@ expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
 						<< shifts[i]) - 1));
 	      cst = vec.build ();
 	      addend = make_ssa_name (type);
-	      stmt = gimple_build_assign (addend, VEC_COND_EXPR, cond,
-					  cst, zero);
+	      stmt
+		= gimple_build_assign (addend, VEC_COND_EXPR, mask, cst, zero);
 	      gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
 	    }
 	}
@@ -950,21 +952,28 @@ expand_vector_condition (gimple_stmt_iterator *gsi)
   tree index = bitsize_int (0);
   tree comp_width = width;
   tree comp_index = index;
-  int i;
   location_t loc = gimple_location (gsi_stmt (*gsi));
+  tree_code code = TREE_CODE (a);
 
-  if (!is_gimple_val (a))
+  if (code == SSA_NAME)
     {
-      gcc_assert (COMPARISON_CLASS_P (a));
-      a_is_comparison = true;
-      a1 = TREE_OPERAND (a, 0);
-      a2 = TREE_OPERAND (a, 1);
-      comp_inner_type = TREE_TYPE (TREE_TYPE (a1));
-      comp_width = vector_element_bits_tree (TREE_TYPE (a1));
+      gimple *assign = SSA_NAME_DEF_STMT (a);
+      if (TREE_CODE_CLASS (gimple_assign_rhs_code (assign)) == tcc_comparison)
+	{
+	  a_is_comparison = true;
+	  a1 = gimple_assign_rhs1 (assign);
+	  a2 = gimple_assign_rhs2 (assign);
+	  code = gimple_assign_rhs_code (assign);
+	  comp_inner_type = TREE_TYPE (TREE_TYPE (a1));
+	  comp_width = vector_element_bits_tree (TREE_TYPE (a1));
+	}
     }
 
-  if (expand_vec_cond_expr_p (type, TREE_TYPE (a1), TREE_CODE (a)))
-    return;
+  if (expand_vec_cond_expr_p (type, TREE_TYPE (a1), code))
+    {
+      gcc_assert (TREE_CODE (a) == SSA_NAME || TREE_CODE (a) == VECTOR_CST);
+      return;
+    }
 
   /* Handle vector boolean types with bitmasks.  If there is a comparison
      and we can expand the comparison into the vector boolean bitmask,
@@ -987,7 +996,7 @@ expand_vector_condition (gimple_stmt_iterator *gsi)
 	  : expand_vec_cmp_expr_p (TREE_TYPE (a1), type, TREE_CODE (a))))
     {
       if (a_is_comparison)
-	a = gimplify_build2 (gsi, TREE_CODE (a), type, a1, a2);
+	a = gimplify_build2 (gsi, code, type, a1, a2);
       a1 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a, b);
       a2 = gimplify_build1 (gsi, BIT_NOT_EXPR, type, a);
       a2 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a2, c);
@@ -1018,7 +1027,7 @@ expand_vector_condition (gimple_stmt_iterator *gsi)
 
   int nunits = nunits_for_known_piecewise_op (type);
   vec_alloc (v, nunits);
-  for (i = 0; i < nunits; i++)
+  for (int i = 0; i < nunits; i++)
     {
       tree aa, result;
       tree bb = tree_vec_extract (gsi, inner_type, b, width, index);
@@ -1029,7 +1038,7 @@ expand_vector_condition (gimple_stmt_iterator *gsi)
 				       comp_width, comp_index);
 	  tree aa2 = tree_vec_extract (gsi, comp_inner_type, a2,
 				       comp_width, comp_index);
-	  aa = fold_build2 (TREE_CODE (a), cond_type, aa1, aa2);
+	  aa = fold_build2 (code, cond_type, aa1, aa2);
 	}
       else if (a_is_scalar_bitmask)
 	{
diff --git a/gcc/tree-vect-isel.c b/gcc/tree-vect-isel.c
new file mode 100644
index 00000000000..97f92080503
--- /dev/null
+++ b/gcc/tree-vect-isel.c
@@ -0,0 +1,244 @@
+/* Schedule GIMPLE vector statements.
+   Copyright (C) 2020 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 3, or (at your option) any
+later version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+#include "rtl.h"
+#include "tree.h"
+#include "gimple.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "expmed.h"
+#include "optabs-tree.h"
+#include "tree-eh.h"
+#include "gimple-iterator.h"
+#include "gimplify-me.h"
+#include "gimplify.h"
+#include "tree-cfg.h"
+
+/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
+   function based on type of selected expansion.  */
+
+static gimple *
+gimple_expand_vec_cond_expr (gimple_stmt_iterator *gsi,
+			     hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
+{
+  tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
+  enum tree_code code;
+  enum tree_code tcode;
+  machine_mode cmp_op_mode;
+  bool unsignedp;
+  enum insn_code icode;
+  imm_use_iterator imm_iter;
+
+  /* Only consider code == GIMPLE_ASSIGN.  */
+  gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
+  if (!stmt)
+    return NULL;
+
+  code = gimple_assign_rhs_code (stmt);
+  if (code != VEC_COND_EXPR)
+    return NULL;
+
+  tree op0 = gimple_assign_rhs1 (stmt);
+  tree op1 = gimple_assign_rhs2 (stmt);
+  tree op2 = gimple_assign_rhs3 (stmt);
+  lhs = gimple_assign_lhs (stmt);
+  machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
+
+  gcc_assert (!COMPARISON_CLASS_P (op0));
+  if (TREE_CODE (op0) == SSA_NAME)
+    {
+      unsigned int used_vec_cond_exprs = 0;
+      unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
+      if (slot)
+	used_vec_cond_exprs = *slot;
+      else
+	{
+	  gimple *use_stmt;
+	  FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
+	    {
+	      gassign *assign = dyn_cast<gassign *> (use_stmt);
+	      if (assign != NULL
+		  && gimple_assign_rhs_code (assign) == VEC_COND_EXPR
+		  && gimple_assign_rhs1 (assign) == op0)
+		used_vec_cond_exprs++;
+	    }
+	  vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
+	}
+
+      gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
+      if (def_stmt)
+	{
+	  tcode = gimple_assign_rhs_code (def_stmt);
+	  op0a = gimple_assign_rhs1 (def_stmt);
+	  op0b = gimple_assign_rhs2 (def_stmt);
+
+	  tree op0a_type = TREE_TYPE (op0a);
+	  if (used_vec_cond_exprs >= 2
+	      && (get_vcond_mask_icode (mode, TYPE_MODE (op0a_type))
+		  != CODE_FOR_nothing)
+	      && expand_vec_cmp_expr_p (op0a_type, TREE_TYPE (lhs), tcode))
+	    {
+	      /* Keep the SSA name and use vcond_mask.  */
+	      tcode = TREE_CODE (op0);
+	    }
+	}
+      else
+	tcode = TREE_CODE (op0);
+    }
+  else
+    tcode = TREE_CODE (op0);
+
+  if (TREE_CODE_CLASS (tcode) != tcc_comparison)
+    {
+      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
+      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
+	  != CODE_FOR_nothing)
+	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
+      /* Fake op0 < 0.  */
+      else
+	{
+	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
+		      == MODE_VECTOR_INT);
+	  op0a = op0;
+	  op0b = build_zero_cst (TREE_TYPE (op0));
+	  tcode = LT_EXPR;
+	}
+    }
+  cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
+  unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
+
+
+  gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
+	      && known_eq (GET_MODE_NUNITS (mode),
+			   GET_MODE_NUNITS (cmp_op_mode)));
+
+  icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
+  if (icode == CODE_FOR_nothing)
+    {
+      if (tcode == LT_EXPR
+	  && op0a == op0
+	  && TREE_CODE (op0) == VECTOR_CST)
+	{
+	  /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
+	     into a constant when only get_vcond_eq_icode is supported.
+	     Verify < 0 and != 0 behave the same and change it to NE_EXPR.  */
+	  unsigned HOST_WIDE_INT nelts;
+	  if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
+	    {
+	      if (VECTOR_CST_STEPPED_P (op0))
+		gcc_unreachable ();
+	      nelts = vector_cst_encoded_nelts (op0);
+	    }
+	  for (unsigned int i = 0; i < nelts; ++i)
+	    if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
+	      gcc_unreachable ();
+	  tcode = NE_EXPR;
+	}
+      if (tcode == EQ_EXPR || tcode == NE_EXPR)
+	{
+	  tree tcode_tree = build_int_cst (integer_type_node, tcode);
+	  return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
+					     op2, tcode_tree);
+	}
+    }
+
+  gcc_assert (icode != CODE_FOR_nothing);
+  tree tcode_tree = build_int_cst (integer_type_node, tcode);
+  return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
+				     5, op0a, op0b, op1, op2, tcode_tree);
+}
+
+
+
+/* Iterate all gimple statements and try to expand
+   VEC_COND_EXPR assignments.  */
+
+static unsigned int
+gimple_expand_vec_cond_exprs (void)
+{
+  gimple_stmt_iterator gsi;
+  basic_block bb;
+  bool cfg_changed = false;
+  hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
+
+  FOR_EACH_BB_FN (bb, cfun)
+    {
+      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+	{
+	  gimple *g = gimple_expand_vec_cond_expr (&gsi,
+						   &vec_cond_ssa_name_uses);
+	  if (g != NULL)
+	    {
+	      tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
+	      gimple_set_lhs (g, lhs);
+	      gsi_replace (&gsi, g, false);
+	    }
+	}
+    }
+
+  return cfg_changed ? TODO_cleanup_cfg : 0;
+}
+
+namespace {
+
+const pass_data pass_data_gimple_isel =
+{
+  GIMPLE_PASS, /* type */
+  "isel", /* name */
+  OPTGROUP_VEC, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  PROP_cfg, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  TODO_update_ssa, /* todo_flags_finish */
+};
+
+class pass_gimple_isel : public gimple_opt_pass
+{
+public:
+  pass_gimple_isel (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_gimple_isel, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  virtual bool gate (function *)
+    {
+      return true;
+    }
+
+  virtual unsigned int execute (function *)
+    {
+      return gimple_expand_vec_cond_exprs ();
+    }
+
+}; // class pass_gimple_isel
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_gimple_isel (gcc::context *ctxt)
+{
+  return new pass_gimple_isel (ctxt);
+}
+
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index cf2d979fea1..710b17a7c5c 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -9937,8 +9937,8 @@ vectorizable_condition (vec_info *vinfo,
 	{
 	  vec_cond_rhs = vec_oprnds1[i];
 	  if (bitop1 == NOP_EXPR)
-	    vec_compare = build2 (cond_code, vec_cmp_type,
-				  vec_cond_lhs, vec_cond_rhs);
+	    vec_compare = gimplify_build2 (gsi, cond_code, vec_cmp_type,
+					   vec_cond_lhs, vec_cond_rhs);
 	  else
 	    {
 	      new_temp = make_ssa_name (vec_cmp_type);
-- 
2.27.0