diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index abd905b78f3661f80168c3866d7c3e68a9c15521..eef2bb50c1505f5cf802d5d80300affc2cbe69f6 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -3512,11 +3512,14 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, epilog = vect_epilogues ? get_loop_copy (loop) : scalar_loop; edge epilog_e = vect_epilogues ? e : scalar_e; edge new_epilog_e = NULL; - epilog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog, - epilog_e, e, - &new_epilog_e); + auto_vec doms; + epilog + = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog, epilog_e, e, + &new_epilog_e, true, &doms); + LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo) = new_epilog_e; gcc_assert (epilog); + gcc_assert (new_epilog_e); epilog->force_vectorize = false; bb_before_epilog = loop_preheader_edge (epilog)->src; @@ -3610,10 +3613,16 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, } - if (skip_epilog) + if (skip_epilog || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) { - guard_cond = fold_build2 (EQ_EXPR, boolean_type_node, + /* For the case where a different exit was chosen we must execute + the scalar loop with the remaining iterations. */ + if (inversed_iv) + guard_cond = boolean_false_node; + else + guard_cond = fold_build2 (EQ_EXPR, boolean_type_node, niters, niters_vector_mult_vf); + guard_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest; edge epilog_e = LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo); guard_to = split_edge (epilog_e); @@ -3621,11 +3630,13 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, skip_vector ? anchor : guard_bb, prob_epilog.invert (), irred_flag); + doms.safe_push (guard_to); if (vect_epilogues) epilogue_vinfo->skip_this_loop_edge = guard_e; edge main_iv = LOOP_VINFO_IV_EXIT (loop_vinfo); slpeel_update_phi_nodes_for_guard2 (loop, epilog, main_iv, guard_e, epilog_e); + /* Only need to handle basic block before epilog loop if it's not the guard_bb, which is the case when skip_vector is true. */ if (guard_bb != bb_before_epilog) @@ -3637,6 +3648,10 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, scale_loop_profile (epilog, prob_epilog, -1); } + /* Recalculate the dominators after adding the guard edge. */ + if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) + iterate_fix_dominators (CDI_DOMINATORS, doms, false); + unsigned HOST_WIDE_INT bound; if (bound_scalar.is_constant (&bound)) {