Re: [PATCH 1/3]middle-end: Refactor vectorizer loop conditionals and separate out IV to new variables

[Richard Biener <rguenther@suse.de>]

On Mon, 2 Oct 2023, Tamar Christina wrote:

> Hi All,
> 
> This is extracted out of the patch series to support early break vectorization
> in order to simplify the review of that patch series.
> 
> The goal of this one is to separate out the refactoring from the new
> functionality.
> 
> This first patch separates out the vectorizer's definition of an exit to their
> own values inside loop_vinfo.  During vectorization we can have three separate
> copies for each loop: scalar, vectorized, epilogue.  The scalar loop can also be
> the versioned loop before peeling.
> 
> Because of this we track 3 different exits inside loop_vinfo corresponding to
> each of these loops.  Additionally each function that uses an exit, when not
> obviously clear which exit is needed will now take the exit explicitly as an
> argument.
> 
> This is because often times the callers switch the loops being passed around.
> While the caller knows which loops it is, the callee does not.
> 
> For now the loop exits are simply initialized to same value as before determined
> by single_exit (..).
> 
> No change in functionality is expected throughout this patch series.
> 
> Bootstrapped Regtested on aarch64-none-linux-gnu, x86_64-linux-gnu, and
> no issues.
> 
> Ok for master?
> 
> Thanks,
> Tamar
> 
> gcc/ChangeLog:
> 
> 	* tree-loop-distribution.cc (copy_loop_before): Pass exit explicitly.
> 	(loop_distribution::distribute_loop): Bail out of not single exit.
> 	* tree-scalar-evolution.cc (get_loop_exit_condition): New.
> 	* tree-scalar-evolution.h (get_loop_exit_condition): New.
> 	* tree-vect-data-refs.cc (vect_enhance_data_refs_alignment): Pass exit
> 	explicitly.
> 	* tree-vect-loop-manip.cc (vect_set_loop_condition_partial_vectors,
> 	vect_set_loop_condition_partial_vectors_avx512,
> 	vect_set_loop_condition_normal, vect_set_loop_condition): Explicitly
> 	take exit.
> 	(slpeel_tree_duplicate_loop_to_edge_cfg): Explicitly take exit and
> 	return new peeled corresponding peeled exit.
> 	(slpeel_can_duplicate_loop_p): Explicitly take exit.
> 	(find_loop_location): Handle not knowing an explicit exit.
> 	(vect_update_ivs_after_vectorizer, vect_gen_vector_loop_niters_mult_vf,
> 	find_guard_arg, slpeel_update_phi_nodes_for_loops,
> 	slpeel_update_phi_nodes_for_guard2): Use new exits.
> 	(vect_do_peeling): Update bookkeeping to keep track of exits.
> 	* tree-vect-loop.cc (vect_get_loop_niters): Explicitly take exit to
> 	analyze.
> 	(vec_init_loop_exit_info): New.
> 	(_loop_vec_info::_loop_vec_info): Initialize vec_loop_iv,
> 	vec_epilogue_loop_iv, scalar_loop_iv.
> 	(vect_analyze_loop_form): Initialize exits.
> 	(vect_create_loop_vinfo): Set main exit.
> 	(vect_create_epilog_for_reduction, vectorizable_live_operation,
> 	vect_transform_loop): Use it.
> 	(scale_profile_for_vect_loop): Explicitly take exit to scale.
> 	* tree-vectorizer.cc (set_uid_loop_bbs): Initialize loop exit.
> 	* tree-vectorizer.h (LOOP_VINFO_IV_EXIT, LOOP_VINFO_EPILOGUE_IV_EXIT,
> 	LOOP_VINFO_SCALAR_IV_EXIT): New.
> 	(struct loop_vec_info): Add vec_loop_iv, vec_epilogue_loop_iv,
> 	scalar_loop_iv.
> 	(vect_set_loop_condition, slpeel_can_duplicate_loop_p,
> 	slpeel_tree_duplicate_loop_to_edge_cfg): Take explicit exits.
> 	(vec_init_loop_exit_info): New.
> 	(struct vect_loop_form_info): Add loop_exit.
> 
> --- inline copy of patch -- 
> diff --git a/gcc/tree-loop-distribution.cc b/gcc/tree-loop-distribution.cc
> index a28470b66ea935741a61fb73961ed7c927543a3d..902edc49ab588152a5b845f2c8a42a7e2a1d6080 100644
> --- a/gcc/tree-loop-distribution.cc
> +++ b/gcc/tree-loop-distribution.cc
> @@ -949,7 +949,8 @@ copy_loop_before (class loop *loop, bool redirect_lc_phi_defs)
>    edge preheader = loop_preheader_edge (loop);
>  
>    initialize_original_copy_tables ();
> -  res = slpeel_tree_duplicate_loop_to_edge_cfg (loop, NULL, preheader);
> +  res = slpeel_tree_duplicate_loop_to_edge_cfg (loop, single_exit (loop), NULL,
> +						NULL, preheader, NULL);
>    gcc_assert (res != NULL);
>  
>    /* When a not last partition is supposed to keep the LC PHIs computed
> @@ -3043,6 +3044,24 @@ loop_distribution::distribute_loop (class loop *loop,
>        return 0;
>      }
>  
> +  /* Loop distribution only does prologue peeling but we still need to
> +     initialize loop exit information.  However we only support single exits at
> +     the moment.  As such, should exit information not have been provided and we
> +     have more than one exit, bail out.  */
> +  if (!single_exit (loop))
> +    {
> +      if (dump_file && (dump_flags & TDF_DETAILS))
> +	fprintf (dump_file,
> +		 "Loop %d not distributed: too many exits.\n",
> +		 loop->num);
> +
> +      free_rdg (rdg);
> +      loop_nest.release ();
> +      free_data_refs (datarefs_vec);
> +      delete ddrs_table;
> +      return 0;
> +    }

We are checking single_exit in the caller:

unsigned int
loop_distribution::execute (function *fun)
{
...
  /* We can at the moment only distribute non-nested loops, thus restrict
     walking to innermost loops.  */
  for (auto loop : loops_list (cfun, LI_ONLY_INNERMOST))
    {
      /* Don't distribute multiple exit edges loop, or cold loop when
         not doing pattern detection.  */
      if (!single_exit (loop)
          || (!flag_tree_loop_distribute_patterns
              && !optimize_loop_for_speed_p (loop)))
        continue;

so this hunk shouldn't be necessary.

> +
>    data_reference_p dref;
>    for (i = 0; datarefs_vec.iterate (i, &dref); ++i)
>      dref->aux = (void *) (uintptr_t) i;
> diff --git a/gcc/tree-scalar-evolution.h b/gcc/tree-scalar-evolution.h
> index c58a8a16e81573aada38e912b7c58b3e1b23b66d..f35ca1bded0b841179e4958645d264ad23684019 100644
> --- a/gcc/tree-scalar-evolution.h
> +++ b/gcc/tree-scalar-evolution.h
> @@ -23,6 +23,7 @@ along with GCC; see the file COPYING3.  If not see
>  
>  extern tree number_of_latch_executions (class loop *);
>  extern gcond *get_loop_exit_condition (const class loop *);
> +extern gcond *get_loop_exit_condition (const_edge);
>  
>  extern void scev_initialize (void);
>  extern bool scev_initialized_p (void);
> diff --git a/gcc/tree-scalar-evolution.cc b/gcc/tree-scalar-evolution.cc
> index 3fb6951e6085352c027d32c3548246042b98b64b..7cafe5ce576079921e380aaab5c5c4aa84cea372 100644
> --- a/gcc/tree-scalar-evolution.cc
> +++ b/gcc/tree-scalar-evolution.cc
> @@ -1292,9 +1292,17 @@ scev_dfs::follow_ssa_edge_expr (gimple *at_stmt, tree expr,
>  
>  gcond *
>  get_loop_exit_condition (const class loop *loop)
> +{
> +  return get_loop_exit_condition (single_exit (loop));
> +}
> +
> +/* If the statement just before the EXIT_EDGE contains a condition then
> +   return the condition, otherwise NULL. */
> +
> +gcond *
> +get_loop_exit_condition (const_edge exit_edge)
>  {
>    gcond *res = NULL;
> -  edge exit_edge = single_exit (loop);
>  
>    if (dump_file && (dump_flags & TDF_SCEV))
>      fprintf (dump_file, "(get_loop_exit_condition \n  ");
> diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
> index 40ab568fe355964b878d770010aa9eeaef63eeac..9607a9fb25da26591ffd8071a02495f2042e0579 100644
> --- a/gcc/tree-vect-data-refs.cc
> +++ b/gcc/tree-vect-data-refs.cc
> @@ -2078,7 +2078,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
>  
>    /* Check if we can possibly peel the loop.  */
>    if (!vect_can_advance_ivs_p (loop_vinfo)
> -      || !slpeel_can_duplicate_loop_p (loop, single_exit (loop))
> +      || !slpeel_can_duplicate_loop_p (loop, LOOP_VINFO_IV_EXIT (loop_vinfo),
> +				       LOOP_VINFO_IV_EXIT (loop_vinfo))
>        || loop->inner)
>      do_peeling = false;
>  
> diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
> index 09641901ff1e5c03dd07ab6f85dd67288f940ea2..e06717272aafc6d31cbdcb94840ac25de616da6d 100644
> --- a/gcc/tree-vect-loop-manip.cc
> +++ b/gcc/tree-vect-loop-manip.cc
> @@ -803,7 +803,7 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo,
>     final gcond.  */
>  
>  static gcond *
> -vect_set_loop_condition_partial_vectors (class loop *loop,
> +vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge,
>  					 loop_vec_info loop_vinfo, tree niters,
>  					 tree final_iv, bool niters_maybe_zero,
>  					 gimple_stmt_iterator loop_cond_gsi)
> @@ -904,7 +904,6 @@ vect_set_loop_condition_partial_vectors (class loop *loop,
>    add_header_seq (loop, header_seq);
>  
>    /* Get a boolean result that tells us whether to iterate.  */
> -  edge exit_edge = single_exit (loop);
>    gcond *cond_stmt;
>    if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo)
>        && !LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
> @@ -935,7 +934,7 @@ vect_set_loop_condition_partial_vectors (class loop *loop,
>    if (final_iv)
>      {
>        gassign *assign = gimple_build_assign (final_iv, orig_niters);
> -      gsi_insert_on_edge_immediate (single_exit (loop), assign);
> +      gsi_insert_on_edge_immediate (exit_edge, assign);
>      }
>  
>    return cond_stmt;
> @@ -953,6 +952,7 @@ vect_set_loop_condition_partial_vectors (class loop *loop,
>  
>  static gcond *
>  vect_set_loop_condition_partial_vectors_avx512 (class loop *loop,
> +					 edge exit_edge,
>  					 loop_vec_info loop_vinfo, tree niters,
>  					 tree final_iv,
>  					 bool niters_maybe_zero,
> @@ -1144,7 +1144,6 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop,
>    add_preheader_seq (loop, preheader_seq);
>  
>    /* Adjust the exit test using the decrementing IV.  */
> -  edge exit_edge = single_exit (loop);
>    tree_code code = (exit_edge->flags & EDGE_TRUE_VALUE) ? LE_EXPR : GT_EXPR;
>    /* When we peel for alignment with niter_skip != 0 this can
>       cause niter + niter_skip to wrap and since we are comparing the
> @@ -1183,7 +1182,8 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop,
>     loop handles exactly VF scalars per iteration.  */
>  
>  static gcond *
> -vect_set_loop_condition_normal (class loop *loop, tree niters, tree step,
> +vect_set_loop_condition_normal (loop_vec_info /* loop_vinfo */, edge exit_edge,
> +				class loop *loop, tree niters, tree step,
>  				tree final_iv, bool niters_maybe_zero,
>  				gimple_stmt_iterator loop_cond_gsi)
>  {
> @@ -1191,13 +1191,12 @@ vect_set_loop_condition_normal (class loop *loop, tree niters, tree step,
>    gcond *cond_stmt;
>    gcond *orig_cond;
>    edge pe = loop_preheader_edge (loop);
> -  edge exit_edge = single_exit (loop);
>    gimple_stmt_iterator incr_gsi;
>    bool insert_after;
>    enum tree_code code;
>    tree niters_type = TREE_TYPE (niters);
>  
> -  orig_cond = get_loop_exit_condition (loop);
> +  orig_cond = get_loop_exit_condition (exit_edge);
>    gcc_assert (orig_cond);
>    loop_cond_gsi = gsi_for_stmt (orig_cond);
>  
> @@ -1305,19 +1304,18 @@ vect_set_loop_condition_normal (class loop *loop, tree niters, tree step,
>    if (final_iv)
>      {
>        gassign *assign;
> -      edge exit = single_exit (loop);
> -      gcc_assert (single_pred_p (exit->dest));
> +      gcc_assert (single_pred_p (exit_edge->dest));
>        tree phi_dest
>  	= integer_zerop (init) ? final_iv : copy_ssa_name (indx_after_incr);
>        /* Make sure to maintain LC SSA form here and elide the subtraction
>  	 if the value is zero.  */
> -      gphi *phi = create_phi_node (phi_dest, exit->dest);
> -      add_phi_arg (phi, indx_after_incr, exit, UNKNOWN_LOCATION);
> +      gphi *phi = create_phi_node (phi_dest, exit_edge->dest);
> +      add_phi_arg (phi, indx_after_incr, exit_edge, UNKNOWN_LOCATION);
>        if (!integer_zerop (init))
>  	{
>  	  assign = gimple_build_assign (final_iv, MINUS_EXPR,
>  					phi_dest, init);
> -	  gimple_stmt_iterator gsi = gsi_after_labels (exit->dest);
> +	  gimple_stmt_iterator gsi = gsi_after_labels (exit_edge->dest);
>  	  gsi_insert_before (&gsi, assign, GSI_SAME_STMT);
>  	}
>      }
> @@ -1348,29 +1346,33 @@ vect_set_loop_condition_normal (class loop *loop, tree niters, tree step,
>     Assumption: the exit-condition of LOOP is the last stmt in the loop.  */
>  
>  void
> -vect_set_loop_condition (class loop *loop, loop_vec_info loop_vinfo,
> +vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo,
>  			 tree niters, tree step, tree final_iv,
>  			 bool niters_maybe_zero)
>  {
>    gcond *cond_stmt;
> -  gcond *orig_cond = get_loop_exit_condition (loop);
> +  gcond *orig_cond = get_loop_exit_condition (loop_e);
>    gimple_stmt_iterator loop_cond_gsi = gsi_for_stmt (orig_cond);
>  
>    if (loop_vinfo && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
>      {
>        if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_avx512)
> -	cond_stmt = vect_set_loop_condition_partial_vectors_avx512 (loop, loop_vinfo,
> +	cond_stmt = vect_set_loop_condition_partial_vectors_avx512 (loop, loop_e,
> +								    loop_vinfo,
>  								    niters, final_iv,
>  								    niters_maybe_zero,
>  								    loop_cond_gsi);
>        else
> -	cond_stmt = vect_set_loop_condition_partial_vectors (loop, loop_vinfo,
> +	cond_stmt = vect_set_loop_condition_partial_vectors (loop, loop_e,
> +							     loop_vinfo,
>  							     niters, final_iv,
>  							     niters_maybe_zero,
>  							     loop_cond_gsi);
>      }
>    else
> -    cond_stmt = vect_set_loop_condition_normal (loop, niters, step, final_iv,
> +    cond_stmt = vect_set_loop_condition_normal (loop_vinfo, loop_e, loop,
> +						niters,
> +						step, final_iv,
>  						niters_maybe_zero,
>  						loop_cond_gsi);
>  
> @@ -1439,7 +1441,6 @@ slpeel_duplicate_current_defs_from_edges (edge from, edge to)
>  		     get_current_def (PHI_ARG_DEF_FROM_EDGE (from_phi, from)));
>  }
>  
> -
>  /* Given LOOP this function generates a new copy of it and puts it
>     on E which is either the entry or exit of LOOP.  If SCALAR_LOOP is
>     non-NULL, assume LOOP and SCALAR_LOOP are equivalent and copy the
> @@ -1447,8 +1448,9 @@ slpeel_duplicate_current_defs_from_edges (edge from, edge to)
>     entry or exit of LOOP.  */
>  
>  class loop *
> -slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop,
> -					class loop *scalar_loop, edge e)
> +slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
> +					class loop *scalar_loop,
> +					edge scalar_exit, edge e, edge *new_e)
>  {
>    class loop *new_loop;
>    basic_block *new_bbs, *bbs, *pbbs;
> @@ -1458,13 +1460,16 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop,
>    edge exit, new_exit;
>    bool duplicate_outer_loop = false;
>  
> -  exit = single_exit (loop);
> +  exit = loop_exit;
>    at_exit = (e == exit);
>    if (!at_exit && e != loop_preheader_edge (loop))
>      return NULL;
>  
>    if (scalar_loop == NULL)
> -    scalar_loop = loop;
> +    {
> +      scalar_loop = loop;
> +      scalar_exit = loop_exit;
> +    }
>  
>    bbs = XNEWVEC (basic_block, scalar_loop->num_nodes + 1);
>    pbbs = bbs + 1;
> @@ -1490,13 +1495,15 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop,
>    bbs[0] = preheader;
>    new_bbs = XNEWVEC (basic_block, scalar_loop->num_nodes + 1);
>  
> -  exit = single_exit (scalar_loop);
>    copy_bbs (bbs, scalar_loop->num_nodes + 1, new_bbs,
> -	    &exit, 1, &new_exit, NULL,
> +	    &scalar_exit, 1, &new_exit, NULL,
>  	    at_exit ? loop->latch : e->src, true);
> -  exit = single_exit (loop);
> +  exit = loop_exit;
>    basic_block new_preheader = new_bbs[0];
>  
> +  if (new_e)
> +    *new_e = new_exit;
> +
>    /* Before installing PHI arguments make sure that the edges
>       into them match that of the scalar loop we analyzed.  This
>       makes sure the SLP tree matches up between the main vectorized
> @@ -1537,8 +1544,7 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop,
>  	 but LOOP will not.  slpeel_update_phi_nodes_for_guard{1,2} expects
>  	 the LOOP SSA_NAMEs (on the exit edge and edge from latch to
>  	 header) to have current_def set, so copy them over.  */
> -      slpeel_duplicate_current_defs_from_edges (single_exit (scalar_loop),
> -						exit);
> +      slpeel_duplicate_current_defs_from_edges (scalar_exit, exit);
>        slpeel_duplicate_current_defs_from_edges (EDGE_SUCC (scalar_loop->latch,
>  							   0),
>  						EDGE_SUCC (loop->latch, 0));
> @@ -1696,11 +1702,11 @@ slpeel_add_loop_guard (basic_block guard_bb, tree cond,
>   */
>  
>  bool
> -slpeel_can_duplicate_loop_p (const class loop *loop, const_edge e)
> +slpeel_can_duplicate_loop_p (const class loop *loop, const_edge exit_e,
> +			     const_edge e)
>  {
> -  edge exit_e = single_exit (loop);
>    edge entry_e = loop_preheader_edge (loop);
> -  gcond *orig_cond = get_loop_exit_condition (loop);
> +  gcond *orig_cond = get_loop_exit_condition (exit_e);
>    gimple_stmt_iterator loop_exit_gsi = gsi_last_bb (exit_e->src);
>    unsigned int num_bb = loop->inner? 5 : 2;
>  
> @@ -1709,7 +1715,7 @@ slpeel_can_duplicate_loop_p (const class loop *loop, const_edge e)
>    if (!loop_outer (loop)
>        || loop->num_nodes != num_bb
>        || !empty_block_p (loop->latch)
> -      || !single_exit (loop)
> +      || !exit_e
>        /* Verify that new loop exit condition can be trivially modified.  */
>        || (!orig_cond || orig_cond != gsi_stmt (loop_exit_gsi))
>        || (e != exit_e && e != entry_e))
> @@ -1722,7 +1728,7 @@ slpeel_can_duplicate_loop_p (const class loop *loop, const_edge e)
>    return ret;
>  }
>  
> -/* Function vect_get_loop_location.
> +/* Function find_loop_location.
>  
>     Extract the location of the loop in the source code.
>     If the loop is not well formed for vectorization, an estimated
> @@ -1739,11 +1745,19 @@ find_loop_location (class loop *loop)
>    if (!loop)
>      return dump_user_location_t ();
>  
> -  stmt = get_loop_exit_condition (loop);
> +  if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
> +    {
> +      /* We only care about the loop location, so use any exit with location
> +	 information.  */
> +      for (edge e : get_loop_exit_edges (loop))
> +	{
> +	  stmt = get_loop_exit_condition (e);
>  
> -  if (stmt
> -      && LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
> -    return stmt;
> +	  if (stmt
> +	      && LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
> +	    return stmt;
> +	}
> +    }
>  
>    /* If we got here the loop is probably not "well formed",
>       try to estimate the loop location */
> @@ -1962,7 +1976,8 @@ vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo,
>    gphi_iterator gsi, gsi1;
>    class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
>    basic_block update_bb = update_e->dest;
> -  basic_block exit_bb = single_exit (loop)->dest;
> +
> +  basic_block exit_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
>  
>    /* Make sure there exists a single-predecessor exit bb:  */
>    gcc_assert (single_pred_p (exit_bb));
> @@ -2529,10 +2544,9 @@ vect_gen_vector_loop_niters_mult_vf (loop_vec_info loop_vinfo,
>  {
>    /* We should be using a step_vector of VF if VF is variable.  */
>    int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ();
> -  class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
>    tree type = TREE_TYPE (niters_vector);
>    tree log_vf = build_int_cst (type, exact_log2 (vf));
> -  basic_block exit_bb = single_exit (loop)->dest;
> +  basic_block exit_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
>  
>    gcc_assert (niters_vector_mult_vf_ptr != NULL);
>    tree niters_vector_mult_vf = fold_build2 (LSHIFT_EXPR, type,
> @@ -2555,11 +2569,11 @@ vect_gen_vector_loop_niters_mult_vf (loop_vec_info loop_vinfo,
>     NULL.  */
>  
>  static tree
> -find_guard_arg (class loop *loop, class loop *epilog ATTRIBUTE_UNUSED,
> -		gphi *lcssa_phi)
> +find_guard_arg (class loop *loop ATTRIBUTE_UNUSED,
> +		class loop *epilog ATTRIBUTE_UNUSED,
> +		const_edge e, gphi *lcssa_phi)

please order 'e' after the corresponding loop argument

>  {
>    gphi_iterator gsi;
> -  edge e = single_exit (loop);
>  
>    gcc_assert (single_pred_p (e->dest));
>    for (gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi); gsi_next (&gsi))
> @@ -2620,7 +2634,8 @@ find_guard_arg (class loop *loop, class loop *epilog ATTRIBUTE_UNUSED,
>  
>  static void
>  slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
> -				   class loop *first, class loop *second,
> +				   class loop *first, edge first_loop_e,
> +				   class loop *second, edge second_loop_e,
>  				   bool create_lcssa_for_iv_phis)
>  {
>    gphi_iterator gsi_update, gsi_orig;
> @@ -2628,7 +2643,7 @@ slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
>  
>    edge first_latch_e = EDGE_SUCC (first->latch, 0);
>    edge second_preheader_e = loop_preheader_edge (second);
> -  basic_block between_bb = single_exit (first)->dest;
> +  basic_block between_bb = first_loop_e->dest;
>  
>    gcc_assert (between_bb == second_preheader_e->src);
>    gcc_assert (single_pred_p (between_bb) && single_succ_p (between_bb));
> @@ -2651,7 +2666,7 @@ slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
>  	{
>  	  tree new_res = copy_ssa_name (PHI_RESULT (orig_phi));
>  	  gphi *lcssa_phi = create_phi_node (new_res, between_bb);
> -	  add_phi_arg (lcssa_phi, arg, single_exit (first), UNKNOWN_LOCATION);
> +	  add_phi_arg (lcssa_phi, arg, first_loop_e, UNKNOWN_LOCATION);
>  	  arg = new_res;
>  	}
>  
> @@ -2664,7 +2679,7 @@ slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
>       for correct vectorization of live stmts.  */
>    if (loop == first)
>      {
> -      basic_block orig_exit = single_exit (second)->dest;
> +      basic_block orig_exit = second_loop_e->dest;
>        for (gsi_orig = gsi_start_phis (orig_exit);
>  	   !gsi_end_p (gsi_orig); gsi_next (&gsi_orig))
>  	{
> @@ -2673,13 +2688,14 @@ slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
>  	  if (TREE_CODE (orig_arg) != SSA_NAME || virtual_operand_p  (orig_arg))
>  	    continue;
>  
> +	  const_edge exit_e = LOOP_VINFO_IV_EXIT (loop_vinfo);
>  	  /* Already created in the above loop.   */
> -	  if (find_guard_arg (first, second, orig_phi))
> +	  if (find_guard_arg (first, second, exit_e, orig_phi))
>  	    continue;
>  
>  	  tree new_res = copy_ssa_name (orig_arg);
>  	  gphi *lcphi = create_phi_node (new_res, between_bb);
> -	  add_phi_arg (lcphi, orig_arg, single_exit (first), UNKNOWN_LOCATION);
> +	  add_phi_arg (lcphi, orig_arg, first_loop_e, UNKNOWN_LOCATION);
>  	}
>      }
>  }
> @@ -2847,7 +2863,8 @@ slpeel_update_phi_nodes_for_guard2 (class loop *loop, class loop *epilog,
>        if (!merge_arg)
>  	merge_arg = old_arg;
>  
> -      tree guard_arg = find_guard_arg (loop, epilog, update_phi);
> +      tree guard_arg
> +	= find_guard_arg (loop, epilog, single_exit (loop), update_phi);

missed adjustment?  you are introducing a single_exit call here ...

>        /* If the var is live after loop but not a reduction, we simply
>  	 use the old arg.  */
>        if (!guard_arg)
> @@ -3201,27 +3218,37 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
>      }
>  
>    if (vect_epilogues)
> -    /* Make sure to set the epilogue's epilogue scalar loop, such that we can
> -       use the original scalar loop as remaining epilogue if necessary.  */
> -    LOOP_VINFO_SCALAR_LOOP (epilogue_vinfo)
> -      = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
> +    {
> +      /* Make sure to set the epilogue's epilogue scalar loop, such that we can
> +	 use the original scalar loop as remaining epilogue if necessary.  */
> +      LOOP_VINFO_SCALAR_LOOP (epilogue_vinfo)
> +	= LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
> +      LOOP_VINFO_SCALAR_IV_EXIT (epilogue_vinfo)
> +	= LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo);
> +    }
>  
>    if (prolog_peeling)
>      {
>        e = loop_preheader_edge (loop);
> -      gcc_checking_assert (slpeel_can_duplicate_loop_p (loop, e));
> +      edge exit_e = LOOP_VINFO_IV_EXIT (loop_vinfo);
> +      gcc_checking_assert (slpeel_can_duplicate_loop_p (loop, exit_e, e));
>  
>        /* Peel prolog and put it on preheader edge of loop.  */
> -      prolog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, scalar_loop, e);
> +      edge scalar_e = LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo);
> +      edge prolog_e = NULL;
> +      prolog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, exit_e,
> +						       scalar_loop, scalar_e,
> +						       e, &prolog_e);
>        gcc_assert (prolog);
>        prolog->force_vectorize = false;
> -      slpeel_update_phi_nodes_for_loops (loop_vinfo, prolog, loop, true);
> +      slpeel_update_phi_nodes_for_loops (loop_vinfo, prolog, prolog_e, loop,
> +					 exit_e, true);
>        first_loop = prolog;
>        reset_original_copy_tables ();
>  
>        /* Update the number of iterations for prolog loop.  */
>        tree step_prolog = build_one_cst (TREE_TYPE (niters_prolog));
> -      vect_set_loop_condition (prolog, NULL, niters_prolog,
> +      vect_set_loop_condition (prolog, prolog_e, loop_vinfo, niters_prolog,
>  			       step_prolog, NULL_TREE, false);
>  
>        /* Skip the prolog loop.  */
> @@ -3275,8 +3302,8 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
>  
>    if (epilog_peeling)
>      {
> -      e = single_exit (loop);
> -      gcc_checking_assert (slpeel_can_duplicate_loop_p (loop, e));
> +      e = LOOP_VINFO_IV_EXIT (loop_vinfo);
> +      gcc_checking_assert (slpeel_can_duplicate_loop_p (loop, e, e));
>  
>        /* Peel epilog and put it on exit edge of loop.  If we are vectorizing
>  	 said epilog then we should use a copy of the main loop as a starting
> @@ -3285,12 +3312,18 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
>  	 If we are not vectorizing the epilog then we should use the scalar loop
>  	 as the transformations mentioned above make less or no sense when not
>  	 vectorizing.  */
> +      edge scalar_e = LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo);
>        epilog = vect_epilogues ? get_loop_copy (loop) : scalar_loop;
> -      epilog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, epilog, e);
> +      edge epilog_e = vect_epilogues ? e : scalar_e;
> +      edge new_epilog_e = NULL;
> +      epilog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, e, epilog,
> +						       epilog_e, e,
> +						       &new_epilog_e);
> +      LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo) = new_epilog_e;
>        gcc_assert (epilog);
> -
>        epilog->force_vectorize = false;
> -      slpeel_update_phi_nodes_for_loops (loop_vinfo, loop, epilog, false);
> +      slpeel_update_phi_nodes_for_loops (loop_vinfo, loop, e, epilog,
> +					 new_epilog_e, false);
>        bb_before_epilog = loop_preheader_edge (epilog)->src;
>  
>        /* Scalar version loop may be preferred.  In this case, add guard
> @@ -3374,16 +3407,16 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
>  	{
>  	  guard_cond = fold_build2 (EQ_EXPR, boolean_type_node,
>  				    niters, niters_vector_mult_vf);
> -	  guard_bb = single_exit (loop)->dest;
> -	  guard_to = split_edge (single_exit (epilog));
> +	  guard_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
> +	  edge epilog_e = LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo);
> +	  guard_to = split_edge (epilog_e);
>  	  guard_e = slpeel_add_loop_guard (guard_bb, guard_cond, guard_to,
>  					   skip_vector ? anchor : guard_bb,
>  					   prob_epilog.invert (),
>  					   irred_flag);
>  	  if (vect_epilogues)
>  	    epilogue_vinfo->skip_this_loop_edge = guard_e;
> -	  slpeel_update_phi_nodes_for_guard2 (loop, epilog, guard_e,
> -					      single_exit (epilog));
> +	  slpeel_update_phi_nodes_for_guard2 (loop, epilog, guard_e, epilog_e);
>  	  /* Only need to handle basic block before epilog loop if it's not
>  	     the guard_bb, which is the case when skip_vector is true.  */
>  	  if (guard_bb != bb_before_epilog)
> @@ -3416,6 +3449,8 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
>      {
>        epilog->aux = epilogue_vinfo;
>        LOOP_VINFO_LOOP (epilogue_vinfo) = epilog;
> +      LOOP_VINFO_IV_EXIT (epilogue_vinfo)
> +	= LOOP_VINFO_EPILOGUE_IV_EXIT (loop_vinfo);
>  
>        loop_constraint_clear (epilog, LOOP_C_INFINITE);
>  
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 23c6e8259e7b133cd7acc6bcf0bad26423e9993a..6e60d84143626a8e1d801bb580f4dcebc73c7ba7 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -855,10 +855,9 @@ vect_fixup_scalar_cycles_with_patterns (loop_vec_info loop_vinfo)
>  
>  
>  static gcond *
> -vect_get_loop_niters (class loop *loop, tree *assumptions,
> +vect_get_loop_niters (class loop *loop, edge exit, tree *assumptions,
>  		      tree *number_of_iterations, tree *number_of_iterationsm1)
>  {
> -  edge exit = single_exit (loop);
>    class tree_niter_desc niter_desc;
>    tree niter_assumptions, niter, may_be_zero;
>    gcond *cond = get_loop_exit_condition (loop);
> @@ -927,6 +926,20 @@ vect_get_loop_niters (class loop *loop, tree *assumptions,
>    return cond;
>  }
>  
> +/*  Determine the main loop exit for the vectorizer.  */
> +
> +edge

can't this be 'static'?

> +vec_init_loop_exit_info (class loop *loop)
> +{
> +  /* Before we begin we must first determine which exit is the main one and
> +     which are auxilary exits.  */
> +  auto_vec<edge> exits = get_loop_exit_edges (loop);
> +  if (exits.length () == 1)
> +    return exits[0];
> +  else
> +    return NULL;
> +}
> +
>  /* Function bb_in_loop_p
>  
>     Used as predicate for dfs order traversal of the loop bbs.  */
> @@ -987,7 +1000,10 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
>      has_mask_store (false),
>      scalar_loop_scaling (profile_probability::uninitialized ()),
>      scalar_loop (NULL),
> -    orig_loop_info (NULL)
> +    orig_loop_info (NULL),
> +    vec_loop_iv (NULL),
> +    vec_epilogue_loop_iv (NULL),
> +    scalar_loop_iv (NULL)
>  {
>    /* CHECKME: We want to visit all BBs before their successors (except for
>       latch blocks, for which this assertion wouldn't hold).  In the simple
> @@ -1646,6 +1662,18 @@ vect_analyze_loop_form (class loop *loop, vect_loop_form_info *info)
>  {
>    DUMP_VECT_SCOPE ("vect_analyze_loop_form");
>  
> +  edge exit_e = vec_init_loop_exit_info (loop);
> +  if (!exit_e)
> +    return opt_result::failure_at (vect_location,
> +				   "not vectorized:"
> +				   " could not determine main exit from"
> +				   " loop with multiple exits.\n");
> +  info->loop_exit = exit_e;
> +  if (dump_enabled_p ())
> +      dump_printf_loc (MSG_NOTE, vect_location,
> +		       "using as main loop exit: %d -> %d [AUX: %p]\n",
> +		       exit_e->src->index, exit_e->dest->index, exit_e->aux);
> +
>    /* Different restrictions apply when we are considering an inner-most loop,
>       vs. an outer (nested) loop.
>       (FORNOW. May want to relax some of these restrictions in the future).  */
> @@ -1767,7 +1795,7 @@ vect_analyze_loop_form (class loop *loop, vect_loop_form_info *info)
>  				   " abnormal loop exit edge.\n");
>  
>    info->loop_cond
> -    = vect_get_loop_niters (loop, &info->assumptions,
> +    = vect_get_loop_niters (loop, e, &info->assumptions,
>  			    &info->number_of_iterations,
>  			    &info->number_of_iterationsm1);
>    if (!info->loop_cond)
> @@ -1821,6 +1849,9 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
>  
>    stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (info->loop_cond);
>    STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type;
> +
> +  LOOP_VINFO_IV_EXIT (loop_vinfo) = info->loop_exit;
> +
>    if (info->inner_loop_cond)
>      {
>        stmt_vec_info inner_loop_cond_info
> @@ -3063,9 +3094,9 @@ start_over:
>        if (dump_enabled_p ())
>          dump_printf_loc (MSG_NOTE, vect_location, "epilog loop required\n");
>        if (!vect_can_advance_ivs_p (loop_vinfo)
> -	  || !slpeel_can_duplicate_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
> -					   single_exit (LOOP_VINFO_LOOP
> -							 (loop_vinfo))))
> +	  || !slpeel_can_duplicate_loop_p (loop,
> +					   LOOP_VINFO_IV_EXIT (loop_vinfo),
> +					   LOOP_VINFO_IV_EXIT (loop_vinfo)))
>          {
>  	  ok = opt_result::failure_at (vect_location,
>  				       "not vectorized: can't create required "
> @@ -6002,7 +6033,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
>           Store them in NEW_PHIS.  */
>    if (double_reduc)
>      loop = outer_loop;
> -  exit_bb = single_exit (loop)->dest;
> +  exit_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
>    exit_gsi = gsi_after_labels (exit_bb);
>    reduc_inputs.create (slp_node ? vec_num : ncopies);
>    for (unsigned i = 0; i < vec_num; i++)
> @@ -6018,7 +6049,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
>  	  phi = create_phi_node (new_def, exit_bb);
>  	  if (j)
>  	    def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]);
> -	  SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
> +	  SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, def);
>  	  new_def = gimple_convert (&stmts, vectype, new_def);
>  	  reduc_inputs.quick_push (new_def);
>  	}
> @@ -10416,12 +10447,12 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>  	   lhs' = new_tree;  */
>  
>        class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> -      basic_block exit_bb = single_exit (loop)->dest;
> +      basic_block exit_bb = LOOP_VINFO_IV_EXIT (loop_vinfo)->dest;
>        gcc_assert (single_pred_p (exit_bb));
>  
>        tree vec_lhs_phi = copy_ssa_name (vec_lhs);
>        gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
> -      SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, vec_lhs);
> +      SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, vec_lhs);
>  
>        gimple_seq stmts = NULL;
>        tree new_tree;
> @@ -10965,7 +10996,7 @@ vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
>     profile.  */
>  
>  static void
> -scale_profile_for_vect_loop (class loop *loop, unsigned vf, bool flat)
> +scale_profile_for_vect_loop (class loop *loop, edge exit_e, unsigned vf, bool flat)
>  {
>    /* For flat profiles do not scale down proportionally by VF and only
>       cap by known iteration count bounds.  */
> @@ -10980,7 +11011,6 @@ scale_profile_for_vect_loop (class loop *loop, unsigned vf, bool flat)
>        return;
>      }
>    /* Loop body executes VF fewer times and exit increases VF times.  */
> -  edge exit_e = single_exit (loop);
>    profile_count entry_count = loop_preheader_edge (loop)->count ();
>  
>    /* If we have unreliable loop profile avoid dropping entry
> @@ -11350,7 +11380,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
>  
>    /* Make sure there exists a single-predecessor exit bb.  Do this before 
>       versioning.   */
> -  edge e = single_exit (loop);
> +  edge e = LOOP_VINFO_IV_EXIT (loop_vinfo);
>    if (! single_pred_p (e->dest))
>      {
>        split_loop_exit_edge (e, true);
> @@ -11376,7 +11406,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
>       loop closed PHI nodes on the exit.  */
>    if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
>      {
> -      e = single_exit (LOOP_VINFO_SCALAR_LOOP (loop_vinfo));
> +      e = LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo);
>        if (! single_pred_p (e->dest))
>  	{
>  	  split_loop_exit_edge (e, true);
> @@ -11625,8 +11655,9 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
>       a zero NITERS becomes a nonzero NITERS_VECTOR.  */
>    if (integer_onep (step_vector))
>      niters_no_overflow = true;
> -  vect_set_loop_condition (loop, loop_vinfo, niters_vector, step_vector,
> -			   niters_vector_mult_vf, !niters_no_overflow);
> +  vect_set_loop_condition (loop, LOOP_VINFO_IV_EXIT (loop_vinfo), loop_vinfo,
> +			   niters_vector, step_vector, niters_vector_mult_vf,
> +			   !niters_no_overflow);
>  
>    unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
>  
> @@ -11699,7 +11730,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
>  			  assumed_vf) - 1
>  	 : wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
>  			   assumed_vf) - 1);
> -  scale_profile_for_vect_loop (loop, assumed_vf, flat);
> +  scale_profile_for_vect_loop (loop, LOOP_VINFO_IV_EXIT (loop_vinfo),
> +			       assumed_vf, flat);
>  
>    if (dump_enabled_p ())
>      {
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index f1d0cd79961abb095bc79d3b59a81930f0337e59..afa7a8e30891c782a0e5e3740ecc4377f5a31e54 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -919,10 +919,24 @@ public:
>       analysis.  */
>    vec<_loop_vec_info *> epilogue_vinfos;
>  
> +  /* The controlling loop IV for the current loop when vectorizing.  This IV
> +     controls the natural exits of the loop.  */
> +  edge vec_loop_iv;
> +
> +  /* The controlling loop IV for the epilogue loop when vectorizing.  This IV
> +     controls the natural exits of the loop.  */
> +  edge vec_epilogue_loop_iv;
> +
> +  /* The controlling loop IV for the scalar loop being vectorized.  This IV
> +     controls the natural exits of the loop.  */
> +  edge scalar_loop_iv;

all of the above sound as if they were IVs, the access macros have
_EXIT at the end, can you make the above as well?

Otherwise looks good to me.

Feel free to push approved patches of the series, no need to wait
until everything is approved.

Thanks,
Richard.

>  } *loop_vec_info;
>  
>  /* Access Functions.  */
>  #define LOOP_VINFO_LOOP(L)                 (L)->loop
> +#define LOOP_VINFO_IV_EXIT(L)              (L)->vec_loop_iv
> +#define LOOP_VINFO_EPILOGUE_IV_EXIT(L)     (L)->vec_epilogue_loop_iv
> +#define LOOP_VINFO_SCALAR_IV_EXIT(L)       (L)->scalar_loop_iv
>  #define LOOP_VINFO_BBS(L)                  (L)->bbs
>  #define LOOP_VINFO_NITERSM1(L)             (L)->num_itersm1
>  #define LOOP_VINFO_NITERS(L)               (L)->num_iters
> @@ -2155,11 +2169,13 @@ class auto_purge_vect_location
>  
>  /* Simple loop peeling and versioning utilities for vectorizer's purposes -
>     in tree-vect-loop-manip.cc.  */
> -extern void vect_set_loop_condition (class loop *, loop_vec_info,
> +extern void vect_set_loop_condition (class loop *, edge, loop_vec_info,
>  				     tree, tree, tree, bool);
> -extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge);
> -class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *,
> -						     class loop *, edge);
> +extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge,
> +					 const_edge);
> +class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *, edge,
> +						    class loop *, edge,
> +						    edge, edge *);
>  class loop *vect_loop_versioning (loop_vec_info, gimple *);
>  extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
>  				    tree *, tree *, tree *, int, bool, bool,
> @@ -2169,6 +2185,7 @@ extern void vect_prepare_for_masked_peels (loop_vec_info);
>  extern dump_user_location_t find_loop_location (class loop *);
>  extern bool vect_can_advance_ivs_p (loop_vec_info);
>  extern void vect_update_inits_of_drs (loop_vec_info, tree, tree_code);
> +extern edge vec_init_loop_exit_info (class loop *);
>  
>  /* In tree-vect-stmts.cc.  */
>  extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
> @@ -2358,6 +2375,7 @@ struct vect_loop_form_info
>    tree assumptions;
>    gcond *loop_cond;
>    gcond *inner_loop_cond;
> +  edge loop_exit;
>  };
>  extern opt_result vect_analyze_loop_form (class loop *, vect_loop_form_info *);
>  extern loop_vec_info vect_create_loop_vinfo (class loop *, vec_info_shared *,
> diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
> index a048e9d89178a37455bd7b83ab0f2a238a4ce69e..d97e2b54c25ac60378935392aa7b73476efed74b 100644
> --- a/gcc/tree-vectorizer.cc
> +++ b/gcc/tree-vectorizer.cc
> @@ -943,6 +943,8 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call,
>    class loop *scalar_loop = get_loop (fun, tree_to_shwi (arg));
>  
>    LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
> +  LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo)
> +    = vec_init_loop_exit_info (scalar_loop);
>    gcc_checking_assert (vect_loop_vectorized_call (scalar_loop)
>  		       == loop_vectorized_call);
>    /* If we are going to vectorize outer loop, prevent vectorization
> 
> 
> 
> 
> 

-- 
Richard Biener <rguenther@suse.de>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Ivo Totev, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)