* [PATCH 3/n] add vec_info * parameters where needed
@ 2020-03-16 12:18 Richard Biener
0 siblings, 0 replies; only message in thread
From: Richard Biener @ 2020-03-16 12:18 UTC (permalink / raw)
To: gcc-patches
This is a part of a WIP series doing vectorizer rework. To be committed
when stage1 opens.
This avoids adding ->vinfo to SLP nodes and taking it from there
when stmt_vinfo is NULL. Instead it does what we should have done
in the first place - pass down vinfo. I refrained from coming up
with an ad-hoc class structure or simply making all the functions
member functions. Some may become so in the future though.
* tree-vectorizer.h (_stmt_vec_info::vinfo): Remove.
(STMT_VINFO_LOOP_VINFO): Likewise.
(STMT_VINFO_BB_VINFO): Likewise.
* tree-vect-data-refs.c: Adjust for the above, adding vec_info *
parameters and adjusting calls.
* tree-vect-loop-manip.c: Likewise.
* tree-vect-loop.c: Likewise.
* tree-vect-patterns.c: Likewise.
* tree-vect-slp.c: Likewise.
* tree-vect-stmts.c: Likewise.
* tree-vectorizer.c: Likewise.
* target.def (add_stmt_cost): Add vec_info * parameter.
* target.h (stmt_in_inner_loop_p): Likewise.
* targhooks.c (default_add_stmt_cost): Adjust.
* config/aarch64/aarch64.c (aarch64_add_stmt_cost): Adjust.
* config/arm/arm.c (arm_add_stmt_cost): Likewise.
* config/i386/i386.c (ix86_add_stmt_cost): Likewise.
* config/rs6000/rs6000.c (rs6000_add_stmt_cost): Likewise.
---
gcc/config/aarch64/aarch64.c | 6 +-
gcc/config/arm/arm.c | 6 +-
gcc/config/i386/i386.c | 6 +-
gcc/config/rs6000/rs6000.c | 6 +-
gcc/target.def | 2 +-
gcc/target.h | 2 +-
gcc/targhooks.c | 6 +-
gcc/tree-vect-data-refs.c | 197 ++++-----
gcc/tree-vect-loop-manip.c | 6 +-
gcc/tree-vect-loop.c | 221 +++++-----
gcc/tree-vect-patterns.c | 474 +++++++++++----------
gcc/tree-vect-slp.c | 106 ++---
gcc/tree-vect-stmts.c | 983 +++++++++++++++++++++++--------------------
gcc/tree-vectorizer.c | 1 -
gcc/tree-vectorizer.h | 123 +++---
15 files changed, 1148 insertions(+), 997 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index b0cbb6e2d55..26d976fa8ed 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -13122,7 +13122,8 @@ aarch64_sve_adjust_stmt_cost (vect_cost_for_stmt kind, stmt_vec_info stmt_info,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
-aarch64_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@@ -13141,7 +13142,8 @@ aarch64_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ if (where == vect_body && stmt_info
+ && stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME */
retval = (unsigned) (count * stmt_cost);
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index 9cc7bc0e562..4f1c0a48371 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -11994,7 +11994,8 @@ arm_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
-arm_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+arm_add_stmt_cost (class vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@@ -12009,7 +12010,8 @@ arm_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ if (where == vect_body && stmt_info
+ && stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index d1910b42b1b..6ec0a5111b1 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -21874,7 +21874,8 @@ ix86_init_cost (class loop *)
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
-ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@@ -22035,7 +22036,8 @@ ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ if (where == vect_body && stmt_info
+ && stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index 5798f924472..f7d5b41e8c3 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -5023,7 +5023,8 @@ adjust_vectorization_cost (enum vect_cost_for_stmt kind,
/* Implement targetm.vectorize.add_stmt_cost. */
static unsigned
-rs6000_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@@ -5039,7 +5040,8 @@ rs6000_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ if (where == vect_body && stmt_info
+ && stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);
diff --git a/gcc/target.def b/gcc/target.def
index b5e82ff826e..6d5f4b0a108 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -2030,7 +2030,7 @@ DEFHOOK
"return value should be viewed as a tentative cost that may later be "
"revised.",
unsigned,
- (void *data, int count, enum vect_cost_for_stmt kind,
+ (class vec_info *, void *data, int count, enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where),
default_add_stmt_cost)
diff --git a/gcc/target.h b/gcc/target.h
index 2f47c577d00..440cd25f297 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -157,7 +157,7 @@ class predefined_function_abi;
/* These are defined in tree-vect-stmts.c. */
extern tree stmt_vectype (class _stmt_vec_info *);
-extern bool stmt_in_inner_loop_p (class _stmt_vec_info *);
+extern bool stmt_in_inner_loop_p (class vec_info *, class _stmt_vec_info *);
/* Assembler instructions for creating various kinds of integer object. */
diff --git a/gcc/targhooks.c b/gcc/targhooks.c
index 7cb04f30bdb..4caab8cfbfa 100644
--- a/gcc/targhooks.c
+++ b/gcc/targhooks.c
@@ -1348,7 +1348,8 @@ default_init_cost (class loop *loop_info ATTRIBUTE_UNUSED)
it into the cost specified by WHERE, and returns the cost added. */
unsigned
-default_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+default_add_stmt_cost (class vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
class _stmt_vec_info *stmt_info, int misalign,
enum vect_cost_model_location where)
{
@@ -1361,7 +1362,8 @@ default_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
/* Statements in an inner loop relative to the loop being
vectorized are weighted more heavily. The value here is
arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ if (where == vect_body && stmt_info
+ && stmt_in_inner_loop_p (vinfo, stmt_info))
count *= 50; /* FIXME. */
retval = (unsigned) (count * stmt_cost);
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index db92c818287..d41ba49fabf 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -695,7 +695,7 @@ vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
disambiguating the loads. */
static bool
-vect_slp_analyze_node_dependences (slp_tree node,
+vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node,
vec<stmt_vec_info> stores,
stmt_vec_info last_store_info)
{
@@ -703,7 +703,6 @@ vect_slp_analyze_node_dependences (slp_tree node,
in NODE verifying we can sink them up to the last stmt in the
group. */
stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
- vec_info *vinfo = last_access_info->vinfo;
for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
{
stmt_vec_info access_info = SLP_TREE_SCALAR_STMTS (node)[k];
@@ -781,7 +780,7 @@ vect_slp_analyze_node_dependences (slp_tree node,
the maximum vectorization factor the data dependences allow. */
bool
-vect_slp_analyze_instance_dependence (slp_instance instance)
+vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
@@ -794,7 +793,7 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
stmt_vec_info last_store_info = NULL;
if (store)
{
- if (! vect_slp_analyze_node_dependences (store, vNULL, NULL))
+ if (! vect_slp_analyze_node_dependences (vinfo, store, vNULL, NULL))
return false;
/* Mark stores in this instance and remember the last one. */
@@ -810,7 +809,7 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
slp_tree load;
unsigned int i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, load)
- if (! vect_slp_analyze_node_dependences (load,
+ if (! vect_slp_analyze_node_dependences (vinfo, load,
store
? SLP_TREE_SCALAR_STMTS (store)
: vNULL, last_store_info))
@@ -831,10 +830,9 @@ vect_slp_analyze_instance_dependence (slp_instance instance)
in STMT_INFO. */
static void
-vect_record_base_alignment (stmt_vec_info stmt_info,
+vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
innermost_loop_behavior *drb)
{
- vec_info *vinfo = stmt_info->vinfo;
bool existed;
innermost_loop_behavior *&entry
= vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
@@ -877,13 +875,13 @@ vect_record_base_alignments (vec_info *vinfo)
&& STMT_VINFO_VECTORIZABLE (stmt_info)
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
- vect_record_base_alignment (stmt_info, &DR_INNERMOST (dr));
+ vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
/* If DR is nested in the loop that is being vectorized, we can also
record the alignment of the base wrt the outer loop. */
if (loop && nested_in_vect_loop_p (loop, stmt_info))
vect_record_base_alignment
- (stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
+ (vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
}
}
}
@@ -908,11 +906,11 @@ vect_calculate_target_alignment (dr_vec_info *dr_info)
only for trivial cases. TODO. */
static void
-vect_compute_data_ref_alignment (dr_vec_info *dr_info)
+vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_info = dr_info->stmt;
- vec_base_alignments *base_alignments = &stmt_info->vinfo->base_alignments;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ vec_base_alignments *base_alignments = &vinfo->base_alignments;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
tree ref = DR_REF (dr_info->dr);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
@@ -930,7 +928,7 @@ vect_compute_data_ref_alignment (dr_vec_info *dr_info)
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
return;
- innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
+ innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
bool step_preserves_misalignment_p;
poly_uint64 vector_alignment
@@ -1137,10 +1135,10 @@ vect_update_misalignment_for_peel (dr_vec_info *dr_info,
Return TRUE if DR_INFO can be handled with respect to alignment. */
static opt_result
-verify_data_ref_alignment (dr_vec_info *dr_info)
+verify_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
enum dr_alignment_support supportable_dr_alignment
- = vect_supportable_dr_alignment (dr_info, false);
+ = vect_supportable_dr_alignment (vinfo, dr_info, false);
if (!supportable_dr_alignment)
return opt_result::failure_at
(dr_info->stmt->stmt,
@@ -1187,7 +1185,7 @@ vect_verify_datarefs_alignment (loop_vec_info vinfo)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- opt_result res = verify_data_ref_alignment (dr_info);
+ opt_result res = verify_data_ref_alignment (vinfo, dr_info);
if (!res)
return res;
}
@@ -1278,14 +1276,14 @@ vector_alignment_reachable_p (dr_vec_info *dr_info)
/* Calculate the cost of the memory access represented by DR_INFO. */
static void
-vect_get_data_access_cost (dr_vec_info *dr_info,
+vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
unsigned int *inside_cost,
unsigned int *outside_cost,
stmt_vector_for_cost *body_cost_vec,
stmt_vector_for_cost *prologue_cost_vec)
{
stmt_vec_info stmt_info = dr_info->stmt;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
int ncopies;
if (PURE_SLP_STMT (stmt_info))
@@ -1294,10 +1292,10 @@ vect_get_data_access_cost (dr_vec_info *dr_info,
ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
if (DR_IS_READ (dr_info->dr))
- vect_get_load_cost (stmt_info, ncopies, true, inside_cost, outside_cost,
- prologue_cost_vec, body_cost_vec, false);
+ vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost,
+ outside_cost, prologue_cost_vec, body_cost_vec, false);
else
- vect_get_store_cost (stmt_info, ncopies, inside_cost, body_cost_vec);
+ vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -1315,6 +1313,7 @@ typedef struct _vect_peel_info
typedef struct _vect_peel_extended_info
{
+ vec_info *vinfo;
struct _vect_peel_info peel_info;
unsigned int inside_cost;
unsigned int outside_cost;
@@ -1352,7 +1351,7 @@ vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
struct _vect_peel_info elem, *slot;
_vect_peel_info **new_slot;
bool supportable_dr_alignment
- = vect_supportable_dr_alignment (dr_info, true);
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
elem.npeel = npeel;
slot = peeling_htab->find (&elem);
@@ -1440,7 +1439,7 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
SET_DR_MISALIGNMENT (dr_info, 0);
else
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
- vect_get_data_access_cost (dr_info, inside_cost, outside_cost,
+ vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost,
body_cost_vec, prologue_cost_vec);
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
}
@@ -1456,8 +1455,7 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
vect_peel_info elem = *slot;
int dummy;
unsigned int inside_cost = 0, outside_cost = 0;
- stmt_vec_info stmt_info = elem->dr_info->stmt;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
stmt_vector_for_cost prologue_cost_vec, body_cost_vec,
epilogue_cost_vec;
@@ -1509,6 +1507,7 @@ vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_hta
struct _vect_peel_extended_info res;
res.peel_info.dr_info = NULL;
+ res.vinfo = loop_vinfo;
if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
{
@@ -1565,7 +1564,7 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
save_misalignment = DR_MISALIGNMENT (dr_info);
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
supportable_dr_alignment
- = vect_supportable_dr_alignment (dr_info, false);
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
if (!supportable_dr_alignment)
@@ -1753,7 +1752,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- supportable_dr_alignment = vect_supportable_dr_alignment (dr_info, true);
+ supportable_dr_alignment
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
do_peeling = vector_alignment_reachable_p (dr_info);
if (do_peeling)
{
@@ -2217,7 +2217,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
}
supportable_dr_alignment
- = vect_supportable_dr_alignment (dr_info, false);
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
if (!supportable_dr_alignment)
{
@@ -2415,7 +2415,7 @@ vect_analyze_data_refs_alignment (loop_vec_info vinfo)
{
dr_vec_info *dr_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
- vect_compute_data_ref_alignment (dr_info);
+ vect_compute_data_ref_alignment (vinfo, dr_info);
}
return opt_result::success ();
@@ -2425,7 +2425,7 @@ vect_analyze_data_refs_alignment (loop_vec_info vinfo)
/* Analyze alignment of DRs of stmts in NODE. */
static bool
-vect_slp_analyze_and_verify_node_alignment (slp_tree node)
+vect_slp_analyze_and_verify_node_alignment (vec_info *vinfo, slp_tree node)
{
/* We vectorize from the first scalar stmt in the node unless
the node is permuted in which case we start from the first
@@ -2436,12 +2436,12 @@ vect_slp_analyze_and_verify_node_alignment (slp_tree node)
first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
- vect_compute_data_ref_alignment (dr_info);
+ vect_compute_data_ref_alignment (vinfo, dr_info);
/* For creating the data-ref pointer we need alignment of the
first element anyway. */
if (dr_info != first_dr_info)
- vect_compute_data_ref_alignment (first_dr_info);
- if (! verify_data_ref_alignment (dr_info))
+ vect_compute_data_ref_alignment (vinfo, first_dr_info);
+ if (! verify_data_ref_alignment (vinfo, dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2459,20 +2459,21 @@ vect_slp_analyze_and_verify_node_alignment (slp_tree node)
Return FALSE if a data reference is found that cannot be vectorized. */
bool
-vect_slp_analyze_and_verify_instance_alignment (slp_instance instance)
+vect_slp_analyze_and_verify_instance_alignment (vec_info *vinfo,
+ slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_and_verify_instance_alignment");
slp_tree node;
unsigned i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
- if (! vect_slp_analyze_and_verify_node_alignment (node))
+ if (! vect_slp_analyze_and_verify_node_alignment (vinfo, node))
return false;
node = SLP_INSTANCE_TREE (instance);
if (STMT_VINFO_DATA_REF (SLP_TREE_SCALAR_STMTS (node)[0])
&& ! vect_slp_analyze_and_verify_node_alignment
- (SLP_INSTANCE_TREE (instance)))
+ (vinfo, SLP_INSTANCE_TREE (instance)))
return false;
return true;
@@ -2486,15 +2487,15 @@ vect_slp_analyze_and_verify_instance_alignment (slp_instance instance)
Worker for vect_analyze_group_access. */
static bool
-vect_analyze_group_access_1 (dr_vec_info *dr_info)
+vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
{
data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
stmt_vec_info stmt_info = dr_info->stmt;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
HOST_WIDE_INT dr_step = -1;
HOST_WIDE_INT groupsize, last_accessed_element = 1;
bool slp_impossible = false;
@@ -2696,9 +2697,9 @@ vect_analyze_group_access_1 (dr_vec_info *dr_info)
Collect groups of strided stores for further use in SLP analysis. */
static bool
-vect_analyze_group_access (dr_vec_info *dr_info)
+vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
{
- if (!vect_analyze_group_access_1 (dr_info))
+ if (!vect_analyze_group_access_1 (vinfo, dr_info))
{
/* Dissolve the group if present. */
stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
@@ -2719,13 +2720,13 @@ vect_analyze_group_access (dr_vec_info *dr_info)
analyze groups of accesses. */
static bool
-vect_analyze_data_ref_access (dr_vec_info *dr_info)
+vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
{
data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
stmt_vec_info stmt_info = dr_info->stmt;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
@@ -2804,10 +2805,10 @@ vect_analyze_data_ref_access (dr_vec_info *dr_info)
if (TREE_CODE (step) != INTEGER_CST)
return (STMT_VINFO_STRIDED_P (stmt_info)
&& (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
- || vect_analyze_group_access (dr_info)));
+ || vect_analyze_group_access (vinfo, dr_info)));
/* Not consecutive access - check if it's a part of interleaving group. */
- return vect_analyze_group_access (dr_info);
+ return vect_analyze_group_access (vinfo, dr_info);
}
/* Compare two data-references DRA and DRB to group them into chunks
@@ -3153,7 +3154,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo)
{
dr_vec_info *dr_info = vinfo->lookup_dr (dr);
if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
- && !vect_analyze_data_ref_access (dr_info))
+ && !vect_analyze_data_ref_access (vinfo, dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -3204,7 +3205,7 @@ vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
gives the worst-case number of bytes covered by the segment. */
static unsigned HOST_WIDE_INT
-vect_vfa_access_size (dr_vec_info *dr_info)
+vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_vinfo = dr_info->stmt;
tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
@@ -3216,7 +3217,7 @@ vect_vfa_access_size (dr_vec_info *dr_info)
access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
}
if (STMT_VINFO_VEC_STMT (stmt_vinfo)
- && (vect_supportable_dr_alignment (dr_info, false)
+ && (vect_supportable_dr_alignment (vinfo, dr_info, false)
== dr_explicit_realign_optimized))
{
/* We might access a full vector's worth. */
@@ -3592,8 +3593,8 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
}
- access_size_a = vect_vfa_access_size (dr_info_a);
- access_size_b = vect_vfa_access_size (dr_info_b);
+ access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
+ access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
align_a = vect_vfa_align (dr_info_a);
align_b = vect_vfa_align (dr_info_b);
@@ -4580,7 +4581,7 @@ vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
FORNOW: We are only handling array accesses with step 1. */
tree
-vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
+vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_seq *new_stmt_list,
tree offset,
tree byte_offset)
@@ -4593,11 +4594,11 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
gimple_seq seq = NULL;
tree vect_ptr_type;
tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
- tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true));
+ tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
@@ -4714,14 +4715,14 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
3. Return the pointer. */
tree
-vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
- class loop *at_loop, tree offset,
+vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree aggr_type, class loop *at_loop, tree offset,
tree *initial_address, gimple_stmt_iterator *gsi,
gimple **ptr_incr, bool only_init,
tree byte_offset, tree iv_step)
{
const char *base_name;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
bool nested_in_vect_loop = false;
class loop *containing_loop = NULL;
@@ -4739,7 +4740,7 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
bool insert_after;
tree indx_before_incr, indx_after_incr;
gimple *incr;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
gcc_assert (iv_step != NULL_TREE
|| TREE_CODE (aggr_type) == ARRAY_TYPE
@@ -4848,7 +4849,8 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
/* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */
- new_temp = vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
+ new_temp = vect_create_addr_base_for_vector_ref (vinfo,
+ stmt_info, &new_stmt_list,
offset, byte_offset);
if (new_stmt_list)
{
@@ -4875,7 +4877,7 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
{
/* Accesses to invariant addresses should be handled specially
by the caller. */
- tree step = vect_dr_behavior (dr_info)->step;
+ tree step = vect_dr_behavior (vinfo, dr_info)->step;
gcc_assert (!integer_zerop (step));
if (iv_step == NULL_TREE)
@@ -4977,7 +4979,8 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
*/
tree
-bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
+bump_vector_ptr (vec_info *vinfo,
+ tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info, tree bump)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
@@ -4997,7 +5000,7 @@ bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
dataref_ptr, update);
- vect_finish_stmt_generation (stmt_info, incr_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
@@ -5277,7 +5280,7 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
I4: 6 14 22 30 7 15 23 31. */
void
-vect_permute_store_chain (vec<tree> dr_chain,
+vect_permute_store_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@@ -5344,7 +5347,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect1 = data_ref;
vect2 = dr_chain[2];
@@ -5355,7 +5358,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j] = data_ref;
}
}
@@ -5394,7 +5397,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
vect2, perm_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j] = high;
/* Create interleaving stmt:
@@ -5404,7 +5407,7 @@ vect_permute_store_chain (vec<tree> dr_chain,
low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
vect2, perm_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j+1] = low;
}
memcpy (dr_chain.address (), result_chain->address (),
@@ -5465,14 +5468,14 @@ vect_permute_store_chain (vec<tree> dr_chain,
Return value - the result of the loop-header phi node. */
tree
-vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- tree *realignment_token,
+vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, tree *realignment_token,
enum dr_alignment_support alignment_support_scheme,
tree init_addr,
class loop **at_loop)
{
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
struct data_reference *dr = dr_info->dr;
class loop *loop = NULL;
@@ -5579,7 +5582,7 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (!compute_in_loop);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- ptr = vect_create_data_ref_ptr (stmt_info, vectype,
+ ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
loop_for_initial_load, NULL_TREE,
&init_addr, NULL, &inc, true);
if (TREE_CODE (ptr) == SSA_NAME)
@@ -5626,7 +5629,8 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (!init_addr)
{
/* Generate the INIT_ADDR computation outside LOOP. */
- init_addr = vect_create_addr_base_for_vector_ref (stmt_info, &stmts,
+ init_addr = vect_create_addr_base_for_vector_ref (vinfo,
+ stmt_info, &stmts,
NULL_TREE);
if (loop)
{
@@ -5900,7 +5904,7 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
4th vec (E4): 3 7 11 15 19 23 27 31. */
static void
-vect_permute_load_chain (vec<tree> dr_chain,
+vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@@ -5953,7 +5957,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
/* Create interleaving stmt (high part of):
high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
@@ -5963,7 +5967,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[k] = data_ref;
}
}
@@ -5998,7 +6002,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_even);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
/* data_ref = permute_odd (first_data_ref, second_data_ref); */
@@ -6006,7 +6010,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_odd);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2+length/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
@@ -6103,7 +6107,7 @@ vect_permute_load_chain (vec<tree> dr_chain,
*/
static bool
-vect_shift_permute_load_chain (vec<tree> dr_chain,
+vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
@@ -6116,7 +6120,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
unsigned int i;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
unsigned HOST_WIDE_INT nelt, vf;
if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
@@ -6205,26 +6209,26 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, first_vect,
perm2_mask1);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[0] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
second_vect, second_vect,
perm2_mask2);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[1] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], shift1_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2 + length/2] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], select_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
@@ -6321,7 +6325,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
dr_chain[k], dr_chain[k],
perm3_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
@@ -6331,7 +6335,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[k % 3], vect[(k + 1) % 3],
shift1_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect_shift[k] = data_ref;
}
@@ -6342,7 +6346,7 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
vect_shift[(4 - k) % 3],
vect_shift[(3 - k) % 3],
shift2_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
@@ -6351,13 +6355,13 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
vect[0], shift3_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[nelt % 3] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
vect[1], shift4_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[0] = data_ref;
return true;
}
@@ -6372,7 +6376,8 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
*/
void
-vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
+vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
+ vec<tree> dr_chain,
int size, gimple_stmt_iterator *gsi)
{
machine_mode mode;
@@ -6389,10 +6394,11 @@ vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
|| pow2p_hwi (size)
- || !vect_shift_permute_load_chain (dr_chain, size, stmt_info,
+ || !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
gsi, &result_chain))
- vect_permute_load_chain (dr_chain, size, stmt_info, gsi, &result_chain);
- vect_record_grouped_load_vectors (stmt_info, result_chain);
+ vect_permute_load_chain (vinfo, dr_chain,
+ size, stmt_info, gsi, &result_chain);
+ vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
result_chain.release ();
}
@@ -6401,10 +6407,9 @@ vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
for each vector to the associated scalar statement. */
void
-vect_record_grouped_load_vectors (stmt_vec_info stmt_info,
+vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
vec<tree> result_chain)
{
- vec_info *vinfo = stmt_info->vinfo;
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
unsigned int i, gap_count;
tree tmp_data_ref;
@@ -6493,14 +6498,14 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
alignment. */
enum dr_alignment_support
-vect_supportable_dr_alignment (dr_vec_info *dr_info,
+vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
bool check_aligned_accesses)
{
data_reference *dr = dr_info->dr;
stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
machine_mode mode = TYPE_MODE (vectype);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *vect_loop = NULL;
bool nested_in_vect_loop = false;
diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c
index 0ee1ab45c07..8c5e696b995 100644
--- a/gcc/tree-vect-loop-manip.c
+++ b/gcc/tree-vect-loop-manip.c
@@ -1568,7 +1568,8 @@ get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
tree offset = (negative
? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1)
: size_zero_node);
- tree start_addr = vect_create_addr_base_for_vector_ref (stmt_info, seq,
+ tree start_addr = vect_create_addr_base_for_vector_ref (loop_vinfo,
+ stmt_info, seq,
offset);
tree type = unsigned_type_for (TREE_TYPE (start_addr));
if (target_align.is_constant (&target_align_c))
@@ -3057,7 +3058,8 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo,
/* create: addr_tmp = (int)(address_of_first_vector) */
addr_base =
- vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
+ vect_create_addr_base_for_vector_ref (loop_vinfo,
+ stmt_info, &new_stmt_list,
offset);
if (new_stmt_list != NULL)
gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list);
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index fb9e98f0ced..67f1513e495 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -161,7 +161,7 @@ static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
may already be set for general statements (not just data refs). */
static opt_result
-vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
+vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
bool vectype_maybe_set_p,
poly_uint64 *vf)
{
@@ -177,7 +177,8 @@ vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
}
tree stmt_vectype, nunits_vectype;
- opt_result res = vect_get_vector_types_for_stmt (stmt_info, &stmt_vectype,
+ opt_result res = vect_get_vector_types_for_stmt (vinfo, stmt_info,
+ &stmt_vectype,
&nunits_vectype);
if (!res)
return res;
@@ -207,13 +208,13 @@ vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
or false if something prevented vectorization. */
static opt_result
-vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
+vect_determine_vf_for_stmt (vec_info *vinfo,
+ stmt_vec_info stmt_info, poly_uint64 *vf)
{
- vec_info *vinfo = stmt_info->vinfo;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
stmt_info->stmt);
- opt_result res = vect_determine_vf_for_stmt_1 (stmt_info, false, vf);
+ opt_result res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, false, vf);
if (!res)
return res;
@@ -232,7 +233,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern def stmt: %G",
def_stmt_info->stmt);
- res = vect_determine_vf_for_stmt_1 (def_stmt_info, true, vf);
+ res = vect_determine_vf_for_stmt_1 (vinfo, def_stmt_info, true, vf);
if (!res)
return res;
}
@@ -241,7 +242,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern statement: %G",
stmt_info->stmt);
- res = vect_determine_vf_for_stmt_1 (stmt_info, true, vf);
+ res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, true, vf);
if (!res)
return res;
}
@@ -343,7 +344,8 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
{
stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
opt_result res
- = vect_determine_vf_for_stmt (stmt_info, &vectorization_factor);
+ = vect_determine_vf_for_stmt (loop_vinfo,
+ stmt_info, &vectorization_factor);
if (!res)
return res;
}
@@ -440,9 +442,8 @@ vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
this function would then return true for x_2. */
static bool
-vect_inner_phi_in_double_reduction_p (stmt_vec_info stmt_info, gphi *phi)
+vect_inner_phi_in_double_reduction_p (loop_vec_info loop_vinfo, gphi *phi)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
use_operand_p use_p;
ssa_op_iter op_iter;
FOR_EACH_PHI_ARG (use_p, phi, op_iter, SSA_OP_USE)
@@ -505,7 +506,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
}
if (!access_fn
- || vect_inner_phi_in_double_reduction_p (stmt_vinfo, phi)
+ || vect_inner_phi_in_double_reduction_p (loop_vinfo, phi)
|| !vect_is_simple_iv_evolution (loop->num, access_fn, &init, &step)
|| (LOOP_VINFO_LOOP (loop_vinfo) != loop
&& TREE_CODE (step) != INTEGER_CST))
@@ -1122,7 +1123,7 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
- (void) add_stmt_cost (target_cost_data, si->count,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
si->kind, si->stmt_info, si->misalign,
vect_body);
unsigned dummy, body_cost = 0;
@@ -1529,7 +1530,8 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
|| (STMT_VINFO_DEF_TYPE (stmt_info)
== vect_double_reduction_def))
- && !vectorizable_lc_phi (stmt_info, NULL, NULL))
+ && !vectorizable_lc_phi (loop_vinfo,
+ stmt_info, NULL, NULL))
return opt_result::failure_at (phi, "unsupported phi\n");
}
@@ -1551,21 +1553,24 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
need_to_vectorize = true;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
&& ! PURE_SLP_STMT (stmt_info))
- ok = vectorizable_induction (stmt_info, NULL, NULL, NULL,
+ ok = vectorizable_induction (loop_vinfo,
+ stmt_info, NULL, NULL, NULL,
&cost_vec);
else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
|| (STMT_VINFO_DEF_TYPE (stmt_info)
== vect_double_reduction_def)
|| STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
&& ! PURE_SLP_STMT (stmt_info))
- ok = vectorizable_reduction (stmt_info, NULL, NULL, &cost_vec);
+ ok = vectorizable_reduction (loop_vinfo,
+ stmt_info, NULL, NULL, &cost_vec);
}
/* SLP PHIs are tested by vect_slp_analyze_node_operations. */
if (ok
&& STMT_VINFO_LIVE_P (stmt_info)
&& !PURE_SLP_STMT (stmt_info))
- ok = vectorizable_live_operation (stmt_info, NULL, NULL, NULL,
+ ok = vectorizable_live_operation (loop_vinfo,
+ stmt_info, NULL, NULL, NULL,
-1, false, &cost_vec);
if (!ok)
@@ -1582,7 +1587,8 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
if (!gimple_clobber_p (stmt))
{
opt_result res
- = vect_analyze_stmt (loop_vinfo->lookup_stmt (stmt),
+ = vect_analyze_stmt (loop_vinfo,
+ loop_vinfo->lookup_stmt (stmt),
&need_to_vectorize,
NULL, NULL, &cost_vec);
if (!res)
@@ -1591,7 +1597,7 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
}
} /* bbs */
- add_stmt_costs (loop_vinfo->target_cost_data, &cost_vec);
+ add_stmt_costs (loop_vinfo, loop_vinfo->target_cost_data, &cost_vec);
/* All operations in the loop are either irrelevant (deal with loop
control, or dead), or only used outside the loop and can be moved
@@ -3371,8 +3377,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
{
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
- (void) add_stmt_cost (target_cost_data, len, vector_stmt, NULL, 0,
- vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
+ NULL, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@@ -3384,13 +3390,13 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
{
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
- (void) add_stmt_cost (target_cost_data, len, vector_stmt, NULL, 0,
- vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
+ NULL, 0, vect_prologue);
len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
if (len)
/* Count LEN - 1 ANDs and LEN comparisons. */
- (void) add_stmt_cost (target_cost_data, len * 2 - 1, scalar_stmt,
- NULL, 0, vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, len * 2 - 1,
+ scalar_stmt, NULL, 0, vect_prologue);
len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
if (len)
{
@@ -3400,8 +3406,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
for (unsigned int i = 0; i < len; ++i)
if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
nstmts += 1;
- (void) add_stmt_cost (target_cost_data, nstmts, scalar_stmt,
- NULL, 0, vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, nstmts,
+ scalar_stmt, NULL, 0, vect_prologue);
}
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
@@ -3413,8 +3419,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
{
/* FIXME: Make cost depend on complexity of individual check. */
- (void) add_stmt_cost (target_cost_data, 1, vector_stmt, NULL, 0,
- vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, vector_stmt,
+ NULL, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
@@ -3422,8 +3428,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
}
if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
- (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken, NULL, 0,
- vect_prologue);
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
+ NULL, 0, vect_prologue);
/* Count statements in scalar loop. Using this as scalar cost for a single
iteration for now.
@@ -3458,7 +3464,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
- (void) add_stmt_cost (target_cost_data, si->count,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
si->kind, si->stmt_info, si->misalign,
vect_epilogue);
}
@@ -3484,9 +3490,11 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
simpler and safer to use the worst-case cost; if this ends up
being the tie-breaker between vectorizing or not, then it's
probably better not to vectorize. */
- (void) add_stmt_cost (target_cost_data, num_masks, vector_stmt,
+ (void) add_stmt_cost (loop_vinfo,
+ target_cost_data, num_masks, vector_stmt,
NULL, 0, vect_prologue);
- (void) add_stmt_cost (target_cost_data, num_masks - 1, vector_stmt,
+ (void) add_stmt_cost (loop_vinfo,
+ target_cost_data, num_masks - 1, vector_stmt,
NULL, 0, vect_body);
}
else if (npeel < 0)
@@ -3508,23 +3516,25 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
branch per peeled loop. Even if scalar loop iterations are known,
vector iterations are not known since peeled prologue iterations are
not known. Hence guards remain the same. */
- (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
NULL, 0, vect_prologue);
- (void) add_stmt_cost (target_cost_data, 1, cond_branch_not_taken,
+ (void) add_stmt_cost (loop_vinfo,
+ target_cost_data, 1, cond_branch_not_taken,
NULL, 0, vect_prologue);
- (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
NULL, 0, vect_epilogue);
- (void) add_stmt_cost (target_cost_data, 1, cond_branch_not_taken,
+ (void) add_stmt_cost (loop_vinfo,
+ target_cost_data, 1, cond_branch_not_taken,
NULL, 0, vect_epilogue);
stmt_info_for_cost *si;
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
{
- (void) add_stmt_cost (target_cost_data,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_prologue,
si->kind, si->stmt_info, si->misalign,
vect_prologue);
- (void) add_stmt_cost (target_cost_data,
+ (void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_epilogue,
si->kind, si->stmt_info, si->misalign,
vect_epilogue);
@@ -3549,11 +3559,13 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
&epilogue_cost_vec);
FOR_EACH_VEC_ELT (prologue_cost_vec, j, si)
- (void) add_stmt_cost (data, si->count, si->kind, si->stmt_info,
+ (void) add_stmt_cost (loop_vinfo,
+ data, si->count, si->kind, si->stmt_info,
si->misalign, vect_prologue);
FOR_EACH_VEC_ELT (epilogue_cost_vec, j, si)
- (void) add_stmt_cost (data, si->count, si->kind, si->stmt_info,
+ (void) add_stmt_cost (loop_vinfo,
+ data, si->count, si->kind, si->stmt_info,
si->misalign, vect_epilogue);
prologue_cost_vec.release ();
@@ -3884,7 +3896,8 @@ have_whole_vector_shift (machine_mode mode)
the loop, and the epilogue code that must be generated. */
static void
-vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
+vect_model_reduction_cost (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, internal_fn reduc_fn,
vect_reduction_type reduction_type,
int ncopies, stmt_vector_for_cost *cost_vec)
{
@@ -3893,7 +3906,6 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
optab optab;
tree vectype;
machine_mode mode;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = NULL;
if (loop_vinfo)
@@ -4122,11 +4134,11 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
A cost model should help decide between these two schemes. */
static tree
-get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
+get_initial_def_for_reduction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_vinfo,
enum tree_code code, tree init_val,
tree *adjustment_def)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree scalar_type = TREE_TYPE (init_val);
tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
@@ -4226,14 +4238,14 @@ get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
value will not change the result. */
static void
-get_initial_defs_for_reduction (slp_tree slp_node,
+get_initial_defs_for_reduction (vec_info *vinfo,
+ slp_tree slp_node,
vec<tree> *vec_oprnds,
unsigned int number_of_vectors,
bool reduc_chain, tree neutral_op)
{
vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
stmt_vec_info stmt_vinfo = stmts[0];
- vec_info *vinfo = stmt_vinfo->vinfo;
unsigned HOST_WIDE_INT nunits;
unsigned j, number_of_places_left_in_vector;
tree vector_type;
@@ -4346,7 +4358,7 @@ get_initial_defs_for_reduction (slp_tree slp_node,
the stmt_vec_info the meta information is stored on. */
stmt_vec_info
-info_for_reduction (stmt_vec_info stmt_info)
+info_for_reduction (vec_info *vinfo, stmt_vec_info stmt_info)
{
stmt_info = vect_orig_stmt (stmt_info);
gcc_assert (STMT_VINFO_REDUC_DEF (stmt_info));
@@ -4362,7 +4374,7 @@ info_for_reduction (stmt_vec_info stmt_info)
{
edge pe = loop_preheader_edge (gimple_bb (phi)->loop_father);
stmt_vec_info info
- = stmt_info->vinfo->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi, pe));
+ = vinfo->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi, pe));
if (info && STMT_VINFO_DEF_TYPE (info) == vect_double_reduction_def)
stmt_info = info;
}
@@ -4417,13 +4429,13 @@ info_for_reduction (stmt_vec_info stmt_info)
*/
static void
-vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
+vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info,
slp_tree slp_node,
slp_instance slp_node_instance)
{
- stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+ stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
/* For double reductions we need to get at the inner loop reduction
stmt which has the meta info attached. Our stmt_info is that of the
loop-closed PHI of the inner loop which we remember as
@@ -5633,7 +5645,8 @@ get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
that should be used to control the operation in a fully-masked loop. */
static bool
-vectorize_fold_left_reduction (stmt_vec_info stmt_info,
+vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
gimple *reduc_def_stmt,
@@ -5641,7 +5654,6 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
tree ops[3], tree vectype_in,
int reduc_index, vec_loop_masks *masks)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
stmt_vec_info new_stmt_info = NULL;
@@ -5669,7 +5681,7 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
if (slp_node)
{
auto_vec<vec<tree> > vec_defs (2);
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (loop_vinfo, slp_node, &vec_defs);
vec_oprnds0.safe_splice (vec_defs[1 - reduc_index]);
vec_defs[0].release ();
vec_defs[1].release ();
@@ -5678,7 +5690,8 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
}
else
{
- tree loop_vec_def0 = vect_get_vec_def_for_operand (op0, stmt_info);
+ tree loop_vec_def0 = vect_get_vec_def_for_operand (loop_vinfo,
+ op0, stmt_info);
vec_oprnds0.create (1);
vec_oprnds0.quick_push (loop_vec_def0);
scalar_dest_def_info = stmt_info;
@@ -5756,11 +5769,13 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
if (i == vec_num - 1)
{
gimple_set_lhs (new_stmt, scalar_dest);
- new_stmt_info = vect_finish_replace_stmt (scalar_dest_def_info,
+ new_stmt_info = vect_finish_replace_stmt (loop_vinfo,
+ scalar_dest_def_info,
new_stmt);
}
else
- new_stmt_info = vect_finish_stmt_generation (scalar_dest_def_info,
+ new_stmt_info = vect_finish_stmt_generation (loop_vinfo,
+ scalar_dest_def_info,
new_stmt, gsi);
if (slp_node)
@@ -5927,13 +5942,13 @@ build_vect_cond_expr (enum tree_code code, tree vop[3], tree mask,
does *NOT* necessarily hold for reduction patterns. */
bool
-vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
+vectorizable_reduction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, slp_tree slp_node,
slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec)
{
tree scalar_dest;
tree vectype_in = NULL_TREE;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
stmt_vec_info cond_stmt_vinfo = NULL;
@@ -5955,7 +5970,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
return false;
/* The stmt we store reduction analysis meta on. */
- stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+ stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
reduc_info->is_reduc_info = true;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
@@ -6688,8 +6703,8 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
else
vec_num = 1;
- vect_model_reduction_cost (stmt_info, reduc_fn, reduction_type, ncopies,
- cost_vec);
+ vect_model_reduction_cost (loop_vinfo, stmt_info, reduc_fn,
+ reduction_type, ncopies, cost_vec);
if (dump_enabled_p ()
&& reduction_type == FOLD_LEFT_REDUCTION)
dump_printf_loc (MSG_NOTE, vect_location,
@@ -6753,18 +6768,18 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
value. */
bool
-vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vect_transform_reduction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
int j;
int vec_num;
- stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+ stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (nested_in_vect_loop_p (loop, stmt_info))
@@ -6839,7 +6854,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
return vectorize_fold_left_reduction
- (stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
+ (loop_vinfo, stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
reduc_fn, ops, vectype_in, reduc_index, masks);
}
@@ -6872,7 +6887,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Get vec defs for all the operands except the reduction index,
ensuring the ordering of the ops in the vector is kept. */
auto_vec<vec<tree>, 3> vec_defs;
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (loop_vinfo, slp_node, &vec_defs);
vec_oprnds0.safe_splice (vec_defs[0]);
vec_defs[0].release ();
vec_oprnds1.safe_splice (vec_defs[1]);
@@ -6886,12 +6901,12 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else
{
vec_oprnds0.quick_push
- (vect_get_vec_def_for_operand (ops[0], stmt_info));
+ (vect_get_vec_def_for_operand (loop_vinfo, ops[0], stmt_info));
vec_oprnds1.quick_push
- (vect_get_vec_def_for_operand (ops[1], stmt_info));
+ (vect_get_vec_def_for_operand (loop_vinfo, ops[1], stmt_info));
if (op_type == ternary_op)
vec_oprnds2.quick_push
- (vect_get_vec_def_for_operand (ops[2], stmt_info));
+ (vect_get_vec_def_for_operand (loop_vinfo, ops[2], stmt_info));
}
}
else
@@ -6944,7 +6959,8 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (loop_vinfo,
+ stmt_info, call, gsi);
}
else
{
@@ -6964,7 +6980,8 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (loop_vinfo,
+ stmt_info, new_stmt, gsi);
}
if (slp_node)
@@ -6991,11 +7008,11 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Transform phase of a cycle PHI. */
bool
-vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
+vect_transform_cycle_phi (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
slp_tree slp_node, slp_instance slp_node_instance)
{
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int i;
int ncopies;
@@ -7012,7 +7029,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info);
- stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+ stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
@@ -7062,7 +7079,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
= neutral_op_for_slp_reduction (slp_node, vectype_out,
STMT_VINFO_REDUC_CODE (reduc_info),
first != NULL);
- get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
+ get_initial_defs_for_reduction (loop_vinfo, slp_node_instance->reduc_phis,
&vec_initial_defs, vec_num,
first != NULL, neutral_op);
}
@@ -7096,7 +7113,8 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
{
/* Do not use an adjustment def as that case is not supported
correctly if ncopies is not one. */
- vec_initial_def = vect_get_vec_def_for_operand (initial_def,
+ vec_initial_def = vect_get_vec_def_for_operand (loop_vinfo,
+ initial_def,
reduc_stmt_info);
}
else
@@ -7107,7 +7125,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
adjustment_defp = NULL;
vec_initial_def
- = get_initial_def_for_reduction (reduc_stmt_info, code,
+ = get_initial_def_for_reduction (loop_vinfo, reduc_stmt_info, code,
initial_def, adjustment_defp);
STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info) = adjustment_def;
}
@@ -7155,10 +7173,10 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
/* Vectorizes LC PHIs. */
bool
-vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
+vectorizable_lc_phi (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
slp_tree slp_node)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
if (!loop_vinfo
|| !is_a <gphi *> (stmt_info->stmt)
|| gimple_phi_num_args (stmt_info->stmt) != 1)
@@ -7180,7 +7198,8 @@ vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
edge e = single_pred_edge (bb);
tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
vec<tree> vec_oprnds = vNULL;
- vect_get_vec_defs (gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE,
+ vect_get_vec_defs (loop_vinfo,
+ gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE,
stmt_info, &vec_oprnds, NULL, slp_node);
if (slp_node)
{
@@ -7268,12 +7287,12 @@ vect_worthwhile_without_simd_p (vec_info *vinfo, tree_code code)
Return true if STMT_INFO is vectorizable in this way. */
bool
-vectorizable_induction (stmt_vec_info stmt_info,
+vectorizable_induction (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
unsigned ncopies;
bool nested_in_vect_loop = false;
@@ -7468,10 +7487,11 @@ vectorizable_induction (stmt_vec_info stmt_info,
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
expr, step_expr);
if (! CONSTANT_CLASS_P (new_name))
- new_name = vect_init_vector (stmt_info, new_name,
+ new_name = vect_init_vector (loop_vinfo, stmt_info, new_name,
TREE_TYPE (step_expr), NULL);
new_vec = build_vector_from_val (step_vectype, new_name);
- vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
+ vec_step = vect_init_vector (loop_vinfo, stmt_info,
+ new_vec, step_vectype, NULL);
/* Now generate the IVs. */
unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
@@ -7542,10 +7562,11 @@ vectorizable_induction (stmt_vec_info stmt_info,
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
expr, step_expr);
if (! CONSTANT_CLASS_P (new_name))
- new_name = vect_init_vector (stmt_info, new_name,
+ new_name = vect_init_vector (loop_vinfo, stmt_info, new_name,
TREE_TYPE (step_expr), NULL);
new_vec = build_vector_from_val (step_vectype, new_name);
- vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
+ vec_step = vect_init_vector (loop_vinfo, stmt_info, new_vec,
+ step_vectype, NULL);
for (; ivn < nvects; ++ivn)
{
gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt;
@@ -7580,7 +7601,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
/* iv_loop is nested in the loop to be vectorized. init_expr had already
been created during vectorization of previous stmts. We obtain it
from the STMT_VINFO_VEC_STMT of the defining stmt. */
- vec_init = vect_get_vec_def_for_operand (init_expr, stmt_info);
+ vec_init = vect_get_vec_def_for_operand (loop_vinfo,
+ init_expr, stmt_info);
/* If the initial value is not of proper type, convert it. */
if (!useless_type_conversion_p (vectype, TREE_TYPE (vec_init)))
{
@@ -7683,7 +7705,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
gcc_assert (CONSTANT_CLASS_P (new_name)
|| TREE_CODE (new_name) == SSA_NAME);
new_vec = build_vector_from_val (step_vectype, t);
- vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
+ vec_step = vect_init_vector (loop_vinfo, stmt_info,
+ new_vec, step_vectype, NULL);
/* Create the following def-use cycle:
@@ -7752,7 +7775,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
gcc_assert (CONSTANT_CLASS_P (new_name)
|| TREE_CODE (new_name) == SSA_NAME);
new_vec = build_vector_from_val (step_vectype, t);
- vec_step = vect_init_vector (stmt_info, new_vec, step_vectype, NULL);
+ vec_step = vect_init_vector (loop_vinfo, stmt_info,
+ new_vec, step_vectype, NULL);
vec_def = induc_def;
prev_stmt_vinfo = induction_phi_info;
@@ -7821,13 +7845,13 @@ vectorizable_induction (stmt_vec_info stmt_info,
it can be supported. */
bool
-vectorizable_live_operation (stmt_vec_info stmt_info,
+vectorizable_live_operation (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
slp_tree slp_node, slp_instance slp_node_instance,
int slp_index, bool vec_stmt_p,
stmt_vector_for_cost *)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
imm_use_iterator imm_iter;
tree lhs, lhs_type, bitsize, vec_bitsize;
@@ -7859,12 +7883,12 @@ vectorizable_live_operation (stmt_vec_info stmt_info,
else if (slp_index != 0)
return true;
}
- stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+ stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
gcc_assert (reduc_info->is_reduc_info);
if (STMT_VINFO_REDUC_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
|| STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
return true;
- vect_create_epilog_for_reduction (stmt_info, slp_node,
+ vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
slp_node_instance);
return true;
}
@@ -8307,7 +8331,7 @@ vect_transform_loop_stmt (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
- if (vect_transform_stmt (stmt_info, gsi, NULL, NULL))
+ if (vect_transform_stmt (loop_vinfo, stmt_info, gsi, NULL, NULL))
*seen_store = stmt_info;
}
@@ -8666,7 +8690,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform phi.\n");
- vect_transform_stmt (stmt_info, NULL, NULL, NULL);
+ vect_transform_stmt (loop_vinfo, stmt_info, NULL, NULL, NULL);
}
}
@@ -8717,7 +8741,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
/* Interleaving. If IS_STORE is TRUE, the
vectorization of the interleaving chain was
completed - free all the stores in the chain. */
- vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store));
+ vect_remove_stores (loop_vinfo,
+ DR_GROUP_FIRST_ELEMENT (seen_store));
else
/* Free the attached stmt_vec_info and remove the stmt. */
loop_vinfo->remove_stmt (stmt_info);
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index 60dc1bfeaa4..91b8f22a30e 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -98,13 +98,12 @@ vect_pattern_detected (const char *name, gimple *stmt)
VECTYPE if it doesn't have one already. */
static stmt_vec_info
-vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
- tree vectype)
+vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
+ stmt_vec_info orig_stmt_info, tree vectype)
{
- vec_info *vinfo = orig_stmt_info->vinfo;
stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
if (pattern_stmt_info == NULL)
- pattern_stmt_info = orig_stmt_info->vinfo->add_stmt (pattern_stmt);
+ pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
pattern_stmt_info->pattern_stmt_p = true;
@@ -126,12 +125,12 @@ vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
have one already. */
static void
-vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
- tree vectype)
+vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
+ stmt_vec_info orig_stmt_info, tree vectype)
{
STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
STMT_VINFO_RELATED_STMT (orig_stmt_info)
- = vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype);
+ = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
}
/* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
@@ -141,13 +140,13 @@ vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
from which it was derived. */
static inline void
-append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
+append_pattern_def_seq (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple *new_stmt,
tree vectype = NULL_TREE,
tree scalar_type_for_mask = NULL_TREE)
{
gcc_assert (!scalar_type_for_mask
== (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
- vec_info *vinfo = stmt_info->vinfo;
if (vectype)
{
stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
@@ -256,7 +255,7 @@ vect_get_internal_def (vec_info *vinfo, tree op)
unsigned. */
static bool
-type_conversion_p (tree name, stmt_vec_info stmt_vinfo, bool check_sign,
+type_conversion_p (vec_info *vinfo, tree name, bool check_sign,
tree *orig_type, gimple **def_stmt, bool *promotion)
{
tree type = TREE_TYPE (name);
@@ -264,8 +263,7 @@ type_conversion_p (tree name, stmt_vec_info stmt_vinfo, bool check_sign,
enum vect_def_type dt;
stmt_vec_info def_stmt_info;
- if (!vect_is_simple_use (name, stmt_vinfo->vinfo, &dt, &def_stmt_info,
- def_stmt))
+ if (!vect_is_simple_use (name, vinfo, &dt, &def_stmt_info, def_stmt))
return false;
if (dt != vect_internal_def
@@ -293,7 +291,7 @@ type_conversion_p (tree name, stmt_vec_info stmt_vinfo, bool check_sign,
else
*promotion = false;
- if (!vect_is_simple_use (oprnd0, stmt_vinfo->vinfo, &dt))
+ if (!vect_is_simple_use (oprnd0, vinfo, &dt))
return false;
return true;
@@ -538,13 +536,12 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type)
exists. */
static unsigned int
-vect_widened_op_tree (stmt_vec_info stmt_info, tree_code code,
+vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
tree_code widened_code, bool shift_p,
unsigned int max_nops,
vect_unpromoted_value *unprom, tree *common_type)
{
/* Check for an integer operation with the right code. */
- vec_info *vinfo = stmt_info->vinfo;
gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
if (!assign)
return 0;
@@ -581,8 +578,7 @@ vect_widened_op_tree (stmt_vec_info stmt_info, tree_code code,
if (shift_p && i == 1)
return 0;
- if (!vect_look_through_possible_promotion (stmt_info->vinfo, op,
- this_unprom))
+ if (!vect_look_through_possible_promotion (vinfo, op, this_unprom))
return 0;
if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type))
@@ -602,9 +598,9 @@ vect_widened_op_tree (stmt_vec_info stmt_info, tree_code code,
/* Recursively process the definition of the operand. */
stmt_vec_info def_stmt_info
= vinfo->lookup_def (this_unprom->op);
- nops = vect_widened_op_tree (def_stmt_info, code, widened_code,
- shift_p, max_nops, this_unprom,
- common_type);
+ nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
+ widened_code, shift_p, max_nops,
+ this_unprom, common_type);
if (nops == 0)
return 0;
@@ -645,16 +641,15 @@ vect_recog_temp_ssa_var (tree type, gimple *stmt)
success. */
static bool
-vect_split_statement (stmt_vec_info stmt2_info, tree new_rhs,
+vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
gimple *stmt1, tree vectype)
{
- vec_info *vinfo = stmt2_info->vinfo;
if (is_pattern_stmt_p (stmt2_info))
{
/* STMT2_INFO is part of a pattern. Get the statement to which
the pattern is attached. */
stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
- vect_init_pattern_stmt (stmt1, orig_stmt2_info, vectype);
+ vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -702,13 +697,13 @@ vect_split_statement (stmt_vec_info stmt2_info, tree new_rhs,
/* Add STMT1 as a singleton pattern definition sequence. */
gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
- vect_init_pattern_stmt (stmt1, stmt2_info, vectype);
+ vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
gimple_seq_add_stmt_without_update (def_seq, stmt1);
/* Build the second of the two pattern statements. */
tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
- vect_set_pattern_stmt (new_stmt2, stmt2_info, lhs_vectype);
+ vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
if (dump_enabled_p ())
{
@@ -726,11 +721,9 @@ vect_split_statement (stmt_vec_info stmt2_info, tree new_rhs,
available. VECTYPE is the vector form of TYPE. */
static tree
-vect_convert_input (stmt_vec_info stmt_info, tree type,
+vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
vect_unpromoted_value *unprom, tree vectype)
{
- vec_info *vinfo = stmt_info->vinfo;
-
/* Check for a no-op conversion. */
if (types_compatible_p (type, TREE_TYPE (unprom->op)))
return unprom->op;
@@ -774,9 +767,10 @@ vect_convert_input (stmt_vec_info stmt_info, tree type,
input = vect_recog_temp_ssa_var (midtype, NULL);
gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
unprom->op);
- if (!vect_split_statement (unprom->caster, input, new_stmt,
+ if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
vec_midtype))
- append_pattern_def_seq (stmt_info, new_stmt, vec_midtype);
+ append_pattern_def_seq (vinfo, stmt_info,
+ new_stmt, vec_midtype);
}
}
@@ -792,7 +786,7 @@ vect_convert_input (stmt_vec_info stmt_info, tree type,
/* If OP is an external value, see if we can insert the new statement
on an incoming edge. */
if (input == unprom->op && unprom->dt == vect_external_def)
- if (edge e = vect_get_external_def_edge (stmt_info->vinfo, input))
+ if (edge e = vect_get_external_def_edge (vinfo, input))
{
basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
gcc_assert (!new_bb);
@@ -800,7 +794,7 @@ vect_convert_input (stmt_vec_info stmt_info, tree type,
}
/* As a (common) last resort, add the statement to the pattern itself. */
- append_pattern_def_seq (stmt_info, new_stmt, vectype);
+ append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
return new_op;
}
@@ -808,7 +802,7 @@ vect_convert_input (stmt_vec_info stmt_info, tree type,
result in the corresponding elements of RESULT. */
static void
-vect_convert_inputs (stmt_vec_info stmt_info, unsigned int n,
+vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
tree *result, tree type, vect_unpromoted_value *unprom,
tree vectype)
{
@@ -821,7 +815,8 @@ vect_convert_inputs (stmt_vec_info stmt_info, unsigned int n,
if (j < i)
result[i] = result[j];
else
- result[i] = vect_convert_input (stmt_info, type, &unprom[i], vectype);
+ result[i] = vect_convert_input (vinfo, stmt_info,
+ type, &unprom[i], vectype);
}
}
@@ -833,13 +828,13 @@ vect_convert_inputs (stmt_vec_info stmt_info, unsigned int n,
VECITYPE is the vector form of PATTERN_STMT's result type. */
static gimple *
-vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt,
- tree vecitype)
+vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
+ gimple *pattern_stmt, tree vecitype)
{
tree lhs = gimple_get_lhs (pattern_stmt);
if (!types_compatible_p (type, TREE_TYPE (lhs)))
{
- append_pattern_def_seq (stmt_info, pattern_stmt, vecitype);
+ append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
tree cast_var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
}
@@ -855,10 +850,11 @@ vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt,
*OP0_OUT and *OP1_OUT. */
static bool
-vect_reassociating_reduction_p (stmt_vec_info stmt_info, tree_code code,
+vect_reassociating_reduction_p (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree_code code,
tree *op0_out, tree *op1_out)
{
- loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
if (!loop_info)
return false;
@@ -932,11 +928,11 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_info, tree_code code,
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_dot_prod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_dot_prod_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
tree oprnd0, oprnd1;
gimple *last_stmt = stmt_vinfo->stmt;
- vec_info *vinfo = stmt_vinfo->vinfo;
tree type, half_type;
gimple *pattern_stmt;
tree var;
@@ -965,7 +961,7 @@ vect_recog_dot_prod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
- if (!vect_reassociating_reduction_p (stmt_vinfo, PLUS_EXPR,
+ if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
&oprnd0, &oprnd1))
return NULL;
@@ -988,7 +984,7 @@ vect_recog_dot_prod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
inside the loop (in case we are analyzing an outer-loop). */
vect_unpromoted_value unprom0[2];
- if (!vect_widened_op_tree (mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
+ if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
false, 2, unprom0, &half_type))
return NULL;
@@ -1007,7 +1003,7 @@ vect_recog_dot_prod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* Get the inputs in the appropriate types. */
tree mult_oprnd[2];
- vect_convert_inputs (stmt_vinfo, 2, mult_oprnd, half_type,
+ vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
unprom0, half_vectype);
var = vect_recog_temp_ssa_var (type, NULL);
@@ -1056,10 +1052,10 @@ vect_recog_dot_prod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
*/
static gimple *
-vect_recog_sad_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_sad_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
- vec_info *vinfo = stmt_vinfo->vinfo;
tree half_type;
/* Look for the following pattern
@@ -1090,7 +1086,7 @@ vect_recog_sad_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
of the above pattern. */
tree plus_oprnd0, plus_oprnd1;
- if (!vect_reassociating_reduction_p (stmt_vinfo, PLUS_EXPR,
+ if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
&plus_oprnd0, &plus_oprnd1))
return NULL;
@@ -1152,7 +1148,7 @@ vect_recog_sad_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
inside the loop (in case we are analyzing an outer-loop). */
vect_unpromoted_value unprom[2];
- if (!vect_widened_op_tree (diff_stmt_vinfo, MINUS_EXPR, MINUS_EXPR,
+ if (!vect_widened_op_tree (vinfo, diff_stmt_vinfo, MINUS_EXPR, MINUS_EXPR,
false, 2, unprom, &half_type))
return NULL;
@@ -1165,7 +1161,7 @@ vect_recog_sad_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* Get the inputs to the SAD_EXPR in the appropriate types. */
tree sad_oprnd[2];
- vect_convert_inputs (stmt_vinfo, 2, sad_oprnd, half_type,
+ vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
unprom, half_vectype);
tree var = vect_recog_temp_ssa_var (sum_type, NULL);
@@ -1201,16 +1197,16 @@ vect_recog_sad_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
name of the pattern being matched, for dump purposes. */
static gimple *
-vect_recog_widen_op_pattern (stmt_vec_info last_stmt_info, tree *type_out,
+vect_recog_widen_op_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out,
tree_code orig_code, tree_code wide_code,
bool shift_p, const char *name)
{
- vec_info *vinfo = last_stmt_info->vinfo;
gimple *last_stmt = last_stmt_info->stmt;
vect_unpromoted_value unprom[2];
tree half_type;
- if (!vect_widened_op_tree (last_stmt_info, orig_code, orig_code,
+ if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
shift_p, 2, unprom, &half_type))
return NULL;
@@ -1232,7 +1228,7 @@ vect_recog_widen_op_pattern (stmt_vec_info last_stmt_info, tree *type_out,
auto_vec<tree> dummy_vec;
if (!vectype
|| !vecitype
- || !supportable_widening_operation (wide_code, last_stmt_info,
+ || !supportable_widening_operation (vinfo, wide_code, last_stmt_info,
vecitype, vectype,
&dummy_code, &dummy_code,
&dummy_int, &dummy_vec))
@@ -1243,23 +1239,26 @@ vect_recog_widen_op_pattern (stmt_vec_info last_stmt_info, tree *type_out,
return NULL;
tree oprnd[2];
- vect_convert_inputs (last_stmt_info, 2, oprnd, half_type, unprom, vectype);
+ vect_convert_inputs (vinfo, last_stmt_info,
+ 2, oprnd, half_type, unprom, vectype);
tree var = vect_recog_temp_ssa_var (itype, NULL);
gimple *pattern_stmt = gimple_build_assign (var, wide_code,
oprnd[0], oprnd[1]);
- return vect_convert_output (last_stmt_info, type, pattern_stmt, vecitype);
+ return vect_convert_output (vinfo, last_stmt_info,
+ type, pattern_stmt, vecitype);
}
/* Try to detect multiplication on widened inputs, converting MULT_EXPR
to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */
static gimple *
-vect_recog_widen_mult_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
+ tree *type_out)
{
- return vect_recog_widen_op_pattern (last_stmt_info, type_out, MULT_EXPR,
- WIDEN_MULT_EXPR, false,
+ return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
+ MULT_EXPR, WIDEN_MULT_EXPR, false,
"vect_recog_widen_mult_pattern");
}
@@ -1288,9 +1287,9 @@ vect_recog_widen_mult_pattern (stmt_vec_info last_stmt_info, tree *type_out)
*/
static gimple *
-vect_recog_pow_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_pow_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
- vec_info *vinfo = stmt_vinfo->vinfo;
gimple *last_stmt = stmt_vinfo->stmt;
tree base, exp;
gimple *stmt;
@@ -1364,7 +1363,7 @@ vect_recog_pow_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
return NULL;
tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
- append_pattern_def_seq (stmt_vinfo, g);
+ append_pattern_def_seq (vinfo, stmt_vinfo, g);
tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
g = gimple_build_call (exp_decl, 1, def);
gimple_call_set_lhs (g, res);
@@ -1452,11 +1451,11 @@ vect_recog_pow_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
inner-loop nested in an outer-loop that us being vectorized). */
static gimple *
-vect_recog_widen_sum_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_widen_sum_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
tree oprnd0, oprnd1;
- vec_info *vinfo = stmt_vinfo->vinfo;
tree type;
gimple *pattern_stmt;
tree var;
@@ -1471,7 +1470,7 @@ vect_recog_widen_sum_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
- if (!vect_reassociating_reduction_p (stmt_vinfo, PLUS_EXPR,
+ if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
&oprnd0, &oprnd1))
return NULL;
@@ -1540,7 +1539,8 @@ vect_recog_widen_sum_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
by users of the result. */
static gimple *
-vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_over_widening_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out)
{
gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
if (!last_stmt)
@@ -1552,7 +1552,6 @@ vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out)
if (!new_precision)
return NULL;
- vec_info *vinfo = last_stmt_info->vinfo;
tree lhs = gimple_assign_lhs (last_stmt);
tree type = TREE_TYPE (lhs);
tree_code code = gimple_assign_rhs_code (last_stmt);
@@ -1716,7 +1715,7 @@ vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out)
tree ops[3] = {};
for (unsigned int i = 1; i < first_op; ++i)
ops[i - 1] = gimple_op (last_stmt, i);
- vect_convert_inputs (last_stmt_info, nops, &ops[first_op - 1],
+ vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
op_type, &unprom[0], op_vectype);
/* Use the operation to produce a result of type OP_TYPE. */
@@ -1732,11 +1731,11 @@ vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out)
/* Convert back to the original signedness, if OP_TYPE is different
from NEW_TYPE. */
if (op_type != new_type)
- pattern_stmt = vect_convert_output (last_stmt_info, new_type,
+ pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
pattern_stmt, op_vectype);
/* Promote the result to the original type. */
- pattern_stmt = vect_convert_output (last_stmt_info, type,
+ pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
pattern_stmt, new_vectype);
return pattern_stmt;
@@ -1755,14 +1754,14 @@ vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out)
where only the bottom half of res is used. */
static gimple *
-vect_recog_mulhs_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_mulhs_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out)
{
/* Check for a right shift. */
gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
if (!last_stmt
|| gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
return NULL;
- vec_info *vinfo = last_stmt_info->vinfo;
/* Check that the shift result is wider than the users of the
result need (i.e. that narrowing would be a natural choice). */
@@ -1868,7 +1867,7 @@ vect_recog_mulhs_pattern (stmt_vec_info last_stmt_info, tree *type_out)
vect_unpromoted_value unprom_mult[2];
tree new_type;
unsigned int nops
- = vect_widened_op_tree (mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
+ = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
false, 2, unprom_mult, &new_type);
if (nops != 2)
return NULL;
@@ -1896,7 +1895,7 @@ vect_recog_mulhs_pattern (stmt_vec_info last_stmt_info, tree *type_out)
/* Generate the IFN_MULHRS call. */
tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
tree new_ops[2];
- vect_convert_inputs (last_stmt_info, 2, new_ops, new_type,
+ vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
unprom_mult, new_vectype);
gcall *mulhrs_stmt
= gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
@@ -1907,7 +1906,7 @@ vect_recog_mulhs_pattern (stmt_vec_info last_stmt_info, tree *type_out)
dump_printf_loc (MSG_NOTE, vect_location,
"created pattern stmt: %G", mulhrs_stmt);
- return vect_convert_output (last_stmt_info, lhs_type,
+ return vect_convert_output (vinfo, last_stmt_info, lhs_type,
mulhrs_stmt, new_vectype);
}
@@ -1934,11 +1933,11 @@ vect_recog_mulhs_pattern (stmt_vec_info last_stmt_info, tree *type_out)
over plus and add a carry. */
static gimple *
-vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_average_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out)
{
/* Check for a shift right by one bit. */
gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
- vec_info *vinfo = last_stmt_info->vinfo;
if (!last_stmt
|| gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
|| !integer_onep (gimple_assign_rhs2 (last_stmt)))
@@ -1976,7 +1975,7 @@ vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
internal_fn ifn = IFN_AVG_FLOOR;
vect_unpromoted_value unprom[3];
tree new_type;
- unsigned int nops = vect_widened_op_tree (plus_stmt_info, PLUS_EXPR,
+ unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
PLUS_EXPR, false, 3,
unprom, &new_type);
if (nops == 0)
@@ -2059,7 +2058,7 @@ vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
tree new_ops[2];
- vect_convert_inputs (last_stmt_info, 2, new_ops, new_type,
+ vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
unprom, new_vectype);
if (fallback_p)
@@ -2079,28 +2078,28 @@ vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
- append_pattern_def_seq (last_stmt_info, g, new_vectype);
+ append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
- append_pattern_def_seq (last_stmt_info, g, new_vectype);
+ append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
shifted_op0, shifted_op1);
- append_pattern_def_seq (last_stmt_info, g, new_vectype);
+ append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
- append_pattern_def_seq (last_stmt_info, g, new_vectype);
+ append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
tree carry = vect_recog_temp_ssa_var (new_type, NULL);
g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
- append_pattern_def_seq (last_stmt_info, g, new_vectype);
+ append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
- return vect_convert_output (last_stmt_info, type, g, new_vectype);
+ return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
}
/* Generate the IFN_AVG* call. */
@@ -2113,7 +2112,8 @@ vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
dump_printf_loc (MSG_NOTE, vect_location,
"created pattern stmt: %G", average_stmt);
- return vect_convert_output (last_stmt_info, type, average_stmt, new_vectype);
+ return vect_convert_output (vinfo, last_stmt_info,
+ type, average_stmt, new_vectype);
}
/* Recognize cases in which the input to a cast is wider than its
@@ -2136,7 +2136,8 @@ vect_recog_average_pattern (stmt_vec_info last_stmt_info, tree *type_out)
input doesn't. */
static gimple *
-vect_recog_cast_forwprop_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_cast_forwprop_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out)
{
/* Check for a cast, including an integer-to-float conversion. */
gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
@@ -2165,7 +2166,6 @@ vect_recog_cast_forwprop_pattern (stmt_vec_info last_stmt_info, tree *type_out)
return NULL;
/* Try to find an unpromoted input. */
- vec_info *vinfo = last_stmt_info->vinfo;
vect_unpromoted_value unprom;
if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
|| TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
@@ -2196,10 +2196,11 @@ vect_recog_cast_forwprop_pattern (stmt_vec_info last_stmt_info, tree *type_out)
to WIDEN_LSHIFT_EXPR. See vect_recog_widen_op_pattern for details. */
static gimple *
-vect_recog_widen_shift_pattern (stmt_vec_info last_stmt_info, tree *type_out)
+vect_recog_widen_shift_pattern (vec_info *vinfo,
+ stmt_vec_info last_stmt_info, tree *type_out)
{
- return vect_recog_widen_op_pattern (last_stmt_info, type_out, LSHIFT_EXPR,
- WIDEN_LSHIFT_EXPR, true,
+ return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
+ LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
"vect_recog_widen_shift_pattern");
}
@@ -2231,13 +2232,13 @@ vect_recog_widen_shift_pattern (stmt_vec_info last_stmt_info, tree *type_out)
S0 stmt. */
static gimple *
-vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_rotate_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
gimple *pattern_stmt, *def_stmt;
enum tree_code rhs_code;
- vec_info *vinfo = stmt_vinfo->vinfo;
enum vect_def_type dt;
optab optab1, optab2;
edge ext_def = NULL;
@@ -2315,7 +2316,7 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
def = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
oprnd0 = def;
}
@@ -2375,7 +2376,7 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
def = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
oprnd0 = def;
}
@@ -2428,7 +2429,7 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
def = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
oprnd0 = def;
}
@@ -2452,7 +2453,7 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
def = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
}
stype = TREE_TYPE (def);
scalar_int_mode smode = SCALAR_INT_TYPE_MODE (stype);
@@ -2481,7 +2482,7 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
gcc_assert (!new_bb);
}
else
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecstype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
def2 = vect_recog_temp_ssa_var (stype, NULL);
tree mask = build_int_cst (stype, GET_MODE_PRECISION (smode) - 1);
@@ -2494,20 +2495,20 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
gcc_assert (!new_bb);
}
else
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecstype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
}
var1 = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
? LSHIFT_EXPR : RSHIFT_EXPR,
oprnd0, def);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
var2 = vect_recog_temp_ssa_var (type, NULL);
def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
? RSHIFT_EXPR : LSHIFT_EXPR,
oprnd0, def2);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
/* Pattern detected. */
vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
@@ -2558,14 +2559,14 @@ vect_recog_rotate_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
S3 stmt. */
static gimple *
-vect_recog_vector_vector_shift_pattern (stmt_vec_info stmt_vinfo,
+vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo,
tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
tree oprnd0, oprnd1, lhs, var;
gimple *pattern_stmt;
enum tree_code rhs_code;
- vec_info *vinfo = stmt_vinfo->vinfo;
if (!is_gimple_assign (last_stmt))
return NULL;
@@ -2622,7 +2623,7 @@ vect_recog_vector_vector_shift_pattern (stmt_vec_info stmt_vinfo,
def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
tree vecstype = get_vectype_for_scalar_type (vinfo,
TREE_TYPE (rhs1));
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecstype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
}
}
}
@@ -2631,7 +2632,7 @@ vect_recog_vector_vector_shift_pattern (stmt_vec_info stmt_vinfo,
{
def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
}
/* Pattern detected. */
@@ -2715,8 +2716,9 @@ target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
VINFO. Return the last statement. */
static gimple *
-synth_lshift_by_additions (tree dest, tree op, HOST_WIDE_INT amnt,
- stmt_vec_info vinfo)
+synth_lshift_by_additions (vec_info *vinfo,
+ tree dest, tree op, HOST_WIDE_INT amnt,
+ stmt_vec_info stmt_info)
{
HOST_WIDE_INT i;
tree itype = TREE_TYPE (op);
@@ -2730,7 +2732,7 @@ synth_lshift_by_additions (tree dest, tree op, HOST_WIDE_INT amnt,
= gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
prev_res = tmp_var;
if (i < amnt - 1)
- append_pattern_def_seq (vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_info, stmt);
else
return stmt;
}
@@ -2746,7 +2748,8 @@ synth_lshift_by_additions (tree dest, tree op, HOST_WIDE_INT amnt,
left shifts using additions. */
static tree
-apply_binop_and_append_stmt (tree_code code, tree op1, tree op2,
+apply_binop_and_append_stmt (vec_info *vinfo,
+ tree_code code, tree op1, tree op2,
stmt_vec_info stmt_vinfo, bool synth_shift_p)
{
if (integer_zerop (op2)
@@ -2764,14 +2767,14 @@ apply_binop_and_append_stmt (tree_code code, tree op1, tree op2,
if (code == LSHIFT_EXPR
&& synth_shift_p)
{
- stmt = synth_lshift_by_additions (tmp_var, op1, TREE_INT_CST_LOW (op2),
- stmt_vinfo);
- append_pattern_def_seq (stmt_vinfo, stmt);
+ stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
+ TREE_INT_CST_LOW (op2), stmt_vinfo);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
return tmp_var;
}
stmt = gimple_build_assign (tmp_var, code, op1, op2);
- append_pattern_def_seq (stmt_vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
return tmp_var;
}
@@ -2783,10 +2786,9 @@ apply_binop_and_append_stmt (tree_code code, tree op1, tree op2,
works on tree-ssa form. */
static gimple *
-vect_synth_mult_by_constant (tree op, tree val,
+vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
stmt_vec_info stmt_vinfo)
{
- vec_info *vinfo = stmt_vinfo->vinfo;
tree itype = TREE_TYPE (op);
machine_mode mode = TYPE_MODE (itype);
struct algorithm alg;
@@ -2832,7 +2834,7 @@ vect_synth_mult_by_constant (tree op, tree val,
{
tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
- append_pattern_def_seq (stmt_vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
op = tmp_op;
}
@@ -2855,23 +2857,23 @@ vect_synth_mult_by_constant (tree op, tree val,
case alg_shift:
if (synth_shift_p)
stmt
- = synth_lshift_by_additions (accum_tmp, accumulator, alg.log[i],
- stmt_vinfo);
+ = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
+ alg.log[i], stmt_vinfo);
else
stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
shft_log);
break;
case alg_add_t_m2:
tmp_var
- = apply_binop_and_append_stmt (LSHIFT_EXPR, op, shft_log,
- stmt_vinfo, synth_shift_p);
+ = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
+ stmt_vinfo, synth_shift_p);
stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
tmp_var);
break;
case alg_sub_t_m2:
- tmp_var = apply_binop_and_append_stmt (LSHIFT_EXPR, op,
- shft_log, stmt_vinfo,
- synth_shift_p);
+ tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
+ shft_log, stmt_vinfo,
+ synth_shift_p);
/* In some algorithms the first step involves zeroing the
accumulator. If subtracting from such an accumulator
just emit the negation directly. */
@@ -2883,27 +2885,27 @@ vect_synth_mult_by_constant (tree op, tree val,
break;
case alg_add_t2_m:
tmp_var
- = apply_binop_and_append_stmt (LSHIFT_EXPR, accumulator, shft_log,
- stmt_vinfo, synth_shift_p);
+ = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
+ shft_log, stmt_vinfo, synth_shift_p);
stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
break;
case alg_sub_t2_m:
tmp_var
- = apply_binop_and_append_stmt (LSHIFT_EXPR, accumulator, shft_log,
- stmt_vinfo, synth_shift_p);
+ = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
+ shft_log, stmt_vinfo, synth_shift_p);
stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
break;
case alg_add_factor:
tmp_var
- = apply_binop_and_append_stmt (LSHIFT_EXPR, accumulator, shft_log,
- stmt_vinfo, synth_shift_p);
+ = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
+ shft_log, stmt_vinfo, synth_shift_p);
stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
tmp_var);
break;
case alg_sub_factor:
tmp_var
- = apply_binop_and_append_stmt (LSHIFT_EXPR, accumulator, shft_log,
- stmt_vinfo, synth_shift_p);
+ = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
+ shft_log, stmt_vinfo, synth_shift_p);
stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
accumulator);
break;
@@ -2914,7 +2916,7 @@ vect_synth_mult_by_constant (tree op, tree val,
but rather return it directly. */
if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
- append_pattern_def_seq (stmt_vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
accumulator = accum_tmp;
}
if (variant == negate_variant)
@@ -2923,7 +2925,7 @@ vect_synth_mult_by_constant (tree op, tree val,
stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
accumulator = accum_tmp;
if (cast_to_unsigned_p)
- append_pattern_def_seq (stmt_vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
}
else if (variant == add_variant)
{
@@ -2931,7 +2933,7 @@ vect_synth_mult_by_constant (tree op, tree val,
stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
accumulator = accum_tmp;
if (cast_to_unsigned_p)
- append_pattern_def_seq (stmt_vinfo, stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, stmt);
}
/* Move back to a signed if needed. */
if (cast_to_unsigned_p)
@@ -2960,9 +2962,9 @@ vect_synth_mult_by_constant (tree op, tree val,
the multiplication. */
static gimple *
-vect_recog_mult_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_mult_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
- vec_info *vinfo = stmt_vinfo->vinfo;
gimple *last_stmt = stmt_vinfo->stmt;
tree oprnd0, oprnd1, vectype, itype;
gimple *pattern_stmt;
@@ -2998,7 +3000,8 @@ vect_recog_mult_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
return NULL;
}
- pattern_stmt = vect_synth_mult_by_constant (oprnd0, oprnd1, stmt_vinfo);
+ pattern_stmt = vect_synth_mult_by_constant (vinfo,
+ oprnd0, oprnd1, stmt_vinfo);
if (!pattern_stmt)
return NULL;
@@ -3049,9 +3052,9 @@ vect_recog_mult_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
S1 or modulo S4 stmt. */
static gimple *
-vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_divmod_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
- vec_info *vinfo = stmt_vinfo->vinfo;
gimple *last_stmt = stmt_vinfo->stmt;
tree oprnd0, oprnd1, vectype, itype, cond;
gimple *pattern_stmt, *def_stmt;
@@ -3126,11 +3129,11 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (rhs_code == TRUNC_MOD_EXPR)
{
- append_pattern_def_seq (stmt_vinfo, div_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
def_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
LSHIFT_EXPR, var_div, shift);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
pattern_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
MINUS_EXPR, oprnd0,
@@ -3155,12 +3158,12 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
fold_build2 (MINUS_EXPR, itype, oprnd1,
build_int_cst (itype, 1)),
build_int_cst (itype, 0));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
var = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
= gimple_build_assign (var, PLUS_EXPR, oprnd0,
gimple_assign_lhs (def_stmt));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
shift = build_int_cst (itype, tree_log2 (oprnd1));
pattern_stmt
@@ -3176,7 +3179,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
build_int_cst (itype, 1),
build_int_cst (itype, 0));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
}
else
{
@@ -3191,27 +3194,27 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
def_stmt = gimple_build_assign (var, COND_EXPR, cond,
build_int_cst (utype, -1),
build_int_cst (utype, 0));
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecutype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
var = vect_recog_temp_ssa_var (utype, NULL);
def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
gimple_assign_lhs (def_stmt),
shift);
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecutype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
signmask = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
= gimple_build_assign (signmask, NOP_EXPR, var);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
}
def_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
PLUS_EXPR, oprnd0, signmask);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
def_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
fold_build2 (MINUS_EXPR, itype, oprnd1,
build_int_cst (itype, 1)));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
pattern_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
@@ -3270,17 +3273,17 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
t1 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
build_int_cst (itype, ml));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t2 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
= gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t3 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
= gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t4 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
@@ -3288,7 +3291,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (post_shift != 1)
{
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
q = vect_recog_temp_ssa_var (itype, NULL);
pattern_stmt
@@ -3315,7 +3318,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
def_stmt
= gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
build_int_cst (NULL, pre_shift));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
}
else
t1 = oprnd0;
@@ -3326,7 +3329,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (post_shift)
{
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
q = vect_recog_temp_ssa_var (itype, NULL);
def_stmt
@@ -3387,7 +3390,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (add)
{
/* t2 = t1 + oprnd0; */
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t2 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
}
@@ -3397,7 +3400,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (post_shift)
{
/* t3 = t2 >> post_shift; */
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t3 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
build_int_cst (itype, post_shift));
@@ -3428,7 +3431,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
t4 = 0;
or if we know from VRP that oprnd0 < 0
t4 = -1; */
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
t4 = vect_recog_temp_ssa_var (itype, NULL);
if (msb != 1)
def_stmt = gimple_build_assign (t4, INTEGER_CST,
@@ -3436,7 +3439,7 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
else
def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
build_int_cst (itype, prec - 1));
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
/* q = t3 - t4; or q = t4 - t3; */
q = vect_recog_temp_ssa_var (itype, NULL);
@@ -3452,11 +3455,11 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
/* We divided. Now finish by:
t1 = q * oprnd1;
r = oprnd0 - t1; */
- append_pattern_def_seq (stmt_vinfo, pattern_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
t1 = vect_recog_temp_ssa_var (itype, NULL);
def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
- append_pattern_def_seq (stmt_vinfo, def_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
r = vect_recog_temp_ssa_var (itype, NULL);
pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
@@ -3498,9 +3501,9 @@ vect_recog_divmod_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
a_T = (TYPE) a_it; */
static gimple *
-vect_recog_mixed_size_cond_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_mixed_size_cond_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
- vec_info *vinfo = stmt_vinfo->vinfo;
gimple *last_stmt = stmt_vinfo->stmt;
tree cond_expr, then_clause, else_clause;
tree type, vectype, comp_vectype, itype = NULL_TREE, vecitype;
@@ -3536,11 +3539,11 @@ vect_recog_mixed_size_cond_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
return NULL;
if ((TREE_CODE (then_clause) != INTEGER_CST
- && !type_conversion_p (then_clause, stmt_vinfo, false, &orig_type0,
- &def_stmt0, &promotion))
+ && !type_conversion_p (vinfo, then_clause, false,
+ &orig_type0, &def_stmt0, &promotion))
|| (TREE_CODE (else_clause) != INTEGER_CST
- && !type_conversion_p (else_clause, stmt_vinfo, false, &orig_type1,
- &def_stmt1, &promotion)))
+ && !type_conversion_p (vinfo, else_clause, false,
+ &orig_type1, &def_stmt1, &promotion)))
return NULL;
if (orig_type0 && orig_type1
@@ -3609,7 +3612,7 @@ vect_recog_mixed_size_cond_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type, NULL),
NOP_EXPR, gimple_assign_lhs (def_stmt));
- append_pattern_def_seq (stmt_vinfo, def_stmt, vecitype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecitype);
*type_out = vectype;
vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt);
@@ -3722,12 +3725,12 @@ check_bool_pattern (tree var, vec_info *vinfo, hash_set<gimple *> &stmts)
pattern sequence. */
static tree
-adjust_bool_pattern_cast (tree type, tree var, stmt_vec_info stmt_info)
+adjust_bool_pattern_cast (vec_info *vinfo,
+ tree type, tree var, stmt_vec_info stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *cast_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type, NULL),
NOP_EXPR, var);
- append_pattern_def_seq (stmt_info, cast_stmt,
+ append_pattern_def_seq (vinfo, stmt_info, cast_stmt,
get_vectype_for_scalar_type (vinfo, type));
return gimple_assign_lhs (cast_stmt);
}
@@ -3739,10 +3742,9 @@ adjust_bool_pattern_cast (tree type, tree var, stmt_vec_info stmt_info)
be associated with. DEFS is a map of pattern defs. */
static void
-adjust_bool_pattern (tree var, tree out_type,
+adjust_bool_pattern (vec_info *vinfo, tree var, tree out_type,
stmt_vec_info stmt_info, hash_map <tree, tree> &defs)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *stmt = SSA_NAME_DEF_STMT (var);
enum tree_code rhs_code, def_rhs_code;
tree itype, cond_expr, rhs1, rhs2, irhs1, irhs2;
@@ -3858,15 +3860,17 @@ adjust_bool_pattern (tree var, tree out_type,
int prec2 = TYPE_PRECISION (TREE_TYPE (irhs2));
int out_prec = TYPE_PRECISION (out_type);
if (absu_hwi (out_prec - prec1) < absu_hwi (out_prec - prec2))
- irhs2 = adjust_bool_pattern_cast (TREE_TYPE (irhs1), irhs2,
+ irhs2 = adjust_bool_pattern_cast (vinfo, TREE_TYPE (irhs1), irhs2,
stmt_info);
else if (absu_hwi (out_prec - prec1) > absu_hwi (out_prec - prec2))
- irhs1 = adjust_bool_pattern_cast (TREE_TYPE (irhs2), irhs1,
+ irhs1 = adjust_bool_pattern_cast (vinfo, TREE_TYPE (irhs2), irhs1,
stmt_info);
else
{
- irhs1 = adjust_bool_pattern_cast (out_type, irhs1, stmt_info);
- irhs2 = adjust_bool_pattern_cast (out_type, irhs2, stmt_info);
+ irhs1 = adjust_bool_pattern_cast (vinfo,
+ out_type, irhs1, stmt_info);
+ irhs2 = adjust_bool_pattern_cast (vinfo,
+ out_type, irhs2, stmt_info);
}
}
itype = TREE_TYPE (irhs1);
@@ -3903,7 +3907,7 @@ adjust_bool_pattern (tree var, tree out_type,
}
gimple_set_location (pattern_stmt, loc);
- append_pattern_def_seq (stmt_info, pattern_stmt,
+ append_pattern_def_seq (vinfo, stmt_info, pattern_stmt,
get_vectype_for_scalar_type (vinfo, itype));
defs.put (var, gimple_assign_lhs (pattern_stmt));
}
@@ -3923,7 +3927,7 @@ sort_after_uid (const void *p1, const void *p2)
OUT_TYPE. Return the def of the pattern root. */
static tree
-adjust_bool_stmts (hash_set <gimple *> &bool_stmt_set,
+adjust_bool_stmts (vec_info *vinfo, hash_set <gimple *> &bool_stmt_set,
tree out_type, stmt_vec_info stmt_info)
{
/* Gather original stmts in the bool pattern in their order of appearance
@@ -3937,7 +3941,7 @@ adjust_bool_stmts (hash_set <gimple *> &bool_stmt_set,
/* Now process them in that order, producing pattern stmts. */
hash_map <tree, tree> defs;
for (unsigned i = 0; i < bool_stmts.length (); ++i)
- adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]),
+ adjust_bool_pattern (vinfo, gimple_assign_lhs (bool_stmts[i]),
out_type, stmt_info, defs);
/* Pop the last pattern seq stmt and install it as pattern root for STMT. */
@@ -4012,12 +4016,12 @@ integer_type_for_mask (tree var, vec_info *vinfo)
but the above is more efficient. */
static gimple *
-vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_bool_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
enum tree_code rhs_code;
tree var, lhs, rhs, vectype;
- vec_info *vinfo = stmt_vinfo->vinfo;
gimple *pattern_stmt;
if (!is_gimple_assign (last_stmt))
@@ -4043,7 +4047,8 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (check_bool_pattern (var, vinfo, bool_stmts))
{
- rhs = adjust_bool_stmts (bool_stmts, TREE_TYPE (lhs), stmt_vinfo);
+ rhs = adjust_bool_stmts (vinfo, bool_stmts,
+ TREE_TYPE (lhs), stmt_vinfo);
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
if (useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
@@ -4075,7 +4080,8 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
{
tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
- append_pattern_def_seq (stmt_vinfo, pattern_stmt, new_vectype);
+ append_pattern_def_seq (vinfo, stmt_vinfo,
+ pattern_stmt, new_vectype);
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
pattern_stmt = gimple_build_assign (lhs, CONVERT_EXPR, tmp);
@@ -4110,7 +4116,7 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
if (!check_bool_pattern (var, vinfo, bool_stmts))
return NULL;
- rhs = adjust_bool_stmts (bool_stmts, type, stmt_vinfo);
+ rhs = adjust_bool_stmts (vinfo, bool_stmts, type, stmt_vinfo);
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
pattern_stmt
@@ -4129,13 +4135,14 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
stmt_vec_info pattern_stmt_info;
tree nunits_vectype;
- if (!vect_get_vector_types_for_stmt (stmt_vinfo, &vectype,
+ if (!vect_get_vector_types_for_stmt (vinfo, stmt_vinfo, &vectype,
&nunits_vectype)
|| !VECTOR_MODE_P (TYPE_MODE (vectype)))
return NULL;
if (check_bool_pattern (var, vinfo, bool_stmts))
- rhs = adjust_bool_stmts (bool_stmts, TREE_TYPE (vectype), stmt_vinfo);
+ rhs = adjust_bool_stmts (vinfo, bool_stmts,
+ TREE_TYPE (vectype), stmt_vinfo);
else
{
tree type = integer_type_for_mask (var, vinfo);
@@ -4153,7 +4160,7 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
rhs = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
- append_pattern_def_seq (stmt_vinfo, pattern_stmt, new_vectype);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype);
}
lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
@@ -4161,7 +4168,7 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
{
tree rhs2 = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
gimple *cast_stmt = gimple_build_assign (rhs2, NOP_EXPR, rhs);
- append_pattern_def_seq (stmt_vinfo, cast_stmt);
+ append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt);
rhs = rhs2;
}
pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
@@ -4185,7 +4192,8 @@ vect_recog_bool_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
Return converted mask. */
static tree
-build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo)
+build_mask_conversion (vec_info *vinfo,
+ tree mask, tree vectype, stmt_vec_info stmt_vinfo)
{
gimple *stmt;
tree masktype, tmp;
@@ -4193,7 +4201,8 @@ build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo)
masktype = truth_type_for (vectype);
tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
- append_pattern_def_seq (stmt_vinfo, stmt, masktype, TREE_TYPE (vectype));
+ append_pattern_def_seq (vinfo, stmt_vinfo,
+ stmt, masktype, TREE_TYPE (vectype));
return tmp;
}
@@ -4225,14 +4234,14 @@ build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo)
S4' c_1' = m_3'' ? c_2 : c_3; */
static gimple *
-vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+vect_recog_mask_conversion_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
{
gimple *last_stmt = stmt_vinfo->stmt;
enum tree_code rhs_code;
tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
tree vectype1, vectype2;
stmt_vec_info pattern_stmt_info;
- vec_info *vinfo = stmt_vinfo->vinfo;
/* Check for MASK_LOAD ans MASK_STORE calls requiring mask conversion. */
if (is_gimple_call (last_stmt)
@@ -4269,7 +4278,7 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
TYPE_VECTOR_SUBPARTS (vectype2)))
return NULL;
- tmp = build_mask_conversion (mask_arg, vectype1, stmt_vinfo);
+ tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
auto_vec<tree, 8> args;
unsigned int nargs = gimple_call_num_args (last_stmt);
@@ -4388,13 +4397,13 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
tmp = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
pattern_stmt = gimple_build_assign (tmp, rhs1);
rhs1 = tmp;
- append_pattern_def_seq (stmt_vinfo, pattern_stmt, vectype2,
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype2,
rhs1_type);
}
if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
TYPE_VECTOR_SUBPARTS (vectype2)))
- tmp = build_mask_conversion (rhs1, vectype1, stmt_vinfo);
+ tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
else
tmp = rhs1;
@@ -4434,14 +4443,14 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
if (!vectype1)
return NULL;
- rhs2 = build_mask_conversion (rhs2, vectype1, stmt_vinfo);
+ rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo);
}
else
{
vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
if (!vectype1)
return NULL;
- rhs1 = build_mask_conversion (rhs1, vectype1, stmt_vinfo);
+ rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
}
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
@@ -4491,7 +4500,7 @@ vect_convert_mask_for_vectype (tree mask, tree vectype,
if (mask_vectype
&& maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
TYPE_VECTOR_SUBPARTS (mask_vectype)))
- mask = build_mask_conversion (mask, vectype, stmt_info);
+ mask = build_mask_conversion (vinfo, mask, vectype, stmt_info);
}
return mask;
}
@@ -4505,15 +4514,15 @@ vect_convert_mask_for_vectype (tree mask, tree vectype,
to STMT_INFO. */
static tree
-vect_add_conversion_to_pattern (tree type, tree value, stmt_vec_info stmt_info)
+vect_add_conversion_to_pattern (vec_info *vinfo,
+ tree type, tree value, stmt_vec_info stmt_info)
{
if (useless_type_conversion_p (type, TREE_TYPE (value)))
return value;
- vec_info *vinfo = stmt_info->vinfo;
tree new_value = vect_recog_temp_ssa_var (type, NULL);
gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
- append_pattern_def_seq (stmt_info, conversion,
+ append_pattern_def_seq (vinfo, stmt_info, conversion,
get_vectype_for_scalar_type (vinfo, type));
return new_value;
}
@@ -4526,10 +4535,11 @@ vect_add_conversion_to_pattern (tree type, tree value, stmt_vec_info stmt_info)
as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
static gimple *
-vect_recog_gather_scatter_pattern (stmt_vec_info stmt_info, tree *type_out)
+vect_recog_gather_scatter_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree *type_out)
{
/* Currently we only support this for loop vectorization. */
- loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (stmt_info->vinfo);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
if (!loop_vinfo)
return NULL;
@@ -4560,8 +4570,8 @@ vect_recog_gather_scatter_pattern (stmt_vec_info stmt_info, tree *type_out)
latter to the same width as the vector elements. */
tree base = gs_info.base;
tree offset_type = TREE_TYPE (gs_info.offset_vectype);
- tree offset = vect_add_conversion_to_pattern (offset_type, gs_info.offset,
- stmt_info);
+ tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
+ gs_info.offset, stmt_info);
/* Build the new pattern statement. */
tree scale = size_int (gs_info.scale);
@@ -4705,10 +4715,10 @@ vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
whose result is LHS. */
static bool
-vect_determine_min_output_precision_1 (stmt_vec_info stmt_info, tree lhs)
+vect_determine_min_output_precision_1 (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree lhs)
{
/* Take the maximum precision required by users of the result. */
- vec_info *vinfo = stmt_info->vinfo;
unsigned int precision = 0;
imm_use_iterator iter;
use_operand_p use;
@@ -4742,7 +4752,7 @@ vect_determine_min_output_precision_1 (stmt_vec_info stmt_info, tree lhs)
/* Calculate min_output_precision for STMT_INFO. */
static void
-vect_determine_min_output_precision (stmt_vec_info stmt_info)
+vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
{
/* We're only interested in statements with a narrowable result. */
tree lhs = gimple_get_lhs (stmt_info->stmt);
@@ -4751,7 +4761,7 @@ vect_determine_min_output_precision (stmt_vec_info stmt_info)
|| !vect_narrowable_type_p (TREE_TYPE (lhs)))
return;
- if (!vect_determine_min_output_precision_1 (stmt_info, lhs))
+ if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
}
@@ -4962,10 +4972,8 @@ possible_vector_mask_operation_p (stmt_vec_info stmt_info)
result in STMT_INFO->mask_precision. */
static void
-vect_determine_mask_precision (stmt_vec_info stmt_info)
+vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
-
if (!possible_vector_mask_operation_p (stmt_info)
|| stmt_info->mask_precision)
return;
@@ -5070,15 +5078,15 @@ vect_determine_mask_precision (stmt_vec_info stmt_info)
have already done so for the users of its result. */
void
-vect_determine_stmt_precisions (stmt_vec_info stmt_info)
+vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
{
- vect_determine_min_output_precision (stmt_info);
+ vect_determine_min_output_precision (vinfo, stmt_info);
if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
{
vect_determine_precisions_from_range (stmt_info, stmt);
vect_determine_precisions_from_users (stmt_info, stmt);
}
- vect_determine_mask_precision (stmt_info);
+ vect_determine_mask_precision (vinfo, stmt_info);
}
/* Walk backwards through the vectorizable region to determine the
@@ -5106,7 +5114,7 @@ vect_determine_precisions (vec_info *vinfo)
for (gimple_stmt_iterator si = gsi_last_bb (bb);
!gsi_end_p (si); gsi_prev (&si))
vect_determine_stmt_precisions
- (vinfo->lookup_stmt (gsi_stmt (si)));
+ (vinfo, vinfo->lookup_stmt (gsi_stmt (si)));
}
}
else
@@ -5123,13 +5131,13 @@ vect_determine_precisions (vec_info *vinfo)
stmt = gsi_stmt (si);
stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
- vect_determine_stmt_precisions (stmt_info);
+ vect_determine_stmt_precisions (vinfo, stmt_info);
}
while (stmt != gsi_stmt (bb_vinfo->region_begin));
}
}
-typedef gimple *(*vect_recog_func_ptr) (stmt_vec_info, tree *);
+typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
struct vect_recog_func
{
@@ -5171,7 +5179,8 @@ const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
/* Mark statements that are involved in a pattern. */
static inline void
-vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
+vect_mark_pattern_stmts (vec_info *vinfo,
+ stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
tree pattern_vectype)
{
stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
@@ -5213,7 +5222,7 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
dump_printf_loc (MSG_NOTE, vect_location,
"extra pattern stmt: %G", gsi_stmt (si));
stmt_vec_info pattern_stmt_info
- = vect_init_pattern_stmt (gsi_stmt (si),
+ = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
orig_stmt_info, pattern_vectype);
/* Stmts in the def sequence are not vectorizable cycle or
induction defs, instead they should all be vect_internal_def
@@ -5223,7 +5232,8 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
if (orig_pattern_stmt)
{
- vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
+ vect_init_pattern_stmt (vinfo, pattern_stmt,
+ orig_stmt_info, pattern_vectype);
/* Insert all the new pattern statements before the original one. */
gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
@@ -5236,12 +5246,12 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
gsi_remove (&gsi, false);
}
else
- vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
+ vect_set_pattern_stmt (vinfo,
+ pattern_stmt, orig_stmt_info, pattern_vectype);
/* Transfer reduction path info to the pattern. */
if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
{
- vec_info *vinfo = orig_stmt_info_saved->vinfo;
tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
/* Search the pattern def sequence and the main pattern stmt. Note
@@ -5312,9 +5322,9 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
for vect_recog_pattern. */
static void
-vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
+vect_pattern_recog_1 (vec_info *vinfo,
+ vect_recog_func *recog_func, stmt_vec_info stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *pattern_stmt;
loop_vec_info loop_vinfo;
tree pattern_vectype;
@@ -5328,12 +5338,13 @@ vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
gimple_stmt_iterator gsi;
for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
!gsi_end_p (gsi); gsi_next (&gsi))
- vect_pattern_recog_1 (recog_func, vinfo->lookup_stmt (gsi_stmt (gsi)));
+ vect_pattern_recog_1 (vinfo, recog_func,
+ vinfo->lookup_stmt (gsi_stmt (gsi)));
return;
}
gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
- pattern_stmt = recog_func->fn (stmt_info, &pattern_vectype);
+ pattern_stmt = recog_func->fn (vinfo, stmt_info, &pattern_vectype);
if (!pattern_stmt)
{
/* Clear any half-formed pattern definition sequence. */
@@ -5341,7 +5352,7 @@ vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
return;
}
- loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
gcc_assert (pattern_vectype);
/* Found a vectorizable pattern. */
@@ -5351,7 +5362,7 @@ vect_pattern_recog_1 (vect_recog_func *recog_func, stmt_vec_info stmt_info)
recog_func->name, pattern_stmt);
/* Mark the stmts that are involved in the pattern. */
- vect_mark_pattern_stmts (stmt_info, pattern_stmt, pattern_vectype);
+ vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
@@ -5471,7 +5482,7 @@ vect_pattern_recog (vec_info *vinfo)
stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
/* Scan over all generic vect_recog_xxx_pattern functions. */
for (j = 0; j < NUM_PATTERNS; j++)
- vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j],
+ vect_pattern_recog_1 (vinfo, &vect_vect_recog_func_ptrs[j],
stmt_info);
}
}
@@ -5489,7 +5500,8 @@ vect_pattern_recog (vec_info *vinfo)
/* Scan over all generic vect_recog_xxx_pattern functions. */
for (j = 0; j < NUM_PATTERNS; j++)
- vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
+ vect_pattern_recog_1 (vinfo,
+ &vect_vect_recog_func_ptrs[j], stmt_info);
}
}
}
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 32e6a0beaed..41510d2218e 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -673,10 +673,10 @@ vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype)
Used only for BB vectorization. */
static bool
-vect_update_all_shared_vectypes (vec<stmt_vec_info> stmts)
+vect_update_all_shared_vectypes (vec_info *vinfo, vec<stmt_vec_info> stmts)
{
tree vectype, nunits_vectype;
- if (!vect_get_vector_types_for_stmt (stmts[0], &vectype,
+ if (!vect_get_vector_types_for_stmt (vinfo, stmts[0], &vectype,
&nunits_vectype, stmts.length ()))
return false;
@@ -732,7 +732,8 @@ compatible_calls_p (gcall *call1, gcall *call2)
vect_build_slp_tree. */
static bool
-vect_record_max_nunits (stmt_vec_info stmt_info, unsigned int group_size,
+vect_record_max_nunits (vec_info *vinfo, stmt_vec_info stmt_info,
+ unsigned int group_size,
tree vectype, poly_uint64 *max_nunits)
{
if (!vectype)
@@ -749,7 +750,7 @@ vect_record_max_nunits (stmt_vec_info stmt_info, unsigned int group_size,
before adjusting *max_nunits for basic-block vectorization. */
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
unsigned HOST_WIDE_INT const_nunits;
- if (STMT_VINFO_BB_VINFO (stmt_info)
+ if (is_a <bb_vec_info> (vinfo)
&& (!nunits.is_constant (&const_nunits)
|| const_nunits > group_size))
{
@@ -810,7 +811,7 @@ vect_two_operations_perm_ok_p (vec<stmt_vec_info> stmts,
to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
static bool
-vect_build_slp_tree_1 (unsigned char *swap,
+vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
vec<stmt_vec_info> stmts, unsigned int group_size,
poly_uint64 *max_nunits, bool *matches,
bool *two_operators)
@@ -835,7 +836,6 @@ vect_build_slp_tree_1 (unsigned char *swap,
stmt_vec_info stmt_info;
FOR_EACH_VEC_ELT (stmts, i, stmt_info)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *stmt = stmt_info->stmt;
swap[i] = 0;
matches[i] = false;
@@ -868,10 +868,10 @@ vect_build_slp_tree_1 (unsigned char *swap,
}
tree nunits_vectype;
- if (!vect_get_vector_types_for_stmt (stmt_info, &vectype,
+ if (!vect_get_vector_types_for_stmt (vinfo, stmt_info, &vectype,
&nunits_vectype, group_size)
|| (nunits_vectype
- && !vect_record_max_nunits (stmt_info, group_size,
+ && !vect_record_max_nunits (vinfo, stmt_info, group_size,
nunits_vectype, max_nunits)))
{
/* Fatal mismatch. */
@@ -1302,7 +1302,8 @@ vect_build_slp_tree_2 (vec_info *vinfo,
{
tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
- if (!vect_record_max_nunits (stmt_info, group_size, vectype, max_nunits))
+ if (!vect_record_max_nunits (vinfo, stmt_info, group_size, vectype,
+ max_nunits))
return NULL;
vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
@@ -1334,7 +1335,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
bool two_operators = false;
unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
- if (!vect_build_slp_tree_1 (swap, stmts, group_size,
+ if (!vect_build_slp_tree_1 (vinfo, swap, stmts, group_size,
&this_max_nunits, matches, &two_operators))
return NULL;
@@ -1444,7 +1445,8 @@ vect_build_slp_tree_2 (vec_info *vinfo,
if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def)
break;
if (!grandchild
- && vect_update_all_shared_vectypes (oprnd_info->def_stmts))
+ && vect_update_all_shared_vectypes (vinfo,
+ oprnd_info->def_stmts))
{
/* Roll back. */
this_tree_size = old_tree_size;
@@ -1486,7 +1488,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
scalar version. */
&& !is_pattern_stmt_p (stmt_info)
&& !oprnd_info->any_pattern
- && vect_update_all_shared_vectypes (oprnd_info->def_stmts))
+ && vect_update_all_shared_vectypes (vinfo, oprnd_info->def_stmts))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -1586,7 +1588,7 @@ vect_build_slp_tree_2 (vec_info *vinfo,
break;
if (!grandchild
&& (vect_update_all_shared_vectypes
- (oprnd_info->def_stmts)))
+ (vinfo, oprnd_info->def_stmts)))
{
/* Roll back. */
this_tree_size = old_tree_size;
@@ -2540,7 +2542,8 @@ vect_make_slp_decision (loop_vec_info loop_vinfo)
can't be SLPed) in the tree rooted at NODE. Mark such stmts as HYBRID. */
static void
-vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype,
+vect_detect_hybrid_slp_stmts (loop_vec_info loop_vinfo, slp_tree node,
+ unsigned i, slp_vect_type stype,
hash_map<slp_tree, unsigned> &visited)
{
stmt_vec_info stmt_vinfo = SLP_TREE_SCALAR_STMTS (node)[i];
@@ -2608,7 +2611,7 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype,
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
if (SLP_TREE_DEF_TYPE (child) != vect_external_def
&& SLP_TREE_DEF_TYPE (child) != vect_constant_def)
- vect_detect_hybrid_slp_stmts (child, i, stype, visited);
+ vect_detect_hybrid_slp_stmts (loop_vinfo, child, i, stype, visited);
}
/* Helpers for vect_detect_hybrid_slp walking pattern stmt uses. */
@@ -2701,7 +2704,8 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
if (j < SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)).length ())
{
any = true;
- vect_detect_hybrid_slp_stmts (SLP_INSTANCE_TREE (instance),
+ vect_detect_hybrid_slp_stmts (loop_vinfo,
+ SLP_INSTANCE_TREE (instance),
j, pure_slp, visited);
}
if (!any)
@@ -2791,7 +2795,8 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
}
bool dummy;
- return vect_analyze_stmt (stmt_info, &dummy, node, node_instance, cost_vec);
+ return vect_analyze_stmt (vinfo, stmt_info, &dummy,
+ node, node_instance, cost_vec);
}
/* Try to build NODE from scalars, returning true on success.
@@ -2958,7 +2963,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
visited.add (*x);
i++;
- add_stmt_costs (vinfo->target_cost_data, &cost_vec);
+ add_stmt_costs (vinfo, vinfo->target_cost_data, &cost_vec);
cost_vec.release ();
}
}
@@ -2972,7 +2977,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
update LIFE according to uses of NODE. */
static void
-vect_bb_slp_scalar_cost (basic_block bb,
+vect_bb_slp_scalar_cost (vec_info *vinfo, basic_block bb,
slp_tree node, vec<bool, va_heap> *life,
stmt_vector_for_cost *cost_vec,
hash_set<slp_tree> &visited)
@@ -2987,7 +2992,6 @@ vect_bb_slp_scalar_cost (basic_block bb,
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
{
gimple *stmt = stmt_info->stmt;
- vec_info *vinfo = stmt_info->vinfo;
ssa_op_iter op_iter;
def_operand_p def_p;
@@ -3045,7 +3049,7 @@ vect_bb_slp_scalar_cost (basic_block bb,
/* Do not directly pass LIFE to the recursive call, copy it to
confine changes in the callee to the current child/subtree. */
subtree_life.safe_splice (*life);
- vect_bb_slp_scalar_cost (bb, child, &subtree_life, cost_vec,
+ vect_bb_slp_scalar_cost (vinfo, bb, child, &subtree_life, cost_vec,
visited);
subtree_life.truncate (0);
}
@@ -3072,12 +3076,12 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo)
auto_vec<bool, 20> life;
life.safe_grow_cleared
(SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)).length ());
- vect_bb_slp_scalar_cost (BB_VINFO_BB (bb_vinfo),
+ vect_bb_slp_scalar_cost (bb_vinfo, BB_VINFO_BB (bb_vinfo),
SLP_INSTANCE_TREE (instance),
&life, &scalar_costs, visited);
}
void *target_cost_data = init_cost (NULL);
- add_stmt_costs (target_cost_data, &scalar_costs);
+ add_stmt_costs (bb_vinfo, target_cost_data, &scalar_costs);
scalar_costs.release ();
unsigned dummy;
finish_cost (target_cost_data, &dummy, &scalar_cost, &dummy);
@@ -3233,8 +3237,8 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal)
dependence in the SLP instances. */
for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
{
- if (! vect_slp_analyze_and_verify_instance_alignment (instance)
- || ! vect_slp_analyze_instance_dependence (instance))
+ if (! vect_slp_analyze_and_verify_instance_alignment (bb_vinfo, instance)
+ || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
{
slp_tree node = SLP_INSTANCE_TREE (instance);
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
@@ -3472,7 +3476,8 @@ vect_slp_bb (basic_block bb)
/* Return 1 if vector type STMT_VINFO is a boolean vector. */
static bool
-vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
+vect_mask_constant_operand_p (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, unsigned op_num)
{
enum tree_code code = gimple_expr_code (stmt_vinfo->stmt);
tree op, vectype;
@@ -3485,7 +3490,7 @@ vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
gassign *stmt = as_a <gassign *> (stmt_vinfo->stmt);
op = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use (op, stmt_vinfo->vinfo, &dt, &vectype))
+ if (!vect_is_simple_use (op, vinfo, &dt, &vectype))
gcc_unreachable ();
return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype);
@@ -3509,7 +3514,7 @@ vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, unsigned op_num)
op = TREE_OPERAND (cond, 0);
}
- if (!vect_is_simple_use (op, stmt_vinfo->vinfo, &dt, &vectype))
+ if (!vect_is_simple_use (op, vinfo, &dt, &vectype))
gcc_unreachable ();
return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype);
@@ -3638,12 +3643,12 @@ duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
operands. */
static void
-vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
+vect_get_constant_vectors (vec_info *vinfo,
+ slp_tree slp_node, unsigned op_num,
vec<tree> *vec_oprnds)
{
slp_tree op_node = SLP_TREE_CHILDREN (slp_node)[op_num];
stmt_vec_info stmt_vinfo = SLP_TREE_SCALAR_STMTS (slp_node)[0];
- vec_info *vinfo = stmt_vinfo->vinfo;
unsigned HOST_WIDE_INT nunits;
tree vec_cst;
unsigned j, number_of_places_left_in_vector;
@@ -3663,7 +3668,7 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
/* Check if vector type is a boolean vector. */
tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op))
- && vect_mask_constant_operand_p (stmt_vinfo, op_num))
+ && vect_mask_constant_operand_p (vinfo, stmt_vinfo, op_num))
vector_type = truth_type_for (stmt_vectype);
else
vector_type = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), op_node);
@@ -3772,8 +3777,8 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
constant_p = false;
if (TREE_CODE (orig_op) == SSA_NAME
&& !SSA_NAME_IS_DEFAULT_DEF (orig_op)
- && STMT_VINFO_BB_VINFO (stmt_vinfo)
- && (STMT_VINFO_BB_VINFO (stmt_vinfo)->bb
+ && is_a <bb_vec_info> (vinfo)
+ && (as_a <bb_vec_info> (vinfo)->bb
== gimple_bb (SSA_NAME_DEF_STMT (orig_op))))
place_after_defs = true;
@@ -3798,12 +3803,12 @@ vect_get_constant_vectors (slp_tree slp_node, unsigned op_num,
stmt_vec_info last_stmt_info
= vect_find_last_scalar_stmt_in_slp (slp_node);
gsi = gsi_for_stmt (last_stmt_info->stmt);
- init = vect_init_vector (stmt_vinfo, vec_cst, vector_type,
- &gsi);
+ init = vect_init_vector (vinfo, stmt_vinfo, vec_cst,
+ vector_type, &gsi);
}
else
- init = vect_init_vector (stmt_vinfo, vec_cst, vector_type,
- NULL);
+ init = vect_init_vector (vinfo, stmt_vinfo, vec_cst,
+ vector_type, NULL);
if (ctor_seq != NULL)
{
gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (init));
@@ -3877,7 +3882,8 @@ vect_get_slp_vect_defs (slp_tree slp_node, vec<tree> *vec_oprnds)
vect_get_slp_vect_defs () to retrieve them. */
void
-vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
+vect_get_slp_defs (vec_info *vinfo,
+ slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
{
if (n == -1U)
n = SLP_TREE_CHILDREN (slp_node).length ();
@@ -3896,7 +3902,7 @@ vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
vect_get_slp_vect_defs (child, &vec_defs);
}
else
- vect_get_constant_vectors (slp_node, i, &vec_defs);
+ vect_get_constant_vectors (vinfo, slp_node, i, &vec_defs);
vec_oprnds->quick_push (vec_defs);
}
@@ -3907,13 +3913,13 @@ vect_get_slp_defs (slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
permute statements for the SLP node NODE. */
bool
-vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
+vect_transform_slp_perm_load (vec_info *vinfo,
+ slp_tree node, vec<tree> dr_chain,
gimple_stmt_iterator *gsi, poly_uint64 vf,
bool analyze_only,
unsigned *n_perms)
{
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
- vec_info *vinfo = stmt_info->vinfo;
int vec_index = 0;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
unsigned int group_size = SLP_TREE_SCALAR_STMTS (node).length ();
@@ -4090,7 +4096,8 @@ vect_transform_slp_perm_load (slp_tree node, vec<tree> dr_chain,
first_vec, second_vec,
mask_vec);
perm_stmt_info
- = vect_finish_stmt_generation (stmt_info, perm_stmt,
+ = vect_finish_stmt_generation (vinfo,
+ stmt_info, perm_stmt,
gsi);
}
else
@@ -4243,7 +4250,7 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
}
}
if (!done_p)
- vect_transform_stmt (stmt_info, &si, node, instance);
+ vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
/* Restore stmt def-types. */
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
@@ -4261,7 +4268,8 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance)
SLP instances may refer to the same scalar stmt. */
static void
-vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
+vect_remove_slp_scalar_calls (vec_info *vinfo,
+ slp_tree node, hash_set<slp_tree> &visited)
{
gimple *new_stmt;
gimple_stmt_iterator gsi;
@@ -4277,7 +4285,7 @@ vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
return;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
- vect_remove_slp_scalar_calls (child, visited);
+ vect_remove_slp_scalar_calls (vinfo, child, visited);
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
{
@@ -4290,16 +4298,16 @@ vect_remove_slp_scalar_calls (slp_tree node, hash_set<slp_tree> &visited)
lhs = gimple_call_lhs (stmt);
new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
gsi = gsi_for_stmt (stmt);
- stmt_info->vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
+ vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
}
}
static void
-vect_remove_slp_scalar_calls (slp_tree node)
+vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
{
hash_set<slp_tree> visited;
- vect_remove_slp_scalar_calls (node, visited);
+ vect_remove_slp_scalar_calls (vinfo, node, visited);
}
/* Vectorize the instance root. */
@@ -4390,7 +4398,7 @@ vect_schedule_slp (vec_info *vinfo)
stmts starting from the SLP tree root if they have no
uses. */
if (is_a <loop_vec_info> (vinfo))
- vect_remove_slp_scalar_calls (root);
+ vect_remove_slp_scalar_calls (vinfo, root);
for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
{
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index f34ac151681..438835c7e04 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -70,11 +70,11 @@ stmt_vectype (class _stmt_vec_info *stmt_info)
/* Return TRUE iff the given statement is in an inner loop relative to
the loop being vectorized. */
bool
-stmt_in_inner_loop_p (class _stmt_vec_info *stmt_info)
+stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
{
gimple *stmt = STMT_VINFO_STMT (stmt_info);
basic_block bb = gimple_bb (stmt);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop* loop;
if (!loop_vinfo)
@@ -124,7 +124,8 @@ create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
with scalar destination SCALAR_DEST. */
static tree
-read_vector_array (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+read_vector_array (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree scalar_dest, tree array, unsigned HOST_WIDE_INT n)
{
tree vect_type, vect, vect_name, array_ref;
@@ -140,7 +141,7 @@ read_vector_array (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_stmt = gimple_build_assign (vect, array_ref);
vect_name = make_ssa_name (vect, new_stmt);
gimple_assign_set_lhs (new_stmt, vect_name);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
return vect_name;
}
@@ -150,7 +151,8 @@ read_vector_array (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
The store is part of the vectorization of STMT_INFO. */
static void
-write_vector_array (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+write_vector_array (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vect, tree array, unsigned HOST_WIDE_INT n)
{
tree array_ref;
@@ -161,7 +163,7 @@ write_vector_array (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
NULL_TREE, NULL_TREE);
new_stmt = gimple_build_assign (array_ref, vect);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
/* PTR is a pointer to an array of type TYPE. Return a representation
@@ -183,12 +185,12 @@ create_array_ref (tree type, tree ptr, tree alias_ptr_type)
Emit the clobber before *GSI. */
static void
-vect_clobber_variable (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- tree var)
+vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, tree var)
{
tree clobber = build_clobber (TREE_TYPE (var));
gimple *new_stmt = gimple_build_assign (var, clobber);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
/* Utility functions used by vect_mark_stmts_to_be_vectorized. */
@@ -787,11 +789,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
/* Compute the prologue cost for invariant or constant operands. */
static unsigned
-vect_prologue_cost_for_slp_op (slp_tree node, stmt_vec_info stmt_info,
+vect_prologue_cost_for_slp_op (vec_info *vinfo,
+ slp_tree node, stmt_vec_info stmt_info,
unsigned opno, enum vect_def_type dt,
stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *stmt = SLP_TREE_SCALAR_STMTS (node)[0]->stmt;
tree op = gimple_op (stmt, opno);
unsigned prologue_cost = 0;
@@ -856,7 +858,8 @@ vect_prologue_cost_for_slp_op (slp_tree node, stmt_vec_info stmt_info,
be generated for the single vector op. We will handle that shortly. */
static void
-vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
+vect_model_simple_cost (vec_info *vinfo,
+ stmt_vec_info stmt_info, int ncopies,
enum vect_def_type *dt,
int ndts,
slp_tree node,
@@ -884,9 +887,10 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
enum vect_def_type dt;
if (!op || op == lhs)
continue;
- if (vect_is_simple_use (op, stmt_info->vinfo, &dt)
+ if (vect_is_simple_use (op, vinfo, &dt)
&& (dt == vect_constant_def || dt == vect_external_def))
- prologue_cost += vect_prologue_cost_for_slp_op (node, stmt_info,
+ prologue_cost += vect_prologue_cost_for_slp_op (vinfo, node,
+ stmt_info,
i, dt, cost_vec);
}
}
@@ -990,7 +994,7 @@ cfun_returns (tree decl)
has the overhead of the grouped access attributed to it. */
static void
-vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
+vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
enum vect_def_type dt,
vect_memory_access_type memory_access_type,
vec_load_store_type vls_type, slp_tree slp_node,
@@ -1007,7 +1011,8 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
if (vls_type == VLS_STORE_INVARIANT)
{
if (slp_node)
- prologue_cost += vect_prologue_cost_for_slp_op (slp_node, stmt_info,
+ prologue_cost += vect_prologue_cost_for_slp_op (vinfo, slp_node,
+ stmt_info,
1, dt, cost_vec);
else
prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
@@ -1056,7 +1061,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
scalar_store, stmt_info, 0, vect_body);
}
else
- vect_get_store_cost (stmt_info, ncopies, &inside_cost, cost_vec);
+ vect_get_store_cost (vinfo, stmt_info, ncopies, &inside_cost, cost_vec);
if (memory_access_type == VMAT_ELEMENTWISE
|| memory_access_type == VMAT_STRIDED_SLP)
@@ -1108,13 +1113,13 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
/* Calculate cost of DR's memory access. */
void
-vect_get_store_cost (stmt_vec_info stmt_info, int ncopies,
+vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
unsigned int *inside_cost,
stmt_vector_for_cost *body_cost_vec)
{
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
int alignment_support_scheme
- = vect_supportable_dr_alignment (dr_info, false);
+ = vect_supportable_dr_alignment (vinfo, dr_info, false);
switch (alignment_support_scheme)
{
@@ -1168,7 +1173,8 @@ vect_get_store_cost (stmt_vec_info stmt_info, int ncopies,
access scheme chosen. */
static void
-vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
+vect_model_load_cost (vec_info *vinfo,
+ stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
vect_memory_access_type memory_access_type,
slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
@@ -1191,7 +1197,8 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
unsigned n_perms;
unsigned assumed_nunits
= vect_nunits_for_cost (STMT_VINFO_VECTYPE (first_stmt_info));
- vect_transform_slp_perm_load (slp_node, vNULL, NULL, vf, true, &n_perms);
+ vect_transform_slp_perm_load (vinfo,
+ slp_node, vNULL, NULL, vf, true, &n_perms);
inside_cost += record_stmt_cost (cost_vec, n_perms, vec_perm,
first_stmt_info, 0, vect_body);
/* And adjust the number of loads performed. This handles
@@ -1265,7 +1272,7 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
scalar_load, stmt_info, 0, vect_body);
}
else
- vect_get_load_cost (stmt_info, ncopies, first_stmt_p,
+ vect_get_load_cost (vinfo, stmt_info, ncopies, first_stmt_p,
&inside_cost, &prologue_cost,
cost_vec, cost_vec, true);
if (memory_access_type == VMAT_ELEMENTWISE
@@ -1282,7 +1289,7 @@ vect_model_load_cost (stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
/* Calculate cost of DR's memory access. */
void
-vect_get_load_cost (stmt_vec_info stmt_info, int ncopies,
+vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
bool add_realign_cost, unsigned int *inside_cost,
unsigned int *prologue_cost,
stmt_vector_for_cost *prologue_cost_vec,
@@ -1291,7 +1298,7 @@ vect_get_load_cost (stmt_vec_info stmt_info, int ncopies,
{
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
int alignment_support_scheme
- = vect_supportable_dr_alignment (dr_info, false);
+ = vect_supportable_dr_alignment (vinfo, dr_info, false);
switch (alignment_support_scheme)
{
@@ -1398,14 +1405,14 @@ vect_get_load_cost (stmt_vec_info stmt_info, int ncopies,
the loop preheader for the vectorized stmt STMT_VINFO. */
static void
-vect_init_vector_1 (stmt_vec_info stmt_vinfo, gimple *new_stmt,
+vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
gimple_stmt_iterator *gsi)
{
if (gsi)
- vect_finish_stmt_generation (stmt_vinfo, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
else
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
if (loop_vinfo)
{
@@ -1422,7 +1429,7 @@ vect_init_vector_1 (stmt_vec_info stmt_vinfo, gimple *new_stmt,
}
else
{
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
basic_block bb;
gimple_stmt_iterator gsi_bb_start;
@@ -1449,7 +1456,7 @@ vect_init_vector_1 (stmt_vec_info stmt_vinfo, gimple *new_stmt,
It will be used in the vectorization of STMT_INFO. */
tree
-vect_init_vector (stmt_vec_info stmt_info, tree val, tree type,
+vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
gimple_stmt_iterator *gsi)
{
gimple *init_stmt;
@@ -1475,7 +1482,7 @@ vect_init_vector (stmt_vec_info stmt_info, tree val, tree type,
new_temp = make_ssa_name (TREE_TYPE (type));
init_stmt = gimple_build_assign (new_temp, COND_EXPR,
val, true_val, false_val);
- vect_init_vector_1 (stmt_info, init_stmt, gsi);
+ vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
val = new_temp;
}
}
@@ -1494,7 +1501,7 @@ vect_init_vector (stmt_vec_info stmt_info, tree val, tree type,
{
init_stmt = gsi_stmt (gsi2);
gsi_remove (&gsi2, false);
- vect_init_vector_1 (stmt_info, init_stmt, gsi);
+ vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
}
}
}
@@ -1503,7 +1510,7 @@ vect_init_vector (stmt_vec_info stmt_info, tree val, tree type,
new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
init_stmt = gimple_build_assign (new_temp, val);
- vect_init_vector_1 (stmt_info, init_stmt, gsi);
+ vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
return new_temp;
}
@@ -1575,12 +1582,13 @@ vect_get_vec_def_for_operand_1 (stmt_vec_info def_stmt_info,
vector invariant. */
tree
-vect_get_vec_def_for_operand (tree op, stmt_vec_info stmt_vinfo, tree vectype)
+vect_get_vec_def_for_operand (vec_info *vinfo,
+ tree op, stmt_vec_info stmt_vinfo, tree vectype)
{
gimple *def_stmt;
enum vect_def_type dt;
bool is_simple_use;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -1607,7 +1615,7 @@ vect_get_vec_def_for_operand (tree op, stmt_vec_info stmt_vinfo, tree vectype)
vector_type = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op));
gcc_assert (vector_type);
- return vect_init_vector (stmt_vinfo, op, vector_type, NULL);
+ return vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL);
}
else
return vect_get_vec_def_for_operand_1 (def_stmt_info, dt);
@@ -1712,7 +1720,7 @@ vect_get_vec_defs_for_stmt_copy (vec_info *vinfo,
/* Get vectorized definitions for OP0 and OP1. */
void
-vect_get_vec_defs (tree op0, tree op1, stmt_vec_info stmt_info,
+vect_get_vec_defs (vec_info *vinfo, tree op0, tree op1, stmt_vec_info stmt_info,
vec<tree> *vec_oprnds0,
vec<tree> *vec_oprnds1,
slp_tree slp_node)
@@ -1720,7 +1728,7 @@ vect_get_vec_defs (tree op0, tree op1, stmt_vec_info stmt_info,
if (slp_node)
{
auto_vec<vec<tree> > vec_defs (SLP_TREE_CHILDREN (slp_node).length ());
- vect_get_slp_defs (slp_node, &vec_defs, op1 ? 2 : 1);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs, op1 ? 2 : 1);
*vec_oprnds0 = vec_defs[0];
if (op1)
*vec_oprnds1 = vec_defs[1];
@@ -1730,13 +1738,13 @@ vect_get_vec_defs (tree op0, tree op1, stmt_vec_info stmt_info,
tree vec_oprnd;
vec_oprnds0->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (op0, stmt_info);
+ vec_oprnd = vect_get_vec_def_for_operand (vinfo, op0, stmt_info);
vec_oprnds0->quick_push (vec_oprnd);
if (op1)
{
vec_oprnds1->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (op1, stmt_info);
+ vec_oprnd = vect_get_vec_def_for_operand (vinfo, op1, stmt_info);
vec_oprnds1->quick_push (vec_oprnd);
}
}
@@ -1747,10 +1755,9 @@ vect_get_vec_defs (tree op0, tree op1, stmt_vec_info stmt_info,
statement and create and return a stmt_vec_info for it. */
static stmt_vec_info
-vect_finish_stmt_generation_1 (stmt_vec_info stmt_info, gimple *vec_stmt)
+vect_finish_stmt_generation_1 (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple *vec_stmt)
{
- vec_info *vinfo = stmt_info->vinfo;
-
stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt);
if (dump_enabled_p ())
@@ -1773,7 +1780,8 @@ vect_finish_stmt_generation_1 (stmt_vec_info stmt_info, gimple *vec_stmt)
stmt_vec_info for VEC_STMT. */
stmt_vec_info
-vect_finish_replace_stmt (stmt_vec_info stmt_info, gimple *vec_stmt)
+vect_finish_replace_stmt (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple *vec_stmt)
{
gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
@@ -1781,14 +1789,15 @@ vect_finish_replace_stmt (stmt_vec_info stmt_info, gimple *vec_stmt)
gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
gsi_replace (&gsi, vec_stmt, true);
- return vect_finish_stmt_generation_1 (stmt_info, vec_stmt);
+ return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
}
/* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
stmt_vec_info
-vect_finish_stmt_generation (stmt_vec_info stmt_info, gimple *vec_stmt,
+vect_finish_stmt_generation (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple *vec_stmt,
gimple_stmt_iterator *gsi)
{
gcc_assert (gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
@@ -1820,7 +1829,7 @@ vect_finish_stmt_generation (stmt_vec_info stmt_info, gimple *vec_stmt,
}
}
gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
- return vect_finish_stmt_generation_1 (stmt_info, vec_stmt);
+ return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
}
/* We want to vectorize a call to combined function CFN with function
@@ -1853,7 +1862,7 @@ vectorizable_internal_function (combined_fn cfn, tree fndecl,
}
-static tree permute_vec_elements (tree, tree, tree, stmt_vec_info,
+static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
gimple_stmt_iterator *);
/* Check whether a load or store statement in the loop described by
@@ -2119,10 +2128,10 @@ vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info,
is negative, 0 if it is zero, and 1 if it is greater than zero. */
static int
-compare_step_with_zero (stmt_vec_info stmt_info)
+compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
{
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
- return tree_int_cst_compare (vect_dr_behavior (dr_info)->step,
+ return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
size_zero_node);
}
@@ -2150,7 +2159,8 @@ perm_mask_for_reverse (tree vectype)
accesses consecutive elements with a negative step. */
static vect_memory_access_type
-get_negative_load_store_type (stmt_vec_info stmt_info, tree vectype,
+get_negative_load_store_type (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree vectype,
vec_load_store_type vls_type,
unsigned int ncopies)
{
@@ -2165,7 +2175,8 @@ get_negative_load_store_type (stmt_vec_info stmt_info, tree vectype,
return VMAT_ELEMENTWISE;
}
- alignment_support_scheme = vect_supportable_dr_alignment (dr_info, false);
+ alignment_support_scheme = vect_supportable_dr_alignment (vinfo,
+ dr_info, false);
if (alignment_support_scheme != dr_aligned
&& alignment_support_scheme != dr_unaligned_supported)
{
@@ -2226,13 +2237,13 @@ vect_get_store_rhs (stmt_vec_info stmt_info)
as well as at the end. */
static bool
-get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
+get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree vectype, bool slp,
bool masked_p, vec_load_store_type vls_type,
vect_memory_access_type *memory_access_type,
gather_scatter_info *gs_info)
{
- vec_info *vinfo = stmt_info->vinfo;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
@@ -2301,7 +2312,8 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
if (overrun_p
&& !masked_p
&& (((alignment_support_scheme
- = vect_supportable_dr_alignment (first_dr_info, false)))
+ = vect_supportable_dr_alignment (vinfo,
+ first_dr_info, false)))
== dr_aligned
|| alignment_support_scheme == dr_unaligned_supported)
&& known_eq (nunits, (group_size - gap) * 2)
@@ -2321,10 +2333,10 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
"Peeling for outer loop is not supported\n");
return false;
}
- int cmp = compare_step_with_zero (stmt_info);
+ int cmp = compare_step_with_zero (vinfo, stmt_info);
if (cmp < 0)
*memory_access_type = get_negative_load_store_type
- (stmt_info, vectype, vls_type, 1);
+ (vinfo, stmt_info, vectype, vls_type, 1);
else
{
gcc_assert (!loop_vinfo || cmp > 0);
@@ -2353,7 +2365,7 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
if (!STMT_VINFO_STRIDED_P (first_stmt_info)
&& (can_overrun_p || !would_overrun_p)
- && compare_step_with_zero (stmt_info) > 0)
+ && compare_step_with_zero (vinfo, stmt_info) > 0)
{
/* First cope with the degenerate case of a single-element
vector. */
@@ -2442,14 +2454,14 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
NCOPIES is the number of vector statements that will be needed. */
static bool
-get_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
+get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree vectype, bool slp,
bool masked_p, vec_load_store_type vls_type,
unsigned int ncopies,
vect_memory_access_type *memory_access_type,
gather_scatter_info *gs_info)
{
- vec_info *vinfo = stmt_info->vinfo;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
@@ -2469,7 +2481,7 @@ get_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
}
else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
{
- if (!get_group_load_store_type (stmt_info, vectype, slp, masked_p,
+ if (!get_group_load_store_type (vinfo, stmt_info, vectype, slp, masked_p,
vls_type, memory_access_type, gs_info))
return false;
}
@@ -2485,10 +2497,10 @@ get_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
}
else
{
- int cmp = compare_step_with_zero (stmt_info);
+ int cmp = compare_step_with_zero (vinfo, stmt_info);
if (cmp < 0)
*memory_access_type = get_negative_load_store_type
- (stmt_info, vectype, vls_type, ncopies);
+ (vinfo, stmt_info, vectype, vls_type, ncopies);
else if (cmp == 0)
{
gcc_assert (vls_type == VLS_LOAD);
@@ -2535,11 +2547,10 @@ get_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
in *MASK_VECTYPE_OUT. */
static bool
-vect_check_scalar_mask (stmt_vec_info stmt_info, tree mask,
+vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, tree mask,
vect_def_type *mask_dt_out,
tree *mask_vectype_out)
{
- vec_info *vinfo = stmt_info->vinfo;
if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask)))
{
if (dump_enabled_p ())
@@ -2558,7 +2569,7 @@ vect_check_scalar_mask (stmt_vec_info stmt_info, tree mask,
enum vect_def_type mask_dt;
tree mask_vectype;
- if (!vect_is_simple_use (mask, stmt_info->vinfo, &mask_dt, &mask_vectype))
+ if (!vect_is_simple_use (mask, vinfo, &mask_dt, &mask_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2601,7 +2612,7 @@ vect_check_scalar_mask (stmt_vec_info stmt_info, tree mask,
*RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
static bool
-vect_check_store_rhs (stmt_vec_info stmt_info, tree rhs,
+vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, tree rhs,
vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
vec_load_store_type *vls_type_out)
{
@@ -2617,7 +2628,7 @@ vect_check_store_rhs (stmt_vec_info stmt_info, tree rhs,
enum vect_def_type rhs_dt;
tree rhs_vectype;
- if (!vect_is_simple_use (rhs, stmt_info->vinfo, &rhs_dt, &rhs_vectype))
+ if (!vect_is_simple_use (rhs, vinfo, &rhs_dt, &rhs_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2648,7 +2659,8 @@ vect_check_store_rhs (stmt_vec_info stmt_info, tree rhs,
floats are interpreted as a bitmask. */
static tree
-vect_build_all_ones_mask (stmt_vec_info stmt_info, tree masktype)
+vect_build_all_ones_mask (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree masktype)
{
if (TREE_CODE (masktype) == INTEGER_TYPE)
return build_int_cst (masktype, -1);
@@ -2656,7 +2668,7 @@ vect_build_all_ones_mask (stmt_vec_info stmt_info, tree masktype)
{
tree mask = build_int_cst (TREE_TYPE (masktype), -1);
mask = build_vector_from_val (masktype, mask);
- return vect_init_vector (stmt_info, mask, masktype, NULL);
+ return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
}
else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
{
@@ -2667,7 +2679,7 @@ vect_build_all_ones_mask (stmt_vec_info stmt_info, tree masktype)
real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
tree mask = build_real (TREE_TYPE (masktype), r);
mask = build_vector_from_val (masktype, mask);
- return vect_init_vector (stmt_info, mask, masktype, NULL);
+ return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
}
gcc_unreachable ();
}
@@ -2676,7 +2688,8 @@ vect_build_all_ones_mask (stmt_vec_info stmt_info, tree masktype)
STMT_INFO as a gather load. */
static tree
-vect_build_zero_merge_argument (stmt_vec_info stmt_info, tree vectype)
+vect_build_zero_merge_argument (vec_info *vinfo,
+ stmt_vec_info stmt_info, tree vectype)
{
tree merge;
if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
@@ -2693,7 +2706,7 @@ vect_build_zero_merge_argument (stmt_vec_info stmt_info, tree vectype)
else
gcc_unreachable ();
merge = build_vector_from_val (vectype, merge);
- return vect_init_vector (stmt_info, merge, vectype, NULL);
+ return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
}
/* Build a gather load call while vectorizing STMT_INFO. Insert new
@@ -2703,13 +2716,13 @@ vect_build_zero_merge_argument (stmt_vec_info stmt_info, tree vectype)
MASK is null. */
static void
-vect_build_gather_load_calls (stmt_vec_info stmt_info,
+vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt,
gather_scatter_info *gs_info,
tree mask)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
@@ -2806,19 +2819,19 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
if (!mask)
{
- src_op = vect_build_zero_merge_argument (stmt_info, rettype);
- mask_op = vect_build_all_ones_mask (stmt_info, masktype);
+ src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
+ mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
}
for (int j = 0; j < ncopies; ++j)
{
tree op, var;
if (modifier == WIDEN && (j & 1))
- op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
+ op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
perm_mask, stmt_info, gsi);
else if (j == 0)
op = vec_oprnd0
- = vect_get_vec_def_for_operand (gs_info->offset, stmt_info);
+ = vect_get_vec_def_for_operand (vinfo, gs_info->offset, stmt_info);
else
op = vec_oprnd0 = vect_get_vec_def_for_stmt_copy (loop_vinfo,
vec_oprnd0);
@@ -2830,19 +2843,19 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
var = vect_get_new_ssa_name (idxtype, vect_simple_var);
op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
op = var;
}
if (mask)
{
if (mask_perm_mask && (j & 1))
- mask_op = permute_vec_elements (mask_op, mask_op,
+ mask_op = permute_vec_elements (vinfo, mask_op, mask_op,
mask_perm_mask, stmt_info, gsi);
else
{
if (j == 0)
- vec_mask = vect_get_vec_def_for_operand (mask, stmt_info);
+ vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info);
else if (modifier != NARROW || (j & 1) == 0)
vec_mask = vect_get_vec_def_for_stmt_copy (loop_vinfo,
vec_mask);
@@ -2857,7 +2870,7 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op);
gassign *new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_op);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_op = var;
}
}
@@ -2868,7 +2881,7 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
= gimple_build_assign (var, (j & 1) ? VEC_UNPACK_HI_EXPR
: VEC_UNPACK_LO_EXPR,
mask_op);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_op = var;
}
src_op = mask_op;
@@ -2886,7 +2899,7 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_op);
gassign *new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_arg = var;
if (!useless_type_conversion_p (real_masktype, utype))
{
@@ -2894,7 +2907,7 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
<= TYPE_PRECISION (real_masktype));
var = vect_get_new_ssa_name (real_masktype, vect_scalar_var);
new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_arg = var;
}
src_op = build_zero_cst (srctype);
@@ -2909,19 +2922,19 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
TYPE_VECTOR_SUBPARTS (rettype)));
op = vect_get_new_ssa_name (rettype, vect_simple_var);
gimple_call_set_lhs (new_call, op);
- vect_finish_stmt_generation (stmt_info, new_call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
var = make_ssa_name (vec_dest);
op = build1 (VIEW_CONVERT_EXPR, vectype, op);
gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
var = make_ssa_name (vec_dest, new_call);
gimple_call_set_lhs (new_call, var);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
}
if (modifier == NARROW)
@@ -2931,7 +2944,7 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
prev_res = var;
continue;
}
- var = permute_vec_elements (prev_res, var, perm_mask,
+ var = permute_vec_elements (vinfo, prev_res, var, perm_mask,
stmt_info, gsi);
new_stmt_info = loop_vinfo->lookup_def (var);
}
@@ -2951,7 +2964,8 @@ vect_build_gather_load_calls (stmt_vec_info stmt_info,
containing loop. */
static void
-vect_get_gather_scatter_ops (class loop *loop, stmt_vec_info stmt_info,
+vect_get_gather_scatter_ops (vec_info *vinfo,
+ class loop *loop, stmt_vec_info stmt_info,
gather_scatter_info *gs_info,
tree *dataref_ptr, tree *vec_offset)
{
@@ -2964,7 +2978,7 @@ vect_get_gather_scatter_ops (class loop *loop, stmt_vec_info stmt_info,
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
gcc_assert (!new_bb);
}
- *vec_offset = vect_get_vec_def_for_operand (gs_info->offset, stmt_info,
+ *vec_offset = vect_get_vec_def_for_operand (vinfo, gs_info->offset, stmt_info,
gs_info->offset_vectype);
}
@@ -3019,14 +3033,15 @@ vect_get_strided_load_store_ops (stmt_vec_info stmt_info,
vectorization. */
static tree
-vect_get_data_ptr_increment (dr_vec_info *dr_info, tree aggr_type,
+vect_get_data_ptr_increment (vec_info *vinfo,
+ dr_vec_info *dr_info, tree aggr_type,
vect_memory_access_type memory_access_type)
{
if (memory_access_type == VMAT_INVARIANT)
return size_zero_node;
tree iv_step = TYPE_SIZE_UNIT (aggr_type);
- tree step = vect_dr_behavior (dr_info)->step;
+ tree step = vect_dr_behavior (vinfo, dr_info)->step;
if (tree_int_cst_sgn (step) == -1)
iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
return iv_step;
@@ -3035,14 +3050,14 @@ vect_get_data_ptr_increment (dr_vec_info *dr_info, tree aggr_type,
/* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */
static bool
-vectorizable_bswap (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_bswap (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
tree vectype_in, stmt_vector_for_cost *cost_vec)
{
tree op, vectype;
gcall *stmt = as_a <gcall *> (stmt_info->stmt);
- vec_info *vinfo = stmt_info->vinfo;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
unsigned ncopies;
op = gimple_call_arg (stmt, 0);
@@ -3102,7 +3117,8 @@ vectorizable_bswap (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
/* Handle uses. */
if (j == 0)
- vect_get_vec_defs (op, NULL, stmt_info, &vec_oprnds, NULL, slp_node);
+ vect_get_vec_defs (vinfo, op, NULL, stmt_info, &vec_oprnds, NULL,
+ slp_node);
else
vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds, NULL);
@@ -3115,16 +3131,16 @@ vectorizable_bswap (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree tem = make_ssa_name (char_vectype);
new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
char_vectype, vop));
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
tree tem2 = make_ssa_name (char_vectype);
new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
tem, tem, bswap_vconst);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
tem = make_ssa_name (vectype);
new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
vectype, tem2));
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
@@ -3177,7 +3193,8 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_call (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
@@ -3190,9 +3207,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vectype_out, vectype_in;
poly_uint64 nunits_in;
poly_uint64 nunits_out;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
tree fndecl, new_temp, rhs_type;
enum vect_def_type dt[4]
= { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
@@ -3261,7 +3277,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if ((int) i == mask_opno)
{
- if (!vect_check_scalar_mask (stmt_info, op, &dt[i], &vectypes[i]))
+ if (!vect_check_scalar_mask (vinfo,
+ stmt_info, op, &dt[i], &vectypes[i]))
return false;
continue;
}
@@ -3402,7 +3419,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
&& (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
|| gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
|| gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)))
- return vectorizable_bswap (stmt_info, gsi, vec_stmt, slp_node,
+ return vectorizable_bswap (vinfo, stmt_info, gsi, vec_stmt, slp_node,
vectype_in, cost_vec);
else
{
@@ -3429,7 +3446,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_call");
- vect_model_simple_cost (stmt_info, ncopies, dt, ndts, slp_node, cost_vec);
+ vect_model_simple_cost (vinfo, stmt_info,
+ ncopies, dt, ndts, slp_node, cost_vec);
if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
record_stmt_cost (cost_vec, ncopies / 2,
vec_promote_demote, stmt_info, 0, vect_body);
@@ -3472,7 +3490,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
auto_vec<vec<tree> > vec_defs (nargs);
vec<tree> vec_oprnds0;
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_oprnds0 = vec_defs[0];
/* Arguments are ready. Create the new vector stmt. */
@@ -3494,7 +3512,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= gimple_build_call_internal_vec (ifn, vargs);
gimple_call_set_lhs (call, half_res);
gimple_call_set_nothrow (call, true);
- vect_finish_stmt_generation (stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
if ((i & 1) == 0)
{
prev_res = half_res;
@@ -3505,8 +3523,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= gimple_build_assign (new_temp, convert_code,
prev_res, half_res);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt,
- gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
else
{
@@ -3530,7 +3548,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ call, gsi);
}
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
@@ -3548,7 +3567,8 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
op = gimple_call_arg (stmt, i);
if (j == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt_info, vectypes[i]);
+ = vect_get_vec_def_for_operand (vinfo,
+ op, stmt_info, vectypes[i]);
else
vec_oprnd0
= vect_get_vec_def_for_stmt_copy (vinfo, orig_vargs[i]);
@@ -3571,11 +3591,11 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree new_var
= vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
gimple *init_stmt = gimple_build_assign (new_var, cst);
- vect_init_vector_1 (stmt_info, init_stmt, NULL);
+ vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
new_temp = make_ssa_name (vec_dest);
gimple *new_stmt = gimple_build_assign (new_temp, new_var);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else if (modifier == NARROW)
{
@@ -3586,7 +3606,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcall *call = gimple_build_call_internal_vec (ifn, vargs);
gimple_call_set_lhs (call, half_res);
gimple_call_set_nothrow (call, true);
- vect_finish_stmt_generation (stmt_info, call, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
if ((j & 1) == 0)
{
prev_res = half_res;
@@ -3596,7 +3616,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt = gimple_build_assign (new_temp, convert_code,
prev_res, half_res);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
@@ -3609,7 +3629,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
}
if (j == (modifier == NARROW ? 1 : 0))
@@ -3637,7 +3657,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
auto_vec<vec<tree> > vec_defs (nargs);
vec<tree> vec_oprnds0;
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_oprnds0 = vec_defs[0];
/* Arguments are ready. Create the new vector stmt. */
@@ -3660,7 +3680,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
@@ -3678,7 +3698,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (j == 0)
{
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt_info,
+ = vect_get_vec_def_for_operand (vinfo, op, stmt_info,
vectypes[i]);
vec_oprnd1
= vect_get_vec_def_for_stmt_copy (vinfo, vec_oprnd0);
@@ -3701,7 +3721,7 @@ vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_call_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (j == 0)
STMT_VINFO_VEC_STMT (stmt_info) = new_stmt_info;
@@ -3835,7 +3855,7 @@ simd_clone_subparts (tree vectype)
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_simd_clone_call (stmt_vec_info stmt_info,
+vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *)
@@ -3847,9 +3867,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
stmt_vec_info prev_stmt_info;
tree vectype;
unsigned int nunits;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
tree fndecl, new_temp;
int ncopies, j;
@@ -4146,7 +4165,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
}
STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
-/* vect_model_simple_cost (stmt_info, ncopies, dt, slp_node, cost_vec); */
+/* vect_model_simple_cost (vinfo, stmt_info, ncopies,
+ dt, slp_node, cost_vec); */
return true;
}
@@ -4201,7 +4221,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gcc_assert ((k & (k - 1)) == 0);
if (m == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt_info);
+ = vect_get_vec_def_for_operand (vinfo, op, stmt_info);
else
{
vec_oprnd0 = arginfo[i].op;
@@ -4218,7 +4238,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gassign *new_stmt
= gimple_build_assign (make_ssa_name (atype),
vec_oprnd0);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
vargs.safe_push (gimple_assign_lhs (new_stmt));
}
else
@@ -4235,7 +4256,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
{
if (m == 0 && l == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt_info);
+ = vect_get_vec_def_for_operand (vinfo,
+ op, stmt_info);
else
vec_oprnd0
= vect_get_vec_def_for_stmt_copy (vinfo,
@@ -4254,8 +4276,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gassign *new_stmt
= gimple_build_assign (make_ssa_name (atype),
vec_oprnd0);
- vect_finish_stmt_generation (stmt_info, new_stmt,
- gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
vargs.safe_push (gimple_assign_lhs (new_stmt));
}
}
@@ -4324,7 +4346,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gassign *new_stmt
= gimple_build_assign (new_temp, code,
arginfo[i].op, tcst);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
vargs.safe_push (new_temp);
}
break;
@@ -4353,7 +4375,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple_call_set_lhs (new_call, new_temp);
}
stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
if (vec_dest)
{
@@ -4379,7 +4401,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple *new_stmt
= gimple_build_assign (make_ssa_name (vectype), t);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
if (j == 0 && l == 0)
STMT_VINFO_VEC_STMT (stmt_info)
@@ -4391,7 +4414,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
}
if (ratype)
- vect_clobber_variable (stmt_info, gsi, new_temp);
+ vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
continue;
}
else if (simd_clone_subparts (vectype) > nunits)
@@ -4411,12 +4434,12 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple *new_stmt
= gimple_build_assign (make_ssa_name (rtype), tem);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt,
- gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
gimple_assign_lhs (new_stmt));
}
- vect_clobber_variable (stmt_info, gsi, new_temp);
+ vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
}
else
CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
@@ -4426,7 +4449,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple *new_stmt
= gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if ((unsigned) j == k - 1)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
@@ -4444,8 +4467,8 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple *new_stmt
= gimple_build_assign (make_ssa_name (vec_dest), t);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
- vect_clobber_variable (stmt_info, gsi, new_temp);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+ vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
}
}
@@ -4493,7 +4516,7 @@ vectorizable_simd_clone_call (stmt_vec_info stmt_info,
STMT_INFO is the original scalar stmt that we are vectorizing. */
static gimple *
-vect_gen_widened_results_half (enum tree_code code,
+vect_gen_widened_results_half (vec_info *vinfo, enum tree_code code,
tree vec_oprnd0, tree vec_oprnd1, int op_type,
tree vec_dest, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info)
@@ -4508,7 +4531,7 @@ vect_gen_widened_results_half (enum tree_code code,
new_stmt = gimple_build_assign (vec_dest, code, vec_oprnd0, vec_oprnd1);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
return new_stmt;
}
@@ -4522,17 +4545,16 @@ vect_gen_widened_results_half (enum tree_code code,
The vectors are collected into VEC_OPRNDS. */
static void
-vect_get_loop_based_defs (tree *oprnd, stmt_vec_info stmt_info,
+vect_get_loop_based_defs (vec_info *vinfo, tree *oprnd, stmt_vec_info stmt_info,
vec<tree> *vec_oprnds, int multi_step_cvt)
{
- vec_info *vinfo = stmt_info->vinfo;
tree vec_oprnd;
/* Get first vector operand. */
/* All the vector operands except the very first one (that is scalar oprnd)
are stmt copies. */
if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
- vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt_info);
+ vec_oprnd = vect_get_vec_def_for_operand (vinfo, *oprnd, stmt_info);
else
vec_oprnd = vect_get_vec_def_for_stmt_copy (vinfo, *oprnd);
@@ -4547,7 +4569,7 @@ vect_get_loop_based_defs (tree *oprnd, stmt_vec_info stmt_info,
/* For conversion in multiple steps, continue to get operands
recursively. */
if (multi_step_cvt)
- vect_get_loop_based_defs (oprnd, stmt_info, vec_oprnds,
+ vect_get_loop_based_defs (vinfo, oprnd, stmt_info, vec_oprnds,
multi_step_cvt - 1);
}
@@ -4557,7 +4579,7 @@ vect_get_loop_based_defs (tree *oprnd, stmt_vec_info stmt_info,
recursively. */
static void
-vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
+vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
int multi_step_cvt,
stmt_vec_info stmt_info,
vec<tree> vec_dsts,
@@ -4579,7 +4601,7 @@ vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
new_tmp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_tmp);
stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (multi_step_cvt)
/* Store the resulting vector for next recursive call. */
@@ -4612,7 +4634,8 @@ vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
/* At each level of recursion we have half of the operands we had at the
previous level. */
vec_oprnds->truncate ((i+1)/2);
- vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
+ vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
+ multi_step_cvt - 1,
stmt_info, vec_dsts, gsi,
slp_node, VEC_PACK_TRUNC_EXPR,
prev_stmt_info);
@@ -4628,7 +4651,8 @@ vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
call the function recursively. */
static void
-vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
+vect_create_vectorized_promotion_stmts (vec_info *vinfo,
+ vec<tree> *vec_oprnds0,
vec<tree> *vec_oprnds1,
stmt_vec_info stmt_info, tree vec_dest,
gimple_stmt_iterator *gsi,
@@ -4649,10 +4673,10 @@ vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
vop1 = NULL_TREE;
/* Generate the two halves of promotion operation. */
- new_stmt1 = vect_gen_widened_results_half (code1, vop0, vop1,
+ new_stmt1 = vect_gen_widened_results_half (vinfo, code1, vop0, vop1,
op_type, vec_dest, gsi,
stmt_info);
- new_stmt2 = vect_gen_widened_results_half (code2, vop0, vop1,
+ new_stmt2 = vect_gen_widened_results_half (vinfo, code2, vop0, vop1,
op_type, vec_dest, gsi,
stmt_info);
if (is_gimple_call (new_stmt1))
@@ -4682,7 +4706,8 @@ vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_conversion (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
@@ -4690,7 +4715,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree scalar_dest;
tree op0, op1 = NULL_TREE;
tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
tree new_temp;
@@ -4706,8 +4731,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
tree vop0;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
int multi_step_cvt = 0;
vec<tree> interm_types = vNULL;
tree last_oprnd, intermediate_type, cvt_type = NULL_TREE;
@@ -4873,7 +4897,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return false;
case WIDEN:
- if (supportable_widening_operation (code, stmt_info, vectype_out,
+ if (supportable_widening_operation (vinfo, code, stmt_info, vectype_out,
vectype_in, &code1, &code2,
&multi_step_cvt, &interm_types))
{
@@ -4906,7 +4930,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
cvt_type, &codecvt1))
goto unsupported;
}
- else if (!supportable_widening_operation (code, stmt_info,
+ else if (!supportable_widening_operation (vinfo, code, stmt_info,
vectype_out, cvt_type,
&codecvt1, &codecvt2,
&multi_step_cvt,
@@ -4915,7 +4939,8 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else
gcc_assert (multi_step_cvt == 0);
- if (supportable_widening_operation (NOP_EXPR, stmt_info, cvt_type,
+ if (supportable_widening_operation (vinfo, NOP_EXPR, stmt_info,
+ cvt_type,
vectype_in, &code1, &code2,
&multi_step_cvt, &interm_types))
{
@@ -4972,7 +4997,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (modifier == NONE)
{
STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
- vect_model_simple_cost (stmt_info, ncopies, dt, ndts, slp_node,
+ vect_model_simple_cost (vinfo, stmt_info, ncopies, dt, ndts, slp_node,
cost_vec);
}
else if (modifier == NARROW)
@@ -5064,7 +5089,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
for (j = 0; j < ncopies; j++)
{
if (j == 0)
- vect_get_vec_defs (op0, NULL, stmt_info, &vec_oprnds0,
+ vect_get_vec_defs (vinfo, op0, NULL, stmt_info, &vec_oprnds0,
NULL, slp_node);
else
vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds0, NULL);
@@ -5078,7 +5103,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
@@ -5118,16 +5143,17 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
vec_oprnds1.quick_push (vec_oprnd1);
- vect_get_vec_defs (op0, NULL_TREE, stmt_info,
+ vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info,
&vec_oprnds0, NULL, slp_node);
}
else
- vect_get_vec_defs (op0, op1, stmt_info, &vec_oprnds0,
+ vect_get_vec_defs (vinfo, op0, op1, stmt_info, &vec_oprnds0,
&vec_oprnds1, slp_node);
}
else
{
- vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt_info);
+ vec_oprnd0 = vect_get_vec_def_for_operand (vinfo,
+ op0, stmt_info);
vec_oprnds0.quick_push (vec_oprnd0);
if (op_type == binary_op)
{
@@ -5135,7 +5161,8 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vec_oprnd1 = op1;
else
vec_oprnd1
- = vect_get_vec_def_for_operand (op1, stmt_info);
+ = vect_get_vec_def_for_operand (vinfo,
+ op1, stmt_info);
vec_oprnds1.quick_push (vec_oprnd1);
}
}
@@ -5167,7 +5194,7 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
c1 = codecvt1;
c2 = codecvt2;
}
- vect_create_vectorized_promotion_stmts (&vec_oprnds0,
+ vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
&vec_oprnds1, stmt_info,
this_dest, gsi,
c1, c2, op_type);
@@ -5183,7 +5210,8 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt
= gimple_build_assign (new_temp, codecvt1, vop0);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
else
new_stmt_info = vinfo->lookup_def (vop0);
@@ -5213,12 +5241,13 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
/* Handle uses. */
if (slp_node)
- vect_get_vec_defs (op0, NULL_TREE, stmt_info, &vec_oprnds0, NULL,
- slp_node);
+ vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info, &vec_oprnds0,
+ NULL, slp_node);
else
{
vec_oprnds0.truncate (0);
- vect_get_loop_based_defs (&last_oprnd, stmt_info, &vec_oprnds0,
+ vect_get_loop_based_defs (vinfo,
+ &last_oprnd, stmt_info, &vec_oprnds0,
vect_pow2 (multi_step_cvt) - 1);
}
@@ -5230,11 +5259,12 @@ vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest);
gassign *new_stmt
= gimple_build_assign (new_temp, codecvt1, vop0);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
vec_oprnds0[i] = new_temp;
}
- vect_create_vectorized_demotion_stmts (&vec_oprnds0, multi_step_cvt,
+ vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
+ multi_step_cvt,
stmt_info, vec_dsts, gsi,
slp_node, code1,
&prev_stmt_info);
@@ -5283,14 +5313,15 @@ vect_nop_conversion_p (stmt_vec_info stmt_info)
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_assignment (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
tree vec_dest;
tree scalar_dest;
tree op;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
tree new_temp;
enum vect_def_type dt[1] = {vect_unknown_def_type};
int ndts = 1;
@@ -5298,8 +5329,7 @@ vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
int i, j;
vec<tree> vec_oprnds = vNULL;
tree vop;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
stmt_vec_info prev_stmt_info = NULL;
enum tree_code code;
tree vectype_in;
@@ -5390,7 +5420,7 @@ vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_assignment");
if (!vect_nop_conversion_p (stmt_info))
- vect_model_simple_cost (stmt_info, ncopies, dt, ndts, slp_node,
+ vect_model_simple_cost (vinfo, stmt_info, ncopies, dt, ndts, slp_node,
cost_vec);
return true;
}
@@ -5407,7 +5437,8 @@ vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
/* Handle uses. */
if (j == 0)
- vect_get_vec_defs (op, NULL, stmt_info, &vec_oprnds, NULL, slp_node);
+ vect_get_vec_defs (vinfo, op, NULL, stmt_info, &vec_oprnds, NULL,
+ slp_node);
else
vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds, NULL);
@@ -5422,7 +5453,7 @@ vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
@@ -5487,7 +5518,8 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_shift (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
@@ -5496,7 +5528,7 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree op0, op1 = NULL;
tree vec_oprnd1 = NULL_TREE;
tree vectype;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum tree_code code;
machine_mode vec_mode;
tree new_temp;
@@ -5517,8 +5549,7 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vop0, vop1;
unsigned int k;
bool scalar_shift_arg = true;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
bool incompatible_op1_vectype_p = false;
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
@@ -5758,7 +5789,7 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_shift");
- vect_model_simple_cost (stmt_info, ncopies, dt,
+ vect_model_simple_cost (vinfo, stmt_info, ncopies, dt,
scalar_shift_arg ? 1 : ndts, slp_node, cost_vec);
return true;
}
@@ -5773,7 +5804,7 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
op1 = fold_convert (TREE_TYPE (vectype), op1);
if (dt[1] != vect_constant_def)
- op1 = vect_init_vector (stmt_info, op1,
+ op1 = vect_init_vector (vinfo, stmt_info, op1,
TREE_TYPE (vectype), NULL);
}
@@ -5820,9 +5851,10 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* If the argument was the same in all lanes create
the correctly typed vector shift amount directly. */
op1 = fold_convert (TREE_TYPE (vectype), op1);
- op1 = vect_init_vector (stmt_info, op1, TREE_TYPE (vectype),
+ op1 = vect_init_vector (vinfo, stmt_info,
+ op1, TREE_TYPE (vectype),
!loop_vinfo ? gsi : NULL);
- vec_oprnd1 = vect_init_vector (stmt_info, op1, vectype,
+ vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
!loop_vinfo ? gsi : NULL);
vec_oprnds1.create (slp_node->vec_stmts_size);
for (k = 0; k < slp_node->vec_stmts_size; k++)
@@ -5851,11 +5883,11 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
(a special case for certain kind of vector shifts); otherwise,
operand 1 should be of a vector type (the usual case). */
if (vec_oprnd1)
- vect_get_vec_defs (op0, NULL_TREE, stmt_info, &vec_oprnds0, NULL,
- slp_node);
+ vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info,
+ &vec_oprnds0, NULL, slp_node);
else
- vect_get_vec_defs (op0, op1, stmt_info, &vec_oprnds0, &vec_oprnds1,
- slp_node);
+ vect_get_vec_defs (vinfo, op0, op1, stmt_info,
+ &vec_oprnds0, &vec_oprnds1, slp_node);
}
else
vect_get_vec_defs_for_stmt_copy (vinfo, &vec_oprnds0, &vec_oprnds1);
@@ -5869,7 +5901,7 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
}
@@ -5900,7 +5932,8 @@ vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_operation (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
@@ -5908,7 +5941,7 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree scalar_dest;
tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
tree vectype;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum tree_code code, orig_code;
machine_mode vec_mode;
tree new_temp;
@@ -5928,8 +5961,7 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vec<tree> vec_oprnds1 = vNULL;
vec<tree> vec_oprnds2 = vNULL;
tree vop0, vop1, vop2;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
@@ -6178,7 +6210,8 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
DUMP_VECT_SCOPE ("vectorizable_operation");
- vect_model_simple_cost (stmt_info, ncopies, dt, ndts, slp_node, cost_vec);
+ vect_model_simple_cost (vinfo, stmt_info,
+ ncopies, dt, ndts, slp_node, cost_vec);
return true;
}
@@ -6264,29 +6297,29 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (j == 0)
{
if (op_type == binary_op)
- vect_get_vec_defs (op0, op1, stmt_info, &vec_oprnds0, &vec_oprnds1,
- slp_node);
+ vect_get_vec_defs (vinfo, op0, op1, stmt_info,
+ &vec_oprnds0, &vec_oprnds1, slp_node);
else if (op_type == ternary_op)
{
if (slp_node)
{
auto_vec<vec<tree> > vec_defs(3);
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_oprnds0 = vec_defs[0];
vec_oprnds1 = vec_defs[1];
vec_oprnds2 = vec_defs[2];
}
else
{
- vect_get_vec_defs (op0, op1, stmt_info, &vec_oprnds0,
+ vect_get_vec_defs (vinfo, op0, op1, stmt_info, &vec_oprnds0,
&vec_oprnds1, NULL);
- vect_get_vec_defs (op2, NULL_TREE, stmt_info, &vec_oprnds2,
- NULL, NULL);
+ vect_get_vec_defs (vinfo, op2, NULL_TREE, stmt_info,
+ &vec_oprnds2, NULL, NULL);
}
}
else
- vect_get_vec_defs (op0, NULL_TREE, stmt_info, &vec_oprnds0, NULL,
- slp_node);
+ vect_get_vec_defs (vinfo, op0, NULL_TREE, stmt_info, &vec_oprnds0,
+ NULL, slp_node);
}
else
{
@@ -6321,7 +6354,7 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, new_temp);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
}
else
{
@@ -6330,7 +6363,7 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (vec_cvt_dest)
{
new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
@@ -6339,8 +6372,8 @@ vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp);
new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
- new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
if (slp_node)
@@ -6608,11 +6641,11 @@ scan_store_can_perm_p (tree vectype, tree init,
Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
static bool
-check_scan_store (stmt_vec_info stmt_info, tree vectype,
+check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
enum vect_def_type rhs_dt, bool slp, tree mask,
vect_memory_access_type memory_access_type)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
tree ref_type;
@@ -6625,7 +6658,7 @@ check_scan_store (stmt_vec_info stmt_info, tree vectype,
|| loop_vinfo == NULL
|| LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
|| STMT_VINFO_GROUPED_ACCESS (stmt_info)
- || !integer_zerop (get_dr_vinfo_offset (dr_info))
+ || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
|| !integer_zerop (DR_INIT (dr_info->dr))
|| !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
|| !alias_sets_conflict_p (get_alias_set (vectype),
@@ -7012,13 +7045,13 @@ check_scan_store (stmt_vec_info stmt_info, tree vectype,
Handle only the transformation, checking is done in check_scan_store. */
static bool
-vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_scan_store (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, int ncopies)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
- vec_info *vinfo = stmt_info->vinfo;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
if (dump_enabled_p ())
@@ -7120,7 +7153,8 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vec_oprnd3 = NULL_TREE;
tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
tree dataref_offset = build_int_cst (ref_type, 0);
- tree bump = vect_get_data_ptr_increment (dr_info, vectype, VMAT_CONTIGUOUS);
+ tree bump = vect_get_data_ptr_increment (vinfo, dr_info,
+ vectype, VMAT_CONTIGUOUS);
tree ldataref_ptr = NULL_TREE;
tree orig = NULL_TREE;
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
@@ -7130,10 +7164,10 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info new_stmt_info;
if (j == 0)
{
- vec_oprnd1 = vect_get_vec_def_for_operand (*init, stmt_info);
+ vec_oprnd1 = vect_get_vec_def_for_operand (vinfo, *init, stmt_info);
if (ldataref_ptr == NULL)
- vec_oprnd2 = vect_get_vec_def_for_operand (rhs1, stmt_info);
- vec_oprnd3 = vect_get_vec_def_for_operand (rhs2, stmt_info);
+ vec_oprnd2 = vect_get_vec_def_for_operand (vinfo, rhs1, stmt_info);
+ vec_oprnd3 = vect_get_vec_def_for_operand (vinfo, rhs2, stmt_info);
orig = vec_oprnd3;
}
else
@@ -7154,7 +7188,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
dataref_offset);
vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
if (prev_stmt_info == NULL)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
else
@@ -7172,7 +7206,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
!= scan_store_kind_perm))
? zero_vec : vec_oprnd1, v,
perms[i]);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
if (prev_stmt_info == NULL)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
else
@@ -7192,7 +7226,8 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree new_temp2 = make_ssa_name (vectype);
g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
new_temp, vec_oprnd1);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
new_temp = new_temp2;
@@ -7211,7 +7246,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree new_temp2 = make_ssa_name (vectype);
g = gimple_build_assign (new_temp2, code, v, new_temp);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
@@ -7220,7 +7255,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree new_temp = make_ssa_name (vectype);
gimple *g = gimple_build_assign (new_temp, code, orig, v);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
@@ -7232,7 +7267,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
last_perm_arg = make_ssa_name (vectype);
g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
}
@@ -7240,7 +7275,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
orig = make_ssa_name (vectype);
g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
last_perm_arg, perms[units_log2]);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
@@ -7251,7 +7286,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
dataref_offset);
vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
g = gimple_build_assign (data_ref, new_temp);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, g, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
}
@@ -7269,7 +7304,7 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
gimple *g = gimple_build_assign (data_ref, orig);
stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (stmt_info, g, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
prev_stmt_info = new_stmt_info;
}
@@ -7286,7 +7321,8 @@ vectorizable_scan_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_store (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec)
{
@@ -7294,7 +7330,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree op;
tree vec_oprnd = NULL_TREE;
tree elem_type;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
machine_mode vec_mode;
tree dummy;
@@ -7316,8 +7352,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vec<tree> vec_oprnds = vNULL;
bool slp = (slp_node != NULL);
unsigned int vec_num;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- vec_info *vinfo = stmt_info->vinfo;
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
tree aggr_type;
gather_scatter_info gs_info;
poly_uint64 vf;
@@ -7371,7 +7406,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (mask_index >= 0)
{
mask = gimple_call_arg (call, mask_index);
- if (!vect_check_scalar_mask (stmt_info, mask, &mask_dt,
+ if (!vect_check_scalar_mask (vinfo, stmt_info, mask, &mask_dt,
&mask_vectype))
return false;
}
@@ -7413,7 +7448,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return false;
}
- if (!vect_check_store_rhs (stmt_info, op, &rhs_dt, &rhs_vectype, &vls_type))
+ if (!vect_check_store_rhs (vinfo, stmt_info,
+ op, &rhs_dt, &rhs_vectype, &vls_type))
return false;
elem_type = TREE_TYPE (vectype);
@@ -7423,8 +7459,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return false;
vect_memory_access_type memory_access_type;
- if (!get_load_store_type (stmt_info, vectype, slp, mask, vls_type, ncopies,
- &memory_access_type, &gs_info))
+ if (!get_load_store_type (vinfo, stmt_info, vectype, slp, mask, vls_type,
+ ncopies, &memory_access_type, &gs_info))
return false;
if (mask)
@@ -7473,7 +7509,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && !vec_stmt)
{
- if (!check_scan_store (stmt_info, vectype, rhs_dt, slp, mask,
+ if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp, mask,
memory_access_type))
return false;
}
@@ -7488,8 +7524,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
memory_access_type, &gs_info, mask);
STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
- vect_model_store_cost (stmt_info, ncopies, rhs_dt, memory_access_type,
- vls_type, slp_node, cost_vec);
+ vect_model_store_cost (vinfo, stmt_info, ncopies, rhs_dt,
+ memory_access_type, vls_type, slp_node, cost_vec);
return true;
}
gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
@@ -7574,7 +7610,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (mask == NULL_TREE)
{
mask_arg = build_int_cst (masktype, -1);
- mask_arg = vect_init_vector (stmt_info, mask_arg, masktype, NULL);
+ mask_arg = vect_init_vector (vinfo, stmt_info,
+ mask_arg, masktype, NULL);
}
scale = build_int_cst (scaletype, gs_info.scale);
@@ -7584,11 +7621,13 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
if (j == 0)
{
- src = vec_oprnd1 = vect_get_vec_def_for_operand (op, stmt_info);
- op = vec_oprnd0 = vect_get_vec_def_for_operand (gs_info.offset,
+ src = vec_oprnd1 = vect_get_vec_def_for_operand (vinfo,
+ op, stmt_info);
+ op = vec_oprnd0 = vect_get_vec_def_for_operand (vinfo,
+ gs_info.offset,
stmt_info);
if (mask)
- mask_op = vec_mask = vect_get_vec_def_for_operand (mask,
+ mask_op = vec_mask = vect_get_vec_def_for_operand (vinfo, mask,
stmt_info);
}
else if (modifier != NONE && (j & 1))
@@ -7598,8 +7637,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
src
= vec_oprnd1 = vect_get_vec_def_for_stmt_copy (vinfo,
vec_oprnd1);
- op = permute_vec_elements (vec_oprnd0, vec_oprnd0, perm_mask,
- stmt_info, gsi);
+ op = permute_vec_elements (vinfo, vec_oprnd0, vec_oprnd0,
+ perm_mask, stmt_info, gsi);
if (mask)
mask_op
= vec_mask = vect_get_vec_def_for_stmt_copy (vinfo,
@@ -7607,8 +7646,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
else if (modifier == NARROW)
{
- src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask,
- stmt_info, gsi);
+ src = permute_vec_elements (vinfo, vec_oprnd1, vec_oprnd1,
+ perm_mask, stmt_info, gsi);
op = vec_oprnd0 = vect_get_vec_def_for_stmt_copy (vinfo,
vec_oprnd0);
}
@@ -7634,7 +7673,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
src = build1 (VIEW_CONVERT_EXPR, srctype, src);
gassign *new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
src = var;
}
@@ -7646,7 +7685,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
gassign *new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
op = var;
}
@@ -7662,7 +7701,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= gimple_build_assign (var, (j & 1) ? VEC_UNPACK_HI_EXPR
: VEC_UNPACK_LO_EXPR,
mask_op);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_arg = var;
}
tree optype = TREE_TYPE (mask_arg);
@@ -7674,7 +7713,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
gassign *new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_arg = var;
if (!useless_type_conversion_p (masktype, utype))
{
@@ -7682,7 +7721,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
<= TYPE_PRECISION (masktype));
var = vect_get_new_ssa_name (masktype, vect_scalar_var);
new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
mask_arg = var;
}
}
@@ -7690,7 +7729,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcall *new_stmt
= gimple_build_call (gs_info.decl, 5, ptr, mask_arg, op, src, scale);
stmt_vec_info new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (prev_stmt_info == NULL)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
@@ -7701,7 +7740,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return true;
}
else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
- return vectorizable_scan_store (stmt_info, gsi, vec_stmt, ncopies);
+ return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
DR_GROUP_STORE_COUNT (DR_GROUP_FIRST_ELEMENT (stmt_info))++;
@@ -7766,7 +7805,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
- dr_offset = get_dr_vinfo_offset (first_dr_info);
+ dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
stride_base
= fold_build_pointer_plus
(DR_BASE_ADDRESS (first_dr_info->dr),
@@ -7891,7 +7930,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree newoff = copy_ssa_name (running_off, NULL);
incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
running_off, pos);
- vect_finish_stmt_generation (stmt_info, incr, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
running_off = newoff;
}
unsigned int group_el = 0;
@@ -7905,7 +7944,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
if (slp)
{
- vect_get_vec_defs (op, NULL_TREE, stmt_info,
+ vect_get_vec_defs (vinfo, op, NULL_TREE, stmt_info,
&vec_oprnds, NULL, slp_node);
vec_oprnd = vec_oprnds[0];
}
@@ -7913,7 +7952,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
op = vect_get_store_rhs (next_stmt_info);
vec_oprnd = vect_get_vec_def_for_operand
- (op, next_stmt_info);
+ (vinfo, op, next_stmt_info);
}
}
else
@@ -7931,7 +7970,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple *pun
= gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
lvectype, vec_oprnd));
- vect_finish_stmt_generation (stmt_info, pun, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
vec_oprnd = tem;
}
for (i = 0; i < nstores; i++)
@@ -7958,7 +7997,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* And store it to *running_off. */
assign = gimple_build_assign (newref, elem);
stmt_vec_info assign_info
- = vect_finish_stmt_generation (stmt_info, assign, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ assign, gsi);
group_el += lnel;
if (! slp
@@ -7967,7 +8007,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
newoff = copy_ssa_name (running_off, NULL);
incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
running_off, stride_step);
- vect_finish_stmt_generation (stmt_info, incr, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
running_off = newoff;
group_el = 0;
@@ -7997,7 +8037,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
oprnds.create (group_size);
alignment_support_scheme
- = vect_supportable_dr_alignment (first_dr_info, false);
+ = vect_supportable_dr_alignment (vinfo, first_dr_info, false);
gcc_assert (alignment_support_scheme);
vec_loop_masks *loop_masks
= (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
@@ -8035,7 +8075,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
else
aggr_type = vectype;
- bump = vect_get_data_ptr_increment (dr_info, aggr_type,
+ bump = vect_get_data_ptr_increment (vinfo, dr_info, aggr_type,
memory_access_type);
}
@@ -8091,7 +8131,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp)
{
/* Get vectorized arguments for SLP_NODE. */
- vect_get_vec_defs (op, NULL_TREE, stmt_info, &vec_oprnds,
+ vect_get_vec_defs (vinfo, op, NULL_TREE, stmt_info, &vec_oprnds,
NULL, slp_node);
vec_oprnd = vec_oprnds[0];
@@ -8115,13 +8155,13 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
and only one iteration of the loop will be executed. */
op = vect_get_store_rhs (next_stmt_info);
vec_oprnd = vect_get_vec_def_for_operand
- (op, next_stmt_info);
+ (vinfo, op, next_stmt_info);
dr_chain.quick_push (vec_oprnd);
oprnds.quick_push (vec_oprnd);
next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
}
if (mask)
- vec_mask = vect_get_vec_def_for_operand (mask, stmt_info,
+ vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info,
mask_vectype);
}
@@ -8134,7 +8174,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
&& !loop_masks
&& TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
&& VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
- && integer_zerop (get_dr_vinfo_offset (first_dr_info))
+ && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
&& integer_zerop (DR_INIT (first_dr_info->dr))
&& alias_sets_conflict_p (get_alias_set (aggr_type),
get_alias_set (TREE_TYPE (ref_type))))
@@ -8143,11 +8183,11 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
dataref_offset = build_int_cst (ref_type, 0);
}
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vect_get_gather_scatter_ops (loop, stmt_info, &gs_info,
+ vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
&dataref_ptr, &vec_offset);
else
dataref_ptr
- = vect_create_data_ref_ptr (first_stmt_info, aggr_type,
+ = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
simd_lane_access_p ? loop : NULL,
offset, &dummy, gsi, &ptr_incr,
simd_lane_access_p, NULL_TREE, bump);
@@ -8176,7 +8216,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
vec_offset = vect_get_vec_def_for_stmt_copy (vinfo, vec_offset);
else
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, bump);
}
@@ -8190,13 +8230,14 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Invalidate the current contents of VEC_ARRAY. This should
become an RTL clobber too, which prevents the vector registers
from being upward-exposed. */
- vect_clobber_variable (stmt_info, gsi, vec_array);
+ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
/* Store the individual vectors into the array. */
for (i = 0; i < vec_num; i++)
{
vec_oprnd = dr_chain[i];
- write_vector_array (stmt_info, gsi, vec_oprnd, vec_array, i);
+ write_vector_array (vinfo, stmt_info,
+ gsi, vec_oprnd, vec_array, i);
}
tree final_mask = NULL;
@@ -8229,10 +8270,11 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (call, data_ref);
}
gimple_call_set_nothrow (call, true);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, call, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ call, gsi);
/* Record that VEC_ARRAY is now dead. */
- vect_clobber_variable (stmt_info, gsi, vec_array);
+ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
}
else
{
@@ -8242,8 +8284,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (j == 0)
result_chain.create (group_size);
/* Permute. */
- vect_permute_store_chain (dr_chain, group_size, stmt_info, gsi,
- &result_chain);
+ vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
+ gsi, &result_chain);
}
stmt_vec_info next_stmt_info = first_stmt_info;
@@ -8275,14 +8317,14 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
scale, vec_oprnd);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
break;
}
if (i > 0)
/* Bump the vector pointer. */
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
+ gsi, stmt_info, bump);
if (slp)
vec_oprnd = vec_oprnds[i];
@@ -8296,7 +8338,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
misalign = 0;
else if (DR_MISALIGNMENT (first_dr_info) == -1)
{
- align = dr_alignment (vect_dr_behavior (first_dr_info));
+ align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
misalign = 0;
}
else
@@ -8317,7 +8359,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple *perm_stmt
= gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
vec_oprnd, perm_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
perm_stmt = SSA_NAME_DEF_STMT (new_temp);
vec_oprnd = new_temp;
@@ -8334,7 +8376,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
final_mask, vec_oprnd);
gimple_call_set_nothrow (call, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, call, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
}
else
{
@@ -8357,7 +8399,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt
= gimple_build_assign (data_ref, vec_oprnd);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
if (slp)
@@ -8418,7 +8460,8 @@ vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
permuted vector variable. */
static tree
-permute_vec_elements (tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
+permute_vec_elements (vec_info *vinfo,
+ tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi)
{
tree vectype = TREE_TYPE (x);
@@ -8434,7 +8477,7 @@ permute_vec_elements (tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
/* Generate the permute statement. */
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
return data_ref;
}
@@ -8503,7 +8546,8 @@ hoist_defs_of_uses (stmt_vec_info stmt_info, class loop *loop)
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_load (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec)
@@ -8512,7 +8556,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vec_dest = NULL;
tree data_ref = NULL;
stmt_vec_info prev_stmt_info;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
class loop *loop = NULL;
class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
bool nested_in_vect_loop = false;
@@ -8542,11 +8586,10 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
int vec_num;
bool slp = (slp_node != NULL);
bool slp_perm = false;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
poly_uint64 vf;
tree aggr_type;
gather_scatter_info gs_info;
- vec_info *vinfo = stmt_info->vinfo;
tree ref_type;
enum vect_def_type mask_dt = vect_unknown_def_type;
@@ -8593,7 +8636,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (mask_index >= 0)
{
mask = gimple_call_arg (call, mask_index);
- if (!vect_check_scalar_mask (stmt_info, mask, &mask_dt,
+ if (!vect_check_scalar_mask (vinfo, stmt_info, mask, &mask_dt,
&mask_vectype))
return false;
}
@@ -8709,7 +8752,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
auto_vec<tree> tem;
unsigned n_perms;
- if (!vect_transform_slp_perm_load (slp_node, tem, NULL, vf,
+ if (!vect_transform_slp_perm_load (vinfo, slp_node, tem, NULL, vf,
true, &n_perms))
{
if (dump_enabled_p ())
@@ -8738,8 +8781,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
group_size = 1;
vect_memory_access_type memory_access_type;
- if (!get_load_store_type (stmt_info, vectype, slp, mask, VLS_LOAD, ncopies,
- &memory_access_type, &gs_info))
+ if (!get_load_store_type (vinfo, stmt_info, vectype, slp, mask, VLS_LOAD,
+ ncopies, &memory_access_type, &gs_info))
return false;
if (mask)
@@ -8773,7 +8816,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
memory_access_type, &gs_info, mask);
STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
- vect_model_load_cost (stmt_info, ncopies, vf, memory_access_type,
+ vect_model_load_cost (vinfo, stmt_info, ncopies, vf, memory_access_type,
slp_node, cost_vec);
return true;
}
@@ -8793,7 +8836,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
{
- vect_build_gather_load_calls (stmt_info, gsi, vec_stmt, &gs_info, mask);
+ vect_build_gather_load_calls (vinfo,
+ stmt_info, gsi, vec_stmt, &gs_info, mask);
return true;
}
@@ -8829,14 +8873,14 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info new_stmt_info;
if (hoist_p)
{
- new_temp = vect_init_vector (stmt_info, scalar_dest,
+ new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
vectype, NULL);
gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp);
new_stmt_info = vinfo->add_stmt (new_stmt);
}
else
{
- new_temp = vect_init_vector (stmt_info, scalar_dest,
+ new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
vectype, &gsi2);
new_stmt_info = vinfo->lookup_def (new_temp);
}
@@ -8896,7 +8940,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
}
- dr_offset = get_dr_vinfo_offset (first_dr_info);
+ dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
stride_base
= fold_build_pointer_plus
(DR_BASE_ADDRESS (first_dr_info->dr),
@@ -9036,7 +9080,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt
= gimple_build_assign (make_ssa_name (ltype), data_ref);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (nloads > 1)
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
gimple_assign_lhs (new_stmt));
@@ -9048,7 +9092,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree newoff = copy_ssa_name (running_off);
gimple *incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
running_off, stride_step);
- vect_finish_stmt_generation (stmt_info, incr, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
running_off = newoff;
group_el = 0;
@@ -9057,7 +9101,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (nloads > 1)
{
tree vec_inv = build_constructor (lvectype, v);
- new_temp = vect_init_vector (stmt_info, vec_inv, lvectype, gsi);
+ new_temp = vect_init_vector (vinfo, stmt_info,
+ vec_inv, lvectype, gsi);
new_stmt_info = vinfo->lookup_def (new_temp);
if (lvectype != vectype)
{
@@ -9067,7 +9112,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
build1 (VIEW_CONVERT_EXPR,
vectype, new_temp));
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
@@ -9090,7 +9136,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp_perm)
{
unsigned n_perms;
- vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
+ vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
false, &n_perms);
}
return true;
@@ -9179,7 +9225,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
alignment_support_scheme
- = vect_supportable_dr_alignment (first_dr_info, false);
+ = vect_supportable_dr_alignment (vinfo, first_dr_info, false);
gcc_assert (alignment_support_scheme);
vec_loop_masks *loop_masks
= (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
@@ -9317,7 +9363,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
initialized yet, use first_stmt_info_for_drptr DR by bumping the
distance from first_stmt_info DR instead as below. */
if (!diff_first_stmt_info)
- msq = vect_setup_realignment (first_stmt_info, gsi, &realignment_token,
+ msq = vect_setup_realignment (loop_vinfo,
+ first_stmt_info, gsi, &realignment_token,
alignment_support_scheme, NULL_TREE,
&at_loop);
if (alignment_support_scheme == dr_explicit_realign_optimized)
@@ -9353,7 +9400,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
else
aggr_type = vectype;
- bump = vect_get_data_ptr_increment (dr_info, aggr_type,
+ bump = vect_get_data_ptr_increment (vinfo, dr_info, aggr_type,
memory_access_type);
}
@@ -9371,7 +9418,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (simd_lane_access_p
&& TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
&& VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
- && integer_zerop (get_dr_vinfo_offset (first_dr_info))
+ && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
&& integer_zerop (DR_INIT (first_dr_info->dr))
&& alias_sets_conflict_p (get_alias_set (aggr_type),
get_alias_set (TREE_TYPE (ref_type)))
@@ -9384,7 +9431,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else if (diff_first_stmt_info)
{
dataref_ptr
- = vect_create_data_ref_ptr (first_stmt_info_for_drptr,
+ = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
aggr_type, at_loop, offset, &dummy,
gsi, &ptr_incr, simd_lane_access_p,
byte_offset, bump);
@@ -9396,11 +9443,12 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
size_binop (MINUS_EXPR,
DR_INIT (first_dr_info->dr),
DR_INIT (ptrdr)));
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, diff);
if (alignment_support_scheme == dr_explicit_realign)
{
- msq = vect_setup_realignment (first_stmt_info_for_drptr, gsi,
+ msq = vect_setup_realignment (vinfo,
+ first_stmt_info_for_drptr, gsi,
&realignment_token,
alignment_support_scheme,
dataref_ptr, &at_loop);
@@ -9408,11 +9456,12 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
}
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
- vect_get_gather_scatter_ops (loop, stmt_info, &gs_info,
+ vect_get_gather_scatter_ops (vinfo, loop, stmt_info, &gs_info,
&dataref_ptr, &vec_offset);
else
dataref_ptr
- = vect_create_data_ref_ptr (first_stmt_info, aggr_type, at_loop,
+ = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
+ at_loop,
offset, &dummy, gsi, &ptr_incr,
simd_lane_access_p,
byte_offset, bump);
@@ -9421,11 +9470,11 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp_node)
{
auto_vec<vec<tree> > vec_defs (1);
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_mask = vec_defs[0][0];
}
else
- vec_mask = vect_get_vec_def_for_operand (mask, stmt_info,
+ vec_mask = vect_get_vec_def_for_operand (vinfo, mask, stmt_info,
mask_vectype);
}
}
@@ -9437,7 +9486,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
vec_offset = vect_get_vec_def_for_stmt_copy (vinfo, vec_offset);
else
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, bump);
if (mask)
vec_mask = vect_get_vec_def_for_stmt_copy (vinfo, vec_mask);
@@ -9481,21 +9530,22 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
gimple_call_set_lhs (call, vec_array);
gimple_call_set_nothrow (call, true);
- new_stmt_info = vect_finish_stmt_generation (stmt_info, call, gsi);
+ new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info,
+ call, gsi);
/* Extract each vector into an SSA_NAME. */
for (i = 0; i < vec_num; i++)
{
- new_temp = read_vector_array (stmt_info, gsi, scalar_dest,
+ new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
vec_array, i);
dr_chain.quick_push (new_temp);
}
/* Record the mapping between SSA_NAMEs and statements. */
- vect_record_grouped_load_vectors (stmt_info, dr_chain);
+ vect_record_grouped_load_vectors (vinfo, stmt_info, dr_chain);
/* Record that VEC_ARRAY is now dead. */
- vect_clobber_variable (stmt_info, gsi, vec_array);
+ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
}
else
{
@@ -9512,8 +9562,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vec_mask, gsi);
if (i > 0)
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
+ gsi, stmt_info, bump);
/* 2. Create the vector-load in the loop. */
gimple *new_stmt = NULL;
@@ -9554,7 +9604,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
else if (DR_MISALIGNMENT (first_dr_info) == -1)
{
align = dr_alignment
- (vect_dr_behavior (first_dr_info));
+ (vect_dr_behavior (vinfo, first_dr_info));
misalign = 0;
}
else
@@ -9615,7 +9665,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
tree tem = make_ssa_name (ltype);
new_stmt = gimple_build_assign (tem, data_ref);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
data_ref = NULL;
vec<constructor_elt, va_gc> *v;
vec_alloc (v, 2);
@@ -9637,7 +9688,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
if (compute_in_loop)
- msq = vect_setup_realignment (first_stmt_info, gsi,
+ msq = vect_setup_realignment (vinfo, first_stmt_info, gsi,
&realignment_token,
dr_explicit_realign,
dataref_ptr, NULL);
@@ -9655,7 +9706,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
build_int_cst
(TREE_TYPE (dataref_ptr),
-(HOST_WIDE_INT) align));
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
data_ref
= build2 (MEM_REF, vectype, ptr,
build_int_cst (ref_type, 0));
@@ -9666,13 +9718,14 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
gimple_move_vops (new_stmt, stmt_info->stmt);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
msq = new_temp;
bump = size_binop (MULT_EXPR, vs,
TYPE_SIZE_UNIT (elem_type));
bump = size_binop (MINUS_EXPR, bump, size_one_node);
- ptr = bump_vector_ptr (dataref_ptr, NULL, gsi,
+ ptr = bump_vector_ptr (vinfo, dataref_ptr, NULL, gsi,
stmt_info, bump);
new_stmt = gimple_build_assign
(NULL_TREE, BIT_AND_EXPR, ptr,
@@ -9680,7 +9733,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
(TREE_TYPE (ptr), -(HOST_WIDE_INT) align));
ptr = copy_ssa_name (ptr, new_stmt);
gimple_assign_set_lhs (new_stmt, ptr);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
data_ref
= build2 (MEM_REF, vectype, ptr,
build_int_cst (ref_type, 0));
@@ -9700,7 +9754,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
(new_temp, BIT_AND_EXPR, dataref_ptr,
build_int_cst (TREE_TYPE (dataref_ptr),
-(HOST_WIDE_INT) align));
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
data_ref
= build2 (MEM_REF, vectype, new_temp,
build_int_cst (ref_type, 0));
@@ -9719,7 +9774,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
/* 3. Handle explicit realignment if necessary/supported.
Create in loop:
@@ -9736,7 +9792,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
if (alignment_support_scheme == dr_explicit_realign_optimized)
{
@@ -9752,7 +9809,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
{
tree perm_mask = perm_mask_for_reverse (vectype);
- new_temp = permute_vec_elements (new_temp, new_temp,
+ new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
perm_mask, stmt_info, gsi);
new_stmt_info = vinfo->lookup_def (new_temp);
}
@@ -9778,8 +9835,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= (wi::to_wide (TYPE_SIZE_UNIT (elem_type))
* group_gap_adj);
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
- stmt_info, bump);
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
+ gsi, stmt_info, bump);
group_elt = 0;
}
}
@@ -9791,7 +9848,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= (wi::to_wide (TYPE_SIZE_UNIT (elem_type))
* group_gap_adj);
tree bump = wide_int_to_tree (sizetype, bump_val);
- dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
stmt_info, bump);
}
}
@@ -9802,8 +9859,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp_perm)
{
unsigned n_perms;
- bool ok = vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
- false, &n_perms);
+ bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
+ gsi, vf, false, &n_perms);
gcc_assert (ok);
}
else
@@ -9811,7 +9868,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (grouped_load)
{
if (memory_access_type != VMAT_LOAD_STORE_LANES)
- vect_transform_grouped_load (stmt_info, dr_chain,
+ vect_transform_grouped_load (vinfo, stmt_info, dr_chain,
group_size, gsi);
*vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
}
@@ -9927,11 +9984,11 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, slp_tree slp_node,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_condition (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt,
slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
tree scalar_dest = NULL_TREE;
tree vec_dest = NULL_TREE;
tree cond_expr, cond_expr0 = NULL_TREE, cond_expr1 = NULL_TREE;
@@ -9941,7 +9998,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
tree vec_compare;
tree new_temp;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum vect_def_type dts[4]
= {vect_unknown_def_type, vect_unknown_def_type,
vect_unknown_def_type, vect_unknown_def_type};
@@ -9951,7 +10008,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
stmt_vec_info prev_stmt_info = NULL;
int i, j;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
vec<tree> vec_oprnds2 = vNULL;
@@ -9980,7 +10037,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
{
if (STMT_SLP_TYPE (stmt_info))
return false;
- reduc_info = info_for_reduction (stmt_info);
+ reduc_info = info_for_reduction (vinfo, stmt_info);
reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
reduc_index = STMT_VINFO_REDUC_IDX (stmt_info);
gcc_assert (reduction_type != EXTRACT_LAST_REDUCTION
@@ -10023,14 +10080,14 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
then_clause = gimple_assign_rhs2 (stmt);
else_clause = gimple_assign_rhs3 (stmt);
- if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo, slp_node,
+ if (!vect_is_simple_cond (cond_expr, vinfo, slp_node,
&comp_vectype, &dts[0], slp_node ? NULL : vectype)
|| !comp_vectype)
return false;
- if (!vect_is_simple_use (then_clause, stmt_info->vinfo, &dts[2], &vectype1))
+ if (!vect_is_simple_use (then_clause, vinfo, &dts[2], &vectype1))
return false;
- if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &dts[3], &vectype2))
+ if (!vect_is_simple_use (else_clause, vinfo, &dts[3], &vectype2))
return false;
if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
@@ -10162,7 +10219,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
return false;
STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
- vect_model_simple_cost (stmt_info, ncopies, dts, ndts, slp_node,
+ vect_model_simple_cost (vinfo, stmt_info, ncopies, dts, ndts, slp_node,
cost_vec, kind);
return true;
}
@@ -10220,7 +10277,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp_node)
{
auto_vec<vec<tree>, 4> vec_defs;
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_oprnds3 = vec_defs.pop ();
vec_oprnds2 = vec_defs.pop ();
if (!masked)
@@ -10232,22 +10289,24 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (masked)
{
vec_cond_lhs
- = vect_get_vec_def_for_operand (cond_expr, stmt_info,
+ = vect_get_vec_def_for_operand (vinfo, cond_expr, stmt_info,
comp_vectype);
}
else
{
vec_cond_lhs
- = vect_get_vec_def_for_operand (cond_expr0,
+ = vect_get_vec_def_for_operand (vinfo, cond_expr0,
stmt_info, comp_vectype);
vec_cond_rhs
- = vect_get_vec_def_for_operand (cond_expr1,
+ = vect_get_vec_def_for_operand (vinfo, cond_expr1,
stmt_info, comp_vectype);
}
- vec_then_clause = vect_get_vec_def_for_operand (then_clause,
+ vec_then_clause = vect_get_vec_def_for_operand (vinfo,
+ then_clause,
stmt_info);
if (reduction_type != EXTRACT_LAST_REDUCTION)
- vec_else_clause = vect_get_vec_def_for_operand (else_clause,
+ vec_else_clause = vect_get_vec_def_for_operand (vinfo,
+ else_clause,
stmt_info);
}
}
@@ -10302,7 +10361,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_stmt
= gimple_build_assign (new_temp, bitop1, vec_cond_lhs,
vec_cond_rhs);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (bitop2 == NOP_EXPR)
vec_compare = new_temp;
else if (bitop2 == BIT_NOT_EXPR)
@@ -10317,7 +10376,8 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_stmt
= gimple_build_assign (vec_compare, bitop2,
vec_cond_lhs, new_temp);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
}
@@ -10354,7 +10414,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree vec_compare_name = make_ssa_name (vec_cmp_type);
gassign *new_stmt = gimple_build_assign (vec_compare_name,
vec_compare);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
vec_compare = vec_compare_name;
}
@@ -10364,7 +10424,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt = gimple_build_assign (vec_compare_name,
BIT_NOT_EXPR,
vec_compare);
- vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
vec_compare = vec_compare_name;
}
@@ -10378,7 +10438,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *g
= gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
loop_mask);
- vect_finish_stmt_generation (stmt_info, g, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
vec_compare = tmp2;
}
}
@@ -10393,7 +10453,8 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_call_set_lhs (new_stmt, lhs);
SSA_NAME_DEF_STMT (lhs) = new_stmt;
if (old_stmt == gsi_stmt (*gsi))
- new_stmt_info = vect_finish_replace_stmt (stmt_info, new_stmt);
+ new_stmt_info = vect_finish_replace_stmt (vinfo,
+ stmt_info, new_stmt);
else
{
/* In this case we're moving the definition to later in the
@@ -10402,7 +10463,8 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
gsi_remove (&old_gsi, true);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
else
@@ -10412,7 +10474,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
= gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
vec_then_clause, vec_else_clause);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
if (slp_node)
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info);
@@ -10446,17 +10508,17 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
Return true if STMT_INFO is vectorizable in this way. */
static bool
-vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vectorizable_comparison (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt,
slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
tree lhs, rhs1, rhs2;
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
tree new_temp;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
int ndts = 2;
poly_uint64 nunits;
@@ -10464,7 +10526,7 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
enum tree_code code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
stmt_vec_info prev_stmt_info = NULL;
int i, j;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
tree mask_type;
@@ -10508,10 +10570,10 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
rhs1 = gimple_assign_rhs1 (stmt);
rhs2 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use (rhs1, stmt_info->vinfo, &dts[0], &vectype1))
+ if (!vect_is_simple_use (rhs1, vinfo, &dts[0], &vectype1))
return false;
- if (!vect_is_simple_use (rhs2, stmt_info->vinfo, &dts[1], &vectype2))
+ if (!vect_is_simple_use (rhs2, vinfo, &dts[1], &vectype2))
return false;
if (vectype1 && vectype2
@@ -10602,7 +10664,8 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
- vect_model_simple_cost (stmt_info, ncopies * (1 + (bitop2 != NOP_EXPR)),
+ vect_model_simple_cost (vinfo, stmt_info,
+ ncopies * (1 + (bitop2 != NOP_EXPR)),
dts, ndts, slp_node, cost_vec);
return true;
}
@@ -10627,7 +10690,7 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
if (slp_node)
{
auto_vec<vec<tree>, 2> vec_defs;
- vect_get_slp_defs (slp_node, &vec_defs);
+ vect_get_slp_defs (vinfo, slp_node, &vec_defs);
vec_oprnds1 = vec_defs.pop ();
vec_oprnds0 = vec_defs.pop ();
if (swap_p)
@@ -10635,9 +10698,9 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
}
else
{
- vec_rhs1 = vect_get_vec_def_for_operand (rhs1, stmt_info,
+ vec_rhs1 = vect_get_vec_def_for_operand (vinfo, rhs1, stmt_info,
vectype);
- vec_rhs2 = vect_get_vec_def_for_operand (rhs2, stmt_info,
+ vec_rhs2 = vect_get_vec_def_for_operand (vinfo, rhs2, stmt_info,
vectype);
}
}
@@ -10668,7 +10731,7 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gassign *new_stmt = gimple_build_assign (new_temp, code,
vec_rhs1, vec_rhs2);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
}
else
{
@@ -10679,7 +10742,7 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
vec_rhs2);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
if (bitop2 != NOP_EXPR)
{
tree res = make_ssa_name (mask);
@@ -10689,7 +10752,8 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
new_temp);
new_stmt_info
- = vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+ = vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
}
}
if (slp_node)
@@ -10719,7 +10783,8 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
GSI and VEC_STMT_P are as for vectorizable_live_operation. */
static bool
-can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+can_vectorize_live_stmts (loop_vec_info loop_vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
slp_tree slp_node, slp_instance slp_node_instance,
bool vec_stmt_p,
stmt_vector_for_cost *cost_vec)
@@ -10731,15 +10796,16 @@ can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
{
if (STMT_VINFO_LIVE_P (slp_stmt_info)
- && !vectorizable_live_operation (slp_stmt_info, gsi, slp_node,
+ && !vectorizable_live_operation (loop_vinfo,
+ slp_stmt_info, gsi, slp_node,
slp_node_instance, i,
vec_stmt_p, cost_vec))
return false;
}
}
else if (STMT_VINFO_LIVE_P (stmt_info)
- && !vectorizable_live_operation (stmt_info, gsi, slp_node,
- slp_node_instance, -1,
+ && !vectorizable_live_operation (loop_vinfo, stmt_info, gsi,
+ slp_node, slp_node_instance, -1,
vec_stmt_p, cost_vec))
return false;
@@ -10749,12 +10815,12 @@ can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Make sure the statement is vectorizable. */
opt_result
-vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
+vect_analyze_stmt (vec_info *vinfo,
+ stmt_vec_info stmt_info, bool *need_to_vectorize,
slp_tree node, slp_instance node_instance,
stmt_vector_for_cost *cost_vec)
{
- vec_info *vinfo = stmt_info->vinfo;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
gimple_seq pattern_def_seq;
@@ -10789,7 +10855,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
pattern_def_stmt_info->stmt);
opt_result res
- = vect_analyze_stmt (pattern_def_stmt_info,
+ = vect_analyze_stmt (vinfo, pattern_def_stmt_info,
need_to_vectorize, node, node_instance,
cost_vec);
if (!res)
@@ -10849,7 +10915,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
pattern_stmt_info->stmt);
opt_result res
- = vect_analyze_stmt (pattern_stmt_info, need_to_vectorize, node,
+ = vect_analyze_stmt (vinfo, pattern_stmt_info, need_to_vectorize, node,
node_instance, cost_vec);
if (!res)
return res;
@@ -10906,39 +10972,50 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
/* Prefer vectorizable_call over vectorizable_simd_clone_call so
-mveclibabi= takes preference over library functions with
the simd attribute. */
- ok = (vectorizable_call (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_simd_clone_call (stmt_info, NULL, NULL, node,
+ ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+ || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_conversion (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_operation (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_assignment (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_load (stmt_info, NULL, NULL, node, node_instance,
- cost_vec)
- || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_reduction (stmt_info, node, node_instance, cost_vec)
- || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_condition (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_comparison (stmt_info, NULL, NULL, node,
+ || vectorizable_conversion (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_operation (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_assignment (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_load (vinfo, stmt_info,
+ NULL, NULL, node, node_instance, cost_vec)
+ || vectorizable_store (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+ || vectorizable_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
+ node, node_instance, cost_vec)
+ || vectorizable_induction (as_a <loop_vec_info> (vinfo), stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_shift (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+ || vectorizable_condition (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_lc_phi (stmt_info, NULL, node));
+ || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
+ stmt_info, NULL, node));
else
{
if (bb_vinfo)
- ok = (vectorizable_call (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_simd_clone_call (stmt_info, NULL, NULL, node,
- cost_vec)
- || vectorizable_conversion (stmt_info, NULL, NULL, node,
+ ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+ || vectorizable_simd_clone_call (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_conversion (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_operation (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_assignment (stmt_info, NULL, NULL, node,
+ || vectorizable_shift (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_operation (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_assignment (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_load (stmt_info, NULL, NULL, node, node_instance,
- cost_vec)
- || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_condition (stmt_info, NULL, NULL, node, cost_vec)
- || vectorizable_comparison (stmt_info, NULL, NULL, node,
+ || vectorizable_load (vinfo, stmt_info,
+ NULL, NULL, node, node_instance, cost_vec)
+ || vectorizable_store (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_condition (vinfo, stmt_info,
+ NULL, NULL, node, cost_vec)
+ || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
cost_vec));
}
@@ -10953,7 +11030,8 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
if (!bb_vinfo
&& STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
&& STMT_VINFO_TYPE (stmt_info) != lc_phi_info_type
- && !can_vectorize_live_stmts (stmt_info, NULL, node, node_instance,
+ && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
+ stmt_info, NULL, node, node_instance,
false, cost_vec))
return opt_result::failure_at (stmt_info->stmt,
"not vectorized:"
@@ -10969,10 +11047,10 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
bool
-vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
+vect_transform_stmt (vec_info *vinfo,
+ stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
slp_tree slp_node, slp_instance slp_node_instance)
{
- vec_info *vinfo = stmt_info->vinfo;
bool is_store = false;
stmt_vec_info vec_stmt = NULL;
bool done;
@@ -10980,10 +11058,10 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
stmt_vec_info old_vec_stmt_info = STMT_VINFO_VEC_STMT (stmt_info);
- bool nested_p = (STMT_VINFO_LOOP_VINFO (stmt_info)
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ bool nested_p = (loop_vinfo
&& nested_in_vect_loop_p
- (LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info)),
- stmt_info));
+ (LOOP_VINFO_LOOP (loop_vinfo), stmt_info));
gimple *stmt = stmt_info->stmt;
switch (STMT_VINFO_TYPE (stmt_info))
@@ -10991,42 +11069,45 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
case type_demotion_vec_info_type:
case type_promotion_vec_info_type:
case type_conversion_vec_info_type:
- done = vectorizable_conversion (stmt_info, gsi, &vec_stmt, slp_node,
- NULL);
+ done = vectorizable_conversion (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
gcc_assert (done);
break;
case induc_vec_info_type:
- done = vectorizable_induction (stmt_info, gsi, &vec_stmt, slp_node,
+ done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
+ stmt_info, gsi, &vec_stmt, slp_node,
NULL);
gcc_assert (done);
break;
case shift_vec_info_type:
- done = vectorizable_shift (stmt_info, gsi, &vec_stmt, slp_node, NULL);
+ done = vectorizable_shift (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
gcc_assert (done);
break;
case op_vec_info_type:
- done = vectorizable_operation (stmt_info, gsi, &vec_stmt, slp_node,
+ done = vectorizable_operation (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
NULL);
gcc_assert (done);
break;
case assignment_vec_info_type:
- done = vectorizable_assignment (stmt_info, gsi, &vec_stmt, slp_node,
- NULL);
+ done = vectorizable_assignment (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
gcc_assert (done);
break;
case load_vec_info_type:
- done = vectorizable_load (stmt_info, gsi, &vec_stmt, slp_node,
+ done = vectorizable_load (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
slp_node_instance, NULL);
gcc_assert (done);
break;
case store_vec_info_type:
- done = vectorizable_store (stmt_info, gsi, &vec_stmt, slp_node, NULL);
+ done = vectorizable_store (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
gcc_assert (done);
if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && !slp_node)
{
@@ -11043,40 +11124,44 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
break;
case condition_vec_info_type:
- done = vectorizable_condition (stmt_info, gsi, &vec_stmt, slp_node, NULL);
+ done = vectorizable_condition (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
gcc_assert (done);
break;
case comparison_vec_info_type:
- done = vectorizable_comparison (stmt_info, gsi, &vec_stmt,
+ done = vectorizable_comparison (vinfo, stmt_info, gsi, &vec_stmt,
slp_node, NULL);
gcc_assert (done);
break;
case call_vec_info_type:
- done = vectorizable_call (stmt_info, gsi, &vec_stmt, slp_node, NULL);
+ done = vectorizable_call (vinfo, stmt_info,
+ gsi, &vec_stmt, slp_node, NULL);
stmt = gsi_stmt (*gsi);
break;
case call_simd_clone_vec_info_type:
- done = vectorizable_simd_clone_call (stmt_info, gsi, &vec_stmt,
+ done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi, &vec_stmt,
slp_node, NULL);
stmt = gsi_stmt (*gsi);
break;
case reduc_vec_info_type:
- done = vect_transform_reduction (stmt_info, gsi, &vec_stmt, slp_node);
+ done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
+ gsi, &vec_stmt, slp_node);
gcc_assert (done);
break;
case cycle_phi_info_type:
- done = vect_transform_cycle_phi (stmt_info, &vec_stmt, slp_node,
- slp_node_instance);
+ done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
+ &vec_stmt, slp_node, slp_node_instance);
gcc_assert (done);
break;
case lc_phi_info_type:
- done = vectorizable_lc_phi (stmt_info, &vec_stmt, slp_node);
+ done = vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
+ stmt_info, &vec_stmt, slp_node);
gcc_assert (done);
break;
@@ -11105,8 +11190,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|| STMT_VINFO_RELEVANT (stmt_info) ==
vect_used_in_outer_by_reduction))
{
- class loop *innerloop = LOOP_VINFO_LOOP (
- STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
+ class loop *innerloop = LOOP_VINFO_LOOP (loop_vinfo)->inner;
imm_use_iterator imm_iter;
use_operand_p use_p;
tree scalar_dest;
@@ -11144,7 +11228,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info reduc_info;
if (STMT_VINFO_REDUC_DEF (orig_stmt_info)
&& vect_stmt_to_vectorize (orig_stmt_info) == stmt_info
- && (reduc_info = info_for_reduction (orig_stmt_info))
+ && (reduc_info = info_for_reduction (vinfo, orig_stmt_info))
&& STMT_VINFO_REDUC_TYPE (reduc_info) != FOLD_LEFT_REDUCTION
&& STMT_VINFO_REDUC_TYPE (reduc_info) != EXTRACT_LAST_REDUCTION)
{
@@ -11190,8 +11274,10 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Handle stmts whose DEF is used outside the loop-nest that is
being vectorized. */
- done = can_vectorize_live_stmts (stmt_info, gsi, slp_node,
- slp_node_instance, true, NULL);
+ if (is_a <loop_vec_info> (vinfo))
+ done = can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
+ stmt_info, gsi, slp_node,
+ slp_node_instance, true, NULL);
gcc_assert (done);
return false;
@@ -11202,9 +11288,8 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info. */
void
-vect_remove_stores (stmt_vec_info first_stmt_info)
+vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
{
- vec_info *vinfo = first_stmt_info->vinfo;
stmt_vec_info next_stmt_info = first_stmt_info;
while (next_stmt_info)
@@ -11657,13 +11742,14 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
widening operation (short in the above example). */
bool
-supportable_widening_operation (enum tree_code code, stmt_vec_info stmt_info,
+supportable_widening_operation (vec_info *vinfo,
+ enum tree_code code, stmt_vec_info stmt_info,
tree vectype_out, tree vectype_in,
enum tree_code *code1, enum tree_code *code2,
int *multi_step_cvt,
vec<tree> *interm_types)
{
- loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
class loop *vect_loop = NULL;
machine_mode vec_mode;
enum insn_code icode1, icode2;
@@ -11719,7 +11805,7 @@ supportable_widening_operation (enum tree_code code, stmt_vec_info stmt_info,
if (vect_loop
&& STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
&& !nested_in_vect_loop_p (vect_loop, stmt_info)
- && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
+ && supportable_widening_operation (vinfo, VEC_WIDEN_MULT_EVEN_EXPR,
stmt_info, vectype_out,
vectype_in, code1, code2,
multi_step_cvt, interm_types))
@@ -12122,12 +12208,11 @@ vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
statement does not help to determine the overall number of units. */
opt_result
-vect_get_vector_types_for_stmt (stmt_vec_info stmt_info,
+vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
tree *stmt_vectype_out,
tree *nunits_vectype_out,
unsigned int group_size)
{
- vec_info *vinfo = stmt_info->vinfo;
gimple *stmt = stmt_info->stmt;
/* For BB vectorization, we should always have a group size once we've
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 8f9444d58a3..41ff6791966 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -631,7 +631,6 @@ stmt_vec_info
vec_info::new_stmt_vec_info (gimple *stmt)
{
stmt_vec_info res = XCNEW (class _stmt_vec_info);
- res->vinfo = this;
res->stmt = stmt;
STMT_VINFO_TYPE (res) = undef_vec_info_type;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 9e75098d1f2..20784251b6f 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -946,9 +946,6 @@ public:
/* The stmt to which this info struct refers to. */
gimple *stmt;
- /* The vec_info with respect to which STMT is vectorized. */
- vec_info *vinfo;
-
/* The vector type to be used for the LHS of this statement. */
tree vectype;
@@ -1153,20 +1150,6 @@ struct gather_scatter_info {
/* Access Functions. */
#define STMT_VINFO_TYPE(S) (S)->type
#define STMT_VINFO_STMT(S) (S)->stmt
-inline loop_vec_info
-STMT_VINFO_LOOP_VINFO (stmt_vec_info stmt_vinfo)
-{
- if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (stmt_vinfo->vinfo))
- return loop_vinfo;
- return NULL;
-}
-inline bb_vec_info
-STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
-{
- if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (stmt_vinfo->vinfo))
- return bb_vinfo;
- return NULL;
-}
#define STMT_VINFO_RELEVANT(S) (S)->relevant
#define STMT_VINFO_LIVE_P(S) (S)->live
#define STMT_VINFO_VECTYPE(S) (S)->vectype
@@ -1378,11 +1361,12 @@ extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt,
/* Alias targetm.vectorize.add_stmt_cost. */
static inline unsigned
-add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+add_stmt_cost (vec_info *vinfo, void *data, int count,
+ enum vect_cost_for_stmt kind,
stmt_vec_info stmt_info, int misalign,
enum vect_cost_model_location where)
{
- unsigned cost = targetm.vectorize.add_stmt_cost (data, count, kind,
+ unsigned cost = targetm.vectorize.add_stmt_cost (vinfo, data, count, kind,
stmt_info, misalign, where);
if (dump_file && (dump_flags & TDF_DETAILS))
dump_stmt_cost (dump_file, data, count, kind, stmt_info, misalign,
@@ -1408,12 +1392,12 @@ destroy_cost_data (void *data)
}
inline void
-add_stmt_costs (void *data, stmt_vector_for_cost *cost_vec)
+add_stmt_costs (vec_info *vinfo, void *data, stmt_vector_for_cost *cost_vec)
{
stmt_info_for_cost *cost;
unsigned i;
FOR_EACH_VEC_ELT (*cost_vec, i, cost)
- add_stmt_cost (data, cost->count, cost->kind, cost->stmt_info,
+ add_stmt_cost (vinfo, data, cost->count, cost->kind, cost->stmt_info,
cost->misalign, cost->where);
}
@@ -1481,10 +1465,10 @@ vect_known_alignment_in_bytes (dr_vec_info *dr_info)
in DR_INFO itself). */
static inline innermost_loop_behavior *
-vect_dr_behavior (dr_vec_info *dr_info)
+vect_dr_behavior (vec_info *vinfo, dr_vec_info *dr_info)
{
stmt_vec_info stmt_info = dr_info->stmt;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
if (loop_vinfo == NULL
|| !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
return &DR_INNERMOST (dr_info->dr);
@@ -1497,11 +1481,12 @@ vect_dr_behavior (dr_vec_info *dr_info)
vect_dr_behavior to select the appropriate data_reference to use. */
inline tree
-get_dr_vinfo_offset (dr_vec_info *dr_info, bool check_outer = false)
+get_dr_vinfo_offset (vec_info *vinfo,
+ dr_vec_info *dr_info, bool check_outer = false)
{
innermost_loop_behavior *base;
if (check_outer)
- base = vect_dr_behavior (dr_info);
+ base = vect_dr_behavior (vinfo, dr_info);
else
base = &dr_info->dr->innermost;
@@ -1706,7 +1691,8 @@ extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
tree *, stmt_vec_info * = NULL,
gimple ** = NULL);
-extern bool supportable_widening_operation (enum tree_code, stmt_vec_info,
+extern bool supportable_widening_operation (vec_info *,
+ enum tree_code, stmt_vec_info,
tree, tree, enum tree_code *,
enum tree_code *, int *,
vec<tree> *);
@@ -1716,31 +1702,36 @@ extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info,
int, enum vect_cost_model_location);
-extern stmt_vec_info vect_finish_replace_stmt (stmt_vec_info, gimple *);
-extern stmt_vec_info vect_finish_stmt_generation (stmt_vec_info, gimple *,
+extern stmt_vec_info vect_finish_replace_stmt (vec_info *,
+ stmt_vec_info, gimple *);
+extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
+ stmt_vec_info, gimple *,
gimple_stmt_iterator *);
extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
extern tree vect_get_store_rhs (stmt_vec_info);
extern tree vect_get_vec_def_for_operand_1 (stmt_vec_info, enum vect_def_type);
-extern tree vect_get_vec_def_for_operand (tree, stmt_vec_info, tree = NULL);
-extern void vect_get_vec_defs (tree, tree, stmt_vec_info, vec<tree> *,
- vec<tree> *, slp_tree);
+extern tree vect_get_vec_def_for_operand (vec_info *, tree,
+ stmt_vec_info, tree = NULL);
+extern void vect_get_vec_defs (vec_info *, tree, tree, stmt_vec_info,
+ vec<tree> *, vec<tree> *, slp_tree);
extern void vect_get_vec_defs_for_stmt_copy (vec_info *,
vec<tree> *, vec<tree> *);
-extern tree vect_init_vector (stmt_vec_info, tree, tree,
+extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
gimple_stmt_iterator *);
extern tree vect_get_vec_def_for_stmt_copy (vec_info *, tree);
-extern bool vect_transform_stmt (stmt_vec_info, gimple_stmt_iterator *,
+extern bool vect_transform_stmt (vec_info *, stmt_vec_info,
+ gimple_stmt_iterator *,
slp_tree, slp_instance);
-extern void vect_remove_stores (stmt_vec_info);
+extern void vect_remove_stores (vec_info *, stmt_vec_info);
extern bool vect_nop_conversion_p (stmt_vec_info);
-extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree,
+extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
+ slp_tree,
slp_instance, stmt_vector_for_cost *);
-extern void vect_get_load_cost (stmt_vec_info, int, bool,
+extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, bool,
unsigned int *, unsigned int *,
stmt_vector_for_cost *,
stmt_vector_for_cost *, bool);
-extern void vect_get_store_cost (stmt_vec_info, int,
+extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
unsigned int *, stmt_vector_for_cost *);
extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);
@@ -1748,22 +1739,24 @@ extern tree vect_gen_perm_mask_checked (tree, const vec_perm_indices &);
extern void optimize_mask_stores (class loop*);
extern gcall *vect_gen_while (tree, tree, tree);
extern tree vect_gen_while_not (gimple_seq *, tree, tree, tree);
-extern opt_result vect_get_vector_types_for_stmt (stmt_vec_info, tree *,
+extern opt_result vect_get_vector_types_for_stmt (vec_info *,
+ stmt_vec_info, tree *,
tree *, unsigned int = 0);
extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
extern enum dr_alignment_support vect_supportable_dr_alignment
- (dr_vec_info *, bool);
+ (vec_info *, dr_vec_info *, bool);
extern tree vect_get_smallest_scalar_type (stmt_vec_info, HOST_WIDE_INT *,
HOST_WIDE_INT *);
extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
-extern bool vect_slp_analyze_instance_dependence (slp_instance);
+extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
extern opt_result vect_enhance_data_refs_alignment (loop_vec_info);
extern opt_result vect_analyze_data_refs_alignment (loop_vec_info);
extern opt_result vect_verify_datarefs_alignment (loop_vec_info);
-extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance);
+extern bool vect_slp_analyze_and_verify_instance_alignment (vec_info *,
+ slp_instance);
extern opt_result vect_analyze_data_ref_accesses (vec_info *);
extern opt_result vect_prune_runtime_alias_test_list (loop_vec_info);
extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
@@ -1774,11 +1767,12 @@ extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
vec<data_reference_p> *);
extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *);
extern void vect_record_base_alignments (vec_info *);
-extern tree vect_create_data_ref_ptr (stmt_vec_info, tree, class loop *, tree,
+extern tree vect_create_data_ref_ptr (vec_info *,
+ stmt_vec_info, tree, class loop *, tree,
tree *, gimple_stmt_iterator *,
gimple **, bool,
tree = NULL_TREE, tree = NULL_TREE);
-extern tree bump_vector_ptr (tree, gimple *, gimple_stmt_iterator *,
+extern tree bump_vector_ptr (vec_info *, tree, gimple *, gimple_stmt_iterator *,
stmt_vec_info, tree);
extern void vect_copy_ref_info (tree, tree);
extern tree vect_create_destination_var (tree, tree);
@@ -1786,18 +1780,22 @@ extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
-extern void vect_permute_store_chain (vec<tree> ,unsigned int, stmt_vec_info,
- gimple_stmt_iterator *, vec<tree> *);
-extern tree vect_setup_realignment (stmt_vec_info, gimple_stmt_iterator *,
+extern void vect_permute_store_chain (vec_info *,
+ vec<tree> ,unsigned int, stmt_vec_info,
+ gimple_stmt_iterator *, vec<tree> *);
+extern tree vect_setup_realignment (vec_info *,
+ stmt_vec_info, gimple_stmt_iterator *,
tree *, enum dr_alignment_support, tree,
class loop **);
-extern void vect_transform_grouped_load (stmt_vec_info, vec<tree> , int,
- gimple_stmt_iterator *);
-extern void vect_record_grouped_load_vectors (stmt_vec_info, vec<tree>);
+extern void vect_transform_grouped_load (vec_info *, stmt_vec_info, vec<tree>,
+ int, gimple_stmt_iterator *);
+extern void vect_record_grouped_load_vectors (vec_info *,
+ stmt_vec_info, vec<tree>);
extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
const char * = NULL);
-extern tree vect_create_addr_base_for_vector_ref (stmt_vec_info, gimple_seq *,
+extern tree vect_create_addr_base_for_vector_ref (vec_info *,
+ stmt_vec_info, gimple_seq *,
tree, tree = NULL_TREE);
/* In tree-vect-loop.c. */
@@ -1819,25 +1817,31 @@ extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
unsigned int, tree, tree);
extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
unsigned int, tree, unsigned int);
-extern stmt_vec_info info_for_reduction (stmt_vec_info);
+extern stmt_vec_info info_for_reduction (vec_info *, stmt_vec_info);
/* Drive for loop transformation stage. */
extern class loop *vect_transform_loop (loop_vec_info, gimple *);
extern opt_loop_vec_info vect_analyze_loop_form (class loop *,
vec_info_shared *);
-extern bool vectorizable_live_operation (stmt_vec_info, gimple_stmt_iterator *,
+extern bool vectorizable_live_operation (loop_vec_info,
+ stmt_vec_info, gimple_stmt_iterator *,
slp_tree, slp_instance, int,
bool, stmt_vector_for_cost *);
-extern bool vectorizable_reduction (stmt_vec_info, slp_tree, slp_instance,
+extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
+ slp_tree, slp_instance,
stmt_vector_for_cost *);
-extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *,
+extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
+ gimple_stmt_iterator *,
stmt_vec_info *, slp_tree,
stmt_vector_for_cost *);
-extern bool vect_transform_reduction (stmt_vec_info, gimple_stmt_iterator *,
+extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
+ gimple_stmt_iterator *,
stmt_vec_info *, slp_tree);
-extern bool vect_transform_cycle_phi (stmt_vec_info, stmt_vec_info *,
+extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
+ stmt_vec_info *,
slp_tree, slp_instance);
-extern bool vectorizable_lc_phi (stmt_vec_info, stmt_vec_info *, slp_tree);
+extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info,
+ stmt_vec_info *, slp_tree);
extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code);
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
stmt_vector_for_cost *,
@@ -1847,7 +1851,7 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
/* In tree-vect-slp.c. */
extern void vect_free_slp_instance (slp_instance, bool);
-extern bool vect_transform_slp_perm_load (slp_tree, vec<tree> ,
+extern bool vect_transform_slp_perm_load (vec_info *, slp_tree, vec<tree>,
gimple_stmt_iterator *, poly_uint64,
bool, unsigned *);
extern bool vect_slp_analyze_operations (vec_info *);
@@ -1856,7 +1860,8 @@ extern opt_result vect_analyze_slp (vec_info *, unsigned);
extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_optimize_slp (vec_info *);
-extern void vect_get_slp_defs (slp_tree, vec<vec<tree> > *, unsigned n = -1U);
+extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
+ unsigned n = -1U);
extern bool vect_slp_bb (basic_block);
extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree);
extern bool is_simple_and_all_uses_invariant (stmt_vec_info, loop_vec_info);
--
2.13.7
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2020-03-16 12:18 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-03-16 12:18 [PATCH 3/n] add vec_info * parameters where needed Richard Biener
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).