public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc r12-4509] Refactor vect_supportable_dr_alignment
@ 2021-10-19 14:09 Richard Biener
0 siblings, 0 replies; only message in thread
From: Richard Biener @ 2021-10-19 14:09 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:93bd0213885739a1073f8c98911f8a00c0eb5597
commit r12-4509-g93bd0213885739a1073f8c98911f8a00c0eb5597
Author: Richard Biener <rguenther@suse.de>
Date: Mon Oct 18 15:55:22 2021 +0200
Refactor vect_supportable_dr_alignment
This refactors vect_supportable_dr_alignment to get the misalignment
as input parameter which allows us to elide modifying/restoring
of DR_MISALIGNMENT during alignment peeling analysis which eventually
makes it more straight-forward to split out the negative step
handling.
2021-10-19 Richard Biener <rguenther@suse.de>
* tree-vectorizer.h (vect_supportable_dr_alignment): Add
misalignment parameter.
* tree-vect-data-refs.c (vect_get_peeling_costs_all_drs):
Do not change DR_MISALIGNMENT in place, instead pass the
adjusted misalignment to vect_supportable_dr_alignment.
(vect_peeling_supportable): Likewise.
(vect_peeling_hash_get_lowest_cost): Adjust.
(vect_enhance_data_refs_alignment): Likewise.
(vect_vfa_access_size): Likewise.
(vect_supportable_dr_alignment): Add misalignment
parameter and simplify.
* tree-vect-stmts.c (get_negative_load_store_type): Adjust.
(get_group_load_store_type): Likewise.
(get_load_store_type): Likewise.
Diff:
---
gcc/tree-vect-data-refs.c | 113 ++++++++++++++++++++++++++++------------------
gcc/tree-vect-stmts.c | 26 ++++++-----
gcc/tree-vectorizer.h | 2 +-
3 files changed, 85 insertions(+), 56 deletions(-)
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 0db6aec7312..556ae9725f1 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -1529,37 +1529,49 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
unsigned int *outside_cost,
stmt_vector_for_cost *body_cost_vec,
stmt_vector_for_cost *prologue_cost_vec,
- unsigned int npeel,
- bool unknown_misalignment)
+ unsigned int npeel)
{
vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ bool dr0_alignment_known_p
+ = (dr0_info
+ && known_alignment_for_access_p (dr0_info,
+ STMT_VINFO_VECTYPE (dr0_info->stmt)));
+
for (data_reference *dr : datarefs)
{
dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
if (!vect_relevant_for_alignment_p (dr_info))
continue;
- int save_misalignment;
- save_misalignment = dr_info->misalignment;
+ tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
+ dr_alignment_support alignment_support_scheme;
+ int misalignment;
+ unsigned HOST_WIDE_INT alignment;
+
if (npeel == 0)
- ;
- else if (unknown_misalignment && dr_info == dr0_info)
- SET_DR_MISALIGNMENT (dr_info,
- vect_dr_misalign_for_aligned_access (dr0_info));
+ misalignment = dr_misalignment (dr_info, vectype);
+ else if (dr_info == dr0_info
+ || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
+ misalignment = 0;
+ else if (!dr0_alignment_known_p
+ || !known_alignment_for_access_p (dr_info, vectype)
+ || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
+ misalignment = DR_MISALIGNMENT_UNKNOWN;
else
- vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
- /* ??? We should be able to avoid both the adjustment before and the
- call to vect_supportable_dr_alignment below. */
- tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
- int misalignment = dr_misalignment (dr_info, vectype);
- dr_alignment_support alignment_support_scheme
- = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
+ {
+ misalignment = dr_misalignment (dr_info, vectype);
+ misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
+ misalignment &= alignment - 1;
+ }
+ alignment_support_scheme
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
+ misalignment);
+
vect_get_data_access_cost (loop_vinfo, dr_info,
alignment_support_scheme, misalignment,
inside_cost, outside_cost,
body_cost_vec, prologue_cost_vec);
- SET_DR_MISALIGNMENT (dr_info, save_misalignment);
}
}
@@ -1583,7 +1595,7 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost,
&outside_cost, &body_cost_vec,
- &prologue_cost_vec, elem->npeel, false);
+ &prologue_cost_vec, elem->npeel);
body_cost_vec.release ();
@@ -1655,25 +1667,37 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
enum dr_alignment_support supportable_dr_alignment;
+ bool dr0_alignment_known_p
+ = known_alignment_for_access_p (dr0_info,
+ STMT_VINFO_VECTYPE (dr0_info->stmt));
+
/* Ensure that all data refs can be vectorized after the peel. */
for (data_reference *dr : datarefs)
{
- int save_misalignment;
-
if (dr == dr0_info->dr)
continue;
dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
- if (!vect_relevant_for_alignment_p (dr_info))
+ if (!vect_relevant_for_alignment_p (dr_info)
+ || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
continue;
- save_misalignment = dr_info->misalignment;
- vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
+ int misalignment;
+ unsigned HOST_WIDE_INT alignment;
+ if (!dr0_alignment_known_p
+ || !known_alignment_for_access_p (dr_info, vectype)
+ || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
+ misalignment = DR_MISALIGNMENT_UNKNOWN;
+ else
+ {
+ misalignment = dr_misalignment (dr_info, vectype);
+ misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
+ misalignment &= alignment - 1;
+ }
supportable_dr_alignment
- = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
- SET_DR_MISALIGNMENT (dr_info, save_misalignment);
-
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
+ misalignment);
if (supportable_dr_alignment == dr_unaligned_unsupported)
return false;
}
@@ -2017,7 +2041,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
/* Check for data refs with unsupportable alignment that
can be peeled. */
enum dr_alignment_support supportable_dr_alignment
- = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
+ DR_MISALIGNMENT_UNKNOWN);
if (supportable_dr_alignment == dr_unaligned_unsupported)
{
one_dr_unsupportable = true;
@@ -2074,7 +2099,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info,
&load_inside_cost,
&load_outside_cost,
- &dummy, &dummy, estimated_npeels, true);
+ &dummy, &dummy, estimated_npeels);
dummy.release ();
if (first_store)
@@ -2084,7 +2109,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
&store_inside_cost,
&store_outside_cost,
&dummy, &dummy,
- estimated_npeels, true);
+ estimated_npeels);
dummy.release ();
}
else
@@ -2172,8 +2197,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
stmt_vector_for_cost dummy;
dummy.create (2);
vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
- &nopeel_outside_cost, &dummy, &dummy,
- 0, false);
+ &nopeel_outside_cost, &dummy, &dummy, 0);
dummy.release ();
/* Add epilogue costs. As we do not peel for alignment here, no prologue
@@ -2362,7 +2386,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- if (aligned_access_p (dr_info, vectype)
+ int misalignment;
+ if ((misalignment = dr_misalignment (dr_info, vectype)) == 0
|| !vect_relevant_for_alignment_p (dr_info))
continue;
@@ -2373,12 +2398,13 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
}
enum dr_alignment_support supportable_dr_alignment
- = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
+ misalignment);
if (supportable_dr_alignment == dr_unaligned_unsupported)
{
- if (known_alignment_for_access_p (dr_info, vectype)
- || LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
- >= (unsigned) param_vect_max_version_for_alignment_checks)
+ if (misalignment != DR_MISALIGNMENT_UNKNOWN
+ || (LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
+ >= (unsigned) param_vect_max_version_for_alignment_checks))
{
do_versioning = false;
break;
@@ -3321,8 +3347,10 @@ vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
}
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
+ int misalignment;
if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists ()
- && (vect_supportable_dr_alignment (vinfo, dr_info, vectype)
+ && ((misalignment = dr_misalignment (dr_info, vectype)), true)
+ && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
== dr_explicit_realign_optimized))
{
/* We might access a full vector's worth. */
@@ -6638,7 +6666,6 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
}
-
/* Return whether the data reference DR_INFO is supported with respect to its
alignment.
If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
@@ -6647,7 +6674,7 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
enum dr_alignment_support
vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
- tree vectype)
+ tree vectype, int misalignment)
{
data_reference *dr = dr_info->dr;
stmt_vec_info stmt_info = dr_info->stmt;
@@ -6656,7 +6683,7 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
class loop *vect_loop = NULL;
bool nested_in_vect_loop = false;
- if (aligned_access_p (dr_info, vectype))
+ if (misalignment == 0)
return dr_aligned;
/* For now assume all conditional loads/stores support unaligned
@@ -6762,11 +6789,11 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
}
bool is_packed = false;
- tree type = (TREE_TYPE (DR_REF (dr)));
- if (!known_alignment_for_access_p (dr_info, vectype))
+ tree type = TREE_TYPE (DR_REF (dr));
+ if (misalignment == DR_MISALIGNMENT_UNKNOWN)
is_packed = not_size_aligned (DR_REF (dr));
- if (targetm.vectorize.support_vector_misalignment
- (mode, type, dr_misalignment (dr_info, vectype), is_packed))
+ if (targetm.vectorize.support_vector_misalignment (mode, type, misalignment,
+ is_packed))
return dr_unaligned_supported;
/* Unsupported. */
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index afc3ef17834..9cbc1af4cc9 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -1986,8 +1986,9 @@ get_negative_load_store_type (vec_info *vinfo,
return VMAT_ELEMENTWISE;
}
- alignment_support_scheme = vect_supportable_dr_alignment (vinfo, dr_info,
- vectype);
+ int misalignment = dr_misalignment (dr_info, vectype);
+ alignment_support_scheme
+ = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
if (alignment_support_scheme != dr_aligned
&& alignment_support_scheme != dr_unaligned_supported)
{
@@ -2184,15 +2185,15 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
can do half-vector operations avoid the epilogue peeling
by simply loading half of the vector only. Usually
the construction with an upper zero half will be elided. */
- dr_alignment_support alignment_support_scheme;
+ dr_alignment_support alss;
+ int misalign = dr_misalignment (first_dr_info, vectype);
tree half_vtype;
if (overrun_p
&& !masked_p
- && (((alignment_support_scheme
- = vect_supportable_dr_alignment (vinfo, first_dr_info,
- vectype)))
+ && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
+ vectype, misalign)))
== dr_aligned
- || alignment_support_scheme == dr_unaligned_supported)
+ || alss == dr_unaligned_supported)
&& known_eq (nunits, (group_size - gap) * 2)
&& known_eq (nunits, group_size)
&& (vector_vector_composition_type (vectype, 2, &half_vtype)
@@ -2304,9 +2305,10 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
}
else
{
- *alignment_support_scheme
- = vect_supportable_dr_alignment (vinfo, first_dr_info, vectype);
*misalignment = dr_misalignment (first_dr_info, vectype);
+ *alignment_support_scheme
+ = vect_supportable_dr_alignment (vinfo, first_dr_info, vectype,
+ *misalignment);
}
if (vls_type != VLS_LOAD && first_stmt_info == stmt_info)
@@ -2452,12 +2454,12 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
(vinfo, stmt_info, vectype, vls_type, ncopies, poffset);
else
*memory_access_type = VMAT_CONTIGUOUS;
+ *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info),
+ vectype);
*alignment_support_scheme
= vect_supportable_dr_alignment (vinfo,
STMT_VINFO_DR_INFO (stmt_info),
- vectype);
- *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info),
- vectype);
+ vectype, *misalignment);
}
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 746e39207d0..866d813a12c 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1980,7 +1980,7 @@ extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
extern enum dr_alignment_support vect_supportable_dr_alignment
- (vec_info *, dr_vec_info *, tree);
+ (vec_info *, dr_vec_info *, tree, int);
extern tree vect_get_smallest_scalar_type (stmt_vec_info, tree);
extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2021-10-19 14:09 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-10-19 14:09 [gcc r12-4509] Refactor vect_supportable_dr_alignment Richard Biener
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).