From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1666) id 9017A385803B; Tue, 19 Oct 2021 14:09:12 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 9017A385803B MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="utf-8" From: Richard Biener To: gcc-cvs@gcc.gnu.org Subject: [gcc r12-4509] Refactor vect_supportable_dr_alignment X-Act-Checkin: gcc X-Git-Author: Richard Biener X-Git-Refname: refs/heads/master X-Git-Oldrev: 5a8832b1659e311437d25b7ec8b078be27ae54b8 X-Git-Newrev: 93bd0213885739a1073f8c98911f8a00c0eb5597 Message-Id: <20211019140912.9017A385803B@sourceware.org> Date: Tue, 19 Oct 2021 14:09:12 +0000 (GMT) X-BeenThere: gcc-cvs@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 19 Oct 2021 14:09:12 -0000 https://gcc.gnu.org/g:93bd0213885739a1073f8c98911f8a00c0eb5597 commit r12-4509-g93bd0213885739a1073f8c98911f8a00c0eb5597 Author: Richard Biener Date: Mon Oct 18 15:55:22 2021 +0200 Refactor vect_supportable_dr_alignment This refactors vect_supportable_dr_alignment to get the misalignment as input parameter which allows us to elide modifying/restoring of DR_MISALIGNMENT during alignment peeling analysis which eventually makes it more straight-forward to split out the negative step handling. 2021-10-19 Richard Biener * tree-vectorizer.h (vect_supportable_dr_alignment): Add misalignment parameter. * tree-vect-data-refs.c (vect_get_peeling_costs_all_drs): Do not change DR_MISALIGNMENT in place, instead pass the adjusted misalignment to vect_supportable_dr_alignment. (vect_peeling_supportable): Likewise. (vect_peeling_hash_get_lowest_cost): Adjust. (vect_enhance_data_refs_alignment): Likewise. (vect_vfa_access_size): Likewise. (vect_supportable_dr_alignment): Add misalignment parameter and simplify. * tree-vect-stmts.c (get_negative_load_store_type): Adjust. (get_group_load_store_type): Likewise. (get_load_store_type): Likewise. Diff: --- gcc/tree-vect-data-refs.c | 113 ++++++++++++++++++++++++++++------------------ gcc/tree-vect-stmts.c | 26 ++++++----- gcc/tree-vectorizer.h | 2 +- 3 files changed, 85 insertions(+), 56 deletions(-) diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 0db6aec7312..556ae9725f1 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -1529,37 +1529,49 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo, unsigned int *outside_cost, stmt_vector_for_cost *body_cost_vec, stmt_vector_for_cost *prologue_cost_vec, - unsigned int npeel, - bool unknown_misalignment) + unsigned int npeel) { vec datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); + bool dr0_alignment_known_p + = (dr0_info + && known_alignment_for_access_p (dr0_info, + STMT_VINFO_VECTYPE (dr0_info->stmt))); + for (data_reference *dr : datarefs) { dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); if (!vect_relevant_for_alignment_p (dr_info)) continue; - int save_misalignment; - save_misalignment = dr_info->misalignment; + tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt); + dr_alignment_support alignment_support_scheme; + int misalignment; + unsigned HOST_WIDE_INT alignment; + if (npeel == 0) - ; - else if (unknown_misalignment && dr_info == dr0_info) - SET_DR_MISALIGNMENT (dr_info, - vect_dr_misalign_for_aligned_access (dr0_info)); + misalignment = dr_misalignment (dr_info, vectype); + else if (dr_info == dr0_info + || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info)) + misalignment = 0; + else if (!dr0_alignment_known_p + || !known_alignment_for_access_p (dr_info, vectype) + || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)) + misalignment = DR_MISALIGNMENT_UNKNOWN; else - vect_update_misalignment_for_peel (dr_info, dr0_info, npeel); - /* ??? We should be able to avoid both the adjustment before and the - call to vect_supportable_dr_alignment below. */ - tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt); - int misalignment = dr_misalignment (dr_info, vectype); - dr_alignment_support alignment_support_scheme - = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype); + { + misalignment = dr_misalignment (dr_info, vectype); + misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr)); + misalignment &= alignment - 1; + } + alignment_support_scheme + = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype, + misalignment); + vect_get_data_access_cost (loop_vinfo, dr_info, alignment_support_scheme, misalignment, inside_cost, outside_cost, body_cost_vec, prologue_cost_vec); - SET_DR_MISALIGNMENT (dr_info, save_misalignment); } } @@ -1583,7 +1595,7 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot, vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost, &outside_cost, &body_cost_vec, - &prologue_cost_vec, elem->npeel, false); + &prologue_cost_vec, elem->npeel); body_cost_vec.release (); @@ -1655,25 +1667,37 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info, vec datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); enum dr_alignment_support supportable_dr_alignment; + bool dr0_alignment_known_p + = known_alignment_for_access_p (dr0_info, + STMT_VINFO_VECTYPE (dr0_info->stmt)); + /* Ensure that all data refs can be vectorized after the peel. */ for (data_reference *dr : datarefs) { - int save_misalignment; - if (dr == dr0_info->dr) continue; dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); - if (!vect_relevant_for_alignment_p (dr_info)) + if (!vect_relevant_for_alignment_p (dr_info) + || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info)) continue; - save_misalignment = dr_info->misalignment; - vect_update_misalignment_for_peel (dr_info, dr0_info, npeel); tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt); + int misalignment; + unsigned HOST_WIDE_INT alignment; + if (!dr0_alignment_known_p + || !known_alignment_for_access_p (dr_info, vectype) + || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)) + misalignment = DR_MISALIGNMENT_UNKNOWN; + else + { + misalignment = dr_misalignment (dr_info, vectype); + misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr)); + misalignment &= alignment - 1; + } supportable_dr_alignment - = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype); - SET_DR_MISALIGNMENT (dr_info, save_misalignment); - + = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype, + misalignment); if (supportable_dr_alignment == dr_unaligned_unsupported) return false; } @@ -2017,7 +2041,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) /* Check for data refs with unsupportable alignment that can be peeled. */ enum dr_alignment_support supportable_dr_alignment - = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype); + = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype, + DR_MISALIGNMENT_UNKNOWN); if (supportable_dr_alignment == dr_unaligned_unsupported) { one_dr_unsupportable = true; @@ -2074,7 +2099,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info, &load_inside_cost, &load_outside_cost, - &dummy, &dummy, estimated_npeels, true); + &dummy, &dummy, estimated_npeels); dummy.release (); if (first_store) @@ -2084,7 +2109,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) &store_inside_cost, &store_outside_cost, &dummy, &dummy, - estimated_npeels, true); + estimated_npeels); dummy.release (); } else @@ -2172,8 +2197,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) stmt_vector_for_cost dummy; dummy.create (2); vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost, - &nopeel_outside_cost, &dummy, &dummy, - 0, false); + &nopeel_outside_cost, &dummy, &dummy, 0); dummy.release (); /* Add epilogue costs. As we do not peel for alignment here, no prologue @@ -2362,7 +2386,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); stmt_vec_info stmt_info = dr_info->stmt; tree vectype = STMT_VINFO_VECTYPE (stmt_info); - if (aligned_access_p (dr_info, vectype) + int misalignment; + if ((misalignment = dr_misalignment (dr_info, vectype)) == 0 || !vect_relevant_for_alignment_p (dr_info)) continue; @@ -2373,12 +2398,13 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) } enum dr_alignment_support supportable_dr_alignment - = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype); + = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype, + misalignment); if (supportable_dr_alignment == dr_unaligned_unsupported) { - if (known_alignment_for_access_p (dr_info, vectype) - || LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length () - >= (unsigned) param_vect_max_version_for_alignment_checks) + if (misalignment != DR_MISALIGNMENT_UNKNOWN + || (LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length () + >= (unsigned) param_vect_max_version_for_alignment_checks)) { do_versioning = false; break; @@ -3321,8 +3347,10 @@ vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info) access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo); } tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo); + int misalignment; if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists () - && (vect_supportable_dr_alignment (vinfo, dr_info, vectype) + && ((misalignment = dr_misalignment (dr_info, vectype)), true) + && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment) == dr_explicit_realign_optimized)) { /* We might access a full vector's worth. */ @@ -6638,7 +6666,6 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment) return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT)); } - /* Return whether the data reference DR_INFO is supported with respect to its alignment. If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even @@ -6647,7 +6674,7 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment) enum dr_alignment_support vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info, - tree vectype) + tree vectype, int misalignment) { data_reference *dr = dr_info->dr; stmt_vec_info stmt_info = dr_info->stmt; @@ -6656,7 +6683,7 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info, class loop *vect_loop = NULL; bool nested_in_vect_loop = false; - if (aligned_access_p (dr_info, vectype)) + if (misalignment == 0) return dr_aligned; /* For now assume all conditional loads/stores support unaligned @@ -6762,11 +6789,11 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info, } bool is_packed = false; - tree type = (TREE_TYPE (DR_REF (dr))); - if (!known_alignment_for_access_p (dr_info, vectype)) + tree type = TREE_TYPE (DR_REF (dr)); + if (misalignment == DR_MISALIGNMENT_UNKNOWN) is_packed = not_size_aligned (DR_REF (dr)); - if (targetm.vectorize.support_vector_misalignment - (mode, type, dr_misalignment (dr_info, vectype), is_packed)) + if (targetm.vectorize.support_vector_misalignment (mode, type, misalignment, + is_packed)) return dr_unaligned_supported; /* Unsupported. */ diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index afc3ef17834..9cbc1af4cc9 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1986,8 +1986,9 @@ get_negative_load_store_type (vec_info *vinfo, return VMAT_ELEMENTWISE; } - alignment_support_scheme = vect_supportable_dr_alignment (vinfo, dr_info, - vectype); + int misalignment = dr_misalignment (dr_info, vectype); + alignment_support_scheme + = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment); if (alignment_support_scheme != dr_aligned && alignment_support_scheme != dr_unaligned_supported) { @@ -2184,15 +2185,15 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, can do half-vector operations avoid the epilogue peeling by simply loading half of the vector only. Usually the construction with an upper zero half will be elided. */ - dr_alignment_support alignment_support_scheme; + dr_alignment_support alss; + int misalign = dr_misalignment (first_dr_info, vectype); tree half_vtype; if (overrun_p && !masked_p - && (((alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, first_dr_info, - vectype))) + && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info, + vectype, misalign))) == dr_aligned - || alignment_support_scheme == dr_unaligned_supported) + || alss == dr_unaligned_supported) && known_eq (nunits, (group_size - gap) * 2) && known_eq (nunits, group_size) && (vector_vector_composition_type (vectype, 2, &half_vtype) @@ -2304,9 +2305,10 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, } else { - *alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, first_dr_info, vectype); *misalignment = dr_misalignment (first_dr_info, vectype); + *alignment_support_scheme + = vect_supportable_dr_alignment (vinfo, first_dr_info, vectype, + *misalignment); } if (vls_type != VLS_LOAD && first_stmt_info == stmt_info) @@ -2452,12 +2454,12 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, (vinfo, stmt_info, vectype, vls_type, ncopies, poffset); else *memory_access_type = VMAT_CONTIGUOUS; + *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info), + vectype); *alignment_support_scheme = vect_supportable_dr_alignment (vinfo, STMT_VINFO_DR_INFO (stmt_info), - vectype); - *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info), - vectype); + vectype, *misalignment); } } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 746e39207d0..866d813a12c 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1980,7 +1980,7 @@ extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0); /* In tree-vect-data-refs.c. */ extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64); extern enum dr_alignment_support vect_supportable_dr_alignment - (vec_info *, dr_vec_info *, tree); + (vec_info *, dr_vec_info *, tree, int); extern tree vect_get_smallest_scalar_type (stmt_vec_info, tree); extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *); extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);