From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1666) id 080CD3858D3C; Tue, 19 Oct 2021 11:35:16 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 080CD3858D3C MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="utf-8" From: Richard Biener To: gcc-cvs@gcc.gnu.org Subject: [gcc r12-4501] Refactor load/store costing X-Act-Checkin: gcc X-Git-Author: Richard Biener X-Git-Refname: refs/heads/master X-Git-Oldrev: 9890b12c72c02828c691f22198c3e0afd8678991 X-Git-Newrev: 793d2549b173a0a2da6dd20ffc27acb9fd2de73e Message-Id: <20211019113516.080CD3858D3C@sourceware.org> Date: Tue, 19 Oct 2021 11:35:16 +0000 (GMT) X-BeenThere: gcc-cvs@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 19 Oct 2021 11:35:16 -0000 https://gcc.gnu.org/g:793d2549b173a0a2da6dd20ffc27acb9fd2de73e commit r12-4501-g793d2549b173a0a2da6dd20ffc27acb9fd2de73e Author: Richard Biener Date: Tue Oct 19 12:40:59 2021 +0200 Refactor load/store costing This passes down the already available alignment scheme and misalignment to the load/store costing routines, removing redundant queries. 2021-10-19 Richard Biener * tree-vectorizer.h (vect_get_store_cost): Adjust signature. (vect_get_load_cost): Likewise. * tree-vect-data-refs.c (vect_get_data_access_cost): Get alignment support scheme and misalignment as arguments and pass them down. (vect_get_peeling_costs_all_drs): Compute that info here and note that we shouldn't need to. * tree-vect-stmts.c (vect_model_store_cost): Get alignment support scheme and misalignment as arguments. (vect_get_store_cost): Likewise. (vect_model_load_cost): Likewise. (vect_get_load_cost): Likewise. (vectorizable_store): Pass down alignment support scheme and misalignment to costing. (vectorizable_load): Likewise. Diff: --- gcc/tree-vect-data-refs.c | 20 ++++++++++++++++---- gcc/tree-vect-stmts.c | 41 +++++++++++++++++++++-------------------- gcc/tree-vectorizer.h | 4 +++- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 7c95f9ad69e..0db6aec7312 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -1396,7 +1396,9 @@ vector_alignment_reachable_p (dr_vec_info *dr_info) static void vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info, - unsigned int *inside_cost, + dr_alignment_support alignment_support_scheme, + int misalignment, + unsigned int *inside_cost, unsigned int *outside_cost, stmt_vector_for_cost *body_cost_vec, stmt_vector_for_cost *prologue_cost_vec) @@ -1411,10 +1413,12 @@ vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info, ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info)); if (DR_IS_READ (dr_info->dr)) - vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost, + vect_get_load_cost (vinfo, stmt_info, ncopies, alignment_support_scheme, + misalignment, true, inside_cost, outside_cost, prologue_cost_vec, body_cost_vec, false); else - vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec); + vect_get_store_cost (vinfo,stmt_info, ncopies, alignment_support_scheme, + misalignment, inside_cost, body_cost_vec); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1545,7 +1549,15 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo, vect_dr_misalign_for_aligned_access (dr0_info)); else vect_update_misalignment_for_peel (dr_info, dr0_info, npeel); - vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost, + /* ??? We should be able to avoid both the adjustment before and the + call to vect_supportable_dr_alignment below. */ + tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt); + int misalignment = dr_misalignment (dr_info, vectype); + dr_alignment_support alignment_support_scheme + = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype); + vect_get_data_access_cost (loop_vinfo, dr_info, + alignment_support_scheme, misalignment, + inside_cost, outside_cost, body_cost_vec, prologue_cost_vec); SET_DR_MISALIGNMENT (dr_info, save_misalignment); } diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 09a97b44c91..afc3ef17834 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -909,6 +909,8 @@ cfun_returns (tree decl) static void vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, vect_memory_access_type memory_access_type, + dr_alignment_support alignment_support_scheme, + int misalignment, vec_load_store_type vls_type, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { @@ -969,7 +971,8 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, scalar_store, stmt_info, 0, vect_body); } else - vect_get_store_cost (vinfo, stmt_info, ncopies, &inside_cost, cost_vec); + vect_get_store_cost (vinfo, stmt_info, ncopies, alignment_support_scheme, + misalignment, &inside_cost, cost_vec); if (memory_access_type == VMAT_ELEMENTWISE || memory_access_type == VMAT_STRIDED_SLP) @@ -1021,15 +1024,12 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, /* Calculate cost of DR's memory access. */ void -vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, +vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, int ncopies, + dr_alignment_support alignment_support_scheme, + int misalignment, unsigned int *inside_cost, stmt_vector_for_cost *body_cost_vec) { - dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); - dr_alignment_support alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, dr_info, vectype); - switch (alignment_support_scheme) { case dr_aligned: @@ -1049,8 +1049,7 @@ vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, /* Here, we assign an additional cost for the unaligned store. */ *inside_cost += record_stmt_cost (body_cost_vec, ncopies, unaligned_store, stmt_info, - dr_misalignment (dr_info, vectype), - vect_body); + misalignment, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "vect_model_store_cost: unaligned supported by " @@ -1085,6 +1084,8 @@ static void vect_model_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf, vect_memory_access_type memory_access_type, + dr_alignment_support alignment_support_scheme, + int misalignment, gather_scatter_info *gs_info, slp_tree slp_node, stmt_vector_for_cost *cost_vec) @@ -1144,7 +1145,8 @@ vect_model_load_cost (vec_info *vinfo, dump_printf_loc (MSG_NOTE, vect_location, "vect_model_load_cost: %d unused vectors.\n", gaps); - vect_get_load_cost (vinfo, stmt_info, ncopies * gaps, false, + vect_get_load_cost (vinfo, stmt_info, ncopies * gaps, + alignment_support_scheme, misalignment, false, &inside_cost, &prologue_cost, cost_vec, cost_vec, true); } @@ -1190,7 +1192,8 @@ vect_model_load_cost (vec_info *vinfo, scalar_load, stmt_info, 0, vect_body); } else - vect_get_load_cost (vinfo, stmt_info, ncopies, first_stmt_p, + vect_get_load_cost (vinfo, stmt_info, ncopies, + alignment_support_scheme, misalignment, first_stmt_p, &inside_cost, &prologue_cost, cost_vec, cost_vec, true); if (memory_access_type == VMAT_ELEMENTWISE @@ -1209,18 +1212,15 @@ vect_model_load_cost (vec_info *vinfo, /* Calculate cost of DR's memory access. */ void -vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, +vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, int ncopies, + dr_alignment_support alignment_support_scheme, + int misalignment, bool add_realign_cost, unsigned int *inside_cost, unsigned int *prologue_cost, stmt_vector_for_cost *prologue_cost_vec, stmt_vector_for_cost *body_cost_vec, bool record_prologue_costs) { - dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); - dr_alignment_support alignment_support_scheme - = vect_supportable_dr_alignment (vinfo, dr_info, vectype); - switch (alignment_support_scheme) { case dr_aligned: @@ -1239,8 +1239,7 @@ vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies, /* Here, we assign an additional cost for the unaligned load. */ *inside_cost += record_stmt_cost (body_cost_vec, ncopies, unaligned_load, stmt_info, - dr_misalignment (dr_info, vectype), - vect_body); + misalignment, vect_body); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -7443,7 +7442,8 @@ vectorizable_store (vec_info *vinfo, STMT_VINFO_TYPE (stmt_info) = store_vec_info_type; vect_model_store_cost (vinfo, stmt_info, ncopies, - memory_access_type, vls_type, slp_node, cost_vec); + memory_access_type, alignment_support_scheme, + misalignment, vls_type, slp_node, cost_vec); return true; } gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)); @@ -8779,6 +8779,7 @@ vectorizable_load (vec_info *vinfo, STMT_VINFO_TYPE (stmt_info) = load_vec_info_type; vect_model_load_cost (vinfo, stmt_info, ncopies, vf, memory_access_type, + alignment_support_scheme, misalignment, &gs_info, slp_node, cost_vec); return true; } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 954a0e18d03..746e39207d0 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1957,11 +1957,13 @@ extern bool vect_nop_conversion_p (stmt_vec_info); extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *); -extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, bool, +extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, + dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool); extern void vect_get_store_cost (vec_info *, stmt_vec_info, int, + dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *); extern bool vect_supportable_shift (vec_info *, enum tree_code, tree); extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);