From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1851) id AB3F9388E826; Mon, 29 Jun 2020 09:16:21 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org AB3F9388E826 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gcc.gnu.org; s=default; t=1593422181; bh=aJCEji9wNgnV+QjQEehrlFVv8ESaudc1QzqVS7kN8MI=; h=From:To:Subject:Date:From; b=x7Omt/fGlcpdKS0MHQMQtYBMKeUil3TT28kbILq5LIQVmOgBfSe0fAMIWljwHZftv 9qAug2P7HkFFR0MPblMyV4zSruQuszP2yFmXUxwZ0DUYwnmYfftoqjuntStXFtVrKM qfcDBhf75BIeovQ8cQnWk0Gz5YeVRFdmCX/9/9H0= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Martin Liska To: gcc-cvs@gcc.gnu.org Subject: [gcc(refs/users/marxin/heads/slp-function-v7)] SLP: vectorize entire function. X-Act-Checkin: gcc X-Git-Author: Martin Liska X-Git-Refname: refs/users/marxin/heads/slp-function-v7 X-Git-Oldrev: 69273534c658949129ac6c389ea25c9cddec3767 X-Git-Newrev: 6dd5e8c2b0e5e5136f709ef4e7263a0e841a3a8e Message-Id: <20200629091621.AB3F9388E826@sourceware.org> Date: Mon, 29 Jun 2020 09:16:21 +0000 (GMT) X-BeenThere: gcc-cvs@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 29 Jun 2020 09:16:21 -0000 https://gcc.gnu.org/g:6dd5e8c2b0e5e5136f709ef4e7263a0e841a3a8e commit 6dd5e8c2b0e5e5136f709ef4e7263a0e841a3a8e Author: Martin Liska Date: Thu Jun 11 13:25:40 2020 +0200 SLP: vectorize entire function. Diff: --- gcc/tree-vect-data-refs.c | 30 ++++++--- gcc/tree-vect-loop.c | 5 +- gcc/tree-vect-slp.c | 161 +++++++++++++++++++++++++++++++++++++--------- gcc/tree-vect-stmts.c | 5 +- gcc/tree-vectorizer.c | 23 ++++++- gcc/tree-vectorizer.h | 29 +++++++-- 6 files changed, 205 insertions(+), 48 deletions(-) diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index eb8288e7a85..92ad181addb 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -2853,10 +2853,10 @@ dr_group_sort_cmp (const void *dra_, const void *drb_) return 0; /* DRs in different loops never belong to the same group. */ - loop_p loopa = gimple_bb (DR_STMT (dra))->loop_father; - loop_p loopb = gimple_bb (DR_STMT (drb))->loop_father; - if (loopa != loopb) - return loopa->num < loopb->num ? -1 : 1; + int bb_index1 = gimple_bb (DR_STMT (dra))->index; + int bb_index2 = gimple_bb (DR_STMT (drb))->index; + if (bb_index1 != bb_index2) + return bb_index1 < bb_index2 ? -1 : 1; /* Ordering of DRs according to base. */ cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra), @@ -2962,7 +2962,8 @@ can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info, FORNOW: handle only arrays and pointer accesses. */ opt_result -vect_analyze_data_ref_accesses (vec_info *vinfo) +vect_analyze_data_ref_accesses (vec_info *vinfo, + vec *dataref_groups) { unsigned int i; vec datarefs = vinfo->shared->datarefs; @@ -2984,6 +2985,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo) for (i = 0; i < datarefs_copy.length () - 1;) { data_reference_p dra = datarefs_copy[i]; + int dra_group_id = dataref_groups ? (*dataref_groups)[i] : 0; dr_vec_info *dr_info_a = vinfo->lookup_dr (dra); stmt_vec_info stmtinfo_a = dr_info_a->stmt; stmt_vec_info lastinfo = NULL; @@ -2996,6 +2998,7 @@ vect_analyze_data_ref_accesses (vec_info *vinfo) for (i = i + 1; i < datarefs_copy.length (); ++i) { data_reference_p drb = datarefs_copy[i]; + int drb_group_id = dataref_groups ? (*dataref_groups)[i] : 0; dr_vec_info *dr_info_b = vinfo->lookup_dr (drb); stmt_vec_info stmtinfo_b = dr_info_b->stmt; if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b) @@ -3008,10 +3011,14 @@ vect_analyze_data_ref_accesses (vec_info *vinfo) matters we can push those to a worklist and re-iterate over them. The we can just skip ahead to the next DR here. */ - /* DRs in a different loop should not be put into the same + /* DRs in a different BBs should not be put into the same interleaving group. */ - if (gimple_bb (DR_STMT (dra))->loop_father - != gimple_bb (DR_STMT (drb))->loop_father) + int bb_index1 = gimple_bb (DR_STMT (dra))->index; + int bb_index2 = gimple_bb (DR_STMT (drb))->index; + if (bb_index1 != bb_index2) + break; + + if (dra_group_id != drb_group_id) break; /* Check that the data-refs have same first location (except init) @@ -4070,7 +4077,8 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, opt_result vect_find_stmt_data_reference (loop_p loop, gimple *stmt, - vec *datarefs) + vec *datarefs, + vec *dataref_groups, int group_id) { /* We can ignore clobbers for dataref analysis - they are removed during loop vectorization and BB vectorization checks dependences with a @@ -4197,6 +4205,8 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt, newdr->aux = (void *) (-1 - tree_to_uhwi (arg2)); free_data_ref (dr); datarefs->safe_push (newdr); + if (dataref_groups) + dataref_groups->safe_push (group_id); return opt_result::success (); } } @@ -4206,6 +4216,8 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt, } datarefs->safe_push (dr); + if (dataref_groups) + dataref_groups->safe_push (group_id); return opt_result::success (); } diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index bc913eeeb36..e60b9bd718b 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -1788,7 +1788,8 @@ vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs, if (is_gimple_debug (stmt)) continue; ++(*n_stmts); - opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs); + opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs, + NULL, 0); if (!res) { if (is_gimple_call (stmt) && loop->safelen) @@ -2010,7 +2011,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, unsigned *n_stmts) /* Analyze the access patterns of the data-refs in the loop (consecutive, complex, etc.). FORNOW: Only handle consecutive access pattern. */ - ok = vect_analyze_data_ref_accesses (loop_vinfo); + ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL); if (!ok) { if (dump_enabled_p ()) diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index b223956e3af..192803eddfb 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -2636,8 +2636,6 @@ _bb_vec_info::_bb_vec_info (gimple_stmt_iterator region_begin_in, continue; add_stmt (stmt); } - - bb->aux = this; } @@ -3127,7 +3125,8 @@ vect_slp_check_for_constructors (bb_vec_info bb_vinfo) region. */ static bool -vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal) +vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal, + vec *dataref_groups) { DUMP_VECT_SCOPE ("vect_slp_analyze_bb"); @@ -3149,7 +3148,7 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal) return false; } - if (!vect_analyze_data_ref_accesses (bb_vinfo)) + if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -3258,10 +3257,11 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal) given by DATAREFS. */ static bool -vect_slp_bb_region (gimple_stmt_iterator region_begin, - gimple_stmt_iterator region_end, - vec datarefs, - unsigned int n_stmts) +vect_slp_region (gimple_stmt_iterator region_begin, + gimple_stmt_iterator region_end, + vec datarefs, + vec *dataref_groups, + unsigned int n_stmts) { bb_vec_info bb_vinfo; auto_vector_modes vector_modes; @@ -3288,7 +3288,7 @@ vect_slp_bb_region (gimple_stmt_iterator region_begin, bb_vinfo->shared->check_datarefs (); bb_vinfo->vector_mode = next_vector_mode; - if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal) + if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups) && dbg_cnt (vect_slp)) { if (dump_enabled_p ()) @@ -3408,7 +3408,7 @@ vect_slp_bb (basic_block bb) if (gimple_location (stmt) != UNKNOWN_LOCATION) vect_location = stmt; - if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs)) + if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs, NULL, 0)) break; } if (gsi_end_p (region_begin)) @@ -3430,7 +3430,7 @@ vect_slp_bb (basic_block bb) "not vectorized: too many instructions in " "basic block.\n"); } - else if (vect_slp_bb_region (region_begin, region_end, datarefs, insns)) + else if (vect_slp_region (region_begin, region_end, datarefs, NULL, insns)) any_vectorized = true; if (gsi_end_p (region_end)) @@ -3443,6 +3443,62 @@ vect_slp_bb (basic_block bb) return any_vectorized; } +bool +vect_slp_function () +{ + basic_block bb; + vec_info_shared shared; + vec datarefs = vNULL; + vec dataref_groups = vNULL; + bool any_vectorized = false; + int insns = 0; + gimple_stmt_iterator region_begin + = gsi_after_labels (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb); + // TODO + while (gsi_stmt (region_begin) == NULL) + { + basic_block next_bb = region_begin.bb->next_bb; + if (next_bb == NULL) + return false; + region_begin = gsi_after_labels (next_bb); + } + + gimple_stmt_iterator region_end + = gsi_last_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb); + if (!gsi_end_p (region_end)) + gsi_next (®ion_end); + int current_group = 0; + + FOR_EACH_BB_FN (bb, cfun) + { + for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi); + gsi_next (&gsi)) + { + gimple *stmt = gsi_stmt (gsi); + if (is_gimple_debug (stmt)) + continue; + + insns++; + + if (gimple_location (stmt) != UNKNOWN_LOCATION) + vect_location = stmt; + + if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs, + &dataref_groups, current_group)) + ++current_group; + } + + ++current_group; + } + + if (vect_slp_region (region_begin, region_end, datarefs, &dataref_groups, + insns)) + any_vectorized = true; + + return any_vectorized; +} + + /* Build a variable-length vector in which the elements in ELTS are repeated to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in @@ -3611,7 +3667,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) constant_p = true; tree_vector_builder elts (vector_type, nunits, 1); elts.quick_grow (nunits); - stmt_vec_info insert_after = NULL; + gimple_stmt_iterator insert_before = gsi_none (); for (j = 0; j < number_of_copies; j++) { tree op; @@ -3675,15 +3731,67 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) when a def is inside the analyzed region since we cannot simply insert at the BB start in this case. */ stmt_vec_info opdef; - if (TREE_CODE (orig_op) == SSA_NAME - && !SSA_NAME_IS_DEFAULT_DEF (orig_op) - && is_a (vinfo) - && (opdef = vinfo->lookup_def (orig_op))) + if (is_a (vinfo)) { - if (!insert_after) - insert_after = opdef; - else - insert_after = get_later_stmt (insert_after, opdef); + gimple_stmt_iterator to_insert = gsi_none (); + if (TREE_CODE (orig_op) == SSA_NAME) + { + if (SSA_NAME_IS_DEFAULT_DEF (orig_op)) + to_insert = gsi_after_labels (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb); + else if ((opdef = vinfo->lookup_def (orig_op))) + { + to_insert = gsi_for_stmt (opdef->stmt); + gsi_next (&to_insert); + } + + else + { + gimple *def = SSA_NAME_DEF_STMT (orig_op); + if (is_a (def)) + to_insert = gsi_after_labels (gimple_bb (def)); + else + { + to_insert = gsi_for_stmt (def); + gsi_next (&to_insert); + } + } + } + else if (CONSTANT_CLASS_P (orig_op) + || TREE_CODE (orig_op) == ADDR_EXPR) + to_insert = gsi_after_labels (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb); + + if (gsi_bb (to_insert)) + { + if (gsi_bb (insert_before)) + { + bool dom0 = vect_stmt_dominates_stmt_p (gsi_stmt (insert_before), + gsi_bb (insert_before), + gsi_stmt (to_insert), + gsi_bb (to_insert)); + bool dom1 = vect_stmt_dominates_stmt_p (gsi_stmt (to_insert), + gsi_bb (to_insert), + gsi_stmt (insert_before), + gsi_bb (insert_before)); + if (dom0 ==false && dom1 == false) + { + gcc_assert (gsi_bb (to_insert) == gsi_bb (insert_before)); + gimple_stmt_iterator it = to_insert; + while (!gsi_end_p (it)) + { + if (gsi_stmt (it) == gsi_stmt (insert_before)) + { + insert_before = to_insert; + break; + } + gsi_prev (&it); + } + } + else if (dom0) + insert_before = to_insert; + } + else + insert_before = to_insert; + } } if (number_of_places_left_in_vector == 0) @@ -3701,14 +3809,9 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) vec_cst = permute_results[number_of_vectors - j - 1]; } tree init; - if (insert_after) - { - gimple_stmt_iterator gsi = gsi_for_stmt (insert_after->stmt); - /* vect_init_vector inserts before. */ - gsi_next (&gsi); - init = vect_init_vector (vinfo, NULL, vec_cst, - vector_type, &gsi); - } + if (gsi_bb (insert_before)) + init = vect_init_vector (vinfo, NULL, vec_cst, + vector_type, &insert_before); else init = vect_init_vector (vinfo, NULL, vec_cst, vector_type, NULL); @@ -3720,7 +3823,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) ctor_seq = NULL; } voprnds.quick_push (init); - insert_after = NULL; + insert_before = gsi_none (); number_of_places_left_in_vector = nunits; constant_p = true; elts.new_vector (vector_type, nunits, 1); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 9b7b04ce2d3..20ca1bd55f7 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1334,8 +1334,9 @@ vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt, else { bb_vec_info bb_vinfo = dyn_cast (vinfo); - gimple_stmt_iterator gsi_region_begin = bb_vinfo->region_begin; - gsi_insert_before (&gsi_region_begin, new_stmt, GSI_SAME_STMT); + gimple_stmt_iterator gsi + = gsi_after_labels (gsi_bb (bb_vinfo->region_begin)); + gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT); } } diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 78d9da689c8..6f11d98e3f1 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -717,7 +717,16 @@ bool vect_stmt_dominates_stmt_p (gimple *s1, gimple *s2) { basic_block bb1 = gimple_bb (s1), bb2 = gimple_bb (s2); + return vect_stmt_dominates_stmt_p (s1, bb1, s2, bb2); +} + +/* Returns true if S1 dominates S2. Any of S1 and S2 can be null + and then basic blocks BB1 and BB2 are compared. */ +bool +vect_stmt_dominates_stmt_p (gimple *s1, basic_block bb1, + gimple *s2, basic_block bb2) +{ /* If bb1 is NULL, it should be a GIMPLE_NOP def stmt of an (D) SSA_NAME. Assume it lives at the beginning of function and thus dominates everything. */ @@ -731,6 +740,11 @@ vect_stmt_dominates_stmt_p (gimple *s1, gimple *s2) if (bb1 != bb2) return dominated_by_p (CDI_DOMINATORS, bb2, bb1); + if (s1 == NULL) + return false; + else if (s2 == NULL) + return true; + /* PHIs in the same basic block are assumed to be executed all in parallel, if only one stmt is a PHI, it dominates the other stmt in the same basic block. */ @@ -1379,12 +1393,17 @@ pass_slp_vectorize::execute (function *fun) } } - FOR_EACH_BB_FN (bb, fun) + if (vect_slp_function ()) { + // TODO: change to function vectorized + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, "basic block vectorized\n"); + } + else + FOR_EACH_BB_FN (bb, fun) if (vect_slp_bb (bb)) if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "basic block vectorized\n"); - } if (!in_loop_pipeline) { diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index d9f6a67264d..f353b4bbc42 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -802,7 +802,17 @@ public: const const_iterator & operator++ () { - gsi_next (&gsi); return *this; + basic_block bb = gimple_bb (gsi_stmt (gsi)); + gsi_next (&gsi); + while (gsi_end_p (gsi)) + { + bb = bb->next_bb; + if (bb == NULL) + break; + gsi = gsi_after_labels (bb); + } + + return *this; } gimple *operator* () const { return gsi_stmt (gsi); } @@ -831,7 +841,15 @@ public: const const_reverse_iterator & operator++ () { - gsi_prev (&gsi); return *this; + basic_block bb = gimple_bb (gsi_stmt (gsi)); + gsi_prev (&gsi); + while (gsi_end_p (gsi) && bb->prev_bb != NULL) + { + bb = bb->prev_bb; + gsi = gsi_after_labels (bb); + } + + return *this; } gimple *operator* () const { return gsi_stmt (gsi); } @@ -1893,14 +1911,15 @@ extern opt_result vect_analyze_data_refs_alignment (loop_vec_info); extern opt_result vect_verify_datarefs_alignment (loop_vec_info); extern bool vect_slp_analyze_and_verify_instance_alignment (vec_info *, slp_instance); -extern opt_result vect_analyze_data_ref_accesses (vec_info *); +extern opt_result vect_analyze_data_ref_accesses (vec_info *, vec *); extern opt_result vect_prune_runtime_alias_test_list (loop_vec_info); extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *); extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info, gather_scatter_info *); extern opt_result vect_find_stmt_data_reference (loop_p, gimple *, - vec *); + vec *, + vec *, int); extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *); extern void vect_record_base_alignments (vec_info *); extern tree vect_create_data_ref_ptr (vec_info *, @@ -1999,6 +2018,7 @@ extern void vect_get_slp_defs (slp_tree, vec *); extern void vect_get_slp_defs (vec_info *, slp_tree, vec > *, unsigned n = -1U); extern bool vect_slp_bb (basic_block); +extern bool vect_slp_function (); extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree); extern stmt_vec_info vect_find_first_scalar_stmt_in_slp (slp_tree); extern bool is_simple_and_all_uses_invariant (stmt_vec_info, loop_vec_info); @@ -2020,5 +2040,6 @@ unsigned vectorize_loops (void); void vect_free_loop_info_assumptions (class loop *); gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL); bool vect_stmt_dominates_stmt_p (gimple *, gimple *); +bool vect_stmt_dominates_stmt_p (gimple *, basic_block, gimple *, basic_block); #endif /* GCC_TREE_VECTORIZER_H */