public inbox for gcc-cvs@sourceware.org help / color / mirror / Atom feed
From: Richard Biener <rguenth@gcc.gnu.org> To: gcc-cvs@gcc.gnu.org Subject: [gcc r12-2967] Fall back to masked_gather_load/masked_scatter_store Date: Tue, 17 Aug 2021 16:54:39 +0000 (GMT) [thread overview] Message-ID: <20210817165439.7C5053839C7A@sourceware.org> (raw) https://gcc.gnu.org/g:e7e17be3be3d43640a9a4a33d59315dec69fdda4 commit r12-2967-ge7e17be3be3d43640a9a4a33d59315dec69fdda4 Author: Richard Biener <rguenther@suse.de> Date: Tue Aug 17 15:50:31 2021 +0200 Fall back to masked_gather_load/masked_scatter_store This adds a fallback to the masked_ variants for gather_load and scatter_store if the latter are not available. 2021-08-17 Richard Biener <rguenther@suse.de> * optabs-query.c (supports_vec_gather_load_p): Also check for masked optabs. (supports_vec_scatter_store_p): Likewise. * tree-vect-data-refs.c (vect_gather_scatter_fn_p): Fall back to masked variants if non-masked are not supported. * tree-vect-patterns.c (vect_recog_gather_scatter_pattern): When we need to use masked gather/scatter but do not have a mask set up a constant true one. * tree-vect-stmts.c (vect_check_scalar_mask): Also allow non-SSA_NAME masks. Diff: --- gcc/optabs-query.c | 6 ++++-- gcc/tree-vect-data-refs.c | 22 +++++++++++++++++++--- gcc/tree-vect-patterns.c | 7 +++++-- gcc/tree-vect-stmts.c | 8 -------- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/gcc/optabs-query.c b/gcc/optabs-query.c index 05ee5f517da..a6dd0fed610 100644 --- a/gcc/optabs-query.c +++ b/gcc/optabs-query.c @@ -740,7 +740,8 @@ supports_vec_gather_load_p () this_fn_optabs->supports_vec_gather_load_cached = true; this_fn_optabs->supports_vec_gather_load - = supports_vec_convert_optab_p (gather_load_optab); + = (supports_vec_convert_optab_p (gather_load_optab) + || supports_vec_convert_optab_p (mask_gather_load_optab)); return this_fn_optabs->supports_vec_gather_load; } @@ -757,7 +758,8 @@ supports_vec_scatter_store_p () this_fn_optabs->supports_vec_scatter_store_cached = true; this_fn_optabs->supports_vec_scatter_store - = supports_vec_convert_optab_p (scatter_store_optab); + = (supports_vec_convert_optab_p (scatter_store_optab) + || supports_vec_convert_optab_p (mask_scatter_store_optab)); return this_fn_optabs->supports_vec_scatter_store; } diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index d594c0a1b1e..b7dde07fc5e 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -3735,11 +3735,17 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p, return false; /* Work out which function we need. */ - internal_fn ifn; + internal_fn ifn, alt_ifn; if (read_p) - ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD; + { + ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD; + alt_ifn = IFN_MASK_GATHER_LOAD; + } else - ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE; + { + ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE; + alt_ifn = IFN_MASK_SCATTER_STORE; + } for (;;) { @@ -3755,6 +3761,16 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p, *offset_vectype_out = offset_vectype; return true; } + else if (!masked_p + && internal_gather_scatter_fn_supported_p (alt_ifn, vectype, + memory_type, + offset_vectype, + scale)) + { + *ifn_out = alt_ifn; + *offset_vectype_out = offset_vectype; + return true; + } if (TYPE_PRECISION (offset_type) >= POINTER_SIZE && TYPE_PRECISION (offset_type) >= element_bits) diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 25de97bd9b0..899734005ce 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -4820,6 +4820,9 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo, if (mask) mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info, loop_vinfo); + else if (gs_info.ifn == IFN_MASK_SCATTER_STORE + || gs_info.ifn == IFN_MASK_GATHER_LOAD) + mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1); /* Get the invariant base and non-invariant offset, converting the latter to the same width as the vector elements. */ @@ -4847,11 +4850,11 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo, { tree rhs = vect_get_store_rhs (stmt_info); if (mask != NULL) - pattern_stmt = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5, + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base, offset, scale, rhs, mask); else - pattern_stmt = gimple_build_call_internal (IFN_SCATTER_STORE, 4, + pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4, base, offset, scale, rhs); } gimple_call_set_nothrow (pattern_stmt, true); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index cc6c091e41e..4e0b2adf1dc 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -2510,14 +2510,6 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info, return false; } - if (TREE_CODE (*mask) != SSA_NAME) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "mask argument is not an SSA name.\n"); - return false; - } - /* If the caller is not prepared for adjusting an external/constant SLP mask vector type fail. */ if (slp_node
reply other threads:[~2021-08-17 16:54 UTC|newest] Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210817165439.7C5053839C7A@sourceware.org \ --to=rguenth@gcc.gnu.org \ --cc=gcc-cvs@gcc.gnu.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).