From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1005) id 8CE83393BA5E; Tue, 15 Nov 2022 21:22:54 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 8CE83393BA5E DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gcc.gnu.org; s=default; t=1668547374; bh=zV7Zf4Q9kRQez9Tp3hSU/PPc35dY3T4q2QCm9oIUi1Y=; h=From:To:Subject:Date:From; b=SuGMHCNQwBd88/JQlzEvY32ypBC6bqboGMfVMnhLzhzIZBn0rsqK+sDDE6zDiKpfI tmKa36zhgN/4XUZh+clEAN2LFpVqQs0FxBdaflfoUUDEJlSNukH4AiAYwQ04i2EEws QsqcdUB3dDxup6fBVUMibLrobVTAN6XnIEFUwToQ= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Michael Meissner To: gcc-cvs@gcc.gnu.org Subject: [gcc(refs/users/meissner/heads/dmf004)] Allow for inline code for memcpy to move more than 16 bytes. X-Act-Checkin: gcc X-Git-Author: Michael Meissner X-Git-Refname: refs/users/meissner/heads/dmf004 X-Git-Oldrev: 77a62f8a5dbe5a5838d7b8e258f775e30eda4ead X-Git-Newrev: dcd80589b3fe13697d32ab85b9ff2df66774b757 Message-Id: <20221115212254.8CE83393BA5E@sourceware.org> Date: Tue, 15 Nov 2022 21:22:54 +0000 (GMT) List-Id: https://gcc.gnu.org/g:dcd80589b3fe13697d32ab85b9ff2df66774b757 commit dcd80589b3fe13697d32ab85b9ff2df66774b757 Author: Michael Meissner Date: Tue Nov 15 16:22:36 2022 -0500 Allow for inline code for memcpy to move more than 16 bytes. 2022-11-15 Michael Meissner gcc/ * config/rs6000/rs6000-string.cc (toplevel): Include optabs.h. (expand_block_move_variable): New helper function, move variable byte copy support here. Add support to move more than 16 bytes. (expand_block_move): Move variable copy support to expand_block_move_variable. Diff: --- gcc/config/rs6000/rs6000-string.cc | 112 ++++++++++++++++++++++++------------- gcc/config/rs6000/rs6000.opt | 4 +- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/gcc/config/rs6000/rs6000-string.cc b/gcc/config/rs6000/rs6000-string.cc index 243be4f396d..6e1e51c15fb 100644 --- a/gcc/config/rs6000/rs6000-string.cc +++ b/gcc/config/rs6000/rs6000-string.cc @@ -37,6 +37,7 @@ #include "target.h" #include "profile-count.h" #include "predict.h" +#include "optabs.h" /* Expand a block clear operation, and return 1 if successful. Return 0 if we should let the compiler generate normal code. @@ -2734,6 +2735,75 @@ gen_lxvl_stxvl_move (rtx dest, rtx src, int length) return gen_lxvl (dest, addr, len); } +/* Expand a block move operation with a variable count using lxvl and stxvl + instructions. */ + +static void +expand_block_move_variable (rtx orig_dest, /* destiation address. */ + rtx orig_src, /* source address. */ + rtx bytes_rtx) /* bytes to move. */ + +{ + rtx join_label = gen_label_rtx (); + rtx inline_label = gen_label_rtx (); + rtx dest_addr = copy_addr_to_reg (XEXP (orig_dest, 0)); + rtx src_addr = copy_addr_to_reg (XEXP (orig_src, 0)); + + /* Check if we want to handle this with inline code. */ + bytes_rtx = (GET_MODE (bytes_rtx) == Pmode + ? copy_to_reg (bytes_rtx) + : convert_to_mode (Pmode, bytes_rtx, true)); + + rtx cr = gen_reg_rtx (CCUNSmode); + rtx max_size = GEN_INT (rs6000_memcpy_inline_bytes); + emit_insn (gen_rtx_SET (cr, + gen_rtx_COMPARE (CCUNSmode, bytes_rtx, max_size))); + + do_ifelse (CCUNSmode, LEU, NULL_RTX, NULL_RTX, cr, inline_label, + profile_probability::likely ()); + + /* Call memcpy if the size is too large. */ + tree fun = builtin_decl_explicit (BUILT_IN_MEMCPY); + emit_library_call_value (XEXP (DECL_RTL (fun), 0), + NULL_RTX, LCT_NORMAL, Pmode, + dest_addr, Pmode, + src_addr, Pmode, + bytes_rtx, Pmode); + + rtx join_ref = gen_rtx_LABEL_REF (VOIDmode, join_label); + emit_jump_insn (gen_rtx_SET (pc_rtx, join_ref)); + emit_barrier (); + + emit_label (inline_label); + + /* We want to move bytes inline. Move 0..16 bytes now. */ + rtx vreg = gen_reg_rtx (V16QImode); + emit_insn (gen_lxvl (vreg, src_addr, bytes_rtx)); + emit_insn (gen_stxvl (vreg, dest_addr, bytes_rtx)); + + /* If we want to move more byes, adjust the pointers and lengths and + loop back. */ + if (rs6000_memcpy_inline_bytes > GET_MODE_SIZE (V16QImode)) + { + rtx vec_size = GEN_INT (GET_MODE_SIZE (V16QImode)); + rtx neg_vec_size = GEN_INT (- GET_MODE_SIZE (V16QImode)); + emit_insn (gen_add2_insn (src_addr, vec_size)); + emit_insn (gen_add2_insn (dest_addr, vec_size)); + emit_insn (gen_add2_insn (bytes_rtx, neg_vec_size)); + + profile_probability probability + = (rs6000_memcpy_inline_bytes >= 2 * GET_MODE_SIZE (V16QImode) + ? profile_probability::likely () + : profile_probability::unlikely ()); + + do_ifelse (CCmode, GT, bytes_rtx, const0_rtx, NULL_RTX, inline_label, + probability); + } + + emit_label (join_label); + return; +} + /* Expand a block move operation, and return 1 if successful. Return 0 if we should let the compiler generate normal code. @@ -2767,48 +2837,10 @@ expand_block_move (rtx operands[], bool might_overlap) { if (TARGET_BLOCK_OPS_UNALIGNED_VSX && TARGET_P9_VECTOR && TARGET_64BIT && rs6000_memcpy_inline_bytes > 0 - && rs6000_memcpy_inline_bytes <= GET_MODE_SIZE (V16QImode) + && rs6000_memcpy_inline_bytes <= 255 && optimize && !optimize_size) { - rtx join_label = gen_label_rtx (); - rtx inline_label = gen_label_rtx (); - rtx dest_addr = copy_addr_to_reg (XEXP (orig_dest, 0)); - rtx src_addr = copy_addr_to_reg (XEXP (orig_src, 0)); - - /* Check if we want to handle this with inline code. */ - bytes_rtx = (GET_MODE (bytes_rtx) == Pmode - ? copy_to_reg (bytes_rtx) - : convert_to_mode (Pmode, bytes_rtx, true)); - - rtx cr = gen_reg_rtx (CCUNSmode); - rtx max_size = GEN_INT (rs6000_memcpy_inline_bytes); - emit_insn (gen_rtx_SET (cr, - gen_rtx_COMPARE (CCUNSmode, bytes_rtx, - max_size))); - - do_ifelse (CCUNSmode, LEU, NULL_RTX, NULL_RTX, cr, - inline_label, profile_probability::likely ()); - - /* Call memcpy if the size is too large. */ - tree fun = builtin_decl_explicit (BUILT_IN_MEMCPY); - emit_library_call_value (XEXP (DECL_RTL (fun), 0), - NULL_RTX, LCT_NORMAL, Pmode, - dest_addr, Pmode, - src_addr, Pmode, - bytes_rtx, Pmode); - - rtx join_ref = gen_rtx_LABEL_REF (VOIDmode, join_label); - emit_jump_insn (gen_rtx_SET (pc_rtx, join_ref)); - emit_barrier (); - - emit_label (inline_label); - - /* We want to move bytes inline. Move 0..16 bytes now. */ - rtx vreg = gen_reg_rtx (V16QImode); - emit_insn (gen_lxvl (vreg, src_addr, bytes_rtx)); - emit_insn (gen_stxvl (vreg, dest_addr, bytes_rtx)); - - emit_label (join_label); + expand_block_move_variable (orig_dest, orig_src, bytes_rtx); return 1; } diff --git a/gcc/config/rs6000/rs6000.opt b/gcc/config/rs6000/rs6000.opt index 90dc91a277f..c594877ebc6 100644 --- a/gcc/config/rs6000/rs6000.opt +++ b/gcc/config/rs6000/rs6000.opt @@ -689,6 +689,6 @@ When reduction factor computed for a loop exceeds the threshold specified by this parameter, prefer to unroll this loop. The default value is 1. -param=rs6000-memcpy-inline-bytes= -Target Undocumented Joined UInteger Var(rs6000_memcpy_inline_bytes) Init(0) Param +Target Undocumented Joined UInteger Var(rs6000_memcpy_inline_bytes) Init(16) Param Maximum number of bytes to move with inline code before calling the memcpy -library function. The default value is 0. +library function. The default value is 16.