* Re: [PATCH] x86-64: Optimize memset for zeroing [not found] ` <CAFUsyfJ5wHJHkfcL84j+GLCX-Dba4qVt_RuCpu=aLy7-_YbHLA@mail.gmail.com> @ 2021-12-31 20:35 ` H.J. Lu 2021-12-31 20:43 ` Florian Weimer 2021-12-31 22:14 ` Noah Goldstein 0 siblings, 2 replies; 9+ messages in thread From: H.J. Lu @ 2021-12-31 20:35 UTC (permalink / raw) To: Noah Goldstein, GCC Development; +Cc: GNU C Library, Arjan van de Ven On Fri, Dec 31, 2021 at 12:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > On Fri, Dec 31, 2021 at 12:20 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > Update MEMSET_VDUP_TO_VEC0_AND_SET_RETURN to use PXOR, which has lower > > lantency and higher throughput than VPBROADCAST, for zero constant. > > Since the most common usage of memset is to zero a block of memory, the > > branch predictor will make the compare/jmp basically free and PXOR is > > almost like being executed unconditionally. > > Any benchmark results? Is the broadcast on the critical path for any size? Can you run your workloads to see how many memset calls are zeroing? > Also imagine the vast majority of memset zero are compile time known. > > I think it might make more sense to give bzero() the fall-through instead and bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero like __memcmpeq? It should be almost free in glibc. GCC can use __memsetzero if it is available. > add a patch in GCC to prefer bzero > memset. > > > > --- > > sysdeps/x86_64/memset.S | 14 ++++++++++++-- > > .../x86_64/multiarch/memset-avx2-unaligned-erms.S | 14 ++++++++++++-- > > .../multiarch/memset-avx512-unaligned-erms.S | 10 ++++++++++ > > .../x86_64/multiarch/memset-evex-unaligned-erms.S | 10 ++++++++++ > > .../x86_64/multiarch/memset-vec-unaligned-erms.S | 13 +++++++++++++ > > 5 files changed, 57 insertions(+), 4 deletions(-) > > > > diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S > > index 0137eba4cd..513f9c703d 100644 > > --- a/sysdeps/x86_64/memset.S > > +++ b/sysdeps/x86_64/memset.S > > @@ -29,15 +29,25 @@ > > #define VMOVA movaps > > > > #define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > - movd d, %xmm0; \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + pxor %xmm0, %xmm0 > > + > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > + movd d, %xmm0; \ > > punpcklbw %xmm0, %xmm0; \ > > punpcklwd %xmm0, %xmm0; \ > > pshufd $0, %xmm0, %xmm0 > > > > #define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > - movd d, %xmm0; \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + pxor %xmm0, %xmm0 > > + > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > + movd d, %xmm0; \ > > pshufd $0, %xmm0, %xmm0 > > > > #define SECTION(p) p > > diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > index 1af668af0a..8004a27750 100644 > > --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > @@ -11,13 +11,23 @@ > > # define VMOVA vmovdqa > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > - vmovd d, %xmm0; \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxor %xmm0, %xmm0, %xmm0 > > + > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > + vmovd d, %xmm0; \ > > vpbroadcastb %xmm0, %ymm0 > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > - vmovd d, %xmm0; \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxor %xmm0, %xmm0, %xmm0 > > + > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > + vmovd d, %xmm0; \ > > vpbroadcastd %xmm0, %ymm0 > > > > # ifndef SECTION > > diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > index f14d6f8493..61ff9ccf6f 100644 > > --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > @@ -17,10 +17,20 @@ > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxorq %XMM0, %XMM0, %XMM0 > > + > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > vpbroadcastb d, %VEC0 > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxorq %XMM0, %XMM0, %XMM0 > > + > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > vpbroadcastd d, %VEC0 > > > > # define SECTION(p) p##.evex512 > > diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > index 64b09e77cc..85544fb0fc 100644 > > --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > @@ -17,10 +17,20 @@ > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxorq %XMM0, %XMM0, %XMM0 > > + > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > vpbroadcastb d, %VEC0 > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > movq r, %rax; \ > > + testl d, d; \ > > + jnz 1f; \ > > + vpxorq %XMM0, %XMM0, %XMM0 > > + > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > vpbroadcastd d, %VEC0 > > > > # define SECTION(p) p##.evex > > diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > index e723413a66..4ca34a19ba 100644 > > --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > @@ -112,6 +112,9 @@ ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) > > shl $2, %RDX_LP > > WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > jmp L(entry_from_bzero) > > +1: > > + WMEMSET_VDUP_TO_VEC0 (%esi) > > + jmp L(entry_from_bzero) > > END (WMEMSET_SYMBOL (__wmemset, unaligned)) > > #endif > > > > @@ -124,6 +127,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) > > > > ENTRY (MEMSET_SYMBOL (__memset, unaligned)) > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > +2: > > # ifdef __ILP32__ > > /* Clear the upper 32 bits. */ > > mov %edx, %edx > > @@ -137,6 +141,10 @@ L(entry_from_bzero): > > VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) > > VMOVU %VEC(0), (%rdi) > > VZEROUPPER_RETURN > > + > > +1: > > + MEMSET_VDUP_TO_VEC0 (%esi) > > + jmp 2b > > #if defined USE_MULTIARCH && IS_IN (libc) > > END (MEMSET_SYMBOL (__memset, unaligned)) > > > > @@ -180,6 +188,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) > > > > ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > +2: > > # ifdef __ILP32__ > > /* Clear the upper 32 bits. */ > > mov %edx, %edx > > @@ -193,6 +202,10 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > VMOVU %VEC(0), (%rax) > > VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) > > VZEROUPPER_RETURN > > + > > +1: > > + MEMSET_VDUP_TO_VEC0 (%esi) > > + jmp 2b > > #endif > > > > .p2align 4,, 10 > > -- > > 2.33.1 > > -- H.J. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 20:35 ` [PATCH] x86-64: Optimize memset for zeroing H.J. Lu @ 2021-12-31 20:43 ` Florian Weimer 2021-12-31 20:52 ` H.J. Lu 2021-12-31 22:14 ` Noah Goldstein 1 sibling, 1 reply; 9+ messages in thread From: Florian Weimer @ 2021-12-31 20:43 UTC (permalink / raw) To: H.J. Lu via Libc-alpha Cc: Noah Goldstein, GCC Development, H.J. Lu, Arjan van de Ven * H. J. Lu via Libc-alpha: > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > like __memcmpeq? It should be almost free in glibc. GCC can use > __memsetzero if it is available. bzero does not have the interface ambiguity that bcmp has. So the only reason for not using it would be namespace cleanliness. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 20:43 ` Florian Weimer @ 2021-12-31 20:52 ` H.J. Lu 2021-12-31 21:02 ` Florian Weimer 0 siblings, 1 reply; 9+ messages in thread From: H.J. Lu @ 2021-12-31 20:52 UTC (permalink / raw) To: Florian Weimer Cc: H.J. Lu via Libc-alpha, Noah Goldstein, GCC Development, Arjan van de Ven On Fri, Dec 31, 2021 at 12:43 PM Florian Weimer <fw@deneb.enyo.de> wrote: > > * H. J. Lu via Libc-alpha: > > > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > > like __memcmpeq? It should be almost free in glibc. GCC can use > > __memsetzero if it is available. > > bzero does not have the interface ambiguity that bcmp has. So the > only reason for not using it would be namespace cleanliness. bzero isn't a standard C function and it isn't optimized like memset in glibc. -- H.J. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 20:52 ` H.J. Lu @ 2021-12-31 21:02 ` Florian Weimer 2021-12-31 21:15 ` Noah Goldstein 0 siblings, 1 reply; 9+ messages in thread From: Florian Weimer @ 2021-12-31 21:02 UTC (permalink / raw) To: H.J. Lu Cc: H.J. Lu via Libc-alpha, Noah Goldstein, GCC Development, Arjan van de Ven * H. J. Lu: > On Fri, Dec 31, 2021 at 12:43 PM Florian Weimer <fw@deneb.enyo.de> wrote: >> >> * H. J. Lu via Libc-alpha: >> >> > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero >> > like __memcmpeq? It should be almost free in glibc. GCC can use >> > __memsetzero if it is available. >> >> bzero does not have the interface ambiguity that bcmp has. So the >> only reason for not using it would be namespace cleanliness. > > bzero isn't a standard C function and it isn't optimized like memset > in glibc. GCC already uses non-standard functions whose names are not implementation-defined for optimization purposes if a suitable prototype is available. stpcpy is an example, for: strcpy (a, b); return a + strlen (a); ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 21:02 ` Florian Weimer @ 2021-12-31 21:15 ` Noah Goldstein 2021-12-31 22:05 ` Florian Weimer 0 siblings, 1 reply; 9+ messages in thread From: Noah Goldstein @ 2021-12-31 21:15 UTC (permalink / raw) To: Florian Weimer Cc: H.J. Lu, H.J. Lu via Libc-alpha, GCC Development, Arjan van de Ven On Fri, Dec 31, 2021 at 3:02 PM Florian Weimer <fw@deneb.enyo.de> wrote: > > * H. J. Lu: > > > On Fri, Dec 31, 2021 at 12:43 PM Florian Weimer <fw@deneb.enyo.de> wrote: > >> > >> * H. J. Lu via Libc-alpha: > >> > >> > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > >> > like __memcmpeq? It should be almost free in glibc. GCC can use > >> > __memsetzero if it is available. Think __memsetzero makes sense. > >> > >> bzero does not have the interface ambiguity that bcmp has. So the > >> only reason for not using it would be namespace cleanliness. > > > > bzero isn't a standard C function and it isn't optimized like memset > > in glibc. It could be an issue if the "optimization" was made and then the binary was run with an older version of GCC that was still using the aliasing bzero. Would end up being a deoptimization then. > > GCC already uses non-standard functions whose names are not > implementation-defined for optimization purposes if a suitable > prototype is available. stpcpy is an example, for: > > strcpy (a, b); > return a + strlen (a); ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 21:15 ` Noah Goldstein @ 2021-12-31 22:05 ` Florian Weimer 0 siblings, 0 replies; 9+ messages in thread From: Florian Weimer @ 2021-12-31 22:05 UTC (permalink / raw) To: Noah Goldstein Cc: H.J. Lu, H.J. Lu via Libc-alpha, GCC Development, Arjan van de Ven * Noah Goldstein: >> >> bzero does not have the interface ambiguity that bcmp has. So the >> >> only reason for not using it would be namespace cleanliness. >> > >> > bzero isn't a standard C function and it isn't optimized like memset >> > in glibc. > > It could be an issue if the "optimization" was made and then the > binary was run with an older version of GCC that was still using > the aliasing bzero. Would end up being a deoptimization then. That's a fair point. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 20:35 ` [PATCH] x86-64: Optimize memset for zeroing H.J. Lu 2021-12-31 20:43 ` Florian Weimer @ 2021-12-31 22:14 ` Noah Goldstein 2021-12-31 22:19 ` Noah Goldstein 1 sibling, 1 reply; 9+ messages in thread From: Noah Goldstein @ 2021-12-31 22:14 UTC (permalink / raw) To: H.J. Lu; +Cc: GCC Development, GNU C Library, Arjan van de Ven On Fri, Dec 31, 2021 at 2:36 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > On Fri, Dec 31, 2021 at 12:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > On Fri, Dec 31, 2021 at 12:20 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > > > Update MEMSET_VDUP_TO_VEC0_AND_SET_RETURN to use PXOR, which has lower > > > lantency and higher throughput than VPBROADCAST, for zero constant. > > > Since the most common usage of memset is to zero a block of memory, the > > > branch predictor will make the compare/jmp basically free and PXOR is > > > almost like being executed unconditionally. > > > > Any benchmark results? Is the broadcast on the critical path for any size? > > Can you run your workloads to see how many memset calls are zeroing? Python3.7.7 running pyperf 99.6% of calls are zero. GCC11.2 compiling llvm 99.1% of calls are zero. > > > Also imagine the vast majority of memset zero are compile time known. > > > > I think it might make more sense to give bzero() the fall-through instead and > > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > like __memcmpeq? It should be almost free in glibc. GCC can use > __memsetzero if it is available. > > > add a patch in GCC to prefer bzero > memset. > > > > > > > --- > > > sysdeps/x86_64/memset.S | 14 ++++++++++++-- > > > .../x86_64/multiarch/memset-avx2-unaligned-erms.S | 14 ++++++++++++-- > > > .../multiarch/memset-avx512-unaligned-erms.S | 10 ++++++++++ > > > .../x86_64/multiarch/memset-evex-unaligned-erms.S | 10 ++++++++++ > > > .../x86_64/multiarch/memset-vec-unaligned-erms.S | 13 +++++++++++++ > > > 5 files changed, 57 insertions(+), 4 deletions(-) > > > > > > diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S > > > index 0137eba4cd..513f9c703d 100644 > > > --- a/sysdeps/x86_64/memset.S > > > +++ b/sysdeps/x86_64/memset.S > > > @@ -29,15 +29,25 @@ > > > #define VMOVA movaps > > > > > > #define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > - movd d, %xmm0; \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + pxor %xmm0, %xmm0 > > > + > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > + movd d, %xmm0; \ > > > punpcklbw %xmm0, %xmm0; \ > > > punpcklwd %xmm0, %xmm0; \ > > > pshufd $0, %xmm0, %xmm0 > > > > > > #define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > - movd d, %xmm0; \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + pxor %xmm0, %xmm0 > > > + > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > + movd d, %xmm0; \ > > > pshufd $0, %xmm0, %xmm0 > > > > > > #define SECTION(p) p > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > index 1af668af0a..8004a27750 100644 > > > --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > @@ -11,13 +11,23 @@ > > > # define VMOVA vmovdqa > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > - vmovd d, %xmm0; \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxor %xmm0, %xmm0, %xmm0 > > > + > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > + vmovd d, %xmm0; \ > > > vpbroadcastb %xmm0, %ymm0 > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > - vmovd d, %xmm0; \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxor %xmm0, %xmm0, %xmm0 > > > + > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > + vmovd d, %xmm0; \ > > > vpbroadcastd %xmm0, %ymm0 > > > > > > # ifndef SECTION > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > index f14d6f8493..61ff9ccf6f 100644 > > > --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > @@ -17,10 +17,20 @@ > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > + > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > vpbroadcastb d, %VEC0 > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > + > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > vpbroadcastd d, %VEC0 > > > > > > # define SECTION(p) p##.evex512 > > > diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > index 64b09e77cc..85544fb0fc 100644 > > > --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > @@ -17,10 +17,20 @@ > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > + > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > vpbroadcastb d, %VEC0 > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > movq r, %rax; \ > > > + testl d, d; \ > > > + jnz 1f; \ > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > + > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > vpbroadcastd d, %VEC0 > > > > > > # define SECTION(p) p##.evex > > > diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > index e723413a66..4ca34a19ba 100644 > > > --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > @@ -112,6 +112,9 @@ ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > shl $2, %RDX_LP > > > WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > jmp L(entry_from_bzero) > > > +1: > > > + WMEMSET_VDUP_TO_VEC0 (%esi) > > > + jmp L(entry_from_bzero) > > > END (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > #endif > > > > > > @@ -124,6 +127,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) > > > > > > ENTRY (MEMSET_SYMBOL (__memset, unaligned)) > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > +2: > > > # ifdef __ILP32__ > > > /* Clear the upper 32 bits. */ > > > mov %edx, %edx > > > @@ -137,6 +141,10 @@ L(entry_from_bzero): > > > VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) > > > VMOVU %VEC(0), (%rdi) > > > VZEROUPPER_RETURN > > > + > > > +1: > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > + jmp 2b > > > #if defined USE_MULTIARCH && IS_IN (libc) > > > END (MEMSET_SYMBOL (__memset, unaligned)) > > > > > > @@ -180,6 +188,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) > > > > > > ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > +2: > > > # ifdef __ILP32__ > > > /* Clear the upper 32 bits. */ > > > mov %edx, %edx > > > @@ -193,6 +202,10 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > VMOVU %VEC(0), (%rax) > > > VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) > > > VZEROUPPER_RETURN > > > + > > > +1: > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > + jmp 2b > > > #endif > > > > > > .p2align 4,, 10 > > > -- > > > 2.33.1 > > > > > > > -- > H.J. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 22:14 ` Noah Goldstein @ 2021-12-31 22:19 ` Noah Goldstein 2021-12-31 22:21 ` H.J. Lu 0 siblings, 1 reply; 9+ messages in thread From: Noah Goldstein @ 2021-12-31 22:19 UTC (permalink / raw) To: H.J. Lu; +Cc: GCC Development, GNU C Library, Arjan van de Ven On Fri, Dec 31, 2021 at 4:14 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > On Fri, Dec 31, 2021 at 2:36 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > On Fri, Dec 31, 2021 at 12:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > > > On Fri, Dec 31, 2021 at 12:20 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > > > > > Update MEMSET_VDUP_TO_VEC0_AND_SET_RETURN to use PXOR, which has lower > > > > lantency and higher throughput than VPBROADCAST, for zero constant. > > > > Since the most common usage of memset is to zero a block of memory, the > > > > branch predictor will make the compare/jmp basically free and PXOR is > > > > almost like being executed unconditionally. > > > > > > Any benchmark results? Is the broadcast on the critical path for any size? > > > > Can you run your workloads to see how many memset calls are zeroing? > > Python3.7.7 running pyperf 99.6% of calls are zero. > GCC11.2 compiling llvm 99.1% of calls are zero. I like the idea of this optimization, just don't think we want to implement it with a branch like this. Even though it will be hyperpredictable under heavy usage, its extra interference, will add a second branch to the first BTB prediction, and will likely incur more misses than that rates above because the BHT entry may be overwritten by other branches in the application between calls. '__memsetzero' makes sense to me and then we can just organize the code so that __memsetzero gets the fallthrough path. > > > > > Also imagine the vast majority of memset zero are compile time known. > > > > > > I think it might make more sense to give bzero() the fall-through instead and > > > > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > > like __memcmpeq? It should be almost free in glibc. GCC can use > > __memsetzero if it is available. > > > > > add a patch in GCC to prefer bzero > memset. > > > > > > > > > > --- > > > > sysdeps/x86_64/memset.S | 14 ++++++++++++-- > > > > .../x86_64/multiarch/memset-avx2-unaligned-erms.S | 14 ++++++++++++-- > > > > .../multiarch/memset-avx512-unaligned-erms.S | 10 ++++++++++ > > > > .../x86_64/multiarch/memset-evex-unaligned-erms.S | 10 ++++++++++ > > > > .../x86_64/multiarch/memset-vec-unaligned-erms.S | 13 +++++++++++++ > > > > 5 files changed, 57 insertions(+), 4 deletions(-) > > > > > > > > diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S > > > > index 0137eba4cd..513f9c703d 100644 > > > > --- a/sysdeps/x86_64/memset.S > > > > +++ b/sysdeps/x86_64/memset.S > > > > @@ -29,15 +29,25 @@ > > > > #define VMOVA movaps > > > > > > > > #define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > - movd d, %xmm0; \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + pxor %xmm0, %xmm0 > > > > + > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > + movd d, %xmm0; \ > > > > punpcklbw %xmm0, %xmm0; \ > > > > punpcklwd %xmm0, %xmm0; \ > > > > pshufd $0, %xmm0, %xmm0 > > > > > > > > #define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > - movd d, %xmm0; \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + pxor %xmm0, %xmm0 > > > > + > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > + movd d, %xmm0; \ > > > > pshufd $0, %xmm0, %xmm0 > > > > > > > > #define SECTION(p) p > > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > index 1af668af0a..8004a27750 100644 > > > > --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > @@ -11,13 +11,23 @@ > > > > # define VMOVA vmovdqa > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > - vmovd d, %xmm0; \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxor %xmm0, %xmm0, %xmm0 > > > > + > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > + vmovd d, %xmm0; \ > > > > vpbroadcastb %xmm0, %ymm0 > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > - vmovd d, %xmm0; \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxor %xmm0, %xmm0, %xmm0 > > > > + > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > + vmovd d, %xmm0; \ > > > > vpbroadcastd %xmm0, %ymm0 > > > > > > > > # ifndef SECTION > > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > index f14d6f8493..61ff9ccf6f 100644 > > > > --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > @@ -17,10 +17,20 @@ > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > + > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > vpbroadcastb d, %VEC0 > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > + > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > vpbroadcastd d, %VEC0 > > > > > > > > # define SECTION(p) p##.evex512 > > > > diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > index 64b09e77cc..85544fb0fc 100644 > > > > --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > @@ -17,10 +17,20 @@ > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > + > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > vpbroadcastb d, %VEC0 > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > movq r, %rax; \ > > > > + testl d, d; \ > > > > + jnz 1f; \ > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > + > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > vpbroadcastd d, %VEC0 > > > > > > > > # define SECTION(p) p##.evex > > > > diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > index e723413a66..4ca34a19ba 100644 > > > > --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > @@ -112,6 +112,9 @@ ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > > shl $2, %RDX_LP > > > > WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > jmp L(entry_from_bzero) > > > > +1: > > > > + WMEMSET_VDUP_TO_VEC0 (%esi) > > > > + jmp L(entry_from_bzero) > > > > END (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > > #endif > > > > > > > > @@ -124,6 +127,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) > > > > > > > > ENTRY (MEMSET_SYMBOL (__memset, unaligned)) > > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > +2: > > > > # ifdef __ILP32__ > > > > /* Clear the upper 32 bits. */ > > > > mov %edx, %edx > > > > @@ -137,6 +141,10 @@ L(entry_from_bzero): > > > > VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) > > > > VMOVU %VEC(0), (%rdi) > > > > VZEROUPPER_RETURN > > > > + > > > > +1: > > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > > + jmp 2b > > > > #if defined USE_MULTIARCH && IS_IN (libc) > > > > END (MEMSET_SYMBOL (__memset, unaligned)) > > > > > > > > @@ -180,6 +188,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) > > > > > > > > ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > +2: > > > > # ifdef __ILP32__ > > > > /* Clear the upper 32 bits. */ > > > > mov %edx, %edx > > > > @@ -193,6 +202,10 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > > VMOVU %VEC(0), (%rax) > > > > VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) > > > > VZEROUPPER_RETURN > > > > + > > > > +1: > > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > > + jmp 2b > > > > #endif > > > > > > > > .p2align 4,, 10 > > > > -- > > > > 2.33.1 > > > > > > > > > > > > -- > > H.J. ^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH] x86-64: Optimize memset for zeroing 2021-12-31 22:19 ` Noah Goldstein @ 2021-12-31 22:21 ` H.J. Lu 0 siblings, 0 replies; 9+ messages in thread From: H.J. Lu @ 2021-12-31 22:21 UTC (permalink / raw) To: Noah Goldstein; +Cc: GCC Development, GNU C Library, Arjan van de Ven On Fri, Dec 31, 2021 at 2:19 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > On Fri, Dec 31, 2021 at 4:14 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > On Fri, Dec 31, 2021 at 2:36 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > > > On Fri, Dec 31, 2021 at 12:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > > > > > On Fri, Dec 31, 2021 at 12:20 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > > > > > > > Update MEMSET_VDUP_TO_VEC0_AND_SET_RETURN to use PXOR, which has lower > > > > > lantency and higher throughput than VPBROADCAST, for zero constant. > > > > > Since the most common usage of memset is to zero a block of memory, the > > > > > branch predictor will make the compare/jmp basically free and PXOR is > > > > > almost like being executed unconditionally. > > > > > > > > Any benchmark results? Is the broadcast on the critical path for any size? > > > > > > Can you run your workloads to see how many memset calls are zeroing? > > > > Python3.7.7 running pyperf 99.6% of calls are zero. > > GCC11.2 compiling llvm 99.1% of calls are zero. > > I like the idea of this optimization, just don't think we want to implement > it with a branch like this. Even though it will be hyperpredictable under > heavy usage, its extra interference, will add a second branch to the > first BTB prediction, and will likely incur more misses than > that rates above because the BHT entry may be overwritten by other > branches in the application between calls. > > '__memsetzero' makes sense to me and then we > can just organize the code so that __memsetzero gets the fallthrough > path. We can do it for glibc 2.36. > > > > > > > Also imagine the vast majority of memset zero are compile time known. > > > > > > > > I think it might make more sense to give bzero() the fall-through instead and > > > > > > bzero is an alias of SSE2 memset in glibc. Should we add __memsetzero > > > like __memcmpeq? It should be almost free in glibc. GCC can use > > > __memsetzero if it is available. > > > > > > > add a patch in GCC to prefer bzero > memset. > > > > > > > > > > > > > --- > > > > > sysdeps/x86_64/memset.S | 14 ++++++++++++-- > > > > > .../x86_64/multiarch/memset-avx2-unaligned-erms.S | 14 ++++++++++++-- > > > > > .../multiarch/memset-avx512-unaligned-erms.S | 10 ++++++++++ > > > > > .../x86_64/multiarch/memset-evex-unaligned-erms.S | 10 ++++++++++ > > > > > .../x86_64/multiarch/memset-vec-unaligned-erms.S | 13 +++++++++++++ > > > > > 5 files changed, 57 insertions(+), 4 deletions(-) > > > > > > > > > > diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S > > > > > index 0137eba4cd..513f9c703d 100644 > > > > > --- a/sysdeps/x86_64/memset.S > > > > > +++ b/sysdeps/x86_64/memset.S > > > > > @@ -29,15 +29,25 @@ > > > > > #define VMOVA movaps > > > > > > > > > > #define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > - movd d, %xmm0; \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + pxor %xmm0, %xmm0 > > > > > + > > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > > + movd d, %xmm0; \ > > > > > punpcklbw %xmm0, %xmm0; \ > > > > > punpcklwd %xmm0, %xmm0; \ > > > > > pshufd $0, %xmm0, %xmm0 > > > > > > > > > > #define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > - movd d, %xmm0; \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + pxor %xmm0, %xmm0 > > > > > + > > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > > + movd d, %xmm0; \ > > > > > pshufd $0, %xmm0, %xmm0 > > > > > > > > > > #define SECTION(p) p > > > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > > index 1af668af0a..8004a27750 100644 > > > > > --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > > +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S > > > > > @@ -11,13 +11,23 @@ > > > > > # define VMOVA vmovdqa > > > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > - vmovd d, %xmm0; \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxor %xmm0, %xmm0, %xmm0 > > > > > + > > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > > + vmovd d, %xmm0; \ > > > > > vpbroadcastb %xmm0, %ymm0 > > > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > - vmovd d, %xmm0; \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxor %xmm0, %xmm0, %xmm0 > > > > > + > > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > > + vmovd d, %xmm0; \ > > > > > vpbroadcastd %xmm0, %ymm0 > > > > > > > > > > # ifndef SECTION > > > > > diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > > index f14d6f8493..61ff9ccf6f 100644 > > > > > --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > > +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S > > > > > @@ -17,10 +17,20 @@ > > > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > > + > > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > > vpbroadcastb d, %VEC0 > > > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > > + > > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > > vpbroadcastd d, %VEC0 > > > > > > > > > > # define SECTION(p) p##.evex512 > > > > > diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > > index 64b09e77cc..85544fb0fc 100644 > > > > > --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > > +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S > > > > > @@ -17,10 +17,20 @@ > > > > > > > > > > # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > > + > > > > > +# define MEMSET_VDUP_TO_VEC0(d) \ > > > > > vpbroadcastb d, %VEC0 > > > > > > > > > > # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ > > > > > movq r, %rax; \ > > > > > + testl d, d; \ > > > > > + jnz 1f; \ > > > > > + vpxorq %XMM0, %XMM0, %XMM0 > > > > > + > > > > > +# define WMEMSET_VDUP_TO_VEC0(d) \ > > > > > vpbroadcastd d, %VEC0 > > > > > > > > > > # define SECTION(p) p##.evex > > > > > diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > > index e723413a66..4ca34a19ba 100644 > > > > > --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > > +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S > > > > > @@ -112,6 +112,9 @@ ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > > > shl $2, %RDX_LP > > > > > WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > > jmp L(entry_from_bzero) > > > > > +1: > > > > > + WMEMSET_VDUP_TO_VEC0 (%esi) > > > > > + jmp L(entry_from_bzero) > > > > > END (WMEMSET_SYMBOL (__wmemset, unaligned)) > > > > > #endif > > > > > > > > > > @@ -124,6 +127,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) > > > > > > > > > > ENTRY (MEMSET_SYMBOL (__memset, unaligned)) > > > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > > +2: > > > > > # ifdef __ILP32__ > > > > > /* Clear the upper 32 bits. */ > > > > > mov %edx, %edx > > > > > @@ -137,6 +141,10 @@ L(entry_from_bzero): > > > > > VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) > > > > > VMOVU %VEC(0), (%rdi) > > > > > VZEROUPPER_RETURN > > > > > + > > > > > +1: > > > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > > > + jmp 2b > > > > > #if defined USE_MULTIARCH && IS_IN (libc) > > > > > END (MEMSET_SYMBOL (__memset, unaligned)) > > > > > > > > > > @@ -180,6 +188,7 @@ END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) > > > > > > > > > > ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > > > MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) > > > > > +2: > > > > > # ifdef __ILP32__ > > > > > /* Clear the upper 32 bits. */ > > > > > mov %edx, %edx > > > > > @@ -193,6 +202,10 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) > > > > > VMOVU %VEC(0), (%rax) > > > > > VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx) > > > > > VZEROUPPER_RETURN > > > > > + > > > > > +1: > > > > > + MEMSET_VDUP_TO_VEC0 (%esi) > > > > > + jmp 2b > > > > > #endif > > > > > > > > > > .p2align 4,, 10 > > > > > -- > > > > > 2.33.1 > > > > > > > > > > > > > > > > > -- > > > H.J. -- H.J. ^ permalink raw reply [flat|nested] 9+ messages in thread
end of thread, other threads:[~2021-12-31 22:22 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20211231182010.107040-1-hjl.tools@gmail.com>
[not found] ` <CAFUsyfJ5wHJHkfcL84j+GLCX-Dba4qVt_RuCpu=aLy7-_YbHLA@mail.gmail.com>
2021-12-31 20:35 ` [PATCH] x86-64: Optimize memset for zeroing H.J. Lu
2021-12-31 20:43 ` Florian Weimer
2021-12-31 20:52 ` H.J. Lu
2021-12-31 21:02 ` Florian Weimer
2021-12-31 21:15 ` Noah Goldstein
2021-12-31 22:05 ` Florian Weimer
2021-12-31 22:14 ` Noah Goldstein
2021-12-31 22:19 ` Noah Goldstein
2021-12-31 22:21 ` H.J. Lu
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).