From: "H.J. Lu" <hjl.tools@gmail.com>
To: Noah Goldstein <goldstein.w.n@gmail.com>
Cc: GNU C Library <libc-alpha@sourceware.org>,
"Carlos O'Donell" <carlos@systemhalted.org>
Subject: Re: [PATCH v6 7/8] x86: Shrink code size of memchr-avx2.S
Date: Tue, 7 Jun 2022 11:18:44 -0700 [thread overview]
Message-ID: <CAMe9rOrgX_tcy=-8dQ5Lh_bYzCX6f3Hzr4z0aeRx0hxn-hLupw@mail.gmail.com> (raw)
In-Reply-To: <20220607041134.2369903-7-goldstein.w.n@gmail.com>
On Mon, Jun 6, 2022 at 9:11 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> This is not meant as a performance optimization. The previous code was
> far to liberal in aligning targets and wasted code size unnecissarily.
>
> The total code size saving is: 59 bytes
>
> There are no major changes in the benchmarks.
> Geometric Mean of all benchmarks New / Old: 0.967
>
> Full xcheck passes on x86_64.
> ---
> sysdeps/x86_64/multiarch/memchr-avx2-rtm.S | 1 +
> sysdeps/x86_64/multiarch/memchr-avx2.S | 109 +++++++++++----------
> 2 files changed, 60 insertions(+), 50 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S
> index 87b076c7c4..c4d71938c5 100644
> --- a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S
> +++ b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S
> @@ -2,6 +2,7 @@
> # define MEMCHR __memchr_avx2_rtm
> #endif
>
> +#define COND_VZEROUPPER COND_VZEROUPPER_XTEST
> #define ZERO_UPPER_VEC_REGISTERS_RETURN \
> ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
>
> diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S
> index 75bd7262e0..28a01280ec 100644
> --- a/sysdeps/x86_64/multiarch/memchr-avx2.S
> +++ b/sysdeps/x86_64/multiarch/memchr-avx2.S
> @@ -57,7 +57,7 @@
> # define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
>
> .section SECTION(.text),"ax",@progbits
> -ENTRY (MEMCHR)
> +ENTRY_P2ALIGN (MEMCHR, 5)
> # ifndef USE_AS_RAWMEMCHR
> /* Check for zero length. */
> # ifdef __ILP32__
> @@ -87,12 +87,14 @@ ENTRY (MEMCHR)
> # endif
> testl %eax, %eax
> jz L(aligned_more)
> - tzcntl %eax, %eax
> + bsfl %eax, %eax
> addq %rdi, %rax
> - VZEROUPPER_RETURN
> +L(return_vzeroupper):
> + ZERO_UPPER_VEC_REGISTERS_RETURN
> +
>
> # ifndef USE_AS_RAWMEMCHR
> - .p2align 5
> + .p2align 4
> L(first_vec_x0):
> /* Check if first match was before length. */
> tzcntl %eax, %eax
> @@ -100,58 +102,31 @@ L(first_vec_x0):
> /* NB: Multiply length by 4 to get byte count. */
> sall $2, %edx
> # endif
> - xorl %ecx, %ecx
> + COND_VZEROUPPER
> + /* Use branch instead of cmovcc so L(first_vec_x0) fits in one fetch
> + block. branch here as opposed to cmovcc is not that costly. Common
> + usage of memchr is to check if the return was NULL (if string was
> + known to contain CHAR user would use rawmemchr). This branch will be
> + highly correlated with the user branch and can be used by most
> + modern branch predictors to predict the user branch. */
> cmpl %eax, %edx
> - leaq (%rdi, %rax), %rax
> - cmovle %rcx, %rax
> - VZEROUPPER_RETURN
> -
> -L(null):
> - xorl %eax, %eax
> - ret
> -# endif
> - .p2align 4
> -L(cross_page_boundary):
> - /* Save pointer before aligning as its original value is
> - necessary for computer return address if byte is found or
> - adjusting length if it is not and this is memchr. */
> - movq %rdi, %rcx
> - /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr
> - and rdi for rawmemchr. */
> - orq $(VEC_SIZE - 1), %ALGN_PTR_REG
> - VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> -# ifndef USE_AS_RAWMEMCHR
> - /* Calculate length until end of page (length checked for a
> - match). */
> - leaq 1(%ALGN_PTR_REG), %rsi
> - subq %RRAW_PTR_REG, %rsi
> -# ifdef USE_AS_WMEMCHR
> - /* NB: Divide bytes by 4 to get wchar_t count. */
> - shrl $2, %esi
> -# endif
> -# endif
> - /* Remove the leading bytes. */
> - sarxl %ERAW_PTR_REG, %eax, %eax
> -# ifndef USE_AS_RAWMEMCHR
> - /* Check the end of data. */
> - cmpq %rsi, %rdx
> - jbe L(first_vec_x0)
> + jle L(null)
> + addq %rdi, %rax
> + ret
> # endif
> - testl %eax, %eax
> - jz L(cross_page_continue)
> - tzcntl %eax, %eax
> - addq %RRAW_PTR_REG, %rax
> -L(return_vzeroupper):
> - ZERO_UPPER_VEC_REGISTERS_RETURN
>
> - .p2align 4
> + .p2align 4,, 10
> L(first_vec_x1):
> - tzcntl %eax, %eax
> + bsfl %eax, %eax
> incq %rdi
> addq %rdi, %rax
> VZEROUPPER_RETURN
> -
> +# ifndef USE_AS_RAWMEMCHR
> + /* First in aligning bytes here. */
> +L(null):
> + xorl %eax, %eax
> + ret
> +# endif
> .p2align 4
> L(first_vec_x2):
> tzcntl %eax, %eax
> @@ -340,7 +315,7 @@ L(first_vec_x1_check):
> incq %rdi
> addq %rdi, %rax
> VZEROUPPER_RETURN
> - .p2align 4
> + .p2align 4,, 6
> L(set_zero_end):
> xorl %eax, %eax
> VZEROUPPER_RETURN
> @@ -428,5 +403,39 @@ L(last_vec_x3):
> VZEROUPPER_RETURN
> # endif
>
> + .p2align 4
> +L(cross_page_boundary):
> + /* Save pointer before aligning as its original value is necessary for
> + computer return address if byte is found or adjusting length if it
> + is not and this is memchr. */
> + movq %rdi, %rcx
> + /* Align data to VEC_SIZE. ALGN_PTR_REG is rcx for memchr and rdi for
> + rawmemchr. */
> + andq $-VEC_SIZE, %ALGN_PTR_REG
> + VPCMPEQ (%ALGN_PTR_REG), %ymm0, %ymm1
> + vpmovmskb %ymm1, %eax
> +# ifndef USE_AS_RAWMEMCHR
> + /* Calculate length until end of page (length checked for a match). */
> + leal VEC_SIZE(%ALGN_PTR_REG), %esi
> + subl %ERAW_PTR_REG, %esi
> +# ifdef USE_AS_WMEMCHR
> + /* NB: Divide bytes by 4 to get wchar_t count. */
> + shrl $2, %esi
> +# endif
> +# endif
> + /* Remove the leading bytes. */
> + sarxl %ERAW_PTR_REG, %eax, %eax
> +# ifndef USE_AS_RAWMEMCHR
> + /* Check the end of data. */
> + cmpq %rsi, %rdx
> + jbe L(first_vec_x0)
> +# endif
> + testl %eax, %eax
> + jz L(cross_page_continue)
> + bsfl %eax, %eax
> + addq %RRAW_PTR_REG, %rax
> + VZEROUPPER_RETURN
> +
> +
> END (MEMCHR)
> #endif
> --
> 2.34.1
>
LGTM.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Thanks.
--
H.J.
next prev parent reply other threads:[~2022-06-07 18:19 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-03 4:42 [PATCH v1 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-03 23:12 ` H.J. Lu
2022-06-03 23:33 ` Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-03 23:09 ` [PATCH v2 1/8] x86: Create header for VEC classes in x86 strings library H.J. Lu
2022-06-03 23:49 ` Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 " Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-06 21:30 ` H.J. Lu
2022-06-06 22:38 ` Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 23:50 ` [PATCH v3 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 23:50 ` [PATCH v3 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 2:45 ` H.J. Lu
2022-07-14 2:12 ` Sunil Pandey
2022-06-06 22:37 ` [PATCH v4 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 2:44 ` H.J. Lu
2022-06-07 4:10 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 2:41 ` H.J. Lu
2022-06-07 4:09 ` Noah Goldstein
2022-06-07 4:12 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 2:35 ` H.J. Lu
2022-06-07 4:06 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 18:03 ` H.J. Lu
2022-06-07 4:11 ` [PATCH v6 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-07 18:04 ` H.J. Lu
2022-07-14 2:19 ` Sunil Pandey
2022-06-07 4:11 ` [PATCH v6 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 18:21 ` H.J. Lu
2022-07-14 2:21 ` Sunil Pandey
2022-06-07 4:11 ` [PATCH v6 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 18:17 ` H.J. Lu
2022-07-14 2:26 ` Sunil Pandey
2022-07-14 2:43 ` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-07 18:18 ` H.J. Lu [this message]
2022-07-14 2:31 ` Sunil Pandey
2022-07-14 2:41 ` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 18:19 ` H.J. Lu
2022-07-14 2:32 ` Sunil Pandey
2022-06-07 18:04 ` [PATCH v6 1/8] x86: Create header for VEC classes in x86 strings library H.J. Lu
2022-07-14 2:07 ` Sunil Pandey
2022-06-03 4:42 ` [PATCH v1 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 4:47 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 4:49 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 4:50 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-03 4:51 ` [PATCH v1 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to='CAMe9rOrgX_tcy=-8dQ5Lh_bYzCX6f3Hzr4z0aeRx0hxn-hLupw@mail.gmail.com' \
--to=hjl.tools@gmail.com \
--cc=carlos@systemhalted.org \
--cc=goldstein.w.n@gmail.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).