From: Noah Goldstein <goldstein.w.n@gmail.com>
To: libc-alpha@sourceware.org
Subject: [PATCH v3 8/8] x86: Shrink code size of memchr-evex.S
Date: Fri, 3 Jun 2022 18:50:01 -0500 [thread overview]
Message-ID: <20220603235001.398195-8-goldstein.w.n@gmail.com> (raw)
In-Reply-To: <20220603235001.398195-1-goldstein.w.n@gmail.com>
This is not meant as a performance optimization. The previous code was
far to liberal in aligning targets and wasted code size unnecissarily.
The total code size saving is: 64 bytes
There are no non-negligible changes in the benchmarks.
Geometric Mean of all benchmarks New / Old: 1.000
Full xcheck passes on x86_64.
---
sysdeps/x86_64/multiarch/memchr-evex.S | 46 ++++++++++++++------------
1 file changed, 25 insertions(+), 21 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S
index cfaf02907d..0fd11b7632 100644
--- a/sysdeps/x86_64/multiarch/memchr-evex.S
+++ b/sysdeps/x86_64/multiarch/memchr-evex.S
@@ -88,7 +88,7 @@
# define PAGE_SIZE 4096
.section SECTION(.text),"ax",@progbits
-ENTRY (MEMCHR)
+ENTRY_P2ALIGN (MEMCHR, 6)
# ifndef USE_AS_RAWMEMCHR
/* Check for zero length. */
test %RDX_LP, %RDX_LP
@@ -131,22 +131,24 @@ L(zero):
xorl %eax, %eax
ret
- .p2align 5
+ .p2align 4
L(first_vec_x0):
- /* Check if first match was before length. */
- tzcntl %eax, %eax
- xorl %ecx, %ecx
- cmpl %eax, %edx
- leaq (%rdi, %rax, CHAR_SIZE), %rax
- cmovle %rcx, %rax
+ /* Check if first match was before length. NB: tzcnt has false data-
+ dependency on destination. eax already had a data-dependency on esi
+ so this should have no affect here. */
+ tzcntl %eax, %esi
+# ifdef USE_AS_WMEMCHR
+ leaq (%rdi, %rsi, CHAR_SIZE), %rdi
+# else
+ addq %rsi, %rdi
+# endif
+ xorl %eax, %eax
+ cmpl %esi, %edx
+ cmovg %rdi, %rax
ret
-# else
- /* NB: first_vec_x0 is 17 bytes which will leave
- cross_page_boundary (which is relatively cold) close enough
- to ideal alignment. So only realign L(cross_page_boundary) if
- rawmemchr. */
- .p2align 4
# endif
+
+ .p2align 4
L(cross_page_boundary):
/* Save pointer before aligning as its original value is
necessary for computer return address if byte is found or
@@ -400,10 +402,14 @@ L(last_2x_vec):
L(zero_end):
ret
+L(set_zero_end):
+ xorl %eax, %eax
+ ret
.p2align 4
L(first_vec_x1_check):
- tzcntl %eax, %eax
+ /* eax must be non-zero. Use bsfl to save code size. */
+ bsfl %eax, %eax
/* Adjust length. */
subl $-(CHAR_PER_VEC * 4), %edx
/* Check if match within remaining length. */
@@ -412,9 +418,6 @@ L(first_vec_x1_check):
/* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax
ret
-L(set_zero_end):
- xorl %eax, %eax
- ret
.p2align 4
L(loop_4x_vec_end):
@@ -464,7 +467,7 @@ L(loop_4x_vec_end):
# endif
ret
- .p2align 4
+ .p2align 4,, 10
L(last_vec_x1_return):
tzcntl %eax, %eax
# if defined USE_AS_WMEMCHR || RET_OFFSET != 0
@@ -496,6 +499,7 @@ L(last_vec_x3_return):
# endif
# ifndef USE_AS_RAWMEMCHR
+ .p2align 4,, 5
L(last_4x_vec_or_less_cmpeq):
VPCMP $0, (VEC_SIZE * 5)(%rdi), %YMMMATCH, %k0
kmovd %k0, %eax
@@ -546,7 +550,7 @@ L(last_4x_vec):
# endif
andl %ecx, %eax
jz L(zero_end2)
- tzcntl %eax, %eax
+ bsfl %eax, %eax
leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
L(zero_end2):
ret
@@ -562,6 +566,6 @@ L(last_vec_x3):
leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
ret
# endif
-
+ /* 7 bytes from next cache line. */
END (MEMCHR)
#endif
--
2.34.1
next prev parent reply other threads:[~2022-06-03 23:50 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-03 4:42 [PATCH v1 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-03 23:12 ` H.J. Lu
2022-06-03 23:33 ` Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 20:04 ` [PATCH v2 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-03 23:09 ` [PATCH v2 1/8] x86: Create header for VEC classes in x86 strings library H.J. Lu
2022-06-03 23:49 ` Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 " Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-06 21:30 ` H.J. Lu
2022-06-06 22:38 ` Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 23:49 ` [PATCH v3 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 23:50 ` [PATCH v3 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 23:50 ` Noah Goldstein [this message]
2022-06-06 22:37 ` [PATCH v4 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 2:45 ` H.J. Lu
2022-07-14 2:12 ` Sunil Pandey
2022-06-06 22:37 ` [PATCH v4 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 2:44 ` H.J. Lu
2022-06-07 4:10 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 2:41 ` H.J. Lu
2022-06-07 4:09 ` Noah Goldstein
2022-06-07 4:12 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 2:35 ` H.J. Lu
2022-06-07 4:06 ` Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-06 22:37 ` [PATCH v4 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-07 4:05 ` [PATCH v5 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 2/8] x86: Add COND_VZEROUPPER that can replace vzeroupper if no `ret` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-07 18:03 ` H.J. Lu
2022-06-07 4:11 ` [PATCH v6 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-07 18:04 ` H.J. Lu
2022-07-14 2:19 ` Sunil Pandey
2022-06-07 4:11 ` [PATCH v6 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-07 18:21 ` H.J. Lu
2022-07-14 2:21 ` Sunil Pandey
2022-06-07 4:11 ` [PATCH v6 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-07 18:17 ` H.J. Lu
2022-07-14 2:26 ` Sunil Pandey
2022-07-14 2:43 ` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-07 18:18 ` H.J. Lu
2022-07-14 2:31 ` Sunil Pandey
2022-07-14 2:41 ` Noah Goldstein
2022-06-07 4:11 ` [PATCH v6 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-07 18:19 ` H.J. Lu
2022-07-14 2:32 ` Sunil Pandey
2022-06-07 18:04 ` [PATCH v6 1/8] x86: Create header for VEC classes in x86 strings library H.J. Lu
2022-07-14 2:07 ` Sunil Pandey
2022-06-03 4:42 ` [PATCH v1 3/8] Benchtests: Improve memrchr benchmarks Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 4/8] x86: Optimize memrchr-sse2.S Noah Goldstein
2022-06-03 4:47 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 5/8] x86: Optimize memrchr-evex.S Noah Goldstein
2022-06-03 4:49 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 6/8] x86: Optimize memrchr-avx2.S Noah Goldstein
2022-06-03 4:50 ` Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 7/8] x86: Shrink code size of memchr-avx2.S Noah Goldstein
2022-06-03 4:42 ` [PATCH v1 8/8] x86: Shrink code size of memchr-evex.S Noah Goldstein
2022-06-03 4:51 ` [PATCH v1 1/8] x86: Create header for VEC classes in x86 strings library Noah Goldstein
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220603235001.398195-8-goldstein.w.n@gmail.com \
--to=goldstein.w.n@gmail.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).