From: Noah Goldstein <goldstein.w.n@gmail.com>
To: libc-alpha@sourceware.org
Subject: [PATCH v1 3/6] x86_64: Add support for __memcmpeq using sse2, avx2, and evex
Date: Tue, 26 Oct 2021 21:43:20 -0500 [thread overview]
Message-ID: <20211027024323.1199441-3-goldstein.w.n@gmail.com> (raw)
In-Reply-To: <20211027024323.1199441-1-goldstein.w.n@gmail.com>
No bug. This commit adds support for __memcmpeq to be implemented
seperately from memcmp. Support is added for versions optimized with
sse2, avx2, and evex.
---
sysdeps/generic/ifunc-init.h | 5 +-
sysdeps/x86_64/memcmp.S | 9 ++--
sysdeps/x86_64/multiarch/Makefile | 4 ++
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 21 +++++++++
sysdeps/x86_64/multiarch/ifunc-memcmpeq.h | 49 ++++++++++++++++++++
sysdeps/x86_64/multiarch/memcmp-sse2.S | 4 +-
sysdeps/x86_64/multiarch/memcmp.c | 3 --
sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S | 12 +++++
sysdeps/x86_64/multiarch/memcmpeq-avx2.S | 23 +++++++++
sysdeps/x86_64/multiarch/memcmpeq-evex.S | 23 +++++++++
sysdeps/x86_64/multiarch/memcmpeq-sse2.S | 23 +++++++++
sysdeps/x86_64/multiarch/memcmpeq.c | 35 ++++++++++++++
12 files changed, 202 insertions(+), 9 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/ifunc-memcmpeq.h
create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-evex.S
create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-sse2.S
create mode 100644 sysdeps/x86_64/multiarch/memcmpeq.c
diff --git a/sysdeps/generic/ifunc-init.h b/sysdeps/generic/ifunc-init.h
index 7f69485de8..ee8a8289c8 100644
--- a/sysdeps/generic/ifunc-init.h
+++ b/sysdeps/generic/ifunc-init.h
@@ -50,5 +50,8 @@
'__<symbol>_<variant>' as the optimized implementation and
'<symbol>_ifunc_selector' as the IFUNC selector. */
#define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME)
-#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name)
#define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector)
+#define OPTIMIZE1(name) EVALUATOR1 (SYMBOL_NAME, name)
+#define OPTIMIZE2(name) EVALUATOR2 (SYMBOL_NAME, name)
+/* Default is to use OPTIMIZE2. */
+#define OPTIMIZE(name) OPTIMIZE2(name)
diff --git a/sysdeps/x86_64/memcmp.S b/sysdeps/x86_64/memcmp.S
index 8a03e572e8..b53f2c0866 100644
--- a/sysdeps/x86_64/memcmp.S
+++ b/sysdeps/x86_64/memcmp.S
@@ -356,9 +356,10 @@ L(ATR32res):
.p2align 4,, 4
END(memcmp)
-#undef bcmp
+#ifdef USE_AS_MEMCMPEQ
+libc_hidden_def (memcmp)
+#else
+# undef bcmp
weak_alias (memcmp, bcmp)
-#undef __memcmpeq
-strong_alias (memcmp, __memcmpeq)
libc_hidden_builtin_def (memcmp)
-libc_hidden_def (__memcmpeq)
+#endif
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 26be40959c..044778585b 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -7,7 +7,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \
memrchr-sse2 memrchr-avx2 \
memcmp-sse2 \
+ memcmpeq-sse2 \
memcmp-avx2-movbe \
+ memcmpeq-avx2 \
memcmp-sse4 memcpy-ssse3 \
memmove-ssse3 \
memcpy-ssse3-back \
@@ -42,6 +44,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memset-avx512-unaligned-erms \
memchr-avx2-rtm \
memcmp-avx2-movbe-rtm \
+ memcmpeq-avx2-rtm \
memmove-avx-unaligned-erms-rtm \
memrchr-avx2-rtm \
memset-avx2-unaligned-erms-rtm \
@@ -61,6 +64,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
strrchr-avx2-rtm \
memchr-evex \
memcmp-evex-movbe \
+ memcmpeq-evex \
memmove-evex-unaligned-erms \
memrchr-evex \
memset-evex-unaligned-erms \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 39ab10613b..f7f3806d1d 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -38,6 +38,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
size_t i = 0;
+ /* Support sysdeps/x86_64/multiarch/memcmpeq.c. */
+ IFUNC_IMPL (i, name, __memcmpeq,
+ IFUNC_IMPL_ADD (array, i, __memcmpeq,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (MOVBE)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __memcmpeq_avx2)
+ IFUNC_IMPL_ADD (array, i, __memcmpeq,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (MOVBE)
+ && CPU_FEATURE_USABLE (RTM)),
+ __memcmpeq_avx2_rtm)
+ IFUNC_IMPL_ADD (array, i, __memcmpeq,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (MOVBE)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __memcmpeq_evex)
+ IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2))
+
/* Support sysdeps/x86_64/multiarch/memchr.c. */
IFUNC_IMPL (i, name, memchr,
IFUNC_IMPL_ADD (array, i, memchr,
diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h
new file mode 100644
index 0000000000..3319a9568a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h
@@ -0,0 +1,49 @@
+/* Common definition for __memcmpeq ifunc selections.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+# include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE1 (sse2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2_rtm) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE1 (evex) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features* cpu_features = __get_cpu_features ();
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ return OPTIMIZE1 (evex);
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE1 (avx2_rtm);
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ return OPTIMIZE1 (avx2);
+ }
+
+ return OPTIMIZE1 (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/memcmp-sse2.S b/sysdeps/x86_64/multiarch/memcmp-sse2.S
index 7b30b7ca2e..132d6fb339 100644
--- a/sysdeps/x86_64/multiarch/memcmp-sse2.S
+++ b/sysdeps/x86_64/multiarch/memcmp-sse2.S
@@ -17,7 +17,9 @@
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
-# define memcmp __memcmp_sse2
+# ifndef memcmp
+# define memcmp __memcmp_sse2
+# endif
# ifdef SHARED
# undef libc_hidden_builtin_def
diff --git a/sysdeps/x86_64/multiarch/memcmp.c b/sysdeps/x86_64/multiarch/memcmp.c
index 7b3409b1dd..fe725f3563 100644
--- a/sysdeps/x86_64/multiarch/memcmp.c
+++ b/sysdeps/x86_64/multiarch/memcmp.c
@@ -29,9 +29,6 @@
libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ());
# undef bcmp
weak_alias (memcmp, bcmp)
-# undef __memcmpeq
-strong_alias (memcmp, __memcmpeq)
-libc_hidden_def (__memcmpeq)
# ifdef SHARED
__hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp)
diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S
new file mode 100644
index 0000000000..24b6a0c9ff
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S
@@ -0,0 +1,12 @@
+#ifndef MEMCMP
+# define MEMCMP __memcmpeq_avx2_rtm
+#endif
+
+#define ZERO_UPPER_VEC_REGISTERS_RETURN \
+ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
+
+#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
+
+#define SECTION(p) p##.avx.rtm
+
+#include "memcmpeq-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S
new file mode 100644
index 0000000000..0181ea0d8d
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S
@@ -0,0 +1,23 @@
+/* __memcmpeq optimized with AVX2.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef MEMCMP
+# define MEMCMP __memcmpeq_avx2
+#endif
+
+#include "memcmp-avx2-movbe.S"
diff --git a/sysdeps/x86_64/multiarch/memcmpeq-evex.S b/sysdeps/x86_64/multiarch/memcmpeq-evex.S
new file mode 100644
index 0000000000..951e1e9560
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmpeq-evex.S
@@ -0,0 +1,23 @@
+/* __memcmpeq optimized with EVEX.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef MEMCMP
+# define MEMCMP __memcmpeq_evex
+#endif
+
+#include "memcmp-evex-movbe.S"
diff --git a/sysdeps/x86_64/multiarch/memcmpeq-sse2.S b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S
new file mode 100644
index 0000000000..c488cbbcd9
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S
@@ -0,0 +1,23 @@
+/* __memcmpeq optimized with SSE2.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef memcmp
+# define memcmp __memcmpeq_sse2
+#endif
+#define USE_AS_MEMCMPEQ 1
+#include "memcmp-sse2.S"
diff --git a/sysdeps/x86_64/multiarch/memcmpeq.c b/sysdeps/x86_64/multiarch/memcmpeq.c
new file mode 100644
index 0000000000..163e56047e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmpeq.c
@@ -0,0 +1,35 @@
+/* Multiple versions of __memcmpeq.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define __memcmpeq __redirect___memcmpeq
+# include <string.h>
+# undef __memcmpeq
+
+# define SYMBOL_NAME __memcmpeq
+# include "ifunc-memcmpeq.h"
+
+libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, IFUNC_SELECTOR ());
+
+# ifdef SHARED
+__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq)
+ __attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq);
+# endif
+#endif
--
2.25.1
next prev parent reply other threads:[~2021-10-27 2:43 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-27 2:43 [PATCH v1 1/6] String: Add __memcmpeq as build target Noah Goldstein
2021-10-27 2:43 ` [PATCH v1 2/6] Benchtests: Add benchtests for __memcmpeq Noah Goldstein
2021-10-27 12:45 ` H.J. Lu
2021-10-27 16:08 ` Noah Goldstein
2021-10-27 16:07 ` [PATCH v2 " Noah Goldstein
2021-10-27 17:59 ` H.J. Lu
2021-10-27 2:43 ` Noah Goldstein [this message]
2021-10-27 12:47 ` [PATCH v1 3/6] x86_64: Add support for __memcmpeq using sse2, avx2, and evex H.J. Lu
2021-10-27 2:43 ` [PATCH v1 4/6] x86_64: Add sse2 optimized __memcmpeq in memcmp-sse2.S Noah Goldstein
2021-10-27 12:48 ` H.J. Lu
2021-10-27 2:43 ` [PATCH v1 5/6] x86_64: Add avx2 optimized __memcmpeq in memcmpeq-avx2.S Noah Goldstein
2021-10-27 12:48 ` H.J. Lu
2021-10-27 2:43 ` [PATCH v1 6/6] x86_64: Add evex optimized __memcmpeq in memcmpeq-evex.S Noah Goldstein
2021-10-27 2:44 ` Noah Goldstein
2021-10-27 12:49 ` H.J. Lu
2021-10-27 12:42 ` [PATCH v1 1/6] String: Add __memcmpeq as build target H.J. Lu
2021-10-27 18:46 ` Noah Goldstein
2021-10-28 17:57 ` Joseph Myers
2021-10-28 18:25 ` Noah Goldstein
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211027024323.1199441-3-goldstein.w.n@gmail.com \
--to=goldstein.w.n@gmail.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).