public inbox for glibc-cvs@sourceware.org
help / color / mirror / Atom feed
* [glibc] x86: Add support to build st{p|r}{n}{cpy|cat} with explicit ISA level
@ 2022-07-16 12:00 Noah Goldstein
0 siblings, 0 replies; only message in thread
From: Noah Goldstein @ 2022-07-16 12:00 UTC (permalink / raw)
To: glibc-cvs
https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=49889fb256a7f9b894b2d16fea23de1ac25b65e2
commit 49889fb256a7f9b894b2d16fea23de1ac25b65e2
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed Jul 13 16:33:01 2022 -0700
x86: Add support to build st{p|r}{n}{cpy|cat} with explicit ISA level
1. Add default ISA level selection in non-multiarch/rtld
implementations.
2. Add ISA level build guards to different implementations.
- I.e strcpy-avx2.S which is ISA level 3 will only build if
compiled ISA level <= 3. Otherwise there is no reason to
include it as we will always use one of the ISA level 4
implementations (strcpy-evex.S).
3. Refactor the ifunc selector and ifunc implementation list to use
the ISA level aware wrapper macros that allow functions below the
compiled ISA level (with a guranteed replacement) to be skipped.
Tested with and without multiarch on x86_64 for ISA levels:
{generic, x86-64-v2, x86-64-v3, x86-64-v4}
And m32 with and without multiarch.
Diff:
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 183 +++++++++++++---------
sysdeps/x86_64/multiarch/ifunc-strcpy.h | 27 ++--
sysdeps/x86_64/multiarch/ifunc-strncpy.h | 22 +--
sysdeps/x86_64/multiarch/stpcpy-avx2.S | 6 +-
sysdeps/x86_64/multiarch/stpcpy-evex.S | 6 +-
sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S | 6 +-
sysdeps/x86_64/multiarch/stpcpy-sse2.S | 25 +--
sysdeps/x86_64/multiarch/stpncpy-avx2.S | 6 +-
sysdeps/x86_64/multiarch/stpncpy-evex.S | 6 +-
sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S | 6 +-
sysdeps/x86_64/multiarch/strcat-avx2.S | 5 +-
sysdeps/x86_64/multiarch/strcat-evex.S | 5 +-
sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S | 7 +-
sysdeps/x86_64/multiarch/strcat-sse2.S | 18 ++-
sysdeps/x86_64/multiarch/strcpy-avx2.S | 5 +-
sysdeps/x86_64/multiarch/strcpy-evex.S | 5 +-
sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S | 7 +-
sysdeps/x86_64/multiarch/strcpy-sse2.S | 23 +--
sysdeps/x86_64/multiarch/strncat-avx2.S | 6 +-
sysdeps/x86_64/multiarch/strncat-evex.S | 6 +-
sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S | 6 +-
sysdeps/x86_64/multiarch/strncpy-avx2.S | 6 +-
sysdeps/x86_64/multiarch/strncpy-evex.S | 6 +-
sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S | 6 +-
sysdeps/x86_64/stpcpy.S | 26 ++-
sysdeps/x86_64/stpncpy.S | 28 ++++
sysdeps/x86_64/strcat.S | 15 +-
sysdeps/x86_64/strcpy.S | 12 +-
sysdeps/x86_64/strncat.S | 28 ++++
sysdeps/x86_64/strncpy.S | 27 ++++
30 files changed, 382 insertions(+), 158 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 9318e98cc8..a71444eccb 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -403,33 +403,46 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/stpncpy.c. */
IFUNC_IMPL (i, name, stpncpy,
- IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2),
- __stpncpy_avx2)
- IFUNC_IMPL_ADD (array, i, stpncpy,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __stpncpy_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, stpncpy,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __stpncpy_evex)
- IFUNC_IMPL_ADD (array, i, stpncpy, 1,
- __stpncpy_sse2_unaligned))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, stpncpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __stpncpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy,
+ CPU_FEATURE_USABLE (AVX2),
+ __stpncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __stpncpy_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, stpncpy,
+ 1,
+ __stpncpy_sse2_unaligned))
/* Support sysdeps/x86_64/multiarch/stpcpy.c. */
IFUNC_IMPL (i, name, stpcpy,
- IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2),
- __stpcpy_avx2)
- IFUNC_IMPL_ADD (array, i, stpcpy,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __stpcpy_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, stpcpy,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __stpcpy_evex)
- IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned)
- IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, stpcpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __stpcpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy,
+ CPU_FEATURE_USABLE (AVX2),
+ __stpcpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __stpcpy_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, stpcpy,
+ 1,
+ __stpcpy_sse2_unaligned)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, stpcpy,
+ 1,
+ __stpcpy_sse2))
/* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */
IFUNC_IMPL (i, name, strcasecmp,
@@ -477,18 +490,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strcat.c. */
IFUNC_IMPL (i, name, strcat,
- IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2),
- __strcat_avx2)
- IFUNC_IMPL_ADD (array, i, strcat,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __strcat_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, strcat,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __strcat_evex)
- IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned)
- IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, strcat,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __strcat_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strcat,
+ CPU_FEATURE_USABLE (AVX2),
+ __strcat_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strcat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __strcat_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, strcat,
+ 1,
+ __strcat_sse2_unaligned)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, strcat,
+ 1,
+ __strcat_sse2))
/* Support sysdeps/x86_64/multiarch/strchr.c. */
IFUNC_IMPL (i, name, strchr,
@@ -584,18 +605,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strcpy.c. */
IFUNC_IMPL (i, name, strcpy,
- IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2),
- __strcpy_avx2)
- IFUNC_IMPL_ADD (array, i, strcpy,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __strcpy_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, strcpy,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __strcpy_evex)
- IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned)
- IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, strcpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __strcpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy,
+ CPU_FEATURE_USABLE (AVX2),
+ __strcpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __strcpy_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, strcpy,
+ 1,
+ __strcpy_sse2_unaligned)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, strcpy,
+ 1,
+ __strcpy_sse2))
/* Support sysdeps/x86_64/multiarch/strcspn.c. */
IFUNC_IMPL (i, name, strcspn,
@@ -651,33 +680,43 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strncat.c. */
IFUNC_IMPL (i, name, strncat,
- IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2),
- __strncat_avx2)
- IFUNC_IMPL_ADD (array, i, strncat,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __strncat_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, strncat,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __strncat_evex)
- IFUNC_IMPL_ADD (array, i, strncat, 1,
- __strncat_sse2_unaligned))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, strncat,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __strncat_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strncat,
+ CPU_FEATURE_USABLE (AVX2),
+ __strncat_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strncat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __strncat_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, strncat,
+ 1,
+ __strncat_sse2_unaligned))
/* Support sysdeps/x86_64/multiarch/strncpy.c. */
IFUNC_IMPL (i, name, strncpy,
- IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2),
- __strncpy_avx2)
- IFUNC_IMPL_ADD (array, i, strncpy,
- (CPU_FEATURE_USABLE (AVX2)
- && CPU_FEATURE_USABLE (RTM)),
- __strncpy_avx2_rtm)
- IFUNC_IMPL_ADD (array, i, strncpy,
- (CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
- __strncpy_evex)
- IFUNC_IMPL_ADD (array, i, strncpy, 1,
- __strncpy_sse2_unaligned))
+ X86_IFUNC_IMPL_ADD_V4 (array, i, strncpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __strncpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy,
+ CPU_FEATURE_USABLE (AVX2),
+ __strncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __strncpy_avx2_rtm)
+ /* ISA V2 wrapper for sse2_unaligned implementation because
+ the sse2_unaligned implementation is also used at ISA
+ level 2. */
+ X86_IFUNC_IMPL_ADD_V2 (array, i, strncpy,
+ 1,
+ __strncpy_sse2_unaligned))
/* Support sysdeps/x86_64/multiarch/strpbrk.c. */
IFUNC_IMPL (i, name, strpbrk,
diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h
index a15afa44e9..e083f71df3 100644
--- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h
+++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h
@@ -20,33 +20,38 @@
#include <init-arch.h>
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
- attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (sse2_unaligned) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
- const struct cpu_features* cpu_features = __get_cpu_features ();
+ const struct cpu_features *cpu_features = __get_cpu_features ();
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ AVX_Fast_Unaligned_Load, ))
{
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
return OPTIMIZE (avx2_rtm);
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
return OPTIMIZE (avx2);
}
- if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load))
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load, ))
return OPTIMIZE (sse2_unaligned);
return OPTIMIZE (sse2);
diff --git a/sysdeps/x86_64/multiarch/ifunc-strncpy.h b/sysdeps/x86_64/multiarch/ifunc-strncpy.h
index 323225af4d..5c1c46b885 100644
--- a/sysdeps/x86_64/multiarch/ifunc-strncpy.h
+++ b/sysdeps/x86_64/multiarch/ifunc-strncpy.h
@@ -19,28 +19,32 @@
#include <init-arch.h>
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
- attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (sse2_unaligned) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
- const struct cpu_features* cpu_features = __get_cpu_features ();
+ const struct cpu_features *cpu_features = __get_cpu_features ();
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ AVX_Fast_Unaligned_Load, ))
{
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
return OPTIMIZE (avx2_rtm);
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
return OPTIMIZE (avx2);
}
diff --git a/sysdeps/x86_64/multiarch/stpcpy-avx2.S b/sysdeps/x86_64/multiarch/stpcpy-avx2.S
index f0bd3029fe..277f9e73d2 100644
--- a/sysdeps/x86_64/multiarch/stpcpy-avx2.S
+++ b/sysdeps/x86_64/multiarch/stpcpy-avx2.S
@@ -1,3 +1,7 @@
+#ifndef STPCPY
+# define STPCPY __stpcpy_avx2
+#endif
+
#define USE_AS_STPCPY
-#define STRCPY __stpcpy_avx2
+#define STRCPY STPCPY
#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/stpcpy-evex.S b/sysdeps/x86_64/multiarch/stpcpy-evex.S
index 7c6f26cd98..4f1c015424 100644
--- a/sysdeps/x86_64/multiarch/stpcpy-evex.S
+++ b/sysdeps/x86_64/multiarch/stpcpy-evex.S
@@ -1,3 +1,7 @@
+#ifndef STPCPY
+# define STPCPY __stpcpy_evex
+#endif
+
#define USE_AS_STPCPY
-#define STRCPY __stpcpy_evex
+#define STRCPY STPCPY
#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S
index 34231f8b46..4c77e5b51c 100644
--- a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S
@@ -1,3 +1,7 @@
+#ifndef STPCPY
+# define STPCPY __stpcpy_sse2_unaligned
+#endif
+
#define USE_AS_STPCPY
-#define STRCPY __stpcpy_sse2_unaligned
+#define STRCPY STPCPY
#include "strcpy-sse2-unaligned.S"
diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2.S b/sysdeps/x86_64/multiarch/stpcpy-sse2.S
index ea9f973af3..fcd67cada2 100644
--- a/sysdeps/x86_64/multiarch/stpcpy-sse2.S
+++ b/sysdeps/x86_64/multiarch/stpcpy-sse2.S
@@ -1,26 +1,7 @@
-/* stpcpy optimized with SSE2.
- Copyright (C) 2017-2022 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#if IS_IN (libc)
-# ifndef STRCPY
-# define STRCPY __stpcpy_sse2
-# endif
+#ifndef STPCPY
+# define STPCPY __stpcpy_sse2
#endif
#define USE_AS_STPCPY
+#define STRCPY STPCPY
#include "strcpy-sse2.S"
diff --git a/sysdeps/x86_64/multiarch/stpncpy-avx2.S b/sysdeps/x86_64/multiarch/stpncpy-avx2.S
index 032b0407d0..b2f8c19143 100644
--- a/sysdeps/x86_64/multiarch/stpncpy-avx2.S
+++ b/sysdeps/x86_64/multiarch/stpncpy-avx2.S
@@ -1,4 +1,8 @@
+#ifndef STPNCPY
+# define STPNCPY __stpncpy_avx2
+#endif
+
#define USE_AS_STPCPY
#define USE_AS_STRNCPY
-#define STRCPY __stpncpy_avx2
+#define STRCPY STPNCPY
#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/stpncpy-evex.S b/sysdeps/x86_64/multiarch/stpncpy-evex.S
index 1570014d1c..99ea76a372 100644
--- a/sysdeps/x86_64/multiarch/stpncpy-evex.S
+++ b/sysdeps/x86_64/multiarch/stpncpy-evex.S
@@ -1,4 +1,8 @@
+#ifndef STPNCPY
+# define STPNCPY __stpncpy_evex
+#endif
+
#define USE_AS_STPCPY
#define USE_AS_STRNCPY
-#define STRCPY __stpncpy_evex
+#define STRCPY STPNCPY
#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S
index 658520f78f..c83706016f 100644
--- a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S
@@ -1,4 +1,8 @@
+#ifndef STPNCPY
+# define STPNCPY __stpncpy_sse2_unaligned
+#endif
+
#define USE_AS_STPCPY
#define USE_AS_STRNCPY
-#define STRCPY __stpncpy_sse2_unaligned
+#define STRCPY STPNCPY
#include "strcpy-sse2-unaligned.S"
diff --git a/sysdeps/x86_64/multiarch/strcat-avx2.S b/sysdeps/x86_64/multiarch/strcat-avx2.S
index 51d42f89e8..d9b7fb2a43 100644
--- a/sysdeps/x86_64/multiarch/strcat-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcat-avx2.S
@@ -16,7 +16,10 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (3)
+
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcat-evex.S b/sysdeps/x86_64/multiarch/strcat-evex.S
index 110505cb13..0e2df947e9 100644
--- a/sysdeps/x86_64/multiarch/strcat-evex.S
+++ b/sysdeps/x86_64/multiarch/strcat-evex.S
@@ -16,7 +16,10 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (4)
+
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S
index a0e051d08a..9d2ca1d504 100644
--- a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S
@@ -16,7 +16,12 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation
+ so we need this to build for ISA V2 builds. */
+#if ISA_SHOULD_BUILD (2)
+
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcat-sse2.S b/sysdeps/x86_64/multiarch/strcat-sse2.S
index 244c4a6d74..d1d0a3366a 100644
--- a/sysdeps/x86_64/multiarch/strcat-sse2.S
+++ b/sysdeps/x86_64/multiarch/strcat-sse2.S
@@ -1,5 +1,6 @@
-/* strcat optimized with SSE2.
- Copyright (C) 2017-2022 Free Software Foundation, Inc.
+/* strcat(dest, src) -- Append SRC on the end of DEST.
+ Optimized for x86-64.
+ Copyright (C) 2002-2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -16,13 +17,17 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (1)
+
+# include <sysdep.h>
+
# ifndef STRCAT
-# define STRCAT __strcat_sse2
+# define STRCAT __strcat_sse2
# endif
-#endif
-#include <sysdep.h>
+/* Will be removed when new strcpy implementation gets merged. */
.text
ENTRY (STRCAT)
@@ -256,3 +261,4 @@ ENTRY (STRCAT)
movq %rdi, %rax /* Source is return value. */
retq
END (STRCAT)
+#endif
diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2.S b/sysdeps/x86_64/multiarch/strcpy-avx2.S
index 064eafcbee..c725834929 100644
--- a/sysdeps/x86_64/multiarch/strcpy-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcpy-avx2.S
@@ -16,7 +16,10 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (3)
+
# ifndef USE_AS_STRCAT
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcpy-evex.S b/sysdeps/x86_64/multiarch/strcpy-evex.S
index 32229e05d8..82e45ac675 100644
--- a/sysdeps/x86_64/multiarch/strcpy-evex.S
+++ b/sysdeps/x86_64/multiarch/strcpy-evex.S
@@ -16,7 +16,10 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (4)
+
# ifndef USE_AS_STRCAT
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
index d5fbd9570a..a889cd96be 100644
--- a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
@@ -16,7 +16,12 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
+#include <isa-level.h>
+
+/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation
+ so we need this to build for ISA V2 builds. */
+#if ISA_SHOULD_BUILD (2)
+
# ifndef USE_AS_STRCAT
# include <sysdep.h>
diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2.S b/sysdeps/x86_64/multiarch/strcpy-sse2.S
index 8b5db8b13d..e29b411314 100644
--- a/sysdeps/x86_64/multiarch/strcpy-sse2.S
+++ b/sysdeps/x86_64/multiarch/strcpy-sse2.S
@@ -1,5 +1,5 @@
-/* strcpy optimized with SSE2.
- Copyright (C) 2017-2022 Free Software Foundation, Inc.
+/* strcpy/stpcpy implementation for x86-64.
+ Copyright (C) 2002-2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -16,13 +16,15 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#if IS_IN (libc)
-# ifndef STRCPY
+#include <isa-level.h>
+
+#if ISA_SHOULD_BUILD (1)
+
+# include <sysdep.h>
+
+# ifndef STPCPY
# define STRCPY __strcpy_sse2
# endif
-#endif
-
-#include <sysdep.h>
.text
ENTRY (STRCPY)
@@ -144,10 +146,11 @@ ENTRY (STRCPY)
jmp 3b /* and look at next two bytes in %rax. */
4:
-#ifdef USE_AS_STPCPY
+# ifdef USE_AS_STPCPY
movq %rdx, %rax /* Destination is return value. */
-#else
+# else
movq %rdi, %rax /* Source is return value. */
-#endif
+# endif
retq
END (STRCPY)
+#endif
diff --git a/sysdeps/x86_64/multiarch/strncat-avx2.S b/sysdeps/x86_64/multiarch/strncat-avx2.S
index bfefa659bb..52ecbca943 100644
--- a/sysdeps/x86_64/multiarch/strncat-avx2.S
+++ b/sysdeps/x86_64/multiarch/strncat-avx2.S
@@ -1,3 +1,7 @@
+#ifndef STRNCAT
+# define STRNCAT __strncat_avx2
+#endif
+
#define USE_AS_STRNCAT
-#define STRCAT __strncat_avx2
+#define STRCAT STRNCAT
#include "strcat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/strncat-evex.S b/sysdeps/x86_64/multiarch/strncat-evex.S
index 8884f02371..203a19bf21 100644
--- a/sysdeps/x86_64/multiarch/strncat-evex.S
+++ b/sysdeps/x86_64/multiarch/strncat-evex.S
@@ -1,3 +1,7 @@
+#ifndef STRNCAT
+# define STRNCAT __strncat_evex
+#endif
+
#define USE_AS_STRNCAT
-#define STRCAT __strncat_evex
+#define STRCAT STRNCAT
#include "strcat-evex.S"
diff --git a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S
index 133e1d20b0..5982e22677 100644
--- a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S
@@ -1,3 +1,7 @@
+#ifndef STRNCAT
+# define STRNCAT __strncat_sse2_unaligned
+#endif
+
#define USE_AS_STRNCAT
-#define STRCAT __strncat_sse2_unaligned
+#define STRCAT STRNCAT
#include "strcat-sse2-unaligned.S"
diff --git a/sysdeps/x86_64/multiarch/strncpy-avx2.S b/sysdeps/x86_64/multiarch/strncpy-avx2.S
index 9ef8c87627..ce634e94fa 100644
--- a/sysdeps/x86_64/multiarch/strncpy-avx2.S
+++ b/sysdeps/x86_64/multiarch/strncpy-avx2.S
@@ -1,3 +1,7 @@
+#ifndef STRNCPY
+# define STRNCPY __strncpy_avx2
+#endif
+
#define USE_AS_STRNCPY
-#define STRCPY __strncpy_avx2
+#define STRCPY STRNCPY
#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/strncpy-evex.S b/sysdeps/x86_64/multiarch/strncpy-evex.S
index 40e391f0da..1b3426d511 100644
--- a/sysdeps/x86_64/multiarch/strncpy-evex.S
+++ b/sysdeps/x86_64/multiarch/strncpy-evex.S
@@ -1,3 +1,7 @@
+#ifndef STRNCPY
+# define STRNCPY __strncpy_evex
+#endif
+
#define USE_AS_STRNCPY
-#define STRCPY __strncpy_evex
+#define STRCPY STRNCPY
#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S
index fcc23a754a..e3eb15d93a 100644
--- a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S
@@ -1,3 +1,7 @@
+#ifndef STRNCPY
+# define STRNCPY __strncpy_sse2_unaligned
+#endif
+
#define USE_AS_STRNCPY
-#define STRCPY __strncpy_sse2_unaligned
+#define STRCPY STRNCPY
#include "strcpy-sse2-unaligned.S"
diff --git a/sysdeps/x86_64/stpcpy.S b/sysdeps/x86_64/stpcpy.S
index b097c203dd..c7d8d959a7 100644
--- a/sysdeps/x86_64/stpcpy.S
+++ b/sysdeps/x86_64/stpcpy.S
@@ -1,6 +1,28 @@
-#define STRCPY __stpcpy
+/* stpcpy dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
-#include "multiarch/stpcpy-sse2.S"
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#define STPCPY __stpcpy
+
+#define DEFAULT_IMPL_V1 "multiarch/stpcpy-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/stpcpy-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/stpcpy-evex.S"
+
+#include "isa-default-impl.h"
weak_alias (__stpcpy, stpcpy)
libc_hidden_def (__stpcpy)
diff --git a/sysdeps/x86_64/stpncpy.S b/sysdeps/x86_64/stpncpy.S
new file mode 100644
index 0000000000..5c2d6efa3d
--- /dev/null
+++ b/sysdeps/x86_64/stpncpy.S
@@ -0,0 +1,28 @@
+/* stpncpy dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#define STPNCPY __stpncpy
+
+#define DEFAULT_IMPL_V1 "multiarch/stpncpy-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/stpncpy-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/stpncpy-evex.S"
+
+#include "isa-default-impl.h"
+
+weak_alias (__stpncpy, stpncpy)
+libc_hidden_def (__stpncpy)
diff --git a/sysdeps/x86_64/strcat.S b/sysdeps/x86_64/strcat.S
index fc3e8a9bcf..9bca7ecedc 100644
--- a/sysdeps/x86_64/strcat.S
+++ b/sysdeps/x86_64/strcat.S
@@ -1,6 +1,5 @@
-/* strcat(dest, src) -- Append SRC on the end of DEST.
- Optimized for x86-64.
- Copyright (C) 2002-2022 Free Software Foundation, Inc.
+/* strcat dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -17,6 +16,12 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#define STRCAT strcat
-#include "multiarch/strcat-sse2.S"
+#define STRCAT strcat
+
+#define DEFAULT_IMPL_V1 "multiarch/strcat-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/strcat-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/strcat-evex.S"
+
+#include "isa-default-impl.h"
+
libc_hidden_builtin_def (strcat)
diff --git a/sysdeps/x86_64/strcpy.S b/sysdeps/x86_64/strcpy.S
index 05f19e6e94..7c04cc2102 100644
--- a/sysdeps/x86_64/strcpy.S
+++ b/sysdeps/x86_64/strcpy.S
@@ -1,5 +1,5 @@
-/* strcpy/stpcpy implementation for x86-64.
- Copyright (C) 2002-2022 Free Software Foundation, Inc.
+/* strcpy dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -17,5 +17,11 @@
<https://www.gnu.org/licenses/>. */
#define STRCPY strcpy
-#include "multiarch/strcpy-sse2.S"
+
+#define DEFAULT_IMPL_V1 "multiarch/strcpy-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/strcpy-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/strcpy-evex.S"
+
+#include "isa-default-impl.h"
+
libc_hidden_builtin_def (strcpy)
diff --git a/sysdeps/x86_64/strncat.S b/sysdeps/x86_64/strncat.S
new file mode 100644
index 0000000000..3ba2603a28
--- /dev/null
+++ b/sysdeps/x86_64/strncat.S
@@ -0,0 +1,28 @@
+/* strncat dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#define STRNCAT strncat
+
+#define DEFAULT_IMPL_V1 "multiarch/strncat-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/strncat-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/strncat-evex.S"
+
+#include "isa-default-impl.h"
+
+strong_alias (strncat, __strncat)
+libc_hidden_def (__strncat)
diff --git a/sysdeps/x86_64/strncpy.S b/sysdeps/x86_64/strncpy.S
new file mode 100644
index 0000000000..04c904e60d
--- /dev/null
+++ b/sysdeps/x86_64/strncpy.S
@@ -0,0 +1,27 @@
+/* strncpy dispatch for RTLD and non-multiarch build
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#define STRNCPY strncpy
+
+#define DEFAULT_IMPL_V1 "multiarch/strncpy-sse2-unaligned.S"
+#define DEFAULT_IMPL_V3 "multiarch/strncpy-avx2.S"
+#define DEFAULT_IMPL_V4 "multiarch/strncpy-evex.S"
+
+#include "isa-default-impl.h"
+
+libc_hidden_builtin_def (strncpy)
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2022-07-16 12:00 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-07-16 12:00 [glibc] x86: Add support to build st{p|r}{n}{cpy|cat} with explicit ISA level Noah Goldstein
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).