From: "Pawar, Amit" <Amit.Pawar@amd.com>
To: "H.J. Lu" <hjl.tools@gmail.com>
Cc: "libc-alpha@sourceware.org" <libc-alpha@sourceware.org>
Subject: RE: [PATCH x86_64] Update memcpy, mempcpy and memmove selection order for Excavator CPU BZ #19583
Date: Fri, 18 Mar 2016 11:43:00 -0000 [thread overview]
Message-ID: <SN1PR12MB07332500CE527AA6EAC1C360978C0@SN1PR12MB0733.namprd12.prod.outlook.com> (raw)
In-Reply-To: <CAMe9rOqhAUNhvD0=FZm23MDeVMRaYrnkZ51wWB1O4JRu8o2ywg@mail.gmail.com>
[-- Attachment #1: Type: text/plain, Size: 88 bytes --]
As per the comments, attached files are updated. If OK please commit.
Thanks,
Amit
[-- Attachment #2: 0001-x86_64-Update-memcpy-mempcpy-and-memmove-selection-o.patch --]
[-- Type: application/octet-stream, Size: 7681 bytes --]
From 5e20f755973e0ae752c872f1c1215b3d53c05772 Mon Sep 17 00:00:00 2001
From: Amit Pawar <Amit.Pawar@amd.com>
Date: Fri, 18 Mar 2016 16:58:46 +0530
Subject: [PATCH] x86_64 Update memcpy, mempcpy and memmove selection order for
Excavator CPU.
Performance of Fast_Copy_Backward based implementations are better compare to
AVX_Fast_Unaligned_Load based implementation of memcpy, memcpy_chk, mempcpy,
mempcpy_chk, memmove and memmove_chk functions on Excavator cpu. New feature
bit is required to fix this issue in IFUNC functions without affecting other
targets. So defining two new bit_arch_Avoid_AVX_Fast_Unaligned_Load and
index_arch_Avoid_AVX_Fast_Unaligned_Load feature bit macros and selection
order of these functions are updated.
[BZ #19583]
* sysdeps/x86/cpu-features.h (bit_arch_Avoid_AVX_Fast_Unaligned_Load):
New.
(index_arch_Avoid_AVX_Fast_Unaligned_Load) Likewise.
* sysdeps/x86/cpu-features.c (init_cpu_features): Set Fast_Copy_Backward
and Avoid_AVX_Fast_Unaligned_Load bits for Excavator core.
* sysdeps/x86_64/multiarch/memcpy.S
(__new_memcpy, Avoid_AVX_Fast_Unaligned_Load): Add check for
Avoid_AVX_Fast_Unaligned_Load bit and select on Excavator core instead
of AVX_Fast_Unaligned_Load.
* sysdeps/x86_64/multiarch/memcpy_chk.S
(__memcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/mempcpy.S
(__mempcpy, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk.S
(__mempcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/memmove.c
(__libc_memmove, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/memmove_chk.c
(__memmove_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
---
sysdeps/x86/cpu-features.c | 15 +++++++++++++--
sysdeps/x86/cpu-features.h | 3 +++
sysdeps/x86_64/multiarch/memcpy.S | 6 +++++-
sysdeps/x86_64/multiarch/memcpy_chk.S | 3 +++
sysdeps/x86_64/multiarch/memmove.c | 3 ++-
sysdeps/x86_64/multiarch/memmove_chk.c | 4 +++-
sysdeps/x86_64/multiarch/mempcpy.S | 3 +++
sysdeps/x86_64/multiarch/mempcpy_chk.S | 3 +++
8 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 1787716..f0baf0e 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -160,8 +160,19 @@ init_cpu_features (struct cpu_features *cpu_features)
{
/* "Excavator" */
if (model >= 0x60 && model <= 0x7f)
- cpu_features->feature[index_arch_Fast_Unaligned_Load]
- |= bit_arch_Fast_Unaligned_Load;
+ {
+#if index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+# error index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+#endif
+#if index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Load
+# error index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Load
+#endif
+ cpu_features->feature[index_arch_Avoid_AVX_Fast_Unaligned_Load]
+ |= (bit_arch_Fast_Unaligned_Load
+ | bit_arch_Fast_Copy_Backward
+ | bit_arch_Avoid_AVX_Fast_Unaligned_Load);
+
+ }
}
}
else
diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h
index 0624a92..fe59828 100644
--- a/sysdeps/x86/cpu-features.h
+++ b/sysdeps/x86/cpu-features.h
@@ -35,6 +35,7 @@
#define bit_arch_I686 (1 << 15)
#define bit_arch_Prefer_MAP_32BIT_EXEC (1 << 16)
#define bit_arch_Prefer_No_VZEROUPPER (1 << 17)
+#define bit_arch_Avoid_AVX_Fast_Unaligned_Load (1 << 18)
/* CPUID Feature flags. */
@@ -101,6 +102,7 @@
# define index_arch_I686 FEATURE_INDEX_1*FEATURE_SIZE
# define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1*FEATURE_SIZE
# define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1*FEATURE_SIZE
+# define index_arch_Avoid_AVX_Fast_Unaligned_Load FEATURE_INDEX_1*FEATURE_SIZE
# if defined (_LIBC) && !IS_IN (nonlib)
@@ -259,6 +261,7 @@ extern const struct cpu_features *__get_cpu_features (void)
# define index_arch_I686 FEATURE_INDEX_1
# define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1
# define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1
+# define index_arch_Avoid_AVX_Fast_Unaligned_Load FEATURE_INDEX_1
#endif /* !__ASSEMBLER__ */
diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S
index 8882590..3c67da8 100644
--- a/sysdeps/x86_64/multiarch/memcpy.S
+++ b/sysdeps/x86_64/multiarch/memcpy.S
@@ -40,7 +40,7 @@ ENTRY(__new_memcpy)
#endif
1: lea __memcpy_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jnz 2f
+ jnz 3f
lea __memcpy_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Load)
jnz 2f
@@ -52,6 +52,10 @@ ENTRY(__new_memcpy)
jnz 2f
lea __memcpy_ssse3(%rip), %RAX_LP
2: ret
+3: HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ jz 2b
+ lea __memcpy_ssse3_back(%rip), %RAX_LP
+ ret
END(__new_memcpy)
# undef ENTRY
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.S b/sysdeps/x86_64/multiarch/memcpy_chk.S
index 648217e..5b608f6 100644
--- a/sysdeps/x86_64/multiarch/memcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/memcpy_chk.S
@@ -48,6 +48,9 @@ ENTRY(__memcpy_chk)
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz 2f
leaq __memcpy_chk_avx_unaligned(%rip), %rax
+ HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ jz 2f
+ leaq __memcpy_chk_ssse3_back(%rip), %rax
2: ret
END(__memcpy_chk)
# else
diff --git a/sysdeps/x86_64/multiarch/memmove.c b/sysdeps/x86_64/multiarch/memmove.c
index 8da5640..b9d9439 100644
--- a/sysdeps/x86_64/multiarch/memmove.c
+++ b/sysdeps/x86_64/multiarch/memmove.c
@@ -59,7 +59,8 @@ libc_ifunc (__libc_memmove,
:
#endif
(HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- ? __memmove_avx_unaligned
+ ? (HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ ? __memmove_ssse3_back : __memmove_avx_unaligned)
: (HAS_CPU_FEATURE (SSSE3)
? (HAS_ARCH_FEATURE (Fast_Copy_Backward)
? __memmove_ssse3_back : __memmove_ssse3)
diff --git a/sysdeps/x86_64/multiarch/memmove_chk.c b/sysdeps/x86_64/multiarch/memmove_chk.c
index f64da63..b8a4282 100644
--- a/sysdeps/x86_64/multiarch/memmove_chk.c
+++ b/sysdeps/x86_64/multiarch/memmove_chk.c
@@ -39,7 +39,9 @@ libc_ifunc (__memmove_chk,
? __memmove_chk_avx512_no_vzeroupper
:
#endif
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) ? __memmove_chk_avx_unaligned :
+ HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
+ ? (HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ ? __memmove_chk_ssse3_back : __memmove_chk_avx_unaligned) :
(HAS_CPU_FEATURE (SSSE3)
? (HAS_ARCH_FEATURE (Fast_Copy_Backward)
? __memmove_chk_ssse3_back : __memmove_chk_ssse3)
diff --git a/sysdeps/x86_64/multiarch/mempcpy.S b/sysdeps/x86_64/multiarch/mempcpy.S
index ed78623..6a8898d 100644
--- a/sysdeps/x86_64/multiarch/mempcpy.S
+++ b/sysdeps/x86_64/multiarch/mempcpy.S
@@ -46,6 +46,9 @@ ENTRY(__mempcpy)
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz 2f
leaq __mempcpy_avx_unaligned(%rip), %rax
+ HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ jz 2f
+ leaq __mempcpy_ssse3_back(%rip), %rax
2: ret
END(__mempcpy)
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.S b/sysdeps/x86_64/multiarch/mempcpy_chk.S
index 6e8a89d..ecea9c2 100644
--- a/sysdeps/x86_64/multiarch/mempcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/mempcpy_chk.S
@@ -48,6 +48,9 @@ ENTRY(__mempcpy_chk)
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz 2f
leaq __mempcpy_chk_avx_unaligned(%rip), %rax
+ HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+ jz 2f
+ leaq __mempcpy_chk_ssse3_back(%rip), %rax
2: ret
END(__mempcpy_chk)
# else
--
2.1.4
[-- Attachment #3: ChangeLog --]
[-- Type: application/octet-stream, Size: 1022 bytes --]
2016-03-18 Amit Pawar <Amit.Pawar@amd.com>
[BZ #19583]
* sysdeps/x86/cpu-features.h (bit_arch_Avoid_AVX_Fast_Unaligned_Load):
New.
(index_arch_Avoid_AVX_Fast_Unaligned_Load) Likewise.
* sysdeps/x86/cpu-features.c (init_cpu_features): Set Fast_Copy_Backward
and Avoid_AVX_Fast_Unaligned_Load bits for Excavator core.
* sysdeps/x86_64/multiarch/memcpy.S
(__new_memcpy, Avoid_AVX_Fast_Unaligned_Load): Add check for
Avoid_AVX_Fast_Unaligned_Load bit and select on Excavator core instead
of AVX_Fast_Unaligned_Load.
* sysdeps/x86_64/multiarch/memcpy_chk.S
(__memcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/mempcpy.S
(__mempcpy, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk.S
(__mempcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/memmove.c
(__libc_memmove, Avoid_AVX_Fast_Unaligned_Load): Likewise.
* sysdeps/x86_64/multiarch/memmove_chk.c
(__memmove_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
next prev parent reply other threads:[~2016-03-18 11:43 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-03-17 10:52 Pawar, Amit
2016-03-17 11:53 ` H.J. Lu
2016-03-17 14:16 ` Pawar, Amit
2016-03-17 14:46 ` H.J. Lu
2016-03-18 11:43 ` Pawar, Amit [this message]
2016-03-18 11:51 ` H.J. Lu
2016-03-18 12:25 ` Pawar, Amit
2016-03-18 12:34 ` H.J. Lu
2016-03-18 13:22 ` Pawar, Amit
2016-03-18 13:51 ` H.J. Lu
2016-03-18 13:55 ` Adhemerval Zanella
2016-03-18 14:43 ` H.J. Lu
2016-03-18 14:45 ` H.J. Lu
2016-03-18 15:19 ` Pawar, Amit
2016-03-18 15:24 ` H.J. Lu
2016-03-22 11:08 ` Pawar, Amit
2016-03-22 14:50 ` H.J. Lu
2016-03-22 14:57 ` Pawar, Amit
2016-03-22 15:03 ` H.J. Lu
2016-03-23 10:12 ` Pawar, Amit
2016-03-23 17:59 ` H.J. Lu
2016-03-28 7:43 ` Pawar, Amit
2016-03-28 12:12 ` H.J. Lu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=SN1PR12MB07332500CE527AA6EAC1C360978C0@SN1PR12MB0733.namprd12.prod.outlook.com \
--to=amit.pawar@amd.com \
--cc=hjl.tools@gmail.com \
--cc=libc-alpha@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).