From 77b89b605ed498e6ab32132e97b0efb8088fd4a6 Mon Sep 17 00:00:00 2001
From: Amit Pawar <Amit.Pawar@amd.com>
Date: Wed, 23 Mar 2016 15:35:27 +0530
Subject: [PATCH] x86_64 Fix memcpy IFUNC selection order for Excavator CPU.

Performance of memcpy implementation based on Fast_Copy_Backward is better
compare to currently selected Fast_Unaligned_Load based implementation on
Excavator cpu. New feature bit is required to fix this issue in memcpy IFUNC
function without affecting other targets. So defining two new
bit_arch_Fast_Unaligned_Copy and index_arch_Fast_Unaligned_Copy feature bits
macros and selection order of this functions is updated.

	[BZ #19583]
        * sysdeps/x86/cpu-features.h (bit_arch_Fast_Unaligned_Copy):
        New.
        (index_arch_Fast_Unaligned_Copy) Likewise.
        * sysdeps/x86/cpu-features.c
        (init_cpu_features, Fast_Copy_Backward): Set it for Excavator core.
        (init_cpu_features, Fast_Unaligned_Copy): Set it for Excavator core.
        * sysdeps/x86_64/multiarch/memcpy.S
        (__new_memcpy, Fast_Unaligned_Copy): Add check for
        Fast_Unaligned_Copy bit and select it on Excavator core.
---
 sysdeps/x86/cpu-features.c        | 11 ++++++++++-
 sysdeps/x86/cpu-features.h        |  3 +++
 sysdeps/x86_64/multiarch/memcpy.S | 12 +++++++-----
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index c8f81ef..7701548 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -220,10 +220,19 @@ init_cpu_features (struct cpu_features *cpu_features)
 
       if (family == 0x15)
 	{
+#if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
+# error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
+#endif
+#if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+# error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+#endif
 	  /* "Excavator"   */
 	  if (model >= 0x60 && model <= 0x7f)
 	    cpu_features->feature[index_arch_Fast_Unaligned_Load]
-	      |= bit_arch_Fast_Unaligned_Load;
+	      |= (bit_arch_Fast_Unaligned_Load
+		  | bit_arch_Fast_Unaligned_Copy
+		  | bit_arch_Fast_Copy_Backward);
+
 	}
     }
   else
diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h
index e06eb7e..bfe1f4c 100644
--- a/sysdeps/x86/cpu-features.h
+++ b/sysdeps/x86/cpu-features.h
@@ -35,6 +35,7 @@
 #define bit_arch_I686				(1 << 15)
 #define bit_arch_Prefer_MAP_32BIT_EXEC		(1 << 16)
 #define bit_arch_Prefer_No_VZEROUPPER		(1 << 17)
+#define bit_arch_Fast_Unaligned_Copy		(1 << 18)
 
 /* CPUID Feature flags.  */
 
@@ -101,6 +102,7 @@
 # define index_arch_I686		FEATURE_INDEX_1*FEATURE_SIZE
 # define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1*FEATURE_SIZE
 # define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1*FEATURE_SIZE
+# define index_arch_Fast_Unaligned_Copy	FEATURE_INDEX_1*FEATURE_SIZE
 
 
 # if defined (_LIBC) && !IS_IN (nonlib)
@@ -265,6 +267,7 @@ extern const struct cpu_features *__get_cpu_features (void)
 # define index_arch_I686		FEATURE_INDEX_1
 # define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1
 # define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1
+# define index_arch_Fast_Unaligned_Copy	FEATURE_INDEX_1
 
 #endif	/* !__ASSEMBLER__ */
 
diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S
index 8882590..9b37626 100644
--- a/sysdeps/x86_64/multiarch/memcpy.S
+++ b/sysdeps/x86_64/multiarch/memcpy.S
@@ -40,18 +40,20 @@ ENTRY(__new_memcpy)
 #endif
 1:	lea	__memcpy_avx_unaligned(%rip), %RAX_LP
 	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
+	jnz	3f
+	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
 	jnz	2f
 	lea	__memcpy_sse2_unaligned(%rip), %RAX_LP
 	HAS_ARCH_FEATURE (Fast_Unaligned_Load)
-	jnz	2f
-	lea	__memcpy_sse2(%rip), %RAX_LP
+	jnz	3f
+2:	lea	__memcpy_sse2(%rip), %RAX_LP
 	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
+	jz	3f
 	lea    __memcpy_ssse3_back(%rip), %RAX_LP
 	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
+	jnz	3f
 	lea	__memcpy_ssse3(%rip), %RAX_LP
-2:	ret
+3:	ret
 END(__new_memcpy)
 
 # undef ENTRY
-- 
2.1.4

