[2.27 COMMITTED][AArch64] Backport memcmp improvements

[2.27 COMMITTED][AArch64] Backport memcmp improvements

[Wilco Dijkstra]

commit 062139f233a9ef94a86b91850b942d5fa991ecbe
Author: Siddhesh Poyarekar <siddhesh@sourceware.org>
Date:   Tue Mar 6 19:22:39 2018 +0530

    aarch64: Optimized memcmp for medium to large sizes
    
    This improved memcmp provides a fast path for compares up to 16 bytes
    and then compares 16 bytes at a time, thus optimizing loads from both
    sources.  The glibc memcmp microbenchmark retains performance (with an
    error of ~1ns) for smaller compare sizes and reduces up to 31% of
    execution time for compares up to 4K on the APM Mustang.  On Qualcomm
    Falkor this improves to almost 48%, i.e. it is almost 2x improvement
    for sizes of 2K and above.
    
        * sysdeps/aarch64/memcmp.S: Widen comparison to 16 bytes at a
        time.
    
    (cherry picked from commit 30a81dae5b752f8aa5f96e7f7c341ec57cba3585)

commit f3e2add2130797967287ee55eecacd570e456d2a
Author: Siddhesh Poyarekar <siddhesh@sourceware.org>
Date:   Fri Feb 2 10:15:20 2018 +0530

    aarch64: Use the L() macro for labels in memcmp
    
    The L() macro makes the assembly a bit more readable.
    
        * sysdeps/aarch64/memcmp.S: Use L() macro for labels.
    
    (cherry picked from commit 84c94d2fd90d84ae7e67657ee8e22c2d1b796f63)

diff --git a/ChangeLog b/ChangeLog
index cb36feb..0374576 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,10 @@
 2019-09-06  Siddhesh Poyarekar  <siddhesh@sourceware.org>
 
+        * sysdeps/aarch64/memcmp.S: Widen comparison to 16 bytes at a
+        time.
+
+2019-09-06  Siddhesh Poyarekar  <siddhesh@sourceware.org>
+
        * sysdeps/aarch64/memcmp.S: Use L() macro for labels.
 
 2019-07-15  Adhemerval Zanella  <adhemerval.zanella@linaro.org>
diff --git a/sysdeps/aarch64/memcmp.S b/sysdeps/aarch64/memcmp.S
index ecd1206..8325d04 100644
--- a/sysdeps/aarch64/memcmp.S
+++ b/sysdeps/aarch64/memcmp.S
@@ -34,9 +34,12 @@
 /* Internal variables.  */
 #define data1          x3
 #define data1w         w3
-#define data2          x4
-#define data2w         w4
-#define tmp1           x5
+#define data1h         x4
+#define data2          x5
+#define data2w         w5
+#define data2h         x6
+#define tmp1           x7
+#define tmp2           x8
 
 ENTRY_ALIGN (memcmp, 6)
        DELOUSE (0)
@@ -46,39 +49,70 @@ ENTRY_ALIGN (memcmp, 6)
        subs    limit, limit, 8
        b.lo    L(less8)
 
-       /* Limit >= 8, so check first 8 bytes using unaligned loads.  */
        ldr     data1, [src1], 8
        ldr     data2, [src2], 8
-       and     tmp1, src1, 7
-       add     limit, limit, tmp1
+       cmp     data1, data2
+       b.ne    L(return)
+
+       subs    limit, limit, 8
+       b.gt    L(more16)
+
+       ldr     data1, [src1, limit]
+       ldr     data2, [src2, limit]
+       b       L(return)
+
+L(more16):
+       ldr     data1, [src1], 8
+       ldr     data2, [src2], 8
        cmp     data1, data2
        bne     L(return)
 
+       /* Jump directly to comparing the last 16 bytes for 32 byte (or less)
+          strings.  */
+       subs    limit, limit, 16
+       b.ls    L(last_bytes)
+
+       /* We overlap loads between 0-32 bytes at either side of SRC1 when we
+          try to align, so limit it only to strings larger than 128 bytes.  */
+       cmp     limit, 96
+       b.ls    L(loop8)
+
        /* Align src1 and adjust src2 with bytes not yet done.  */
+       and     tmp1, src1, 15
+       add     limit, limit, tmp1
        sub     src1, src1, tmp1
        sub     src2, src2, tmp1
 
-       subs    limit, limit, 8
-       b.ls    L(last_bytes)
-
-       /* Loop performing 8 bytes per iteration using aligned src1.
-          Limit is pre-decremented by 8 and must be larger than zero.
-          Exit if <= 8 bytes left to do or if the data is not equal.  */
+       /* Loop performing 16 bytes per iteration using aligned src1.
+          Limit is pre-decremented by 16 and must be larger than zero.
+          Exit if <= 16 bytes left to do or if the data is not equal.  */
        .p2align 4
-L(loop8):
-       ldr     data1, [src1], 8
-       ldr     data2, [src2], 8
-       subs    limit, limit, 8
-       ccmp    data1, data2, 0, hi  /* NZCV = 0b0000.  */
-       b.eq    L(loop8)
+L(loop16):
+       ldp     data1, data1h, [src1], 16
+       ldp     data2, data2h, [src2], 16
+       subs    limit, limit, 16
+       ccmp    data1, data2, 0, hi
+       ccmp    data1h, data2h, 0, eq
+       b.eq    L(loop16)
 
        cmp     data1, data2
        bne     L(return)
+       mov     data1, data1h
+       mov     data2, data2h
+       cmp     data1, data2
+       bne     L(return)
 
-       /* Compare last 1-8 bytes using unaligned access.  */
+       /* Compare last 1-16 bytes using unaligned access.  */
 L(last_bytes):
-       ldr     data1, [src1, limit]
-       ldr     data2, [src2, limit]
+       add     src1, src1, limit
+       add     src2, src2, limit
+       ldp     data1, data1h, [src1]
+       ldp     data2, data2h, [src2]
+       cmp     data1, data2
+       bne     L(return)
+       mov     data1, data1h
+       mov     data2, data2h
+       cmp     data1, data2
 
        /* Compare data bytes and set return value to 0, -1 or 1.  */
 L(return):