From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1895) id D9EAB385841D; Tue, 17 Jan 2023 16:36:01 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org D9EAB385841D DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1673973361; bh=HwA8WZFxxFUxNDdKOXKd9RwucBAdlw4oTdas31uq4ms=; h=From:To:Subject:Date:From; b=dC8rFzaciU2Yn/nJuLj+DhbTucX5rfaBGMln7pQabCCUwGvhbF/8aa0f0tb2bDEkI XG71Xkxrcu7Hq9YDHRTKvGhodptYN2flYGBnYJ2ORS/OtXIwKe058YxraqOZJKHS7y 4xNKCdDguvHPhtSPOo2WimxCc0AcR3xiw7LOK3nM= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Wilco Dijkstra To: glibc-cvs@sourceware.org Subject: [glibc] AArch64: Optimize strcpy X-Act-Checkin: glibc X-Git-Author: Wilco Dijkstra X-Git-Refname: refs/heads/master X-Git-Oldrev: 09ebd8549b2ce5a3a6c0c7c5f3e62227faf50a99 X-Git-Newrev: 349e48c01e85bd96006860084e76d322e6ca02f1 Message-Id: <20230117163601.D9EAB385841D@sourceware.org> Date: Tue, 17 Jan 2023 16:36:01 +0000 (GMT) List-Id: https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=349e48c01e85bd96006860084e76d322e6ca02f1 commit 349e48c01e85bd96006860084e76d322e6ca02f1 Author: Wilco Dijkstra Date: Wed Jan 11 13:52:39 2023 +0000 AArch64: Optimize strcpy Unroll the main loop. Large strings are around 20% faster on modern CPUs. Reviewed-by: Szabolcs Nagy Diff: --- sysdeps/aarch64/strcpy.S | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/sysdeps/aarch64/strcpy.S b/sysdeps/aarch64/strcpy.S index bc14dd9f79..8b045a92c2 100644 --- a/sysdeps/aarch64/strcpy.S +++ b/sysdeps/aarch64/strcpy.S @@ -30,7 +30,6 @@ * MTE compatible. */ -/* Arguments and results. */ #define dstin x0 #define srcin x1 #define result x0 @@ -76,14 +75,14 @@ ENTRY (STRCPY) ld1 {vdata.16b}, [src] cmeq vhas_nul.16b, vdata.16b, 0 lsl shift, srcin, 2 - shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ + shrn vend.8b, vhas_nul.8h, 4 fmov synd, dend lsr synd, synd, shift cbnz synd, L(tail) ldr dataq, [src, 16]! cmeq vhas_nul.16b, vdata.16b, 0 - shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ + shrn vend.8b, vhas_nul.8h, 4 fmov synd, dend cbz synd, L(start_loop) @@ -102,13 +101,10 @@ ENTRY (STRCPY) IFSTPCPY (add result, dstin, len) ret - .p2align 4,,8 L(tail): rbit synd, synd clz len, synd lsr len, len, 2 - - .p2align 4 L(less16): tbz len, 3, L(less8) sub tmp, len, 7 @@ -141,31 +137,37 @@ L(zerobyte): .p2align 4 L(start_loop): - sub len, src, srcin + sub tmp, srcin, dstin ldr dataq2, [srcin] - add dst, dstin, len + sub dst, src, tmp str dataq2, [dstin] - - .p2align 5 L(loop): - str dataq, [dst], 16 - ldr dataq, [src, 16]! + str dataq, [dst], 32 + ldr dataq, [src, 16] + cmeq vhas_nul.16b, vdata.16b, 0 + umaxp vend.16b, vhas_nul.16b, vhas_nul.16b + fmov synd, dend + cbnz synd, L(loopend) + str dataq, [dst, -16] + ldr dataq, [src, 32]! cmeq vhas_nul.16b, vdata.16b, 0 umaxp vend.16b, vhas_nul.16b, vhas_nul.16b fmov synd, dend cbz synd, L(loop) - + add dst, dst, 16 +L(loopend): shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ fmov synd, dend + sub dst, dst, 31 #ifndef __AARCH64EB__ rbit synd, synd #endif clz len, synd lsr len, len, 2 - sub tmp, len, 15 - ldr dataq, [src, tmp] - str dataq, [dst, tmp] - IFSTPCPY (add result, dst, len) + add dst, dst, len + ldr dataq, [dst, tmp] + str dataq, [dst] + IFSTPCPY (add result, dst, 15) ret END (STRCPY)