From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1895) id C1FC8384646C; Wed, 10 Apr 2024 15:07:16 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org C1FC8384646C DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1712761636; bh=eZ4AAA8XH3wMLs4TL9k3tpn1H6vfGPcL2N3p2hIZ8zM=; h=From:To:Subject:Date:From; b=cMO0AjYWGzUghXIwYVvjGwRmf3LgvLFLHRMciiS02d77JWRBqSUXuMe7gHBteApkP ieg6P3us26T3dqnpXmmsaygUPT3+2AZBRkCm/H88eYrb70TRONIdSX0XoMO2J5t0cm uP4XfJL9PkG7Sdv52FaOL7rarytGYojtwArHW0Zg= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Wilco Dijkstra To: glibc-cvs@sourceware.org Subject: [glibc/release/2.35/master] AArch64: Optimize strlen X-Act-Checkin: glibc X-Git-Author: Wilco Dijkstra X-Git-Refname: refs/heads/release/2.35/master X-Git-Oldrev: 6f2ca6aab9c60f8f74aba8bf8585fb4932863184 X-Git-Newrev: 3a1557efef9d3aa334122f44c495dd0c5d78b787 Message-Id: <20240410150716.C1FC8384646C@sourceware.org> Date: Wed, 10 Apr 2024 15:07:16 +0000 (GMT) List-Id: https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=3a1557efef9d3aa334122f44c495dd0c5d78b787 commit 3a1557efef9d3aa334122f44c495dd0c5d78b787 Author: Wilco Dijkstra Date: Wed Jan 11 13:52:53 2023 +0000 AArch64: Optimize strlen Optimize strlen by unrolling the main loop. Large strings are 64% faster on modern CPUs. Reviewed-by: Szabolcs Nagy (cherry picked from commit 03c8ce5000198947a4dd7b2c14e5131738fda62b) Diff: --- sysdeps/aarch64/strlen.S | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sysdeps/aarch64/strlen.S b/sysdeps/aarch64/strlen.S index 3a5d088407..10b9ec0769 100644 --- a/sysdeps/aarch64/strlen.S +++ b/sysdeps/aarch64/strlen.S @@ -43,12 +43,9 @@ #define dend d2 /* Core algorithm: - - For each 16-byte chunk we calculate a 64-bit nibble mask value with four bits - per byte. We take 4 bits of every comparison byte with shift right and narrow - by 4 instruction. Since the bits in the nibble mask reflect the order in - which things occur in the original string, counting trailing zeros identifies - exactly which byte matched. */ + Process the string in 16-byte aligned chunks. Compute a 64-bit mask with + four bits per byte using the shrn instruction. A count trailing zeros then + identifies the first zero byte. */ ENTRY (STRLEN) PTR_ARG (0) @@ -68,18 +65,25 @@ ENTRY (STRLEN) .p2align 5 L(loop): - ldr data, [src, 16]! + ldr data, [src, 16] + cmeq vhas_nul.16b, vdata.16b, 0 + umaxp vend.16b, vhas_nul.16b, vhas_nul.16b + fmov synd, dend + cbnz synd, L(loop_end) + ldr data, [src, 32]! cmeq vhas_nul.16b, vdata.16b, 0 umaxp vend.16b, vhas_nul.16b, vhas_nul.16b fmov synd, dend cbz synd, L(loop) - + sub src, src, 16 +L(loop_end): shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ sub result, src, srcin fmov synd, dend #ifndef __AARCH64EB__ rbit synd, synd #endif + add result, result, 16 clz tmp, synd add result, result, tmp, lsr 2 ret