[RFC PATCH 3/3] RISC-V: Implement CPU yielding for busy loops with Zihintpause/Zawrs

["Christoph Müllner" <christoph.muellner@vrull.eu>]

The macro atomic_spin_nop can be used to implement arch-specific
CPU yielding that is used in busy loops (e.g. in pthread_spin_lock).
This patch introduces an ifunc-based implementation for RISC-V,
that uses Zihintpause's PAUSE instruction for that matter (as PAUSE
is a HINT instruction there is not dependency to Zihintpause at
runtime).  Further, we test for Zawrs via hwprobe() and if found
we use WRS.STO instead of PAUSE.

Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
---
 sysdeps/riscv/multiarch/cpu-relax_generic.S   | 31 +++++++++++++++
 sysdeps/riscv/multiarch/cpu-relax_zawrs.S     | 28 +++++++++++++
 .../unix/sysv/linux/riscv/atomic-machine.h    |  3 ++
 .../unix/sysv/linux/riscv/multiarch/Makefile  |  8 ++++
 .../sysv/linux/riscv/multiarch/cpu-relax.c    | 39 +++++++++++++++++++
 .../linux/riscv/multiarch/ifunc-impl-list.c   | 32 +++++++++++++--
 6 files changed, 137 insertions(+), 4 deletions(-)
 create mode 100644 sysdeps/riscv/multiarch/cpu-relax_generic.S
 create mode 100644 sysdeps/riscv/multiarch/cpu-relax_zawrs.S
 create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c

diff --git a/sysdeps/riscv/multiarch/cpu-relax_generic.S b/sysdeps/riscv/multiarch/cpu-relax_generic.S
new file mode 100644
index 0000000000..d3ccfdce84
--- /dev/null
+++ b/sysdeps/riscv/multiarch/cpu-relax_generic.S
@@ -0,0 +1,31 @@
+/* CPU strand yielding for busy loops.  RISC-V version.
+   Copyright (C) 2024 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library.  If not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+#include <sys/asm.h>
+
+.option push
+.option arch, +zihintpause
+ENTRY (__cpu_relax_generic)
+	/* While we can use the `pause` instruction without
+	   the need of Zihintpause (because it is a HINT instruction),
+	   we still have to enable Zihintpause for the assembler.  */
+	pause
+	ret
+END (__cpu_relax_generic)
+.option pop
diff --git a/sysdeps/riscv/multiarch/cpu-relax_zawrs.S b/sysdeps/riscv/multiarch/cpu-relax_zawrs.S
new file mode 100644
index 0000000000..6d27b354df
--- /dev/null
+++ b/sysdeps/riscv/multiarch/cpu-relax_zawrs.S
@@ -0,0 +1,28 @@
+/* CPU strand yielding for busy loops.  RISC-V version with Zawrs.
+   Copyright (C) 2024 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library.  If not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+#include <sys/asm.h>
+
+.option push
+.option arch, +zawrs
+ENTRY (__cpu_relax_zawrs)
+	wrs.sto
+	ret
+END (__cpu_relax_zawrs)
+.option pop
diff --git a/sysdeps/unix/sysv/linux/riscv/atomic-machine.h b/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
index c1c9d949a0..02b9b7a421 100644
--- a/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
+++ b/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
@@ -178,4 +178,7 @@
 # error "ISAs that do not subsume the A extension are not supported"
 #endif /* !__riscv_atomic */
 
+extern void __cpu_relax (void);
+#define atomic_spin_nop() __cpu_relax()
+
 #endif /* bits/atomic.h */
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
index fcef5659d4..0cdf37a38b 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
@@ -1,3 +1,11 @@
+# nscd uses atomic_spin_nop which in turn requires cpu_relax
+ifeq ($(subdir),nscd)
+sysdep_routines += \
+  cpu-relax \
+  cpu-relax_generic \
+  cpu-relax_zawrs
+endif
+
 ifeq ($(subdir),string)
 sysdep_routines += \
   memcpy \
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c b/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c
new file mode 100644
index 0000000000..5aeb120e21
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c
@@ -0,0 +1,39 @@
+/* Multiple versions of cpu-relax.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2024 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+# include <ifunc-init.h>
+# include <riscv-ifunc.h>
+# include <sys/hwprobe.h>
+
+void __cpu_relax (void);
+extern __typeof (__cpu_relax) __cpu_relax_generic attribute_hidden;
+extern __typeof (__cpu_relax) __cpu_relax_zawrs attribute_hidden;
+
+static inline __typeof (__cpu_relax) *
+select_cpu_relax_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
+{
+  unsigned long long int v;
+  if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
+      && (v & RISCV_HWPROBE_EXT_ZAWRS))
+    return __cpu_relax_zawrs;
+
+  return __cpu_relax_generic;
+}
+
+riscv_libc_ifunc (__cpu_relax, select_cpu_relax_ifunc);
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
index 9f806d7a9e..9c7a8c2e1f 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
@@ -20,24 +20,48 @@
 #include <string.h>
 #include <sys/hwprobe.h>
 
+#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
+
+void __cpu_relax (void);
+
 size_t
 __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			size_t max)
 {
   size_t i = max;
+  struct riscv_hwprobe pairs[] = {
+    { .key = RISCV_HWPROBE_KEY_IMA_EXT_0 },
+    { .key = RISCV_HWPROBE_KEY_CPUPERF_0 },
+  };
 
   bool fast_unaligned = false;
+  bool has_zawrs = false;
+
+  if (__riscv_hwprobe (pairs, ARRAY_SIZE (pairs), 0, NULL, 0) == 0)
+    {
+      struct riscv_hwprobe *pair;
 
-  struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 };
-  if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0
-      && (pair.value & RISCV_HWPROBE_MISALIGNED_MASK)
+      /* RISCV_HWPROBE_KEY_IMA_EXT_0  */
+      pair = &pairs[0];
+      if (pair->value & RISCV_HWPROBE_EXT_ZAWRS)
+	has_zawrs = true;
+
+      /* RISCV_HWPROBE_KEY_CPUPERF_0  */
+      pair = &pairs[1];
+      if ((pair->value & RISCV_HWPROBE_MISALIGNED_MASK)
           == RISCV_HWPROBE_MISALIGNED_FAST)
-    fast_unaligned = true;
+	fast_unaligned = true;
+    }
 
   IFUNC_IMPL (i, name, memcpy,
 	      IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
 			      __memcpy_noalignment)
 	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic))
 
+  IFUNC_IMPL (i, name, __cpu_relax,
+	      IFUNC_IMPL_ADD (array, i, __cpu_relax, has_zawrs,
+			      __cpu_relax_zawrs)
+	      IFUNC_IMPL_ADD (array, i, __cpu_relax, 1, __cpu_relax_generic))
+
   return 0;
 }
-- 
2.44.0