* [PATCH v3 1/3] riscv: Add Linux hwprobe syscall support
2023-04-07 23:07 [PATCH v3 0/3] RISC-V: ifunced memcpy using new kernel hwprobe interface Evan Green
@ 2023-04-07 23:07 ` Evan Green
2023-06-28 0:20 ` Palmer Dabbelt
2023-06-28 5:13 ` Florian Weimer
2023-04-07 23:07 ` [PATCH v3 2/3] riscv: Add hwprobe vdso call support Evan Green
2023-04-07 23:07 ` [PATCH v3 3/3] riscv: Add and use alignment-ignorant memcpy Evan Green
2 siblings, 2 replies; 6+ messages in thread
From: Evan Green @ 2023-04-07 23:07 UTC (permalink / raw)
To: libc-alpha; +Cc: vineetg, palmer, slewis, Evan Green
Add awareness and a thin wrapper function around a new Linux system call
that allows callers to get architecture and microarchitecture
information about the CPUs from the kernel. This can be used to
do things like dynamically choose a memcpy implementation.
Signed-off-by: Evan Green <evan@rivosinc.com>
Reviewed-by: Palmer Dabbelt <palmer@rivosinc.com>
---
Changes in v3:
- Update argument types to match v4 kernel interface
Changes in v2:
- hwprobe.h: Use __has_include and duplicate Linux content to make
compilation work when Linux headers are absent (Adhemerval)
- hwprobe.h: Put declaration under __USE_GNU (Adhemerval)
- Use INLINE_SYSCALL_CALL (Adhemerval)
- Update versions
- Update UNALIGNED_MASK to match kernel v3 series.
sysdeps/unix/sysv/linux/riscv/Makefile | 4 +-
sysdeps/unix/sysv/linux/riscv/Versions | 3 +
sysdeps/unix/sysv/linux/riscv/hwprobe.c | 30 ++++++++
.../unix/sysv/linux/riscv/rv32/arch-syscall.h | 1 +
.../unix/sysv/linux/riscv/rv32/libc.abilist | 1 +
.../unix/sysv/linux/riscv/rv64/arch-syscall.h | 1 +
.../unix/sysv/linux/riscv/rv64/libc.abilist | 1 +
sysdeps/unix/sysv/linux/riscv/sys/hwprobe.h | 68 +++++++++++++++++++
sysdeps/unix/sysv/linux/syscall-names.list | 1 +
9 files changed, 108 insertions(+), 2 deletions(-)
create mode 100644 sysdeps/unix/sysv/linux/riscv/hwprobe.c
create mode 100644 sysdeps/unix/sysv/linux/riscv/sys/hwprobe.h
diff --git a/sysdeps/unix/sysv/linux/riscv/Makefile b/sysdeps/unix/sysv/linux/riscv/Makefile
index 4b6eacb32f..45cc29e40d 100644
--- a/sysdeps/unix/sysv/linux/riscv/Makefile
+++ b/sysdeps/unix/sysv/linux/riscv/Makefile
@@ -1,6 +1,6 @@
ifeq ($(subdir),misc)
-sysdep_headers += sys/cachectl.h
-sysdep_routines += flush-icache
+sysdep_headers += sys/cachectl.h sys/hwprobe.h
+sysdep_routines += flush-icache hwprobe
endif
ifeq ($(subdir),stdlib)
diff --git a/sysdeps/unix/sysv/linux/riscv/Versions b/sysdeps/unix/sysv/linux/riscv/Versions
index 5625d2a0b8..8717b62a4a 100644
--- a/sysdeps/unix/sysv/linux/riscv/Versions
+++ b/sysdeps/unix/sysv/linux/riscv/Versions
@@ -8,4 +8,7 @@ libc {
GLIBC_2.27 {
__riscv_flush_icache;
}
+ GLIBC_2.39 {
+ __riscv_hwprobe;
+ }
}
diff --git a/sysdeps/unix/sysv/linux/riscv/hwprobe.c b/sysdeps/unix/sysv/linux/riscv/hwprobe.c
new file mode 100644
index 0000000000..e9023e2fba
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/riscv/hwprobe.c
@@ -0,0 +1,30 @@
+/* RISC-V hardware feature probing support on Linux
+ Copyright (C) 2023 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation; either version 2.1 of the
+ License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sys/syscall.h>
+#include <sys/hwprobe.h>
+#include <sysdep.h>
+
+int
+__riscv_hwprobe (struct riscv_hwprobe *pairs, size_t pair_count,
+ size_t cpu_count, unsigned long *cpus, unsigned int flags)
+{
+ return INLINE_SYSCALL_CALL (riscv_hwprobe, pairs, pair_count,
+ cpu_count, cpus, flags);
+}
diff --git a/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h b/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h
index 202520ee25..2416e041c8 100644
--- a/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h
+++ b/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h
@@ -198,6 +198,7 @@
#define __NR_request_key 218
#define __NR_restart_syscall 128
#define __NR_riscv_flush_icache 259
+#define __NR_riscv_hwprobe 258
#define __NR_rseq 293
#define __NR_rt_sigaction 134
#define __NR_rt_sigpending 136
diff --git a/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist b/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist
index b716f5c763..1548587eb1 100644
--- a/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist
@@ -2428,3 +2428,4 @@ GLIBC_2.38 __isoc23_wcstoull F
GLIBC_2.38 __isoc23_wcstoull_l F
GLIBC_2.38 __isoc23_wcstoumax F
GLIBC_2.38 __isoc23_wscanf F
+GLIBC_2.39 __riscv_hwprobe F
diff --git a/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h b/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h
index 4e65f337d4..a32bc82f60 100644
--- a/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h
+++ b/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h
@@ -205,6 +205,7 @@
#define __NR_request_key 218
#define __NR_restart_syscall 128
#define __NR_riscv_flush_icache 259
+#define __NR_riscv_hwprobe 258
#define __NR_rseq 293
#define __NR_rt_sigaction 134
#define __NR_rt_sigpending 136
diff --git a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
index 774e777b65..49c0577b61 100644
--- a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
@@ -2628,3 +2628,4 @@ GLIBC_2.38 __isoc23_wcstoull F
GLIBC_2.38 __isoc23_wcstoull_l F
GLIBC_2.38 __isoc23_wcstoumax F
GLIBC_2.38 __isoc23_wscanf F
+GLIBC_2.39 __riscv_hwprobe F
diff --git a/sysdeps/unix/sysv/linux/riscv/sys/hwprobe.h b/sysdeps/unix/sysv/linux/riscv/sys/hwprobe.h
new file mode 100644
index 0000000000..49e27ee855
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/riscv/sys/hwprobe.h
@@ -0,0 +1,68 @@
+/* RISC-V architecture probe interface
+ Copyright (C) 2023 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_HWPROBE_H
+#define _SYS_HWPROBE_H 1
+
+#include <features.h>
+#include <stddef.h>
+#ifdef __has_include
+# if __has_include (<asm/hwprobe.h>)
+# include <asm/hwprobe.h>
+# endif
+#endif
+
+/*
+ * Define a (probably stale) version of the interface if the Linux headers
+ * aren't present.
+ */
+#ifndef RISCV_HWPROBE_KEY_MVENDORID
+struct riscv_hwprobe {
+ signed long long key;
+ unsigned long long value;
+};
+
+#define RISCV_HWPROBE_KEY_MVENDORID 0
+#define RISCV_HWPROBE_KEY_MARCHID 1
+#define RISCV_HWPROBE_KEY_MIMPID 2
+#define RISCV_HWPROBE_KEY_BASE_BEHAVIOR 3
+#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA (1 << 0)
+#define RISCV_HWPROBE_KEY_IMA_EXT_0 4
+#define RISCV_HWPROBE_IMA_FD (1 << 0)
+#define RISCV_HWPROBE_IMA_C (1 << 1)
+#define RISCV_HWPROBE_KEY_CPUPERF_0 5
+#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
+#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
+#define RISCV_HWPROBE_MISALIGNED_SLOW (2 << 0)
+#define RISCV_HWPROBE_MISALIGNED_FAST (3 << 0)
+#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0)
+
+#endif // RISCV_HWPROBE_KEY_MVENDORID
+
+__BEGIN_DECLS
+
+#ifdef __USE_GNU
+int
+__riscv_hwprobe (struct riscv_hwprobe *pairs, size_t pair_count,
+ size_t cpu_count, unsigned long *cpus, unsigned int flags);
+#endif
+
+__END_DECLS
+
+#endif /* sys/hwprobe.h */
diff --git a/sysdeps/unix/sysv/linux/syscall-names.list b/sysdeps/unix/sysv/linux/syscall-names.list
index 5d27b5279c..bbc0e01d3a 100644
--- a/sysdeps/unix/sysv/linux/syscall-names.list
+++ b/sysdeps/unix/sysv/linux/syscall-names.list
@@ -477,6 +477,7 @@ renameat2
request_key
restart_syscall
riscv_flush_icache
+riscv_hwprobe
rmdir
rseq
rt_sigaction
--
2.25.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v3 2/3] riscv: Add hwprobe vdso call support
2023-04-07 23:07 [PATCH v3 0/3] RISC-V: ifunced memcpy using new kernel hwprobe interface Evan Green
2023-04-07 23:07 ` [PATCH v3 1/3] riscv: Add Linux hwprobe syscall support Evan Green
@ 2023-04-07 23:07 ` Evan Green
2023-04-07 23:07 ` [PATCH v3 3/3] riscv: Add and use alignment-ignorant memcpy Evan Green
2 siblings, 0 replies; 6+ messages in thread
From: Evan Green @ 2023-04-07 23:07 UTC (permalink / raw)
To: libc-alpha; +Cc: vineetg, palmer, slewis, Evan Green
The new riscv_hwprobe syscall also comes with a vDSO for faster answers
to your most common questions. Call in today to speak with a kernel
representative near you!
Signed-off-by: Evan Green <evan@rivosinc.com>
Reviewed-by: Palmer Dabbelt <palmer@rivosinc.com>
---
Changes in v3:
- Add the "return" to the vsyscall
- Fix up vdso arg types to match kernel v4 version
- Remove ifdef around INLINE_VSYSCALL (Adhemerval)
Changes in v2:
- Add vDSO interface
sysdeps/unix/sysv/linux/dl-vdso-setup.c | 10 ++++++++++
sysdeps/unix/sysv/linux/dl-vdso-setup.h | 3 +++
sysdeps/unix/sysv/linux/riscv/hwprobe.c | 5 +++--
sysdeps/unix/sysv/linux/riscv/sysdep.h | 1 +
4 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/sysdeps/unix/sysv/linux/dl-vdso-setup.c b/sysdeps/unix/sysv/linux/dl-vdso-setup.c
index 68fa8de641..3fc558afdd 100644
--- a/sysdeps/unix/sysv/linux/dl-vdso-setup.c
+++ b/sysdeps/unix/sysv/linux/dl-vdso-setup.c
@@ -71,6 +71,16 @@ PROCINFO_CLASS int (*_dl_vdso_clock_getres_time64) (clockid_t,
# ifdef HAVE_GET_TBFREQ
PROCINFO_CLASS uint64_t (*_dl_vdso_get_tbfreq)(void) RELRO;
# endif
+
+/* RISC-V specific ones. */
+# ifdef HAVE_RISCV_HWPROBE
+PROCINFO_CLASS int (*_dl_vdso_riscv_hwprobe)(void *,
+ size_t,
+ size_t,
+ unsigned long *,
+ unsigned int) RELRO;
+# endif
+
#endif
#undef RELRO
diff --git a/sysdeps/unix/sysv/linux/dl-vdso-setup.h b/sysdeps/unix/sysv/linux/dl-vdso-setup.h
index 867072b897..39eafd5316 100644
--- a/sysdeps/unix/sysv/linux/dl-vdso-setup.h
+++ b/sysdeps/unix/sysv/linux/dl-vdso-setup.h
@@ -47,6 +47,9 @@ setup_vdso_pointers (void)
#ifdef HAVE_GET_TBFREQ
GLRO(dl_vdso_get_tbfreq) = dl_vdso_vsym (HAVE_GET_TBFREQ);
#endif
+#ifdef HAVE_RISCV_HWPROBE
+ GLRO(dl_vdso_riscv_hwprobe) = dl_vdso_vsym (HAVE_RISCV_HWPROBE);
+#endif
}
#endif
diff --git a/sysdeps/unix/sysv/linux/riscv/hwprobe.c b/sysdeps/unix/sysv/linux/riscv/hwprobe.c
index e9023e2fba..6bf28f574b 100644
--- a/sysdeps/unix/sysv/linux/riscv/hwprobe.c
+++ b/sysdeps/unix/sysv/linux/riscv/hwprobe.c
@@ -20,11 +20,12 @@
#include <sys/syscall.h>
#include <sys/hwprobe.h>
#include <sysdep.h>
+#include <sysdep-vdso.h>
int
__riscv_hwprobe (struct riscv_hwprobe *pairs, size_t pair_count,
size_t cpu_count, unsigned long *cpus, unsigned int flags)
{
- return INLINE_SYSCALL_CALL (riscv_hwprobe, pairs, pair_count,
- cpu_count, cpus, flags);
+ /* The vDSO may be able to provide the answer without a syscall. */
+ return INLINE_VSYSCALL(riscv_hwprobe, 5, pairs, pair_count, cpu_count, cpus, flags);
}
diff --git a/sysdeps/unix/sysv/linux/riscv/sysdep.h b/sysdeps/unix/sysv/linux/riscv/sysdep.h
index 4af5fe5dbc..ba17aaaff2 100644
--- a/sysdeps/unix/sysv/linux/riscv/sysdep.h
+++ b/sysdeps/unix/sysv/linux/riscv/sysdep.h
@@ -155,6 +155,7 @@
/* List of system calls which are supported as vsyscalls (for RV32 and
RV64). */
# define HAVE_GETCPU_VSYSCALL "__vdso_getcpu"
+# define HAVE_RISCV_HWPROBE "__vdso_riscv_hwprobe"
# undef HAVE_INTERNAL_BRK_ADDR_SYMBOL
# define HAVE_INTERNAL_BRK_ADDR_SYMBOL 1
--
2.25.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v3 3/3] riscv: Add and use alignment-ignorant memcpy
2023-04-07 23:07 [PATCH v3 0/3] RISC-V: ifunced memcpy using new kernel hwprobe interface Evan Green
2023-04-07 23:07 ` [PATCH v3 1/3] riscv: Add Linux hwprobe syscall support Evan Green
2023-04-07 23:07 ` [PATCH v3 2/3] riscv: Add hwprobe vdso call support Evan Green
@ 2023-04-07 23:07 ` Evan Green
2 siblings, 0 replies; 6+ messages in thread
From: Evan Green @ 2023-04-07 23:07 UTC (permalink / raw)
To: libc-alpha; +Cc: vineetg, palmer, slewis, Evan Green
For CPU implementations that can perform unaligned accesses with little
or no performance penalty, create a memcpy implementation that does not
bother aligning buffers. It will use a block of integer registers, a
single integer register, and fall back to bytewise copy for the
remainder.
Signed-off-by: Evan Green <evan@rivosinc.com>
Reviewed-by: Palmer Dabbelt <palmer@rivosinc.com>
---
Changes in v3:
- Word align dest for large memcpy()s.
- Add tags
- Remove spurious blank line from sysdeps/riscv/memcpy.c
Changes in v2:
- Used _MASK instead of _FAST value itself.
---
sysdeps/riscv/memcopy.h | 28 ++++
sysdeps/riscv/memcpy.c | 64 +++++++++
sysdeps/riscv/memcpy_noalignment.S | 121 ++++++++++++++++++
sysdeps/unix/sysv/linux/riscv/Makefile | 4 +
.../unix/sysv/linux/riscv/memcpy-generic.c | 24 ++++
5 files changed, 241 insertions(+)
create mode 100644 sysdeps/riscv/memcopy.h
create mode 100644 sysdeps/riscv/memcpy.c
create mode 100644 sysdeps/riscv/memcpy_noalignment.S
create mode 100644 sysdeps/unix/sysv/linux/riscv/memcpy-generic.c
diff --git a/sysdeps/riscv/memcopy.h b/sysdeps/riscv/memcopy.h
new file mode 100644
index 0000000000..21f6081b5f
--- /dev/null
+++ b/sysdeps/riscv/memcopy.h
@@ -0,0 +1,28 @@
+/* memcopy.h -- definitions for memory copy functions. RISC-V version.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdeps/generic/memcopy.h>
+
+/*
+ * Redefine the generic memcpy implementation to __memcpy_generic, so
+ * the memcpy ifunc can select between generic and special versions.
+ * In rtld, don't bother with all the ifunciness.
+ */
+#if IS_IN (libc)
+#define MEMCPY __memcpy_generic
+#endif
diff --git a/sysdeps/riscv/memcpy.c b/sysdeps/riscv/memcpy.c
new file mode 100644
index 0000000000..fdb8dc3208
--- /dev/null
+++ b/sysdeps/riscv/memcpy.c
@@ -0,0 +1,64 @@
+/* Multiple versions of memcpy.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017-2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#if IS_IN (libc)
+/* Redefine memcpy so that the compiler won't complain about the type
+ mismatch with the IFUNC selector in strong_alias, below. */
+# undef memcpy
+# define memcpy __redirect_memcpy
+# include <string.h>
+#include <ifunc-init.h>
+#include <sys/hwprobe.h>
+
+#define INIT_ARCH()
+
+extern __typeof (__redirect_memcpy) __libc_memcpy;
+
+extern __typeof (__redirect_memcpy) __memcpy_generic attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_noalignment attribute_hidden;
+
+static inline __typeof (__redirect_memcpy) *
+select_memcpy_ifunc (void)
+{
+ INIT_ARCH ();
+
+ struct riscv_hwprobe pair;
+
+ pair.key = RISCV_HWPROBE_KEY_CPUPERF_0;
+ if (__riscv_hwprobe(&pair, 1, 0, NULL, 0) != 0)
+ return __memcpy_generic;
+
+ if ((pair.key > 0) &&
+ (pair.value & RISCV_HWPROBE_MISALIGNED_MASK) ==
+ RISCV_HWPROBE_MISALIGNED_FAST)
+ return __memcpy_noalignment;
+
+ return __memcpy_generic;
+}
+
+libc_ifunc (__libc_memcpy, select_memcpy_ifunc ());
+
+# undef memcpy
+strong_alias (__libc_memcpy, memcpy);
+# ifdef SHARED
+__hidden_ver1 (memcpy, __GI_memcpy, __redirect_memcpy)
+ __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memcpy);
+# endif
+
+#endif
diff --git a/sysdeps/riscv/memcpy_noalignment.S b/sysdeps/riscv/memcpy_noalignment.S
new file mode 100644
index 0000000000..80f5e09ebb
--- /dev/null
+++ b/sysdeps/riscv/memcpy_noalignment.S
@@ -0,0 +1,121 @@
+/* memcpy for RISC-V, ignoring buffer alignment
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+#include <sys/asm.h>
+
+/* void *memcpy(void *, const void *, size_t) */
+ENTRY (__memcpy_noalignment)
+ move t6, a0 /* Preserve return value */
+
+ /* Round down to the nearest "page" size */
+ andi a4, a2, ~((16*SZREG)-1)
+ beqz a4, 2f
+ add a3, a1, a4
+
+ /* Copy the first word to get dest word aligned */
+ andi a5, t6, SZREG-1
+ beqz a5, 1f
+ REG_L a6, (a1)
+ REG_S a6, (t6)
+
+ /* Align dst up to a word, move src and size as well. */
+ addi t6, t6, SZREG-1
+ andi t6, t6, ~(SZREG-1)
+ sub a5, t6, a0
+ add a1, a1, a5
+ sub a2, a2, a5
+
+ /* Recompute page count */
+ andi a4, a2, ~((16*SZREG)-1)
+ beqz a4, 2f
+
+1:
+ /* Copy "pages" (chunks of 16 registers) */
+ REG_L a4, 0(a1)
+ REG_L a5, SZREG(a1)
+ REG_L a6, 2*SZREG(a1)
+ REG_L a7, 3*SZREG(a1)
+ REG_L t0, 4*SZREG(a1)
+ REG_L t1, 5*SZREG(a1)
+ REG_L t2, 6*SZREG(a1)
+ REG_L t3, 7*SZREG(a1)
+ REG_L t4, 8*SZREG(a1)
+ REG_L t5, 9*SZREG(a1)
+ REG_S a4, 0(t6)
+ REG_S a5, SZREG(t6)
+ REG_S a6, 2*SZREG(t6)
+ REG_S a7, 3*SZREG(t6)
+ REG_S t0, 4*SZREG(t6)
+ REG_S t1, 5*SZREG(t6)
+ REG_S t2, 6*SZREG(t6)
+ REG_S t3, 7*SZREG(t6)
+ REG_S t4, 8*SZREG(t6)
+ REG_S t5, 9*SZREG(t6)
+ REG_L a4, 10*SZREG(a1)
+ REG_L a5, 11*SZREG(a1)
+ REG_L a6, 12*SZREG(a1)
+ REG_L a7, 13*SZREG(a1)
+ REG_L t0, 14*SZREG(a1)
+ REG_L t1, 15*SZREG(a1)
+ addi a1, a1, 16*SZREG
+ REG_S a4, 10*SZREG(t6)
+ REG_S a5, 11*SZREG(t6)
+ REG_S a6, 12*SZREG(t6)
+ REG_S a7, 13*SZREG(t6)
+ REG_S t0, 14*SZREG(t6)
+ REG_S t1, 15*SZREG(t6)
+ addi t6, t6, 16*SZREG
+ bltu a1, a3, 1b
+ andi a2, a2, (16*SZREG)-1 /* Update count */
+
+2:
+ /* Remainder is smaller than a page, compute native word count */
+ beqz a2, 6f
+ andi a5, a2, ~(SZREG-1)
+ andi a2, a2, (SZREG-1)
+ add a3, a1, a5
+ /* Jump directly to byte copy if no words. */
+ beqz a5, 4f
+
+3:
+ /* Use single native register copy */
+ REG_L a4, 0(a1)
+ addi a1, a1, SZREG
+ REG_S a4, 0(t6)
+ addi t6, t6, SZREG
+ bltu a1, a3, 3b
+
+ /* Jump directly out if no more bytes */
+ beqz a2, 6f
+
+4:
+ /* Copy the last few individual bytes */
+ add a3, a1, a2
+5:
+ lb a4, 0(a1)
+ addi a1, a1, 1
+ sb a4, 0(t6)
+ addi t6, t6, 1
+ bltu a1, a3, 5b
+6:
+ ret
+
+END (__memcpy_noalignment)
+
+hidden_def (__memcpy_noalignment)
diff --git a/sysdeps/unix/sysv/linux/riscv/Makefile b/sysdeps/unix/sysv/linux/riscv/Makefile
index 45cc29e40d..aa9ea443d6 100644
--- a/sysdeps/unix/sysv/linux/riscv/Makefile
+++ b/sysdeps/unix/sysv/linux/riscv/Makefile
@@ -7,6 +7,10 @@ ifeq ($(subdir),stdlib)
gen-as-const-headers += ucontext_i.sym
endif
+ifeq ($(subdir),string)
+sysdep_routines += memcpy memcpy-generic memcpy_noalignment
+endif
+
abi-variants := ilp32 ilp32d lp64 lp64d
ifeq (,$(filter $(default-abi),$(abi-variants)))
diff --git a/sysdeps/unix/sysv/linux/riscv/memcpy-generic.c b/sysdeps/unix/sysv/linux/riscv/memcpy-generic.c
new file mode 100644
index 0000000000..0abe03f7f5
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/riscv/memcpy-generic.c
@@ -0,0 +1,24 @@
+/* Re-include the default memcpy implementation.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <string.h>
+
+extern __typeof (memcpy) __memcpy_generic;
+hidden_proto(__memcpy_generic)
+
+#include <string/memcpy.c>
--
2.25.1
^ permalink raw reply [flat|nested] 6+ messages in thread