[PATCH v1] linux: Add support for getrandom vDSO

["Jason A. Donenfeld" <Jason@zx2c4.com>]

Linux 6.11 gains support for calling getrandom() from the vDSO. It
operates on a thread-local opaque state allocated with mmap using flags
specified by the vDSO.

Multiple states are allocated at once, as many as fit into a page, and
these are held in an array of available states to be doled out to each
thread upon first use, and recycled when a thread terminates. As these
states run low, more are allocated.

To make this procedure async-signal-safe, a simple guard is used in the
LSB of the opaque state address, falling back to the syscall if there's
reentrancy contention.

This implementation is intentionally kept somewhat basic. We can add
optimizations later, but for now, the idea is to get the bones set.

It's currently enabled for x86_64. As the kernel support gains more
platforms (arm64 is in the works), this can be easily turned on for
those.

Co-authored-by: Adhemerval Zanella <zatrazz@gmail.com>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
---
Hi folks,

This is joint work with Adhemerval. We've been iterating on this in
parallel with the kernel development, and now that the kernel part has
landed, we're ready to submit the glibc implementation here.

Note that I'll be traveling until the end of August, so v2 of this
patch, should there be a need, will likely be submitted by Adhemerval
instead.

This patch currently lives here: https://git.zx2c4.com/glibc/log/?h=vdso

Thanks for reviewing.

Thanks,
Jason

 include/sys/random.h                          |   4 +
 malloc/malloc.c                               |   4 +-
 sysdeps/generic/not-cancel.h                  |   4 +-
 sysdeps/mach/hurd/not-cancel.h                |   4 +-
 sysdeps/nptl/_Fork.c                          |   2 +
 sysdeps/nptl/fork.h                           |   2 +
 sysdeps/unix/sysv/linux/dl-vdso-setup.c       |   5 +
 sysdeps/unix/sysv/linux/dl-vdso-setup.h       |   3 +
 sysdeps/unix/sysv/linux/getrandom.c           | 199 +++++++++++++++++-
 sysdeps/unix/sysv/linux/getrandom_vdso.h      |  36 ++++
 sysdeps/unix/sysv/linux/not-cancel.h          |   7 +-
 sysdeps/unix/sysv/linux/tls-internal-struct.h |  29 +++
 sysdeps/unix/sysv/linux/tls-internal.c        |   2 +
 sysdeps/unix/sysv/linux/x86_64/sysdep.h       |   1 +
 14 files changed, 295 insertions(+), 7 deletions(-)
 create mode 100644 sysdeps/unix/sysv/linux/getrandom_vdso.h
 create mode 100644 sysdeps/unix/sysv/linux/tls-internal-struct.h

diff --git a/include/sys/random.h b/include/sys/random.h
index 6aa313d35d..4cc94c572d 100644
--- a/include/sys/random.h
+++ b/include/sys/random.h
@@ -3,9 +3,13 @@
 
 # ifndef _ISOMAC
 
+# include <stdbool.h>
+
 extern ssize_t __getrandom (void *__buffer, size_t __length,
                             unsigned int __flags) __wur;
 libc_hidden_proto (__getrandom)
 
+extern void __getrandom_fork_subprocess (bool reset_states) attribute_hidden;
+
 # endif /* !_ISOMAC */
 #endif
diff --git a/malloc/malloc.c b/malloc/malloc.c
index bcb6e5b83c..9e577ab900 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -3140,8 +3140,8 @@ static void
 tcache_key_initialize (void)
 {
   /* We need to use the _nostatus version here, see BZ 29624.  */
-  if (__getrandom_nocancel_nostatus (&tcache_key, sizeof(tcache_key),
-				     GRND_NONBLOCK)
+  if (__getrandom_nocancel_nostatus_direct (&tcache_key, sizeof(tcache_key),
+					    GRND_NONBLOCK)
       != sizeof (tcache_key))
     {
       tcache_key = random_bits ();
diff --git a/sysdeps/generic/not-cancel.h b/sysdeps/generic/not-cancel.h
index 2dd1064600..95d0ba1923 100644
--- a/sysdeps/generic/not-cancel.h
+++ b/sysdeps/generic/not-cancel.h
@@ -51,7 +51,9 @@
   __fcntl64 (fd, cmd, __VA_ARGS__)
 #define __getrandom_nocancel(buf, size, flags) \
   __getrandom (buf, size, flags)
-#define __getrandom_nocancel_nostatus(buf, size, flags) \
+#define __getrandom_nocancel_direct(buf, size, flags) \
+  __getrandom_nocancel (buf, size, flags)
+#define __getrandom_nocancel_nostatus_direct(buf, size, flags) \
   __getrandom (buf, size, flags)
 #define __poll_infinity_nocancel(fds, nfds) \
   __poll (fds, nfds, -1)
diff --git a/sysdeps/mach/hurd/not-cancel.h b/sysdeps/mach/hurd/not-cancel.h
index 69fb3c00ef..393febd225 100644
--- a/sysdeps/mach/hurd/not-cancel.h
+++ b/sysdeps/mach/hurd/not-cancel.h
@@ -79,7 +79,7 @@ __typeof (__fcntl) __fcntl_nocancel;
 /* Non cancellable getrandom syscall that does not also set errno in case of
    failure.  */
 static inline ssize_t
-__getrandom_nocancel_nostatus (void *buf, size_t buflen, unsigned int flags)
+__getrandom_nocancel_nostatus_direct (void *buf, size_t buflen, unsigned int flags)
 {
   int save_errno = errno;
   ssize_t r = __getrandom (buf, buflen, flags);
@@ -90,6 +90,8 @@ __getrandom_nocancel_nostatus (void *buf, size_t buflen, unsigned int flags)
 
 #define __getrandom_nocancel(buf, size, flags) \
   __getrandom (buf, size, flags)
+#define __getrandom_nocancel_direct(buf, size, flags) \
+  __getrandom_nocancel (buf, size, flags)
 
 #define __poll_infinity_nocancel(fds, nfds) \
   __poll (fds, nfds, -1)
diff --git a/sysdeps/nptl/_Fork.c b/sysdeps/nptl/_Fork.c
index ef199ddbc3..8375b18535 100644
--- a/sysdeps/nptl/_Fork.c
+++ b/sysdeps/nptl/_Fork.c
@@ -18,6 +18,7 @@
 
 #include <arch-fork.h>
 #include <pthreadP.h>
+#include <sys/random.h>
 
 pid_t
 _Fork (void)
@@ -43,6 +44,7 @@ _Fork (void)
       self->robust_head.list = &self->robust_head;
       INTERNAL_SYSCALL_CALL (set_robust_list, &self->robust_head,
 			     sizeof (struct robust_list_head));
+      call_function_static_weak (__getrandom_fork_subprocess, false);
     }
   return pid;
 }
diff --git a/sysdeps/nptl/fork.h b/sysdeps/nptl/fork.h
index 7643926df9..858129f3d2 100644
--- a/sysdeps/nptl/fork.h
+++ b/sysdeps/nptl/fork.h
@@ -26,6 +26,7 @@
 #include <mqueue.h>
 #include <pthreadP.h>
 #include <sysdep.h>
+#include <sys/random.h>
 
 static inline void
 fork_system_setup (void)
@@ -46,6 +47,7 @@ fork_system_setup_after_fork (void)
 
   call_function_static_weak (__mq_notify_fork_subprocess);
   call_function_static_weak (__timer_fork_subprocess);
+  call_function_static_weak (__getrandom_fork_subprocess, true);
 }
 
 /* In case of a fork() call the memory allocation in the child will be
diff --git a/sysdeps/unix/sysv/linux/dl-vdso-setup.c b/sysdeps/unix/sysv/linux/dl-vdso-setup.c
index 3a44944dbb..476c6db75a 100644
--- a/sysdeps/unix/sysv/linux/dl-vdso-setup.c
+++ b/sysdeps/unix/sysv/linux/dl-vdso-setup.c
@@ -66,6 +66,11 @@ PROCINFO_CLASS int (*_dl_vdso_clock_getres) (clockid_t,
 PROCINFO_CLASS int (*_dl_vdso_clock_getres_time64) (clockid_t,
 						    struct __timespec64 *) RELRO;
 # endif
+# ifdef HAVE_GETRANDOM_VSYSCALL
+PROCINFO_CLASS ssize_t (*_dl_vdso_getrandom) (void *buffer, size_t len,
+                                              unsigned int flags, void *state,
+                                              size_t state_len) RELRO;
+# endif
 
 /* PowerPC specific ones.  */
 # ifdef HAVE_GET_TBFREQ
diff --git a/sysdeps/unix/sysv/linux/dl-vdso-setup.h b/sysdeps/unix/sysv/linux/dl-vdso-setup.h
index 8aee5a8212..cde99f608c 100644
--- a/sysdeps/unix/sysv/linux/dl-vdso-setup.h
+++ b/sysdeps/unix/sysv/linux/dl-vdso-setup.h
@@ -50,6 +50,9 @@ setup_vdso_pointers (void)
 #ifdef HAVE_RISCV_HWPROBE
   GLRO(dl_vdso_riscv_hwprobe) = dl_vdso_vsym (HAVE_RISCV_HWPROBE);
 #endif
+#ifdef HAVE_GETRANDOM_VSYSCALL
+  GLRO(dl_vdso_getrandom) = dl_vdso_vsym (HAVE_GETRANDOM_VSYSCALL);
+#endif
 }
 
 #endif
diff --git a/sysdeps/unix/sysv/linux/getrandom.c b/sysdeps/unix/sysv/linux/getrandom.c
index 777d1decf0..b84cd72939 100644
--- a/sysdeps/unix/sysv/linux/getrandom.c
+++ b/sysdeps/unix/sysv/linux/getrandom.c
@@ -21,12 +21,209 @@
 #include <unistd.h>
 #include <sysdep-cancel.h>
 
+#ifdef HAVE_GETRANDOM_VSYSCALL
+# include <getrandom_vdso.h>
+# include <ldsodefs.h>
+# include <libc-lock.h>
+# include <list.h>
+# include <setvmaname.h>
+# include <sys/mman.h>
+# include <sys/sysinfo.h>
+# include <tls-internal.h>
+
+# define ALIGN_PAGE(p) PTR_ALIGN_UP (p, GLRO (dl_pagesize))
+# define READ_ONCE(p) (*((volatile typeof (p) *) (&(p))))
+# define WRITE_ONCE(p, v) (*((volatile typeof (p) *) (&(p))) = (v))
+# define RESERVE_PTR(p) ((void *) ((uintptr_t) (p) | 1UL))
+# define RELEASE_PTR(p) ((void *) ((uintptr_t) (p) & ~1UL))
+# define IS_RESERVED_PTR(p) (!!((uintptr_t) (p) & 1UL))
+
+static struct
+{
+  __libc_lock_define (, lock);
+  void **states;
+  size_t state_size;
+  size_t len;
+  size_t total;
+  size_t cap;
+} grnd_alloc = { .lock = LLL_LOCK_INITIALIZER };
+
+static struct grnd_alloc_state *
+vgetrandom_get_state (void)
+{
+  void *state = NULL;
+
+  __libc_lock_lock (grnd_alloc.lock);
+
+  if (grnd_alloc.len == 0)
+    {
+      struct vgetrandom_opaque_params params;
+      size_t block_size, num = __get_nprocs (); /* Just a decent heuristic. */
+      void *block;
+
+      if (GLRO (dl_vdso_getrandom) (NULL, 0, 0, &params, ~0UL))
+        goto out;
+      grnd_alloc.state_size = params.size_of_opaque_state;
+
+      block_size = ALIGN_PAGE (num * grnd_alloc.state_size);
+      num = (GLRO (dl_pagesize) / grnd_alloc.state_size) *
+            (block_size / GLRO (dl_pagesize));
+      block = __mmap (0, block_size, params.mmap_prot,
+                      params.mmap_flags, -1, 0);
+      if (block == NULL)
+        goto out;
+      __set_vma_name (block, block_size, " glibc: getrandom");
+
+      if (grnd_alloc.total + num > grnd_alloc.cap)
+        {
+          void **states;
+          size_t states_size = ALIGN_PAGE (sizeof(*grnd_alloc.states) *
+                                           (grnd_alloc.total + num));
+          if (grnd_alloc.states)
+            states = __mremap (grnd_alloc.states,
+                               ALIGN_PAGE (sizeof (*grnd_alloc.states) *
+                                           grnd_alloc.cap),
+                               states_size, MREMAP_MAYMOVE);
+          else
+            states = __mmap (NULL, states_size, PROT_READ | PROT_WRITE,
+                             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+          if (states == MAP_FAILED)
+            goto unmap;
+          __set_vma_name (states, states_size, " glibc: getrandom");
+          grnd_alloc.states = states;
+          grnd_alloc.cap = states_size / sizeof (*grnd_alloc.states);
+        }
+
+      for (size_t i = 0; i < num; ++i)
+        {
+          if (((uintptr_t) block & (GLRO (dl_pagesize) - 1)) +
+              grnd_alloc.state_size > GLRO (dl_pagesize))
+            block = ALIGN_PAGE (block);
+          grnd_alloc.states[i] = block;
+          block += grnd_alloc.state_size;
+        }
+      grnd_alloc.len = num;
+      grnd_alloc.total += num;
+      goto success;
+
+    unmap:
+      __munmap (block, block_size);
+      goto out;
+    }
+
+success:
+  state = grnd_alloc.states[--grnd_alloc.len];
+
+out:
+  __libc_lock_unlock (grnd_alloc.lock);
+  return state;
+}
+
+/* Return true if the syscall fallback should be issued in the case the vDSO
+   is not present, in the case of reentrancy, or if any memory allocation
+   fails.  */
+static bool
+__getrandom_internal (ssize_t *ret, void *buffer, size_t length,
+                      unsigned int flags)
+{
+  if (GLRO (dl_vdso_getrandom) == NULL)
+    return false;
+
+  struct tls_internal_t *ti = __glibc_tls_internal ();
+  void *state = READ_ONCE (ti->getrandom_buf);
+  if (IS_RESERVED_PTR (state))
+    return false;
+  WRITE_ONCE (ti->getrandom_buf, RESERVE_PTR (state));
+
+  bool r = false;
+  if (state == NULL)
+    {
+      state = vgetrandom_get_state ();
+      if (state == NULL)
+        goto out;
+    }
+
+  *ret = GLRO (dl_vdso_getrandom) (buffer, length, flags, state,
+                                   grnd_alloc.state_size);
+  if (INTERNAL_SYSCALL_ERROR_P (*ret))
+    {
+      __set_errno (INTERNAL_SYSCALL_ERRNO (*ret));
+      *ret = -1;
+    }
+  r = true;
+
+out:
+  WRITE_ONCE (ti->getrandom_buf, state);
+  return r;
+}
+
+static void
+reset_other_threads (list_t *list)
+{
+  list_t *runp;
+  list_for_each (runp, list)
+    {
+      struct pthread *t = list_entry (runp, struct pthread, list);
+      if (t == THREAD_SELF || !t->tls_state.getrandom_buf)
+        continue;
+      grnd_alloc.states[grnd_alloc.len++]
+          = RELEASE_PTR (t->tls_state.getrandom_buf);
+      t->tls_state.getrandom_buf = NULL;
+    }
+}
+#endif
+
+void
+__getrandom_vdso_release (void)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+  void *state = __glibc_tls_internal ()->getrandom_buf;
+  if (state == NULL)
+    return;
+
+  __libc_lock_lock (grnd_alloc.lock);
+  grnd_alloc.states[grnd_alloc.len++] = state;
+  __libc_lock_unlock (grnd_alloc.lock);
+#endif
+}
+
+void
+__getrandom_fork_subprocess (bool reset_states)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+  grnd_alloc.lock = LLL_LOCK_INITIALIZER;
+  if (reset_states)
+    {
+      reset_other_threads (&GL (dl_stack_used));
+      reset_other_threads (&GL (dl_stack_user));
+    }
+#endif
+}
+
+ssize_t
+__getrandom_nocancel (void *buffer, size_t length, unsigned int flags)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+  ssize_t r;
+  if (__getrandom_internal (&r, buffer, length, flags))
+    return r;
+#endif
+
+  return SYSCALL_CANCEL (getrandom, buffer, length, flags);
+}
+
 /* Write up to LENGTH bytes of randomness starting at BUFFER.
    Return the number of bytes written, or -1 on error.  */
 ssize_t
 __getrandom (void *buffer, size_t length, unsigned int flags)
 {
-  return SYSCALL_CANCEL (getrandom, buffer, length, flags);
+#ifdef HAVE_GETRANDOM_VSYSCALL
+  ssize_t r;
+  if (__getrandom_internal (&r, buffer, length, flags))
+    return r;
+#endif
+
+  return INTERNAL_SYSCALL_CALL (getrandom, buffer, length, flags);
 }
 libc_hidden_def (__getrandom)
 weak_alias (__getrandom, getrandom)
diff --git a/sysdeps/unix/sysv/linux/getrandom_vdso.h b/sysdeps/unix/sysv/linux/getrandom_vdso.h
new file mode 100644
index 0000000000..d3130c1038
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/getrandom_vdso.h
@@ -0,0 +1,36 @@
+/* Linux getrandom vDSO support.
+   Copyright (C) 2024 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef _GETRANDOM_VDSO_H
+#define _GETRANDOM_VDSO_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+extern void __getrandom_vdso_release (void) attribute_hidden;
+
+struct vgetrandom_opaque_params
+{
+  uint32_t size_of_opaque_state;
+  uint32_t mmap_prot;
+  uint32_t mmap_flags;
+  uint32_t reserved[13];
+};
+
+#endif
diff --git a/sysdeps/unix/sysv/linux/not-cancel.h b/sysdeps/unix/sysv/linux/not-cancel.h
index 2a7585b73f..12f26912d3 100644
--- a/sysdeps/unix/sysv/linux/not-cancel.h
+++ b/sysdeps/unix/sysv/linux/not-cancel.h
@@ -27,6 +27,7 @@
 #include <sys/syscall.h>
 #include <sys/wait.h>
 #include <time.h>
+#include <sys/random.h>
 
 /* Non cancellable open syscall.  */
 __typeof (open) __open_nocancel;
@@ -84,15 +85,17 @@ __writev_nocancel_nostatus (int fd, const struct iovec *iov, int iovcnt)
 }
 
 static inline ssize_t
-__getrandom_nocancel (void *buf, size_t buflen, unsigned int flags)
+__getrandom_nocancel_direct (void *buf, size_t buflen, unsigned int flags)
 {
   return INLINE_SYSCALL_CALL (getrandom, buf, buflen, flags);
 }
 
+__typeof (getrandom) __getrandom_nocancel attribute_hidden;
+
 /* Non cancellable getrandom syscall that does not also set errno in case of
    failure.  */
 static inline ssize_t
-__getrandom_nocancel_nostatus (void *buf, size_t buflen, unsigned int flags)
+__getrandom_nocancel_nostatus_direct (void *buf, size_t buflen, unsigned int flags)
 {
   return INTERNAL_SYSCALL_CALL (getrandom, buf, buflen, flags);
 }
diff --git a/sysdeps/unix/sysv/linux/tls-internal-struct.h b/sysdeps/unix/sysv/linux/tls-internal-struct.h
new file mode 100644
index 0000000000..f8d561c987
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tls-internal-struct.h
@@ -0,0 +1,29 @@
+/* Per-thread state.  Linux version.
+   Copyright (C) 2024 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef _TLS_INTERNAL_STRUCT_H
+#define _TLS_INTERNAL_STRUCT_H 1
+
+struct tls_internal_t
+{
+  char *strsignal_buf;
+  char *strerror_l_buf;
+  void *getrandom_buf;
+};
+
+#endif
diff --git a/sysdeps/unix/sysv/linux/tls-internal.c b/sysdeps/unix/sysv/linux/tls-internal.c
index 81ebcdfbf8..886b55daf9 100644
--- a/sysdeps/unix/sysv/linux/tls-internal.c
+++ b/sysdeps/unix/sysv/linux/tls-internal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#include <getrandom_vdso.h>
 #include <string.h>
 #include <tls-internal.h>
 
@@ -25,4 +26,5 @@ __glibc_tls_internal_free (void)
   struct pthread *self = THREAD_SELF;
   free (self->tls_state.strsignal_buf);
   free (self->tls_state.strerror_l_buf);
+  call_function_static_weak (__getrandom_vdso_release);
 }
diff --git a/sysdeps/unix/sysv/linux/x86_64/sysdep.h b/sysdeps/unix/sysv/linux/x86_64/sysdep.h
index a2b021bd86..7dc072ae2d 100644
--- a/sysdeps/unix/sysv/linux/x86_64/sysdep.h
+++ b/sysdeps/unix/sysv/linux/x86_64/sysdep.h
@@ -376,6 +376,7 @@
 # define HAVE_TIME_VSYSCALL             "__vdso_time"
 # define HAVE_GETCPU_VSYSCALL		"__vdso_getcpu"
 # define HAVE_CLOCK_GETRES64_VSYSCALL   "__vdso_clock_getres"
+# define HAVE_GETRANDOM_VSYSCALL        "__vdso_getrandom"
 
 # define HAVE_CLONE3_WRAPPER			1
 
-- 
2.45.2