[glibc] malloc: Add THP/madvise support for sbrk

[glibc] malloc: Add THP/madvise support for sbrk

[Adhemerval Zanella]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=7478c9959ae409f7b3d63146943575d6ee745352

commit 7478c9959ae409f7b3d63146943575d6ee745352
Author: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date:   Fri Aug 13 10:06:04 2021 -0300

    malloc: Add THP/madvise support for sbrk
    
    To increase effectiveness with Transparent Huge Page with madvise, the
    large page size is use instead page size for sbrk increment for the
    main arena.
    
    Checked on x86_64-linux-gnu.
    
    Reviewed-by: DJ Delorie <dj@redhat.com>

Diff:
---
 include/libc-pointer-arith.h |  8 ++++++++
 malloc/malloc.c              | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
index 04ba537617..55dccc10ac 100644
--- a/include/libc-pointer-arith.h
+++ b/include/libc-pointer-arith.h
@@ -60,4 +60,12 @@
 #define PTR_ALIGN_UP(base, size) \
   ((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
 
+/* Check if BASE is aligned on SIZE  */
+#define PTR_IS_ALIGNED(base, size) \
+  ((((uintptr_t) (base)) & (size - 1)) == 0)
+
+/* Returns the ptrdiff_t diference between P1 and P2.  */
+#define PTR_DIFF(p1, p2) \
+  ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
+
 #endif
diff --git a/malloc/malloc.c b/malloc/malloc.c
index c75841b841..57db4dd9a5 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -2023,6 +2023,16 @@ madvise_thp (void *p, INTERNAL_SIZE_T size)
      not active.  */
   if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
     return;
+
+  /* Linux requires the input address to be page-aligned, and unaligned
+     inputs happens only for initial data segment.  */
+  if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
+    {
+      void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
+      size += PTR_DIFF (p, q);
+      p = q;
+    }
+
   __madvise (p, size, MADV_HUGEPAGE);
 #endif
 }
@@ -2609,14 +2619,25 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
         size -= old_size;
 
       /*
-         Round to a multiple of page size.
+         Round to a multiple of page size or huge page size.
          If MORECORE is not contiguous, this ensures that we only call it
          with whole-page arguments.  And if MORECORE is contiguous and
          this is not first time through, this preserves page-alignment of
          previous calls. Otherwise, we correct to page-align below.
        */
 
-      size = ALIGN_UP (size, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+      /* Defined in brk.c.  */
+      extern void *__curbrk;
+      if (__glibc_unlikely (mp_.thp_pagesize != 0))
+	{
+	  uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
+				    mp_.thp_pagesize);
+	  size = top - (uintptr_t) __curbrk;
+	}
+      else
+#endif
+	size = ALIGN_UP (size, GLRO(dl_pagesize));
 
       /*
          Don't try to call MORECORE if argument is so big as to appear
@@ -2899,10 +2920,8 @@ systrim (size_t pad, mstate av)
   long released;         /* Amount actually released */
   char *current_brk;     /* address returned by pre-check sbrk call */
   char *new_brk;         /* address returned by post-check sbrk call */
-  size_t pagesize;
   long top_area;
 
-  pagesize = GLRO (dl_pagesize);
   top_size = chunksize (av->top);
 
   top_area = top_size - MINSIZE - 1;
@@ -2910,7 +2929,12 @@ systrim (size_t pad, mstate av)
     return 0;
 
   /* Release in pagesize units and round down to the nearest page.  */
-  extra = ALIGN_DOWN(top_area - pad, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+  if (__glibc_unlikely (mp_.thp_pagesize != 0))
+    extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
+  else
+#endif
+    extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
 
   if (extra == 0)
     return 0;