[glibc] Correct range checking in mallopt/mxfast/tcache [BZ #25194]

[DJ Delorie <dj@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=16554464bcd9d77b07c6ff419dc54f00e394fa50

commit 16554464bcd9d77b07c6ff419dc54f00e394fa50
Author: DJ Delorie <dj@redhat.com>
Date:   Tue Dec 3 17:44:36 2019 -0500

    Correct range checking in mallopt/mxfast/tcache [BZ #25194]
    
    do_set_tcache_max, do_set_mxfast:
    Fix two instances of comparing "size_t < 0"
    Both cases have upper limit, so the "negative value" case
    is already handled via overflow semantics.
    
    do_set_tcache_max, do_set_tcache_count:
    Fix return value on error.  Note: currently not used.
    
    mallopt:
    pass return value of helper functions to user.  Behavior should
    only be actually changed for mxfast, where we restore the old
    (pre-tunables) behavior.
    
    Reviewed-by: Carlos O'Donell <carlos@redhat.com>

Diff:
---
 malloc/malloc.c | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/malloc/malloc.c b/malloc/malloc.c
index 70cc35a..7d7d30b 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -5086,13 +5086,14 @@ do_set_arena_max (size_t value)
 static __always_inline int
 do_set_tcache_max (size_t value)
 {
-  if (value >= 0 && value <= MAX_TCACHE_SIZE)
+  if (value <= MAX_TCACHE_SIZE)
     {
       LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
       mp_.tcache_max_bytes = value;
       mp_.tcache_bins = csize2tidx (request2size(value)) + 1;
+      return 1;
     }
-  return 1;
+  return 0;
 }
 
 static __always_inline int
@@ -5102,8 +5103,9 @@ do_set_tcache_count (size_t value)
     {
       LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count);
       mp_.tcache_count = value;
+      return 1;
     }
-  return 1;
+  return 0;
 }
 
 static __always_inline int
@@ -5119,7 +5121,7 @@ static inline int
 __always_inline
 do_set_mxfast (size_t value)
 {
-  if (value >= 0 && value <= MAX_FAST_SIZE)
+  if (value <= MAX_FAST_SIZE)
     {
       LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ());
       set_max_fast (value);
@@ -5144,18 +5146,24 @@ __libc_mallopt (int param_number, int value)
      (see definition of set_max_fast).  */
   malloc_consolidate (av);
 
+  /* Many of these helper functions take a size_t.  We do not worry
+     about overflow here, because negative int values will wrap to
+     very large size_t values and the helpers have sufficient range
+     checking for such conversions.  Many of these helpers are also
+     used by the tunables macros in arena.c.  */
+
   switch (param_number)
     {
     case M_MXFAST:
-      do_set_mxfast (value);
+      res = do_set_mxfast (value);
       break;
 
     case M_TRIM_THRESHOLD:
-      do_set_trim_threshold (value);
+      res = do_set_trim_threshold (value);
       break;
 
     case M_TOP_PAD:
-      do_set_top_pad (value);
+      res = do_set_top_pad (value);
       break;
 
     case M_MMAP_THRESHOLD:
@@ -5163,25 +5171,25 @@ __libc_mallopt (int param_number, int value)
       break;
 
     case M_MMAP_MAX:
-      do_set_mmaps_max (value);
+      res = do_set_mmaps_max (value);
       break;
 
     case M_CHECK_ACTION:
-      do_set_mallopt_check (value);
+      res = do_set_mallopt_check (value);
       break;
 
     case M_PERTURB:
-      do_set_perturb_byte (value);
+      res = do_set_perturb_byte (value);
       break;
 
     case M_ARENA_TEST:
       if (value > 0)
-	do_set_arena_test (value);
+	res = do_set_arena_test (value);
       break;
 
     case M_ARENA_MAX:
       if (value > 0)
-	do_set_arena_max (value);
+	res = do_set_arena_max (value);
       break;
     }
   __libc_lock_unlock (av->mutex);