[glibc] Missing ENOMEM in realloc_check wrapper (bug 27870)

[Andreas Schwab <schwab@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=c6b6b4f2c7ff62abf5da617bff9d8080631993c0

commit c6b6b4f2c7ff62abf5da617bff9d8080631993c0
Author: Andreas Schwab <schwab@linux-m68k.org>
Date:   Mon May 17 14:00:19 2021 +0200

    Missing ENOMEM in realloc_check wrapper (bug 27870)
    
    When MALLOC_CHECK_ is non-zero, the realloc hook missed to set errno to
    ENOMEM when called with too big size.  Run the test tst-malloc-too-large
    also with MALLOC_CHECK_=3 to catch that.

Diff:
---
 malloc/Makefile | 2 +-
 malloc/hooks.c  | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/malloc/Makefile b/malloc/Makefile
index afcd296ef6..857e2ebbd9 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -72,7 +72,7 @@ test-srcs = tst-mtrace
 # with MALLOC_CHECK_=3 because they expect a specific failure.
 tests-exclude-mcheck = tst-mcheck tst-malloc-usable \
 	tst-interpose-nothread tst-interpose-static-nothread \
-	tst-interpose-static-thread tst-malloc-too-large \
+	tst-interpose-static-thread \
 	tst-mxfast tst-safe-linking
 
 # Run all tests with MALLOC_CHECK_=3
diff --git a/malloc/hooks.c b/malloc/hooks.c
index c91f9502ba..8080c3f40e 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -321,7 +321,10 @@ realloc_check (void *oldmem, size_t bytes, const void *caller)
   const INTERNAL_SIZE_T oldsize = chunksize (oldp);
 
   if (!checked_request2size (rb, &chnb))
-    goto invert;
+    {
+      __set_errno (ENOMEM);
+      goto invert;
+    }
 
   __libc_lock_lock (main_arena.mutex);