[newlib-cygwin/cygwin-3_4-branch] newlib: Fix memory leak regarding gdtoa-based _ldtoa_r().

[newlib-cygwin/cygwin-3_4-branch] newlib: Fix memory leak regarding gdtoa-based _ldtoa_r().

[Takashi Yano]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=841b7307d60b4b21418df93190d8134262324eb7

commit 841b7307d60b4b21418df93190d8134262324eb7
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Wed Aug 2 14:31:53 2023 +0900

    newlib: Fix memory leak regarding gdtoa-based _ldtoa_r().
    
    After the commit a4705d387f78, printf() for floating-point values
    causes a memory leak. The legacy _ldtoa_r() assumed the char pointer
    returned will be free'ed by Bfree(). However, gdtoa-based _ldtoa_r()
    returns the pointer returned by gdtoa() which should be free'ed by
    freedtoa(). Due to this issue, the caller of _ldtoa_r() fails to free
    the allocated char buffer. This is the cause of the said memory leak.
    https://cygwin.com/pipermail/cygwin/2023-July/254054.html
    
    This patch makes rv_alloc()/freedtoa() allocate/free the buffer in
    a compatible way with legacy _ldtoa_r().
    
    Fixes: a4705d387f78 ("ldtoa: Import gdtoa from OpenBSD.")
    Reported-by: natan_b <natan_b@libero.it>
    Reviewed-by: Corinna Vinschen <corinna@vinschen.de>
    Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>

Diff:
---
 newlib/libc/stdlib/gdtoa-dmisc.c | 23 +++++++++++++----------
 newlib/libc/stdlib/gdtoa-ldtoa.c |  4 +---
 winsup/cygwin/release/3.4.8      |  3 +++
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/newlib/libc/stdlib/gdtoa-dmisc.c b/newlib/libc/stdlib/gdtoa-dmisc.c
index 332023dae..f330f8ae7 100644
--- a/newlib/libc/stdlib/gdtoa-dmisc.c
+++ b/newlib/libc/stdlib/gdtoa-dmisc.c
@@ -46,26 +46,28 @@ rv_alloc(ptr, i) struct _reent *ptr, int i;
 rv_alloc(struct _reent *ptr, int i)
 #endif
 {
-	int j, k, *r;
+	int j;
+	char *r;
 
+	/* Allocate buffer in a compatible way with legacy _ldtoa_r(). */
 	j = sizeof(ULong);
-	for(k = 0;
-		sizeof(Bigint) - sizeof(ULong) - sizeof(int) + j <= i;
-		j <<= 1)
-			k++;
-	r = (int*)Balloc(ptr, k);
+	for (_REENT_MP_RESULT_K (ptr) = 0;
+	     sizeof (Bigint) - sizeof (ULong) + j <= i; j <<= 1)
+		_REENT_MP_RESULT_K (ptr)++;
+	_REENT_MP_RESULT (ptr) = eBalloc (ptr, _REENT_MP_RESULT_K (ptr));
+	r = (char *) _REENT_MP_RESULT (ptr);
+
 	if (r == NULL)
 		return (
 #ifndef MULTIPLE_THREADS
 		dtoa_result =
 #endif
 			NULL);
-	*r = k;
 	return
 #ifndef MULTIPLE_THREADS
 	dtoa_result =
 #endif
-		(char *)(r+1);
+		r;
 	}
 
  char *
@@ -100,8 +102,9 @@ freedtoa(ptr, s) struct _reent *ptr, char *s;
 freedtoa(struct _reent *ptr, char *s)
 #endif
 {
-	Bigint *b = (Bigint *)((int *)s - 1);
-	b->_maxwds = 1 << (b->_k = *(int*)b);
+	/* Free buffer allocated in a compatible way with legacy _ldtoa_r(). */
+	Bigint *b = (Bigint *)s;
+	b->_maxwds = 1 << (b->_k = _REENT_MP_RESULT_K (ptr));
 	Bfree(ptr, b);
 #ifndef MULTIPLE_THREADS
 	if (s == dtoa_result)
diff --git a/newlib/libc/stdlib/gdtoa-ldtoa.c b/newlib/libc/stdlib/gdtoa-ldtoa.c
index 14b99042c..09ba6b34b 100644
--- a/newlib/libc/stdlib/gdtoa-ldtoa.c
+++ b/newlib/libc/stdlib/gdtoa-ldtoa.c
@@ -72,9 +72,7 @@ _ldtoa_r(struct _reent *ptr,
 
 	/* reentrancy addition to use mprec storage pool */
 	if (_REENT_MP_RESULT (ptr)) {
-		_REENT_MP_RESULT (ptr)->_k = _REENT_MP_RESULT_K (ptr);
-		_REENT_MP_RESULT (ptr)->_maxwds = 1 << _REENT_MP_RESULT_K (ptr);
-		Bfree (ptr, _REENT_MP_RESULT (ptr));
+		freedtoa (ptr, _REENT_MP_RESULT (ptr));
 		_REENT_MP_RESULT (ptr) = 0;
 	}
 
diff --git a/winsup/cygwin/release/3.4.8 b/winsup/cygwin/release/3.4.8
index d37272eef..448831c65 100644
--- a/winsup/cygwin/release/3.4.8
+++ b/winsup/cygwin/release/3.4.8
@@ -14,3 +14,6 @@ Bug Fixes
 - Rename internal macros _NL_CTYPE_OUTDIGITSx_MB/WC to GLibc compatible
   _NL_CTYPE_OUTDIGITx_MB/WC.
   Addresses: https://cygwin.com/pipermail/cygwin-developers/2023-July/012637.html
+
+- Fix memory leak in printf() regarding gdtoa-based _ldtoa_r().
+  Addresses: https://cygwin.com/pipermail/cygwin/2023-July/254054.html