[PATCH] [gdb/python] Throw MemoryError in inferior.read_memory if malloc fails

[Tom de Vries <tdevries@suse.de>]

PR python/31631 reports a gdb internal error when doing:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
utils.c:709: internal-error: virtual memory exhausted.
A problem internal to GDB has been detected,
further debugging may prove unreliable.
...

Fix this by throwing a python MemoryError, such that we have instead:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
Python Exception <class 'MemoryError'>:
Error occurred in Python.
(gdb)
...

Likewise for DAP.

Tested on x86_64-linux.

Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=31631
---
 gdb/python/lib/gdb/dap/memory.py         |  9 ++++++++-
 gdb/python/py-inferior.c                 | 11 ++++++++++-
 gdb/testsuite/gdb.dap/memory.exp         |  5 +++++
 gdb/testsuite/gdb.python/py-inferior.exp |  6 ++++++
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/gdb/python/lib/gdb/dap/memory.py b/gdb/python/lib/gdb/dap/memory.py
index dd62b0e7ba6..0fd5c2128e5 100644
--- a/gdb/python/lib/gdb/dap/memory.py
+++ b/gdb/python/lib/gdb/dap/memory.py
@@ -18,13 +18,20 @@ import base64
 import gdb
 
 from .server import capability, request
+from .startup import DAPException
 
 
 @request("readMemory")
 @capability("supportsReadMemoryRequest")
 def read_memory(*, memoryReference: str, offset: int = 0, count: int, **extra):
     addr = int(memoryReference, 0) + offset
-    buf = gdb.selected_inferior().read_memory(addr, count)
+    oom = False
+    try:
+        buf = gdb.selected_inferior().read_memory(addr, count)
+    except MemoryError:
+        oom = True
+    if oom:
+        raise DAPException("Out of memory")
     return {
         "address": hex(addr),
         "data": base64.b64encode(buf).decode("ASCII"),
diff --git a/gdb/python/py-inferior.c b/gdb/python/py-inferior.c
index 795ac655ddd..e825e649533 100644
--- a/gdb/python/py-inferior.c
+++ b/gdb/python/py-inferior.c
@@ -562,7 +562,16 @@ infpy_read_memory (PyObject *self, PyObject *args, PyObject *kw)
       scoped_restore_current_inferior_for_memory restore_inferior
 	(inf->inferior);
 
-      buffer.reset ((gdb_byte *) xmalloc (length));
+      /* We used to use xmalloc, which does this trick to avoid malloc
+	 returning a nullptr for a valid reason.  Keep doing the same.  */
+      if (length == 0)
+	length = 1;
+
+      void *p = malloc (length);
+      if (p == nullptr)
+	return PyErr_NoMemory ();
+
+      buffer.reset ((gdb_byte *) p);
 
       read_memory (addr, buffer.get (), length);
     }
diff --git a/gdb/testsuite/gdb.dap/memory.exp b/gdb/testsuite/gdb.dap/memory.exp
index 2e911f4dc77..4e2e361289a 100644
--- a/gdb/testsuite/gdb.dap/memory.exp
+++ b/gdb/testsuite/gdb.dap/memory.exp
@@ -55,6 +55,11 @@ set obj [dap_check_request_and_response "evaluate global pointer" \
 	     evaluate {o expression [s thirty_two_p]}]
 set addr [dict get [lindex $obj 0] body memoryReference]
 
+set obj [dap_request_and_response \
+	     readMemory [format {o memoryReference [s %s] count [i 18446744073709551615]} $addr]]
+set response [lindex $obj 0]
+gdb_assert { [dict get $response success] == "false" } "read memory, count to big"
+
 set obj [dap_check_request_and_response "read memory" \
 	     readMemory [format {o memoryReference [s %s] count [i 4]} $addr]]
 
diff --git a/gdb/testsuite/gdb.python/py-inferior.exp b/gdb/testsuite/gdb.python/py-inferior.exp
index c14f2d2796c..4c19e259159 100644
--- a/gdb/testsuite/gdb.python/py-inferior.exp
+++ b/gdb/testsuite/gdb.python/py-inferior.exp
@@ -34,6 +34,12 @@ switch [get_endianness] {
     big { set python_pack_char ">" }
 }
 
+gdb_test \
+    "python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)" \
+    [multi_line \
+	 [string_to_regexp "Python Exception <class 'MemoryError'>: "] \
+	 [string_to_regexp "Error occurred in Python."]]
+
 # Test memory read operations without execution.
 
 gdb_py_test_silent_cmd "python addr = gdb.lookup_global_symbol ('int8_global').value().address" \

base-commit: af925905211930677751678183f43c1bda13e013
-- 
2.35.3