[binutils-gdb] [gdb/tdep] Fix avx512 -m32 support in gdbserver

[binutils-gdb] [gdb/tdep] Fix avx512 -m32 support in gdbserver

[Tom de Vries]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=2e187550373de67a20a2584d595ed9bb18e742ee

commit 2e187550373de67a20a2584d595ed9bb18e742ee
Author: Tom de Vries <tdevries@suse.de>
Date:   Thu Dec 2 18:20:13 2021 +0100

    [gdb/tdep] Fix avx512 -m32 support in gdbserver
    
    PR27257 reports a problem that can be reproduced as follows:
    - use x86_64 machine with avx512 support
    - compile a hello world with -m32 to a.out
    - start a gdbserver session with a.out
    - use gdb to connect to the gdbserver session
    
    This makes us run into:
    ...
    Listening on port 2346
    Remote debugging from host ::1, port 34940
    src/gdbserver/regcache.cc:257: \
      A problem internal to GDBserver has been detected.
    Unknown register zmm16h requested
    ...
    
    The problem is that i387_xsave_to_cache in gdbserver/i387-fp.cc can't find a
    register zmm16h in the register cache.
    
    To understand how this happens, first some background.
    
    SSE has 16 128-bit wide xmm registers.
    
    AVX extends the SSE registers set as follows:
    - it extends the 16 existing 128-bit wide xmm registers to 256-bit wide ymm
      registers.
    
    AVX512 extends the AVX register set as follows:
    - it extends the 16 existing 256-bit wide ymm registers to 512-bit wide zmm
      registers.
    - it adds 16 additional 512-bit wide zmm registers (with corresponding ymm and
      xmm subregisters added as well)
    
    However, in 32-bit mode, there are only 8 xmm/ymm/zmm registers.
    
    The problem we're running into is that gdbserver/i387-fp.cc uses these
    constants to describe the size of the register file:
    ...
    static const int num_avx512_zmmh_low_registers = 16;
    static const int num_avx512_zmmh_high_registers = 16;
    static const int num_avx512_ymmh_registers = 16;
    static const int num_avx512_xmm_registers = 16;
    ...
    which are all incorrect for the 32-bit case.
    
    Fix this by replacing the constants with variables that have the appropriate
    values in 64-bit and 32-bit mode.
    
    Tested on x86_64-linux with native and unix/-m32.

Diff:
---
 gdbserver/i387-fp.cc | 50 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/gdbserver/i387-fp.cc b/gdbserver/i387-fp.cc
index 6d9b62ed8ce..9b07ded592d 100644
--- a/gdbserver/i387-fp.cc
+++ b/gdbserver/i387-fp.cc
@@ -23,10 +23,6 @@
 static const int num_mpx_bnd_registers = 4;
 static const int num_mpx_cfg_registers = 2;
 static const int num_avx512_k_registers = 8;
-static const int num_avx512_zmmh_low_registers = 16;
-static const int num_avx512_zmmh_high_registers = 16;
-static const int num_avx512_ymmh_registers = 16;
-static const int num_avx512_xmm_registers = 16;
 static const int num_pkeys_registers = 1;
 
 /* Note: These functions preserve the reserved bits in control registers.
@@ -256,14 +252,22 @@ void
 i387_cache_to_xsave (struct regcache *regcache, void *buf)
 {
   struct i387_xsave *fp = (struct i387_xsave *) buf;
+  bool amd64 = register_size (regcache->tdesc, 0) == 8;
   int i;
   unsigned long val, val2;
   unsigned long long xstate_bv = 0;
   unsigned long long clear_bv = 0;
   char raw[64];
   char *p;
+
   /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
-  int num_xmm_registers = register_size (regcache->tdesc, 0) == 8 ? 16 : 8;
+  int num_xmm_registers = amd64 ? 16 : 8;
+  /* AVX512 extends the existing xmm/ymm registers to a wider mode: zmm.  */
+  int num_avx512_zmmh_low_registers = num_xmm_registers;
+  /* AVX512 adds 16 extra regs in Amd64 mode, but none in I386 mode.*/
+  int num_avx512_zmmh_high_registers = amd64 ? 16 : 0;
+  int num_avx512_ymmh_registers = amd64 ? 16 : 0;
+  int num_avx512_xmm_registers = amd64 ? 16 : 0;
 
   /* The supported bits in `xstat_bv' are 8 bytes.  Clear part in
      vector registers if its bit in xstat_bv is zero.  */
@@ -452,7 +456,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any of ZMM16H-ZMM31H registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int zmm16h_regnum = find_regno (regcache->tdesc, "zmm16h");
+      int zmm16h_regnum = (num_avx512_zmmh_high_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "zmm16h"));
 
       for (i = 0; i < num_avx512_zmmh_high_registers; i++)
 	{
@@ -469,7 +475,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any XMM_AVX512 registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int xmm_avx512_regnum = find_regno (regcache->tdesc, "xmm16");
+      int xmm_avx512_regnum = (num_avx512_xmm_registers == 0
+			       ? -1
+			       : find_regno (regcache->tdesc, "xmm16"));
 
       for (i = 0; i < num_avx512_xmm_registers; i++)
 	{
@@ -486,7 +494,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any YMMH_AVX512 registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int ymmh_avx512_regnum = find_regno (regcache->tdesc, "ymm16h");
+      int ymmh_avx512_regnum = (num_avx512_ymmh_registers == 0
+				? -1
+				: find_regno (regcache->tdesc, "ymm16h"));
 
       for (i = 0; i < num_avx512_ymmh_registers; i++)
 	{
@@ -710,12 +720,20 @@ i387_xsave_to_cache (struct regcache *regcache, const void *buf)
 {
   struct i387_xsave *fp = (struct i387_xsave *) buf;
   struct i387_fxsave *fxp = (struct i387_fxsave *) buf;
+  bool amd64 = register_size (regcache->tdesc, 0) == 8;
   int i, top;
   unsigned long val;
   unsigned long long clear_bv;
   gdb_byte *p;
-  /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
-  int num_xmm_registers = register_size (regcache->tdesc, 0) == 8 ? 16 : 8;
+
+   /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
+  int num_xmm_registers = amd64 ? 16 : 8;
+  /* AVX512 extends the existing xmm/ymm registers to a wider mode: zmm.  */
+  int num_avx512_zmmh_low_registers = num_xmm_registers;
+  /* AVX512 adds 16 extra regs in Amd64 mode, but none in I386 mode.*/
+  int num_avx512_zmmh_high_registers = amd64 ? 16 : 0;
+  int num_avx512_ymmh_registers = amd64 ? 16 : 0;
+  int num_avx512_xmm_registers = amd64 ? 16 : 0;
 
   /* The supported bits in `xstat_bv' are 8 bytes.  Clear part in
      vector registers if its bit in xstat_bv is zero.  */
@@ -845,9 +863,15 @@ i387_xsave_to_cache (struct regcache *regcache, const void *buf)
 
   if ((x86_xcr0 & X86_XSTATE_ZMM) != 0)
     {
-      int zmm16h_regnum = find_regno (regcache->tdesc, "zmm16h");
-      int ymm16h_regnum = find_regno (regcache->tdesc, "ymm16h");
-      int xmm16_regnum = find_regno (regcache->tdesc, "xmm16");
+      int zmm16h_regnum = (num_avx512_zmmh_high_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "zmm16h"));
+      int ymm16h_regnum = (num_avx512_ymmh_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "ymm16h"));
+      int xmm16_regnum = (num_avx512_xmm_registers == 0
+			  ? -1
+			  : find_regno (regcache->tdesc, "xmm16"));
 
       if ((clear_bv & X86_XSTATE_ZMM) != 0)
 	{