[PATCH] Fix call functions command bug in 64-bit programs for AIX

[Aditya Kamath1 <Aditya.Kamath1@ibm.com>]

[-- Attachment #1.1: Type: text/plain, Size: 2366 bytes --]

Hi all,

AIX folks while attempting to debug 64-bit programs using the call command are unable to receive and send the correct return value and parameter info in the program.

The issue can be replicated in the below program:-

#include <stdio.h>


long longit (long a)

{

  printf ("long val = %ld \n", a);

  return a;


}


int intit (int a)

{

  printf ("int val = %d \n", a);

  return a;


}


int main ()

{

  intit (27);

  longit (33);

}

The output before the patch is

Reading symbols from /home/xyz/gdb_tests/callfuncs...

(gdb) b main

Breakpoint 1 at 0x10000780: file /home/xyz/gdb_tests/callfuncs.c, line 19.

(gdb) r

Starting program: /home/xyz/gdb_tests/callfuncs

BFD: /usr/lib/libc.a(/usr/lib/libc.a(shr_64.o)): wrong auxtype 0xff for storage class 0x2

BFD: /usr/lib/libc.a(/usr/lib/libc.a(shr_64.o)): wrong auxtype 0xff for storage class 0x6b


Breakpoint 1, main () at /home/xyz/gdb_tests/callfuncs.c:19

19        intit (27);

(gdb) call intit (21)

int val = 0

$1 = 0

(gdb) q


The output should have been 21 but is not.

The output after applying this patch:-

Reading symbols from /home/XYZ/gdb_tests/callfuncs...

(gdb) b main

Breakpoint 1 at 0x100005f0: file /home/XYZ/gdb_tests/callfuncs.c, line 19.

(gdb) r

Starting program: /home/XYZ/gdb_tests/callfuncs


Breakpoint 1, main () at /home/XYZ/gdb_tests/callfuncs.c:19

19        intit (27);

(gdb) call intit (21)

int val = 21

$1 = 21

(gdb) q


Why does this happen??
The issue is that when a user attempts to test the return type or print statements via the call "FUNC_NAME (parameter A, parameter B)" command, any input be it parameter A or B is taken in little endian format from the GDB cache. But AIX is using Big endian format.

For example, if we have a value 21 as type long then the higher 32 bits [ which is 0 in number 21] were stored in lower 32 bits and lower 32 bits [ represent 21 in the number 21] is stored in higher 32 bits.

Please find attached the patch. [See:- 0001-Fix-call-functions-command-bug-in-64-bit-programs.patch]

In the patch I have written comments on further details.

Kindly let us know if this solution works, if not let us know a better way to handle the same.

Have a nice day ahead.

Thanks and regards,
Aditya.




[-- Attachment #2: 0001-Fix-call-functions-command-bug-in-64-bit-programs.patch --]
[-- Type: application/octet-stream, Size: 6727 bytes --]

From 7d842b6c956840f78871570207f15cf76a4b2354 Mon Sep 17 00:00:00 2001
From: Aditya Vidyadhar Kamath <Aditya.Kamath1@ibm.com>
Date: Mon, 7 Nov 2022 04:53:15 -0600
Subject: [PATCH] Fix call functions command bug in 64-bit programs for AIX

The issue is that when a user attempts to test the return type or print statements via the

call FUNCNAME with parameters command

any input be it parameter A or B is taken in little endian format from the GDB cache

 But AIX is using Big endian format

This patch fixes the same issue
---
 gdb/rs6000-aix-nat.c | 146 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 108 insertions(+), 38 deletions(-)

diff --git a/gdb/rs6000-aix-nat.c b/gdb/rs6000-aix-nat.c
index cb141427696..99e8af03877 100644
--- a/gdb/rs6000-aix-nat.c
+++ b/gdb/rs6000-aix-nat.c
@@ -194,6 +194,7 @@ fetch_register (struct regcache *regcache, int regno)
 {
   struct gdbarch *gdbarch = regcache->arch ();
   int addr[PPC_MAX_REGISTER_SIZE];
+  long long addr64[PPC_MAX_REGISTER_SIZE];
   int nr, isfloat;
   pid_t pid = regcache->ptid ().pid ();
 
@@ -228,14 +229,19 @@ fetch_register (struct regcache *regcache, int regno)
 	  long long buf;
 	  rs6000_ptrace64 (PT_READ_GPR, pid, nr, 0, &buf);
 	  if (register_size (gdbarch, regno) == 8)
-	    memcpy (addr, &buf, 8);
+	    memcpy (addr64, &buf, 8);
 	  else
-	    *addr = buf;
+	    *addr64 = buf;
 	}
     }
 
   if (!errno)
-    regcache->raw_supply (regno, (char *) addr);
+  {
+    if (!ARCH64 ())
+      regcache->raw_supply (regno, (char *) addr);
+    else
+      regcache->raw_supply (regno, (char *) addr64);
+  }
   else
     {
 #if 0
@@ -252,57 +258,121 @@ static void
 store_register (struct regcache *regcache, int regno)
 {
   struct gdbarch *gdbarch = regcache->arch ();
-  int addr[PPC_MAX_REGISTER_SIZE];
-  int nr, isfloat;
-  pid_t pid = regcache->ptid ().pid ();
-
-  /* Fetch the register's value from the register cache.  */
-  regcache->raw_collect (regno, addr);
-
-  /* -1 can be a successful return value, so infer errors from errno.  */
+  int addr32[PPC_MAX_REGISTER_SIZE];
+  long addr64[PPC_MAX_REGISTER_SIZE];
+  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
+  CORE_ADDR addr;
+  CORE_ADDR original_addr;
+  size_t size;
+  gdb_byte *little_end_buf;
+  pid_t pid;
+  int i;
+  int isfloat;
+  long big_end_buf = INT_MIN;
   errno = 0;
 
-  nr = regmap (gdbarch, regno, &isfloat);
+  /* Get the address of the register.  */
+  addr = regmap (gdbarch, regno, &isfloat);
+
+  if (addr == (CORE_ADDR)-1
+      || gdbarch_cannot_store_register (gdbarch, regno))
+    return;
+ 
+  pid = regcache->ptid ().pid ();
+
+  size = register_size (gdbarch, regno);
+  little_end_buf = (gdb_byte *) alloca (size);
+
+  /* Register number R3 to R10 in AIX belongs to 
+     function parameters. If a user attempts to 
+     give input via call command and in 64 bit mode
+     the value in the buffer is little endian but
+     in AIX we use big endian.  Hence if R3 to R10
+     registers are used in 64 bit mode we need to 
+     re align from little endian to big endian.
+     Otherwise it can be in a 64 bit or 32 bit buffer
+     depending on the architecture bit mode of the 
+     user program.  */
+
+  if (regno <= 10 && regno >= 3 && ARCH64 ()) 
+    regcache->raw_collect (regno, little_end_buf);
+  else if (ARCH64 ())
+    regcache->raw_collect (regno, addr64);
+  else
+    regcache->raw_collect (regno, addr32);
+ 
+  /* Save the original address of the register.  */
+  original_addr = addr;
 
   /* Floating-point registers.  */
   if (isfloat)
-    rs6000_ptrace32 (PT_WRITE_FPR, pid, addr, nr, 0);
+    rs6000_ptrace32 (PT_WRITE_FPR, pid, *little_end_buf, addr, 0);
 
   /* Bogus register number.  */
-  else if (nr < 0)
+  if (addr < 0)
     {
       if (regno >= gdbarch_num_regs (gdbarch))
-	gdb_printf (gdb_stderr,
-		    "gdb error: register no %d not implemented.\n",
-		    regno);
+        gdb_printf (gdb_stderr,
+                    "gdb error: register no %d not implemented.\n",
+                    regno);
     }
+  /* Fixed-point registers in 32 bit mode.  */
+  else if (!ARCH64 ())
+    rs6000_ptrace32 (PT_WRITE_GPR, pid, (int *) addr, *addr32, 0);
 
-  /* Fixed-point registers.  */
+  /* Fixed-point registers in 64 bit mode.  */
   else
+  {
+    /* Function parameters to be changed from little to big endian.  */
+    if (regno <= 10 && regno >= 3)
     {
+      for (i = 0; i < size; i += sizeof (PTRACE_TYPE_RET))
+      {
+        size_t chunk = std::min (sizeof (PTRACE_TYPE_RET), size - i);
+        PTRACE_TYPE_RET val;
+
+        val = extract_unsigned_integer (little_end_buf + i, chunk, byte_order);
+        errno = 0;
+         
+        /* Realign chunck by chunk in 8 bits.  */
+   
+        if (register_size (gdbarch, regno) == 8)
+        {
+          if (val != 0)
+          {
+            /* Either higher 32 bits or higher 32 bits are 0.  */
+            if (big_end_buf == INT_MIN || big_end_buf == 0)
+              big_end_buf = val;
+
+            /* Lower 32 bits.  */
+            else if (big_end_buf != INT_MIN)
+            {
+              big_end_buf <<= 8;
+              big_end_buf |= val;
+            }
+          }
+
+          /* Copy 0 only if the actual value is  0.  */
+          else if (val == 0 && big_end_buf == INT_MIN)
+            big_end_buf = val;
+        }
+        else
+          big_end_buf = val;
+        addr += sizeof (PTRACE_TYPE_RET);
+      }
+
       /* The PT_WRITE_GPR operation is rather odd.  For 32-bit inferiors,
-	 the register's value is passed by value, but for 64-bit inferiors,
-	 the address of a buffer containing the value is passed.  */
-      if (!ARCH64 ())
-	rs6000_ptrace32 (PT_WRITE_GPR, pid, (int *) nr, *addr, 0);
-      else
-	{
-	  /* PT_WRITE_GPR requires the buffer parameter to point to an 8-byte
-	     area, even if the register is really only 32 bits.  */
-	  long long buf;
-	  if (register_size (gdbarch, regno) == 8)
-	    memcpy (&buf, addr, 8);
-	  else
-	    buf = *addr;
-	  rs6000_ptrace64 (PT_WRITE_GPR, pid, nr, 0, &buf);
-	}
-    }
+         the register's value is passed by value, but for 64-bit inferiors,
+         the address of a buffer containing the value is passed.  */
 
-  if (errno)
-    {
-      perror (_("ptrace write"));
-      errno = 0;
+      /* PT_WRITE_GPR requires the buffer parameter to point to an 8-byte
+         area, even if the register is really only 32 bits.  */
+
+      rs6000_ptrace64 (PT_WRITE_GPR, pid, (long *)original_addr, 0, &big_end_buf);
     }
+    else
+      rs6000_ptrace64 (PT_WRITE_GPR, pid, (long *)original_addr, 0, addr64);
+  }
 }
 
 /* Read from the inferior all registers if REGNO == -1 and just register
-- 
2.31.1