[PATCH v5 4/7] gdb/record: add support to AVX unpack instructions

[Guinevere Larsen <guinevere@redhat.com>]

This commit adds support to recording instructions to unpack high
or low data from XMM registers, identified by the mnemonics in the
form: VPUNPCK [L|H] [BW|WD|DQ|QDQ].
All these instructions are encoded the exact same way, and only affect
the destination register, making them trivial to implement together.

It also updates the test gdb.reverse/i386-avx-reverse.exp to test these
new instructions.  The test always uses ymm because the vpunpck
instructions overwrite the high bits, so we have to be able to record
the full ymm register, not just the output size.
---
 gdb/i386-tdep.c                               | 16 ++++++
 gdb/testsuite/gdb.reverse/i386-avx-reverse.c  | 55 +++++++++++++++++++
 .../gdb.reverse/i386-avx-reverse.exp          | 51 ++++++++++++++++-
 3 files changed, 121 insertions(+), 1 deletion(-)

diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index ef1c848b1a2..a0e0181a2c5 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -4886,6 +4886,22 @@ i386_record_vex (struct i386_record_s *ir, uint8_t vex_w, uint8_t vex_r,
 	}
       break;
 
+    case 0x60:	/* VPUNPCKLBW  */
+    case 0x61:	/* VPUNPCKLWD  */
+    case 0x62:	/* VPUNPCKLDQ  */
+    case 0x6c:	/* VPUNPCKLQDQ */
+    case 0x68:	/* VPUNPCKHBW  */
+    case 0x69:	/* VPUNPCKHWD  */
+    case 0x6a:	/* VPUNPCKHDQ  */
+    case 0x6d:	/* VPUNPCKHQDQ */
+      {
+	i386_record_modrm (ir);
+	int reg_offset = ir->reg + vex_r * 8;
+	record_full_arch_list_add_reg (ir->regcache,
+				       tdep->ymm0_regnum + reg_offset);
+      }
+      break;
+
     default:
       gdb_printf (gdb_stderr,
 		  _("Process record does not support VEX instruction 0x%02x "
diff --git a/gdb/testsuite/gdb.reverse/i386-avx-reverse.c b/gdb/testsuite/gdb.reverse/i386-avx-reverse.c
index fd1c68ae378..c897436995e 100644
--- a/gdb/testsuite/gdb.reverse/i386-avx-reverse.c
+++ b/gdb/testsuite/gdb.reverse/i386-avx-reverse.c
@@ -78,6 +78,58 @@ vmov_test ()
   return 0; /* end vmov_test */
 }
 
+/* Test if we can properly record (and undo) vpunpck style instructions.
+   Most tests will use xmm0 and xmm1 as sources. The registers xmm15 and xmm2
+   are used as destination to ensure we're reading the VEX.R bit correctly.  */
+int
+vpunpck_test  ()
+{
+  /* Using GDB, load these values onto registers, for ease of testing.
+     ymm0.v2_int128  = {0x1f1e1d1c1b1a19181716151413121110, 0x2f2e2d2c2b2a29282726252423222120}
+     ymm1.v2_int128  = {0x4f4e4d4c4b4a49484746454443424140, 0x3f3e3d3c3b3a39383736353433323130}
+     ymm15.v2_int128 = {0xdead, 0xbeef}
+     so that's easy to confirm that the unpacking went as expected.  */
+
+  /* start vpunpck_test.  */
+
+  /* First try all low bit unpack instructions with xmm registers.  */
+  /* 17 27 16 26 15 25 14 24 ...*/
+  asm volatile ("vpunpcklbw  %xmm0, %xmm1, %xmm15");
+  /* 17 16 27 26 15 14 25 24 ...*/
+  asm volatile ("vpunpcklwd  %0, %%xmm1, %%xmm15"
+      : : "m" (global_buf0));
+  /* 17 16 15 14 27 26 25 24 ...*/
+  asm volatile ("vpunpckldq  %0, %%xmm1,  %%xmm2"
+      : : "m" (global_buf0));
+  /* 17 16 15 14 13 12 11 10 ...*/
+  asm volatile ("vpunpcklqdq %xmm0, %xmm1, %xmm2");
+
+  /* Then try all high bit unpack instructions with xmm registers.  */
+  /* 17 27 16 26 15 25 14 24 ...*/
+  asm volatile ("vpunpckhbw  %xmm0, %xmm1, %xmm15");
+  /* 17 16 27 26 15 14 25 24 ...*/
+  asm volatile ("vpunpckhwd  %0, %%xmm1, %%xmm15"
+      : : "m" (global_buf0));
+  /* 17 16 15 14 27 26 25 24 ...*/
+  asm volatile ("vpunpckhdq  %0, %%xmm1,  %%xmm2"
+      : : "m" (global_buf0));
+  /* 17 16 15 14 13 12 11 10 ...*/
+  asm volatile ("vpunpckhqdq %xmm0, %xmm1, %xmm2");
+
+  /* Lastly, lets test a few unpack instructions with ymm registers.  */
+  /* 17 27 16 26 15 25 14 24 ...*/
+  asm volatile ("vpunpcklbw  %ymm0, %ymm1, %ymm15");
+  /* 17 16 27 26 15 14 25 24 ...*/
+  asm volatile ("vpunpcklwd  %ymm0, %ymm1, %ymm15");
+  /* 17 16 15 14 27 26 25 24 ...*/
+  asm volatile ("vpunpckhdq  %ymm0, %ymm1,  %ymm15");
+  /* 17 16 15 14 13 12 11 10 ...*/
+  asm volatile ("vpunpckhqdq %ymm0, %ymm1, %ymm15");
+  /* We have a return statement to deal with
+     epilogue in different compilers.  */
+  return 0; /* end vpunpck_test */
+}
+
 int
 main ()
 {
@@ -90,8 +142,11 @@ main ()
     }
   /* Zero relevant xmm registers, se we know what to look for.  */
   asm volatile ("vmovq %0, %%xmm0": : "m" (global_buf1));
+  asm volatile ("vmovq %0, %%xmm1": : "m" (global_buf1));
+  asm volatile ("vmovq %0, %%xmm2": : "m" (global_buf1));
   asm volatile ("vmovq %0, %%xmm15": : "m" (global_buf1));
 
   vmov_test ();
+  vpunpck_test ();
   return 0;	/* end of main */
 }
diff --git a/gdb/testsuite/gdb.reverse/i386-avx-reverse.exp b/gdb/testsuite/gdb.reverse/i386-avx-reverse.exp
index 65e982efc63..718dca3429a 100644
--- a/gdb/testsuite/gdb.reverse/i386-avx-reverse.exp
+++ b/gdb/testsuite/gdb.reverse/i386-avx-reverse.exp
@@ -53,8 +53,12 @@ proc test_one_register {insn register value {prefix ""}} {
     gdb_test "reverse-step" "$insn.*" \
 	"${prefix}reverse-step from $insn to test register $register"
 
+    if {[regexp {^ymm} $register]} {
+	set value "\\\{$value\\\}"
+    }
+
     gdb_test "info register $register" \
-	"$register.*uint128 = $value.*" \
+	"$register.*int128 = $value.*" \
 	"${prefix}verify $register before $insn"
 }
 
@@ -188,3 +192,48 @@ if {[record_full_function "vmov"] == true} {
 
 # Move to the end of vmov_test to set up next.
 gdb_test "finish" "Run till exit from.*vmov_test.*" "leaving vmov_test"
+
+# Starting vpunpck tests.
+gdb_test_no_output \
+    "set \$ymm0.v4_int64 = {0x1716151413121110, 0x1f1e1d1c1b1a1918, 0x2726252423222120, 0x2f2e2d2c2b2a2928}"
+gdb_test_no_output \
+    "set \$ymm1.v4_int64 = {0x3736353433323130, 0x3f3e3d3c3b3a3938, 0x4746454443424140, 0x4f4e4d4c4b4a4948}"
+gdb_test_no_output "set \$ymm2.v2_int128 = {0x0, 0x0}"
+gdb_test_no_output "set \$ymm15.v2_int128 = {0xdead, 0xbeef}"
+if {[record_full_function "vpunpck"] == true} {
+    test_one_register "vpunpckhqdq" "ymm15" \
+	"0x1f1e1d1c3f3e3d3c1b1a19183b3a3938, 0x2f2e2d2c4f4e4d4c2b2a29284b4a4948" \
+	"ymm: "
+    test_one_register "vpunpckhdq" "ymm15" \
+	"0x17163736151435341312333211103130, 0x27264746252445442322434221204140" \
+	"ymm: "
+    test_one_register "vpunpcklwd" "ymm15" \
+	"0x17371636153514341333123211311030, 0x27472646254524442343224221412040" \
+	"ymm: "
+    test_one_register "vpunpcklbw" "ymm15" \
+	"0x1f1e3f3e1d1c3d3c1b1a3b3a19183938, 0x0" "ymm: "
+
+    test_one_register "vpunpckhqdq" "ymm2" \
+	"0x1f1e1d1c3f3e3d3c1b1a19183b3a3938, 0x0"
+    test_one_register "vpunpckhdq" "ymm2" \
+	"0x17161514131211103736353433323130, 0x0"
+    test_one_register "vpunpckhwd" "ymm15" \
+	"0x1f3f1e3e1d3d1c3c1b3b1a3a19391838, 0x0"
+    test_one_register "vpunpckhbw" "ymm15" \
+	"0x17163736151435341312333211103130, 0x0"
+
+    test_one_register "vpunpcklqdq" "ymm2" \
+	"0x17161514373635341312111033323130, 0x0"
+    test_one_register "vpunpckldq" "ymm2" "0x0, 0x0"
+    test_one_register "vpunpcklwd" "ymm15" \
+	"0x17371636153514341333123211311030, 0x0"
+    test_one_register "vpunpcklbw" "ymm15" "0xdead, 0xbeef"
+} else {
+    untested "couldn't test vpunpck tests"
+}
+
+# Move to the end of vmov_test to set up next.
+# Stop recording in case of recording errors.
+gdb_test "record stop" "Process record is stopped.*" \
+    "delete history for vpunpck_test"
+gdb_test "finish" "Run till exit from.*vpunpck_test.*" "leaving vpunpck_test"
-- 
2.47.0