[binutils-gdb] x86: flag EVEX masking when destination is GPR(-like)

[Jan Beulich <jbeulich@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=37c5374291224ed7e061fe001094cdacac7d9821

commit 37c5374291224ed7e061fe001094cdacac7d9821
Author: Jan Beulich <jbeulich@suse.com>
Date:   Tue Jul 4 17:01:56 2023 +0200

    x86: flag EVEX masking when destination is GPR(-like)
    
    Masking is not permitted in this case. See the code comment for how this
    is being dealt with.
    
    To avoid excess special casing of modes, have OP_M() call OP_E_memory()
    directly.

Diff:
---
 gas/testsuite/gas/i386/avx512f-nondef.d |  2 ++
 gas/testsuite/gas/i386/avx512f-nondef.s |  6 ++++++
 opcodes/i386-dis.c                      | 17 ++++++++++++++++-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/gas/testsuite/gas/i386/avx512f-nondef.d b/gas/testsuite/gas/i386/avx512f-nondef.d
index 9c1d3272669..aa7d292151e 100644
--- a/gas/testsuite/gas/i386/avx512f-nondef.d
+++ b/gas/testsuite/gas/i386/avx512f-nondef.d
@@ -22,4 +22,6 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:	62 f2 7d 48 92 04 08 	vgatherdps \(%eax,%zmm1(,1)?\),%zmm0/\(bad\)
 [ 	]*[a-f0-9]+:	62 f1 7c cf c2 c0 00 	vcmpeqps %zmm0,%zmm0,%k0\{%k7\}\{z\}/\(bad\)
 [ 	]*[a-f0-9]+:	62 f1 7c cf 29 00    	vmovaps %zmm0,\(%eax\)\{%k7\}\{z\}/\(bad\)
+[ 	]*[a-f0-9]+:	62 f1 7d 0a c5 c8 00 	vpextrw \$(0x)?0,%xmm0,%ecx\{%k2\}/\(bad\)
+[ 	]*[a-f0-9]+:	62 f3 7d 0a 16 01 00 	vpextrd \$(0x)?0,%xmm0,\(%ecx\)\{%k2\}/\(bad\)
 #pass
diff --git a/gas/testsuite/gas/i386/avx512f-nondef.s b/gas/testsuite/gas/i386/avx512f-nondef.s
index 77426b1b7f5..39ba23bcbe9 100644
--- a/gas/testsuite/gas/i386/avx512f-nondef.s
+++ b/gas/testsuite/gas/i386/avx512f-nondef.s
@@ -32,3 +32,9 @@
 
 	# vmovaps %zmm0, (%eax){%k7} with EVEX.z set
 	.insn EVEX.0f 0x29, %zmm0, (%eax){%k7}{z}
+
+	# vpextrw $0, %xmm0, %ecx with non-zero EVEX.aaa
+	.insn EVEX.66.0f 0xc5, $0, %xmm0, %ecx{%k2}
+
+	# vpextrd $0, %xmm0, (%ecx) with non-zero EVEX.aaa
+	.insn EVEX.66.0f3a 0x16, $0, %xmm0, (%ecx){%k2}
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 7c4130245e1..d5e3547be84 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -11641,6 +11641,11 @@ print_register (instr_info *ins, unsigned int reg, unsigned int rexmask,
 {
   const char (*names)[8];
 
+  /* Masking is invalid for insns with GPR destination. Set the flag uniformly,
+     as the consumer will inspect it only for the destination operand.  */
+  if (bytemode != mask_mode && ins->vex.mask_register_specifier)
+    ins->illegal_masking = true;
+
   USED_REX (rexmask);
   if (ins->rex & rexmask)
     reg += 8;
@@ -12375,6 +12380,12 @@ OP_E (instr_info *ins, int bytemode, int sizeflag)
       return true;
     }
 
+  /* Masking is invalid for insns with GPR-like memory destination. Set the
+     flag uniformly, as the consumer will inspect it only for the destination
+     operand.  */
+  if (ins->vex.mask_register_specifier)
+    ins->illegal_masking = true;
+
   return OP_E_memory (ins, bytemode, sizeflag);
 }
 
@@ -13157,10 +13168,14 @@ OP_XS (instr_info *ins, int bytemode, int sizeflag)
 static bool
 OP_M (instr_info *ins, int bytemode, int sizeflag)
 {
+  /* Skip mod/rm byte.  */
+  MODRM_CHECK;
+  ins->codep++;
+
   if (ins->modrm.mod == 3)
     /* bad bound,lea,lds,les,lfs,lgs,lss,cmpxchg8b,vmptrst modrm */
     return BadOp (ins);
-  return OP_E (ins, bytemode, sizeflag);
+  return OP_E_memory (ins, bytemode, sizeflag);
 }
 
 static bool