[PATCH] x86: Check VEX/EVEX encoding before checking vector operands

[PATCH] x86: Check VEX/EVEX encoding before checking vector operands

[H.J. Lu]

Since

commit 837e225ba1992f9745e5bbbd5e8443243a7f475f
Author: Jan Beulich <jbeulich@suse.com>
Date:   Thu Oct 20 10:01:12 2022 +0200

    x86: re-work AVX-VNNI support

moved AVX-VNNI after AVX512-VNNI, vector Disp8 is applied even when VEX
encoding is selected.  Check VEX/EVEX encoding before checking vector
operands to avoid vector Disp8 with VEX encoding.

	PR gas/29708
	* config/tc-i386.c (match_template): Check VEX/EVEX encoding
	before checking vector operands.
	* testsuite/gas/i386/avx-vnni.d: Updated.
	* testsuite/gas/i386/x86-64-avx-vnni.d: Likewise.
	* testsuite/gas/i386/avx-vnni.s: Add a Disp32 test.
	* testsuite/gas/i386/x86-64-avx-vnni.s: Likewise.
---
 gas/config/tc-i386.c                     | 8 ++++----
 gas/testsuite/gas/i386/avx-vnni.d        | 1 +
 gas/testsuite/gas/i386/avx-vnni.s        | 1 +
 gas/testsuite/gas/i386/x86-64-avx-vnni.d | 1 +
 gas/testsuite/gas/i386/x86-64-avx-vnni.s | 1 +
 5 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 526802ed308..02905c17021 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -6821,15 +6821,15 @@ match_template (char mnem_suffix)
 	     slip through to break.  */
 	}
 
-      /* Check if vector operands are valid.  */
-      if (check_VecOperands (t))
+      /* Check if VEX/EVEX encoding requirements can be satisfied.  */
+      if (VEX_check_encoding (t))
 	{
 	  specific_error = progress (i.error);
 	  continue;
 	}
 
-      /* Check if VEX/EVEX encoding requirements can be satisfied.  */
-      if (VEX_check_encoding (t))
+      /* Check if vector operands are valid.  */
+      if (check_VecOperands (t))
 	{
 	  specific_error = progress (i.error);
 	  continue;
diff --git a/gas/testsuite/gas/i386/avx-vnni.d b/gas/testsuite/gas/i386/avx-vnni.d
index 47e3118b1ce..4dc353e89fd 100644
--- a/gas/testsuite/gas/i386/avx-vnni.d
+++ b/gas/testsuite/gas/i386/avx-vnni.d
@@ -39,4 +39,5 @@ Disassembly of section .text:
  +[a-f0-9]+:	c4 e2 7d 50 c0       	\{vex\} vpdpbusd %ymm0,%ymm0,%ymm0
  +[a-f0-9]+:	c4 e2 79 50 c0       	\{vex\} vpdpbusd %xmm0,%xmm0,%xmm0
  +[a-f0-9]+:	62 f2 5d 08 50 d2    	vpdpbusd %xmm2,%xmm4,%xmm2
+ +[a-f0-9]+:	c4 e2 59 50 91 f0 07 00 00 	\{vex\} vpdpbusd 0x7f0\(%ecx\),%xmm4,%xmm2
 #pass
diff --git a/gas/testsuite/gas/i386/avx-vnni.s b/gas/testsuite/gas/i386/avx-vnni.s
index e2b793b24a6..6260330cca4 100644
--- a/gas/testsuite/gas/i386/avx-vnni.s
+++ b/gas/testsuite/gas/i386/avx-vnni.s
@@ -37,3 +37,4 @@ _start:
 	.arch default
 	.arch .avx_vnni
 	 vpdpbusd	%xmm2, %xmm4, %xmm2
+	{vex3} vpdpbusd 2032(%ecx), %xmm4, %xmm2
diff --git a/gas/testsuite/gas/i386/x86-64-avx-vnni.d b/gas/testsuite/gas/i386/x86-64-avx-vnni.d
index 6b3acab5d54..1fa08a5c64a 100644
--- a/gas/testsuite/gas/i386/x86-64-avx-vnni.d
+++ b/gas/testsuite/gas/i386/x86-64-avx-vnni.d
@@ -36,4 +36,5 @@ Disassembly of section .text:
  +[a-f0-9]+:	c4 e2 59 53 11       	\{vex\} vpdpwssds \(%rcx\),%xmm4,%xmm2
  +[a-f0-9]+:	62 b2 5d 08 53 d6    	vpdpwssds %xmm22,%xmm4,%xmm2
  +[a-f0-9]+:	62 d2 5d 08 50 d4    	vpdpbusd %xmm12,%xmm4,%xmm2
+ +[a-f0-9]+:	c4 e2 59 50 91 f0 07 00 00 	\{vex\} vpdpbusd 0x7f0\(%rcx\),%xmm4,%xmm2
 #pass
diff --git a/gas/testsuite/gas/i386/x86-64-avx-vnni.s b/gas/testsuite/gas/i386/x86-64-avx-vnni.s
index 95b6dc2ef33..9c57e6f6a97 100644
--- a/gas/testsuite/gas/i386/x86-64-avx-vnni.s
+++ b/gas/testsuite/gas/i386/x86-64-avx-vnni.s
@@ -19,3 +19,4 @@ _start:
 
 	.arch .avx_vnni
 	vpdpbusd	%xmm12, %xmm4, %xmm2
+	{vex3} vpdpbusd 2032(%rcx), %xmm4, %xmm2
-- 
2.37.3