[PATCH 3/4] x86/APX: correct .insn opcode space determination when REX2 is needed

[Jan Beulich <jbeulich@suse.com>]

In this case spaces 0f38 and 0f3a may not be put in place. To achieve
the intended effect, operand parsing (but not operand processing) needs
pulling ahead, so we know whether eGRP-s are in use.

--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -12849,13 +12849,43 @@ s_insn (int dummy ATTRIBUTE_UNUSED)
 	}
     }
 
+  /* Parse operands, if any, before evaluating encoding space.  */
+  if (*line == ',')
+    {
+      i.memshift = -1;
+
+      ptr = parse_operands (line + 1, &i386_mnemonics[MN__insn]);
+      this_operand = -1;
+      if (!ptr)
+	goto bad;
+      line = ptr;
+
+      if (!i.operands)
+	{
+	  as_bad (_("expecting operand after ','; got nothing"));
+	  goto done;
+	}
+
+      if (i.mem_operands > 1)
+	{
+	  as_bad (_("too many memory references for `%s'"),
+		  &i386_mnemonics[MN__insn]);
+	  goto done;
+	}
+
+      /* No need to distinguish encoding_evex and encoding_evex512.  */
+      if (i.encoding == encoding_evex512)
+	i.encoding = encoding_evex;
+    }
+
   /* Trim off encoding space.  */
   if (j > 1 && !i.insn_opcode_space && (val >> ((j - 1) * 8)) == 0x0f)
     {
       uint8_t byte = val >> ((--j - 1) * 8);
 
       i.insn_opcode_space = SPACE_0F;
-      switch (byte & -(j > 1))
+      switch (byte & -(j > 1 && !i.rex2_encoding
+		       && (i.encoding != encoding_egpr || evex)))
 	{
 	case 0x38:
 	  i.insn_opcode_space = SPACE_0F38;
@@ -12878,42 +12908,17 @@ s_insn (int dummy ATTRIBUTE_UNUSED)
   if (j > 2)
     {
       as_bad (_("opcode residual (%#"PRIx64") too wide"), (uint64_t) val);
-      goto bad;
+      goto done;
     }
   i.opcode_length = j;
 
   /* Handle operands, if any.  */
-  if (*line == ',')
+  if (i.operands)
     {
       i386_operand_type combined;
       expressionS *disp_exp = NULL;
       bool changed;
 
-      i.memshift = -1;
-
-      ptr = parse_operands (line + 1, &i386_mnemonics[MN__insn]);
-      this_operand = -1;
-      if (!ptr)
-	goto bad;
-      line = ptr;
-
-      if (!i.operands)
-	{
-	  as_bad (_("expecting operand after ','; got nothing"));
-	  goto done;
-	}
-
-      if (i.mem_operands > 1)
-	{
-	  as_bad (_("too many memory references for `%s'"),
-		  &i386_mnemonics[MN__insn]);
-	  goto done;
-	}
-
-      /* No need to distinguish encoding_evex and encoding_evex512.  */
-      if (i.encoding == encoding_evex512)
-	i.encoding = encoding_evex;
-
       if (i.encoding == encoding_egpr)
 	{
 	  if (vex || xop)
--- /dev/null
+++ b/gas/testsuite/gas/i386/insn-rex2.l
@@ -0,0 +1,38 @@
+[ 	]*[0-9]+[ 	]+\.text
+[ 	]*[0-9]+[ 	]+insn_rex2:
+[ 	]*[0-9]+ .... D58001C0[ 	]+\.insn \{rex2\} 0x0f01/0, %eax
+[ 	]*[0-9]+ .... D58038C0[ 	]+\.insn \{rex2\} 0x0f38/0, %eax
+[ 	]*[0-9]+ .... D5803801[ 	]+\.insn \{rex2\} 0x0f3801/0, %eax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5803901[ 	]+\.insn \{rex2\} 0x0f3901/0, %eax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5803A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %eax
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D58801C0[ 	]+\.insn \{rex2\} 0x0f01/0, %rax
+[ 	]*[0-9]+ .... D58838C0[ 	]+\.insn \{rex2\} 0x0f38/0, %rax
+[ 	]*[0-9]+ .... D5883801[ 	]+\.insn \{rex2\} 0x0f3801/0, %rax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5883901[ 	]+\.insn \{rex2\} 0x0f3901/0, %rax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5883A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %rax
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D58901C0[ 	]+\.insn \{rex2\} 0x0f01/0, %r8
+[ 	]*[0-9]+ .... D58938C0[ 	]+\.insn \{rex2\} 0x0f38/0, %r8
+[ 	]*[0-9]+ .... D5893801[ 	]+\.insn \{rex2\} 0x0f3801/0, %r8
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5893901[ 	]+\.insn \{rex2\} 0x0f3901/0, %r8
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5893A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %r8
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D59801C0[ 	]+\.insn 0x0f01/0, %r16
+[ 	]*[0-9]+ .... D59838C0[ 	]+\.insn 0x0f38/0, %r16
+[ 	]*[0-9]+ .... D5983801[ 	]+\.insn 0x0f3801/0, %r16
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5983901[ 	]+\.insn 0x0f3901/0, %r16
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5983A01[ 	]+\.insn 0x0f3a01/0, \$0xCC, %r16
+[ 	]*[0-9]+[ 	]+C0CC
+#pass
--- /dev/null
+++ b/gas/testsuite/gas/i386/insn-rex2.s
@@ -0,0 +1,25 @@
+	.text
+insn_rex2:
+	.insn {rex2} 0x0f01/0, %eax
+	.insn {rex2} 0x0f38/0, %eax
+	.insn {rex2} 0x0f3801/0, %eax
+	.insn {rex2} 0x0f3901/0, %eax
+	.insn {rex2} 0x0f3a01/0, $0xCC, %eax
+
+	.insn {rex2} 0x0f01/0, %rax
+	.insn {rex2} 0x0f38/0, %rax
+	.insn {rex2} 0x0f3801/0, %rax
+	.insn {rex2} 0x0f3901/0, %rax
+	.insn {rex2} 0x0f3a01/0, $0xCC, %rax
+
+	.insn {rex2} 0x0f01/0, %r8
+	.insn {rex2} 0x0f38/0, %r8
+	.insn {rex2} 0x0f3801/0, %r8
+	.insn {rex2} 0x0f3901/0, %r8
+	.insn {rex2} 0x0f3a01/0, $0xCC, %r8
+
+	.insn 0x0f01/0, %r16
+	.insn 0x0f38/0, %r16
+	.insn 0x0f3801/0, %r16
+	.insn 0x0f3901/0, %r16
+	.insn 0x0f3a01/0, $0xCC, %r16
--- a/gas/testsuite/gas/i386/x86-64.exp
+++ b/gas/testsuite/gas/i386/x86-64.exp
@@ -126,6 +126,7 @@ run_dump_test "x86-64-sysenter-mixed"
 run_dump_test "x86-64-sysenter-amd"
 run_list_test "x86-64-sysenter-amd" "-mamd64"
 run_dump_test "insn-64"
+run_list_test "insn-rex2" "-aln"
 run_dump_test "noreg64"
 run_list_test "noreg64"
 run_dump_test "noreg64-data16"