[PATCH] x86-64: make "length_vex" also account for VEX.B use by register operand

[PATCH] x86-64: make "length_vex" also account for VEX.B use by register operand

[Jan Beulich]

The length attribute ought to be "the (bounding maximum) length of an
instruction" according to the comment next to its definition. A register
operand encoded using the ModR/M.rm field will additionally use VEX.B
for encoding the highest bit of the register number. Hence for the high
8 GPR registers as well as the [xy]mm{8..15} ones 3-byte VEX encoding
may be needed. Since it isn't known to the function calculating the
length which register goes where in the insn encoding, be conservative
and assume a 3-byte VEX prefix whenever any such register operand is
present and there's no memory operand.

gcc/

	* config/i386/i386.cc (ix86_attr_length_vex_default): Take REX.B
	into account for reg-only insns.

--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -16820,7 +16820,8 @@ int
 ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
 			      bool has_vex_w)
 {
-  int i;
+  int i, reg_only = 2 + 1;
+  bool has_mem = false;
 
   /* Only 0f opcode can use 2 byte VEX prefix and  VEX W bit uses 3
      byte VEX prefix.  */
@@ -16840,16 +16841,23 @@ ix86_attr_length_vex_default (rtx_insn *
 	if (GET_MODE (recog_data.operand[i]) == DImode
 	    && GENERAL_REG_P (recog_data.operand[i]))
 	  return 3 + 1;
+
+	/* REX.B bit requires 3-byte VEX. Right here we don't know which
+	   operand will be encoded using VEX.B, so be conservative.  */
+	if (REX_INT_REGNO_P (recog_data.operand[i])
+	    || REX_SSE_REGNO_P (recog_data.operand[i]))
+	  reg_only = 3 + 1;
       }
-    else
+    else if (MEM_P (recog_data.operand[i]))
       {
 	/* REX.X or REX.B bits use 3 byte VEX prefix.  */
-	if (MEM_P (recog_data.operand[i])
-	    && x86_extended_reg_mentioned_p (recog_data.operand[i]))
+	if (x86_extended_reg_mentioned_p (recog_data.operand[i]))
 	  return 3 + 1;
+
+	has_mem = true;
       }
 
-  return 2 + 1;
+  return has_mem ? 2 + 1 : reg_only;
 }
 \f
 

Re: [PATCH] x86-64: make "length_vex" also account for VEX.B use by register operand

[Uros Bizjak]

On Thu, Jun 2, 2022 at 5:11 PM Jan Beulich <jbeulich@suse.com> wrote:
>
> The length attribute ought to be "the (bounding maximum) length of an
> instruction" according to the comment next to its definition. A register
> operand encoded using the ModR/M.rm field will additionally use VEX.B
> for encoding the highest bit of the register number. Hence for the high
> 8 GPR registers as well as the [xy]mm{8..15} ones 3-byte VEX encoding
> may be needed. Since it isn't known to the function calculating the
> length which register goes where in the insn encoding, be conservative
> and assume a 3-byte VEX prefix whenever any such register operand is
> present and there's no memory operand.
>
> gcc/
>
>         * config/i386/i386.cc (ix86_attr_length_vex_default): Take REX.B
>         into account for reg-only insns.

LGTM.

Thanks,
Uros.

> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -16820,7 +16820,8 @@ int
>  ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
>                               bool has_vex_w)
>  {
> -  int i;
> +  int i, reg_only = 2 + 1;
> +  bool has_mem = false;
>
>    /* Only 0f opcode can use 2 byte VEX prefix and  VEX W bit uses 3
>       byte VEX prefix.  */
> @@ -16840,16 +16841,23 @@ ix86_attr_length_vex_default (rtx_insn *
>         if (GET_MODE (recog_data.operand[i]) == DImode
>             && GENERAL_REG_P (recog_data.operand[i]))
>           return 3 + 1;
> +
> +       /* REX.B bit requires 3-byte VEX. Right here we don't know which
> +          operand will be encoded using VEX.B, so be conservative.  */
> +       if (REX_INT_REGNO_P (recog_data.operand[i])
> +           || REX_SSE_REGNO_P (recog_data.operand[i]))
> +         reg_only = 3 + 1;
>        }
> -    else
> +    else if (MEM_P (recog_data.operand[i]))
>        {
>         /* REX.X or REX.B bits use 3 byte VEX prefix.  */
> -       if (MEM_P (recog_data.operand[i])
> -           && x86_extended_reg_mentioned_p (recog_data.operand[i]))
> +       if (x86_extended_reg_mentioned_p (recog_data.operand[i]))
>           return 3 + 1;
> +
> +       has_mem = true;
>        }
>
> -  return 2 + 1;
> +  return has_mem ? 2 + 1 : reg_only;
>  }
>
>
>