[PATCH 0/3] x86: a little bit of disassembler tidying

[PATCH 0/3] x86: a little bit of disassembler tidying

[Jan Beulich]

1: use fixed-width type for codep and friends
2: disassembling over-long insns
3: convert two pointers to (indexing) integers

Jan

[PATCH 1/3] x86: use fixed-width type for codep and friends

[Jan Beulich]

This first of all removes a dependency on bfd_byte and unsigned char
being the same types. It further eliminates the need to mask by 0xff
when fetching values (which wasn't done fully consistently anyway),
improving code legibility.

While there, where possible add const.

--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -120,8 +120,8 @@ static void ATTRIBUTE_PRINTF_3 i386_dis_
 
 struct dis_private {
   /* Points to first byte not fetched.  */
-  bfd_byte *max_fetched;
-  bfd_byte the_buffer[MAX_MNEM_SIZE];
+  uint8_t *max_fetched;
+  uint8_t the_buffer[MAX_MNEM_SIZE];
   bfd_vma insn_start;
   int orig_sizeflag;
 };
@@ -133,7 +133,7 @@ enum address_mode
   mode_64bit
 };
 
-static const char *prefix_name (enum address_mode, int, int);
+static const char *prefix_name (enum address_mode, uint8_t, int);
 
 enum x86_64_isa
 {
@@ -149,9 +149,9 @@ struct instr_info
   int prefixes;
 
   /* REX prefix the current instruction.  See below.  */
-  unsigned char rex;
+  uint8_t rex;
   /* Bits of REX we've already used.  */
-  unsigned char rex_used;
+  uint8_t rex_used;
 
   bool need_modrm;
   bool need_vex;
@@ -168,10 +168,10 @@ struct instr_info
   char obuf[MAX_OPERAND_BUFFER_SIZE];
   char *obufp;
   char *mnemonicendp;
-  unsigned char *start_codep;
-  unsigned char *insn_codep;
-  unsigned char *codep;
-  unsigned char *end_codep;
+  const uint8_t *start_codep;
+  uint8_t *insn_codep;
+  uint8_t *codep;
+  const uint8_t *end_codep;
   signed char last_lock_prefix;
   signed char last_repz_prefix;
   signed char last_repnz_prefix;
@@ -186,7 +186,7 @@ struct instr_info
 #define MAX_CODE_LENGTH 15
   /* We can up to 14 ins->prefixes since the maximum instruction length is
      15bytes.  */
-  unsigned char all_prefixes[MAX_CODE_LENGTH - 1];
+  uint8_t all_prefixes[MAX_CODE_LENGTH - 1];
   disassemble_info *info;
 
   struct
@@ -288,7 +288,7 @@ struct instr_info
    to ADDR (exclusive) are valid.  Returns true for success, false
    on error.  */
 static bool
-fetch_code (struct disassemble_info *info, bfd_byte *until)
+fetch_code (struct disassemble_info *info, uint8_t *until)
 {
   int status = -1;
   struct dis_private *priv = info->private_data;
@@ -8794,7 +8794,8 @@ static enum {
 }
 ckprefix (instr_info *ins)
 {
-  int newrex, i, length;
+  int i, length;
+  uint8_t newrex;
 
   i = 0;
   length = 0;
@@ -8804,7 +8805,7 @@ ckprefix (instr_info *ins)
       if (!fetch_code (ins->info, ins->codep + 1))
 	return ckp_fetch_error;
       newrex = 0;
-      switch (*ins->codep & 0xff)
+      switch (*ins->codep)
 	{
 	/* REX prefixes family.  */
 	case 0x40:
@@ -8824,7 +8825,7 @@ ckprefix (instr_info *ins)
 	case 0x4e:
 	case 0x4f:
 	  if (ins->address_mode == mode_64bit)
-	    newrex = *ins->codep & 0xff;
+	    newrex = *ins->codep;
 	  else
 	    return ckp_okay;
 	  ins->last_rex_prefix = i;
@@ -8904,8 +8905,8 @@ ckprefix (instr_info *ins)
       /* Rex is ignored when followed by another prefix.  */
       if (ins->rex)
 	return ckp_bogus;
-      if ((*ins->codep & 0xff) != FWAIT_OPCODE)
-	ins->all_prefixes[i++] = *ins->codep & 0xff;
+      if (*ins->codep != FWAIT_OPCODE)
+	ins->all_prefixes[i++] = *ins->codep;
       ins->rex = newrex;
       ins->codep++;
       length++;
@@ -8917,7 +8918,7 @@ ckprefix (instr_info *ins)
    prefix byte.  */
 
 static const char *
-prefix_name (enum address_mode mode, int pref, int sizeflag)
+prefix_name (enum address_mode mode, uint8_t pref, int sizeflag)
 {
   static const char *rexes [16] =
     {
@@ -9136,7 +9137,7 @@ get_valid_dis386 (const struct dis386 *d
     case USE_3BYTE_TABLE:
       if (!fetch_code (ins->info, ins->codep + 2))
 	return &err_opcode;
-      vindex = *ins->codep++ & 0xff;
+      vindex = *ins->codep++;
       dp = &three_byte_table[dp->op[1].bytemode][vindex];
       ins->end_codep = ins->codep;
       if (!fetch_modrm (ins))
@@ -9243,7 +9244,7 @@ get_valid_dis386 (const struct dis386 *d
 	}
       ins->need_vex = true;
       ins->codep++;
-      vindex = *ins->codep++ & 0xff;
+      vindex = *ins->codep++;
       dp = &xop_table[vex_table_index][vindex];
 
       ins->end_codep = ins->codep;
@@ -9308,7 +9309,7 @@ get_valid_dis386 (const struct dis386 *d
 	}
       ins->need_vex = true;
       ins->codep++;
-      vindex = *ins->codep++ & 0xff;
+      vindex = *ins->codep++;
       dp = &vex_table[vex_table_index][vindex];
       ins->end_codep = ins->codep;
       /* There is no MODRM byte for VEX0F 77.  */
@@ -9343,7 +9344,7 @@ get_valid_dis386 (const struct dis386 *d
 	}
       ins->need_vex = true;
       ins->codep++;
-      vindex = *ins->codep++ & 0xff;
+      vindex = *ins->codep++;
       dp = &vex_table[dp->op[1].bytemode][vindex];
       ins->end_codep = ins->codep;
       /* There is no MODRM byte for VEX 77.  */
@@ -9435,7 +9436,7 @@ get_valid_dis386 (const struct dis386 *d
 
       ins->need_vex = true;
       ins->codep++;
-      vindex = *ins->codep++ & 0xff;
+      vindex = *ins->codep++;
       dp = &evex_table[vex_table_index][vindex];
       ins->end_codep = ins->codep;
       if (!fetch_modrm (ins))
@@ -9782,11 +9783,10 @@ print_insn (bfd_vma pc, disassemble_info
       goto out;
     }
 
-  ins.two_source_ops = ((*ins.codep & 0xff) == 0x62
-			|| (*ins.codep & 0xff) == 0xc8);
+  ins.two_source_ops = (*ins.codep == 0x62 || *ins.codep == 0xc8);
 
   if ((ins.prefixes & PREFIX_FWAIT)
-      && ((*ins.codep & 0xff) < 0xd8 || (*ins.codep & 0xff) > 0xdf))
+      && (*ins.codep < 0xd8 || *ins.codep > 0xdf))
     {
       /* Handle ins.prefixes before fwait.  */
       for (i = 0; i < ins.fwait_prefix && ins.all_prefixes[i];
@@ -9799,22 +9799,22 @@ print_insn (bfd_vma pc, disassemble_info
       goto out;
     }
 
-  if ((*ins.codep & 0xff) == 0x0f)
+  if (*ins.codep == 0x0f)
     {
       unsigned char threebyte;
 
       ins.codep++;
       if (!fetch_code (info, ins.codep + 1))
 	goto fetch_error_out;
-      threebyte = *ins.codep & 0xff;
+      threebyte = *ins.codep;
       dp = &dis386_twobyte[threebyte];
       ins.need_modrm = twobyte_has_modrm[threebyte];
       ins.codep++;
     }
   else
     {
-      dp = &dis386[*ins.codep & 0xff];
-      ins.need_modrm = onebyte_has_modrm[*ins.codep & 0xff];
+      dp = &dis386[*ins.codep];
+      ins.need_modrm = onebyte_has_modrm[*ins.codep];
       ins.codep++;
     }
 
@@ -10514,9 +10514,7 @@ static bool
 dofloat (instr_info *ins, int sizeflag)
 {
   const struct dis386 *dp;
-  unsigned char floatop;
-
-  floatop = ins->codep[-1] & 0xff;
+  unsigned char floatop = ins->codep[-1];
 
   if (ins->modrm.mod != 3)
     {
@@ -10537,7 +10535,7 @@ dofloat (instr_info *ins, int sizeflag)
       putop (ins, fgrps[dp->op[0].bytemode][ins->modrm.rm], sizeflag);
 
       /* Instruction fnstsw is only one with strange arg.  */
-      if (floatop == 0xdf && (ins->codep[-1] & 0xff) == 0xe0)
+      if (floatop == 0xdf && ins->codep[-1] == 0xe0)
 	strcpy (ins->op_out[0], att_names16[0] + ins->intel_syntax);
     }
   else
@@ -11710,7 +11708,7 @@ get8s (instr_info *ins, bfd_vma *res)
 {
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  *res = (((bfd_vma) *ins->codep++ & 0xff) ^ 0x80) - 0x80;
+  *res = ((bfd_vma) *ins->codep++ ^ 0x80) - 0x80;
   return true;
 }
 
@@ -11719,8 +11717,8 @@ get16 (instr_info *ins, bfd_vma *res)
 {
   if (!fetch_code (ins->info, ins->codep + 2))
     return false;
-  *res = (bfd_vma) *ins->codep++ & 0xff;
-  *res |= ((bfd_vma) *ins->codep++ & 0xff) << 8;
+  *res = *ins->codep++;
+  *res |= (bfd_vma) *ins->codep++ << 8;
   return true;
 }
 
@@ -11738,10 +11736,10 @@ get32 (instr_info *ins, bfd_vma *res)
 {
   if (!fetch_code (ins->info, ins->codep + 4))
     return false;
-  *res = *ins->codep++ & (bfd_vma) 0xff;
-  *res |= (*ins->codep++ & (bfd_vma) 0xff) << 8;
-  *res |= (*ins->codep++ & (bfd_vma) 0xff) << 16;
-  *res |= (*ins->codep++ & (bfd_vma) 0xff) << 24;
+  *res = *ins->codep++;
+  *res |= (bfd_vma) *ins->codep++ << 8;
+  *res |= (bfd_vma) *ins->codep++ << 16;
+  *res |= (bfd_vma) *ins->codep++ << 24;
   return true;
 }
 
@@ -11764,14 +11762,14 @@ get64 (instr_info *ins, uint64_t *res)
 
   if (!fetch_code (ins->info, ins->codep + 8))
     return false;
-  a = *ins->codep++ & 0xff;
-  a |= (*ins->codep++ & 0xff) << 8;
-  a |= (*ins->codep++ & 0xff) << 16;
-  a |= (*ins->codep++ & 0xffu) << 24;
-  b = *ins->codep++ & 0xff;
-  b |= (*ins->codep++ & 0xff) << 8;
-  b |= (*ins->codep++ & 0xff) << 16;
-  b |= (*ins->codep++ & 0xffu) << 24;
+  a = *ins->codep++;
+  a |= (unsigned int) *ins->codep++ << 8;
+  a |= (unsigned int) *ins->codep++ << 16;
+  a |= (unsigned int) *ins->codep++ << 24;
+  b = *ins->codep++;
+  b |= (unsigned int) *ins->codep++ << 8;
+  b |= (unsigned int) *ins->codep++ << 16;
+  b |= (unsigned int) *ins->codep++ << 24;
   *res = a + ((uint64_t) b << 32);
   return true;
 }
@@ -12450,7 +12448,7 @@ OP_I (instr_info *ins, int bytemode, int
     case b_mode:
       if (!fetch_code (ins->info, ins->codep + 1))
 	return false;
-      op = *ins->codep++ & 0xff;
+      op = *ins->codep++;
       break;
     case v_mode:
       USED_REX (REX_W);
@@ -12737,7 +12735,7 @@ OP_ESreg (instr_info *ins, int code, int
 {
   if (ins->intel_syntax)
     {
-      switch (ins->codep[-1] & 0xff)
+      switch (ins->codep[-1])
 	{
 	case 0x6d:	/* insw/insl */
 	  intel_operand_size (ins, z_mode, sizeflag);
@@ -12763,7 +12761,7 @@ OP_DSreg (instr_info *ins, int code, int
 {
   if (ins->intel_syntax)
     {
-      switch (ins->codep[-1] & 0xff)
+      switch (ins->codep[-1])
 	{
 	case 0x6f:	/* outsw/outsl */
 	  intel_operand_size (ins, z_mode, sizeflag);
@@ -13209,7 +13207,7 @@ OP_3DNowSuffix (instr_info *ins, int byt
      place where an 8-bit immediate would normally go.  ie. the last
      byte of the instruction.  */
   ins->obufp = ins->mnemonicendp;
-  mnemonic = Suffix3DNow[*ins->codep++ & 0xff];
+  mnemonic = Suffix3DNow[*ins->codep++];
   if (mnemonic)
     ins->obufp = stpcpy (ins->obufp, mnemonic);
   else
@@ -13274,7 +13272,7 @@ CMP_Fixup (instr_info *ins, int bytemode
 
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  cmp_type = *ins->codep++ & 0xff;
+  cmp_type = *ins->codep++;
   if (cmp_type < ARRAY_SIZE (simd_cmp_op))
     {
       char suffix[3];
@@ -13724,7 +13722,7 @@ OP_REG_VexI4 (instr_info *ins, int bytem
 
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  reg = *ins->codep++ & 0xff;
+  reg = *ins->codep++;
 
   if (bytemode != x_mode && bytemode != scalar_mode)
     abort ();
@@ -13768,7 +13766,7 @@ VPCMP_Fixup (instr_info *ins, int bytemo
 
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  cmp_type = *ins->codep++ & 0xff;
+  cmp_type = *ins->codep++;
   /* There are aliases for immediates 0, 1, 2, 4, 5, 6.
      If it's the case, print suffix, otherwise - print the immediate.  */
   if (cmp_type < ARRAY_SIZE (simd_cmp_op)
@@ -13823,7 +13821,7 @@ VPCOM_Fixup (instr_info *ins, int bytemo
 
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  cmp_type = *ins->codep++ & 0xff;
+  cmp_type = *ins->codep++;
   if (cmp_type < ARRAY_SIZE (xop_cmp_op))
     {
       char suffix[3];
@@ -13870,7 +13868,7 @@ PCLMUL_Fixup (instr_info *ins, int bytem
 
   if (!fetch_code (ins->info, ins->codep + 1))
     return false;
-  pclmul_type = *ins->codep++ & 0xff;
+  pclmul_type = *ins->codep++;
   switch (pclmul_type)
     {
     case 0x10:

[PATCH 2/3] x86: disassembling over-long insns

[Jan Beulich]

The present way of dealing with them - misusing MAX_MNEM_SIZE, which has
nothing to do with insn length - leads to inconsistent results. Since we
allow for up to MAX_CODE_LENGTH - 1 prefix bytes (which then could be
followed by another MAX_CODE_LENGTH "normal" insn bytes until we're done
decoding), size the_buffer[] accordingly.

Move struct dis_private down to be able to use MAX_CODE_LENGTH without
moving its #define. While doing this also alter the order to have the
potentially large array last.
---
Originally I meant to use just MAX_CODE_LENGTH as the buffer size. But
that broke the four "long insn" testcases, which I thought might make
sense to keep intact. Nevertheless that's certainly another way to get
things into consistent shape. It would further allow to drop the check
associated with a "Check maximum code length" comment, as then we would
never fetch more than this many bytes.

Strictly speaking what can come past suffixes is a little less than
MAX_CODE_LENGTH: Right now it could be EVEX (4 bytes), opcode (1),
ModR/M (<=1), SIB (<=1), displacement (<=4), and immediate (<=4); the
8-byte displacement and 8-byte immediate cases have, respectively, no
immediate or no displacement, and hence disp+imm <= 8 in all cases.
While this sums up to 15, there are no EVEX insns at present with 32-bit
immediate. The longest prefix-less ones currently known are those in the
oversized{16,64} gas testcases (sans their prefixes, of course), i.e. 14
bytes. But I think it is better to cover the potential case of EVEX-
with-32-bit-immediate case right away.

--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -118,14 +118,6 @@ static void ATTRIBUTE_PRINTF_3 i386_dis_
 /* The maximum operand buffer size.  */
 #define MAX_OPERAND_BUFFER_SIZE 128
 
-struct dis_private {
-  /* Points to first byte not fetched.  */
-  uint8_t *max_fetched;
-  uint8_t the_buffer[MAX_MNEM_SIZE];
-  bfd_vma insn_start;
-  int orig_sizeflag;
-};
-
 enum address_mode
 {
   mode_16bit,
@@ -251,6 +243,15 @@ struct instr_info
   enum x86_64_isa isa64;
 };
 
+struct dis_private {
+  bfd_vma insn_start;
+  int orig_sizeflag;
+
+  /* Points to first byte not fetched.  */
+  uint8_t *max_fetched;
+  uint8_t the_buffer[2 * MAX_CODE_LENGTH - 1];
+};
+
 /* Mark parts used in the REX prefix.  When we are testing for
    empty prefix (for 8bit register REX extension), just mask it
    out.  Otherwise test for REX bit is excuse for existence of REX
@@ -297,7 +298,7 @@ fetch_code (struct disassemble_info *inf
   if (until <= priv->max_fetched)
     return true;
 
-  if (until <= priv->the_buffer + MAX_MNEM_SIZE)
+  if (until <= priv->the_buffer + ARRAY_SIZE (priv->the_buffer))
     status = (*info->read_memory_func) (start,
 					priv->max_fetched,
 					until - priv->max_fetched,

[PATCH 3/3] x86: convert two pointers to (indexing) integers

[Jan Beulich]

This in particular reduces the number of pointers to non-const that we
have (and that could potentially be used for undue modification of
state). As a result, fetch_code()'s 2nd parameter can then also become
pointer-to-const.

--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -161,9 +161,9 @@ struct instr_info
   char *obufp;
   char *mnemonicendp;
   const uint8_t *start_codep;
-  uint8_t *insn_codep;
   uint8_t *codep;
   const uint8_t *end_codep;
+  unsigned char nr_prefixes;
   signed char last_lock_prefix;
   signed char last_repz_prefix;
   signed char last_repnz_prefix;
@@ -247,8 +247,8 @@ struct dis_private {
   bfd_vma insn_start;
   int orig_sizeflag;
 
-  /* Points to first byte not fetched.  */
-  uint8_t *max_fetched;
+  /* Indexes first byte not fetched.  */
+  unsigned int fetched;
   uint8_t the_buffer[2 * MAX_CODE_LENGTH - 1];
 };
 
@@ -289,32 +289,31 @@ struct dis_private {
    to ADDR (exclusive) are valid.  Returns true for success, false
    on error.  */
 static bool
-fetch_code (struct disassemble_info *info, uint8_t *until)
+fetch_code (struct disassemble_info *info, const uint8_t *until)
 {
   int status = -1;
   struct dis_private *priv = info->private_data;
-  bfd_vma start = priv->insn_start + (priv->max_fetched - priv->the_buffer);
+  bfd_vma start = priv->insn_start + priv->fetched;
+  uint8_t *fetch_end = priv->the_buffer + priv->fetched;
+  ptrdiff_t needed = until - fetch_end;
 
-  if (until <= priv->max_fetched)
+  if (needed <= 0)
     return true;
 
-  if (until <= priv->the_buffer + ARRAY_SIZE (priv->the_buffer))
-    status = (*info->read_memory_func) (start,
-					priv->max_fetched,
-					until - priv->max_fetched,
-					info);
+  if (priv->fetched + needed <= ARRAY_SIZE (priv->the_buffer))
+    status = (*info->read_memory_func) (start, fetch_end, needed, info);
   if (status != 0)
     {
       /* If we did manage to read at least one byte, then
 	 print_insn_i386 will do something sensible.  Otherwise, print
 	 an error.  We do that here because this is where we know
 	 STATUS.  */
-      if (priv->max_fetched == priv->the_buffer)
+      if (!priv->fetched)
 	(*info->memory_error_func) (status, start, info);
       return false;
     }
 
-  priv->max_fetched = until;
+  priv->fetched += needed;
   return true;
 }
 
@@ -9743,7 +9742,7 @@ print_insn (bfd_vma pc, disassemble_info
   info->bytes_per_line = 7;
 
   info->private_data = &priv;
-  priv.max_fetched = priv.the_buffer;
+  priv.fetched = 0;
   priv.insn_start = pc;
 
   for (i = 0; i < MAX_OPERANDS; ++i)
@@ -9775,7 +9774,7 @@ print_insn (bfd_vma pc, disassemble_info
       goto fetch_error_out;
     }
 
-  ins.insn_codep = ins.codep;
+  ins.nr_prefixes = ins.codep - ins.start_codep;
 
   if (!fetch_code (info, ins.codep + 1))
     {
@@ -11790,7 +11789,9 @@ static bool
 BadOp (instr_info *ins)
 {
   /* Throw away prefixes and 1st. opcode byte.  */
-  ins->codep = ins->insn_codep + 1;
+  struct dis_private *priv = ins->info->private_data;
+
+  ins->codep = priv->the_buffer + ins->nr_prefixes + 1;
   ins->obufp = stpcpy (ins->obufp, "(bad)");
   return true;
 }