MSVC hook function prologue

["Stefan Dösinger" <stefan@codeweavers.com>]

[-- Attachment #1: Type: text/plain, Size: 2229 bytes --]

Hello,
After a rather long break due to other work I tried to revive my work on 
support for the function prologue used in Win32 API functions on Windows - a 
function prologue that some apps running in Wine expect.

This thread from January explains what I am trying to do:
http://gcc.gnu.org/ml/gcc/2009-01/msg00089.html

Essentially I want a function attrib that starts the function with this 
sequence, no matter what other parameters, code in the function, attributes 
or whatever are used:
8b ff          mov    %edi,%edi
55             push   %ebp
8b ec          mov    %esp,%ebp

I have attached the latest version of my patch for comments. It is mainly 
rebased against gcc changes that were made in the meantime. I also improved 
the REG_FRAME_RELATED_EXPR notes a bit, and only set it if the movs and pops 
are used for the frame pointer setup.

I also now know that I don't(or cannot) care about 64 bit right now. The 
windows apps currently do Windows API function hooking only in 32 bit, and 
there is no emerging scheme yet for hooking Win64 functions in the same way.

Currently I still have these problems:
*) There is apparently some plugin framework in the works. Can this 
functionality implemented as a plugin?

*) The way I read the msvc_prologue attribute seems wrong to me. I could read 
it directly in ix86_expand_prologue, but I got lost in the different trees 
gcc uses. I'm yet again trying to find this in the code and in the docs.

*) The code generated if no frame pointer is needed isn't pretty, but Wine 
will always need a frame pointer, so any optimization in that area won't get 
much test exposure.

*) The stack alignment code + msvc_prologue is used by Wine on osx though. 
Currently I pop %ebp after the 5 byte prologue, and the normal code recreates 
the frame pointer afterwards. My understanding is that I can avoid this by 
keeping the original frame pointer, but adjusting a lot of offsets after the 
alignment to find the function parameters and align the stack properly on 
calls. However, this is currently above my head.

*) What other changes are needed to get a functionality like this into 
mainline?

Thank you,
Stefan Dösinger

[-- Attachment #2: msvc-prologue.diff --]
[-- Type: text/x-diff, Size: 6963 bytes --]

Index: gcc/configure.ac
===================================================================
--- gcc/configure.ac	(revision 151348)
+++ gcc/configure.ac	(working copy)
@@ -3035,6 +3035,12 @@
       [AC_DEFINE(HAVE_AS_IX86_SAHF, 1,
         [Define if your assembler supports the sahf mnemonic.])])
 
+    gcc_GAS_CHECK_FEATURE([swap suffix],
+      gcc_cv_as_ix86_swap,,,
+      [movl.s %esp, %ebp],,
+      [AC_DEFINE(HAVE_AS_IX86_SWAP, 1,
+        [Define if your assembler supports the swap suffix.])])
+
     gcc_GAS_CHECK_FEATURE([different section symbol subtraction],
       gcc_cv_as_ix86_diff_sect_delta,,,
       [.section .rodata
Index: gcc/config/i386/i386.h
===================================================================
--- gcc/config/i386/i386.h	(revision 151348)
+++ gcc/config/i386/i386.h	(working copy)
@@ -2388,6 +2388,9 @@
      to be used. MS_ABI means ms abi. Otherwise SYSV_ABI means sysv abi.  */
   enum calling_abi call_abi;
   struct machine_cfa_state cfa;
+  /* This value is used for i386 targets and specifies if the function
+   * should start with the hooking-friendly Win32 function prologue       */
+  int msvc_prologue;
 };
 #endif
 
Index: gcc/config/i386/i386.md
===================================================================
--- gcc/config/i386/i386.md	(revision 151348)
+++ gcc/config/i386/i386.md	(working copy)
@@ -237,6 +237,7 @@
    (UNSPECV_RDTSC		18)
    (UNSPECV_RDTSCP		19)
    (UNSPECV_RDPMC		20)
+   (UNSPECV_VSWAPMOV	21)
   ])
 
 ;; Constants to represent pcomtrue/pcomfalse variants
@@ -15747,6 +15748,16 @@
    (set_attr "length_immediate" "0")
    (set_attr "modrm" "0")])
 
+(define_insn "vswapmov"
+[(unspec_volatile [(match_operand 0 "register_operand" "0")
+                   (match_operand 1 "register_operand" "1")]
+ UNSPECV_VSWAPMOV )]
+  ""
+  "movl.s\t%1,%0"
+  [(set_attr "length" "2")
+   (set_attr "length_immediate" "0")
+   (set_attr "modrm" "0")])
+
 ;; Pad to 16-byte boundary, max skip in op0.  Used to avoid
 ;; branch prediction penalty for the third jump in a 16-byte
 ;; block on K8.
Index: gcc/config/i386/i386.c
===================================================================
--- gcc/config/i386/i386.c	(revision 151348)
+++ gcc/config/i386/i386.c	(working copy)
@@ -4777,6 +4777,19 @@
   return ix86_abi;
 }
 
+static int
+ix86_function_msvc_prologue (const_tree fntype)
+{
+  if (!TARGET_64BIT && fntype != NULL)
+    {
+      if(lookup_attribute ("msvc_prologue", TYPE_ATTRIBUTES (fntype)))
+        {
+          return 1;
+        }
+    }
+  return 0;
+}
+
 static enum calling_abi
 ix86_function_abi (const_tree fndecl)
 {
@@ -4808,6 +4821,11 @@
     cfun->machine->call_abi = ix86_abi;
   else
     cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
+
+  if (fndecl == NULL_TREE)
+    cfun->machine->msvc_prologue = 0;
+  else
+    cfun->machine->msvc_prologue = ix86_function_msvc_prologue (TREE_TYPE (fndecl));
 }
 
 /* MS and SYSV ABI have different set of call used registers.  Avoid expensive
@@ -8316,6 +8334,7 @@
   bool pic_reg_used;
   struct ix86_frame frame;
   HOST_WIDE_INT allocate;
+  int gen_frame_pointer = frame_pointer_needed;
 
   ix86_finalize_stack_realign_flags ();
 
@@ -8328,6 +8347,45 @@
 
   ix86_compute_frame_layout (&frame);
 
+  if(cfun->machine->msvc_prologue)
+  {
+    rtx push, mov;
+    /* Make sure the function starts with
+       8b ff     movl.s %edi,%edi
+       55        push   %ebp
+       8b ec     movl.s %esp,%ebp
+
+       This matches the hookable function prologue generated by msvc. Wine uses this
+       to enable Windows games to hook the Win32 API functions provided by Wine */
+    insn = emit_insn(gen_vswapmov(gen_rtx_REG (Pmode, DI_REG), gen_rtx_REG (Pmode, DI_REG)));
+    push = emit_insn (gen_push (hard_frame_pointer_rtx));
+    mov = emit_insn(gen_vswapmov(hard_frame_pointer_rtx, stack_pointer_rtx));
+
+    if(frame_pointer_needed && !(crtl->drap_reg && crtl->stack_realign_needed))
+    {
+      /* The push %ebp and movl.s %esp, %ebp already set up the frame pointer. No need to do
+         this again. */
+      gen_frame_pointer = 0;
+      RTX_FRAME_RELATED_P (push) = 1;
+      RTX_FRAME_RELATED_P (mov) = 1;
+      add_reg_note (mov, REG_FRAME_RELATED_EXPR,
+                    gen_rtx_SET (VOIDmode, hard_frame_pointer_rtx, stack_pointer_rtx));
+      if (ix86_cfa_state->reg == stack_pointer_rtx)
+        ix86_cfa_state->reg = hard_frame_pointer_rtx;
+    }
+    else
+    {
+      /* If the frame pointer is not needed, pop %ebp again. This could be optimized for cases where
+         ebp needs to be backed up for some other reason.
+
+         If stack realignment is needed, pop the base pointer again, align the stack, and later
+         regenerate the frame pointer setup. The frame pointer generated by the msvc prologue
+         is not aligned, so it can't be used */
+      insn = emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
+    }
+  }
+
+  
   /* Emit prologue code to adjust stack alignment and setup DRAP, in case
      of DRAP is needed and stack realignment is really needed after reload */
   if (crtl->drap_reg && crtl->stack_realign_needed)
@@ -8377,14 +8435,12 @@
   /* Note: AT&T enter does NOT have reversed args.  Enter is probably
      slower on all targets.  Also sdb doesn't like it.  */
 
-  if (frame_pointer_needed)
+  if (gen_frame_pointer)
     {
       insn = emit_insn (gen_push (hard_frame_pointer_rtx));
       RTX_FRAME_RELATED_P (insn) = 1;
-
       insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
       RTX_FRAME_RELATED_P (insn) = 1;
-
       if (ix86_cfa_state->reg == stack_pointer_rtx)
         ix86_cfa_state->reg = hard_frame_pointer_rtx;
     }
@@ -26069,11 +26125,24 @@
     }
   if (!TARGET_64BIT)
     {
-      warning (OPT_Wattributes, "%qE attribute only available for 64-bit",
-	       name);
-      *no_add_attrs = true;
-      return NULL_TREE;
+      if(!is_attribute_p ("msvc_prologue", name))
+        {
+          warning (OPT_Wattributes, "%qE attribute only available for 64-bit",
+	           name);
+          *no_add_attrs = true;
+          return NULL_TREE;
+        }
     }
+  else
+    {
+      if(is_attribute_p ("msvc_prologue", name))
+        {
+          warning (OPT_Wattributes, "%qE attribute only available for 32-bit",
+                   name);
+          *no_add_attrs = true;
+          return NULL_TREE;
+        }
+    }
 
   /* Can combine regparm with all attributes but fastcall.  */
   if (is_attribute_p ("ms_abi", name))
@@ -28983,6 +29052,9 @@
   /* ms_abi and sysv_abi calling convention function attributes.  */
   { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
   { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
+#ifdef HAVE_AS_IX86_SWAP
+  { "msvc_prologue", 0, 0, false, true, true, ix86_handle_abi_attribute },
+#endif
   /* End element.  */
   { NULL,        0, 0, false, false, false, NULL }
 };