[PATCH 1/3] Detect GDB is in cygwin

[Yao Qi <yao@codesourcery.com>]

Hello,
This patch is to detect whether GDB is in cygwin by means of new added
function 'is_in_cygwin'.  In general, the detection is inspired by
Corinna's example on detecting whether GDB is running in a cygwin pty.
I extended the logic a little to handle the case we ssh to a cygwin
machine without allocating a pty.  In this patch, we'll find the file
name of stdin, if its name is prefixed by "\cygwin-", we know GDB is
running in cygwin.

The C code is simple, but getting these simple c code compiled on
various mingw compilers is not simple, due to different header files
they shipped.  I change configure.ac to first check winternl.h, if it
is not found, check other needed headers.  Then invoke AC_TRY_COMPILE
to test whether I can use function NtQueryInformationFile.

gdb:

2013-08-13  Yao Qi  <yao@codesourcery.com>
	    Corinna Vinschen  <vinschen@redhat.com>

	* configure.ac: Invoke AC_CHECK_HEADERS to check winternl.h,
	and ddk/ntddk.h.
	* config.in: Re-generated.
	* configure: Re-generated.
	* defs.h [__MINGW32__] (using_cygwin_pty): Declare.
	* mingw-hdep.c: Inlcude wchar.h.
	[HAVE_WINTERNL_H]: Include winternl.h.  Define
	USE_NTQUERYINFORMATIONFILE.
	[!HAVE_WINTERNL_H] [HAVE_DDK_NTDDK_H]: Include ddk/ntddk.h.
	Define USE_NTQUERYINFORMATIONFILE..
	[USE_NTQUERYINFORMATIONFILE] (get_filename_from_handle): New.
	(using_cygwin_pty): New.
---
 gdb/config.in    |    6 +++
 gdb/configure    |   30 +++++++++++++++++
 gdb/configure.ac |    7 ++++
 gdb/defs.h       |    4 ++
 gdb/mingw-hdep.c |   96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 143 insertions(+), 0 deletions(-)

diff --git a/gdb/config.in b/gdb/config.in
index 76abd04..3d4f1cc 100644
--- a/gdb/config.in
+++ b/gdb/config.in
@@ -84,6 +84,9 @@
 /* Define to 1 if you have the <curses.h> header file. */
 #undef HAVE_CURSES_H
 
+/* Define to 1 if you have the <ddk/ntddk.h> header file. */
+#undef HAVE_DDK_NTDDK_H
+
 /* Define to 1 if you have the declaration of `ADDR_NO_RANDOMIZE', and to 0 if
    you don't. */
 #undef HAVE_DECL_ADDR_NO_RANDOMIZE
@@ -575,6 +578,9 @@
 /* Define to 1 if you have the `wborder' function. */
 #undef HAVE_WBORDER
 
+/* Define to 1 if you have the <winternl.h> header file. */
+#undef HAVE_WINTERNL_H
+
 /* Define to 1 if `fork' works. */
 #undef HAVE_WORKING_FORK
 
diff --git a/gdb/configure b/gdb/configure
index 8067825..e2898d4 100755
--- a/gdb/configure
+++ b/gdb/configure
@@ -9030,6 +9030,36 @@ fi
 done
 
 
+# Check header winternl.h, if not found, check ddk/ntddk.h.
+case "${host}" in
+  *-*-mingw*) for ac_header in winternl.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "winternl.h" "ac_cv_header_winternl_h" "$ac_includes_default"
+if test "x$ac_cv_header_winternl_h" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_WINTERNL_H 1
+_ACEOF
+
+else
+  for ac_header in ddk/ntddk.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "ddk/ntddk.h" "ac_cv_header_ddk_ntddk_h" "$ac_includes_default"
+if test "x$ac_cv_header_ddk_ntddk_h" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_DDK_NTDDK_H 1
+_ACEOF
+
+fi
+
+done
+
+fi
+
+done
+
+		;;
+esac
+
 # ------------------------- #
 # Checks for declarations.  #
 # ------------------------- #
diff --git a/gdb/configure.ac b/gdb/configure.ac
index 667821f..21105a4 100644
--- a/gdb/configure.ac
+++ b/gdb/configure.ac
@@ -1121,6 +1121,13 @@ AC_CHECK_HEADERS(term.h, [], [],
 #endif
 ])
 
+# Check header winternl.h, if not found, check ddk/ntddk.h.
+case "${host}" in
+  *-*-mingw*) AC_CHECK_HEADERS(winternl.h, [],
+  	      	[AC_CHECK_HEADERS(ddk/ntddk.h, [], [],[])], [])
+		;;
+esac
+
 # ------------------------- #
 # Checks for declarations.  #
 # ------------------------- #
diff --git a/gdb/defs.h b/gdb/defs.h
index 014d7d4..71c1c30 100644
--- a/gdb/defs.h
+++ b/gdb/defs.h
@@ -790,6 +790,10 @@ enum block_enum
   FIRST_LOCAL_BLOCK = 2
 };
 
+#ifdef __MINGW32__
+int using_cygwin_pty (void);
+#endif
+
 #include "utils.h"
 
 #endif /* #ifndef DEFS_H */
diff --git a/gdb/mingw-hdep.c b/gdb/mingw-hdep.c
index efc9848..965bda2 100644
--- a/gdb/mingw-hdep.c
+++ b/gdb/mingw-hdep.c
@@ -28,6 +28,19 @@
 #include "readline/readline.h"
 
 #include <windows.h>
+#include <wchar.h>
+
+#ifdef HAVE_WINTERNL_H
+#include <winternl.h>
+#define USE_NTQUERYINFORMATIONFILE 1
+#else
+
+#ifdef HAVE_DDK_NTDDK_H
+#include <ddk/ntddk.h>
+#define USE_NTQUERYINFORMATIONFILE 1
+#endif /*HAVE_DDK_NTDDK_H */
+
+#endif /* HAVE_WINTERNL_H */
 
 /* This event is signalled whenever an asynchronous SIGINT handler
    needs to perform an action in the main thread.  */
@@ -265,6 +278,89 @@ gdb_call_async_signal_handler (struct async_signal_handler *handler,
   SetEvent (sigint_event);
 }
 
+#ifdef USE_NTQUERYINFORMATIONFILE
+
+/* Return the file name of handle FH.  */
+
+static PWCHAR
+get_filename_from_handle (HANDLE fh)
+{
+  IO_STATUS_BLOCK io;
+  NTSTATUS status;
+  long buf[66];	/* NAME_MAX + 1 + sizeof ULONG */
+  PFILE_NAME_INFORMATION pfni = (PFILE_NAME_INFORMATION) buf;
+  static NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE,
+						    PIO_STATUS_BLOCK,
+						    PVOID, ULONG,
+						    FILE_INFORMATION_CLASS);
+
+  /* Calling the native NT function NtQueryInformationFile is required to
+     support pre-Vista systems.  If that's of no concern, Vista introduced
+     the GetFileInformationByHandleEx call with the FileNameInfo info class,
+     which can be used instead. */
+  if (!pNtQueryInformationFile)
+    {
+      pNtQueryInformationFile = (NTSTATUS (NTAPI *)(HANDLE, PIO_STATUS_BLOCK,
+						    PVOID, ULONG, FILE_INFORMATION_CLASS))
+	GetProcAddress (GetModuleHandle ("ntdll.dll"),
+			"NtQueryInformationFile");
+      if (pNtQueryInformationFile == NULL)
+	return NULL;
+    }
+  if (!NT_SUCCESS (pNtQueryInformationFile (fh, &io, pfni, sizeof buf,
+					   FileNameInformation)))
+    return NULL;
+
+  /* The filename is not guaranteed to be NUL-terminated. */
+  pfni->FileName[pfni->FileNameLength / sizeof (WCHAR)] = L'\0';
+
+  return pfni->FileName;
+}
+
+#endif /* USE_NTQUERYINFORMATIONFILE */
+
+/* Return true if GDB is running in Cygwin pseudo-tty.  */
+
+int
+using_cygwin_pty (void)
+{
+  PWCHAR cp;
+  /* Now fetch the underlying HANDLE of stdin. */
+  HANDLE fh = (HANDLE) _get_osfhandle (fileno (stdin));
+  const char *msystem = getenv ("MSYSTEM");
+
+  if (!fh || fh == INVALID_HANDLE_VALUE)
+    return 0;
+
+  /* Return false if environment variable "MSYSTEM" is set, because
+     the code below can't tell GDB runs from MSYS or Cygwin.  GDB
+     shouldn't think it runs in a Cygwin pty when it actually runs
+     from MSYS bash.  */
+  if (msystem != NULL)
+    return 0;
+
+#ifdef USE_NTQUERYINFORMATIONFILE
+  cp = get_filename_from_handle (fh);
+#else
+  cp = NULL;
+#endif
+
+  /* Now check the name pattern.  With pseudo-tty allocated in ssh,
+     the filename of handle of stdin looks like this:
+
+       \cygwin-c5e39b7a9d22bafb-{p,t}ty1-from-master
+
+     Without pseudo-tty allocated in ssh, the filename of handle of
+     stdin looks like this:
+
+       \cygwin-c5e39b7a9d22bafb-pipe-0x14C8-0x3
+
+     If the file name is prefixed with "\cygwin-", GDB is running in
+     cygwin.  */
+
+  return (cp != NULL && wcsncmp (cp, L"\\cygwin-", 8) == 0);
+}
+
 /* -Wmissing-prototypes */
 extern initialize_file_ftype _initialize_mingw_hdep;
 
-- 
1.7.7.6