[PATCH v2 0/3] Return appropriate handle by _get_osfhandle() and GetStdHandle().

[PATCH v2 0/3] Return appropriate handle by _get_osfhandle() and GetStdHandle().

[Takashi Yano]

Takashi Yano (3):
  Cygwin: syscalls.cc: Make _get_osfhandle() return appropriate handle.
  Cygwin: pty: Add hook for GetStdHandle() to return appropriate handle.
  Cygwin: pty: Clear input_available_event if pipe is empty on close.

 winsup/cygwin/fhandler_tty.cc | 31 ++++++++++++++++++++++++-------
 winsup/cygwin/syscalls.cc     | 13 ++++++++++++-
 2 files changed, 36 insertions(+), 8 deletions(-)

-- 
2.30.2

[PATCH v2 1/3] Cygwin: syscalls.cc: Make _get_osfhandle() return appropriate handle.

[Takashi Yano]

- Currently, _get_osfhandle() returns input handle for pty even for
  stdout and stdout. This patch fixes the issue. Also, setup_locale()
  is called to make sure the charset conversion works for output.
---
 winsup/cygwin/syscalls.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/winsup/cygwin/syscalls.cc b/winsup/cygwin/syscalls.cc
index 6ba4f10f7..205d15951 100644
--- a/winsup/cygwin/syscalls.cc
+++ b/winsup/cygwin/syscalls.cc
@@ -3223,7 +3223,18 @@ _get_osfhandle (int fd)
 
   cygheap_fdget cfd (fd);
   if (cfd >= 0)
-    res = (long) cfd->get_handle ();
+    {
+      if (cfd->get_major () == DEV_PTYS_MAJOR)
+	{
+	  fhandler_pty_slave *ptys =
+	    (fhandler_pty_slave *) (fhandler_base *) cfd;
+	  ptys->setup_locale ();
+	}
+      if (fd == 1 || fd == 2)
+	res = (long) cfd->get_output_handle ();
+      else
+	res = (long) cfd->get_handle_cyg ();
+    }
   else
     res = -1;
 
-- 
2.30.2

[PATCH v2 2/3] Cygwin: pty: Add hook for GetStdHandle() to return appropriate handle.

[Takashi Yano]

- Currently, GetStdHandle(STD_INPUT_HANDLE) returns input handle for
  non-cygwin process. If cygwin process read from non-cygwin pipe,
  this causes hang up because master writes input to cygwin pipe.
  Also, setup_locale() is called to make charset conversion for output
  handle work properly.
---
 winsup/cygwin/fhandler_tty.cc | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 02e94efcc..682264130 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -118,7 +118,7 @@ set_switch_to_pcon (HANDLE *in, HANDLE *out, HANDLE *err, bool iscygwin)
   fhandler_pty_slave *ptys_pcon = NULL;
   while ((fd = cfd.next ()) >= 0)
     {
-      if (*in == cfd->get_handle () ||
+      if (*in == cfd->get_handle () || *in == cfd->get_handle_cyg () ||
 	  (fd == 0 && *in == GetStdHandle (STD_INPUT_HANDLE)))
 	replace_in = (fhandler_base *) cfd;
       if (*out == cfd->get_output_handle () ||
@@ -140,12 +140,7 @@ set_switch_to_pcon (HANDLE *in, HANDLE *out, HANDLE *err, bool iscygwin)
   if (!iscygwin && ptys_pcon)
     ptys_pcon->set_switch_to_pcon ();
   if (replace_in)
-    {
-      if (iscygwin && ptys_pcon->pcon_activated ())
-	*in = replace_in->get_handle_cyg ();
-      else
-	*in = replace_in->get_handle ();
-    }
+    *in = replace_in->get_handle ();
   if (replace_out)
     *out = replace_out->get_output_handle ();
   if (replace_err)
@@ -157,6 +152,7 @@ set_switch_to_pcon (HANDLE *in, HANDLE *out, HANDLE *err, bool iscygwin)
 DEF_HOOK (CreateProcessA);
 DEF_HOOK (CreateProcessW);
 DEF_HOOK (exit);
+DEF_HOOK (GetStdHandle);
 
 static BOOL WINAPI
 CreateProcessA_Hooked
@@ -300,6 +296,23 @@ exit_Hooked (int e)
   exit_Orig (e);
 }
 
+static HANDLE WINAPI
+GetStdHandle_Hooked (DWORD h)
+{
+  HANDLE r = GetStdHandle_Orig (h);
+  cygheap_fdenum cfd (false);
+  while (cfd.next () >= 0)
+    if (cfd->get_major () == DEV_PTYS_MAJOR)
+      {
+	fhandler_pty_slave *ptys =
+	  (fhandler_pty_slave *) (fhandler_base *) cfd;
+	ptys->setup_locale ();
+	if (r == cfd->get_handle ())
+	  return cfd->get_handle_cyg ();
+      }
+  return r;
+}
+
 static void
 convert_mb_str (UINT cp_to, char *ptr_to, size_t *len_to,
 		UINT cp_from, const char *ptr_from, size_t len_from,
@@ -2349,6 +2362,7 @@ fhandler_pty_slave::fixup_after_exec ()
   DO_HOOK (NULL, CreateProcessW);
   if (CreateProcessA_Orig || CreateProcessW_Orig)
     DO_HOOK (NULL, exit);
+  DO_HOOK (NULL, GetStdHandle);
 }
 
 /* This thread function handles the master control pipe.  It waits for a
-- 
2.30.2

[PATCH v2 3/3] Cygwin: pty: Clear input_available_event if pipe is empty on close.

[Takashi Yano]

- If apps read input from get_handle_cyg() directly by ReadFile(),
  input_available_event may remains signalled. This patch clears
  input_available_event if the pipe is empty on closing pty slave.
---
 winsup/cygwin/fhandler_tty.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 682264130..43e83b807 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -982,6 +982,9 @@ fhandler_pty_slave::close ()
   termios_printf ("closing last open %s handle", ttyname ());
   if (inuse && !CloseHandle (inuse))
     termios_printf ("CloseHandle (inuse), %E");
+  DWORD n;
+  if (bytes_available (n) && !n)
+    ResetEvent (input_available_event);
   if (!ForceCloseHandle (input_available_event))
     termios_printf ("CloseHandle (input_available_event<%p>), %E", input_available_event);
   if (!ForceCloseHandle (get_output_handle_cyg ()))
-- 
2.30.2