[newlib-cygwin] Cygwin: pty: Stop closing and recreating attach_mutex.

[Takashi Yano <tyan0@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=3af461092e7ee9b76f3a1c18a6d95ed6e226df81

commit 3af461092e7ee9b76f3a1c18a6d95ed6e226df81
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Thu Jan 13 19:44:43 2022 +0900

    Cygwin: pty: Stop closing and recreating attach_mutex.
    
    - Closing attach_mutex and recreating it causes the race issue
      between pty and console codes. With this patch, attach_mutex
      is created only once in a process which opens pty, and never
      closed in order to avoid this issue.
    
    Addresses:
      https://cygwin.com/pipermail/cygwin-developers/2021-December/012548.html

Diff:
---
 winsup/cygwin/fhandler.h          |  3 +++
 winsup/cygwin/fhandler_console.cc | 17 -----------------
 winsup/cygwin/fhandler_tty.cc     | 30 ++++++++++++++++++++----------
 winsup/cygwin/select.cc           | 16 ----------------
 4 files changed, 23 insertions(+), 43 deletions(-)

diff --git a/winsup/cygwin/fhandler.h b/winsup/cygwin/fhandler.h
index 4f70c4c0b..0cea1b7f3 100644
--- a/winsup/cygwin/fhandler.h
+++ b/winsup/cygwin/fhandler.h
@@ -1881,6 +1881,9 @@ class fhandler_serial: public fhandler_base
 #define release_output_mutex() \
   __release_output_mutex (__PRETTY_FUNCTION__, __LINE__)
 
+DWORD acquire_attach_mutex (DWORD t);
+void release_attach_mutex (void);
+
 class tty;
 class tty_min;
 class fhandler_termios: public fhandler_base
diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index 024be522b..0e4b41559 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -56,25 +56,8 @@ fhandler_console::console_state NO_COPY *fhandler_console::shared_console_info;
 
 bool NO_COPY fhandler_console::invisible_console;
 
-/* Mutex for AttachConsole()/FreeConsole() in fhandler_tty.cc */
-HANDLE attach_mutex;
-
 extern DWORD mutex_timeout; /* defined in fhandler_termios.cc */
 
-static inline void
-acquire_attach_mutex (DWORD t)
-{
-  if (attach_mutex)
-    WaitForSingleObject (attach_mutex, t);
-}
-
-static inline void
-release_attach_mutex ()
-{
-  if (attach_mutex)
-    ReleaseMutex (attach_mutex);
-}
-
 /* con_ra is shared in the same process.
    Only one console can exist in a process, therefore, static is suitable. */
 static struct fhandler_base::rabuf_t con_ra;
diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index f68e80df9..1ae4edd63 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -58,8 +58,21 @@ struct pipe_reply {
   DWORD error;
 };
 
-extern HANDLE attach_mutex; /* Defined in fhandler_console.cc */
-static LONG master_cnt = 0;
+HANDLE attach_mutex;
+
+DWORD acquire_attach_mutex (DWORD t)
+{
+  if (!attach_mutex)
+    return WAIT_OBJECT_0;
+  return WaitForSingleObject (attach_mutex, t);
+}
+
+void release_attach_mutex (void)
+{
+  if (!attach_mutex)
+    return;
+  ReleaseMutex (attach_mutex);
+}
 
 inline static bool pcon_pid_alive (DWORD pid);
 
@@ -523,13 +536,13 @@ fhandler_pty_master::accept_input ()
 	{
 	  /* Slave attaches to a different console than master.
 	     Therefore reattach here. */
-	  WaitForSingleObject (attach_mutex, mutex_timeout);
+	  acquire_attach_mutex (mutex_timeout);
 	  FreeConsole ();
 	  AttachConsole (target_pid);
 	  cp_to = GetConsoleCP ();
 	  FreeConsole ();
 	  AttachConsole (resume_pid);
-	  ReleaseMutex (attach_mutex);
+	  release_attach_mutex ();
 	}
       else
 	cp_to = GetConsoleCP ();
@@ -2111,8 +2124,6 @@ fhandler_pty_master::close ()
 	  master_fwd_thread->detach ();
 	}
     }
-  if (InterlockedDecrement (&master_cnt) == 0)
-    CloseHandle (attach_mutex);
 
   /* Check if the last master handle has been closed.  If so, set
      input_available_event to wake up potentially waiting slaves. */
@@ -2838,13 +2849,13 @@ fhandler_pty_master::pty_master_fwd_thread (const master_fwd_thread_param_t *p)
 	{
 	  /* Slave attaches to a different console than master.
 	     Therefore reattach here. */
-	  WaitForSingleObject (attach_mutex, mutex_timeout);
+	  acquire_attach_mutex (mutex_timeout);
 	  FreeConsole ();
 	  AttachConsole (target_pid);
 	  cp_from = GetConsoleOutputCP ();
 	  FreeConsole ();
 	  AttachConsole (resume_pid);
-	  ReleaseMutex (attach_mutex);
+	  release_attach_mutex ();
 	}
       else
 	cp_from = GetConsoleOutputCP ();
@@ -2993,7 +3004,7 @@ fhandler_pty_master::setup ()
   if (!(pcon_mutex = CreateMutex (&sa, FALSE, buf)))
     goto err;
 
-  if (InterlockedIncrement (&master_cnt) == 1)
+  if (!attach_mutex)
     attach_mutex = CreateMutex (&sa, FALSE, NULL);
 
   /* Create master control pipe which allows the master to duplicate
@@ -3057,7 +3068,6 @@ err:
   close_maybe (input_available_event);
   close_maybe (output_mutex);
   close_maybe (input_mutex);
-  close_maybe (attach_mutex);
   close_maybe (from_master_nat);
   close_maybe (from_master);
   close_maybe (to_master_nat);
diff --git a/winsup/cygwin/select.cc b/winsup/cygwin/select.cc
index 0cd62d932..5b8fc0f81 100644
--- a/winsup/cygwin/select.cc
+++ b/winsup/cygwin/select.cc
@@ -1095,22 +1095,6 @@ fhandler_fifo::select_except (select_stuff *ss)
   return s;
 }
 
-extern HANDLE attach_mutex; /* Defined in fhandler_console.cc */
-
-static inline void
-acquire_attach_mutex (DWORD t)
-{
-  if (attach_mutex)
-    WaitForSingleObject (attach_mutex, t);
-}
-
-static inline void
-release_attach_mutex ()
-{
-  if (attach_mutex)
-    ReleaseMutex (attach_mutex);
-}
-
 extern DWORD mutex_timeout; /* defined in fhandler_termios.cc */
 
 static int