[newlib-cygwin/cygwin-3_3-branch] Cygwin: pty: Avoid deadlock when pcon is started on console.

[Takashi Yano <tyan0@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=0f5046ef9f9d48aaaed283e019c20b2265121e69

commit 0f5046ef9f9d48aaaed283e019c20b2265121e69
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Mon May 9 00:37:38 2022 +0900

    Cygwin: pty: Avoid deadlock when pcon is started on console.
    
    - Previously, "env SHELL=cmd script" command in console caused
      deadlock when starting cmd.exe. This patch fixes the issue.

Diff:
---
 winsup/cygwin/fhandler_tty.cc | 18 ++++++++++++------
 winsup/cygwin/select.cc       |  2 --
 winsup/cygwin/tty.cc          |  4 ++--
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index b419cbdd0..31f399f99 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -248,9 +248,7 @@ atexit_func (void)
 		ReleaseMutex (ptys->input_mutex);
 	      }
 	    WaitForSingleObject (ptys->pcon_mutex, INFINITE);
-	    acquire_attach_mutex (mutex_timeout);
 	    ptys->close_pseudoconsole (ttyp, force_switch_to);
-	    release_attach_mutex ();
 	    ReleaseMutex (ptys->pcon_mutex);
 	    break;
 	  }
@@ -1176,9 +1174,7 @@ fhandler_pty_slave::reset_switch_to_pcon (void)
 		return;
 	      bool need_restore_handles = get_ttyp ()->pcon_activated;
 	      WaitForSingleObject (pcon_mutex, INFINITE);
-	      acquire_attach_mutex (mutex_timeout);
 	      close_pseudoconsole (get_ttyp ());
-	      release_attach_mutex ();
 	      ReleaseMutex (pcon_mutex);
 	      if (need_restore_handles)
 		{
@@ -3319,9 +3315,11 @@ fhandler_pty_slave::setup_pseudoconsole (bool nopcon)
 		       GetCurrentProcess (), &hpConOut,
 		       0, TRUE, DUPLICATE_SAME_ACCESS);
       CloseHandle (pcon_owner);
+      acquire_attach_mutex (mutex_timeout);
       FreeConsole ();
       AttachConsole (get_ttyp ()->pcon_pid);
       init_console_handler (false);
+      release_attach_mutex ();
       goto skip_create;
     }
 
@@ -3443,9 +3441,11 @@ fhandler_pty_slave::setup_pseudoconsole (bool nopcon)
       HeapFree (GetProcessHeap (), 0, si.lpAttributeList);
 
       /* Attach to pseudo console */
+      acquire_attach_mutex (mutex_timeout);
       FreeConsole ();
       AttachConsole (pi.dwProcessId);
       init_console_handler (false);
+      release_attach_mutex ();
 
       /* Terminate helper process */
       SetEvent (goodbye);
@@ -3585,8 +3585,10 @@ fhandler_pty_slave::close_pseudoconsole (tty *ttyp, DWORD force_switch_to)
     }
   if (ttyp->pcon_activated)
     {
+      acquire_attach_mutex (mutex_timeout);
       ttyp->previous_code_page = GetConsoleCP ();
       ttyp->previous_output_code_page = GetConsoleOutputCP ();
+      release_attach_mutex ();
       if (pcon_pid_self (ttyp->pcon_pid))
 	{
 	  if (switch_to)
@@ -3628,19 +3630,23 @@ fhandler_pty_slave::close_pseudoconsole (tty *ttyp, DWORD force_switch_to)
 	      ttyp->h_pcon_conhost_process = new_conhost_process;
 	      ttyp->h_pcon_in = new_pcon_in;
 	      ttyp->h_pcon_out = new_pcon_out;
+	      acquire_attach_mutex (mutex_timeout);
 	      FreeConsole ();
 	      pinfo p (myself->ppid);
 	      if (!p || !AttachConsole (p->dwProcessId))
 		AttachConsole (ATTACH_PARENT_PROCESS);
 	      init_console_handler (false);
+	      release_attach_mutex ();
 	    }
 	  else
 	    { /* Close pseudo console */
+	      acquire_attach_mutex (mutex_timeout);
 	      FreeConsole ();
 	      pinfo p (myself->ppid);
 	      if (!p || !AttachConsole (p->dwProcessId))
 		AttachConsole (ATTACH_PARENT_PROCESS);
 	      init_console_handler (false);
+	      release_attach_mutex ();
 	      /* Reconstruct pseudo console handler container here for close */
 	      HPCON_INTERNAL *hp =
 		(HPCON_INTERNAL *) HeapAlloc (GetProcessHeap (), 0,
@@ -3660,11 +3666,13 @@ fhandler_pty_slave::close_pseudoconsole (tty *ttyp, DWORD force_switch_to)
 	}
       else
 	{
+	  acquire_attach_mutex (mutex_timeout);
 	  FreeConsole ();
 	  pinfo p (myself->ppid);
 	  if (!p || !AttachConsole (p->dwProcessId))
 	    AttachConsole (ATTACH_PARENT_PROCESS);
 	  init_console_handler (false);
+	  release_attach_mutex ();
 	}
     }
   else if (pcon_pid_self (ttyp->pcon_pid))
@@ -4085,9 +4093,7 @@ fhandler_pty_slave::setup_for_non_cygwin_app (bool nopcon, PWCHAR envblock,
   if (disable_pcon || !term_has_pcon_cap (envblock))
     nopcon = true;
   WaitForSingleObject (pcon_mutex, INFINITE);
-  acquire_attach_mutex (mutex_timeout);
   bool enable_pcon = setup_pseudoconsole (nopcon);
-  release_attach_mutex ();
   ReleaseMutex (pcon_mutex);
   /* For pcon enabled case, transfer_input() is called in master::write() */
   if (!enable_pcon && get_ttyp ()->getpgid () == myself->pgid
diff --git a/winsup/cygwin/select.cc b/winsup/cygwin/select.cc
index 29a89a0e4..a84d5a21c 100644
--- a/winsup/cygwin/select.cc
+++ b/winsup/cygwin/select.cc
@@ -1125,7 +1125,6 @@ peek_console (select_record *me, bool)
     {
       if (fh->bg_check (SIGTTIN, true) <= bg_eof)
 	{
-	  release_attach_mutex ();
 	  fh->release_input_mutex ();
 	  return me->read_ready = true;
 	}
@@ -1142,7 +1141,6 @@ peek_console (select_record *me, bool)
 	  && global_sigs[SIGWINCH].sa_handler != SIG_DFL)
 	{
 	  set_sig_errno (EINTR);
-	  release_attach_mutex ();
 	  fh->release_input_mutex ();
 	  return -1;
 	}
diff --git a/winsup/cygwin/tty.cc b/winsup/cygwin/tty.cc
index 44be3c8b4..f64c27719 100644
--- a/winsup/cygwin/tty.cc
+++ b/winsup/cygwin/tty.cc
@@ -335,6 +335,7 @@ tty_min::setpgid (int pid)
 	{
 	  bool attach_restore = false;
 	  HANDLE from = ptys->get_handle_nat ();
+	  WaitForSingleObject (ptys->input_mutex, mutex_timeout);
 	  acquire_attach_mutex (mutex_timeout);
 	  if (ttyp->pcon_activated && ttyp->pcon_pid
 	      && !ptys->get_console_process_id (ttyp->pcon_pid, true))
@@ -350,10 +351,8 @@ tty_min::setpgid (int pid)
 	      init_console_handler (false);
 	      attach_restore = true;
 	    }
-	  WaitForSingleObject (ptys->input_mutex, mutex_timeout);
 	  fhandler_pty_slave::transfer_input (tty::to_cyg, from, ttyp,
 				  ptys->get_input_available_event ());
-	  ReleaseMutex (ptys->input_mutex);
 	  if (attach_restore)
 	    {
 	      FreeConsole ();
@@ -363,6 +362,7 @@ tty_min::setpgid (int pid)
 	      init_console_handler (false);
 	    }
 	  release_attach_mutex ();
+	  ReleaseMutex (ptys->input_mutex);
 	}
       ReleaseMutex (ptys->pcon_mutex);
     }