[newlib-cygwin] Cygwin: console: Disable xterm mode for non cygwin process only.

[Corinna Vinschen <corinna@sourceware.org>]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=d7478090d684ad92773330f1abdff660b537da49

commit d7478090d684ad92773330f1abdff660b537da49
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Tue Jan 14 11:09:42 2020 +0900

    Cygwin: console: Disable xterm mode for non cygwin process only.
    
    - Special function keys such as arrow keys or function keys do not
      work in ConEmu with cygwin-connector after commit
      6a06c6bc8f8492ea09aa3ae180fe94e4ac265611. This patch fixes the
      issue.

Diff:
---
 winsup/cygwin/fhandler_console.cc | 19 -------------------
 winsup/cygwin/fhandler_tty.cc     | 10 ----------
 winsup/cygwin/spawn.cc            | 21 ++++++++++++++++++++-
 winsup/cygwin/winlean.h           | 12 ++++++++++++
 4 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index b286c6c..d976691 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -33,17 +33,6 @@ details. */
 #include "child_info.h"
 #include "cygwait.h"
 
-/* Not yet defined in Mingw-w64 */
-#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
-#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
-#endif /* ENABLE_VIRTUAL_TERMINAL_PROCESSING */
-#ifndef DISABLE_NEWLINE_AUTO_RETURN
-#define DISABLE_NEWLINE_AUTO_RETURN 0x0008
-#endif /* DISABLE_NEWLINE_AUTO_RETURN */
-#ifndef ENABLE_VIRTUAL_TERMINAL_INPUT
-#define ENABLE_VIRTUAL_TERMINAL_INPUT 0x0200
-#endif /* ENABLE_VIRTUAL_TERMINAL_INPUT */
-
 /* Don't make this bigger than NT_MAX_PATH as long as the temporary buffer
    is allocated using tmp_pathbuf!!! */
 #define CONVERT_LIMIT NT_MAX_PATH
@@ -2975,14 +2964,6 @@ fhandler_console::fixup_after_fork_exec (bool execing)
 {
   set_unit ();
   setup_io_mutex ();
-  if (wincap.has_con_24bit_colors () && !con_is_legacy)
-    {
-      DWORD dwMode;
-      /* Disable xterm compatible mode in input */
-      GetConsoleMode (get_handle (), &dwMode);
-      dwMode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
-      SetConsoleMode (get_handle (), dwMode);
-    }
 }
 
 // #define WINSTA_ACCESS (WINSTA_READATTRIBUTES | STANDARD_RIGHTS_READ | STANDARD_RIGHTS_WRITE | WINSTA_CREATEDESKTOP | WINSTA_EXITWINDOWS)
diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 410e237..042ffd1 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -31,19 +31,9 @@ details. */
 #define ALWAYS_USE_PCON false
 #define USE_API_HOOK true
 
-/* Not yet defined in Mingw-w64 */
-#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
-#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
-#endif /* ENABLE_VIRTUAL_TERMINAL_PROCESSING */
-#ifndef DISABLE_NEWLINE_AUTO_RETURN
-#define DISABLE_NEWLINE_AUTO_RETURN 0x0008
-#endif /* DISABLE_NEWLINE_AUTO_RETURN */
 #ifndef PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE
 #define PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE 0x00020016
 #endif /* PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE */
-#ifndef ENABLE_VIRTUAL_TERMINAL_INPUT
-#define ENABLE_VIRTUAL_TERMINAL_INPUT 0x0200
-#endif /* ENABLE_VIRTUAL_TERMINAL_INPUT */
 
 extern "C" int sscanf (const char *, const char *, ...);
 extern "C" int ttyname_r (int, char*, size_t);
diff --git a/winsup/cygwin/spawn.cc b/winsup/cygwin/spawn.cc
index cea79e3..6a50342 100644
--- a/winsup/cygwin/spawn.cc
+++ b/winsup/cygwin/spawn.cc
@@ -610,7 +610,26 @@ child_info_spawn::worker (const char *prog_arg, const char *const *argv,
 		}
 	    }
 	  else if (fh && fh->get_major () == DEV_CONS_MAJOR)
-	    attach_to_console = true;
+	    {
+	      attach_to_console = true;
+	      if (wincap.has_con_24bit_colors () && !iscygwin ())
+		{
+		  DWORD dwMode;
+		  if (fd == 0)
+		    {
+		      /* Disable xterm compatible mode in input */
+		      GetConsoleMode (fh->get_handle (), &dwMode);
+		      dwMode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
+		      SetConsoleMode (fh->get_handle (), dwMode);
+		    }
+		  else
+		    {
+		      GetConsoleMode (fh->get_output_handle (), &dwMode);
+		      dwMode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+		      SetConsoleMode (fh->get_output_handle (), dwMode);
+		    }
+		}
+	    }
 	}
 
       /* Set up needed handles for stdio */
diff --git a/winsup/cygwin/winlean.h b/winsup/cygwin/winlean.h
index deb79be..3d79a92 100644
--- a/winsup/cygwin/winlean.h
+++ b/winsup/cygwin/winlean.h
@@ -93,4 +93,16 @@ details. */
    use this function.  Use GetSystemWindowsDirectoryW. */
 #define GetWindowsDirectoryW dont_use_GetWindowsDirectory
 #define GetWindowsDirectoryA dont_use_GetWindowsDirectory
+
+/* For console with xterm compatible mode */
+/* Not yet defined in Mingw-w64 */
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
+#endif /* ENABLE_VIRTUAL_TERMINAL_PROCESSING */
+#ifndef ENABLE_VIRTUAL_TERMINAL_INPUT
+#define ENABLE_VIRTUAL_TERMINAL_INPUT 0x0200
+#endif /* ENABLE_VIRTUAL_TERMINAL_INPUT */
+#ifndef DISABLE_NEWLINE_AUTO_RETURN
+#define DISABLE_NEWLINE_AUTO_RETURN 0x0008
+#endif /* DISABLE_NEWLINE_AUTO_RETURN */
 #endif /*_WINLEAN_H*/