[PATCH] Cygwin: termios: Refactor the function is_console_app().

[PATCH] Cygwin: termios: Refactor the function is_console_app().

[Takashi Yano]

Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
---
 winsup/cygwin/fhandler/termios.cc | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/winsup/cygwin/fhandler/termios.cc b/winsup/cygwin/fhandler/termios.cc
index 789ae0179..d106955dc 100644
--- a/winsup/cygwin/fhandler/termios.cc
+++ b/winsup/cygwin/fhandler/termios.cc
@@ -704,22 +704,20 @@ static bool
 is_console_app (const WCHAR *filename)
 {
   HANDLE h;
-  const int id_offset = 92;
   h = CreateFileW (filename, GENERIC_READ, FILE_SHARE_READ,
 		   NULL, OPEN_EXISTING, 0, NULL);
   char buf[1024];
   DWORD n;
   ReadFile (h, buf, sizeof (buf), &n, 0);
   CloseHandle (h);
-  char *p = (char *) memmem (buf, n, "PE\0\0", 4);
-  if (p && p + id_offset < buf + n)
-    return p[id_offset] == '\003'; /* 02: GUI, 03: console */
-  else
-    {
-      wchar_t *e = wcsrchr (filename, L'.');
-      if (e && (wcscasecmp (e, L".bat") == 0 || wcscasecmp (e, L".cmd") == 0))
-	return true;
-    }
+  /* The offset of Subsystem is the same for both IMAGE_NT_HEADERS32 and
+     IMAGE_NT_HEADERS64, so only IMAGE_NT_HEADERS32 is used here. */
+  IMAGE_NT_HEADERS32 *p = (IMAGE_NT_HEADERS32 *) memmem (buf, n, "PE\0\0", 4);
+  if (p && (char *) &p->OptionalHeader.DllCharacteristics <= buf + n)
+    return p->OptionalHeader.Subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI;
+  wchar_t *e = wcsrchr (filename, L'.');
+  if (e && (wcscasecmp (e, L".bat") == 0 || wcscasecmp (e, L".cmd") == 0))
+    return true;
   return false;
 }
 
-- 
2.39.0

Re: [PATCH] Cygwin: termios: Refactor the function is_console_app().

[Corinna Vinschen]

On Aug 28 18:21, Takashi Yano wrote:
> Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
> ---
>  winsup/cygwin/fhandler/termios.cc | 18 ++++++++----------
>  1 file changed, 8 insertions(+), 10 deletions(-)
> 
> diff --git a/winsup/cygwin/fhandler/termios.cc b/winsup/cygwin/fhandler/termios.cc
> index 789ae0179..d106955dc 100644
> --- a/winsup/cygwin/fhandler/termios.cc
> +++ b/winsup/cygwin/fhandler/termios.cc
> @@ -704,22 +704,20 @@ static bool
>  is_console_app (const WCHAR *filename)
>  {
>    HANDLE h;
> -  const int id_offset = 92;
>    h = CreateFileW (filename, GENERIC_READ, FILE_SHARE_READ,
>  		   NULL, OPEN_EXISTING, 0, NULL);
>    char buf[1024];
>    DWORD n;
>    ReadFile (h, buf, sizeof (buf), &n, 0);
>    CloseHandle (h);
> -  char *p = (char *) memmem (buf, n, "PE\0\0", 4);
> -  if (p && p + id_offset < buf + n)
> -    return p[id_offset] == '\003'; /* 02: GUI, 03: console */
> -  else
> -    {
> -      wchar_t *e = wcsrchr (filename, L'.');
> -      if (e && (wcscasecmp (e, L".bat") == 0 || wcscasecmp (e, L".cmd") == 0))
> -	return true;
> -    }
> +  /* The offset of Subsystem is the same for both IMAGE_NT_HEADERS32 and
> +     IMAGE_NT_HEADERS64, so only IMAGE_NT_HEADERS32 is used here. */
> +  IMAGE_NT_HEADERS32 *p = (IMAGE_NT_HEADERS32 *) memmem (buf, n, "PE\0\0", 4);

Please use PIMAGE_NT_HEADERS instead and just drop the comment.
We don't support 32 bit anyway.


Thanks,
Corinna

Re: [PATCH] Cygwin: termios: Refactor the function is_console_app().

[Takashi Yano]

On Mon, 28 Aug 2023 12:58:31 +0200
Corinna Vinschen wrote:
> On Aug 28 18:21, Takashi Yano wrote:
> > Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
> > ---
> >  winsup/cygwin/fhandler/termios.cc | 18 ++++++++----------
> >  1 file changed, 8 insertions(+), 10 deletions(-)
> > 
> > diff --git a/winsup/cygwin/fhandler/termios.cc b/winsup/cygwin/fhandler/termios.cc
> > index 789ae0179..d106955dc 100644
> > --- a/winsup/cygwin/fhandler/termios.cc
> > +++ b/winsup/cygwin/fhandler/termios.cc
> > @@ -704,22 +704,20 @@ static bool
> >  is_console_app (const WCHAR *filename)
> >  {
> >    HANDLE h;
> > -  const int id_offset = 92;
> >    h = CreateFileW (filename, GENERIC_READ, FILE_SHARE_READ,
> >  		   NULL, OPEN_EXISTING, 0, NULL);
> >    char buf[1024];
> >    DWORD n;
> >    ReadFile (h, buf, sizeof (buf), &n, 0);
> >    CloseHandle (h);
> > -  char *p = (char *) memmem (buf, n, "PE\0\0", 4);
> > -  if (p && p + id_offset < buf + n)
> > -    return p[id_offset] == '\003'; /* 02: GUI, 03: console */
> > -  else
> > -    {
> > -      wchar_t *e = wcsrchr (filename, L'.');
> > -      if (e && (wcscasecmp (e, L".bat") == 0 || wcscasecmp (e, L".cmd") == 0))
> > -	return true;
> > -    }
> > +  /* The offset of Subsystem is the same for both IMAGE_NT_HEADERS32 and
> > +     IMAGE_NT_HEADERS64, so only IMAGE_NT_HEADERS32 is used here. */
> > +  IMAGE_NT_HEADERS32 *p = (IMAGE_NT_HEADERS32 *) memmem (buf, n, "PE\0\0", 4);
> 
> Please use PIMAGE_NT_HEADERS instead and just drop the comment.
> We don't support 32 bit anyway.

This function is used for determining whether a non-cygwin
app is console app or not. Even in 64-bit cygwin, 32-bit
non-cygwin app can be executed. So we should support both
32-bit and 64-bit binary.

However, using IMAGE_NT_HEADERS may be better. Thanks.

-- 
Takashi Yano <takashi.yano@nifty.ne.jp>