[gcc r14-1542] libiberty: On Windows, pass a >32k cmdline through a response file.

[gcc r14-1542] libiberty: On Windows, pass a >32k cmdline through a response file.

[Jonathan Yong]

https://gcc.gnu.org/g:180ebb8a24d24fc5b105f2257d6216f6dfde62df

commit r14-1542-g180ebb8a24d24fc5b105f2257d6216f6dfde62df
Author: Costas Argyris <costas.argyris@gmail.com>
Date:   Mon May 22 13:55:56 2023 +0100

    libiberty: On Windows, pass a >32k cmdline through a response file.
    
    pex-win32.c (win32_spawn): If the command line for CreateProcess
    exceeds the 32k Windows limit, try to store it in a temporary
    response file and call CreateProcess with @file instead (PR71850).
    
    Signed-off-by: Costas Argyris <costas.argyris@gmail.com>
    Signed-off-by: Jonathan Yong <10walls@gmail.com>
    
    libiberty/ChangeLog:
    
            * pex-win32.c (win32_spawn): Check command line length
            and generate a response file if necessary.
            (spawn_script): Adjust parameters.
            (pex_win32_exec_child): Ditto.
    
    Signed-off-by: Jonathan Yong <10walls@gmail.com>

Diff:
---
 libiberty/pex-win32.c | 57 +++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/libiberty/pex-win32.c b/libiberty/pex-win32.c
index 23c6c190a2c..0fd8b38734c 100644
--- a/libiberty/pex-win32.c
+++ b/libiberty/pex-win32.c
@@ -569,7 +569,8 @@ env_compare (const void *a_ptr, const void *b_ptr)
  * target is not actually an executable, such as if it is a shell script. */
 
 static pid_t
-win32_spawn (const char *executable,
+win32_spawn (struct pex_obj *obj,
+         const char *executable,
 	     BOOL search,
 	     char *const *argv,
              char *const *env, /* array of strings of the form: VAR=VALUE */
@@ -624,8 +625,37 @@ win32_spawn (const char *executable,
   cmdline = argv_to_cmdline (argv);
   if (!cmdline)
     goto exit;
-    
-  /* Create the child process.  */  
+  /* If cmdline is too large, CreateProcess will fail with a bad
+     'No such file or directory' error. Try passing it through a
+     temporary response file instead.  */
+  if (strlen (cmdline) > 32767)
+    {
+      char *response_file = make_temp_file ("");
+      /* Register the file for deletion by pex_free.  */
+      ++obj->remove_count;
+      obj->remove = XRESIZEVEC (char *, obj->remove, obj->remove_count);
+      obj->remove[obj->remove_count - 1] = response_file;
+      int fd = pex_win32_open_write (obj, response_file, 0, 0);
+      if (fd == -1)
+        goto exit;
+      FILE *f = pex_win32_fdopenw (obj, fd, 0);
+      /* Don't write argv[0] (program name) to the response file.  */
+      if (writeargv (&argv[1], f))
+        {
+          fclose (f);
+          goto exit;
+        }
+      fclose (f); /* Also closes fd and the underlying OS handle.  */
+      char *response_arg = concat ("@", response_file, NULL);
+      char *response_argv[3] = {argv[0], response_arg, NULL};
+      free (cmdline);
+      cmdline = argv_to_cmdline (response_argv);
+      free (response_arg);
+      if (!cmdline)
+        goto exit;
+    }
+  
+  /* Create the child process.  */
   if (CreateProcess (full_executable, cmdline,
 		      /*lpProcessAttributes=*/NULL,
 		      /*lpThreadAttributes=*/NULL,
@@ -645,7 +675,7 @@ win32_spawn (const char *executable,
   free (env_block);
   free (cmdline);
   free (full_executable);
-
+  
   return pid;
 }
 
@@ -653,7 +683,8 @@ win32_spawn (const char *executable,
    This function is called as a fallback if win32_spawn fails. */
 
 static pid_t
-spawn_script (const char *executable, char *const *argv,
+spawn_script (struct pex_obj *obj,
+              const char *executable, char *const *argv,
               char* const *env,
 	      DWORD dwCreationFlags,
 	      LPSTARTUPINFO si,
@@ -703,20 +734,20 @@ spawn_script (const char *executable, char *const *argv,
 	      executable = strrchr (executable1, '\\') + 1;
 	      if (!executable)
 		executable = executable1;
-	      pid = win32_spawn (executable, TRUE, argv, env,
+	      pid = win32_spawn (obj, executable, TRUE, argv, env,
 				 dwCreationFlags, si, pi);
 #else
 	      if (strchr (executable1, '\\') == NULL)
-		pid = win32_spawn (executable1, TRUE, argv, env,
+		pid = win32_spawn (obj, executable1, TRUE, argv, env,
 				   dwCreationFlags, si, pi);
 	      else if (executable1[0] != '\\')
-		pid = win32_spawn (executable1, FALSE, argv, env,
+		pid = win32_spawn (obj, executable1, FALSE, argv, env,
 				   dwCreationFlags, si, pi);
 	      else
 		{
 		  const char *newex = mingw_rootify (executable1);
 		  *avhere = newex;
-		  pid = win32_spawn (newex, FALSE, argv, env,
+		  pid = win32_spawn (obj, newex, FALSE, argv, env,
 				     dwCreationFlags, si, pi);
 		  if (executable1 != newex)
 		    free ((char *) newex);
@@ -726,7 +757,7 @@ spawn_script (const char *executable, char *const *argv,
 		      if (newex != executable1)
 			{
 			  *avhere = newex;
-			  pid = win32_spawn (newex, FALSE, argv, env,
+			  pid = win32_spawn (obj, newex, FALSE, argv, env,
 					     dwCreationFlags, si, pi);
 			  free ((char *) newex);
 			}
@@ -745,7 +776,7 @@ spawn_script (const char *executable, char *const *argv,
 /* Execute a child.  */
 
 static pid_t
-pex_win32_exec_child (struct pex_obj *obj ATTRIBUTE_UNUSED, int flags,
+pex_win32_exec_child (struct pex_obj *obj, int flags,
 		      const char *executable, char * const * argv,
                       char* const* env,
 		      int in, int out, int errdes,
@@ -841,10 +872,10 @@ pex_win32_exec_child (struct pex_obj *obj ATTRIBUTE_UNUSED, int flags,
   si.hStdError = stderr_handle;
 
   /* Create the child process.  */  
-  pid = win32_spawn (executable, (flags & PEX_SEARCH) != 0,
+  pid = win32_spawn (obj, executable, (flags & PEX_SEARCH) != 0,
 		     argv, env, dwCreationFlags, &si, &pi);
   if (pid == (pid_t) -1)
-    pid = spawn_script (executable, argv, env, dwCreationFlags,
+    pid = spawn_script (obj, executable, argv, env, dwCreationFlags,
                         &si, &pi);
   if (pid == (pid_t) -1)
     {