[RFC] Process files in parallel

[Tom de Vries <tdevries@suse.de>]

Hi,

Add an option -j <n> / --jobs <n> that allows multiple files to be processed
in parallel.  Note that this does not yet parallelize when multifile is used.

Consider the experiment do.sh:
...
ns=$(seq 1 10)

for n in $ns; do
    cp debug/cc1 $n
done

time.sh ./dwz $@ -lnone $ns
...

On a 4 smt-thread, dual core system we get:
...
$ for n in $(seq 1 4); do echo "N: $n"; ./do.sh -j $n; done
N: 1
maxmem: 1261744
real: 56.53
user: 53.24
system: 3.17
N: 2
maxmem: 1260216
real: 31.74
user: 58.84
system: 4.39
N: 3
maxmem: 1261868
real: 28.26
user: 75.65
system: 5.00
N: 4
maxmem: 1262036
real: 26.80
user: 87.31
system: 5.69
...

The sweet spot of real time reduction vs. extra user/system time seems to be
around 2, so we set the default -j to processors / 2.

Any comments?

Thanks,
- Tom

Process files in parallel

2021-03-23  Tom de Vries  <tdevries@suse.de>

	PR dwz/25951
	* args.c (max_forks): New var.
	(dwz_options, dwz_multi_file_options_help, usage): Add entries for -j <n>.
	(parse_args): Handle -j <n>.  Assign default value for max_forks.
	* args.h (max_forks): Declare.
	* dwz.1: Add entries for -j <n> / --jobs <n> entry.
	* dwz.c (dwz_files_1): Handle max_forks.

---
 args.c |  27 +++++++++++++--
 args.h |   1 +
 dwz.1  |   4 +++
 dwz.c  | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
 4 files changed, 137 insertions(+), 18 deletions(-)

diff --git a/args.c b/args.c
index cc8c717..11946b8 100644
--- a/args.c
+++ b/args.c
@@ -25,6 +25,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <error.h>
+#include <sys/sysinfo.h>
 
 #include "args.h"
 
@@ -52,6 +53,7 @@ int progress_p;
 int progress_mem_p;
 int import_opt_p = 1;
 int force_p;
+int max_forks = -1;
 
 enum deduplication_mode deduplication_mode = dm_inter_cu;
 
@@ -161,6 +163,7 @@ static struct option dwz_options[] =
   { "odr",		 no_argument,	    &odr, 1 },
   { "no-odr",		 no_argument,	    &odr, 0 },
   { "odr-mode",		 required_argument, &odr_mode_parsed, 1 },
+  { "jobs",		 required_argument, 0, 'j' },
   { NULL,		 no_argument,	    0, 0 }
 };
 
@@ -219,7 +222,9 @@ static struct option_help dwz_multi_file_options_help[] =
     " to multifile." },
   { "5", "dwarf-5", NULL, NULL,
     "Emit DWARF 5 standardized supplementary object files instead of"
-    " GNU extension .debug_altlink." }
+    " GNU extension .debug_altlink." },
+  { "j", "jobs", "<n>", "number of processors / 2",
+    "Process <n> files in parallel" }
 };
 
 /* Describe misc command line options.  */
@@ -363,7 +368,8 @@ usage (const char *progname, int failing)
 
   fprintf (stream,
 	   ("Usage:\n"
-	    "  %s [common options] [-h] [-m COMMONFILE] [-M NAME | -r] [FILES]\n"
+	    "  %s [common options] [-h] [-m COMMONFILE] [-M NAME | -r] [-j N]"
+	      " [FILES]\n"
 	    "  %s [common options] -o OUTFILE FILE\n"
 	    "  %s [ -v | -? ]\n"),
 	   progname, progname, progname);
@@ -487,7 +493,7 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
   while (1)
     {
       int option_index = -1;
-      int c = getopt_long (argc, argv, "m:o:qhl:L:M:r?v5", dwz_options,
+      int c = getopt_long (argc, argv, "m:o:j:qhl:L:M:r?v5", dwz_options,
 			   &option_index);
       if (c == -1)
 	break;
@@ -619,6 +625,13 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
 	case 'v':
 	  version ();
 	  break;
+
+	case 'j':
+	  l = strtoul (optarg, &end, 0);
+	  if (*end != '\0' || optarg == end || (unsigned int) l != l)
+	    error (1, 0, "invalid argument -j %s", optarg);
+	  max_forks = l;
+	  break;
 	}
     }
 
@@ -634,4 +647,12 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
 
   if (multifile_relative && multifile_name)
     error (1, 0, "-M and -r options can't be specified together");
+
+  if (max_forks == -1)
+    {
+      long nprocs = get_nprocs ();
+      /* Be conservative on max forks: 4 procs may be actually be 4 SMT threads
+	 with only 2 cores.  */
+      max_forks = nprocs / 2;
+    }
 }
diff --git a/args.h b/args.h
index c899003..ca71194 100644
--- a/args.h
+++ b/args.h
@@ -54,6 +54,7 @@ extern int progress_p;
 extern int progress_mem_p;
 extern int import_opt_p;
 extern int force_p;
+extern int max_forks;
 
 enum deduplication_mode
 {
diff --git a/dwz.1 b/dwz.1
index 19dc814..23fe878 100644
--- a/dwz.1
+++ b/dwz.1
@@ -111,6 +111,10 @@ Emit standard DWARF 5 Supplementary Object Files with \fI.debug_sup\fR and
 corresponding forms, instead of the GNU extension \fI.gnu_debugaltlink\fR
 and corresponding forms.
 .TP
+.B \-j <N> \-\-jobs <N>
+Process \fIN\fR files in parallel.  The default is processors / 2.  Disabled
+when multifile is used.
+.TP
 .B \-\-odr / \-\-no-odr
 .B Experimental.
 Enable/disable One-Definition-Rule optimization for C++ compilation units.
diff --git a/dwz.c b/dwz.c
index 245e540..07e1da1 100644
--- a/dwz.c
+++ b/dwz.c
@@ -34,6 +34,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/times.h>
+#include <sys/wait.h>
 
 #include <obstack.h>
 
@@ -16392,22 +16393,114 @@ dwz_files_1 (int nr_files, char *files[], bool hardlink,
   if (hardlink)
     hardlink = detect_hardlinks (nr_files, files, resa);
 
-  for (i = 0; i < nr_files; i++)
+  int nr_forks = 0;
+  if (max_forks > 1 && multifile == NULL)
     {
-      int thisret;
-      file = files[i];
-      struct file_result *res = &resa[i];
-      if (res->res == -2)
-	/* Skip hard links.  */
-	continue;
-      if (stats_p)
-	init_stats (file);
-      bool low_mem_p;
-      thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
-      if (thisret == 1)
-	ret = 1;
-      else if (!low_mem_p && resa[i].res >= 0)
-	successcount++;
+      pid_t pids[nr_files];
+      for (i = 0; i < nr_files; i++)
+	{
+	  int thisret;
+	  file = files[i];
+	  struct file_result *res = &resa[i];
+	  if (res->res == -2)
+	    /* Skip hard links.  */
+	    continue;
+
+	  if (nr_forks == max_forks)
+	    {
+	      int state;
+	      pid_t got_pid = waitpid (-1, &state, 0);
+	      if (!WIFEXITED (state))
+		error (1, 0, "Child dwz process got killed");
+	      thisret = WEXITSTATUS (state) & 0x3;
+	      bool low_mem_p = false;
+	      if (thisret == 2)
+		{
+		  thisret = 0;
+		  low_mem_p = true;
+		}
+	      (void)low_mem_p;
+	      res->res = (int)((WEXITSTATUS (state) & ~0x3) >> 2) - 3;
+	      if (thisret == 1)
+		ret = 1;
+	      nr_forks--;
+	      int j;
+	      for (j = 0; j < i; ++j)
+		if (pids[j] == got_pid)
+		  {
+		    pids[j] = 0;
+		    break;
+		  }
+	      assert (j < i);
+	    }
+
+	  pid_t fork_res = fork ();
+	  assert (fork_res != -1);
+	  if (fork_res == 0)
+	    {
+	      bool low_mem_p;
+	      thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
+	      /* Encode thisret, low_mem_p and res->res into return status.  */
+	      if (thisret == 0 && low_mem_p)
+		thisret = 2;
+	      assert (thisret >= 0 && thisret <= 2);
+	      assert (res->res >= -3);
+	      thisret = thisret + ((res->res + 3) << 2);
+	      return thisret;
+	    }
+	  else
+	    {
+	      pids[i] = fork_res;
+	      nr_forks++;
+	    }
+	}
+      for (i = 0; i < nr_files; i++)
+	{
+	  int thisret;
+	  file = files[i];
+	  struct file_result *res = &resa[i];
+	  if (res->res == -2)
+	    /* Skip hard links.  */
+	    continue;
+	  if (pids[i] == 0)
+	    continue;
+	  int state;
+	  pid_t got_pid = waitpid (pids[i], &state, 0);
+	  assert (got_pid == pids[i]);
+	  if (!WIFEXITED (state))
+	    error (1, 0, "Child dwz process got killed");
+	  thisret = WEXITSTATUS (state) & 0x3;
+	  bool low_mem_p = false;
+	  if (thisret == 2)
+	    {
+	      thisret = 0;
+	      low_mem_p = true;
+	    }
+	  (void)low_mem_p;
+	  res->res = (int)((WEXITSTATUS (state) & ~0x3) >> 2) - 3;
+	  if (thisret == 1)
+	    ret = 1;
+	}
+    }
+  else
+    {
+      for (i = 0; i < nr_files; i++)
+	{
+	  int thisret;
+	  file = files[i];
+	  struct file_result *res = &resa[i];
+	  if (res->res == -2)
+	    /* Skip hard links.  */
+	    continue;
+	  if (stats_p)
+	    init_stats (file);
+	  bool low_mem_p;
+	  thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
+	  if (thisret == 1)
+	    ret = 1;
+	  else if (!low_mem_p && resa[i].res >= 0)
+	    successcount++;
+	}
     }
 
   if (hardlink)