public inbox for gcc-patches@gcc.gnu.org
 help / color / mirror / Atom feed
* [PATCH] Come up with -flto=auto option.
@ 2019-07-23  8:55 Martin Liška
  2019-07-23  9:29 ` Jan Hubicka
                   ` (2 more replies)
  0 siblings, 3 replies; 54+ messages in thread
From: Martin Liška @ 2019-07-23  8:55 UTC (permalink / raw)
  To: gcc-patches; +Cc: Jan Hubicka, Michael Matz, Richard Biener

[-- Attachment #1: Type: text/plain, Size: 616 bytes --]

Hi.

As we as openSUSE started using -flto, I see it very handy to have
an option value that will automatically detect number of cores
that can be used for parallel LTRANS phase.

Thoughts?

gcc/ChangeLog:

2019-07-23  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document the new option value.
	* lto-wrapper.c (cpuset_popcount): New function
	is a copy of libgomp/config/linux/proc.c.
	(init_num_threads): Likewise.
	(run_gcc): Support -flto=auto.
---
 gcc/doc/invoke.texi |   3 ++
 gcc/lto-wrapper.c   | 124 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 1 deletion(-)



[-- Attachment #2: 0001-Come-up-with-flto-auto-option.patch --]
[-- Type: text/x-patch, Size: 5275 bytes --]

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 77a2d561e38..58656fbe1e1 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10398,6 +10398,9 @@ parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
 used.  The default value for @var{n} is 1.
 
+You can specify @var{auto} to automatically detect number of
+cores that will determine the number of parallel jobs.
+
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
 is useful when the Makefile calling GCC is already executing in parallel.
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 946897726d0..5451285f896 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1110,6 +1110,110 @@ cmp_priority (const void *a, const void *b)
   return *((const int *)b)-*((const int *)a);
 }
 
+/* Number of CPUs that can be used for parallel LTRANS phase.  */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+  /* glibc 2.7 and above provide a macro for this.  */
+  return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+  if (cpusetsize == sizeof (cpu_set_t))
+    /* glibc 2.6 and above provide a macro for this.  */
+    return CPU_COUNT (cpusetp);
+#endif
+  size_t i;
+  unsigned long ret = 0;
+  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+    {
+      unsigned long int mask = cpusetp->__bits[i];
+      if (mask == 0)
+	continue;
+      ret += __builtin_popcountl (mask);
+    }
+  return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads.  It would seem
+   this should be related to the number of cpus online.  */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+  cpuset_size = sizeof (cpu_set_t);
+#endif
+
+  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+  do
+    {
+      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+					cpusetp);
+      if (ret == 0)
+	{
+	  /* Count only the CPUs this process can use.  */
+	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+	  if (nthreads_var == 0)
+	    break;
+	  get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+	  unsigned long i;
+	  for (i = cpuset_size * 8; i; i--)
+	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+	      break;
+	  cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+	  return;
+	}
+      if (ret != EINVAL)
+	break;
+#ifdef CPU_ALLOC_SIZE
+      if (cpuset_size < sizeof (cpu_set_t))
+	cpuset_size = sizeof (cpu_set_t);
+      else
+	cpuset_size = cpuset_size * 2;
+      if (cpuset_size < 8 * sizeof (cpu_set_t))
+	cpusetp
+	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+      else
+	{
+	  /* Avoid fatal if too large memory allocation would be
+	     requested, e.g. kernel returning EINVAL all the time.  */
+	  void *p = realloc (cpusetp, cpuset_size);
+	  if (p == NULL)
+	    break;
+	  cpusetp = (cpu_set_t *) p;
+	}
+#else
+      break;
+#endif
+    }
+  while (1);
+  cpuset_size = 0;
+  nthreads_var = 1;
+  free (cpusetp);
+  cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
 
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
@@ -1124,6 +1228,7 @@ run_gcc (unsigned argc, char *argv[])
   const char *collect_gcc, *collect_gcc_options;
   int parallel = 0;
   int jobserver = 0;
+  int auto_parallel = 0;
   bool no_partition = false;
   struct cl_decoded_option *fdecoded_options = NULL;
   struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1251,6 +1356,11 @@ run_gcc (unsigned argc, char *argv[])
 	      jobserver = 1;
 	      parallel = 1;
 	    }
+	  else if (strcmp (option->arg, "auto") == 0)
+	    {
+	      auto_parallel = 1;
+	      parallel = 1;
+	    }
 	  else
 	    {
 	      parallel = atoi (option->arg);
@@ -1291,6 +1401,7 @@ run_gcc (unsigned argc, char *argv[])
     {
       lto_mode = LTO_MODE_LTO;
       jobserver = 0;
+      auto_parallel = 0;
       parallel = 0;
     }
 
@@ -1485,6 +1596,16 @@ cont1:
 
       if (jobserver)
 	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+      else if (auto_parallel)
+	{
+	  char buf[256];
+	  init_num_threads ();
+	  if (verbose)
+	    fprintf (stderr, "LTO parallelism level set to %ld\n",
+		     nthreads_var);
+	  sprintf (buf, "-fwpa=%ld", nthreads_var);
+	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+	}
       else if (parallel > 1)
 	{
 	  char buf[256];
@@ -1692,7 +1813,8 @@ cont:
 	  i = 3;
 	  if (!jobserver)
 	    {
-	      snprintf (jobs, 31, "-j%d", parallel);
+	      snprintf (jobs, 31, "-j%ld",
+			auto_parallel ? nthreads_var : parallel);
 	      new_argv[i++] = jobs;
 	    }
 	  new_argv[i++] = "all";


^ permalink raw reply	[flat|nested] 54+ messages in thread
end of thread, other threads:[~2019-08-12 15:00 UTC | newest]

Thread overview: 54+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-07-23  8:55 [PATCH] Come up with -flto=auto option Martin Liška
2019-07-23  9:29 ` Jan Hubicka
2019-07-23 10:34   ` [PATCH] Deduce automatically number of cores for -flto option Martin Liška
2019-07-24 15:47     ` Jeff Law
2019-07-29 13:37       ` Martin Liška
2019-07-30 13:47         ` Martin Liška
2019-07-31  1:23         ` Jakub Jelinek
2019-07-31  7:24           ` Martin Liška
2019-07-31  7:40             ` Jakub Jelinek
2019-07-31  7:49               ` Jan Hubicka
2019-07-31  7:50                 ` Martin Liška
2019-07-31  7:54               ` Martin Liška
2019-07-31  8:08                 ` Jakub Jelinek
2019-07-31  8:21                 ` Jan Hubicka
2019-07-31  8:37                   ` Martin Liška
2019-07-31  9:12                     ` Jakub Jelinek
2019-07-31  9:15                       ` Jan Hubicka
2019-07-31  9:17                         ` Jakub Jelinek
2019-07-31  9:22                           ` Jan Hubicka
2019-07-31 10:02                         ` Martin Liška
2019-07-31 12:02                           ` Jan Hubicka
2019-07-31 15:42                           ` Martin Liška
2019-08-01 13:19                             ` Jakub Jelinek
2019-08-01 14:34                               ` [PATCH] Properly detect working jobserver in gcc driver Martin Liška
2019-08-01 14:41                                 ` Jakub Jelinek
2019-08-02  6:30                                   ` Martin Liška
2019-08-02  7:45                                     ` Jakub Jelinek
2019-08-02  8:47                                       ` Martin Liška
2019-08-02  8:50                                         ` Jakub Jelinek
2019-08-02  9:04                                           ` Richard Biener
2019-08-02  9:08                                             ` Jan Hubicka
2019-08-02  9:15                                             ` Jan Hubicka
2019-08-02  9:19                                               ` Martin Liška
2019-08-02  9:55                                                 ` Richard Biener
2019-08-05  6:41                                                   ` Martin Liška
2019-08-09  8:14                                                     ` Martin Liška
2019-08-09  8:22                                                       ` Richard Biener
2019-08-09 12:51                                                         ` Martin Liška
2019-08-09 13:56                                                           ` Martin Liška
2019-08-12 15:18                                                             ` Jeff Law
2019-07-23 13:13   ` [PATCH] Come up with -flto=auto option Jeff Law
2019-07-23 13:22     ` Richard Biener
2019-07-23 13:57     ` Michael Matz
2019-07-23 14:00       ` Jeff Law
2019-07-23 14:27         ` Martin Liška
2019-07-23 22:56           ` Jeff Law
2019-07-24  6:59             ` Martin Liška
2019-07-24 15:16               ` Jeff Law
2019-07-23 22:32 ` Allan Sandfeld Jensen
2019-07-24  6:47   ` Martin Liška
2019-07-24  7:12     ` Allan Sandfeld Jensen
2019-07-24  7:15       ` Martin Liška
2019-07-24 11:09         ` Nathan Sidwell
2019-07-24 15:46 ` Jeff Law

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).