[PATCH] Come up with -flto=auto option.

[PATCH] Come up with -flto=auto option.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 616 bytes --]

Hi.

As we as openSUSE started using -flto, I see it very handy to have
an option value that will automatically detect number of cores
that can be used for parallel LTRANS phase.

Thoughts?

gcc/ChangeLog:

2019-07-23  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document the new option value.
	* lto-wrapper.c (cpuset_popcount): New function
	is a copy of libgomp/config/linux/proc.c.
	(init_num_threads): Likewise.
	(run_gcc): Support -flto=auto.
---
 gcc/doc/invoke.texi |   3 ++
 gcc/lto-wrapper.c   | 124 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 1 deletion(-)



[-- Attachment #2: 0001-Come-up-with-flto-auto-option.patch --]
[-- Type: text/x-patch, Size: 5275 bytes --]

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 77a2d561e38..58656fbe1e1 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10398,6 +10398,9 @@ parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
 used.  The default value for @var{n} is 1.
 
+You can specify @var{auto} to automatically detect number of
+cores that will determine the number of parallel jobs.
+
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
 is useful when the Makefile calling GCC is already executing in parallel.
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 946897726d0..5451285f896 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1110,6 +1110,110 @@ cmp_priority (const void *a, const void *b)
   return *((const int *)b)-*((const int *)a);
 }
 
+/* Number of CPUs that can be used for parallel LTRANS phase.  */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+  /* glibc 2.7 and above provide a macro for this.  */
+  return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+  if (cpusetsize == sizeof (cpu_set_t))
+    /* glibc 2.6 and above provide a macro for this.  */
+    return CPU_COUNT (cpusetp);
+#endif
+  size_t i;
+  unsigned long ret = 0;
+  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+    {
+      unsigned long int mask = cpusetp->__bits[i];
+      if (mask == 0)
+	continue;
+      ret += __builtin_popcountl (mask);
+    }
+  return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads.  It would seem
+   this should be related to the number of cpus online.  */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+  cpuset_size = sizeof (cpu_set_t);
+#endif
+
+  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+  do
+    {
+      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+					cpusetp);
+      if (ret == 0)
+	{
+	  /* Count only the CPUs this process can use.  */
+	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+	  if (nthreads_var == 0)
+	    break;
+	  get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+	  unsigned long i;
+	  for (i = cpuset_size * 8; i; i--)
+	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+	      break;
+	  cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+	  return;
+	}
+      if (ret != EINVAL)
+	break;
+#ifdef CPU_ALLOC_SIZE
+      if (cpuset_size < sizeof (cpu_set_t))
+	cpuset_size = sizeof (cpu_set_t);
+      else
+	cpuset_size = cpuset_size * 2;
+      if (cpuset_size < 8 * sizeof (cpu_set_t))
+	cpusetp
+	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+      else
+	{
+	  /* Avoid fatal if too large memory allocation would be
+	     requested, e.g. kernel returning EINVAL all the time.  */
+	  void *p = realloc (cpusetp, cpuset_size);
+	  if (p == NULL)
+	    break;
+	  cpusetp = (cpu_set_t *) p;
+	}
+#else
+      break;
+#endif
+    }
+  while (1);
+  cpuset_size = 0;
+  nthreads_var = 1;
+  free (cpusetp);
+  cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
 
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
@@ -1124,6 +1228,7 @@ run_gcc (unsigned argc, char *argv[])
   const char *collect_gcc, *collect_gcc_options;
   int parallel = 0;
   int jobserver = 0;
+  int auto_parallel = 0;
   bool no_partition = false;
   struct cl_decoded_option *fdecoded_options = NULL;
   struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1251,6 +1356,11 @@ run_gcc (unsigned argc, char *argv[])
 	      jobserver = 1;
 	      parallel = 1;
 	    }
+	  else if (strcmp (option->arg, "auto") == 0)
+	    {
+	      auto_parallel = 1;
+	      parallel = 1;
+	    }
 	  else
 	    {
 	      parallel = atoi (option->arg);
@@ -1291,6 +1401,7 @@ run_gcc (unsigned argc, char *argv[])
     {
       lto_mode = LTO_MODE_LTO;
       jobserver = 0;
+      auto_parallel = 0;
       parallel = 0;
     }
 
@@ -1485,6 +1596,16 @@ cont1:
 
       if (jobserver)
 	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+      else if (auto_parallel)
+	{
+	  char buf[256];
+	  init_num_threads ();
+	  if (verbose)
+	    fprintf (stderr, "LTO parallelism level set to %ld\n",
+		     nthreads_var);
+	  sprintf (buf, "-fwpa=%ld", nthreads_var);
+	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+	}
       else if (parallel > 1)
 	{
 	  char buf[256];
@@ -1692,7 +1813,8 @@ cont:
 	  i = 3;
 	  if (!jobserver)
 	    {
-	      snprintf (jobs, 31, "-j%d", parallel);
+	      snprintf (jobs, 31, "-j%ld",
+			auto_parallel ? nthreads_var : parallel);
 	      new_argv[i++] = jobs;
 	    }
 	  new_argv[i++] = "all";

Re: [PATCH] Come up with -flto=auto option.

[Jan Hubicka]

> Hi.
> 
> As we as openSUSE started using -flto, I see it very handy to have
> an option value that will automatically detect number of cores
> that can be used for parallel LTRANS phase.
> 
> Thoughts?
Hi,
great you found time to make this. It should become the default for
-flto IMO.

I think we also should auto-detect the case where jobserver is available
and in that case let make to connect to the outer jobserver.  (We should
also really convince make developers to invent way to connect to it w/o
the extra + role)

Honza
> 
> gcc/ChangeLog:
> 
> 2019-07-23  Martin Liska  <mliska@suse.cz>
> 
> 	* doc/invoke.texi: Document the new option value.
> 	* lto-wrapper.c (cpuset_popcount): New function
> 	is a copy of libgomp/config/linux/proc.c.
> 	(init_num_threads): Likewise.
> 	(run_gcc): Support -flto=auto.
> ---
>  gcc/doc/invoke.texi |   3 ++
>  gcc/lto-wrapper.c   | 124 +++++++++++++++++++++++++++++++++++++++++++-
>  2 files changed, 126 insertions(+), 1 deletion(-)
> 
> 

> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index 77a2d561e38..58656fbe1e1 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -10398,6 +10398,9 @@ parallel jobs by utilizing an installed @command{make} program.  The
>  environment variable @env{MAKE} may be used to override the program
>  used.  The default value for @var{n} is 1.
>  
> +You can specify @var{auto} to automatically detect number of
> +cores that will determine the number of parallel jobs.
> +
>  You can also specify @option{-flto=jobserver} to use GNU make's
>  job server mode to determine the number of parallel jobs. This
>  is useful when the Makefile calling GCC is already executing in parallel.
> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> index 946897726d0..5451285f896 100644
> --- a/gcc/lto-wrapper.c
> +++ b/gcc/lto-wrapper.c
> @@ -1110,6 +1110,110 @@ cmp_priority (const void *a, const void *b)
>    return *((const int *)b)-*((const int *)a);
>  }
>  
> +/* Number of CPUs that can be used for parallel LTRANS phase.  */
> +
> +static unsigned long nthreads_var = 0;
> +
> +#ifdef HAVE_PTHREAD_AFFINITY_NP
> +unsigned long cpuset_size;
> +static unsigned long get_cpuset_size;
> +cpu_set_t *cpusetp;
> +
> +unsigned long
> +static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
> +{
> +#ifdef CPU_COUNT_S
> +  /* glibc 2.7 and above provide a macro for this.  */
> +  return CPU_COUNT_S (cpusetsize, cpusetp);
> +#else
> +#ifdef CPU_COUNT
> +  if (cpusetsize == sizeof (cpu_set_t))
> +    /* glibc 2.6 and above provide a macro for this.  */
> +    return CPU_COUNT (cpusetp);
> +#endif
> +  size_t i;
> +  unsigned long ret = 0;
> +  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
> +  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
> +    {
> +      unsigned long int mask = cpusetp->__bits[i];
> +      if (mask == 0)
> +	continue;
> +      ret += __builtin_popcountl (mask);
> +    }
> +  return ret;
> +#endif
> +}
> +#endif
> +
> +/* At startup, determine the default number of threads.  It would seem
> +   this should be related to the number of cpus online.  */
> +
> +static void
> +init_num_threads (void)
> +{
> +#ifdef HAVE_PTHREAD_AFFINITY_NP
> +#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
> +  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
> +  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
> +#else
> +  cpuset_size = sizeof (cpu_set_t);
> +#endif
> +
> +  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
> +  do
> +    {
> +      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
> +					cpusetp);
> +      if (ret == 0)
> +	{
> +	  /* Count only the CPUs this process can use.  */
> +	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
> +	  if (nthreads_var == 0)
> +	    break;
> +	  get_cpuset_size = cpuset_size;
> +#ifdef CPU_ALLOC_SIZE
> +	  unsigned long i;
> +	  for (i = cpuset_size * 8; i; i--)
> +	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
> +	      break;
> +	  cpuset_size = CPU_ALLOC_SIZE (i);
> +#endif
> +	  return;
> +	}
> +      if (ret != EINVAL)
> +	break;
> +#ifdef CPU_ALLOC_SIZE
> +      if (cpuset_size < sizeof (cpu_set_t))
> +	cpuset_size = sizeof (cpu_set_t);
> +      else
> +	cpuset_size = cpuset_size * 2;
> +      if (cpuset_size < 8 * sizeof (cpu_set_t))
> +	cpusetp
> +	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
> +      else
> +	{
> +	  /* Avoid fatal if too large memory allocation would be
> +	     requested, e.g. kernel returning EINVAL all the time.  */
> +	  void *p = realloc (cpusetp, cpuset_size);
> +	  if (p == NULL)
> +	    break;
> +	  cpusetp = (cpu_set_t *) p;
> +	}
> +#else
> +      break;
> +#endif
> +    }
> +  while (1);
> +  cpuset_size = 0;
> +  nthreads_var = 1;
> +  free (cpusetp);
> +  cpusetp = NULL;
> +#endif
> +#ifdef _SC_NPROCESSORS_ONLN
> +  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
> +#endif
> +}
>  
>  /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
>  
> @@ -1124,6 +1228,7 @@ run_gcc (unsigned argc, char *argv[])
>    const char *collect_gcc, *collect_gcc_options;
>    int parallel = 0;
>    int jobserver = 0;
> +  int auto_parallel = 0;
>    bool no_partition = false;
>    struct cl_decoded_option *fdecoded_options = NULL;
>    struct cl_decoded_option *offload_fdecoded_options = NULL;
> @@ -1251,6 +1356,11 @@ run_gcc (unsigned argc, char *argv[])
>  	      jobserver = 1;
>  	      parallel = 1;
>  	    }
> +	  else if (strcmp (option->arg, "auto") == 0)
> +	    {
> +	      auto_parallel = 1;
> +	      parallel = 1;
> +	    }
>  	  else
>  	    {
>  	      parallel = atoi (option->arg);
> @@ -1291,6 +1401,7 @@ run_gcc (unsigned argc, char *argv[])
>      {
>        lto_mode = LTO_MODE_LTO;
>        jobserver = 0;
> +      auto_parallel = 0;
>        parallel = 0;
>      }
>  
> @@ -1485,6 +1596,16 @@ cont1:
>  
>        if (jobserver)
>  	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
> +      else if (auto_parallel)
> +	{
> +	  char buf[256];
> +	  init_num_threads ();
> +	  if (verbose)
> +	    fprintf (stderr, "LTO parallelism level set to %ld\n",
> +		     nthreads_var);
> +	  sprintf (buf, "-fwpa=%ld", nthreads_var);
> +	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
> +	}
>        else if (parallel > 1)
>  	{
>  	  char buf[256];
> @@ -1692,7 +1813,8 @@ cont:
>  	  i = 3;
>  	  if (!jobserver)
>  	    {
> -	      snprintf (jobs, 31, "-j%d", parallel);
> +	      snprintf (jobs, 31, "-j%ld",
> +			auto_parallel ? nthreads_var : parallel);
>  	      new_argv[i++] = jobs;
>  	    }
>  	  new_argv[i++] = "all";
> 

[PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 691 bytes --]

On 7/23/19 11:20 AM, Jan Hubicka wrote:
> Hi,
> great you found time to make this. It should become the default for
> -flto IMO.

Works for me. Then I'm suggesting to not come up with -flto=auto and
only document that -flto passed during linking will automatically detect
number of cores. It's the same what clang does.

> 
> I think we also should auto-detect the case where jobserver is available
> and in that case let make to connect to the outer jobserver.  (We should
> also really convince make developers to invent way to connect to it w/o
> the extra + role)

Good idea, however, I don't have any experience with make and I'm leaving that
to you as a follow up improvement.

Martin

[-- Attachment #2: 0001-Deduce-automatically-number-of-cores-for-flto-option.patch --]
[-- Type: text/x-patch, Size: 5874 bytes --]

From 72489809a50a52f37748fe9ca4e7382ef2faa00c Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 23 Jul 2019 10:14:31 +0200
Subject: [PATCH] Deduce automatically number of cores for -flto option.

gcc/ChangeLog:

2019-07-23  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document new behavior.
	* lto-wrapper.c (cpuset_popcount): New function
	is a copy of libgomp/config/linux/proc.c.
	(init_num_threads): Likewise.
	(run_gcc): Automatically detect core count for -flto.
---
 gcc/doc/invoke.texi |   3 +-
 gcc/lto-wrapper.c   | 121 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 122 insertions(+), 2 deletions(-)

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 77a2d561e38..f5bfea3f003 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10396,7 +10396,8 @@ If you specify the optional @var{n}, the optimization and code
 generation done at link time is executed in parallel using @var{n}
 parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
-used.  The default value for @var{n} is 1.
+used.  The default value for @var{n} is automatically detected based
+on number of cores.
 
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 946897726d0..7e29ecb3618 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1110,6 +1110,110 @@ cmp_priority (const void *a, const void *b)
   return *((const int *)b)-*((const int *)a);
 }
 
+/* Number of CPUs that can be used for parallel LTRANS phase.  */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+  /* glibc 2.7 and above provide a macro for this.  */
+  return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+  if (cpusetsize == sizeof (cpu_set_t))
+    /* glibc 2.6 and above provide a macro for this.  */
+    return CPU_COUNT (cpusetp);
+#endif
+  size_t i;
+  unsigned long ret = 0;
+  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+    {
+      unsigned long int mask = cpusetp->__bits[i];
+      if (mask == 0)
+	continue;
+      ret += __builtin_popcountl (mask);
+    }
+  return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads.  It would seem
+   this should be related to the number of cpus online.  */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+  cpuset_size = sizeof (cpu_set_t);
+#endif
+
+  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+  do
+    {
+      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+					cpusetp);
+      if (ret == 0)
+	{
+	  /* Count only the CPUs this process can use.  */
+	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+	  if (nthreads_var == 0)
+	    break;
+	  get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+	  unsigned long i;
+	  for (i = cpuset_size * 8; i; i--)
+	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+	      break;
+	  cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+	  return;
+	}
+      if (ret != EINVAL)
+	break;
+#ifdef CPU_ALLOC_SIZE
+      if (cpuset_size < sizeof (cpu_set_t))
+	cpuset_size = sizeof (cpu_set_t);
+      else
+	cpuset_size = cpuset_size * 2;
+      if (cpuset_size < 8 * sizeof (cpu_set_t))
+	cpusetp
+	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+      else
+	{
+	  /* Avoid fatal if too large memory allocation would be
+	     requested, e.g. kernel returning EINVAL all the time.  */
+	  void *p = realloc (cpusetp, cpuset_size);
+	  if (p == NULL)
+	    break;
+	  cpusetp = (cpu_set_t *) p;
+	}
+#else
+      break;
+#endif
+    }
+  while (1);
+  cpuset_size = 0;
+  nthreads_var = 1;
+  free (cpusetp);
+  cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
 
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
@@ -1124,6 +1228,7 @@ run_gcc (unsigned argc, char *argv[])
   const char *collect_gcc, *collect_gcc_options;
   int parallel = 0;
   int jobserver = 0;
+  int auto_parallel = 0;
   bool no_partition = false;
   struct cl_decoded_option *fdecoded_options = NULL;
   struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1260,6 +1365,8 @@ run_gcc (unsigned argc, char *argv[])
 	  /* Fallthru.  */
 
 	case OPT_flto:
+	  auto_parallel = 1;
+	  parallel = 1;
 	  lto_mode = LTO_MODE_WHOPR;
 	  break;
 
@@ -1291,6 +1398,7 @@ run_gcc (unsigned argc, char *argv[])
     {
       lto_mode = LTO_MODE_LTO;
       jobserver = 0;
+      auto_parallel = 0;
       parallel = 0;
     }
 
@@ -1485,6 +1593,16 @@ cont1:
 
       if (jobserver)
 	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+      else if (auto_parallel)
+	{
+	  char buf[256];
+	  init_num_threads ();
+	  if (verbose)
+	    fprintf (stderr, "LTO parallelism level set to %ld\n",
+		     nthreads_var);
+	  sprintf (buf, "-fwpa=%ld", nthreads_var);
+	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+	}
       else if (parallel > 1)
 	{
 	  char buf[256];
@@ -1692,7 +1810,8 @@ cont:
 	  i = 3;
 	  if (!jobserver)
 	    {
-	      snprintf (jobs, 31, "-j%d", parallel);
+	      snprintf (jobs, 31, "-j%ld",
+			auto_parallel ? nthreads_var : parallel);
 	      new_argv[i++] = jobs;
 	    }
 	  new_argv[i++] = "all";
-- 
2.22.0

Re: [PATCH] Come up with -flto=auto option.

[Jeff Law]

On 7/23/19 3:20 AM, Jan Hubicka wrote:
>> Hi.
>>
>> As we as openSUSE started using -flto, I see it very handy to have
>> an option value that will automatically detect number of cores
>> that can be used for parallel LTRANS phase.
>>
>> Thoughts?
> Hi,
> great you found time to make this. It should become the default for
> -flto IMO.
I was going to hack it into the rpm configury bits since we have access
to the # cores there.  But an auto-selector within GCC is even better.

BTW, isn't this all going to wreck havoc with reproducible builds since
partitioning can affect code generation?  That's one of our open
questions...

Jeff

Re: [PATCH] Come up with -flto=auto option.

[Richard Biener]

On Tue, Jul 23, 2019 at 3:09 PM Jeff Law <law@redhat.com> wrote:
>
> On 7/23/19 3:20 AM, Jan Hubicka wrote:
> >> Hi.
> >>
> >> As we as openSUSE started using -flto, I see it very handy to have
> >> an option value that will automatically detect number of cores
> >> that can be used for parallel LTRANS phase.
> >>
> >> Thoughts?
> > Hi,
> > great you found time to make this. It should become the default for
> > -flto IMO.
> I was going to hack it into the rpm configury bits since we have access
> to the # cores there.  But an auto-selector within GCC is even better.
>
> BTW, isn't this all going to wreck havoc with reproducible builds since
> partitioning can affect code generation?  That's one of our open
> questions...

Partitioning is independent on N in -flto=N it's just dependent on
the LTO partitioning --params.

Richard.

> Jeff

Re: [PATCH] Come up with -flto=auto option.

[Michael Matz]

Hi,

On Tue, 23 Jul 2019, Jeff Law wrote:

> > great you found time to make this. It should become the default for
> > -flto IMO.
> I was going to hack it into the rpm configury bits since we have access
> to the # cores there.  But an auto-selector within GCC is even better.
> 
> BTW, isn't this all going to wreck havoc with reproducible builds since 
> partitioning can affect code generation?  That's one of our open 
> questions...

See Richi for this, but the reason for -flto=auto (or just -flto, or 
whatever the endresult will be) is _also_ reproducible builds, because 
some packages like to encode the compile flags into their binaries and 
hence would change depending on build host just because of "-flto=32" vs. 
"-flto=64" even when the code remains exactly the same.


Ciao,
Michael.

Re: [PATCH] Come up with -flto=auto option.

[Jeff Law]

On 7/23/19 7:50 AM, Michael Matz wrote:
> Hi,
> 
> On Tue, 23 Jul 2019, Jeff Law wrote:
> 
>>> great you found time to make this. It should become the default for
>>> -flto IMO.
>> I was going to hack it into the rpm configury bits since we have access
>> to the # cores there.  But an auto-selector within GCC is even better.
>>
>> BTW, isn't this all going to wreck havoc with reproducible builds since 
>> partitioning can affect code generation?  That's one of our open 
>> questions...
> 
> See Richi for this, but the reason for -flto=auto (or just -flto, or 
> whatever the endresult will be) is _also_ reproducible builds, because 
> some packages like to encode the compile flags into their binaries and 
> hence would change depending on build host just because of "-flto=32" vs. 
> "-flto=64" even when the code remains exactly the same.
Makes sense.

What did you end up doing with old autoconf scripts that aren't LTO
safe?  Many of the older style tests to see if a function exists are
broken by LTO.  I've seen more issues with this than anything in the LTO
space so far.

We're capturing config.h files and comparing them with and without LTO
to at least be able to enumerate where these issues are.   Sadly this
test gets lots of false positives due to flag and timestamp encodings
into the config.h files :(

jeff

Re: [PATCH] Come up with -flto=auto option.

[Martin Liška]

On 7/23/19 3:57 PM, Jeff Law wrote:
> On 7/23/19 7:50 AM, Michael Matz wrote:
>> Hi,
>>
>> On Tue, 23 Jul 2019, Jeff Law wrote:
>>
>>>> great you found time to make this. It should become the default for
>>>> -flto IMO.
>>> I was going to hack it into the rpm configury bits since we have access
>>> to the # cores there.  But an auto-selector within GCC is even better.
>>>
>>> BTW, isn't this all going to wreck havoc with reproducible builds since 
>>> partitioning can affect code generation?  That's one of our open 
>>> questions...
>>
>> See Richi for this, but the reason for -flto=auto (or just -flto, or 
>> whatever the endresult will be) is _also_ reproducible builds, because 
>> some packages like to encode the compile flags into their binaries and 
>> hence would change depending on build host just because of "-flto=32" vs. 
>> "-flto=64" even when the code remains exactly the same.
> Makes sense.
> 
> What did you end up doing with old autoconf scripts that aren't LTO
> safe?  Many of the older style tests to see if a function exists are
> broken by LTO.  I've seen more issues with this than anything in the LTO
> space so far.

Well, I've seen some of these failures, but only a few.

Martin

> 
> We're capturing config.h files and comparing them with and without LTO
> to at least be able to enumerate where these issues are.   Sadly this
> test gets lots of false positives due to flag and timestamp encodings
> into the config.h files :(
> 
> jeff
> 

Re: [PATCH] Come up with -flto=auto option.

[Allan Sandfeld Jensen]

On Dienstag, 23. Juli 2019 10:30:07 CEST Martin Liška wrote:
> Hi.
> 
> As we as openSUSE started using -flto, I see it very handy to have
> an option value that will automatically detect number of cores
> that can be used for parallel LTRANS phase.
> 
> Thoughts?
> 
That's really nice. 

How much extra work would it be to make it support a posix make jobserver? 

As far as I understand, you would need to guess a partition size first (as 
your patch here does), but then only start each job when given a token from 
the jobserver FD.

With that the integration to existing build infrastructure would be optimal.

Cheers
'Allan

Re: [PATCH] Come up with -flto=auto option.

[Jeff Law]

On 7/23/19 8:23 AM, Martin Liška wrote:
> On 7/23/19 3:57 PM, Jeff Law wrote:
>> On 7/23/19 7:50 AM, Michael Matz wrote:
>>> Hi,
>>>
>>> On Tue, 23 Jul 2019, Jeff Law wrote:
>>>
>>>>> great you found time to make this. It should become the default for
>>>>> -flto IMO.
>>>> I was going to hack it into the rpm configury bits since we have access
>>>> to the # cores there.  But an auto-selector within GCC is even better.
>>>>
>>>> BTW, isn't this all going to wreck havoc with reproducible builds since 
>>>> partitioning can affect code generation?  That's one of our open 
>>>> questions...
>>>
>>> See Richi for this, but the reason for -flto=auto (or just -flto, or 
>>> whatever the endresult will be) is _also_ reproducible builds, because 
>>> some packages like to encode the compile flags into their binaries and 
>>> hence would change depending on build host just because of "-flto=32" vs. 
>>> "-flto=64" even when the code remains exactly the same.
>> Makes sense.
>>
>> What did you end up doing with old autoconf scripts that aren't LTO
>> safe?  Many of the older style tests to see if a function exists are
>> broken by LTO.  I've seen more issues with this than anything in the LTO
>> space so far.
> 
> Well, I've seen some of these failures, but only a few.
Many appear to be silent, possibly not really affecting anything (like
all the packages that test for doprnt, but really don't care about it in
the end).    But there were enough real failures that I put in auditing
to detect any cases where we get different config.h files with LTO vs
non-LTO and that is tripping often enough to have my concerns about how
much work it's going to be to get everything fixed.


But still, overall we're moving forward.  Next step is to get everything
classified into buckets and start iterating.  Presumably you'd be open
to a google doc of some kind where we can coordinate any such efforts?

jeff

ps.  I'm on PTO July 25 to Aug 5, so not much is going to happen in the
next couple weeks :-)

Re: [PATCH] Come up with -flto=auto option.

[Martin Liška]

On 7/24/19 12:11 AM, Allan Sandfeld Jensen wrote:
> On Dienstag, 23. Juli 2019 10:30:07 CEST Martin Liška wrote:
>> Hi.
>>
>> As we as openSUSE started using -flto, I see it very handy to have
>> an option value that will automatically detect number of cores
>> that can be used for parallel LTRANS phase.
>>
>> Thoughts?
>>
> That's really nice. 
> 
> How much extra work would it be to make it support a posix make jobserver? 

We do support it via -flto=jobserver:

           You can also specify -flto=jobserver to use GNU make's job server mode to determine the number of parallel jobs. This is useful when the Makefile calling GCC is already executing in parallel.  You must prepend a + to the command recipe in the parent
           Makefile for this to work.  This option likely only works if MAKE is GNU make.

Problem is that nowadays you how much more common make systems like ninja, meson and others
that probably do not support that.

> 
> As far as I understand, you would need to guess a partition size first (as 
> your patch here does), but then only start each job when given a token from 
> the jobserver FD.
> 
> With that the integration to existing build infrastructure would be optimal.

Fully agree with you. Maybe a new (more generic) infrastructure (API) would be welcome.

Martin

> 
> Cheers
> 'Allan
> 
> 

Re: [PATCH] Come up with -flto=auto option.

[Martin Liška]

On 7/24/19 12:32 AM, Jeff Law wrote:
> On 7/23/19 8:23 AM, Martin Liška wrote:
>> On 7/23/19 3:57 PM, Jeff Law wrote:
>>> On 7/23/19 7:50 AM, Michael Matz wrote:
>>>> Hi,
>>>>
>>>> On Tue, 23 Jul 2019, Jeff Law wrote:
>>>>
>>>>>> great you found time to make this. It should become the default for
>>>>>> -flto IMO.
>>>>> I was going to hack it into the rpm configury bits since we have access
>>>>> to the # cores there.  But an auto-selector within GCC is even better.
>>>>>
>>>>> BTW, isn't this all going to wreck havoc with reproducible builds since 
>>>>> partitioning can affect code generation?  That's one of our open 
>>>>> questions...
>>>>
>>>> See Richi for this, but the reason for -flto=auto (or just -flto, or 
>>>> whatever the endresult will be) is _also_ reproducible builds, because 
>>>> some packages like to encode the compile flags into their binaries and 
>>>> hence would change depending on build host just because of "-flto=32" vs. 
>>>> "-flto=64" even when the code remains exactly the same.
>>> Makes sense.
>>>
>>> What did you end up doing with old autoconf scripts that aren't LTO
>>> safe?  Many of the older style tests to see if a function exists are
>>> broken by LTO.  I've seen more issues with this than anything in the LTO
>>> space so far.
>>
>> Well, I've seen some of these failures, but only a few.
> Many appear to be silent, possibly not really affecting anything (like
> all the packages that test for doprnt, but really don't care about it in
> the end).    But there were enough real failures that I put in auditing
> to detect any cases where we get different config.h files with LTO vs
> non-LTO and that is tripping often enough to have my concerns about how
> much work it's going to be to get everything fixed.

I see.

> 
> 
> But still, overall we're moving forward.  Next step is to get everything
> classified into buckets and start iterating.  Presumably you'd be open
> to a google doc of some kind where we can coordinate any such efforts?

Sure, I'm open. In the meantime, I've got a META issue that I use for tracking
of LTO-related issues in openSUSE:
https://bugzilla.opensuse.org/show_bug.cgi?id=1133084

Martin

> 
> jeff
> 
> ps.  I'm on PTO July 25 to Aug 5, so not much is going to happen in the
> next couple weeks :-)
> 

Re: [PATCH] Come up with -flto=auto option.

[Allan Sandfeld Jensen]

On Mittwoch, 24. Juli 2019 08:45:21 CEST Martin Liška wrote:
> On 7/24/19 12:11 AM, Allan Sandfeld Jensen wrote:
> > On Dienstag, 23. Juli 2019 10:30:07 CEST Martin Liška wrote:
> >> Hi.
> >> 
> >> As we as openSUSE started using -flto, I see it very handy to have
> >> an option value that will automatically detect number of cores
> >> that can be used for parallel LTRANS phase.
> >> 
> >> Thoughts?
> > 
> > That's really nice.
> > 
> > How much extra work would it be to make it support a posix make jobserver?
> 
> We do support it via -flto=jobserver:
> 
Good to know :)

> 
> Problem is that nowadays you how much more common make systems like ninja,
> meson and others that probably do not support that.
> 
There are patches to enable it in ninja, and I know some Linux distros apply 
the patches by default. Though that is more listening, so it probably requires 
launching ninja using make, if you want to be able to pass it own to gcc.

'Allan

Re: [PATCH] Come up with -flto=auto option.

[Martin Liška]

On 7/24/19 9:03 AM, Allan Sandfeld Jensen wrote:
> On Mittwoch, 24. Juli 2019 08:45:21 CEST Martin Liška wrote:
>> On 7/24/19 12:11 AM, Allan Sandfeld Jensen wrote:
>>> On Dienstag, 23. Juli 2019 10:30:07 CEST Martin Liška wrote:
>>>> Hi.
>>>>
>>>> As we as openSUSE started using -flto, I see it very handy to have
>>>> an option value that will automatically detect number of cores
>>>> that can be used for parallel LTRANS phase.
>>>>
>>>> Thoughts?
>>>
>>> That's really nice.
>>>
>>> How much extra work would it be to make it support a posix make jobserver?
>>
>> We do support it via -flto=jobserver:
>>
> Good to know :)
> 
>>
>> Problem is that nowadays you how much more common make systems like ninja,
>> meson and others that probably do not support that.
>>
> There are patches to enable it in ninja, and I know some Linux distros apply 
> the patches by default. Though that is more listening, so it probably requires 
> launching ninja using make, if you want to be able to pass it own to gcc.

Btw. I know that Nathan will need a much closer integration between a compiler
and a build system for hit C++ modules work. That can be a good opportunity
for the LTO to utilize the infrastructure as well.

Martin

> 
> 'Allan
> 
> 

Re: [PATCH] Come up with -flto=auto option.

[Nathan Sidwell]

On 7/24/19 3:12 AM, Martin Liška wrote:
> On 7/24/19 9:03 AM, Allan Sandfeld Jensen wrote:

>> There are patches to enable it in ninja, and I know some Linux distros apply
>> the patches by default. Though that is more listening, so it probably requires
>> launching ninja using make, if you want to be able to pass it own to gcc.
> 
> Btw. I know that Nathan will need a much closer integration between a compiler
> and a build system for hit C++ modules work. That can be a good opportunity
> for the LTO to utilize the infrastructure as well.
> 

Yes indeed, I do mention LTO requirements as a foil against 'everything 
is different' complaints about modules :)  FWIW WG21 (C++ std) Study 
Group 15 is the tooling group, which is where build system people hang 
out.  If anyone's interested in joining email me.

nathan

-- 
Nathan Sidwell

Re: [PATCH] Come up with -flto=auto option.

[Jeff Law]

On 7/24/19 12:47 AM, Martin Liška wrote:
> On 7/24/19 12:32 AM, Jeff Law wrote:
>> On 7/23/19 8:23 AM, Martin Liška wrote:
>>> On 7/23/19 3:57 PM, Jeff Law wrote:
>>>> On 7/23/19 7:50 AM, Michael Matz wrote:
>>>>> Hi,
>>>>>
>>>>> On Tue, 23 Jul 2019, Jeff Law wrote:
>>>>>
>>>>>>> great you found time to make this. It should become the default for
>>>>>>> -flto IMO.
>>>>>> I was going to hack it into the rpm configury bits since we have access
>>>>>> to the # cores there.  But an auto-selector within GCC is even better.
>>>>>>
>>>>>> BTW, isn't this all going to wreck havoc with reproducible builds since 
>>>>>> partitioning can affect code generation?  That's one of our open 
>>>>>> questions...
>>>>>
>>>>> See Richi for this, but the reason for -flto=auto (or just -flto, or 
>>>>> whatever the endresult will be) is _also_ reproducible builds, because 
>>>>> some packages like to encode the compile flags into their binaries and 
>>>>> hence would change depending on build host just because of "-flto=32" vs. 
>>>>> "-flto=64" even when the code remains exactly the same.
>>>> Makes sense.
>>>>
>>>> What did you end up doing with old autoconf scripts that aren't LTO
>>>> safe?  Many of the older style tests to see if a function exists are
>>>> broken by LTO.  I've seen more issues with this than anything in the LTO
>>>> space so far.
>>>
>>> Well, I've seen some of these failures, but only a few.
>> Many appear to be silent, possibly not really affecting anything (like
>> all the packages that test for doprnt, but really don't care about it in
>> the end).    But there were enough real failures that I put in auditing
>> to detect any cases where we get different config.h files with LTO vs
>> non-LTO and that is tripping often enough to have my concerns about how
>> much work it's going to be to get everything fixed.
> 
> I see.
> 
>>
>>
>> But still, overall we're moving forward.  Next step is to get everything
>> classified into buckets and start iterating.  Presumably you'd be open
>> to a google doc of some kind where we can coordinate any such efforts?
> 
> Sure, I'm open. In the meantime, I've got a META issue that I use for tracking
> of LTO-related issues in openSUSE:
> https://bugzilla.opensuse.org/show_bug.cgi?id=1133084
Sounds good.  I'll try to get it populated and at least the first level
categorization done today.  Matthew B. may do some additional analysis
while I'm offline.

I've got scripts which can query the jenkins build failure plugin to
help with that first level categorization.  So, in theory, I can query
for all the configury differences or query for any cases where LTO
exposed a fatal warning,  testsuite failures, etc.

I don't want to burn a lot of time on the config.h stuff right now.  At
this stage we're more interested in the other failure categories.  But
I'm going to have to figure out something shortly after returning when I
put in the Fedora 32 change request.

I've got your META BZ in one of my tabs.  I've referred to it several
times over the last few weeks :-)

jeff

Re: [PATCH] Come up with -flto=auto option.

[Jeff Law]

On 7/23/19 2:30 AM, Martin Liška wrote:
> Hi.
> 
> As we as openSUSE started using -flto, I see it very handy to have
> an option value that will automatically detect number of cores
> that can be used for parallel LTRANS phase.
> 
> Thoughts?
> 
> gcc/ChangeLog:
> 
> 2019-07-23  Martin Liska  <mliska@suse.cz>
> 
> 	* doc/invoke.texi: Document the new option value.
> 	* lto-wrapper.c (cpuset_popcount): New function
> 	is a copy of libgomp/config/linux/proc.c.
> 	(init_num_threads): Likewise.
> 	(run_gcc): Support -flto=auto.
OK.  I won't be at all surprised if this causes headaches on some hosts,
so please watch for Solaris, AIX, etc etc issues.

Jeff

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jeff Law]

On 7/23/19 4:15 AM, Martin Liška wrote:
> On 7/23/19 11:20 AM, Jan Hubicka wrote:
>> Hi,
>> great you found time to make this. It should become the default for
>> -flto IMO.
> 
> Works for me. Then I'm suggesting to not come up with -flto=auto and
> only document that -flto passed during linking will automatically detect
> number of cores. It's the same what clang does.
This variant is fine too.  Same caveats apply with the non-linux hosts.

jeff

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 198 bytes --]

Hi.

I'm sending v2 of the patch that can newly auto-detect make
jobserver and use it.

Patch can bootstrap on x86_64-linux-gnu and survives regression tests.

Ready to be installed?
Thanks,
Martin

[-- Attachment #2: 0001-Deduce-automatically-number-of-cores-for-flto-option.patch --]
[-- Type: text/x-patch, Size: 7239 bytes --]

From df747a46241dcdb8ad055071f9e0605c9f469268 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 23 Jul 2019 10:14:31 +0200
Subject: [PATCH 1/2] Deduce automatically number of cores for -flto option.

gcc/ChangeLog:

2019-07-23  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document new behavior.
	* lto-wrapper.c (cpuset_popcount): New function
	is a copy of libgomp/config/linux/proc.c.
	(init_num_threads): Likewise.
	(run_gcc): Automatically detect core count for -flto.
	(jobserver_active_p): New function.
---
 gcc/doc/invoke.texi |   3 +-
 gcc/lto-wrapper.c   | 167 ++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 162 insertions(+), 8 deletions(-)

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 77a2d561e38..f5bfea3f003 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10396,7 +10396,8 @@ If you specify the optional @var{n}, the optimization and code
 generation done at link time is executed in parallel using @var{n}
 parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
-used.  The default value for @var{n} is 1.
+used.  The default value for @var{n} is automatically detected based
+on number of cores.
 
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 946897726d0..353187c6043 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1110,6 +1110,136 @@ cmp_priority (const void *a, const void *b)
   return *((const int *)b)-*((const int *)a);
 }
 
+/* Number of CPUs that can be used for parallel LTRANS phase.  */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+  /* glibc 2.7 and above provide a macro for this.  */
+  return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+  if (cpusetsize == sizeof (cpu_set_t))
+    /* glibc 2.6 and above provide a macro for this.  */
+    return CPU_COUNT (cpusetp);
+#endif
+  size_t i;
+  unsigned long ret = 0;
+  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+    {
+      unsigned long int mask = cpusetp->__bits[i];
+      if (mask == 0)
+	continue;
+      ret += __builtin_popcountl (mask);
+    }
+  return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads.  It would seem
+   this should be related to the number of cpus online.  */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+  cpuset_size = sizeof (cpu_set_t);
+#endif
+
+  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+  do
+    {
+      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+					cpusetp);
+      if (ret == 0)
+	{
+	  /* Count only the CPUs this process can use.  */
+	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+	  if (nthreads_var == 0)
+	    break;
+	  get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+	  unsigned long i;
+	  for (i = cpuset_size * 8; i; i--)
+	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+	      break;
+	  cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+	  return;
+	}
+      if (ret != EINVAL)
+	break;
+#ifdef CPU_ALLOC_SIZE
+      if (cpuset_size < sizeof (cpu_set_t))
+	cpuset_size = sizeof (cpu_set_t);
+      else
+	cpuset_size = cpuset_size * 2;
+      if (cpuset_size < 8 * sizeof (cpu_set_t))
+	cpusetp
+	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+      else
+	{
+	  /* Avoid fatal if too large memory allocation would be
+	     requested, e.g. kernel returning EINVAL all the time.  */
+	  void *p = realloc (cpusetp, cpuset_size);
+	  if (p == NULL)
+	    break;
+	  cpusetp = (cpu_set_t *) p;
+	}
+#else
+      break;
+#endif
+    }
+  while (1);
+  cpuset_size = 0;
+  nthreads_var = 1;
+  free (cpusetp);
+  cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
+
+/* FIXME: once using -std=c11, we can use std::thread::hardware_concurrency.  */
+
+/* Return true when a jobserver is running and can accept a job.  */
+
+static bool
+jobserver_active_p (void)
+{
+  const char *makeflags = getenv ("MAKEFLAGS");
+  if (makeflags == NULL)
+    return false;
+
+  const char *needle = "--jobserver-auth=";
+  const char *n = strstr (makeflags, needle);
+  if (n == NULL)
+    return false;
+
+  int rfd = -1;
+  int wfd = -1;
+
+  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+	  && rfd > 0
+	  && wfd > 0
+	  && fcntl (rfd, F_GETFD) >= 0
+	  && fcntl (wfd, F_GETFD) >= 0);
+}
 
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
@@ -1122,8 +1252,10 @@ run_gcc (unsigned argc, char *argv[])
   char *list_option_full = NULL;
   const char *linker_output = NULL;
   const char *collect_gcc, *collect_gcc_options;
-  int parallel = 0;
+  /* Make linking parallel by default.  */
+  int parallel = 1;
   int jobserver = 0;
+  int auto_parallel = 0;
   bool no_partition = false;
   struct cl_decoded_option *fdecoded_options = NULL;
   struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1247,10 +1379,7 @@ run_gcc (unsigned argc, char *argv[])
 
 	case OPT_flto_:
 	  if (strcmp (option->arg, "jobserver") == 0)
-	    {
-	      jobserver = 1;
-	      parallel = 1;
-	    }
+	    jobserver = 1;
 	  else
 	    {
 	      parallel = atoi (option->arg);
@@ -1291,8 +1420,17 @@ run_gcc (unsigned argc, char *argv[])
     {
       lto_mode = LTO_MODE_LTO;
       jobserver = 0;
+      auto_parallel = 0;
       parallel = 0;
     }
+  else if (!jobserver && parallel)
+    {
+      /* If there's no explicit usage of jobserver and
+	 parallel is enabled, then automatically detect
+	 jobserver or number of cores.  */
+      auto_parallel = 1;
+      jobserver = jobserver_active_p ();
+    }
 
   if (linker_output)
     {
@@ -1484,7 +1622,21 @@ cont1:
       strcpy (tmp, ltrans_output_file);
 
       if (jobserver)
-	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+	{
+	  if (verbose)
+	    fprintf (stderr, "Using make jobserver\n");
+	  obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+	}
+      else if (auto_parallel)
+	{
+	  char buf[256];
+	  init_num_threads ();
+	  if (verbose)
+	    fprintf (stderr, "LTO parallelism level set to %ld\n",
+		     nthreads_var);
+	  sprintf (buf, "-fwpa=%ld", nthreads_var);
+	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+	}
       else if (parallel > 1)
 	{
 	  char buf[256];
@@ -1692,7 +1844,8 @@ cont:
 	  i = 3;
 	  if (!jobserver)
 	    {
-	      snprintf (jobs, 31, "-j%d", parallel);
+	      snprintf (jobs, 31, "-j%ld",
+			auto_parallel ? nthreads_var : parallel);
 	      new_argv[i++] = jobs;
 	    }
 	  new_argv[i++] = "all";
-- 
2.22.0

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

On 7/29/19 3:35 PM, Martin Liška wrote:
> Hi.
> 
> I'm sending v2 of the patch that can newly auto-detect make
> jobserver and use it.
> 
> Patch can bootstrap on x86_64-linux-gnu and survives regression tests.
> 
> Ready to be installed?
> Thanks,
> Martin
> 

Ok, I've discussed the latest version also with Honza and he's fine.
So I'm going to install the patch.

Thanks,
Martin

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Mon, Jul 29, 2019 at 03:35:08PM +0200, Martin Liška wrote:
> I'm sending v2 of the patch that can newly auto-detect make
> jobserver and use it.
> 
> Patch can bootstrap on x86_64-linux-gnu and survives regression tests.
> 
> Ready to be installed?
> Thanks,
> Martin

> >From df747a46241dcdb8ad055071f9e0605c9f469268 Mon Sep 17 00:00:00 2001
> From: Martin Liska <mliska@suse.cz>
> Date: Tue, 23 Jul 2019 10:14:31 +0200
> Subject: [PATCH 1/2] Deduce automatically number of cores for -flto option.
> 
> gcc/ChangeLog:
> 
> 2019-07-23  Martin Liska  <mliska@suse.cz>
> 
> 	* doc/invoke.texi: Document new behavior.
> 	* lto-wrapper.c (cpuset_popcount): New function
> 	is a copy of libgomp/config/linux/proc.c.
> 	(init_num_threads): Likewise.
> 	(run_gcc): Automatically detect core count for -flto.
> 	(jobserver_active_p): New function.

This broke a lot of tests.
The logs show
spawn -ignore SIGHUP /home/jakub/src/gcc/obj31/gcc/xgcc -B/home/jakub/src/gcc/obj31/gcc/ c_lto_pr83954_0.o c_lto_pr83954_1.o -fno-diagnostics-show-
caret -fno-diagnostics-show-line-numbers -fdiagnostics-color=never -O2 -flto -flto-partition=1to1 -o gcc-dg-lto-pr83954-31.exe
make[4]: *** write jobserver: Bad file descriptor.  Stop.
make[4]: *** Waiting for unfinished jobs....
make[4]: *** write jobserver: Bad file descriptor.  Stop.
lto-wrapper: fatal error: make returned 2 exit status
compilation terminated.
collect2: fatal error: lto-wrapper returned 1 exit status
compilation terminated.
compiler exited with status 1
FAIL: gcc.dg/lto/pr83954 c_lto_pr83954_0.o-c_lto_pr83954_1.o link, -O2 -flto -flto-partition=1to1 
and similar, e.g. for x86_64-linux it was following regressions, on
i686-linux also some tests in libgomp etc.
Is -flto now really using all available CPUs for each compilation?  Without
jobserver that would like a fork-bomb, say if I have a CPU with 32 threads
and do make check -j32, does that mean there are 1024 lto1s?
Judging from http://gcc.gnu.org/ml/gcc-testresults/2019-07/msg03610.html
I'm not alone.

+FAIL: gcc.dg/lto/20081111 c_lto_20081111_0.o-c_lto_20081111_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081111 c_lto_20081111_0.o-c_lto_20081111_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081112 c_lto_20081112_0.o-c_lto_20081112_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081112 c_lto_20081112_0.o-c_lto_20081112_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081125 c_lto_20081125_0.o-c_lto_20081125_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081125 c_lto_20081125_0.o-c_lto_20081125_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081222 c_lto_20081222_0.o-c_lto_20081222_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20081222 c_lto_20081222_0.o-c_lto_20081222_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090210 c_lto_20090210_0.o-c_lto_20090210_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090210 c_lto_20090210_0.o-c_lto_20090210_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090213 c_lto_20090213_0.o-c_lto_20090213_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090213 c_lto_20090213_0.o-c_lto_20090213_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218 c_lto_20090218_0.o-c_lto_20090218_3.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218 c_lto_20090218_0.o-c_lto_20090218_3.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218-1 c_lto_20090218-1_0.o-c_lto_20090218-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218-1 c_lto_20090218-1_0.o-c_lto_20090218-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218-2 c_lto_20090218-2_0.o-c_lto_20090218-2_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090218-2 c_lto_20090218-2_0.o-c_lto_20090218-2_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090312 c_lto_20090312_0.o-c_lto_20090312_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090312 c_lto_20090312_0.o-c_lto_20090312_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090717 c_lto_20090717_0.o-c_lto_20090717_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090717 c_lto_20090717_0.o-c_lto_20090717_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090812 c_lto_20090812_0.o-c_lto_20090812_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20090812 c_lto_20090812_0.o-c_lto_20090812_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091005-1 c_lto_20091005-1_0.o-c_lto_20091005-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091005-1 c_lto_20091005-1_0.o-c_lto_20091005-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091006-1 c_lto_20091006-1_0.o-c_lto_20091006-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091006-1 c_lto_20091006-1_0.o-c_lto_20091006-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091006-2 c_lto_20091006-2_0.o-c_lto_20091006-2_2.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091006-2 c_lto_20091006-2_0.o-c_lto_20091006-2_2.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091017-1 c_lto_20091017-1_0.o-c_lto_20091017-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091017-1 c_lto_20091017-1_0.o-c_lto_20091017-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091027-1 c_lto_20091027-1_0.o-c_lto_20091027-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20091027-1 c_lto_20091027-1_0.o-c_lto_20091027-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100227-1 c_lto_20100227-1_0.o-c_lto_20100227-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100227-1 c_lto_20100227-1_0.o-c_lto_20100227-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100423-1 c_lto_20100423-1_0.o-c_lto_20100423-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100423-1 c_lto_20100423-1_0.o-c_lto_20100423-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100709-1 c_lto_20100709-1_0.o-c_lto_20100709-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100709-1 c_lto_20100709-1_0.o-c_lto_20100709-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-1 c_lto_20100720-1_0.o-c_lto_20100720-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-1 c_lto_20100720-1_0.o-c_lto_20100720-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-2 c_lto_20100720-2_0.o-c_lto_20100720-2_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-2 c_lto_20100720-2_0.o-c_lto_20100720-2_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-3 c_lto_20100720-3_0.o-c_lto_20100720-3_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100720-3 c_lto_20100720-3_0.o-c_lto_20100720-3_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100724-1 c_lto_20100724-1_0.o-c_lto_20100724-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20100724-1 c_lto_20100724-1_0.o-c_lto_20100724-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20101009-2 c_lto_20101009-2_0.o-c_lto_20101009-2_2.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20101009-2 c_lto_20101009-2_0.o-c_lto_20101009-2_2.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20101125-1 c_lto_20101125-1_0.o-c_lto_20101125-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/20101125-1 c_lto_20101125-1_0.o-c_lto_20101125-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/attr-weakref-1 c_lto_attr-weakref-1_0.o-c_lto_attr-weakref-1_2.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/attr-weakref-1 c_lto_attr-weakref-1_0.o-c_lto_attr-weakref-1_2.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr34989-1 c_lto_pr34989-1_0.o-c_lto_pr34989-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr34989-1 c_lto_pr34989-1_0.o-c_lto_pr34989-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr52634 c_lto_pr52634_0.o-c_lto_pr52634_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr52634 c_lto_pr52634_0.o-c_lto_pr52634_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr55660 c_lto_pr55660_0.o-c_lto_pr55660_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr55660 c_lto_pr55660_0.o-c_lto_pr55660_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr59626 c_lto_pr59626_0.o-c_lto_pr59626_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr59626 c_lto_pr59626_0.o-c_lto_pr59626_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr60449 c_lto_pr60449_0.o-c_lto_pr60449_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr60449 c_lto_pr60449_0.o-c_lto_pr60449_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr60720 c_lto_pr60720_0.o-c_lto_pr60720_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr60720 c_lto_pr60720_0.o-c_lto_pr60720_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr70955 c_lto_pr70955_0.o-c_lto_pr70955_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr70955 c_lto_pr70955_0.o-c_lto_pr70955_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr81440 c_lto_pr81440_0.o-c_lto_pr81440_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr81440 c_lto_pr81440_0.o-c_lto_pr81440_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr83954 c_lto_pr83954_0.o-c_lto_pr83954_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr83954 c_lto_pr83954_0.o-c_lto_pr83954_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr88077 c_lto_pr88077_0.o-c_lto_pr88077_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gcc.dg/lto/pr88077 c_lto_pr88077_0.o-c_lto_pr88077_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091015-1 f_lto_20091015-1_0.o-f_lto_20091015-1_2.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091015-1 f_lto_20091015-1_0.o-f_lto_20091015-1_2.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091028-1 f_lto_20091028-1_0.o-f_lto_20091028-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091028-1 f_lto_20091028-1_0.o-f_lto_20091028-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091028-2 f_lto_20091028-2_0.o-f_lto_20091028-2_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20091028-2 f_lto_20091028-2_0.o-f_lto_20091028-2_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20100222-1 f_lto_20100222-1_0.o-f_lto_20100222-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/20100222-1 f_lto_20100222-1_0.o-f_lto_20100222-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr40724 f_lto_pr40724_0.o-f_lto_pr40724_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr40724 f_lto_pr40724_0.o-f_lto_pr40724_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr40725 f_lto_pr40725_0.o-f_lto_pr40725_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr40725 f_lto_pr40725_0.o-f_lto_pr40725_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr41069 f_lto_pr41069_0.o-f_lto_pr41069_2.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr41069 f_lto_pr41069_0.o-f_lto_pr41069_2.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr87689 f_lto_pr87689_0.o-f_lto_pr87689_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: gfortran.dg/lto/pr87689 f_lto_pr87689_0.o-f_lto_pr87689_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081022 cp_lto_20081022_0.o-cp_lto_20081022_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081022 cp_lto_20081022_0.o-cp_lto_20081022_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081109 cp_lto_20081109_0.o-cp_lto_20081109_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081109 cp_lto_20081109_0.o-cp_lto_20081109_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081118-1 cp_lto_20081118-1_0.o-cp_lto_20081118-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081118-1 cp_lto_20081118-1_0.o-cp_lto_20081118-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081119 cp_lto_20081119_0.o-cp_lto_20081119_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081119 cp_lto_20081119_0.o-cp_lto_20081119_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081127 cp_lto_20081127_0.o-cp_lto_20081127_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081203 cp_lto_20081203_0.o-cp_lto_20081203_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081203 cp_lto_20081203_0.o-cp_lto_20081203_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081209 cp_lto_20081209_0.o-cp_lto_20081209_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081209 cp_lto_20081209_0.o-cp_lto_20081209_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081211-1 cp_lto_20081211-1_0.o-cp_lto_20081211-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20081211-1 cp_lto_20081211-1_0.o-cp_lto_20081211-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090311 cp_lto_20090311_0.o-cp_lto_20090311_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090311 cp_lto_20090311_0.o-cp_lto_20090311_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090311-1 cp_lto_20090311-1_0.o-cp_lto_20090311-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090311-1 cp_lto_20090311-1_0.o-cp_lto_20090311-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090312 cp_lto_20090312_0.o-cp_lto_20090312_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090312 cp_lto_20090312_0.o-cp_lto_20090312_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090315 cp_lto_20090315_0.o-cp_lto_20090315_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20090315 cp_lto_20090315_0.o-cp_lto_20090315_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20091026-1 cp_lto_20091026-1_0.o-cp_lto_20091026-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20091026-1 cp_lto_20091026-1_0.o-cp_lto_20091026-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20091210-1 cp_lto_20091210-1_0.o-cp_lto_20091210-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20091210-1 cp_lto_20091210-1_0.o-cp_lto_20091210-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20100603-1 cp_lto_20100603-1_0.o-cp_lto_20100603-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20100603-1 cp_lto_20100603-1_0.o-cp_lto_20100603-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20101020-1 cp_lto_20101020-1_0.o-cp_lto_20101020-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20101020-1 cp_lto_20101020-1_0.o-cp_lto_20101020-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20101126-1 cp_lto_20101126-1_0.o-cp_lto_20101126-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/20101126-1 cp_lto_20101126-1_0.o-cp_lto_20101126-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-1 cp_lto_odr-1_0.o-cp_lto_odr-1_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-1 cp_lto_odr-1_0.o-cp_lto_odr-1_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-2 cp_lto_odr-2_0.o-cp_lto_odr-2_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-2 cp_lto_odr-2_0.o-cp_lto_odr-2_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-3 cp_lto_odr-3_0.o-cp_lto_odr-3_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-3 cp_lto_odr-3_0.o-cp_lto_odr-3_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-5 cp_lto_odr-5_0.o-cp_lto_odr-5_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/odr-5 cp_lto_odr-5_0.o-cp_lto_odr-5_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr68057 cp_lto_pr68057_0.o-cp_lto_pr68057_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr68057 cp_lto_pr68057_0.o-cp_lto_pr68057_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr78472 cp_lto_pr78472_0.o-cp_lto_pr78472_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr78472 cp_lto_pr78472_0.o-cp_lto_pr78472_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr79671 cp_lto_pr79671_0.o-cp_lto_pr79671_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr79671 cp_lto_pr79671_0.o-cp_lto_pr79671_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr87089 cp_lto_pr87089_0.o-cp_lto_pr87089_1.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin 
+FAIL: g++.dg/lto/pr87089 cp_lto_pr87089_0.o-cp_lto_pr87089_1.o link, -O2 -flto -flto-partition=1to1 -fno-use-linker-plugin 

	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

On 7/31/19 1:32 AM, Jakub Jelinek wrote:
> This broke a lot of tests.

Whoops.

> The logs show
> spawn -ignore SIGHUP /home/jakub/src/gcc/obj31/gcc/xgcc -B/home/jakub/src/gcc/obj31/gcc/ c_lto_pr83954_0.o c_lto_pr83954_1.o -fno-diagnostics-show-
> caret -fno-diagnostics-show-line-numbers -fdiagnostics-color=never -O2 -flto -flto-partition=1to1 -o gcc-dg-lto-pr83954-31.exe
> make[4]: *** write jobserver: Bad file descriptor.  Stop.
> make[4]: *** Waiting for unfinished jobs....
> make[4]: *** write jobserver: Bad file descriptor.  Stop.
> lto-wrapper: fatal error: make returned 2 exit status
> compilation terminated.
> collect2: fatal error: lto-wrapper returned 1 exit status
> compilation terminated.
> compiler exited with status 1
> FAIL: gcc.dg/lto/pr83954 c_lto_pr83954_0.o-c_lto_pr83954_1.o link, -O2 -flto -flto-partition=1to1 
> and similar, e.g. for x86_64-linux it was following regressions, on
> i686-linux also some tests in libgomp etc.
> Is -flto now really using all available CPUs for each compilation?

Yes, I can confirm that linking happens in N processed for each LTO test now.
It's caused by fact that current Dejagnu machinery does not pass a make jobserver
to gcc command invocations. On the other hand, our LTO tests are so tiny that we
should have always very low number of partitions.

>  Without
> jobserver that would like a fork-bomb, say if I have a CPU with 32 threads
> and do make check -j32, does that mean there are 1024 lto1s?
> Judging from http://gcc.gnu.org/ml/gcc-testresults/2019-07/msg03610.html
> I'm not alone.

One possible solution will be to adjust lto.exp:

	  set LTO_OPTIONS [list	\
	      {-O0 -flto -flto-partition=none -fuse-linker-plugin} \
	      {-O2 -flto -flto-partition=none -fuse-linker-plugin -fno-fat-lto-objects } \
	      {-O0 -flto -flto-partition=1to1 -fno-use-linker-plugin } \
	      {-O2 -flto -flto-partition=1to1 -fno-use-linker-plugin } \

and replace all -flto with -flto=1. But still, many individual tests set -flto by themselves.
Another solution would be to disable the auto-detection with an environment variable:

diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 353187c6043..bb6fb2b53ff 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1423,7 +1423,7 @@ run_gcc (unsigned argc, char *argv[])
       auto_parallel = 0;
       parallel = 0;
     }
-  else if (!jobserver && parallel)
+  else if (!jobserver && parallel && !getenv ("LTO_NO_AUTO_PARALLEL"))
     {
       /* If there's no explicit usage of jobserver and
 	 parallel is enabled, then automatically detect
diff --git a/gcc/testsuite/lib/lto.exp b/gcc/testsuite/lib/lto.exp
index 25c934731df..e303894e0b0 100644
--- a/gcc/testsuite/lib/lto.exp
+++ b/gcc/testsuite/lib/lto.exp
@@ -209,6 +209,8 @@ proc lto_init { args } {
 	  ]
 	}
     }
+
+    setenv LTO_NO_AUTO_PARALLEL 1
 }
 
 #

Can you Jakub test the patch or the s/-flto/-flto=1 solutions please?
Thanks,
Martin

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Wed, Jul 31, 2019 at 09:20:40AM +0200, Martin Liška wrote:
> One possible solution will be to adjust lto.exp:
> 
> 	  set LTO_OPTIONS [list	\
> 	      {-O0 -flto -flto-partition=none -fuse-linker-plugin} \
> 	      {-O2 -flto -flto-partition=none -fuse-linker-plugin -fno-fat-lto-objects } \
> 	      {-O0 -flto -flto-partition=1to1 -fno-use-linker-plugin } \
> 	      {-O2 -flto -flto-partition=1to1 -fno-use-linker-plugin } \
> 
> and replace all -flto with -flto=1. But still, many individual tests set -flto by themselves.
> Another solution would be to disable the auto-detection with an environment variable:
> 
> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> index 353187c6043..bb6fb2b53ff 100644
> --- a/gcc/lto-wrapper.c
> +++ b/gcc/lto-wrapper.c
> @@ -1423,7 +1423,7 @@ run_gcc (unsigned argc, char *argv[])
>        auto_parallel = 0;
>        parallel = 0;
>      }
> -  else if (!jobserver && parallel)
> +  else if (!jobserver && parallel && !getenv ("LTO_NO_AUTO_PARALLEL"))
>      {
>        /* If there's no explicit usage of jobserver and
>  	 parallel is enabled, then automatically detect
> diff --git a/gcc/testsuite/lib/lto.exp b/gcc/testsuite/lib/lto.exp
> index 25c934731df..e303894e0b0 100644
> --- a/gcc/testsuite/lib/lto.exp
> +++ b/gcc/testsuite/lib/lto.exp
> @@ -209,6 +209,8 @@ proc lto_init { args } {
>  	  ]
>  	}
>      }
> +
> +    setenv LTO_NO_AUTO_PARALLEL 1
>  }
>  
>  #
> 
> Can you Jakub test the patch or the s/-flto/-flto=1 solutions please?

Neither will work very well, we have thousands of -flto tests outside of
lto.exp, e.g. dg-torture.exp (or libgomp and others) cycle through various
options including -flto etc.

Some env var would be useful I guess, though shouldn't it have GCC in the
name?  I mean, if we run into these fork-bomb problems in gcc, won't other
projects run into those as well?

Why doesn't the jobserver work in the tests?  Is that because of missing +
somewhere in the Makefiles or is something unsetting MFLAGS or MAKEFLAGS
env vars?

Though, as I said on IRC, I think we might run out of filedescriptors when
using jobserver too, say if on 64 thread machine one does make -j64 -k check
and each test simultaneously tries to create 64 partitions, that would be
4096 connections to the jobserver, right?

	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jan Hubicka]

> Neither will work very well, we have thousands of -flto tests outside of
> lto.exp, e.g. dg-torture.exp (or libgomp and others) cycle through various
> options including -flto etc.
> 
> Some env var would be useful I guess, though shouldn't it have GCC in the
> name?  I mean, if we run into these fork-bomb problems in gcc, won't other
> projects run into those as well?
> 
> Why doesn't the jobserver work in the tests?  Is that because of missing +
> somewhere in the Makefiles or is something unsetting MFLAGS or MAKEFLAGS
> env vars?

Main trouble with make's jobserver is that it works by 
1) defining environment variable saying which file descriptior to
connect to
2) keeping the file descriptor open upon invoking "+" prefixed lines

Adding "+" to GCC invocation is wrong since it breaks dry run (we do not
want to link at that time) but it is only way to access the jobserver.
If "+" is not present, make will keep the environment vairable but will
close file descriptors prior exec.

Make developers said that this is because some old prorams misbehave
when you exec them with more than 3 file descriptors open. I tried to
negotiate for named pipe which would solve this and it would make it
easy to connect to outermost jobserver from anything invoked form
toplevel make, but they was worried about systems w/o named pipes.

I wonder why we do not detect jobserv as unavailable in this case and do
not default to -flto=<numthreads>?
Is it because dejagnu machinery actually opens some other file
descriptor that gets same ID and executes us with it?

Or does LTO wrapper open something prior accessing jobserver?

> 
> Though, as I said on IRC, I think we might run out of filedescriptors when
> using jobserver too, say if on 64 thread machine one does make -j64 -k check
> and each test simultaneously tries to create 64 partitions, that would be
> 4096 connections to the jobserver, right?

Only WPA process connects to jobserver (which is 1 per linker
invokation), so I think this should be safe.

Honza
> 
> 	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

On 7/31/19 9:40 AM, Jan Hubicka wrote:
>> Neither will work very well, we have thousands of -flto tests outside of
>> lto.exp, e.g. dg-torture.exp (or libgomp and others) cycle through various
>> options including -flto etc.
>>
>> Some env var would be useful I guess, though shouldn't it have GCC in the
>> name?  I mean, if we run into these fork-bomb problems in gcc, won't other
>> projects run into those as well?
>>
>> Why doesn't the jobserver work in the tests?  Is that because of missing +
>> somewhere in the Makefiles or is something unsetting MFLAGS or MAKEFLAGS
>> env vars?
> 
> Main trouble with make's jobserver is that it works by 
> 1) defining environment variable saying which file descriptior to
> connect to
> 2) keeping the file descriptor open upon invoking "+" prefixed lines
> 
> Adding "+" to GCC invocation is wrong since it breaks dry run (we do not
> want to link at that time) but it is only way to access the jobserver.
> If "+" is not present, make will keep the environment vairable but will
> close file descriptors prior exec.
> 
> Make developers said that this is because some old prorams misbehave
> when you exec them with more than 3 file descriptors open. I tried to
> negotiate for named pipe which would solve this and it would make it
> easy to connect to outermost jobserver from anything invoked form
> toplevel make, but they was worried about systems w/o named pipes.

Yes a more generic approach would be welcome as other build systems
would also be able to utilize it.

> 
> I wonder why we do not detect jobserv as unavailable in this case and do
> not default to -flto=<numthreads>?

We do not detect jobserver because of Dejagnu is not using it.
And yes, we default to -flto=<numthreads> in LTO tests now. That's
what is causing issues right now.

> Is it because dejagnu machinery actually opens some other file
> descriptor that gets same ID and executes us with it?
> 
> Or does LTO wrapper open something prior accessing jobserver?
> 
>>
>> Though, as I said on IRC, I think we might run out of filedescriptors when
>> using jobserver too, say if on 64 thread machine one does make -j64 -k check
>> and each test simultaneously tries to create 64 partitions, that would be
>> 4096 connections to the jobserver, right?
> 
> Only WPA process connects to jobserver (which is 1 per linker
> invokation), so I think this should be safe.

Yes and it's only about a quick fcntl. But as mentioned, Dejagnu is not passing
us jobserver, so we don't do it right now.

Martin

> 
> Honza
>>
>> 	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

On 7/31/19 9:34 AM, Jakub Jelinek wrote:
> On Wed, Jul 31, 2019 at 09:20:40AM +0200, Martin Liška wrote:
>> One possible solution will be to adjust lto.exp:
>>
>> 	  set LTO_OPTIONS [list	\
>> 	      {-O0 -flto -flto-partition=none -fuse-linker-plugin} \
>> 	      {-O2 -flto -flto-partition=none -fuse-linker-plugin -fno-fat-lto-objects } \
>> 	      {-O0 -flto -flto-partition=1to1 -fno-use-linker-plugin } \
>> 	      {-O2 -flto -flto-partition=1to1 -fno-use-linker-plugin } \
>>
>> and replace all -flto with -flto=1. But still, many individual tests set -flto by themselves.
>> Another solution would be to disable the auto-detection with an environment variable:
>>
>> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
>> index 353187c6043..bb6fb2b53ff 100644
>> --- a/gcc/lto-wrapper.c
>> +++ b/gcc/lto-wrapper.c
>> @@ -1423,7 +1423,7 @@ run_gcc (unsigned argc, char *argv[])
>>        auto_parallel = 0;
>>        parallel = 0;
>>      }
>> -  else if (!jobserver && parallel)
>> +  else if (!jobserver && parallel && !getenv ("LTO_NO_AUTO_PARALLEL"))
>>      {
>>        /* If there's no explicit usage of jobserver and
>>  	 parallel is enabled, then automatically detect
>> diff --git a/gcc/testsuite/lib/lto.exp b/gcc/testsuite/lib/lto.exp
>> index 25c934731df..e303894e0b0 100644
>> --- a/gcc/testsuite/lib/lto.exp
>> +++ b/gcc/testsuite/lib/lto.exp
>> @@ -209,6 +209,8 @@ proc lto_init { args } {
>>  	  ]
>>  	}
>>      }
>> +
>> +    setenv LTO_NO_AUTO_PARALLEL 1
>>  }
>>  
>>  #
>>
>> Can you Jakub test the patch or the s/-flto/-flto=1 solutions please?
> 
> Neither will work very well, we have thousands of -flto tests outside of
> lto.exp, e.g. dg-torture.exp (or libgomp and others) cycle through various
> options including -flto etc.
> 
> Some env var would be useful I guess, though shouldn't it have GCC in the
> name?

Works for me.

>  I mean, if we run into these fork-bomb problems in gcc, won't other
> projects run into those as well?

Right now, we build whole openSUSE distribution with -flto=N (N based on # of threads)
and we haven't seen issues related to fork-bomb.

> 
> Why doesn't the jobserver work in the tests?  Is that because of missing +
> somewhere in the Makefiles or is something unsetting MFLAGS or MAKEFLAGS
> env vars?

Is Dejagnu using make internally to invoke individual tests?

> 
> Though, as I said on IRC, I think we might run out of filedescriptors when
> using jobserver too, say if on 64 thread machine one does make -j64 -k check
> and each test simultaneously tries to create 64 partitions, that would be
> 4096 connections to the jobserver, right?
> 
> 	Jakub
> 

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Wed, Jul 31, 2019 at 09:50:47AM +0200, Martin Liška wrote:
> > Why doesn't the jobserver work in the tests?  Is that because of missing +
> > somewhere in the Makefiles or is something unsetting MFLAGS or MAKEFLAGS
> > env vars?
> 
> Is Dejagnu using make internally to invoke individual tests?

No.  But as I said, the same check-gcc causes many make goals running
simultaneously, as you can see in the testsuite/gcc{,1,2,3,...} directories,
where each runtest invoked by those goals uses
gcc_parallel_test_run_p and O_EXCL files to decide which of the runtests
will perform each test.

	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jan Hubicka]

Hi,

> We do not detect jobserver because of Dejagnu is not using it.
> And yes, we default to -flto=<numthreads> in LTO tests now. That's
> what is causing issues right now.

Why the error messages are 
make[4]: *** write jobserver: Bad file descriptor.  Stop.
make[4]: *** Waiting for unfinished jobs....
make[4]: *** write jobserver: Bad file descriptor.  Stop.
It seems to me that it is internal make from lto-wrapper trying to get
jobserver access?

> Works for me.

We probably also can give it a meaning controlling the default parameter
of -flto.
I.e. setenv GCC_LTO_PARALLELISM <numthreads/auto/jobserv>
which we will default to in case only -flto is passed to command line.

Make's jobserver is quite nice and easy to use. If it supported named
pipe and there was a small library which allowed to initialize jobserver
and connect to it, perhaps other tools needing parallelism during build
(GCC, ninja, llvm, ...) would be able to use common protocol.

Honza

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

On 7/31/19 10:08 AM, Jan Hubicka wrote:
> Hi,
> 
>> We do not detect jobserver because of Dejagnu is not using it.
>> And yes, we default to -flto=<numthreads> in LTO tests now. That's
>> what is causing issues right now.
> 
> Why the error messages are 
> make[4]: *** write jobserver: Bad file descriptor.  Stop.
> make[4]: *** Waiting for unfinished jobs....
> make[4]: *** write jobserver: Bad file descriptor.  Stop.
> It seems to me that it is internal make from lto-wrapper trying to get
> jobserver access?

Hard to guess. Can you Jakub debug that? I don't see the error message.

> 
>> Works for me.
> 
> We probably also can give it a meaning controlling the default parameter
> of -flto.
> I.e. setenv GCC_LTO_PARALLELISM <numthreads/auto/jobserv>
> which we will default to in case only -flto is passed to command line.

Interesting idea, I like it, but:
- if we detect that jobserver will not work, GCC_LTO_PARALLELISM=jobserver is useless
- auto equals now to 'unset GCC_LTO_PARALLELISM'

So the only interesting value is numthreads. Then I would recommend to use
GCC_LTO_MAX_AUTO_THREADS?

> 
> Make's jobserver is quite nice and easy to use. If it supported named
> pipe and there was a small library which allowed to initialize jobserver
> and connect to it, perhaps other tools needing parallelism during build
> (GCC, ninja, llvm, ...) would be able to use common protocol.

Can be a nice GSoC project for next year?

Martin

> 
> Honza
> 

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Wed, Jul 31, 2019 at 10:21:16AM +0200, Martin Liška wrote:
> On 7/31/19 10:08 AM, Jan Hubicka wrote:
> > Hi,
> > 
> >> We do not detect jobserver because of Dejagnu is not using it.
> >> And yes, we default to -flto=<numthreads> in LTO tests now. That's
> >> what is causing issues right now.
> > 
> > Why the error messages are 
> > make[4]: *** write jobserver: Bad file descriptor.  Stop.
> > make[4]: *** Waiting for unfinished jobs....
> > make[4]: *** write jobserver: Bad file descriptor.  Stop.
> > It seems to me that it is internal make from lto-wrapper trying to get
> > jobserver access?
> 
> Hard to guess. Can you Jakub debug that? I don't see the error message.

I can easily reproduce even with
make -j2 -k check-gcc RUNTESTFLAGS=lto.exp=*20081125*

I've added a hack:
--- lto-wrapper.c.jj	2019-07-31 09:46:39.361331843 +0200
+++ lto-wrapper.c	2019-07-31 10:40:57.970302110 +0200
@@ -1430,6 +1430,12 @@ run_gcc (unsigned argc, char *argv[])
 	 jobserver or number of cores.  */
       auto_parallel = 1;
       jobserver = jobserver_active_p ();
+FILE *f = fopen ("/tmp/111x", "a");
+fprintf (f, "%d %s\n", jobserver, getenv ("MAKEFLAGS"));
+fclose (f);
+char buf[1024];
+sprintf (buf, "ls -l /proc/%d/fd >> /tmp/111x", getpid ());
+system (buf);
     }
 
   if (linker_output)

and see that in some cases jobserver is 0 and in other cases jobserver is 1.
Examples of both:
0 kw -j2 --jobserver-auth=3,6 -- 
total 0
lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> pipe:[75026106]
lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/src/gcc/obj/gcc/libgcc_s.so
lr-x------. 1 jakub jakub 64 Jul 31 10:41 13 -> /usr/lib64/libc.so
lr-x------. 1 jakub jakub 64 Jul 31 10:41 18 -> /usr/src/gcc/obj/gcc/libgcc_s.so
l-wx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /tmp/ccX4y4r3.le
l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]
1 kw -j2 --jobserver-auth=3,6 -- 
total 0
lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> /tmp/cchSvmBt
lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/lib64/crtn.o
lrwx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /dev/pts/4
lr-x------. 1 jakub jakub 64 Jul 31 10:41 3 -> /usr/lib64/crt1.o
lr-x------. 1 jakub jakub 64 Jul 31 10:41 4 -> /usr/lib64/crti.o
lr-x------. 1 jakub jakub 64 Jul 31 10:41 5 -> /usr/src/gcc/obj/gcc/crtbegin.o
lr-x------. 1 jakub jakub 64 Jul 31 10:41 6 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_0.o
lr-x------. 1 jakub jakub 64 Jul 31 10:41 7 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_1.o
lr-x------. 1 jakub jakub 64 Jul 31 10:41 8 -> /usr/src/gcc/obj/gcc/crtend.o
l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]

so, if one is lucky enough and at least one of the two file descriptors in
MAKEFLAGS --jobserver-auth= is closed, it will work fine, but if they are
open, even when they have nothing to do with make, it will fail miserably.

As a partial workaround, I wonder if jobserver_active_p couldn't check (with
fstat/fstat64) if the file descriptors are pipes.  It will still not be 100%
reliable, as when you are unlucky the file descriptor could be some
completely unrelated pipe, but at least better than the current state.

	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jan Hubicka]

> and see that in some cases jobserver is 0 and in other cases jobserver is 1.
> Examples of both:
> 0 kw -j2 --jobserver-auth=3,6 -- 
> total 0
> lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
> l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> pipe:[75026106]
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/src/gcc/obj/gcc/libgcc_s.so
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 13 -> /usr/lib64/libc.so
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 18 -> /usr/src/gcc/obj/gcc/libgcc_s.so
> l-wx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /tmp/ccX4y4r3.le
> l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]
> 1 kw -j2 --jobserver-auth=3,6 -- 
> total 0
> lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
> l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> /tmp/cchSvmBt
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/lib64/crtn.o
> lrwx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /dev/pts/4
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 3 -> /usr/lib64/crt1.o
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 4 -> /usr/lib64/crti.o
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 5 -> /usr/src/gcc/obj/gcc/crtbegin.o
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 6 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_0.o
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 7 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_1.o
> lr-x------. 1 jakub jakub 64 Jul 31 10:41 8 -> /usr/src/gcc/obj/gcc/crtend.o
> l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]
> 
> so, if one is lucky enough and at least one of the two file descriptors in
> MAKEFLAGS --jobserver-auth= is closed, it will work fine, but if they are
> open, even when they have nothing to do with make, it will fail miserably.

I see, so the problem is that lto-wrapper is executed via plugin from
gold which already used the file descriptors for something else?

This seems a bit of problem since gold opens it before we get to
lto-plugin (most probably) so i do not see how to reliably detect
presence of make server.

I think the gcc binary can look for jobserv environment, detect jobserv
and if it is not available remove jobserver option from it prior going
to ld.  I think this works since I think link-time gcc can only be
executed via gcc/g++/gfortran... wrappers.

(unlike lto plugin invoked implicitly for ar/nm etc).

Honza
> 
> As a partial workaround, I wonder if jobserver_active_p couldn't check (with
> fstat/fstat64) if the file descriptors are pipes.  It will still not be 100%
> reliable, as when you are unlucky the file descriptor could be some
> completely unrelated pipe, but at least better than the current state.
> 
> 	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Wed, Jul 31, 2019 at 11:12:15AM +0200, Jan Hubicka wrote:
> > so, if one is lucky enough and at least one of the two file descriptors in
> > MAKEFLAGS --jobserver-auth= is closed, it will work fine, but if they are
> > open, even when they have nothing to do with make, it will fail miserably.
> 
> I see, so the problem is that lto-wrapper is executed via plugin from
> gold which already used the file descriptors for something else?

I don't think I'm using gold, I have it installed, but /usr/bin/ld points to
ld.bfd.

	Jakub

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jan Hubicka]

> On Wed, Jul 31, 2019 at 11:12:15AM +0200, Jan Hubicka wrote:
> > > so, if one is lucky enough and at least one of the two file descriptors in
> > > MAKEFLAGS --jobserver-auth= is closed, it will work fine, but if they are
> > > open, even when they have nothing to do with make, it will fail miserably.
> > 
> > I see, so the problem is that lto-wrapper is executed via plugin from
> > gold which already used the file descriptors for something else?
> 
> I don't think I'm using gold, I have it installed, but /usr/bin/ld points to
> ld.bfd.

Particular linker implementation probably does not make much pracitcal
difference.  It seems natural that linker opens couple files prior
asking plugin doing its job. 

Honza

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 3039 bytes --]

On 7/31/19 11:12 AM, Jan Hubicka wrote:
>> and see that in some cases jobserver is 0 and in other cases jobserver is 1.
>> Examples of both:
>> 0 kw -j2 --jobserver-auth=3,6 -- 
>> total 0
>> lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
>> l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> pipe:[75026106]
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/src/gcc/obj/gcc/libgcc_s.so
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 13 -> /usr/lib64/libc.so
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 18 -> /usr/src/gcc/obj/gcc/libgcc_s.so
>> l-wx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /tmp/ccX4y4r3.le
>> l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]
>> 1 kw -j2 --jobserver-auth=3,6 -- 
>> total 0
>> lrwx------. 1 jakub jakub 64 Jul 31 10:41 0 -> /dev/pts/4
>> l-wx------. 1 jakub jakub 64 Jul 31 10:41 1 -> /tmp/cchSvmBt
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 10 -> /usr/lib64/crtn.o
>> lrwx------. 1 jakub jakub 64 Jul 31 10:41 2 -> /dev/pts/4
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 3 -> /usr/lib64/crt1.o
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 4 -> /usr/lib64/crti.o
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 5 -> /usr/src/gcc/obj/gcc/crtbegin.o
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 6 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_0.o
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 7 -> /usr/src/gcc/obj/gcc/testsuite/gcc/c_lto_20081125_1.o
>> lr-x------. 1 jakub jakub 64 Jul 31 10:41 8 -> /usr/src/gcc/obj/gcc/crtend.o
>> l-wx------. 1 jakub jakub 64 Jul 31 10:41 9 -> pipe:[46167136]
>>
>> so, if one is lucky enough and at least one of the two file descriptors in
>> MAKEFLAGS --jobserver-auth= is closed, it will work fine, but if they are
>> open, even when they have nothing to do with make, it will fail miserably.
> 
> I see, so the problem is that lto-wrapper is executed via plugin from
> gold which already used the file descriptors for something else?
> 
> This seems a bit of problem since gold opens it before we get to
> lto-plugin (most probably) so i do not see how to reliably detect
> presence of make server.

Thank you Jakub for debugging!

> 
> I think the gcc binary can look for jobserv environment, detect jobserv
> and if it is not available remove jobserver option from it prior going
> to ld.  I think this works since I think link-time gcc can only be
> executed via gcc/g++/gfortran... wrappers.

Do you mean something like what I'm attaching?

Anyway, the auto-detection of jobserver seems very ugly to me :/
I tend to remove the support and start working on a proper implementation
that can be used not only by make build system.

Martin

> 
> (unlike lto plugin invoked implicitly for ar/nm etc).
> 
> Honza
>>
>> As a partial workaround, I wonder if jobserver_active_p couldn't check (with
>> fstat/fstat64) if the file descriptors are pipes.  It will still not be 100%
>> reliable, as when you are unlucky the file descriptor could be some
>> completely unrelated pipe, but at least better than the current state.
>>
>> 	Jakub


[-- Attachment #2: lto-jobserver.patch --]
[-- Type: text/x-patch, Size: 1831 bytes --]

diff --git a/gcc/gcc.c b/gcc/gcc.c
index a4323eb146e..e43f46246ad 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -8268,6 +8268,44 @@ driver::maybe_run_linker (const char *argv0) const
     {
       int tmp = execution_count;
 
+      /* Detect jobserver and drop it if it's not working.  */
+      const char *makeflags = env.get ("MAKEFLAGS");
+      if (makeflags != NULL)
+	{
+	  const char *needle = "--jobserver-auth=";
+	  const char *n = strstr (makeflags, needle);
+	  if (n != NULL)
+	    {
+	      int rfd = -1;
+	      int wfd = -1;
+
+	      bool jobserver
+		= ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+		   && rfd > 0
+		   && wfd > 0
+		   && fcntl (rfd, F_GETFD) >= 0
+		   && fcntl (wfd, F_GETFD) >= 0);
+
+	      /* Drop the jobserver if it's not working now.  */
+	      if (!jobserver)
+		{
+		  unsigned offset = n - makeflags;
+		  char *dup = xstrdup (makeflags);
+		  dup[offset] = '\0';
+
+		  const char *space = strchr (makeflags + offset, ' ');
+		  if (space != NULL)
+		    strcpy (dup + offset, space);
+		  xputenv (concat ("MAKEFLAGS=", dup, NULL));
+
+		  // TODO: remove
+		  FILE *f = fopen ("/tmp/111x", "a"); 
+		  fprintf (f, "dropping MAKEFLAGS\n"); 
+		  fclose (f); 
+		}
+	    }
+	}
+
       if (! have_c)
 	{
 #if HAVE_LTO_PLUGIN > 0
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 353187c6043..10b74d84af7 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1430,6 +1430,13 @@ run_gcc (unsigned argc, char *argv[])
 	 jobserver or number of cores.  */
       auto_parallel = 1;
       jobserver = jobserver_active_p ();
+
+FILE *f = fopen ("/tmp/111x", "a"); 
+fprintf (f, "%d %s\n", jobserver, getenv ("MAKEFLAGS")); 
+fclose (f); 
+char buf[1024]; 
+sprintf (buf, "ls -l /proc/%d/fd >> /tmp/111x", getpid ()); 
+system (buf); 
     }
 
   if (linker_output)

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jan Hubicka]

> diff --git a/gcc/gcc.c b/gcc/gcc.c
> index a4323eb146e..e43f46246ad 100644
> --- a/gcc/gcc.c
> +++ b/gcc/gcc.c
> @@ -8268,6 +8268,44 @@ driver::maybe_run_linker (const char *argv0) const
>      {
>        int tmp = execution_count;
>  
> +      /* Detect jobserver and drop it if it's not working.  */
> +      const char *makeflags = env.get ("MAKEFLAGS");
> +      if (makeflags != NULL)
> +	{
> +	  const char *needle = "--jobserver-auth=";
> +	  const char *n = strstr (makeflags, needle);
> +	  if (n != NULL)
> +	    {
> +	      int rfd = -1;
> +	      int wfd = -1;
> +
> +	      bool jobserver
> +		= ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> +		   && rfd > 0
> +		   && wfd > 0
> +		   && fcntl (rfd, F_GETFD) >= 0
> +		   && fcntl (wfd, F_GETFD) >= 0);

Yes, I suppose this is the test whether file descriptors are open that
should work in our context.

Honza
> +
> +	      /* Drop the jobserver if it's not working now.  */
> +	      if (!jobserver)
> +		{
> +		  unsigned offset = n - makeflags;
> +		  char *dup = xstrdup (makeflags);
> +		  dup[offset] = '\0';
> +
> +		  const char *space = strchr (makeflags + offset, ' ');
> +		  if (space != NULL)
> +		    strcpy (dup + offset, space);
> +		  xputenv (concat ("MAKEFLAGS=", dup, NULL));
> +
> +		  // TODO: remove
> +		  FILE *f = fopen ("/tmp/111x", "a"); 
> +		  fprintf (f, "dropping MAKEFLAGS\n"); 
> +		  fclose (f); 
> +		}
> +	    }
> +	}
> +
>        if (! have_c)
>  	{
>  #if HAVE_LTO_PLUGIN > 0
> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> index 353187c6043..10b74d84af7 100644
> --- a/gcc/lto-wrapper.c
> +++ b/gcc/lto-wrapper.c
> @@ -1430,6 +1430,13 @@ run_gcc (unsigned argc, char *argv[])
>  	 jobserver or number of cores.  */
>        auto_parallel = 1;
>        jobserver = jobserver_active_p ();
> +
> +FILE *f = fopen ("/tmp/111x", "a"); 
> +fprintf (f, "%d %s\n", jobserver, getenv ("MAKEFLAGS")); 
> +fclose (f); 
> +char buf[1024]; 
> +sprintf (buf, "ls -l /proc/%d/fd >> /tmp/111x", getpid ()); 
> +system (buf); 
>      }
>  
>    if (linker_output)

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 356 bytes --]

On 7/31/19 12:01 PM, Martin Liška wrote:
> Anyway, the auto-detection of jobserver seems very ugly to me :/
> I tend to remove the support and start working on a proper implementation
> that can be used not only by make build system.

Can you Jakub test the attached patch please?
Do it reach a ulimit limit on your testing environment?

Thanks,
Martin

[-- Attachment #2: 0001-flto-do-not-use-jobserver-automatically.patch --]
[-- Type: text/x-patch, Size: 1818 bytes --]

From 2ccefa19d40d136ea04909a2c2b9215ab5362897 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Wed, 31 Jul 2019 16:57:45 +0200
Subject: [PATCH] -flto: do not use jobserver automatically

gcc/ChangeLog:

2019-07-31  Martin Liska  <mliska@suse.cz>

	* lto-wrapper.c (jobserver_active_p): Remove.
	(run_gcc): Only set auto_parallel and do not
	detect jobserverver.
---
 gcc/lto-wrapper.c | 29 +----------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 353187c6043..9a40a366f2b 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1217,30 +1217,6 @@ init_num_threads (void)
 
 /* FIXME: once using -std=c11, we can use std::thread::hardware_concurrency.  */
 
-/* Return true when a jobserver is running and can accept a job.  */
-
-static bool
-jobserver_active_p (void)
-{
-  const char *makeflags = getenv ("MAKEFLAGS");
-  if (makeflags == NULL)
-    return false;
-
-  const char *needle = "--jobserver-auth=";
-  const char *n = strstr (makeflags, needle);
-  if (n == NULL)
-    return false;
-
-  int rfd = -1;
-  int wfd = -1;
-
-  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
-	  && rfd > 0
-	  && wfd > 0
-	  && fcntl (rfd, F_GETFD) >= 0
-	  && fcntl (wfd, F_GETFD) >= 0);
-}
-
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
 static void
@@ -1425,11 +1401,8 @@ run_gcc (unsigned argc, char *argv[])
     }
   else if (!jobserver && parallel)
     {
-      /* If there's no explicit usage of jobserver and
-	 parallel is enabled, then automatically detect
-	 jobserver or number of cores.  */
+      /* Detect number of jobs automatically.  */
       auto_parallel = 1;
-      jobserver = jobserver_active_p ();
     }
 
   if (linker_output)
-- 
2.22.0

Re: [PATCH] Deduce automatically number of cores for -flto option.

[Jakub Jelinek]

On Wed, Jul 31, 2019 at 05:00:37PM +0200, Martin Liška wrote:
> On 7/31/19 12:01 PM, Martin Liška wrote:
> > Anyway, the auto-detection of jobserver seems very ugly to me :/
> > I tend to remove the support and start working on a proper implementation
> > that can be used not only by make build system.
> 
> Can you Jakub test the attached patch please?

That works.  I guess it didn't actually act as a fork-bomb because most of
our lto tests are small and there aren't enough functions to create so many
partitions.

	Jakub

[PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 796 bytes --]

On 8/1/19 3:19 PM, Jakub Jelinek wrote:
> On Wed, Jul 31, 2019 at 05:00:37PM +0200, Martin Liška wrote:
>> On 7/31/19 12:01 PM, Martin Liška wrote:
>>> Anyway, the auto-detection of jobserver seems very ugly to me :/
>>> I tend to remove the support and start working on a proper implementation
>>> that can be used not only by make build system.
>>
>> Can you Jakub test the attached patch please?
> 
> That works.  I guess it didn't actually act as a fork-bomb because most of
> our lto tests are small and there aren't enough functions to create so many
> partitions.
> 
> 	Jakub
> 

Ok, after deeper discussion with Honza, I would like to suggest the original
patch that was about proper detection of jobserver.

Can you please Jakub test the patch in your environment?

Thanks,
Martin

[-- Attachment #2: 0001-Properly-detect-working-jobserver-in-gcc-driver.patch --]
[-- Type: text/x-patch, Size: 1804 bytes --]

From 0a7f15566dda99146784eb3f85e8b4547f2ab71c Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Thu, 1 Aug 2019 16:30:01 +0200
Subject: [PATCH] Properly detect working jobserver in gcc driver.

gcc/ChangeLog:

2019-08-01  Martin Liska  <mliska@suse.cz>

	PR lto/91313
	* gcc.c (driver::maybe_run_linker): Test whether jobserver
	is active from GCC driver. That will prevent situation where
	GCC is invoked from a LD plugin and the linker already uses
	file descriptors suggested by make.  That leads to a wrong
	detection.
---
 gcc/gcc.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/gcc/gcc.c b/gcc/gcc.c
index a4323eb146e..fc11abf1f88 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -8268,6 +8268,39 @@ driver::maybe_run_linker (const char *argv0) const
     {
       int tmp = execution_count;
 
+      /* Detect jobserver and drop it if it's not working.  */
+      const char *makeflags = env.get ("MAKEFLAGS");
+      if (makeflags != NULL)
+	{
+	  const char *needle = "--jobserver-auth=";
+	  const char *n = strstr (makeflags, needle);
+	  if (n != NULL)
+	    {
+	      int rfd = -1;
+	      int wfd = -1;
+
+	      bool jobserver
+		= ((sscanf (n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+		   && rfd > 0
+		   && wfd > 0
+		   && fcntl (rfd, F_GETFD) >= 0
+		   && fcntl (wfd, F_GETFD) >= 0);
+
+	      /* Drop the jobserver if it's not working now.  */
+	      if (!jobserver)
+		{
+		  unsigned offset = n - makeflags;
+		  char *dup = xstrdup (makeflags);
+		  dup[offset] = '\0';
+
+		  const char *space = strchr (makeflags + offset, ' ');
+		  if (space != NULL)
+		    strcpy (dup + offset, space);
+		  xputenv (concat ("MAKEFLAGS=", dup, NULL));
+		}
+	    }
+	}
+
       if (! have_c)
 	{
 #if HAVE_LTO_PLUGIN > 0
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jakub Jelinek]

On Thu, Aug 01, 2019 at 04:34:09PM +0200, Martin Liška wrote:
> Ok, after deeper discussion with Honza, I would like to suggest the original
> patch that was about proper detection of jobserver.
> 
> Can you please Jakub test the patch in your environment?

Isn't this done too late (as in, doesn't the driver at that moment already
have some files newly openend, like e.g. the @ option files?

> +		= ((sscanf (n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)

No need to wrap sscanf (...) == 2 into ()s.  Also, you've already done
a strstr, what is the point in verifying it once again that it starts with
--jobserver-auth= string?
And in the lto-writer.c code there is no space between sscanf and (.

	Jakub

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 873 bytes --]

On 8/1/19 4:41 PM, Jakub Jelinek wrote:
> On Thu, Aug 01, 2019 at 04:34:09PM +0200, Martin Liška wrote:
>> Ok, after deeper discussion with Honza, I would like to suggest the original
>> patch that was about proper detection of jobserver.
>>
>> Can you please Jakub test the patch in your environment?
> 
> Isn't this done too late (as in, doesn't the driver at that moment already
> have some files newly openend, like e.g. the @ option files?

You are right, I've reworked that. Good observation.

> 
>> +		= ((sscanf (n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> 
> No need to wrap sscanf (...) == 2 into ()s.  Also, you've already done
> a strstr, what is the point in verifying it once again that it starts with
> --jobserver-auth= string?
> And in the lto-writer.c code there is no space between sscanf and (.

Yep, I simplified that.

Martin

> 
> 	Jakub
> 


[-- Attachment #2: 0001-Properly-detect-working-jobserver-in-gcc-driver.patch --]
[-- Type: text/x-patch, Size: 3210 bytes --]

From 9f96e16a7798fd3b60fa9ec5b7a66748212146c4 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Thu, 1 Aug 2019 16:30:01 +0200
Subject: [PATCH] Properly detect working jobserver in gcc driver.

gcc/ChangeLog:

2019-08-02  Martin Liska  <mliska@suse.cz>

	* gcc.c (driver::main): Call detect_jobserver right
	at the very start of the function.
	(driver::detect_jobserver): Test whether jobserver
	is active from GCC driver. That will prevent situation where
	GCC is invoked from a LD plugin and the linker already uses
	file descriptors suggested by make.  That leads to a wrong
	detection.
	* gcc.h (driver): Add detect_jobserver.
	* lto-wrapper.c (jobserver_active_p): Simplify sscanf by
	not scanning for --jobserver-auth prefix.
---
 gcc/gcc.c         | 41 +++++++++++++++++++++++++++++++++++++++++
 gcc/gcc.h         |  1 +
 gcc/lto-wrapper.c |  2 +-
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/gcc/gcc.c b/gcc/gcc.c
index a4323eb146e..c8753b9671f 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -7354,6 +7354,7 @@ driver::main (int argc, char **argv)
 {
   bool early_exit;
 
+  detect_jobserver ();
   set_progname (argv[0]);
   expand_at_files (&argc, &argv);
   decode_argv (argc, const_cast <const char **> (argv));
@@ -8357,6 +8358,46 @@ driver::final_actions () const
     }
 }
 
+/* Detect whether jobserver is active and working.  If not drop
+   --jobserver-auth from MAKEFLAGS.  */
+
+void
+driver::detect_jobserver () const
+{
+  /* Detect jobserver and drop it if it's not working.  */
+  const char *makeflags = env.get ("MAKEFLAGS");
+  if (makeflags != NULL)
+    {
+      const char *needle = "--jobserver-auth=";
+      const char *n = strstr (makeflags, needle);
+      if (n != NULL)
+	{
+	  int rfd = -1;
+	  int wfd = -1;
+
+	  bool jobserver
+	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
+	       && rfd > 0
+	       && wfd > 0
+	       && fcntl (rfd, F_GETFD) >= 0
+	       && fcntl (wfd, F_GETFD) >= 0);
+
+	  /* Drop the jobserver if it's not working now.  */
+	  if (!jobserver)
+	    {
+	      unsigned offset = n - makeflags;
+	      char *dup = xstrdup (makeflags);
+	      dup[offset] = '\0';
+
+	      const char *space = strchr (makeflags + offset, ' ');
+	      if (space != NULL)
+		strcpy (dup + offset, space);
+	      xputenv (concat ("MAKEFLAGS=", dup, NULL));
+	    }
+	}
+    }
+}
+
 /* Determine what the exit code of the driver should be.  */
 
 int
diff --git a/gcc/gcc.h b/gcc/gcc.h
index a0a1d94c6e6..dc77dba67fb 100644
--- a/gcc/gcc.h
+++ b/gcc/gcc.h
@@ -51,6 +51,7 @@ class driver
   void do_spec_on_infiles () const;
   void maybe_run_linker (const char *argv0) const;
   void final_actions () const;
+  void detect_jobserver () const;
   int get_exit_code () const;
 
  private:
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 353187c6043..3414adedd26 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1234,7 +1234,7 @@ jobserver_active_p (void)
   int rfd = -1;
   int wfd = -1;
 
-  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
 	  && rfd > 0
 	  && wfd > 0
 	  && fcntl (rfd, F_GETFD) >= 0
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jakub Jelinek]

On Fri, Aug 02, 2019 at 08:30:47AM +0200, Martin Liška wrote:
> On 8/1/19 4:41 PM, Jakub Jelinek wrote:
> > On Thu, Aug 01, 2019 at 04:34:09PM +0200, Martin Liška wrote:
> >> Ok, after deeper discussion with Honza, I would like to suggest the original
> >> patch that was about proper detection of jobserver.
> >>
> >> Can you please Jakub test the patch in your environment?
> > 
> > Isn't this done too late (as in, doesn't the driver at that moment already
> > have some files newly openend, like e.g. the @ option files?
> 
> You are right, I've reworked that. Good observation.

I was actually wrong, because while expandargv fopens new file descriptors
when processing the options, it fcloses them too before it returns.

Sorry for that.

Can you strace if other fds are opened and not closed in the spot you had it
before?  Advantage of doing it there is that it will not be done for all the
-E/-S/-c compilations when the linker is not spawned.

> > 
> >> +		= ((sscanf (n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> > 
> > No need to wrap sscanf (...) == 2 into ()s.  Also, you've already done
> > a strstr, what is the point in verifying it once again that it starts with
> > --jobserver-auth= string?
> > And in the lto-writer.c code there is no space between sscanf and (.
> 
> Yep, I simplified that.

Thanks.  Note, your patch from yesterday also passed testing.

	Jakub

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 1612 bytes --]

On 8/2/19 9:44 AM, Jakub Jelinek wrote:
> On Fri, Aug 02, 2019 at 08:30:47AM +0200, Martin Liška wrote:
>> On 8/1/19 4:41 PM, Jakub Jelinek wrote:
>>> On Thu, Aug 01, 2019 at 04:34:09PM +0200, Martin Liška wrote:
>>>> Ok, after deeper discussion with Honza, I would like to suggest the original
>>>> patch that was about proper detection of jobserver.
>>>>
>>>> Can you please Jakub test the patch in your environment?
>>>
>>> Isn't this done too late (as in, doesn't the driver at that moment already
>>> have some files newly openend, like e.g. the @ option files?
>>
>> You are right, I've reworked that. Good observation.
> 
> I was actually wrong, because while expandargv fopens new file descriptors
> when processing the options, it fcloses them too before it returns.
> 
> Sorry for that.

Ah ok.

> 
> Can you strace if other fds are opened and not closed in the spot you had it
> before?  Advantage of doing it there is that it will not be done for all the
> -E/-S/-c compilations when the linker is not spawned.

I've used the same trick which you used and I'm attaching the output.
I believe it's fine, I can't see any opened fd by GCC.

Martin

> 
>>>
>>>> +		= ((sscanf (n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
>>>
>>> No need to wrap sscanf (...) == 2 into ()s.  Also, you've already done
>>> a strstr, what is the point in verifying it once again that it starts with
>>> --jobserver-auth= string?
>>> And in the lto-writer.c code there is no space between sscanf and (.
>>
>> Yep, I simplified that.
> 
> Thanks.  Note, your patch from yesterday also passed testing.
> 
> 	Jakub
> 


[-- Attachment #2: lto-jobserver-openned-fds.txt --]
[-- Type: text/plain, Size: 18307 bytes --]

dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccdu6rwA
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 17 -> /usr/lib64/libc.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/cc9oWLNw.le
lr-x------ 1 marxin users 64 Aug  2 10:33 22 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 9 -> /usr/lib64/libm.so
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccKEFRGz
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /usr/lib64/libc.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccv4FRky.le
lr-x------ 1 marxin users 64 Aug  2 10:33 22 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccrRRAfS
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 13 -> /usr/lib64/libc.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /dev/shm/objdir/gcc/libgcc_s.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccWJ65uO.le
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccXIkUlY
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /usr/lib64/libc.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccvNMFyU.le
lr-x------ 1 marxin users 64 Aug  2 10:33 22 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lr-x------ 1 marxin users 64 Aug  2 10:33 0 -> pipe:[579323352]
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccozzUjb
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /usr/lib64/libc.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccacysX9.le
lr-x------ 1 marxin users 64 Aug  2 10:33 22 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccozzUjb
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /usr/lib64/libc.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccacysX9.le
lr-x------ 1 marxin users 64 Aug  2 10:33 22 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=11,12
dropping MAKEFLAGS in w -j16 --jobserver-auth=11,12
total 0
lr-x------ 1 marxin users 64 Aug  2 10:33 0 -> pipe:[579328123]
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccDCgNyq
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/crtn.o
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 4 -> /usr/lib64/crt1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 5 -> /usr/lib64/crti.o
lr-x------ 1 marxin users 64 Aug  2 10:33 6 -> /dev/shm/objdir/gcc/crtbegin.o
lr-x------ 1 marxin users 64 Aug  2 10:33 7 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_0.o
lr-x------ 1 marxin users 64 Aug  2 10:33 8 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 9 -> /dev/shm/objdir/gcc/crtend.o
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/5
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccDCgNyq
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/crtn.o
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/5
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 4 -> /usr/lib64/crt1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 5 -> /usr/lib64/crti.o
lr-x------ 1 marxin users 64 Aug  2 10:33 6 -> /dev/shm/objdir/gcc/crtbegin.o
lr-x------ 1 marxin users 64 Aug  2 10:33 7 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_0.o
lr-x------ 1 marxin users 64 Aug  2 10:33 8 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 9 -> /dev/shm/objdir/gcc/crtend.o
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=11,12
dropping MAKEFLAGS in w -j16 --jobserver-auth=11,12
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccG4DfIN
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/crtn.o
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 4 -> /usr/lib64/crt1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 5 -> /usr/lib64/crti.o
lr-x------ 1 marxin users 64 Aug  2 10:33 6 -> /dev/shm/objdir/gcc/crtbegin.o
lr-x------ 1 marxin users 64 Aug  2 10:33 7 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_0.o
lr-x------ 1 marxin users 64 Aug  2 10:33 8 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 9 -> /dev/shm/objdir/gcc/crtend.o
total 0
lr-x------ 1 marxin users 64 Aug  2 10:33 0 -> pipe:[579322564]
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccG4DfIN
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/crtn.o
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 4 -> /usr/lib64/crt1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 5 -> /usr/lib64/crti.o
lr-x------ 1 marxin users 64 Aug  2 10:33 6 -> /dev/shm/objdir/gcc/crtbegin.o
lr-x------ 1 marxin users 64 Aug  2 10:33 7 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_0.o
lr-x------ 1 marxin users 64 Aug  2 10:33 8 -> /dev/shm/objdir/gcc/testsuite/gcc/c_lto_20081125_1.o
lr-x------ 1 marxin users 64 Aug  2 10:33 9 -> /dev/shm/objdir/gcc/crtend.o
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/cc6xSPub
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 13 -> /usr/lib64/libc.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /dev/shm/objdir/gcc/libgcc_s.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccEeSLA7.le
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/cc91ZMov
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 13 -> /usr/lib64/libc.so
lr-x------ 1 marxin users 64 Aug  2 10:33 18 -> /dev/shm/objdir/gcc/libgcc_s.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccXWaZKr.le
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccK5b4bR
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 15 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 19 -> /dev/shm/objdir/gcc/libgcc_s.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/cc5885XO.le
lr-x------ 1 marxin users 64 Aug  2 10:33 24 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 28 -> /dev/shm/objdir/gcc/libgcc_s.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 32 -> /usr/lib64/libc.so
lr-x------ 1 marxin users 64 Aug  2 10:33 36 -> /dev/shm/objdir/gcc/libgcc_s.so
dropping MAKEFLAGS in kw -j2 --jobserver-auth=3,4 -- 
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 1 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 2 -> /dev/pts/4
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
dropping MAKEFLAGS in w -j16 --jobserver-auth=4,5
total 0
lrwx------ 1 marxin users 64 Aug  2 10:33 0 -> /dev/pts/4
l-wx------ 1 marxin users 64 Aug  2 10:33 1 -> /tmp/ccgExLq7
lr-x------ 1 marxin users 64 Aug  2 10:33 10 -> /usr/lib64/libm.so
lr-x------ 1 marxin users 64 Aug  2 10:33 14 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 19 -> /usr/lib64/libm.so
l-wx------ 1 marxin users 64 Aug  2 10:33 2 -> /tmp/ccq6akH8.le
lr-x------ 1 marxin users 64 Aug  2 10:33 23 -> /dev/shm/objdir/gcc/libgcc_s.so
lr-x------ 1 marxin users 64 Aug  2 10:33 27 -> /usr/lib64/libc.so
lrwx------ 1 marxin users 64 Aug  2 10:33 3 -> /dev/pts/2
lr-x------ 1 marxin users 64 Aug  2 10:33 31 -> /dev/shm/objdir/gcc/libgcc_s.so

[-- Attachment #3: 0001-Properly-detect-working-jobserver-in-gcc-driver.patch --]
[-- Type: text/x-patch, Size: 3180 bytes --]

From 295e6aeae929d33fb904439a0fda3b840ab6c980 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Thu, 1 Aug 2019 16:30:01 +0200
Subject: [PATCH] Properly detect working jobserver in gcc driver.

gcc/ChangeLog:

2019-08-02  Martin Liska  <mliska@suse.cz>

	* gcc.c (driver::maybe_run_linker): Call detect_jobserver
	to detect working job server.
	(driver::detect_jobserver): Test whether jobserver
	is active from GCC driver. That will prevent situation where
	GCC is invoked from a LD plugin and the linker already uses
	file descriptors suggested by make.  That leads to a wrong
	detection.
	* gcc.h (driver): Add detect_jobserver.
	* lto-wrapper.c (jobserver_active_p): Simplify sscanf by
	not scanning for --jobserver-auth prefix.
---
 gcc/gcc.c         | 42 ++++++++++++++++++++++++++++++++++++++++++
 gcc/gcc.h         |  1 +
 gcc/lto-wrapper.c |  2 +-
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/gcc/gcc.c b/gcc/gcc.c
index a4323eb146e..18a07426290 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -8268,6 +8268,8 @@ driver::maybe_run_linker (const char *argv0) const
     {
       int tmp = execution_count;
 
+      detect_jobserver ();
+
       if (! have_c)
 	{
 #if HAVE_LTO_PLUGIN > 0
@@ -8357,6 +8359,46 @@ driver::final_actions () const
     }
 }
 
+/* Detect whether jobserver is active and working.  If not drop
+   --jobserver-auth from MAKEFLAGS.  */
+
+void
+driver::detect_jobserver () const
+{
+  /* Detect jobserver and drop it if it's not working.  */
+  const char *makeflags = env.get ("MAKEFLAGS");
+  if (makeflags != NULL)
+    {
+      const char *needle = "--jobserver-auth=";
+      const char *n = strstr (makeflags, needle);
+      if (n != NULL)
+	{
+	  int rfd = -1;
+	  int wfd = -1;
+
+	  bool jobserver
+	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
+	       && rfd > 0
+	       && wfd > 0
+	       && fcntl (rfd, F_GETFD) >= 0
+	       && fcntl (wfd, F_GETFD) >= 0);
+
+	  /* Drop the jobserver if it's not working now.  */
+	  if (!jobserver)
+	    {
+	      unsigned offset = n - makeflags;
+	      char *dup = xstrdup (makeflags);
+	      dup[offset] = '\0';
+
+	      const char *space = strchr (makeflags + offset, ' ');
+	      if (space != NULL)
+		strcpy (dup + offset, space);
+	      xputenv (concat ("MAKEFLAGS=", dup, NULL));
+	    }
+	}
+    }
+}
+
 /* Determine what the exit code of the driver should be.  */
 
 int
diff --git a/gcc/gcc.h b/gcc/gcc.h
index a0a1d94c6e6..dc77dba67fb 100644
--- a/gcc/gcc.h
+++ b/gcc/gcc.h
@@ -51,6 +51,7 @@ class driver
   void do_spec_on_infiles () const;
   void maybe_run_linker (const char *argv0) const;
   void final_actions () const;
+  void detect_jobserver () const;
   int get_exit_code () const;
 
  private:
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 353187c6043..3414adedd26 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1234,7 +1234,7 @@ jobserver_active_p (void)
   int rfd = -1;
   int wfd = -1;
 
-  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
 	  && rfd > 0
 	  && wfd > 0
 	  && fcntl (rfd, F_GETFD) >= 0
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jakub Jelinek]

On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
> > Can you strace if other fds are opened and not closed in the spot you had it
> > before?  Advantage of doing it there is that it will not be done for all the
> > -E/-S/-c compilations when the linker is not spawned.
> 
> I've used the same trick which you used and I'm attaching the output.
> I believe it's fine, I can't see any opened fd by GCC.

LGTM.

> gcc/ChangeLog:
> 
> 2019-08-02  Martin Liska  <mliska@suse.cz>
> 
> 	* gcc.c (driver::maybe_run_linker): Call detect_jobserver
> 	to detect working job server.
> 	(driver::detect_jobserver): Test whether jobserver
> 	is active from GCC driver. That will prevent situation where
> 	GCC is invoked from a LD plugin and the linker already uses
> 	file descriptors suggested by make.  That leads to a wrong
> 	detection.
> 	* gcc.h (driver): Add detect_jobserver.
> 	* lto-wrapper.c (jobserver_active_p): Simplify sscanf by
> 	not scanning for --jobserver-auth prefix.
> ---
>  gcc/gcc.c         | 42 ++++++++++++++++++++++++++++++++++++++++++
>  gcc/gcc.h         |  1 +
>  gcc/lto-wrapper.c |  2 +-
>  3 files changed, 44 insertions(+), 1 deletion(-)
> 
> diff --git a/gcc/gcc.c b/gcc/gcc.c
> index a4323eb146e..18a07426290 100644
> --- a/gcc/gcc.c
> +++ b/gcc/gcc.c
> @@ -8268,6 +8268,8 @@ driver::maybe_run_linker (const char *argv0) const
>      {
>        int tmp = execution_count;
>  
> +      detect_jobserver ();
> +
>        if (! have_c)
>  	{
>  #if HAVE_LTO_PLUGIN > 0
> @@ -8357,6 +8359,46 @@ driver::final_actions () const
>      }
>  }
>  
> +/* Detect whether jobserver is active and working.  If not drop
> +   --jobserver-auth from MAKEFLAGS.  */
> +
> +void
> +driver::detect_jobserver () const
> +{
> +  /* Detect jobserver and drop it if it's not working.  */
> +  const char *makeflags = env.get ("MAKEFLAGS");
> +  if (makeflags != NULL)
> +    {
> +      const char *needle = "--jobserver-auth=";
> +      const char *n = strstr (makeflags, needle);
> +      if (n != NULL)
> +	{
> +	  int rfd = -1;
> +	  int wfd = -1;
> +
> +	  bool jobserver
> +	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> +	       && rfd > 0
> +	       && wfd > 0
> +	       && fcntl (rfd, F_GETFD) >= 0
> +	       && fcntl (wfd, F_GETFD) >= 0);
> +
> +	  /* Drop the jobserver if it's not working now.  */
> +	  if (!jobserver)
> +	    {
> +	      unsigned offset = n - makeflags;
> +	      char *dup = xstrdup (makeflags);
> +	      dup[offset] = '\0';
> +
> +	      const char *space = strchr (makeflags + offset, ' ');
> +	      if (space != NULL)
> +		strcpy (dup + offset, space);
> +	      xputenv (concat ("MAKEFLAGS=", dup, NULL));
> +	    }
> +	}
> +    }
> +}
> +
>  /* Determine what the exit code of the driver should be.  */
>  
>  int
> diff --git a/gcc/gcc.h b/gcc/gcc.h
> index a0a1d94c6e6..dc77dba67fb 100644
> --- a/gcc/gcc.h
> +++ b/gcc/gcc.h
> @@ -51,6 +51,7 @@ class driver
>    void do_spec_on_infiles () const;
>    void maybe_run_linker (const char *argv0) const;
>    void final_actions () const;
> +  void detect_jobserver () const;
>    int get_exit_code () const;
>  
>   private:
> diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> index 353187c6043..3414adedd26 100644
> --- a/gcc/lto-wrapper.c
> +++ b/gcc/lto-wrapper.c
> @@ -1234,7 +1234,7 @@ jobserver_active_p (void)
>    int rfd = -1;
>    int wfd = -1;
>  
> -  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> +  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
>  	  && rfd > 0
>  	  && wfd > 0
>  	  && fcntl (rfd, F_GETFD) >= 0
> -- 
> 2.22.0
> 


	Jakub

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Richard Biener]

On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
>
> On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
> > > Can you strace if other fds are opened and not closed in the spot you had it
> > > before?  Advantage of doing it there is that it will not be done for all the
> > > -E/-S/-c compilations when the linker is not spawned.
> >
> > I've used the same trick which you used and I'm attaching the output.
> > I believe it's fine, I can't see any opened fd by GCC.
>
> LGTM.

Btw, we discussed yesterday on the phone and the conclusion was to
make -flto auto-detect a job-server (but not fall back to # of threads)
and add -flto=auto to auto-detect a job-server and fall back to # of threads.
That basically makes -flto=jobserver the default behavior which means
we should document -flto=1 as a way to override jobserver detection.

We also discussed carrying distribution-local patches to make GNU make
expose the jobserver FD to all jobs, not just those marked to make the
new default effective.  Does anybody have an idea if there's another
common enough make that would benefit from support?  That is,
do other "jobserver" implementations/APIs exist we could support
or is another make used in more than 1% of build systems?

Richard.

> > gcc/ChangeLog:
> >
> > 2019-08-02  Martin Liska  <mliska@suse.cz>
> >
> >       * gcc.c (driver::maybe_run_linker): Call detect_jobserver
> >       to detect working job server.
> >       (driver::detect_jobserver): Test whether jobserver
> >       is active from GCC driver. That will prevent situation where
> >       GCC is invoked from a LD plugin and the linker already uses
> >       file descriptors suggested by make.  That leads to a wrong
> >       detection.
> >       * gcc.h (driver): Add detect_jobserver.
> >       * lto-wrapper.c (jobserver_active_p): Simplify sscanf by
> >       not scanning for --jobserver-auth prefix.
> > ---
> >  gcc/gcc.c         | 42 ++++++++++++++++++++++++++++++++++++++++++
> >  gcc/gcc.h         |  1 +
> >  gcc/lto-wrapper.c |  2 +-
> >  3 files changed, 44 insertions(+), 1 deletion(-)
> >
> > diff --git a/gcc/gcc.c b/gcc/gcc.c
> > index a4323eb146e..18a07426290 100644
> > --- a/gcc/gcc.c
> > +++ b/gcc/gcc.c
> > @@ -8268,6 +8268,8 @@ driver::maybe_run_linker (const char *argv0) const
> >      {
> >        int tmp = execution_count;
> >
> > +      detect_jobserver ();
> > +
> >        if (! have_c)
> >       {
> >  #if HAVE_LTO_PLUGIN > 0
> > @@ -8357,6 +8359,46 @@ driver::final_actions () const
> >      }
> >  }
> >
> > +/* Detect whether jobserver is active and working.  If not drop
> > +   --jobserver-auth from MAKEFLAGS.  */
> > +
> > +void
> > +driver::detect_jobserver () const
> > +{
> > +  /* Detect jobserver and drop it if it's not working.  */
> > +  const char *makeflags = env.get ("MAKEFLAGS");
> > +  if (makeflags != NULL)
> > +    {
> > +      const char *needle = "--jobserver-auth=";
> > +      const char *n = strstr (makeflags, needle);
> > +      if (n != NULL)
> > +     {
> > +       int rfd = -1;
> > +       int wfd = -1;
> > +
> > +       bool jobserver
> > +         = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> > +            && rfd > 0
> > +            && wfd > 0
> > +            && fcntl (rfd, F_GETFD) >= 0
> > +            && fcntl (wfd, F_GETFD) >= 0);
> > +
> > +       /* Drop the jobserver if it's not working now.  */
> > +       if (!jobserver)
> > +         {
> > +           unsigned offset = n - makeflags;
> > +           char *dup = xstrdup (makeflags);
> > +           dup[offset] = '\0';
> > +
> > +           const char *space = strchr (makeflags + offset, ' ');
> > +           if (space != NULL)
> > +             strcpy (dup + offset, space);
> > +           xputenv (concat ("MAKEFLAGS=", dup, NULL));
> > +         }
> > +     }
> > +    }
> > +}
> > +
> >  /* Determine what the exit code of the driver should be.  */
> >
> >  int
> > diff --git a/gcc/gcc.h b/gcc/gcc.h
> > index a0a1d94c6e6..dc77dba67fb 100644
> > --- a/gcc/gcc.h
> > +++ b/gcc/gcc.h
> > @@ -51,6 +51,7 @@ class driver
> >    void do_spec_on_infiles () const;
> >    void maybe_run_linker (const char *argv0) const;
> >    void final_actions () const;
> > +  void detect_jobserver () const;
> >    int get_exit_code () const;
> >
> >   private:
> > diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> > index 353187c6043..3414adedd26 100644
> > --- a/gcc/lto-wrapper.c
> > +++ b/gcc/lto-wrapper.c
> > @@ -1234,7 +1234,7 @@ jobserver_active_p (void)
> >    int rfd = -1;
> >    int wfd = -1;
> >
> > -  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> > +  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> >         && rfd > 0
> >         && wfd > 0
> >         && fcntl (rfd, F_GETFD) >= 0
> > --
> > 2.22.0
> >
>
>
>         Jakub

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jan Hubicka]

> On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
> >
> > On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
> > > > Can you strace if other fds are opened and not closed in the spot you had it
> > > > before?  Advantage of doing it there is that it will not be done for all the
> > > > -E/-S/-c compilations when the linker is not spawned.
> > >
> > > I've used the same trick which you used and I'm attaching the output.
> > > I believe it's fine, I can't see any opened fd by GCC.
> >
> > LGTM.
> 
> Btw, we discussed yesterday on the phone and the conclusion was to
> make -flto auto-detect a job-server (but not fall back to # of threads)
> and add -flto=auto to auto-detect a job-server and fall back to # of threads.
> That basically makes -flto=jobserver the default behavior which means
> we should document -flto=1 as a way to override jobserver detection.
> 
> We also discussed carrying distribution-local patches to make GNU make
> expose the jobserver FD to all jobs, not just those marked to make the
> new default effective.  Does anybody have an idea if there's another
> common enough make that would benefit from support?  That is,
> do other "jobserver" implementations/APIs exist we could support
> or is another make used in more than 1% of build systems?

Ideal solution would be to provide a simple jobserver library for
multithreaded tools to use (GCC is not the only one - llvm, compression
programs etc) and patch make and its replacements to use it.
I think Ninja is used by some larger projects now.

Honza
> 
> Richard.
> 
> > > gcc/ChangeLog:
> > >
> > > 2019-08-02  Martin Liska  <mliska@suse.cz>
> > >
> > >       * gcc.c (driver::maybe_run_linker): Call detect_jobserver
> > >       to detect working job server.
> > >       (driver::detect_jobserver): Test whether jobserver
> > >       is active from GCC driver. That will prevent situation where
> > >       GCC is invoked from a LD plugin and the linker already uses
> > >       file descriptors suggested by make.  That leads to a wrong
> > >       detection.
> > >       * gcc.h (driver): Add detect_jobserver.
> > >       * lto-wrapper.c (jobserver_active_p): Simplify sscanf by
> > >       not scanning for --jobserver-auth prefix.
> > > ---
> > >  gcc/gcc.c         | 42 ++++++++++++++++++++++++++++++++++++++++++
> > >  gcc/gcc.h         |  1 +
> > >  gcc/lto-wrapper.c |  2 +-
> > >  3 files changed, 44 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/gcc/gcc.c b/gcc/gcc.c
> > > index a4323eb146e..18a07426290 100644
> > > --- a/gcc/gcc.c
> > > +++ b/gcc/gcc.c
> > > @@ -8268,6 +8268,8 @@ driver::maybe_run_linker (const char *argv0) const
> > >      {
> > >        int tmp = execution_count;
> > >
> > > +      detect_jobserver ();
> > > +
> > >        if (! have_c)
> > >       {
> > >  #if HAVE_LTO_PLUGIN > 0
> > > @@ -8357,6 +8359,46 @@ driver::final_actions () const
> > >      }
> > >  }
> > >
> > > +/* Detect whether jobserver is active and working.  If not drop
> > > +   --jobserver-auth from MAKEFLAGS.  */
> > > +
> > > +void
> > > +driver::detect_jobserver () const
> > > +{
> > > +  /* Detect jobserver and drop it if it's not working.  */
> > > +  const char *makeflags = env.get ("MAKEFLAGS");
> > > +  if (makeflags != NULL)
> > > +    {
> > > +      const char *needle = "--jobserver-auth=";
> > > +      const char *n = strstr (makeflags, needle);
> > > +      if (n != NULL)
> > > +     {
> > > +       int rfd = -1;
> > > +       int wfd = -1;
> > > +
> > > +       bool jobserver
> > > +         = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> > > +            && rfd > 0
> > > +            && wfd > 0
> > > +            && fcntl (rfd, F_GETFD) >= 0
> > > +            && fcntl (wfd, F_GETFD) >= 0);
> > > +
> > > +       /* Drop the jobserver if it's not working now.  */
> > > +       if (!jobserver)
> > > +         {
> > > +           unsigned offset = n - makeflags;
> > > +           char *dup = xstrdup (makeflags);
> > > +           dup[offset] = '\0';
> > > +
> > > +           const char *space = strchr (makeflags + offset, ' ');
> > > +           if (space != NULL)
> > > +             strcpy (dup + offset, space);
> > > +           xputenv (concat ("MAKEFLAGS=", dup, NULL));
> > > +         }
> > > +     }
> > > +    }
> > > +}
> > > +
> > >  /* Determine what the exit code of the driver should be.  */
> > >
> > >  int
> > > diff --git a/gcc/gcc.h b/gcc/gcc.h
> > > index a0a1d94c6e6..dc77dba67fb 100644
> > > --- a/gcc/gcc.h
> > > +++ b/gcc/gcc.h
> > > @@ -51,6 +51,7 @@ class driver
> > >    void do_spec_on_infiles () const;
> > >    void maybe_run_linker (const char *argv0) const;
> > >    void final_actions () const;
> > > +  void detect_jobserver () const;
> > >    int get_exit_code () const;
> > >
> > >   private:
> > > diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
> > > index 353187c6043..3414adedd26 100644
> > > --- a/gcc/lto-wrapper.c
> > > +++ b/gcc/lto-wrapper.c
> > > @@ -1234,7 +1234,7 @@ jobserver_active_p (void)
> > >    int rfd = -1;
> > >    int wfd = -1;
> > >
> > > -  return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
> > > +  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> > >         && rfd > 0
> > >         && wfd > 0
> > >         && fcntl (rfd, F_GETFD) >= 0
> > > --
> > > 2.22.0
> > >
> >
> >
> >         Jakub

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jan Hubicka]

> On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
> >
> > On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
> > > > Can you strace if other fds are opened and not closed in the spot you had it
> > > > before?  Advantage of doing it there is that it will not be done for all the
> > > > -E/-S/-c compilations when the linker is not spawned.
> > >
> > > I've used the same trick which you used and I'm attaching the output.
> > > I believe it's fine, I can't see any opened fd by GCC.
> >
> > LGTM.
> 
> Btw, we discussed yesterday on the phone and the conclusion was to
> make -flto auto-detect a job-server (but not fall back to # of threads)
> and add -flto=auto to auto-detect a job-server and fall back to # of threads.
> That basically makes -flto=jobserver the default behavior which means
> we should document -flto=1 as a way to override jobserver detection.

And concerning to the yesterday discussion, my preference would still be
-flto to first try presence of jobserver and default to number of
threads otherwise. It seems like user friendly default to me and other
tools with reasonable parallelism support usually behaves this way, too.

Sure one can use -flto=auto everywhere but then one needs to use
different options for differnt compilers.  It would be nice to make
-flto to do right hting for majority of users including those like me
who cut&paste command lines from Make output and execute them by hand.

The fork bombing is IMO relatively rare and it was not what Jakub ran
into (it was broken jobserv detection). 
After all whole distro built with Martin's orriginal approach of
-flto=<nthreads> which forkbombs on every possible occasion. 

Said that, I could live with more conservative defaults.
Honza

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

On 8/2/19 11:15 AM, Jan Hubicka wrote:
>> On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
>>>
>>> On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
>>>>> Can you strace if other fds are opened and not closed in the spot you had it
>>>>> before?  Advantage of doing it there is that it will not be done for all the
>>>>> -E/-S/-c compilations when the linker is not spawned.
>>>>
>>>> I've used the same trick which you used and I'm attaching the output.
>>>> I believe it's fine, I can't see any opened fd by GCC.
>>>
>>> LGTM.
>>

Hi.

>> Btw, we discussed yesterday on the phone and the conclusion was to
>> make -flto auto-detect a job-server (but not fall back to # of threads)
>> and add -flto=auto to auto-detect a job-server and fall back to # of threads.
>> That basically makes -flto=jobserver the default behavior which means
>> we should document -flto=1 as a way to override jobserver detection.
> 
> And concerning to the yesterday discussion, my preference would still be
> -flto to first try presence of jobserver and default to number of
> threads otherwise. It seems like user friendly default to me and other
> tools with reasonable parallelism support usually behaves this way, too.

I also like the default as Honza defined.

> 
> Sure one can use -flto=auto everywhere but then one needs to use
> different options for differnt compilers.  It would be nice to make
> -flto to do right hting for majority of users including those like me
> who cut&paste command lines from Make output and execute them by hand.

Note that our ambition is to ideally backport the patches to our gcc9 package.
The auto-detection of job-server with -flto is a behavior change that will happen
in the gcc9 package.

To be honest I don't like the invention of 'auto' value for -flto command.
That would be useful just for gcc9 right now :/

Martin

> 
> The fork bombing is IMO relatively rare and it was not what Jakub ran
> into (it was broken jobserv detection). 
> After all whole distro built with Martin's orriginal approach of
> -flto=<nthreads> which forkbombs on every possible occasion. 
> 
> Said that, I could live with more conservative defaults.
> Honza
> 

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Richard Biener]

On Fri, Aug 2, 2019 at 11:19 AM Martin Liška <mliska@suse.cz> wrote:
>
> On 8/2/19 11:15 AM, Jan Hubicka wrote:
> >> On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
> >>>
> >>> On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
> >>>>> Can you strace if other fds are opened and not closed in the spot you had it
> >>>>> before?  Advantage of doing it there is that it will not be done for all the
> >>>>> -E/-S/-c compilations when the linker is not spawned.
> >>>>
> >>>> I've used the same trick which you used and I'm attaching the output.
> >>>> I believe it's fine, I can't see any opened fd by GCC.
> >>>
> >>> LGTM.
> >>
>
> Hi.
>
> >> Btw, we discussed yesterday on the phone and the conclusion was to
> >> make -flto auto-detect a job-server (but not fall back to # of threads)
> >> and add -flto=auto to auto-detect a job-server and fall back to # of threads.
> >> That basically makes -flto=jobserver the default behavior which means
> >> we should document -flto=1 as a way to override jobserver detection.
> >
> > And concerning to the yesterday discussion, my preference would still be
> > -flto to first try presence of jobserver and default to number of
> > threads otherwise. It seems like user friendly default to me and other
> > tools with reasonable parallelism support usually behaves this way, too.
>
> I also like the default as Honza defined.
>
> >
> > Sure one can use -flto=auto everywhere but then one needs to use
> > different options for differnt compilers.  It would be nice to make
> > -flto to do right hting for majority of users including those like me
> > who cut&paste command lines from Make output and execute them by hand.
>
> Note that our ambition is to ideally backport the patches to our gcc9 package.
> The auto-detection of job-server with -flto is a behavior change that will happen
> in the gcc9 package.
>
> To be honest I don't like the invention of 'auto' value for -flto command.
> That would be useful just for gcc9 right now :/

Well.  We teached people they need to use -flto=N or -flto=jobserver to
get parallelism.  Like we teached people they need to use -O2 to get
any optimization.

Other compilers behave differently.  So what.

Auto-detecting threads is nice.  Suddenly trashing your system
because you happen to invoke two links in parallel is not.
Which is at least why changing this kind of behavior for 9.2 (or 9.3)
vs. 9.1 is out of the question.  And I'm not sure it is OK for 10.

As with defaulting to use a job-server when present that's more
reasonable.

For cut&pasting you then can use -flto=auto.

But it's just my opinion - I surely can adapt.

Richard.

> Martin
>
> >
> > The fork bombing is IMO relatively rare and it was not what Jakub ran
> > into (it was broken jobserv detection).
> > After all whole distro built with Martin's orriginal approach of
> > -flto=<nthreads> which forkbombs on every possible occasion.
> >
> > Said that, I could live with more conservative defaults.
> > Honza
> >
>

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 3436 bytes --]

On 8/2/19 11:55 AM, Richard Biener wrote:
> On Fri, Aug 2, 2019 at 11:19 AM Martin Liška <mliska@suse.cz> wrote:
>>
>> On 8/2/19 11:15 AM, Jan Hubicka wrote:
>>>> On Fri, Aug 2, 2019 at 10:50 AM Jakub Jelinek <jakub@redhat.com> wrote:
>>>>>
>>>>> On Fri, Aug 02, 2019 at 10:47:10AM +0200, Martin Liška wrote:
>>>>>>> Can you strace if other fds are opened and not closed in the spot you had it
>>>>>>> before?  Advantage of doing it there is that it will not be done for all the
>>>>>>> -E/-S/-c compilations when the linker is not spawned.
>>>>>>
>>>>>> I've used the same trick which you used and I'm attaching the output.
>>>>>> I believe it's fine, I can't see any opened fd by GCC.
>>>>>
>>>>> LGTM.
>>>>
>>
>> Hi.
>>
>>>> Btw, we discussed yesterday on the phone and the conclusion was to
>>>> make -flto auto-detect a job-server (but not fall back to # of threads)
>>>> and add -flto=auto to auto-detect a job-server and fall back to # of threads.
>>>> That basically makes -flto=jobserver the default behavior which means
>>>> we should document -flto=1 as a way to override jobserver detection.
>>>
>>> And concerning to the yesterday discussion, my preference would still be
>>> -flto to first try presence of jobserver and default to number of
>>> threads otherwise. It seems like user friendly default to me and other
>>> tools with reasonable parallelism support usually behaves this way, too.
>>
>> I also like the default as Honza defined.
>>
>>>
>>> Sure one can use -flto=auto everywhere but then one needs to use
>>> different options for differnt compilers.  It would be nice to make
>>> -flto to do right hting for majority of users including those like me
>>> who cut&paste command lines from Make output and execute them by hand.
>>
>> Note that our ambition is to ideally backport the patches to our gcc9 package.
>> The auto-detection of job-server with -flto is a behavior change that will happen
>> in the gcc9 package.
>>
>> To be honest I don't like the invention of 'auto' value for -flto command.
>> That would be useful just for gcc9 right now :/
> 
> Well.  We teached people they need to use -flto=N or -flto=jobserver to
> get parallelism.  Like we teached people they need to use -O2 to get
> any optimization.
> 
> Other compilers behave differently.  So what.
> 
> Auto-detecting threads is nice.  Suddenly trashing your system
> because you happen to invoke two links in parallel is not.
> Which is at least why changing this kind of behavior for 9.2 (or 9.3)
> vs. 9.1 is out of the question.  And I'm not sure it is OK for 10.
> 
> As with defaulting to use a job-server when present that's more
> reasonable.
> 
> For cut&pasting you then can use -flto=auto.
> 
> But it's just my opinion - I surely can adapt.

Hi.

Based on what was written, I feel OK with -flto=auto. I would
like to see jobserver detection to be only enabled with the 'auto'
option value. That will make it consistent in GCC 9.x branch.

Patch can bootstrap on x86_64-linux-gnu and survives regression tests.

Ready to be installed?
Thanks,
Martin

> 
> Richard.
> 
>> Martin
>>
>>>
>>> The fork bombing is IMO relatively rare and it was not what Jakub ran
>>> into (it was broken jobserv detection).
>>> After all whole distro built with Martin's orriginal approach of
>>> -flto=<nthreads> which forkbombs on every possible occasion.
>>>
>>> Said that, I could live with more conservative defaults.
>>> Honza
>>>
>>


[-- Attachment #2: 0001-Add-flto-auto-option-value.patch --]
[-- Type: text/x-patch, Size: 3814 bytes --]

From 41f13e1890cc78292afb36a54982b097a3c6abdf Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Mon, 5 Aug 2019 06:44:25 +0200
Subject: [PATCH] Add -flto=auto option value.

gcc/ChangeLog:

2019-08-05  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document the option value.
	* lto-wrapper.c (run_gcc): Set auto_parallel
	only with -flto=auto.

gcc/testsuite/ChangeLog:

2019-08-05  Martin Liska  <mliska@suse.cz>

	* g++.dg/lto/devirt-19_0.C: Add -flto=auto.
---
 gcc/doc/invoke.texi                    |  8 ++++++--
 gcc/lto-wrapper.c                      | 18 ++++++++----------
 gcc/testsuite/g++.dg/lto/devirt-19_0.C |  2 +-
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 7b3c77b8033..5ea56fbb23a 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10396,8 +10396,7 @@ If you specify the optional @var{n}, the optimization and code
 generation done at link time is executed in parallel using @var{n}
 parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
-used.  The default value for @var{n} is automatically detected based
-on number of cores.
+used.
 
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
@@ -10406,6 +10405,11 @@ You must prepend a @samp{+} to the command recipe in the parent Makefile
 for this to work.  This option likely only works if @env{MAKE} is
 GNU make.
 
+One can also use @option{-flto=auto} to either use GNU make's
+job server mode to determine the number of parallel jobs, if available.
+Or the default value for @var{n} is automatically detected based
+on number of cores.
+
 @item -flto-partition=@var{alg}
 @opindex flto-partition
 Specify the partitioning algorithm used by the link-time optimizer.
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 3414adedd26..b825e0a0aa0 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1252,8 +1252,7 @@ run_gcc (unsigned argc, char *argv[])
   char *list_option_full = NULL;
   const char *linker_output = NULL;
   const char *collect_gcc, *collect_gcc_options;
-  /* Make linking parallel by default.  */
-  int parallel = 1;
+  int parallel = 0;
   int jobserver = 0;
   int auto_parallel = 0;
   bool no_partition = false;
@@ -1380,6 +1379,11 @@ run_gcc (unsigned argc, char *argv[])
 	case OPT_flto_:
 	  if (strcmp (option->arg, "jobserver") == 0)
 	    jobserver = 1;
+	  else if (strcmp (option->arg, "auto") == 0)
+	    {
+	      parallel = 1;
+	      auto_parallel = 1;
+	    }
 	  else
 	    {
 	      parallel = atoi (option->arg);
@@ -1423,14 +1427,8 @@ run_gcc (unsigned argc, char *argv[])
       auto_parallel = 0;
       parallel = 0;
     }
-  else if (!jobserver && parallel)
-    {
-      /* If there's no explicit usage of jobserver and
-	 parallel is enabled, then automatically detect
-	 jobserver or number of cores.  */
-      auto_parallel = 1;
-      jobserver = jobserver_active_p ();
-    }
+  else if (!jobserver && auto_parallel)
+    jobserver = jobserver_active_p ();
 
   if (linker_output)
     {
diff --git a/gcc/testsuite/g++.dg/lto/devirt-19_0.C b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
index 696d8c0fc83..b43527e324e 100644
--- a/gcc/testsuite/g++.dg/lto/devirt-19_0.C
+++ b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
@@ -1,5 +1,5 @@
 /* { dg-lto-do link } */
 /* { dg-lto-options { "-O2 -fdump-ipa-cp -Wno-return-type -flto -r -nostdlib" } } */
-/* { dg-extra-ld-options "-flinker-output=nolto-rel" } */
+/* { dg-extra-ld-options "-flinker-output=nolto-rel -flto=auto" } */
 #include "../ipa/devirt-19.C"
 /* { dg-final { scan-wpa-ipa-dump-times "Discovered a virtual call to a known target" 1 "cp"  } } */
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 108 bytes --]

I'm sending slightly updated version of the patch
where I allow -flto=auto in common_handle_option.

Martin

[-- Attachment #2: 0001-Add-flto-auto-option-value.patch --]
[-- Type: text/x-patch, Size: 4272 bytes --]

From cc04dfc9dbf2ed91a021093d1d27b81848ea726b Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Mon, 5 Aug 2019 06:44:25 +0200
Subject: [PATCH] Add -flto=auto option value.

gcc/ChangeLog:

2019-08-05  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document the option value.
	* lto-wrapper.c (run_gcc): Set auto_parallel
	only with -flto=auto.

gcc/testsuite/ChangeLog:

2019-08-05  Martin Liska  <mliska@suse.cz>

	* g++.dg/lto/devirt-19_0.C: Add -flto=auto.
---
 gcc/doc/invoke.texi                    |  8 ++++++--
 gcc/lto-wrapper.c                      | 18 ++++++++----------
 gcc/opts.c                             |  1 +
 gcc/testsuite/g++.dg/lto/devirt-19_0.C |  2 +-
 4 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 01aab60f895..ba9cca9de84 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10415,8 +10415,7 @@ If you specify the optional @var{n}, the optimization and code
 generation done at link time is executed in parallel using @var{n}
 parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
-used.  The default value for @var{n} is automatically detected based
-on number of cores.
+used.
 
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
@@ -10425,6 +10424,11 @@ You must prepend a @samp{+} to the command recipe in the parent Makefile
 for this to work.  This option likely only works if @env{MAKE} is
 GNU make.
 
+One can also use @option{-flto=auto} to either use GNU make's
+job server mode to determine the number of parallel jobs, if available.
+Or the default value for @var{n} is automatically detected based
+on number of cores.
+
 @item -flto-partition=@var{alg}
 @opindex flto-partition
 Specify the partitioning algorithm used by the link-time optimizer.
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index f1253cdc91c..84f59cf1a1f 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1252,8 +1252,7 @@ run_gcc (unsigned argc, char *argv[])
   char *list_option_full = NULL;
   const char *linker_output = NULL;
   const char *collect_gcc, *collect_gcc_options;
-  /* Make linking parallel by default.  */
-  int parallel = 1;
+  int parallel = 0;
   int jobserver = 0;
   int auto_parallel = 0;
   bool no_partition = false;
@@ -1380,6 +1379,11 @@ run_gcc (unsigned argc, char *argv[])
 	case OPT_flto_:
 	  if (strcmp (option->arg, "jobserver") == 0)
 	    jobserver = 1;
+	  else if (strcmp (option->arg, "auto") == 0)
+	    {
+	      parallel = 1;
+	      auto_parallel = 1;
+	    }
 	  else
 	    {
 	      parallel = atoi (option->arg);
@@ -1423,14 +1427,8 @@ run_gcc (unsigned argc, char *argv[])
       auto_parallel = 0;
       parallel = 0;
     }
-  else if (!jobserver && parallel)
-    {
-      /* If there's no explicit usage of jobserver and
-	 parallel is enabled, then automatically detect
-	 jobserver or number of cores.  */
-      auto_parallel = 1;
-      jobserver = jobserver_active_p ();
-    }
+  else if (!jobserver && auto_parallel)
+    jobserver = jobserver_active_p ();
 
   if (linker_output)
     {
diff --git a/gcc/opts.c b/gcc/opts.c
index a0a77893448..bb0d8b5e7db 100644
--- a/gcc/opts.c
+++ b/gcc/opts.c
@@ -2825,6 +2825,7 @@ common_handle_option (struct gcc_options *opts,
     case OPT_flto_:
       if (strcmp (arg, "none") != 0
 	  && strcmp (arg, "jobserver") != 0
+	  && strcmp (arg, "auto") != 0
 	  && atoi (arg) == 0)
 	error_at (loc,
 		  "unrecognized argument to %<-flto=%> option: %qs", arg);
diff --git a/gcc/testsuite/g++.dg/lto/devirt-19_0.C b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
index 696d8c0fc83..b43527e324e 100644
--- a/gcc/testsuite/g++.dg/lto/devirt-19_0.C
+++ b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
@@ -1,5 +1,5 @@
 /* { dg-lto-do link } */
 /* { dg-lto-options { "-O2 -fdump-ipa-cp -Wno-return-type -flto -r -nostdlib" } } */
-/* { dg-extra-ld-options "-flinker-output=nolto-rel" } */
+/* { dg-extra-ld-options "-flinker-output=nolto-rel -flto=auto" } */
 #include "../ipa/devirt-19.C"
 /* { dg-final { scan-wpa-ipa-dump-times "Discovered a virtual call to a known target" 1 "cp"  } } */
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Richard Biener]

On Fri, Aug 9, 2019 at 10:11 AM Martin Liška <mliska@suse.cz> wrote:
>
> I'm sending slightly updated version of the patch
> where I allow -flto=auto in common_handle_option.

+One can also use @option{-flto=auto} to either use GNU make's
+job server mode to determine the number of parallel jobs, if available.
+Or the default value for @var{n} is automatically detected based
+on number of cores.

"Use @option{-flto=auto} to use GNU make's job server, if available,
or otherwise fall back to autodetection of the number of CPU threads
present in your system."

OK with that.  I still think that making -flto use a jobserver if detected
(but _not_ use the number of CPU cores by default) makes
sense as an independent change.

> Martin

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 302 bytes --]

On 8/9/19 10:19 AM, Richard Biener wrote:
> OK with that.  I still think that making -flto use a jobserver if detected
> (but _not_ use the number of CPU cores by default) makes
> sense as an independent change.

In order to address that, I'm suggesting following patch that I've been
testing.

Martin

[-- Attachment #2: 0001-Automatically-detect-GNU-jobserver-with-flto.patch --]
[-- Type: text/x-patch, Size: 4573 bytes --]

From 0f159f703fa2c918cfa75d5636f9b118b6ca1871 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Fri, 9 Aug 2019 14:03:11 +0200
Subject: [PATCH] Automatically detect GNU jobserver with -flto.

config/ChangeLog:

2019-08-09  Martin Liska  <mliska@suse.cz>

	* bootstrap-lto-lean.mk: Remove -flto=jobserver and use
	only -flto.
	* bootstrap-lto-noplugin.mk: Likewise.
	* bootstrap-lto.mk: Likewise.

gcc/ChangeLog:

2019-08-09  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document automatic detection of jobserver.
	* lto-wrapper.c (run_gcc): Detect jobserver always.
---
 config/bootstrap-lto-lean.mk     |  8 ++++----
 config/bootstrap-lto-noplugin.mk | 10 +++++-----
 config/bootstrap-lto.mk          | 10 +++++-----
 gcc/doc/invoke.texi              |  3 ++-
 gcc/lto-wrapper.c                |  2 +-
 5 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/config/bootstrap-lto-lean.mk b/config/bootstrap-lto-lean.mk
index 79cea50a4c6..9fbaef3f811 100644
--- a/config/bootstrap-lto-lean.mk
+++ b/config/bootstrap-lto-lean.mk
@@ -1,10 +1,10 @@
 # This option enables LTO for stage4 and LTO for generators in stage3 with profiledbootstrap.
 # Otherwise, LTO is used in only stage3.
 
-STAGE3_CFLAGS += -flto=jobserver
-override STAGEtrain_CFLAGS := $(filter-out -flto=jobserver,$(STAGEtrain_CFLAGS))
-STAGEtrain_GENERATOR_CFLAGS += -flto=jobserver
-STAGEfeedback_CFLAGS += -flto=jobserver
+STAGE3_CFLAGS += -flto
+override STAGEtrain_CFLAGS := $(filter-out -flto,$(STAGEtrain_CFLAGS))
+STAGEtrain_GENERATOR_CFLAGS += -flto
+STAGEfeedback_CFLAGS += -flto
 
 # assumes the host supports the linker plugin
 LTO_AR = $$r/$(HOST_SUBDIR)/prev-gcc/gcc-ar$(exeext) -B$$r/$(HOST_SUBDIR)/prev-gcc/
diff --git a/config/bootstrap-lto-noplugin.mk b/config/bootstrap-lto-noplugin.mk
index 0f50708e49d..592a75fba99 100644
--- a/config/bootstrap-lto-noplugin.mk
+++ b/config/bootstrap-lto-noplugin.mk
@@ -1,9 +1,9 @@
 # This option enables LTO for stage2 and stage3 on
 # hosts without linker plugin support.
 
-STAGE2_CFLAGS += -flto=jobserver -frandom-seed=1 -ffat-lto-objects
-STAGE3_CFLAGS += -flto=jobserver -frandom-seed=1 -ffat-lto-objects
-STAGEprofile_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGEtrain_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGEfeedback_CFLAGS += -flto=jobserver -frandom-seed=1
+STAGE2_CFLAGS += -flto -frandom-seed=1 -ffat-lto-objects
+STAGE3_CFLAGS += -flto -frandom-seed=1 -ffat-lto-objects
+STAGEprofile_CFLAGS += -flto -frandom-seed=1
+STAGEtrain_CFLAGS += -flto -frandom-seed=1
+STAGEfeedback_CFLAGS += -flto -frandom-seed=1
 do-compare = /bin/true
diff --git a/config/bootstrap-lto.mk b/config/bootstrap-lto.mk
index 4de07e5b226..09084bd0b8e 100644
--- a/config/bootstrap-lto.mk
+++ b/config/bootstrap-lto.mk
@@ -1,10 +1,10 @@
 # This option enables LTO for stage2 and stage3 in slim mode
 
-STAGE2_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGE3_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGEprofile_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGEtrain_CFLAGS += -flto=jobserver -frandom-seed=1
-STAGEfeedback_CFLAGS += -flto=jobserver -frandom-seed=1
+STAGE2_CFLAGS += -flto -frandom-seed=1
+STAGE3_CFLAGS += -flto -frandom-seed=1
+STAGEprofile_CFLAGS += -flto -frandom-seed=1
+STAGEtrain_CFLAGS += -flto -frandom-seed=1
+STAGEfeedback_CFLAGS += -flto -frandom-seed=1
 
 # assumes the host supports the linker plugin
 LTO_AR = $$r/$(HOST_SUBDIR)/prev-gcc/gcc-ar$(exeext) -B$$r/$(HOST_SUBDIR)/prev-gcc/
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 5b6b824bdd3..0000d358e48 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10422,7 +10422,8 @@ job server mode to determine the number of parallel jobs. This
 is useful when the Makefile calling GCC is already executing in parallel.
 You must prepend a @samp{+} to the command recipe in the parent Makefile
 for this to work.  This option likely only works if @env{MAKE} is
-GNU make.
+GNU make.  Even without the option value, GCC tries to automatically
+detect a running GNU make's job server.
 
 Use @option{-flto=auto} to use GNU make's job server, if available,
 or otherwise fall back to autodetection of the number of CPU threads
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 84f59cf1a1f..339c379d972 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1427,7 +1427,7 @@ run_gcc (unsigned argc, char *argv[])
       auto_parallel = 0;
       parallel = 0;
     }
-  else if (!jobserver && auto_parallel)
+  else if (!jobserver)
     jobserver = jobserver_active_p ();
 
   if (linker_output)
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 407 bytes --]

On 8/9/19 2:38 PM, Martin Liška wrote:
> On 8/9/19 10:19 AM, Richard Biener wrote:
>> OK with that.  I still think that making -flto use a jobserver if detected
>> (but _not_ use the number of CPU cores by default) makes
>> sense as an independent change.
> 
> In order to address that, I'm suggesting following patch that I've been
> testing.
> 
> Martin
> 

Hm, I take back the config changes.

Martin

[-- Attachment #2: 0001-Automatically-detect-GNU-jobserver-with-flto.patch --]
[-- Type: text/x-patch, Size: 1600 bytes --]

From 1f9c9f74a84ec3ca930bbc9525ef2185200e0ce8 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Fri, 9 Aug 2019 14:03:11 +0200
Subject: [PATCH] Automatically detect GNU jobserver with -flto.

gcc/ChangeLog:

2019-08-09  Martin Liska  <mliska@suse.cz>

	* doc/invoke.texi: Document automatic detection of jobserver.
	* lto-wrapper.c (run_gcc): Detect jobserver always.
---
 gcc/doc/invoke.texi | 3 ++-
 gcc/lto-wrapper.c   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 5b6b824bdd3..0000d358e48 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10422,7 +10422,8 @@ job server mode to determine the number of parallel jobs. This
 is useful when the Makefile calling GCC is already executing in parallel.
 You must prepend a @samp{+} to the command recipe in the parent Makefile
 for this to work.  This option likely only works if @env{MAKE} is
-GNU make.
+GNU make.  Even without the option value, GCC tries to automatically
+detect a running GNU make's job server.
 
 Use @option{-flto=auto} to use GNU make's job server, if available,
 or otherwise fall back to autodetection of the number of CPU threads
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 84f59cf1a1f..339c379d972 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1427,7 +1427,7 @@ run_gcc (unsigned argc, char *argv[])
       auto_parallel = 0;
       parallel = 0;
     }
-  else if (!jobserver && auto_parallel)
+  else if (!jobserver)
     jobserver = jobserver_active_p ();
 
   if (linker_output)
-- 
2.22.0

Re: [PATCH] Properly detect working jobserver in gcc driver.

[Jeff Law]

On 8/9/19 7:05 AM, Martin Liška wrote:
> On 8/9/19 2:38 PM, Martin Liška wrote:
>> On 8/9/19 10:19 AM, Richard Biener wrote:
>>> OK with that.  I still think that making -flto use a jobserver if detected
>>> (but _not_ use the number of CPU cores by default) makes
>>> sense as an independent change.
>> In order to address that, I'm suggesting following patch that I've been
>> testing.
>>
>> Martin
>>
> Hm, I take back the config changes.
> 
> Martin
> 
> 
> 0001-Automatically-detect-GNU-jobserver-with-flto.patch
> 
> From 1f9c9f74a84ec3ca930bbc9525ef2185200e0ce8 Mon Sep 17 00:00:00 2001
> From: Martin Liska <mliska@suse.cz>
> Date: Fri, 9 Aug 2019 14:03:11 +0200
> Subject: [PATCH] Automatically detect GNU jobserver with -flto.
> 
> gcc/ChangeLog:
> 
> 2019-08-09  Martin Liska  <mliska@suse.cz>
> 
> 	* doc/invoke.texi: Document automatic detection of jobserver.
> 	* lto-wrapper.c (run_gcc): Detect jobserver always.
OK
jeff