From: "Martin Liška" <mliska@suse.cz>
To: Jeff Law <law@redhat.com>, Jan Hubicka <hubicka@ucw.cz>
Cc: gcc-patches@gcc.gnu.org, Michael Matz <matz@suse.de>,
Richard Biener <richard.guenther@gmail.com>
Subject: Re: [PATCH] Deduce automatically number of cores for -flto option.
Date: Mon, 29 Jul 2019 13:37:00 -0000 [thread overview]
Message-ID: <ae0d9a26-bdf9-c4d3-cf06-f32be2379939@suse.cz> (raw)
In-Reply-To: <69d3f527-4086-da6c-e789-469dc389d9eb@redhat.com>
[-- Attachment #1: Type: text/plain, Size: 198 bytes --]
Hi.
I'm sending v2 of the patch that can newly auto-detect make
jobserver and use it.
Patch can bootstrap on x86_64-linux-gnu and survives regression tests.
Ready to be installed?
Thanks,
Martin
[-- Attachment #2: 0001-Deduce-automatically-number-of-cores-for-flto-option.patch --]
[-- Type: text/x-patch, Size: 7239 bytes --]
From df747a46241dcdb8ad055071f9e0605c9f469268 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 23 Jul 2019 10:14:31 +0200
Subject: [PATCH 1/2] Deduce automatically number of cores for -flto option.
gcc/ChangeLog:
2019-07-23 Martin Liska <mliska@suse.cz>
* doc/invoke.texi: Document new behavior.
* lto-wrapper.c (cpuset_popcount): New function
is a copy of libgomp/config/linux/proc.c.
(init_num_threads): Likewise.
(run_gcc): Automatically detect core count for -flto.
(jobserver_active_p): New function.
---
gcc/doc/invoke.texi | 3 +-
gcc/lto-wrapper.c | 167 ++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 162 insertions(+), 8 deletions(-)
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 77a2d561e38..f5bfea3f003 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10396,7 +10396,8 @@ If you specify the optional @var{n}, the optimization and code
generation done at link time is executed in parallel using @var{n}
parallel jobs by utilizing an installed @command{make} program. The
environment variable @env{MAKE} may be used to override the program
-used. The default value for @var{n} is 1.
+used. The default value for @var{n} is automatically detected based
+on number of cores.
You can also specify @option{-flto=jobserver} to use GNU make's
job server mode to determine the number of parallel jobs. This
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 946897726d0..353187c6043 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1110,6 +1110,136 @@ cmp_priority (const void *a, const void *b)
return *((const int *)b)-*((const int *)a);
}
+/* Number of CPUs that can be used for parallel LTRANS phase. */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+ /* glibc 2.7 and above provide a macro for this. */
+ return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+ if (cpusetsize == sizeof (cpu_set_t))
+ /* glibc 2.6 and above provide a macro for this. */
+ return CPU_COUNT (cpusetp);
+#endif
+ size_t i;
+ unsigned long ret = 0;
+ STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+ for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+ {
+ unsigned long int mask = cpusetp->__bits[i];
+ if (mask == 0)
+ continue;
+ ret += __builtin_popcountl (mask);
+ }
+ return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads. It would seem
+ this should be related to the number of cpus online. */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+ cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+ cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+ cpuset_size = sizeof (cpu_set_t);
+#endif
+
+ cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+ do
+ {
+ int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+ cpusetp);
+ if (ret == 0)
+ {
+ /* Count only the CPUs this process can use. */
+ nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+ if (nthreads_var == 0)
+ break;
+ get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+ unsigned long i;
+ for (i = cpuset_size * 8; i; i--)
+ if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+ break;
+ cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+ return;
+ }
+ if (ret != EINVAL)
+ break;
+#ifdef CPU_ALLOC_SIZE
+ if (cpuset_size < sizeof (cpu_set_t))
+ cpuset_size = sizeof (cpu_set_t);
+ else
+ cpuset_size = cpuset_size * 2;
+ if (cpuset_size < 8 * sizeof (cpu_set_t))
+ cpusetp
+ = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+ else
+ {
+ /* Avoid fatal if too large memory allocation would be
+ requested, e.g. kernel returning EINVAL all the time. */
+ void *p = realloc (cpusetp, cpuset_size);
+ if (p == NULL)
+ break;
+ cpusetp = (cpu_set_t *) p;
+ }
+#else
+ break;
+#endif
+ }
+ while (1);
+ cpuset_size = 0;
+ nthreads_var = 1;
+ free (cpusetp);
+ cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+ nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
+
+/* FIXME: once using -std=c11, we can use std::thread::hardware_concurrency. */
+
+/* Return true when a jobserver is running and can accept a job. */
+
+static bool
+jobserver_active_p (void)
+{
+ const char *makeflags = getenv ("MAKEFLAGS");
+ if (makeflags == NULL)
+ return false;
+
+ const char *needle = "--jobserver-auth=";
+ const char *n = strstr (makeflags, needle);
+ if (n == NULL)
+ return false;
+
+ int rfd = -1;
+ int wfd = -1;
+
+ return ((sscanf(n, "--jobserver-auth=%d,%d", &rfd, &wfd) == 2)
+ && rfd > 0
+ && wfd > 0
+ && fcntl (rfd, F_GETFD) >= 0
+ && fcntl (wfd, F_GETFD) >= 0);
+}
/* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
@@ -1122,8 +1252,10 @@ run_gcc (unsigned argc, char *argv[])
char *list_option_full = NULL;
const char *linker_output = NULL;
const char *collect_gcc, *collect_gcc_options;
- int parallel = 0;
+ /* Make linking parallel by default. */
+ int parallel = 1;
int jobserver = 0;
+ int auto_parallel = 0;
bool no_partition = false;
struct cl_decoded_option *fdecoded_options = NULL;
struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1247,10 +1379,7 @@ run_gcc (unsigned argc, char *argv[])
case OPT_flto_:
if (strcmp (option->arg, "jobserver") == 0)
- {
- jobserver = 1;
- parallel = 1;
- }
+ jobserver = 1;
else
{
parallel = atoi (option->arg);
@@ -1291,8 +1420,17 @@ run_gcc (unsigned argc, char *argv[])
{
lto_mode = LTO_MODE_LTO;
jobserver = 0;
+ auto_parallel = 0;
parallel = 0;
}
+ else if (!jobserver && parallel)
+ {
+ /* If there's no explicit usage of jobserver and
+ parallel is enabled, then automatically detect
+ jobserver or number of cores. */
+ auto_parallel = 1;
+ jobserver = jobserver_active_p ();
+ }
if (linker_output)
{
@@ -1484,7 +1622,21 @@ cont1:
strcpy (tmp, ltrans_output_file);
if (jobserver)
- obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+ {
+ if (verbose)
+ fprintf (stderr, "Using make jobserver\n");
+ obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+ }
+ else if (auto_parallel)
+ {
+ char buf[256];
+ init_num_threads ();
+ if (verbose)
+ fprintf (stderr, "LTO parallelism level set to %ld\n",
+ nthreads_var);
+ sprintf (buf, "-fwpa=%ld", nthreads_var);
+ obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+ }
else if (parallel > 1)
{
char buf[256];
@@ -1692,7 +1844,8 @@ cont:
i = 3;
if (!jobserver)
{
- snprintf (jobs, 31, "-j%d", parallel);
+ snprintf (jobs, 31, "-j%ld",
+ auto_parallel ? nthreads_var : parallel);
new_argv[i++] = jobs;
}
new_argv[i++] = "all";
--
2.22.0
next prev parent reply other threads:[~2019-07-29 13:35 UTC|newest]
Thread overview: 54+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-23 8:55 [PATCH] Come up with -flto=auto option Martin Liška
2019-07-23 9:29 ` Jan Hubicka
2019-07-23 10:34 ` [PATCH] Deduce automatically number of cores for -flto option Martin Liška
2019-07-24 15:47 ` Jeff Law
2019-07-29 13:37 ` Martin Liška [this message]
2019-07-30 13:47 ` Martin Liška
2019-07-31 1:23 ` Jakub Jelinek
2019-07-31 7:24 ` Martin Liška
2019-07-31 7:40 ` Jakub Jelinek
2019-07-31 7:49 ` Jan Hubicka
2019-07-31 7:50 ` Martin Liška
2019-07-31 7:54 ` Martin Liška
2019-07-31 8:08 ` Jakub Jelinek
2019-07-31 8:21 ` Jan Hubicka
2019-07-31 8:37 ` Martin Liška
2019-07-31 9:12 ` Jakub Jelinek
2019-07-31 9:15 ` Jan Hubicka
2019-07-31 9:17 ` Jakub Jelinek
2019-07-31 9:22 ` Jan Hubicka
2019-07-31 10:02 ` Martin Liška
2019-07-31 12:02 ` Jan Hubicka
2019-07-31 15:42 ` Martin Liška
2019-08-01 13:19 ` Jakub Jelinek
2019-08-01 14:34 ` [PATCH] Properly detect working jobserver in gcc driver Martin Liška
2019-08-01 14:41 ` Jakub Jelinek
2019-08-02 6:30 ` Martin Liška
2019-08-02 7:45 ` Jakub Jelinek
2019-08-02 8:47 ` Martin Liška
2019-08-02 8:50 ` Jakub Jelinek
2019-08-02 9:04 ` Richard Biener
2019-08-02 9:08 ` Jan Hubicka
2019-08-02 9:15 ` Jan Hubicka
2019-08-02 9:19 ` Martin Liška
2019-08-02 9:55 ` Richard Biener
2019-08-05 6:41 ` Martin Liška
2019-08-09 8:14 ` Martin Liška
2019-08-09 8:22 ` Richard Biener
2019-08-09 12:51 ` Martin Liška
2019-08-09 13:56 ` Martin Liška
2019-08-12 15:18 ` Jeff Law
2019-07-23 13:13 ` [PATCH] Come up with -flto=auto option Jeff Law
2019-07-23 13:22 ` Richard Biener
2019-07-23 13:57 ` Michael Matz
2019-07-23 14:00 ` Jeff Law
2019-07-23 14:27 ` Martin Liška
2019-07-23 22:56 ` Jeff Law
2019-07-24 6:59 ` Martin Liška
2019-07-24 15:16 ` Jeff Law
2019-07-23 22:32 ` Allan Sandfeld Jensen
2019-07-24 6:47 ` Martin Liška
2019-07-24 7:12 ` Allan Sandfeld Jensen
2019-07-24 7:15 ` Martin Liška
2019-07-24 11:09 ` Nathan Sidwell
2019-07-24 15:46 ` Jeff Law
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=ae0d9a26-bdf9-c4d3-cf06-f32be2379939@suse.cz \
--to=mliska@suse.cz \
--cc=gcc-patches@gcc.gnu.org \
--cc=hubicka@ucw.cz \
--cc=law@redhat.com \
--cc=matz@suse.de \
--cc=richard.guenther@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).