From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1851) id 85A503954C53; Fri, 9 Dec 2022 09:33:40 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 85A503954C53 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gcc.gnu.org; s=default; t=1670578420; bh=dfWaWyhFwOlUqPiFcpSJOMi1pFHk3Kry7/5rp9MayqU=; h=From:To:Subject:Date:From; b=YA/Hz16r1WwZEAE8faOFF2TK1jTrY5eQ8ExGQWsc8MjaoY74iwku7n0KRZn+xXtdR Nm/bczAqrQ5RegQWQaODTW28GC32ss5eMjnPn19cDAmve6A+MVnbrOXw/llvBzfLVk zIpNsHIneebZmKLzKRlpj/kc3NG3+JPN2t0g4WO8= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Martin Liska To: gcc-cvs@gcc.gnu.org Subject: [gcc(refs/users/marxin/heads/backport-12)] lto: respect jobserver in parallel WPA streaming X-Act-Checkin: gcc X-Git-Author: Martin Liska X-Git-Refname: refs/users/marxin/heads/backport-12 X-Git-Oldrev: 35c4567b37af5625526c47034fdb5efc65af3295 X-Git-Newrev: 900a38c527953ee8d5a1c5a7022c9da8e22e8fa5 Message-Id: <20221209093340.85A503954C53@sourceware.org> Date: Fri, 9 Dec 2022 09:33:40 +0000 (GMT) List-Id: https://gcc.gnu.org/g:900a38c527953ee8d5a1c5a7022c9da8e22e8fa5 commit 900a38c527953ee8d5a1c5a7022c9da8e22e8fa5 Author: Martin Liska Date: Tue Aug 9 13:59:39 2022 +0200 lto: respect jobserver in parallel WPA streaming PR lto/106328 gcc/ChangeLog: * opts-jobserver.h (struct jobserver_info): Add pipefd. (jobserver_info::connect): New. (jobserver_info::disconnect): Likewise. (jobserver_info::get_token): Likewise. (jobserver_info::return_token): Likewise. * opts-common.cc: Implement the new functions. gcc/lto/ChangeLog: * lto.cc (wait_for_child): Decrement nruns once a process finishes. (stream_out_partitions): Use job server if active. (do_whole_program_analysis): Likewise. (cherry picked from commit fed766af32ed6cd371016cc24e931131e19b4eb1) Diff: --- gcc/lto/lto.cc | 58 +++++++++++++++++++++++++++++++++++++++------------- gcc/opts-common.cc | 40 ++++++++++++++++++++++++++++++++++++ gcc/opts-jobserver.h | 14 +++++++++++++ 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc index 31b0c1862f7..c82307f4f7e 100644 --- a/gcc/lto/lto.cc +++ b/gcc/lto/lto.cc @@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see . */ +#define INCLUDE_STRING #include "config.h" #include "system.h" #include "coretypes.h" @@ -54,11 +55,17 @@ along with GCC; see the file COPYING3. If not see #include "attribs.h" #include "builtins.h" #include "lto-common.h" +#include "opts-jobserver.h" - -/* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */ +/* Number of parallel tasks to run. */ static int lto_parallelism; +/* Number of active WPA streaming processes. */ +static int nruns = 0; + +/* GNU make's jobserver info. */ +static jobserver_info *jinfo = NULL; + /* Return true when NODE has a clone that is analyzed (i.e. we need to load its body even if the node itself is not needed). */ @@ -205,6 +212,12 @@ wait_for_child () "streaming subprocess was killed by signal"); } while (!WIFEXITED (status) && !WIFSIGNALED (status)); + + --nruns; + + /* Return token to the jobserver if active. */ + if (jinfo != NULL && jinfo->is_active) + jinfo->return_token (); } #endif @@ -228,25 +241,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, bool ARG_UNUSED (last)) { #ifdef HAVE_WORKING_FORK - static int nruns; - if (lto_parallelism <= 1) { stream_out_partitions_1 (temp_filename, blen, min, max); return; } - /* Do not run more than LTO_PARALLELISM streamings - FIXME: we ignore limits on jobserver. */ if (lto_parallelism > 0 && nruns >= lto_parallelism) - { - wait_for_child (); - nruns --; - } + wait_for_child (); + /* If this is not the last parallel partition, execute new streaming process. */ if (!last) { + if (jinfo != NULL && jinfo->is_active) + while (true) + { + if (jinfo->get_token ()) + break; + if (nruns > 0) + wait_for_child (); + else + { + /* There are no free tokens, lets do the job outselves. */ + stream_out_partitions_1 (temp_filename, blen, min, max); + asm_nodes_output = true; + return; + } + } + pid_t cpid = fork (); if (!cpid) @@ -264,10 +287,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, /* Last partition; stream it and wait for all children to die. */ else { - int i; stream_out_partitions_1 (temp_filename, blen, min, max); - for (i = 0; i < nruns; i++) + while (nruns > 0) wait_for_child (); + + if (jinfo != NULL && jinfo->is_active) + jinfo->disconnect (); } asm_nodes_output = true; #else @@ -460,9 +485,14 @@ do_whole_program_analysis (void) lto_parallelism = 1; - /* TODO: jobserver communication is not supported, yet. */ if (!strcmp (flag_wpa, "jobserver")) - lto_parallelism = param_max_lto_streaming_parallelism; + { + jinfo = new jobserver_info (); + if (jinfo->is_active) + jinfo->connect (); + + lto_parallelism = param_max_lto_streaming_parallelism; + } else { lto_parallelism = atoi (flag_wpa); diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc index 7c07d504696..eed0eb7b799 100644 --- a/gcc/opts-common.cc +++ b/gcc/opts-common.cc @@ -2057,3 +2057,43 @@ jobserver_info::jobserver_info () if (!error_msg.empty ()) error_msg = "jobserver is not available: " + error_msg; } + +void +jobserver_info::connect () +{ + if (!pipe_path.empty ()) + pipefd = open (pipe_path.c_str (), O_RDWR); +} + +void +jobserver_info::disconnect () +{ + if (!pipe_path.empty ()) + { + gcc_assert (close (pipefd) == 0); + pipefd = -1; + } +} + +bool +jobserver_info::get_token () +{ + int fd = pipe_path.empty () ? rfd : pipefd; + char c; + unsigned n = read (fd, &c, 1); + if (n != 1) + { + gcc_assert (errno == EAGAIN); + return false; + } + else + return true; +} + +void +jobserver_info::return_token () +{ + int fd = pipe_path.empty () ? wfd : pipefd; + char c = 'G'; + gcc_assert (write (fd, &c, 1) == 1); +} diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h index 98ea2579962..76c1d9b2882 100644 --- a/gcc/opts-jobserver.h +++ b/gcc/opts-jobserver.h @@ -29,6 +29,18 @@ struct jobserver_info /* Default constructor. */ jobserver_info (); + /* Connect to the server. */ + void connect (); + + /* Disconnect from the server. */ + void disconnect (); + + /* Get token from the server. */ + bool get_token (); + + /* Return token to the server. */ + void return_token (); + /* Error message if there is a problem. */ string error_msg = ""; /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ @@ -39,6 +51,8 @@ struct jobserver_info int wfd = -1; /* Named pipe path. */ string pipe_path = ""; + /* Pipe file descriptor. */ + int pipefd = -1; /* Return true if jobserver is active. */ bool is_active = false; };