From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1851) id B96D83856DE7; Wed, 10 Aug 2022 07:18:26 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org B96D83856DE7 Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Martin Liska To: gcc-cvs@gcc.gnu.org Subject: [gcc(refs/users/marxin/heads/jobserver-refactoring)] lto: respect jobserver in parallel WPA streaming X-Act-Checkin: gcc X-Git-Author: Martin Liska X-Git-Refname: refs/users/marxin/heads/jobserver-refactoring X-Git-Oldrev: 83dd262301ec0b5df156e73990bf5854f5db358e X-Git-Newrev: e691c65e2a46c4554111ccc374082bb71c490a6c Message-Id: <20220810071826.B96D83856DE7@sourceware.org> Date: Wed, 10 Aug 2022 07:18:26 +0000 (GMT) X-BeenThere: gcc-cvs@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 10 Aug 2022 07:18:26 -0000 https://gcc.gnu.org/g:e691c65e2a46c4554111ccc374082bb71c490a6c commit e691c65e2a46c4554111ccc374082bb71c490a6c Author: Martin Liska Date: Tue Aug 9 13:59:39 2022 +0200 lto: respect jobserver in parallel WPA streaming PR lto/106328 gcc/ChangeLog: * jobserver.h (struct jobserver_info): Add pipefd. (jobserver_info::connect): New. (jobserver_info::disconnect): Likewise. (jobserver_info::get_token): Likewise. (jobserver_info::return_token): Likewise. gcc/lto/ChangeLog: * lto.cc (wait_for_child): Decrement nruns once a process finishes. (stream_out_partitions): Use job server if active. (do_whole_program_analysis): Likewise. Diff: --- gcc/jobserver.h | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ gcc/lto/lto.cc | 58 +++++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 98 insertions(+), 14 deletions(-) diff --git a/gcc/jobserver.h b/gcc/jobserver.h index d073e5161a1..d232519765a 100644 --- a/gcc/jobserver.h +++ b/gcc/jobserver.h @@ -29,6 +29,18 @@ struct jobserver_info /* Default constructor. */ jobserver_info (); + /* Connect to the server. */ + void connect (); + + /* Disconnect from the server. */ + void disconnect (); + + /* Get token from the server. */ + bool get_token (); + + /* Return token to the server. */ + void return_token (); + /* Error message if there is a problem. */ string error_msg = ""; /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ @@ -39,6 +51,8 @@ struct jobserver_info int wfd = -1; /* Named pipe path. */ string pipe_path = ""; + /* Pipe file descriptor. */ + int pipefd = -1; /* Return true if jobserver is active. */ bool is_active = false; }; @@ -95,4 +109,44 @@ jobserver_info::jobserver_info () error_msg = "jobserver is not available: " + error_msg; } +void +jobserver_info::connect () +{ + if (!pipe_path.empty ()) + pipefd = open (pipe_path.c_str (), O_RDWR); +} + +void +jobserver_info::disconnect () +{ + if (!pipe_path.empty ()) + { + gcc_assert (close (pipefd) == 0); + pipefd = -1; + } +} + +bool +jobserver_info::get_token () +{ + int fd = pipe_path.empty () ? rfd : pipefd; + char c; + unsigned n = read (fd, &c, 1); + if (n != 1) + { + gcc_assert (errno == EAGAIN); + return false; + } + else + return true; +} + +void +jobserver_info::return_token () +{ + int fd = pipe_path.empty () ? wfd : pipefd; + char c = 'G'; + gcc_assert (write (fd, &c, 1) == 1); +} + #endif /* GCC_JOBSERVER_H */ diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc index 31b0c1862f7..868f0a788e9 100644 --- a/gcc/lto/lto.cc +++ b/gcc/lto/lto.cc @@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see . */ +#define INCLUDE_STRING #include "config.h" #include "system.h" #include "coretypes.h" @@ -54,11 +55,17 @@ along with GCC; see the file COPYING3. If not see #include "attribs.h" #include "builtins.h" #include "lto-common.h" +#include "jobserver.h" - -/* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */ +/* Number of parallel tasks to run. */ static int lto_parallelism; +/* Number of active WPA streaming processes. */ +static int nruns = 0; + +/* GNU make's jobserver info. */ +static jobserver_info *jinfo = NULL; + /* Return true when NODE has a clone that is analyzed (i.e. we need to load its body even if the node itself is not needed). */ @@ -205,6 +212,12 @@ wait_for_child () "streaming subprocess was killed by signal"); } while (!WIFEXITED (status) && !WIFSIGNALED (status)); + + --nruns; + + /* Return token to the jobserver if active. */ + if (jinfo != NULL && jinfo->is_active) + jinfo->return_token (); } #endif @@ -228,25 +241,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, bool ARG_UNUSED (last)) { #ifdef HAVE_WORKING_FORK - static int nruns; - if (lto_parallelism <= 1) { stream_out_partitions_1 (temp_filename, blen, min, max); return; } - /* Do not run more than LTO_PARALLELISM streamings - FIXME: we ignore limits on jobserver. */ if (lto_parallelism > 0 && nruns >= lto_parallelism) - { - wait_for_child (); - nruns --; - } + wait_for_child (); + /* If this is not the last parallel partition, execute new streaming process. */ if (!last) { + if (jinfo != NULL && jinfo->is_active) + while (true) + { + if (jinfo->get_token ()) + break; + if (nruns > 0) + wait_for_child (); + else + { + /* There are no free tokens, lets do the job outselves. */ + stream_out_partitions_1 (temp_filename, blen, min, max); + asm_nodes_output = true; + return; + } + } + pid_t cpid = fork (); if (!cpid) @@ -264,10 +287,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, /* Last partition; stream it and wait for all children to die. */ else { - int i; stream_out_partitions_1 (temp_filename, blen, min, max); - for (i = 0; i < nruns; i++) + while (nruns > 0) wait_for_child (); + + if (jinfo != NULL && jinfo->is_active) + jinfo->disconnect (); } asm_nodes_output = true; #else @@ -460,9 +485,14 @@ do_whole_program_analysis (void) lto_parallelism = 1; - /* TODO: jobserver communication is not supported, yet. */ if (!strcmp (flag_wpa, "jobserver")) - lto_parallelism = param_max_lto_streaming_parallelism; + { + jinfo = new jobserver_info (); + if (jinfo->is_active) + jinfo->connect (); + + lto_parallelism = param_max_lto_streaming_parallelism; + } else { lto_parallelism = atoi (flag_wpa);