From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-ed1-x533.google.com (mail-ed1-x533.google.com [IPv6:2a00:1450:4864:20::533]) by sourceware.org (Postfix) with ESMTPS id 539183858D1E for ; Wed, 10 Aug 2022 10:52:36 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 539183858D1E Received: by mail-ed1-x533.google.com with SMTP id a89so18559675edf.5 for ; Wed, 10 Aug 2022 03:52:36 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:cc:to:subject:message-id:date:from :in-reply-to:references:mime-version:x-gm-message-state:from:to:cc; bh=giY1V40a03kDTtb4TJKD4D/yi/dT2zqSwBa2/HJ11mE=; b=pNSGT6ujmqcsKEtYcM5YclyR55dLhfPJkscbwXDzbNHvOqDoZF+XvgkxJtnMY9Fiig D9El3fY6dQbwmQ1cLX6qOTks37w+oefg9P3EEbM6/KDjERLn5nUyvU1SlyTEZK1/GZMe dFsKmT/WH5uL0uBiAvUs8nHrhEQ3AuyFU6eDEHPbKjG0FTzpzKsxKjbBGjnQ7rZuiQgf 5fkl/K5eNRuxOEe1BmHyg7CBZ/vb4k91PYgsO6scL1UehWYdaVX6wuqfVBByZq8gs9vM QtymhBOb1Xo7KhPrEE0O6AFCVOCQTbcNBmS+ctxjPpfzme2k/EEET5Ihcr0mY5yeZK2H bBoA== X-Gm-Message-State: ACgBeo0ccr1LFqFz57833dYKVl/bhfy0ab4yRgwXe1EmQWWQaGVJD84M kF4jQgbp9T231gD3WpVEfok4O6y6fiK52YzC9pE= X-Google-Smtp-Source: AA6agR52zhogk9LxuDQXuzTB618EpXpQwEdkDb5IUBi59t/MQAeKcZmfFZF58WQIHyJ4zHQtvpBZ68QAD8rrsU2YW5o= X-Received: by 2002:aa7:d447:0:b0:43f:b049:8e00 with SMTP id q7-20020aa7d447000000b0043fb0498e00mr22588586edr.218.1660128754618; Wed, 10 Aug 2022 03:52:34 -0700 (PDT) MIME-Version: 1.0 References: <95d2ec7222d50c8ca1e6535cb8c272d47abbb08d.1660046383.git.mliska@suse.cz> <7ad7af4bb978bffdbc220c754002c667df1b0209.1660046383.git.mliska@suse.cz> In-Reply-To: From: Richard Biener Date: Wed, 10 Aug 2022 12:52:22 +0200 Message-ID: Subject: Re: [PATCH 3/3] lto: respect jobserver in parallel WPA streaming To: =?UTF-8?Q?Martin_Li=C5=A1ka?= Cc: GCC Patches Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable X-Spam-Status: No, score=-8.0 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, FREEMAIL_FROM, GIT_PATCH_0, RCVD_IN_DNSWL_NONE, SPF_HELO_NONE, SPF_PASS, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: gcc-patches@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-patches mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 10 Aug 2022 10:52:38 -0000 On Tue, Aug 9, 2022 at 2:02 PM Martin Li=C5=A1ka wrote: > > Patch can bootstrap on x86_64-linux-gnu and survives regression tests. > > Ready to be installed? > Thanks, > Martin > > PR lto/106328 > > gcc/ChangeLog: > > * jobserver.h (struct jobserver_info): Add pipefd. > (jobserver_info::connect): New. > (jobserver_info::disconnect): Likewise. > (jobserver_info::get_token): Likewise. > (jobserver_info::return_token): Likewise. > > gcc/lto/ChangeLog: > > * lto.cc (wait_for_child): Decrement nruns once a process > finishes. > (stream_out_partitions): Use job server if active. > (do_whole_program_analysis): Likewise. > --- > gcc/jobserver.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ > gcc/lto/lto.cc | 55 +++++++++++++++++++++++++++++++++++++------------ > 2 files changed, 96 insertions(+), 13 deletions(-) > > diff --git a/gcc/jobserver.h b/gcc/jobserver.h > index 856e326ddfc..2a7dc9f4113 100644 > --- a/gcc/jobserver.h > +++ b/gcc/jobserver.h > @@ -31,6 +31,18 @@ struct jobserver_info > /* Default constructor. */ > jobserver_info (); > > + /* Connect to the server. */ > + void connect (); > + > + /* Disconnect from the server. */ > + void disconnect (); > + > + /* Get token from the server. */ > + bool get_token (); > + > + /* Return token to the server. */ > + void return_token (); > + > /* Error message if there is a problem. */ > string error_msg =3D ""; > /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ > @@ -41,6 +53,8 @@ struct jobserver_info > int wfd =3D -1; > /* Named pipe path. */ > string pipe_path =3D ""; > + /* Pipe file descriptor. */ > + int pipefd =3D -1; > /* Return true if jobserver is active. */ > bool is_active =3D false; > }; > @@ -97,4 +111,44 @@ jobserver_info::jobserver_info () > error_msg =3D "jobserver is not available: " + error_msg; > } > > +void > +jobserver_info::connect () > +{ > + if (!pipe_path.empty ()) > + pipefd =3D open (pipe_path.c_str (), O_RDWR); > +} > + > +void > +jobserver_info::disconnect () If it's not a template and the methods not inline won't we get multiple definition errors when this include file is included in multiple places from an executable? opts-common.cc might be a good place to stuff it, and opts.h to declare (if it were not for std::string, so possibl= y jobserver.h is good enough, or opts-jobserver.h?) > +{ > + if (!pipe_path.empty ()) > + { > + gcc_assert (close (pipefd) =3D=3D 0); > + pipefd =3D -1; > + } > +} > + > +bool > +jobserver_info::get_token () > +{ > + int fd =3D pipe_path.empty () ? rfd : pipefd; > + char c; > + unsigned n =3D read (fd, &c, 1); > + if (n !=3D 1) > + { > + gcc_assert (errno =3D=3D EAGAIN); > + return false; > + } > + else > + return true; > +} > + > +void > +jobserver_info::return_token () > +{ > + int fd =3D pipe_path.empty () ? wfd : pipefd; > + char c =3D 'G'; > + gcc_assert (write (fd, &c, 1) =3D=3D 1); > +} > + > #endif /* GCC_JOBSERVER_H */ > diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc > index 31b0c1862f7..56266195ead 100644 > --- a/gcc/lto/lto.cc > +++ b/gcc/lto/lto.cc > @@ -54,11 +54,17 @@ along with GCC; see the file COPYING3. If not see > #include "attribs.h" > #include "builtins.h" > #include "lto-common.h" > - > +#include "jobserver.h" > > /* Number of parallel tasks to run, -1 if we want to use GNU Make jobser= ver. */ > static int lto_parallelism; > > +/* Number of active WPA streaming processes. */ > +static int nruns =3D 0; > + > +/* GNU make's jobserver info. */ > +static jobserver_info *jinfo =3D NULL; > + > /* Return true when NODE has a clone that is analyzed (i.e. we need > to load its body even if the node itself is not needed). */ > > @@ -205,6 +211,12 @@ wait_for_child () > "streaming subprocess was killed by signal"); > } > while (!WIFEXITED (status) && !WIFSIGNALED (status)); > + > + --nruns; > + > + /* Return token to the jobserver if active. */ > + if (jinfo !=3D NULL && jinfo->is_active) > + jinfo->return_token (); > } > #endif > > @@ -228,25 +240,35 @@ stream_out_partitions (char *temp_filename, int ble= n, int min, int max, > bool ARG_UNUSED (last)) > { > #ifdef HAVE_WORKING_FORK > - static int nruns; > - > if (lto_parallelism <=3D 1) > { > stream_out_partitions_1 (temp_filename, blen, min, max); > return; > } > > - /* Do not run more than LTO_PARALLELISM streamings > - FIXME: we ignore limits on jobserver. */ > if (lto_parallelism > 0 && nruns >=3D lto_parallelism) > - { > - wait_for_child (); > - nruns --; > - } > + wait_for_child (); > + > /* If this is not the last parallel partition, execute new > streaming process. */ > if (!last) > { > + if (jinfo !=3D NULL && jinfo->is_active) > + while (true) > + { > + if (jinfo->get_token ()) > + break; > + if (nruns > 0) > + wait_for_child (); > + else > + { > + /* There are no free tokens, lets do the job outselves. = */ > + stream_out_partitions_1 (temp_filename, blen, min, max); > + asm_nodes_output =3D true; > + return; > + } > + } > + > pid_t cpid =3D fork (); > > if (!cpid) > @@ -264,10 +286,12 @@ stream_out_partitions (char *temp_filename, int ble= n, int min, int max, > /* Last partition; stream it and wait for all children to die. */ > else > { > - int i; > stream_out_partitions_1 (temp_filename, blen, min, max); > - for (i =3D 0; i < nruns; i++) > + while (nruns > 0) > wait_for_child (); > + > + if (jinfo !=3D NULL && jinfo->is_active) > + jinfo->disconnect (); > } > asm_nodes_output =3D true; > #else > @@ -460,9 +484,14 @@ do_whole_program_analysis (void) > > lto_parallelism =3D 1; > > - /* TODO: jobserver communication is not supported, yet. */ > if (!strcmp (flag_wpa, "jobserver")) > - lto_parallelism =3D param_max_lto_streaming_parallelism; > + { > + jinfo =3D new jobserver_info (); > + if (jinfo->is_active) > + jinfo->connect (); > + > + lto_parallelism =3D param_max_lto_streaming_parallelism; > + } > else > { > lto_parallelism =3D atoi (flag_wpa); > -- > 2.37.1 >