[libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[Rainer Orth]

[-- Attachment #1: Type: text/plain, Size: 1707 bytes --]

As reported in the PR, with the addition of all those OpenACC tests,
libgomp make check times have skyrocketed since the testsuite is still
run sequentially.

Even on a reasonably fast x86 machine (4 x 2.0 Ghz Xeon E7450) the run
takes 4286 seconds.

On slower sparc boxes (1.2 GHz UltraSPARC-T2) we're at 29406 seconds,
compared to 7825 seconds on the 4.9 branch.

Thus, the libgomp tests massively slow down the whole testsuite run,
being the last part to finish.

Fixing this proved trivial: I managed to almost literally copy the
solution from libstdc++-v3/testsuite/Makefile.am, with a minimal change
to libgomp.exp so the generated libgomp-test-support.exp file is found
in both the sequential and parallel cases.  This isn't an issue in
libstdc++ since all necessary variables are stored in a single
site.exp.

Tested with make (1:13:53.80) and make -j128 (4:58.78) on
i386-pc-solaris2.11 (4 x 2.0 Ghz Xeon E7-4850), no differences in
results according to contrib/dg-cmp-results.sh.

Ok for mainline and gcc-5 branch after some soak time?

	Rainer


2015-05-06  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>

	PR libgomp/66005
	* testsuite/Makefile.am (PWD_COMMAND): New variable.
	(%/site.exp): New target.
	(check_p_numbers0, check_p_numbers1, check_p_numbers2)
	(check_p_numbers3, check_p_numbers4, check_p_numbers5)
	(check_p_numbers6, check_p_numbers, check_p_subdirs)
	(check_DEJAGNU_normal_targets): New variables.
	($(check_DEJAGNU_normal_targets)): New target.
	($(check_DEJAGNU_normal_targets)): New dependency.
	(check-DEJAGNU $(check_DEJAGNU_normal_targets)): New targets.
	* testsuite/Makefile.in: Regenerate.
	* testsuite/lib/libgomp.exp: Also search in .. for
	libgomp-test-support.exp.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: libgomp-testsuite-parallel.patch --]
[-- Type: text/x-patch, Size: 4130 bytes --]

# HG changeset patch
# Parent 56a827256364c7b567b751287defdb0c9eabc666
Support parallel testing in libgomp (PR libgomp/66005)

diff --git a/libgomp/testsuite/Makefile.am b/libgomp/testsuite/Makefile.am
--- a/libgomp/testsuite/Makefile.am
+++ b/libgomp/testsuite/Makefile.am
@@ -12,6 +12,71 @@ EXPECT = $(shell if test -f $(top_buildd
 	     echo $(top_srcdir)/../dejagnu/runtest; else echo runtest; fi)
 RUNTEST = "$(_RUNTEST) $(AM_RUNTESTFLAGS)"
 
+PWD_COMMAND = $${PWDCMD-pwd}
+
+%/site.exp: site.exp
+	-@test -d $* || mkdir $*
+	@srcdir=`cd $(srcdir); ${PWD_COMMAND}`;
+	@objdir=`${PWD_COMMAND}`/$*; \
+	sed -e "s|^set srcdir .*$$|set srcdir $$srcdir|" \
+	    -e "s|^set objdir .*$$|set objdir $$objdir|" \
+	    site.exp > $*/site.exp.tmp
+	@-rm -f $*/site.bak
+	@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
+	@mv $*/site.exp.tmp $*/site.exp
+
+check_p_numbers0:=1 2 3 4 5 6 7 8 9
+check_p_numbers1:=0 $(check_p_numbers0)
+check_p_numbers2:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers1)))
+check_p_numbers3:=$(addprefix 0,$(check_p_numbers1)) $(check_p_numbers2)
+check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers3)))
+check_p_numbers5:=$(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4)
+check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5)))
+check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
+check_p_subdirs=$(wordlist 1,$(if $(GCC_TEST_PARALLEL_SLOTS),$(GCC_TEST_PARALLEL_SLOTS),128),$(check_p_numbers))
+check_DEJAGNU_normal_targets = $(addprefix check-DEJAGNUnormal,$(check_p_subdirs))
+$(check_DEJAGNU_normal_targets): check-DEJAGNUnormal%: normal%/site.exp
+
+check-DEJAGNU $(check_DEJAGNU_normal_targets): check-DEJAGNU%: site.exp
+	$(if $*,@)AR="$(AR)"; export AR; \
+	RANLIB="$(RANLIB)"; export RANLIB; \
+	if [ -z "$*" ] && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
+	  rm -rf normal-parallel || true; \
+	  mkdir normal-parallel; \
+	  $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \
+	  rm -rf normal-parallel || true; \
+	  for idx in $(check_p_subdirs); do \
+	    if [ -d normal$$idx ]; then \
+	      mv -f normal$$idx/libgomp.sum normal$$idx/libgomp.sum.sep; \
+	      mv -f normal$$idx/libgomp.log normal$$idx/libgomp.log.sep; \
+	    fi; \
+	  done; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh \
+	    normal[0-9]*/libgomp.sum.sep > libgomp.sum; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh -L \
+	    normal[0-9]*/libgomp.log.sep > libgomp.log; \
+	  exit 0; \
+	fi; \
+	srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \
+	EXPECT=$(EXPECT); export EXPECT; \
+	runtest=$(RUNTEST); \
+	if [ -z "$$runtest" ]; then runtest=runtest; fi; \
+	tool=libgomp; \
+	if [ -n "$*" ]; then \
+	  if [ -f normal-parallel/finished ]; then rm -rf "$*"; exit 0; fi; \
+	  GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/normal-parallel; \
+	  export GCC_RUNTEST_PARALLELIZE_DIR; \
+	  cd "$*"; \
+	fi; \
+	if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
+	  $$runtest $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) \
+		    $(RUNTESTFLAGS); \
+	  if [ -n "$*" ]; then \
+	    touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished; \
+	  fi; \
+	else \
+	  echo "WARNING: could not find \`runtest'" 1>&2; :;\
+	fi
 
 # Instead of directly in ../testsuite/libgomp-test-support.exp.in, the
 # following variables have to be "routed through" this Makefile, for expansion
diff --git a/libgomp/testsuite/lib/libgomp.exp b/libgomp/testsuite/lib/libgomp.exp
--- a/libgomp/testsuite/lib/libgomp.exp
+++ b/libgomp/testsuite/lib/libgomp.exp
@@ -33,7 +33,8 @@ load_gcc_lib torture-options.exp
 load_gcc_lib fortran-modules.exp
 
 # Try to load a test support file, built during libgomp configuration.
-load_file libgomp-test-support.exp
+# Search in both .. and . to support parallel and sequential testing.
+load_file -1 ../libgomp-test-support.exp libgomp-test-support.exp
 
 # Populate offload_targets_s (offloading targets separated by a space), and
 # offload_targets_s_openacc (the same, but with OpenACC names; OpenACC spells

[-- Attachment #3: Type: text/plain, Size: 143 bytes --]


-- 
-----------------------------------------------------------------------------
Rainer Orth, Center for Biotechnology, Bielefeld University

Re: [libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[Jakub Jelinek]

On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
> As reported in the PR, with the addition of all those OpenACC tests,
> libgomp make check times have skyrocketed since the testsuite is still
> run sequentially.
> 
> Even on a reasonably fast x86 machine (4 x 2.0 Ghz Xeon E7450) the run
> takes 4286 seconds.
> 
> On slower sparc boxes (1.2 GHz UltraSPARC-T2) we're at 29406 seconds,
> compared to 7825 seconds on the 4.9 branch.
> 
> Thus, the libgomp tests massively slow down the whole testsuite run,
> being the last part to finish.
> 
> Fixing this proved trivial: I managed to almost literally copy the
> solution from libstdc++-v3/testsuite/Makefile.am, with a minimal change
> to libgomp.exp so the generated libgomp-test-support.exp file is found
> in both the sequential and parallel cases.  This isn't an issue in
> libstdc++ since all necessary variables are stored in a single
> site.exp.

It is far from trivial though.
The point is that most of the OpenMP tests are parallelized with the
default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
machine a lot, the higher number of hw threads the more.

If we go forward with some parallelization of the tests, we at least should
try to export something like OMP_WAIT_POLICY=passive so that the
oversubscribed machine would at least not spend too much time in spinning.

And perhaps reconsider running all OpenACC threads 3 times, just allow
user to select which offloading target they want to test (host fallback,
the host nonshm hack, PTX, XeonPHI in the future?), and test just that
(that is pretty much how OpenMP offloading testing works).  For tests that
always want to test host fallback, I hope OpenACC offers clauses to force
the host fallback.

	Jakub

Re: [libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[Mike Stump]

On May 7, 2015, at 4:39 AM, Jakub Jelinek <jakub@redhat.com> wrote:
> On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
>> As reported in the PR, with the addition of all those OpenACC tests,
>> libgomp make check times have skyrocketed since the testsuite is still
>> run sequentially.
>> 
>> Even on a reasonably fast x86 machine (4 x 2.0 Ghz Xeon E7450) the run
>> takes 4286 seconds.
>> 
>> On slower sparc boxes (1.2 GHz UltraSPARC-T2) we're at 29406 seconds,
>> compared to 7825 seconds on the 4.9 branch.
>> 
>> Thus, the libgomp tests massively slow down the whole testsuite run,
>> being the last part to finish.
>> 
>> Fixing this proved trivial: I managed to almost literally copy the
>> solution from libstdc++-v3/testsuite/Makefile.am, with a minimal change
>> to libgomp.exp so the generated libgomp-test-support.exp file is found
>> in both the sequential and parallel cases.  This isn't an issue in
>> libstdc++ since all necessary variables are stored in a single
>> site.exp.
> 
> It is far from trivial though.
> The point is that most of the OpenMP tests are parallelized with the
> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
> machine a lot

If OpenMP cannot keep the machine busy, then the test suite should.  A 15x speed up means that OpenMP cannot keep the machine busy.  I’d not expect OpenMP to fill the gap here, so that leave just the test suite.  So, unless someone wants to try their hand at getting some serious time from OpenMP, I think the patch lies on the path of goodness.

Re: [libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[Thomas Schwinge]

[-- Attachment #1: Type: text/plain, Size: 3184 bytes --]

Hi!

On Thu, 7 May 2015 13:39:40 +0200, Jakub Jelinek <jakub@redhat.com> wrote:
> On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
> > As reported in the PR, with the addition of all those OpenACC tests,
> > libgomp make check times have skyrocketed since the testsuite is still
> > run sequentially.

ACK.  And, thanks for looking into that!

> > Fixing this proved trivial: I managed to almost literally copy the
> > solution from libstdc++-v3/testsuite/Makefile.am, with a minimal change
> > to libgomp.exp so the generated libgomp-test-support.exp file is found
> > in both the sequential and parallel cases.  This isn't an issue in
> > libstdc++ since all necessary variables are stored in a single
> > site.exp.
> 
> It is far from trivial though.
> The point is that most of the OpenMP tests are parallelized with the
> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
> machine a lot, the higher number of hw threads the more.

Do you agree that we have two classes of test cases in libgomp: 1) test
cases that don't place a considerably higher load on the machine compared
to "normal" (single-threaded) execution tests, because they're just
testing some functionality that is not expected to actively depend
on/interfere with parallelism.  If needed, and/or if not already done,
such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
num_workers, vector_length clauses, and so on) for low parallelism
levels.  And, 2) test cases that place a considerably higher load on the
machine compared to "normal" (single-threaded) execution tests, because
they're testing some functionality that actively depends on/interferes
with some kind of parallelism.  What about marking such tests specially,
such that DejaGnu will only ever schedule one of them for execution at
the same time?  For example, a new dg-* directive to run them wrapped
through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?

> If we go forward with some parallelization of the tests, we at least should
> try to export something like OMP_WAIT_POLICY=passive so that the
> oversubscribed machine would at least not spend too much time in spinning.

(Will again have the problem that DejaGnu doesn't provide infrastructure
to communicate environment variables to boards in remote testing.)

> And perhaps reconsider running all OpenACC threads 3 times, just allow
> user to select which offloading target they want to test (host fallback,
> the host nonshm hack, PTX, XeonPHI in the future?), and test just that
> (that is pretty much how OpenMP offloading testing works).

My rationale is: if you configure GCC to support a set of offloading
devices (more than one), you'll also want to get the test coverage that
indeed all these work as expected.  (It currently doesn't matter, but...)
that's something I'd like to see improved in the libgomp OpenMP
offloading testing (once it supports more than one architecture for
offloading).

> For tests that
> always want to test host fallback, I hope OpenACC offers clauses to force
> the host fallback.

Yes.


Grüße,
 Thomas

[-- Attachment #2: Type: application/pgp-signature, Size: 472 bytes --]

Re: [libgomp, testsuite] Support parallel testing in libgomp (PR libgomp/66005)

[Martin Liška]

[-- Attachment #1: Type: text/plain, Size: 3856 bytes --]

On 05/08/2015 10:40 AM, Thomas Schwinge wrote:
> Hi!
> 
> On Thu, 7 May 2015 13:39:40 +0200, Jakub Jelinek <jakub@redhat.com> wrote:
>> On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
>>> As reported in the PR, with the addition of all those OpenACC tests,
>>> libgomp make check times have skyrocketed since the testsuite is still
>>> run sequentially.
> 
> ACK.  And, thanks for looking into that!
> 
>>> Fixing this proved trivial: I managed to almost literally copy the
>>> solution from libstdc++-v3/testsuite/Makefile.am, with a minimal change
>>> to libgomp.exp so the generated libgomp-test-support.exp file is found
>>> in both the sequential and parallel cases.  This isn't an issue in
>>> libstdc++ since all necessary variables are stored in a single
>>> site.exp.
>>
>> It is far from trivial though.
>> The point is that most of the OpenMP tests are parallelized with the
>> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
>> machine a lot, the higher number of hw threads the more.
> 
> Do you agree that we have two classes of test cases in libgomp: 1) test
> cases that don't place a considerably higher load on the machine compared
> to "normal" (single-threaded) execution tests, because they're just
> testing some functionality that is not expected to actively depend
> on/interfere with parallelism.  If needed, and/or if not already done,
> such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
> num_workers, vector_length clauses, and so on) for low parallelism
> levels.  And, 2) test cases that place a considerably higher load on the
> machine compared to "normal" (single-threaded) execution tests, because
> they're testing some functionality that actively depends on/interferes
> with some kind of parallelism.  What about marking such tests specially,
> such that DejaGnu will only ever schedule one of them for execution at
> the same time?  For example, a new dg-* directive to run them wrapped
> through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?

Looks the thread got stuck. Anyway I've just noticed how slow libgomp.exp tests
are on a recent Intel Machine with 160 HT cores. I'm attaching graph with CPU
utilization and 'ps ax | grep expect' log file that shows which tests are running.
Roughly, after 10 minutes I see drop in utilization and then libgomp.exp is running
mainly serially.

So I believe splitting tests in libgomp.exp to serial and parallel would make sense.
Another another idea is to overwrite OMP_NUM_THREADS to a reasonable number which
will enable also parallel execution of parallel tests?

Thanks,
Martin

> 
>> If we go forward with some parallelization of the tests, we at least should
>> try to export something like OMP_WAIT_POLICY=passive so that the
>> oversubscribed machine would at least not spend too much time in spinning.
> 
> (Will again have the problem that DejaGnu doesn't provide infrastructure
> to communicate environment variables to boards in remote testing.)
> 
>> And perhaps reconsider running all OpenACC threads 3 times, just allow
>> user to select which offloading target they want to test (host fallback,
>> the host nonshm hack, PTX, XeonPHI in the future?), and test just that
>> (that is pretty much how OpenMP offloading testing works).
> 
> My rationale is: if you configure GCC to support a set of offloading
> devices (more than one), you'll also want to get the test coverage that
> indeed all these work as expected.  (It currently doesn't matter, but...)
> that's something I'd like to see improved in the libgomp OpenMP
> offloading testing (once it supports more than one architecture for
> offloading).
> 
>> For tests that
>> always want to test host fallback, I hope OpenACC offers clauses to force
>> the host fallback.
> 
> Yes.
> 
> 
> Grüße,
>  Thomas
> 


[-- Attachment #2: make-check.tar.bz2 --]
[-- Type: application/x-bzip, Size: 87635 bytes --]

Support parallel testing in libgomp, part I [PR66005]

[Thomas Schwinge]

[-- Attachment #1: Type: text/plain, Size: 5410 bytes --]

Hi!

[Putting Bernhard, Honza, Segher in CC, as they are eager to test this,
based on recent comments on IRC.]  ;-P


First, establish the parallel testing infrastructure -- while still
hard-coding the number of parallel slots to one.

On 2015-05-08T10:40:02+0200, I wrote:
> On Thu, 7 May 2015 13:39:40 +0200, Jakub Jelinek <jakub@redhat.com> wrote:
>> On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
>> > As reported in the PR, with the addition of all those OpenACC tests,
>> > libgomp make check times have skyrocketed since the testsuite is still
>> > run sequentially.
>
> ACK.  And, thanks for looking into that!
>
>> > Fixing this proved trivial: I managed to almost literally copy the
>> > solution from libstdc++-v3/testsuite/Makefile.am

So I recently had re-created this patch independently, before remembering
that Rainer had -- just eight years ago... ;-) -- already submitted this.
Fortunately, we've done it in almost the exact same way, just that a few
libgomp testsuite infrastructure as well as libstdc++ parallel testing
infrastructure changes (which is where these changes have been copied
from) have since gone in, which I've adjusted for.  Additionally, use
'check_DEJAGNU_libgomp_targets' instead of 'check_DEJAGNU_normal_targets'
etc. (where "normal" is a libstdc++ detail), and regarding:

>> > with a minimal change
>> > to libgomp.exp so the generated libgomp-test-support.exp file is found
>> > in both the sequential and parallel cases.  This isn't an issue in
>> > libstdc++ since all necessary variables are stored in a single
>> > site.exp.

... in 'libgomp/testsuite/lib/libgomp.exp', I've changed:

    -load_file libgomp-test-support.exp
    +# Search in both .. and . to support parallel and sequential testing.
    +load_file -1 ../libgomp-test-support.exp libgomp-test-support.exp

... into the more explicit:

    -load_file libgomp-test-support.exp
    +# Search in '..' vs. '.' to support parallel vs. sequential testing.
    +if [info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
    +    load_file ../libgomp-test-support.exp
    +} else {
    +    load_file libgomp-test-support.exp
    +}

And, for now, I hard-code the number of parallel slots to one.  This
means that libgomp for 'make -j' now does use the parallel testing code
paths, but is restricted to just one slot.  That is, no actual change in
behavior, other than that 'libgomp.sum' then is filtered through
'contrib/dg-extract-results.sh'.

OK to push the attached
"Support parallel testing in libgomp, part I [PR66005]"?


Grüße
 Thomas


>> It is far from trivial though.
>> The point is that most of the OpenMP tests are parallelized with the
>> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
>> machine a lot, the higher number of hw threads the more.
>
> Do you agree that we have two classes of test cases in libgomp: 1) test
> cases that don't place a considerably higher load on the machine compared
> to "normal" (single-threaded) execution tests, because they're just
> testing some functionality that is not expected to actively depend
> on/interfere with parallelism.  If needed, and/or if not already done,
> such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
> num_workers, vector_length clauses, and so on) for low parallelism
> levels.  And, 2) test cases that place a considerably higher load on the
> machine compared to "normal" (single-threaded) execution tests, because
> they're testing some functionality that actively depends on/interferes
> with some kind of parallelism.  What about marking such tests specially,
> such that DejaGnu will only ever schedule one of them for execution at
> the same time?  For example, a new dg-* directive to run them wrapped
> through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?
>
>> If we go forward with some parallelization of the tests, we at least should
>> try to export something like OMP_WAIT_POLICY=passive so that the
>> oversubscribed machine would at least not spend too much time in spinning.
>
> (Will again have the problem that DejaGnu doesn't provide infrastructure
> to communicate environment variables to boards in remote testing.)
>
>> And perhaps reconsider running all OpenACC threads 3 times, just allow
>> user to select which offloading target they want to test (host fallback,
>> the host nonshm hack, PTX, XeonPHI in the future?), and test just that
>> (that is pretty much how OpenMP offloading testing works).
>
> My rationale is: if you configure GCC to support a set of offloading
> devices (more than one), you'll also want to get the test coverage that
> indeed all these work as expected.  (It currently doesn't matter, but...)
> that's something I'd like to see improved in the libgomp OpenMP
> offloading testing (once it supports more than one architecture for
> offloading).
>
>> For tests that
>> always want to test host fallback, I hope OpenACC offers clauses to force
>> the host fallback.
>
> Yes.
>
>
> Grüße,
>  Thomas


-----------------
Siemens Electronic Design Automation GmbH; Anschrift: Arnulfstraße 201, 80634 München; Gesellschaft mit beschränkter Haftung; Geschäftsführer: Thomas Heurung, Frank Thürauf; Sitz der Gesellschaft: München; Registergericht München, HRB 106955

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Support-parallel-testing-in-libgomp-part-I-PR66005.patch --]
[-- Type: text/x-diff, Size: 10736 bytes --]

From ec0b945d2a24524b987e2e0324d3bd86d9d554bd Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@CeBiTec.Uni-Bielefeld.DE>
Date: Thu, 7 May 2015 13:26:57 +0200
Subject: [PATCH] Support parallel testing in libgomp, part I [PR66005]

..., while still hard-coding the number of parallel slots to one.

	PR libgomp/66005
	libgomp/
	* testsuite/Makefile.am (PWD_COMMAND): New variable.
	(%/site.exp): New target.
	(check_p_numbers0, check_p_numbers1, check_p_numbers2)
	(check_p_numbers3, check_p_numbers4, check_p_numbers5)
	(check_p_numbers6, check_p_numbers, gcc_test_parallel_slots)
	(check_p_subdirs)
	(check_DEJAGNU_libgomp_targets): New variables.
	($(check_DEJAGNU_libgomp_targets)): New target.
	($(check_DEJAGNU_libgomp_targets)): New dependency.
	(check-DEJAGNU $(check_DEJAGNU_libgomp_targets)): New targets.
	* testsuite/Makefile.in: Regenerate.
	* testsuite/lib/libgomp.exp: For parallel testing,
	'load_file ../libgomp-test-support.exp'.

Co-authored-by: Thomas Schwinge <thomas@codesourcery.com>
---
 libgomp/testsuite/Makefile.am     | 78 ++++++++++++++++++++++++++-----
 libgomp/testsuite/Makefile.in     | 76 +++++++++++++++++++++++++-----
 libgomp/testsuite/lib/libgomp.exp |  7 ++-
 3 files changed, 138 insertions(+), 23 deletions(-)

diff --git a/libgomp/testsuite/Makefile.am b/libgomp/testsuite/Makefile.am
index 655a413c160..eef02f4a8c2 100644
--- a/libgomp/testsuite/Makefile.am
+++ b/libgomp/testsuite/Makefile.am
@@ -12,6 +12,8 @@ _RUNTEST = $(shell if test -f $(top_srcdir)/../dejagnu/runtest; then \
 	     echo $(top_srcdir)/../dejagnu/runtest; else echo runtest; fi)
 RUNTESTDEFAULTFLAGS = --tool $$tool --srcdir $$srcdir
 
+PWD_COMMAND = $${PWDCMD-pwd}
+
 EXTRA_DEJAGNU_SITE_CONFIG = libgomp-site-extra.exp
 
 # Instead of directly in ../testsuite/libgomp-test-support.exp.in, the
@@ -25,17 +27,6 @@ libgomp-test-support.exp: libgomp-test-support.pt.exp Makefile
 	  'set offload_additional_lib_paths "$(offload_additional_lib_paths)"'
 	mv $@.tmp $@
 
-check-DEJAGNU: site.exp
-	srcdir='$(srcdir)'; export srcdir; \
-	EXPECT=$(EXPECT); export EXPECT; \
-	if $(SHELL) -c "$(_RUNTEST) --version" > /dev/null 2>&1; then \
-	  exit_status=0; l='$(PACKAGE)'; for tool in $$l; do \
-	    if $(_RUNTEST) $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) $(RUNTESTFLAGS); \
-	    then :; else exit_status=1; fi; \
-	  done; \
-	else echo "WARNING: could not find '$(_RUNTEST)'" 1>&2; :;\
-	fi; \
-	exit $$exit_status
 site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG)
 	@echo 'Making a new site.exp file ...'
 	@echo '## these variables are automatically generated by make ##' >site.tmp
@@ -63,6 +54,71 @@ site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG)
 	@test ! -f site.exp || mv site.exp site.bak
 	@mv site.tmp site.exp
 
+%/site.exp: site.exp
+	-@test -d $* || mkdir $*
+	@srcdir=`cd $(srcdir); ${PWD_COMMAND}`;
+	@objdir=`${PWD_COMMAND}`/$*; \
+	sed -e "s|^set srcdir .*$$|set srcdir $$srcdir|" \
+	    -e "s|^set objdir .*$$|set objdir $$objdir|" \
+	    site.exp > $*/site.exp.tmp
+	@-rm -f $*/site.bak
+	@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
+	@mv $*/site.exp.tmp $*/site.exp
+
+check_p_numbers0:=1 2 3 4 5 6 7 8 9
+check_p_numbers1:=0 $(check_p_numbers0)
+check_p_numbers2:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers1)))
+check_p_numbers3:=$(addprefix 0,$(check_p_numbers1)) $(check_p_numbers2)
+check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers3)))
+check_p_numbers5:=$(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4)
+check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5)))
+check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
+gcc_test_parallel_slots:=1
+check_p_subdirs=$(wordlist 1,$(gcc_test_parallel_slots),$(check_p_numbers))
+check_DEJAGNU_libgomp_targets = $(addprefix check-DEJAGNUlibgomp,$(check_p_subdirs))
+$(check_DEJAGNU_libgomp_targets): check-DEJAGNUlibgomp%: libgomp%/site.exp
+
+check-DEJAGNU $(check_DEJAGNU_libgomp_targets): check-DEJAGNU%: site.exp
+	$(if $*,@)AR="$(AR)"; export AR; \
+	RANLIB="$(RANLIB)"; export RANLIB; \
+	if [ -z "$*" ] && [ -n "$(filter -j%, $(MFLAGS))" ]; then \
+	  rm -rf libgomp-parallel || true; \
+	  mkdir libgomp-parallel; \
+	  $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_libgomp_targets); \
+	  rm -rf libgomp-parallel || true; \
+	  for idx in $(check_p_subdirs); do \
+	    if [ -d libgomp$$idx ]; then \
+	      mv -f libgomp$$idx/libgomp.sum libgomp$$idx/libgomp.sum.sep; \
+	      mv -f libgomp$$idx/libgomp.log libgomp$$idx/libgomp.log.sep; \
+	    fi; \
+	  done; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh \
+	    libgomp[0-9]*/libgomp.sum.sep > libgomp.sum; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh -L \
+	    libgomp[0-9]*/libgomp.log.sep > libgomp.log; \
+	  exit 0; \
+	fi; \
+	srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \
+	EXPECT=$(EXPECT); export EXPECT; \
+	runtest=$(_RUNTEST); \
+	if [ -z "$$runtest" ]; then runtest=runtest; fi; \
+	tool=libgomp; \
+	if [ -n "$*" ]; then \
+	  if [ -f libgomp-parallel/finished ]; then rm -rf "$*"; exit 0; fi; \
+	  GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/libgomp-parallel; \
+	  export GCC_RUNTEST_PARALLELIZE_DIR; \
+	  cd "$*"; \
+	fi; \
+	if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
+	  $$runtest $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) \
+		    $(RUNTESTFLAGS); \
+	  if [ -n "$*" ]; then \
+	    touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished; \
+	  fi; \
+	else \
+	  echo "WARNING: could not find \`runtest'" 1>&2; :;\
+	fi
+
 distclean-DEJAGNU:
 	-rm -f site.exp site.bak
 	-l='$(PACKAGE)'; for tool in $$l; do \
diff --git a/libgomp/testsuite/Makefile.in b/libgomp/testsuite/Makefile.in
index 7a88f0fe5c6..3b5903daed9 100644
--- a/libgomp/testsuite/Makefile.in
+++ b/libgomp/testsuite/Makefile.in
@@ -299,7 +299,19 @@ _RUNTEST = $(shell if test -f $(top_srcdir)/../dejagnu/runtest; then \
 	     echo $(top_srcdir)/../dejagnu/runtest; else echo runtest; fi)
 
 RUNTESTDEFAULTFLAGS = --tool $$tool --srcdir $$srcdir
+PWD_COMMAND = $${PWDCMD-pwd}
 EXTRA_DEJAGNU_SITE_CONFIG = libgomp-site-extra.exp
+check_p_numbers0 := 1 2 3 4 5 6 7 8 9
+check_p_numbers1 := 0 $(check_p_numbers0)
+check_p_numbers2 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers1)))
+check_p_numbers3 := $(addprefix 0,$(check_p_numbers1)) $(check_p_numbers2)
+check_p_numbers4 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers3)))
+check_p_numbers5 := $(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4)
+check_p_numbers6 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5)))
+check_p_numbers := $(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
+gcc_test_parallel_slots := 1
+check_p_subdirs = $(wordlist 1,$(gcc_test_parallel_slots),$(check_p_numbers))
+check_DEJAGNU_libgomp_targets = $(addprefix check-DEJAGNUlibgomp,$(check_p_subdirs))
 all: all-am
 
 .SUFFIXES:
@@ -474,17 +486,6 @@ libgomp-test-support.exp: libgomp-test-support.pt.exp Makefile
 	  'set offload_additional_lib_paths "$(offload_additional_lib_paths)"'
 	mv $@.tmp $@
 
-check-DEJAGNU: site.exp
-	srcdir='$(srcdir)'; export srcdir; \
-	EXPECT=$(EXPECT); export EXPECT; \
-	if $(SHELL) -c "$(_RUNTEST) --version" > /dev/null 2>&1; then \
-	  exit_status=0; l='$(PACKAGE)'; for tool in $$l; do \
-	    if $(_RUNTEST) $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) $(RUNTESTFLAGS); \
-	    then :; else exit_status=1; fi; \
-	  done; \
-	else echo "WARNING: could not find '$(_RUNTEST)'" 1>&2; :;\
-	fi; \
-	exit $$exit_status
 site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG)
 	@echo 'Making a new site.exp file ...'
 	@echo '## these variables are automatically generated by make ##' >site.tmp
@@ -512,6 +513,59 @@ site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG)
 	@test ! -f site.exp || mv site.exp site.bak
 	@mv site.tmp site.exp
 
+%/site.exp: site.exp
+	-@test -d $* || mkdir $*
+	@srcdir=`cd $(srcdir); ${PWD_COMMAND}`;
+	@objdir=`${PWD_COMMAND}`/$*; \
+	sed -e "s|^set srcdir .*$$|set srcdir $$srcdir|" \
+	    -e "s|^set objdir .*$$|set objdir $$objdir|" \
+	    site.exp > $*/site.exp.tmp
+	@-rm -f $*/site.bak
+	@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
+	@mv $*/site.exp.tmp $*/site.exp
+$(check_DEJAGNU_libgomp_targets): check-DEJAGNUlibgomp%: libgomp%/site.exp
+
+check-DEJAGNU $(check_DEJAGNU_libgomp_targets): check-DEJAGNU%: site.exp
+	$(if $*,@)AR="$(AR)"; export AR; \
+	RANLIB="$(RANLIB)"; export RANLIB; \
+	if [ -z "$*" ] && [ -n "$(filter -j%, $(MFLAGS))" ]; then \
+	  rm -rf libgomp-parallel || true; \
+	  mkdir libgomp-parallel; \
+	  $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_libgomp_targets); \
+	  rm -rf libgomp-parallel || true; \
+	  for idx in $(check_p_subdirs); do \
+	    if [ -d libgomp$$idx ]; then \
+	      mv -f libgomp$$idx/libgomp.sum libgomp$$idx/libgomp.sum.sep; \
+	      mv -f libgomp$$idx/libgomp.log libgomp$$idx/libgomp.log.sep; \
+	    fi; \
+	  done; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh \
+	    libgomp[0-9]*/libgomp.sum.sep > libgomp.sum; \
+	  $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh -L \
+	    libgomp[0-9]*/libgomp.log.sep > libgomp.log; \
+	  exit 0; \
+	fi; \
+	srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \
+	EXPECT=$(EXPECT); export EXPECT; \
+	runtest=$(_RUNTEST); \
+	if [ -z "$$runtest" ]; then runtest=runtest; fi; \
+	tool=libgomp; \
+	if [ -n "$*" ]; then \
+	  if [ -f libgomp-parallel/finished ]; then rm -rf "$*"; exit 0; fi; \
+	  GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/libgomp-parallel; \
+	  export GCC_RUNTEST_PARALLELIZE_DIR; \
+	  cd "$*"; \
+	fi; \
+	if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
+	  $$runtest $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) \
+		    $(RUNTESTFLAGS); \
+	  if [ -n "$*" ]; then \
+	    touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished; \
+	  fi; \
+	else \
+	  echo "WARNING: could not find \`runtest'" 1>&2; :;\
+	fi
+
 distclean-DEJAGNU:
 	-rm -f site.exp site.bak
 	-l='$(PACKAGE)'; for tool in $$l; do \
diff --git a/libgomp/testsuite/lib/libgomp.exp b/libgomp/testsuite/lib/libgomp.exp
index e12236e9083..4ea6f26c674 100644
--- a/libgomp/testsuite/lib/libgomp.exp
+++ b/libgomp/testsuite/lib/libgomp.exp
@@ -42,7 +42,12 @@ load_gcc_lib torture-options.exp
 load_gcc_lib fortran-modules.exp
 
 # Try to load a test support file, built during libgomp configuration.
-load_file libgomp-test-support.exp
+# Search in '..' vs. '.' to support parallel vs. sequential testing.
+if [info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
+    load_file ../libgomp-test-support.exp
+} else {
+    load_file libgomp-test-support.exp
+}
 
 set dg-do-what-default run
 
-- 
2.34.1

Support parallel testing in libgomp, part II [PR66005]

[Thomas Schwinge]

[-- Attachment #1: Type: text/plain, Size: 4601 bytes --]

Hi!

On 2023-05-05T10:55:41+0200, I wrote:
> [Putting Bernhard, Honza, Segher in CC, as they are eager to test this,
> based on recent comments on IRC.]  ;-P


> First, establish the parallel testing infrastructure -- while still
> hard-coding the number of parallel slots to one.

> OK to push the attached
> "Support parallel testing in libgomp, part I [PR66005]"?

On top of that, second, enable parallel testing.

> On 2015-05-08T10:40:02+0200, I wrote:
>> On Thu, 7 May 2015 13:39:40 +0200, Jakub Jelinek <jakub@redhat.com> wrote:
>>> On Thu, May 07, 2015 at 01:26:57PM +0200, Rainer Orth wrote:
>>> It is far from trivial though.
>>> The point is that most of the OpenMP tests are parallelized with the
>>> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
>>> machine a lot, the higher number of hw threads the more.
>>
>> Do you agree that we have two classes of test cases in libgomp: 1) test
>> cases that don't place a considerably higher load on the machine compared
>> to "normal" (single-threaded) execution tests, because they're just
>> testing some functionality that is not expected to actively depend
>> on/interfere with parallelism.  If needed, and/or if not already done,
>> such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
>> num_workers, vector_length clauses, and so on) for low parallelism
>> levels.  And, 2) test cases that place a considerably higher load on the
>> machine compared to "normal" (single-threaded) execution tests, because
>> they're testing some functionality that actively depends on/interferes
>> with some kind of parallelism.  What about marking such tests specially,
>> such that DejaGnu will only ever schedule one of them for execution at
>> the same time?  For example, a new dg-* directive to run them wrapped
>> through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?

Bernhard on GCC IRC also suggested:

    2023-04-25T19:32:57+0200: <blindvt> tschwinge, we could have a dg-do
      run-serial for tests that themselves occupy more/all CPUs. Or maybe
      it would be better to look at the testcases to see if they do that
      and put them in a "serial queue". I did not look, but there
      certainly are at least some tests which we could run in parallel.

So while there certainly is potential for using more parallelism in
execution testing, I've however now implemented what I'd described in
<https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66005#c2>:

| [...] parallelize *all* compilation, while just allowing for *one*
| execution test job slot.  That will require some GCC DejaGnu test
| harness hackery which I've [now] gotten to look into.  That is, enable
| the usual GCC/DejaGnu parallel testing, but also have some kind of
| mutex for the execution test invocation.  This has to play nicely with
| DejaGnu timeout handling, etc.

OK to push the attached
"Support parallel testing in libgomp, part II [PR66005]"?
See the Git commit log for further discussion.


Grüße
 Thomas


>>> If we go forward with some parallelization of the tests, we at least should
>>> try to export something like OMP_WAIT_POLICY=passive so that the
>>> oversubscribed machine would at least not spend too much time in spinning.
>>
>> (Will again have the problem that DejaGnu doesn't provide infrastructure
>> to communicate environment variables to boards in remote testing.)
>>
>>> And perhaps reconsider running all OpenACC threads 3 times, just allow
>>> user to select which offloading target they want to test (host fallback,
>>> the host nonshm hack, PTX, XeonPHI in the future?), and test just that
>>> (that is pretty much how OpenMP offloading testing works).
>>
>> My rationale is: if you configure GCC to support a set of offloading
>> devices (more than one), you'll also want to get the test coverage that
>> indeed all these work as expected.  (It currently doesn't matter, but...)
>> that's something I'd like to see improved in the libgomp OpenMP
>> offloading testing (once it supports more than one architecture for
>> offloading).
>>
>>> For tests that
>>> always want to test host fallback, I hope OpenACC offers clauses to force
>>> the host fallback.
>>
>> Yes.
>>
>>
>> Grüße,
>>  Thomas


-----------------
Siemens Electronic Design Automation GmbH; Anschrift: Arnulfstraße 201, 80634 München; Gesellschaft mit beschränkter Haftung; Geschäftsführer: Thomas Heurung, Frank Thürauf; Sitz der Gesellschaft: München; Registergericht München, HRB 106955

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Support-parallel-testing-in-libgomp-part-II-PR66005.patch --]
[-- Type: text/x-diff, Size: 15502 bytes --]

From f1ae4a3675ad1147aaa88405be9f000ceed703bc Mon Sep 17 00:00:00 2001
From: Thomas Schwinge <thomas@codesourcery.com>
Date: Tue, 25 Apr 2023 23:53:12 +0200
Subject: [PATCH] Support parallel testing in libgomp, part II [PR66005]

..., and enable if 'flock' is available for serializing execution testing.

Regarding the default of 19 parallel slots, this turned out to be a local
minimum for wall time when testing this on:

    $ uname -srvi
    Linux 4.2.0-42-generic #49~14.04.1-Ubuntu SMP Wed Jun 29 20:22:11 UTC 2016 x86_64
    $ grep '^model name' < /proc/cpuinfo | uniq -c
         32 model name      : Intel(R) Xeon(R) CPU E5-2640 v3 @ 2.60GHz

... in two configurations: case (a) standard configuration, no offloading
configured, case (b) offloading for GCN and nvptx configured but no devices
available.  For both cases, default plus '-m32' variant.

    $ \time make check-target-libgomp RUNTESTFLAGS="--target_board=unix\{,-m32\}"

Case (a), baseline:

    6432.23user 332.38system 47:32.28elapsed 237%CPU (0avgtext+0avgdata 505044maxresident)k
    6382.43user 319.21system 47:06.04elapsed 237%CPU (0avgtext+0avgdata 505172maxresident)k

This is what people have been complaining about, rightly so, in
<https://gcc.gnu.org/PR66005> "libgomp make check time is excessive" and
elsewhere.

Case (a), parallelized:

    -j12 GCC_TEST_PARALLEL_SLOTS=10
    3088.49user 267.74system 6:43.82elapsed 831%CPU (0avgtext+0avgdata 505188maxresident)k
    -j15 GCC_TEST_PARALLEL_SLOTS=15
    3308.08user 294.79system 5:56.04elapsed 1011%CPU (0avgtext+0avgdata 505360maxresident)k
    -j17 GCC_TEST_PARALLEL_SLOTS=17
    3539.93user 298.99system 5:27.86elapsed 1170%CPU (0avgtext+0avgdata 505112maxresident)k
    -j18 GCC_TEST_PARALLEL_SLOTS=18
    3697.50user 317.18system 5:14.63elapsed 1275%CPU (0avgtext+0avgdata 505360maxresident)k
    -j19 GCC_TEST_PARALLEL_SLOTS=19
    3765.94user 324.27system 5:13.22elapsed 1305%CPU (0avgtext+0avgdata 505128maxresident)k
    -j20 GCC_TEST_PARALLEL_SLOTS=20
    3684.66user 312.32system 5:15.26elapsed 1267%CPU (0avgtext+0avgdata 505100maxresident)k
    -j23 GCC_TEST_PARALLEL_SLOTS=23
    4040.59user 347.10system 5:29.12elapsed 1333%CPU (0avgtext+0avgdata 505200maxresident)k
    -j26 GCC_TEST_PARALLEL_SLOTS=26
    3973.24user 377.96system 5:24.70elapsed 1340%CPU (0avgtext+0avgdata 505160maxresident)k
    -j32 GCC_TEST_PARALLEL_SLOTS=32
    4004.42user 346.10system 5:16.11elapsed 1376%CPU (0avgtext+0avgdata 505160maxresident)k

Yay!

Case (b), baseline; 2+ h:

    7227.58user 700.54system 2:14:33elapsed 98%CPU (0avgtext+0avgdata 994264maxresident)k

Case (b), parallelized:

    -j12 GCC_TEST_PARALLEL_SLOTS=10
    7377.46user 777.52system 16:06.63elapsed 843%CPU (0avgtext+0avgdata 994344maxresident)k
    -j15 GCC_TEST_PARALLEL_SLOTS=15
    8019.18user 721.42system 12:13.56elapsed 1191%CPU (0avgtext+0avgdata 994228maxresident)k
    -j17 GCC_TEST_PARALLEL_SLOTS=17
    8530.11user 716.95system 10:45.92elapsed 1431%CPU (0avgtext+0avgdata 994176maxresident)k
    -j18 GCC_TEST_PARALLEL_SLOTS=18
    8776.79user 645.89system 10:27.20elapsed 1502%CPU (0avgtext+0avgdata 994248maxresident)k
    -j19 GCC_TEST_PARALLEL_SLOTS=19
    9332.37user 641.76system 10:15.09elapsed 1621%CPU (0avgtext+0avgdata 994260maxresident)k
    -j20 GCC_TEST_PARALLEL_SLOTS=20
    9609.54user 789.88system 10:26.94elapsed 1658%CPU (0avgtext+0avgdata 994284maxresident)k
    -j23 GCC_TEST_PARALLEL_SLOTS=23
    10362.40user 911.14system 10:44.47elapsed 1749%CPU (0avgtext+0avgdata 994208maxresident)k
    -j26 GCC_TEST_PARALLEL_SLOTS=26
    11159.44user 850.99system 11:09.25elapsed 1794%CPU (0avgtext+0avgdata 994256maxresident)k
    -j32 GCC_TEST_PARALLEL_SLOTS=32
    11453.50user 939.52system 11:00.38elapsed 1876%CPU (0avgtext+0avgdata 994240maxresident)k

On my Dell Precision 7530 laptop:

    $ uname -srvi
    Linux 5.15.0-71-generic #78-Ubuntu SMP Tue Apr 18 09:00:29 UTC 2023 x86_64
    $ grep '^model name' < /proc/cpuinfo | uniq -c
         12 model name      : Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz
    $ nvidia-smi -L
    GPU 0: Quadro P1000 (UUID: GPU-e043973b-b52a-d02b-c066-a8fdbf64e8ea)

... in two configurations: case (c) standard configuration, no offloading
configured, case (d) offloading for nvptx configured and device available.
For both cases, only default variant, no '-m32'.

    $ \time make check-target-libgomp

Case (c), baseline; roughly half of case (a) (just one variant):

    1180.98user 110.80system 19:36.40elapsed 109%CPU (0avgtext+0avgdata 505148maxresident)k
    1133.22user 111.08system 19:35.75elapsed 105%CPU (0avgtext+0avgdata 505212maxresident)k

Case (c), parallelized:

    -j12 GCC_TEST_PARALLEL_SLOTS=2
    1143.83user 110.76system 10:20.46elapsed 202%CPU (0avgtext+0avgdata 505216maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=6
    1737.08user 143.94system 4:59.48elapsed 628%CPU (0avgtext+0avgdata 505200maxresident)k
    1730.31user 143.02system 4:58.75elapsed 627%CPU (0avgtext+0avgdata 505152maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=8
    2192.63user 169.34system 4:52.96elapsed 806%CPU (0avgtext+0avgdata 505216maxresident)k
    2219.04user 167.67system 4:53.19elapsed 814%CPU (0avgtext+0avgdata 505152maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=10
    2463.93user 184.98system 4:48.39elapsed 918%CPU (0avgtext+0avgdata 505200maxresident)k
    2455.62user 183.68system 4:47.40elapsed 918%CPU (0avgtext+0avgdata 505216maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=12
    2591.04user 192.64system 4:44.98elapsed 976%CPU (0avgtext+0avgdata 505216maxresident)k
    2581.23user 195.21system 4:47.51elapsed 965%CPU (0avgtext+0avgdata 505212maxresident)k
    -j20 GCC_TEST_PARALLEL_SLOTS=20 [oversubscribe]
    2613.18user 199.51system 4:44.06elapsed 990%CPU (0avgtext+0avgdata 505216maxresident)k

Case (d), baseline (compared to case (b): only nvptx offloading compilation,
but also nvptx offloading execution); ~1 h:

    2841.93user 653.68system 1:02:26elapsed 93%CPU (0avgtext+0avgdata 909792maxresident)k
    2842.03user 654.39system 1:02:24elapsed 93%CPU (0avgtext+0avgdata 909880maxresident)k

Case (d), parallelized:

    -j12 GCC_TEST_PARALLEL_SLOTS=2
    2856.39user 606.87system 33:58.64elapsed 169%CPU (0avgtext+0avgdata 909948maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=6
    3444.90user 666.86system 18:37.57elapsed 367%CPU (0avgtext+0avgdata 909856maxresident)k
    3462.13user 667.13system 18:36.87elapsed 369%CPU (0avgtext+0avgdata 909872maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=8
    3929.74user 716.22system 18:02.36elapsed 429%CPU (0avgtext+0avgdata 909832maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=10
    4152.84user 736.16system 17:43.05elapsed 459%CPU (0avgtext+0avgdata 909872maxresident)k
    -j12 GCC_TEST_PARALLEL_SLOTS=12
    4209.60user 749.00system 17:35.20elapsed 469%CPU (0avgtext+0avgdata 909840maxresident)k
    -j20 GCC_TEST_PARALLEL_SLOTS=20 [oversubscribe]
    4255.54user 756.78system 17:29.06elapsed 477%CPU (0avgtext+0avgdata 909868maxresident)k

Worth noting is that with nvptx offloading, there is one execution test case
that times out ('libgomp.fortran/reverse-offload-5.f90').  This effectively
stalls progress for almost 5 min: quickly other executions test cases queue up
on the lock for all parallel slots.  That's working as expected; just noting
this as it accordingly does skew the wall time numbers.

	PR libgomp/66005
	libgomp/
	* configure.ac: Look for 'flock'.
	* testsuite/Makefile.am (gcc_test_parallel_slots): Enable parallel testing.
	* testsuite/config/default.exp: Don't 'load_lib "standard.exp"' here...
	* testsuite/lib/libgomp.exp: ... but here, instead.
	(libgomp_load): Override for parallel testing.
	* testsuite/libgomp-site-extra.exp.in (FLOCK): Set.
	* configure: Regenerate.
	* Makefile.in: Regenerate.
	* testsuite/Makefile.in: Regenerate.
---
 libgomp/Makefile.in                         |  1 +
 libgomp/configure                           | 48 ++++++++++++++++++++-
 libgomp/configure.ac                        |  2 +
 libgomp/testsuite/Makefile.am               |  3 +-
 libgomp/testsuite/Makefile.in               |  4 +-
 libgomp/testsuite/config/default.exp        |  2 -
 libgomp/testsuite/lib/libgomp.exp           | 29 +++++++++++++
 libgomp/testsuite/libgomp-site-extra.exp.in |  1 +
 8 files changed, 84 insertions(+), 6 deletions(-)

diff --git a/libgomp/Makefile.in b/libgomp/Makefile.in
index 2c81ccacc1d..f1afb5ef57f 100644
--- a/libgomp/Makefile.in
+++ b/libgomp/Makefile.in
@@ -382,6 +382,7 @@ EXEEXT = @EXEEXT@
 FC = @FC@
 FCFLAGS = @FCFLAGS@
 FGREP = @FGREP@
+FLOCK = @FLOCK@
 GREP = @GREP@
 INSTALL = @INSTALL@
 INSTALL_DATA = @INSTALL_DATA@
diff --git a/libgomp/configure b/libgomp/configure
index fd0e337b578..489c9a00421 100755
--- a/libgomp/configure
+++ b/libgomp/configure
@@ -656,6 +656,7 @@ tmake_file
 XLDFLAGS
 XCFLAGS
 config_path
+FLOCK
 CPU_COUNT
 LIBGOMP_BUILD_VERSIONED_SHLIB_SUN_FALSE
 LIBGOMP_BUILD_VERSIONED_SHLIB_SUN_TRUE
@@ -11418,7 +11419,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 11421 "configure"
+#line 11422 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -11524,7 +11525,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 11527 "configure"
+#line 11528 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -16486,6 +16487,49 @@ $as_echo "unable to detect (assuming 1)" >&6; }
 fi
 
 
+for ac_prog in flock
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_FLOCK+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$FLOCK"; then
+  ac_cv_prog_FLOCK="$FLOCK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_FLOCK="$ac_prog"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+FLOCK=$ac_cv_prog_FLOCK
+if test -n "$FLOCK"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FLOCK" >&5
+$as_echo "$FLOCK" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+  test -n "$FLOCK" && break
+done
+
+
 # Get target configury.
 . ${srcdir}/configure.tgt
 CFLAGS="$save_CFLAGS $XCFLAGS"
diff --git a/libgomp/configure.ac b/libgomp/configure.ac
index a9b1f3973f7..0fab7168cfa 100644
--- a/libgomp/configure.ac
+++ b/libgomp/configure.ac
@@ -339,6 +339,8 @@ fi
 AX_COUNT_CPUS
 AC_SUBST(CPU_COUNT)
 
+AC_CHECK_PROGS(FLOCK, flock)
+
 # Get target configury.
 . ${srcdir}/configure.tgt
 CFLAGS="$save_CFLAGS $XCFLAGS"
diff --git a/libgomp/testsuite/Makefile.am b/libgomp/testsuite/Makefile.am
index eef02f4a8c2..0cc91ccc4d1 100644
--- a/libgomp/testsuite/Makefile.am
+++ b/libgomp/testsuite/Makefile.am
@@ -73,7 +73,8 @@ check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_num
 check_p_numbers5:=$(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4)
 check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5)))
 check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
-gcc_test_parallel_slots:=1
+# If unable to serialize execution testing, use just one parallel slot.
+gcc_test_parallel_slots:=$(if $(FLOCK),$(if $(GCC_TEST_PARALLEL_SLOTS),$(GCC_TEST_PARALLEL_SLOTS),19),1)
 check_p_subdirs=$(wordlist 1,$(gcc_test_parallel_slots),$(check_p_numbers))
 check_DEJAGNU_libgomp_targets = $(addprefix check-DEJAGNUlibgomp,$(check_p_subdirs))
 $(check_DEJAGNU_libgomp_targets): check-DEJAGNUlibgomp%: libgomp%/site.exp
diff --git a/libgomp/testsuite/Makefile.in b/libgomp/testsuite/Makefile.in
index 3b5903daed9..70cb5e42d7b 100644
--- a/libgomp/testsuite/Makefile.in
+++ b/libgomp/testsuite/Makefile.in
@@ -161,6 +161,7 @@ EXEEXT = @EXEEXT@
 FC = @FC@
 FCFLAGS = @FCFLAGS@
 FGREP = @FGREP@
+FLOCK = @FLOCK@
 GREP = @GREP@
 INSTALL = @INSTALL@
 INSTALL_DATA = @INSTALL_DATA@
@@ -309,7 +310,8 @@ check_p_numbers4 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_n
 check_p_numbers5 := $(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4)
 check_p_numbers6 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5)))
 check_p_numbers := $(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
-gcc_test_parallel_slots := 1
+# If unable to serialize execution testing, use just one parallel slot.
+gcc_test_parallel_slots := $(if $(FLOCK),$(if $(GCC_TEST_PARALLEL_SLOTS),$(GCC_TEST_PARALLEL_SLOTS),19),1)
 check_p_subdirs = $(wordlist 1,$(gcc_test_parallel_slots),$(check_p_numbers))
 check_DEJAGNU_libgomp_targets = $(addprefix check-DEJAGNUlibgomp,$(check_p_subdirs))
 all: all-am
diff --git a/libgomp/testsuite/config/default.exp b/libgomp/testsuite/config/default.exp
index b7afc82ff03..01569e6ab62 100644
--- a/libgomp/testsuite/config/default.exp
+++ b/libgomp/testsuite/config/default.exp
@@ -13,5 +13,3 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; see the file COPYING3.  If not see
 # <http://www.gnu.org/licenses/>.
-
-load_lib "standard.exp"
diff --git a/libgomp/testsuite/lib/libgomp.exp b/libgomp/testsuite/lib/libgomp.exp
index 4ea6f26c674..8512f2c1e64 100644
--- a/libgomp/testsuite/lib/libgomp.exp
+++ b/libgomp/testsuite/lib/libgomp.exp
@@ -9,6 +9,7 @@ proc load_gcc_lib { filename } {
 }
 
 load_lib dg.exp
+load_lib standard.exp
 
 # Required to use gcc-dg.exp - however, the latter should NOT be
 # loaded until ${tool}_target_compile is defined since it uses that
@@ -297,6 +298,34 @@ proc libgomp_option_proc { option } {
     }
 }
 
+if ![info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
+    # No parallel testing.
+} elseif { $FLOCK == "" } {
+    # Using just one parallel slot.
+} else {
+    # Using several parallel slots.  Override DejaGnu
+    # 'standard.exp:${tool}_load'...
+    rename libgomp_load standard_libgomp_load
+    proc libgomp_load { program args } {
+	# ... in order to serialize execution testing via an exclusive lock.
+	set lock_file ../lock
+	set lock_kind --exclusive
+	set lock_fd [open $lock_file a+]
+	set lock_clock_begin [clock seconds]
+	global FLOCK
+	exec $FLOCK $lock_kind 0 <@ $lock_fd
+	set lock_clock_end [clock seconds]
+	verbose -log "Got ${FLOCK}('$lock_file', '$lock_kind') at [clock format $lock_clock_end] after [expr $lock_clock_end - $lock_clock_begin] s" 2
+
+	set result [standard_libgomp_load $program $args]
+
+	# Unlock (implicit with 'close').
+	close $lock_fd
+
+	return $result
+    }
+}
+
 # Translate offload target to OpenACC device type.  Return the empty string if
 # not supported, and 'host' for offload target 'disable'.
 proc offload_target_to_openacc_device_type { offload_target } {
diff --git a/libgomp/testsuite/libgomp-site-extra.exp.in b/libgomp/testsuite/libgomp-site-extra.exp.in
index c0d26660bad..0a3ba059c21 100644
--- a/libgomp/testsuite/libgomp-site-extra.exp.in
+++ b/libgomp/testsuite/libgomp-site-extra.exp.in
@@ -1 +1,2 @@
+set FLOCK {@FLOCK@}
 set GCC_UNDER_TEST {@CC@}
-- 
2.34.1

Re: Support parallel testing in libgomp, part I [PR66005]

[Bernhard Reutner-Fischer]

On Fri, 5 May 2023 10:55:41 +0200
Thomas Schwinge <thomas@codesourcery.com> wrote:

> So I recently had re-created this patch independently, before remembering
> that Rainer had -- just eight years ago... ;-) -- already submitted this.

thanks to you both :)

> etc. (where "normal" is a libstdc++ detail), and regarding:
> 
> >> > with a minimal change
> >> > to libgomp.exp so the generated libgomp-test-support.exp file is found
> >> > in both the sequential and parallel cases.  This isn't an issue in
> >> > libstdc++ since all necessary variables are stored in a single
> >> > site.exp.  
> 
> ... in 'libgomp/testsuite/lib/libgomp.exp', I've changed:
> 
>     -load_file libgomp-test-support.exp
>     +# Search in both .. and . to support parallel and sequential testing.
>     +load_file -1 ../libgomp-test-support.exp libgomp-test-support.exp
> 
> ... into the more explicit:
> 
>     -load_file libgomp-test-support.exp
>     +# Search in '..' vs. '.' to support parallel vs. sequential testing.
>     +if [info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
>     +    load_file ../libgomp-test-support.exp
>     +} else {
>     +    load_file libgomp-test-support.exp
>     +}

Do we have to re-read those? Otherwise this would be load_lib:

We have libdirs in the minimum deja we require.

Speaking of which. IIRC i additionally deleted all load_gcc_lib:
https://gcc.gnu.org/legacy-ml/fortran/2012-03/msg00094.html
in lib{atomic,ffi,go,gomp,itm,phobos,stdc++-v3,vtv}

> And, for now, I hard-code the number of parallel slots to one.  This
> means that libgomp for 'make -j' now does use the parallel testing code
> paths, but is restricted to just one slot.  That is, no actual change in
> behavior, other than that 'libgomp.sum' then is filtered through
> 'contrib/dg-extract-results.sh'.
> 
> OK to push the attached
> "Support parallel testing in libgomp, part I [PR66005]"?

Some cosmetic nits.
See Jakubs one_to_9999.

+	@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
that's twisted

+	  rm -rf libgomp-parallel || true; \

just || :; \
I count 4 times.

There seems to be a mixture of ${PWD_COMMAND} and am__cd && pwd:
+	@objdir=`${PWD_COMMAND}`/$*; \
+	srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \

+	runtest=$(_RUNTEST); \
+	if [ -z "$$runtest" ]; then runtest=runtest; fi; \
I think I have plain $${RUNTEST-runtest}
off the default wildcard $(top_srcdir)/../dejagnu/runtest

> >> It is far from trivial though.
> >> The point is that most of the OpenMP tests are parallelized with the
> >> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
> >> machine a lot, the higher number of hw threads the more.  
> >
> > Do you agree that we have two classes of test cases in libgomp: 1) test
> > cases that don't place a considerably higher load on the machine compared
> > to "normal" (single-threaded) execution tests, because they're just
> > testing some functionality that is not expected to actively depend
> > on/interfere with parallelism.  If needed, and/or if not already done,
> > such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
> > num_workers, vector_length clauses, and so on) for low parallelism
> > levels.  And, 2) test cases that place a considerably higher load on the
> > machine compared to "normal" (single-threaded) execution tests, because
> > they're testing some functionality that actively depends on/interferes
> > with some kind of parallelism.  What about marking such tests specially,
> > such that DejaGnu will only ever schedule one of them for execution at
> > the same time?  For example, a new dg-* directive to run them wrapped
> > through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?

I think we all agree one that, yes.

> >  
> >> If we go forward with some parallelization of the tests, we at least should
> >> try to export something like OMP_WAIT_POLICY=passive so that the
> >> oversubscribed machine would at least not spend too much time in spinning.  
> >
> > (Will again have the problem that DejaGnu doesn't provide infrastructure
> > to communicate environment variables to boards in remote testing.)

Are you sure? I'm pretty confident that this worked fine at least at
one point in the past for certain targets.

The rest of these 2 patches LGTM. Let's see what others have to say.
thanks,

Re: Support parallel testing in libgomp, part I [PR66005]

[Thomas Schwinge]

Hi Bernhard!

Thanks for your comments.


On 2023-05-06T16:15:45+0200, Bernhard Reutner-Fischer via Gcc-patches <gcc-patches@gcc.gnu.org> wrote:
> On Fri, 5 May 2023 10:55:41 +0200
> Thomas Schwinge <thomas@codesourcery.com> wrote:
>
>> >> > with a minimal change
>> >> > to libgomp.exp so the generated libgomp-test-support.exp file is found
>> >> > in both the sequential and parallel cases.  This isn't an issue in
>> >> > libstdc++ since all necessary variables are stored in a single
>> >> > site.exp.
>>
>> ... in 'libgomp/testsuite/lib/libgomp.exp', I've changed:
>>
>>     -load_file libgomp-test-support.exp
>>     +# Search in both .. and . to support parallel and sequential testing.
>>     +load_file -1 ../libgomp-test-support.exp libgomp-test-support.exp
>>
>> ... into the more explicit:
>>
>>     -load_file libgomp-test-support.exp
>>     +# Search in '..' vs. '.' to support parallel vs. sequential testing.
>>     +if [info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
>>     +    load_file ../libgomp-test-support.exp
>>     +} else {
>>     +    load_file libgomp-test-support.exp
>>     +}
>
> Do we have to re-read those? Otherwise this would be load_lib:

Indeed there is some confusion there; conceptually the content of that
file has to be expected to vary per libgomp (multilib) build variant.
On the other hand, given the GCC target libraries testing setup, we're
running testing always via the default variant's build, so always read
the default variant's file.  I agree that should get un-confused, however
it's a pre-existing issue that I suggest we tackle independently of this
mechanical change.

> We have libdirs in the minimum deja we require.
>
> Speaking of which. IIRC i additionally deleted all load_gcc_lib:
> https://gcc.gnu.org/legacy-ml/fortran/2012-03/msg00094.html
> in lib{atomic,ffi,go,gomp,itm,phobos,stdc++-v3,vtv}

Aha, another pre-existing/independent thing to look into.


>> And, for now, I hard-code the number of parallel slots to one.  This
>> means that libgomp for 'make -j' now does use the parallel testing code
>> paths, but is restricted to just one slot.  That is, no actual change in
>> behavior, other than that 'libgomp.sum' then is filtered through
>> 'contrib/dg-extract-results.sh'.
>>
>> OK to push the attached
>> "Support parallel testing in libgomp, part I [PR66005]"?
>
> Some cosmetic nits.
> See Jakubs one_to_9999.

Thanks.  That got installed while I'd already finished my patch.  ;-)
Note that the new 'one_to_9999' etc. is just the existing
'check_p_numbers' etc. renamed and moved, as it now has a second use.
I'm happy to incorporate that into my changes -- if we agree that it
should also go into other GCC target libraries' parallel testing code:
libphobos, libstdc++-v3.

> +     @test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
> that's twisted
>
> +       rm -rf libgomp-parallel || true; \
>
> just || :; \
> I count 4 times.
>
> There seems to be a mixture of ${PWD_COMMAND} and am__cd && pwd:
> +     @objdir=`${PWD_COMMAND}`/$*; \
> +     srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \
>
> +     runtest=$(_RUNTEST); \
> +     if [ -z "$$runtest" ]; then runtest=runtest; fi; \
> I think I have plain $${RUNTEST-runtest}
> off the default wildcard $(top_srcdir)/../dejagnu/runtest

As I've written: "libstdc++ parallel testing infrastructure [...] is
where these changes have been copied from", and that one had it copied
from GCC compilers parallel testing infrastructure, I presume.  I do
agree that there's value in unifying and, as feasible, simplifying this
code, but again that to me sounds like a pre-existing/separate thing to
do?


>> >> It is far from trivial though.
>> >> The point is that most of the OpenMP tests are parallelized with the
>> >> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
>> >> machine a lot, the higher number of hw threads the more.
>> >
>> > Do you agree that we have two classes of test cases in libgomp: 1) test
>> > cases that don't place a considerably higher load on the machine compared
>> > to "normal" (single-threaded) execution tests, because they're just
>> > testing some functionality that is not expected to actively depend
>> > on/interfere with parallelism.  If needed, and/or if not already done,
>> > such test cases can be parameterized (OMP_NUM_THREADS, OpenACC num_gangs,
>> > num_workers, vector_length clauses, and so on) for low parallelism
>> > levels.  And, 2) test cases that place a considerably higher load on the
>> > machine compared to "normal" (single-threaded) execution tests, because
>> > they're testing some functionality that actively depends on/interferes
>> > with some kind of parallelism.  What about marking such tests specially,
>> > such that DejaGnu will only ever schedule one of them for execution at
>> > the same time?  For example, a new dg-* directive to run them wrapped
>> > through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?
>
> I think we all agree one that, yes.

Conceptually, yes.  However, given that my
"Support parallel testing in libgomp, part II [PR66005]" changes already
seem to enable a great amount of parallelism, occupying CPUs almost
fully,  I'm not actually sure now if adding more parallelism via
tedious-to-maintain new dg-* directives is actually necessary/useful.  As
long as libgomp testing now no longer is at the long tail end of overall
testing time, I'd rather keep it simple?


>> >> If we go forward with some parallelization of the tests, we at least should
>> >> try to export something like OMP_WAIT_POLICY=passive so that the
>> >> oversubscribed machine would at least not spend too much time in spinning.
>> >
>> > (Will again have the problem that DejaGnu doesn't provide infrastructure
>> > to communicate environment variables to boards in remote testing.)
>
> Are you sure? I'm pretty confident that this worked fine at least at
> one point in the past for certain targets.

For example, see the comments/references in recent
<https://inbox.sourceware.org/018bcdeb-b3bb-1859-cd0b-a8a92e26d3a0@codesourcery.com>.


> The rest of these 2 patches LGTM. Let's see what others have to say.

Thanks!


Grüße
 Thomas
-----------------
Siemens Electronic Design Automation GmbH; Anschrift: Arnulfstraße 201, 80634 München; Gesellschaft mit beschränkter Haftung; Geschäftsführer: Thomas Heurung, Frank Thürauf; Sitz der Gesellschaft: München; Registergericht München, HRB 106955

Re: Support parallel testing in libgomp, part I [PR66005]

[Bernhard Reutner-Fischer]

On Mon, 8 May 2023 12:42:33 +0200
Thomas Schwinge <thomas@codesourcery.com> wrote:

> >>     +if [info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
> >>     +    load_file ../libgomp-test-support.exp
> >>     +} else {
> >>     +    load_file libgomp-test-support.exp
> >>     +}  
> >
> > Do we have to re-read those? Otherwise this would be load_lib:  
> 
> Indeed there is some confusion there; conceptually the content of that
> file has to be expected to vary per libgomp (multilib) build variant.
> On the other hand, given the GCC target libraries testing setup, we're
> running testing always via the default variant's build, so always read
> the default variant's file.  I agree that should get un-confused, however
> it's a pre-existing issue that I suggest we tackle independently of this
> mechanical change.

Sure. One thing at a time.

> > Some cosmetic nits.
> > See Jakubs one_to_9999.  
> 
> Thanks.  That got installed while I'd already finished my patch.  ;-)
> Note that the new 'one_to_9999' etc. is just the existing
> 'check_p_numbers' etc. renamed and moved, as it now has a second use.
> I'm happy to incorporate that into my changes -- if we agree that it
> should also go into other GCC target libraries' parallel testing code:
> libphobos, libstdc++-v3.

Yes. Streamlining these can be done in follow-ups if the respective
maintainers agree.

> >> >> It is far from trivial though.
> >> >> The point is that most of the OpenMP tests are parallelized with the
> >> >> default OMP_NUM_THREADS, so running the tests in parallel oversubscribes the
> >> >> machine a lot, the higher number of hw threads the more.  
[]
> >> > the same time?  For example, a new dg-* directive to run them wrapped
> >> > through »flock [libgomp/testsuite/serial.lock] [a.out]« or some such?  
> >
> > I think we all agree one that, yes.  
> 
> Conceptually, yes.  However, given that my
> "Support parallel testing in libgomp, part II [PR66005]" changes already
> seem to enable a great amount of parallelism, occupying CPUs almost
> fully,  I'm not actually sure now if adding more parallelism via
> tedious-to-maintain new dg-* directives is actually necessary/useful.  As
> long as libgomp testing now no longer is at the long tail end of overall
> testing time, I'd rather keep it simple?

If the testsuite runtime is fine with your part II as is then we
should keep it as simple as possible.

> >> > (Will again have the problem that DejaGnu doesn't provide infrastructure
> >> > to communicate environment variables to boards in remote testing.)  
> >
> > Are you sure? I'm pretty confident that this worked fine at least at
> > one point in the past for certain targets.  
> 
> For example, see the comments/references in recent
> <https://inbox.sourceware.org/018bcdeb-b3bb-1859-cd0b-a8a92e26d3a0@codesourcery.com>.

oh, i think i had an rsh2 remote that uploaded $program.env and the rsh
itself was something like
 $RSH $rsh_useropts $hostname sh -c 'test -r $program.env && .
 $program.env \\; $program $pargs \\; echo ...

but that was ages ago and was properly implemented in the meantime, one
would hope? Well, apparently not.

PS: Back then I usually needed very different per-program env and not a
single, static env. So for me it made no sense to have a set of default
env and have tests add their own variables on-demand on top of the
default. The situation in gcc might be the exact opposite?

thanks,

Re: Support parallel testing in libgomp, part II [PR66005]

[Bernhard Reutner-Fischer]

Thomas,

On Fri, 5 May 2023 10:59:31 +0200
Thomas Schwinge <thomas@codesourcery.com> wrote:

> Worth noting is that with nvptx offloading, there is one execution test case
> that times out ('libgomp.fortran/reverse-offload-5.f90').  This effectively
> stalls progress for almost 5 min:

Short of fixing it for real you could shorten the timeout for this
single test to a handful of seconds: dg-timeout 3
or set it to a 1/50 fraction or the like: dg-timeout-factor 0.02

If it's unbroken then it should run to completion in far less than 10
or 30 seconds, from the looks?

just a thought..

Support parallel testing in libgomp: fallback Perl 'flock' [PR66005]

[Thomas Schwinge]

[-- Attachment #1: Type: text/plain, Size: 1873 bytes --]

Hi!

On 2023-05-05T10:59:31+0200, I wrote:
> On 2023-05-05T10:55:41+0200, I wrote:
>> [Putting Bernhard, Honza, Segher in CC, as they are eager to test this,
>> based on recent comments on IRC.]  ;-P


>> First, establish the parallel testing infrastructure -- while still
>> hard-coding the number of parallel slots to one.

>> "Support parallel testing in libgomp, part I [PR66005]"

> On top of that, second, enable parallel testing.

> implemented what I'd described in
> <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66005#c2>:
>
> | [...] parallelize *all* compilation, while just allowing for *one*
> | execution test job slot.  That will require some GCC DejaGnu test
> | harness hackery which I've [now] gotten to look into.  That is, enable
> | the usual GCC/DejaGnu parallel testing, but also have some kind of
> | mutex for the execution test invocation.  This has to play nicely with
> | DejaGnu timeout handling, etc.

> Subject: [PATCH] Support parallel testing in libgomp, part II [PR66005]
>
> ..., and enable if 'flock' is available for serializing execution testing.

OK to push the attached
"Support parallel testing in libgomp: fallback Perl 'flock' [PR66005]"?

Per the PR66005 discussion, if 'flock' is not available, having a
fallback Perl 'flock' for parallelizing 'check-target-libgomp' wasn't met
with the greatest of all enthusiasm -- but in my opinion it's still
better than continued all-serial 'check-target-libgomp'?

We may then proceed working on a more integrated solution, using TCL or
shell features.


Grüße
 Thomas


-----------------
Siemens Electronic Design Automation GmbH; Anschrift: Arnulfstraße 201, 80634 München; Gesellschaft mit beschränkter Haftung; Geschäftsführer: Thomas Heurung, Frank Thürauf; Sitz der Gesellschaft: München; Registergericht München, HRB 106955

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Support-parallel-testing-in-libgomp-fallback-Perl-fl.patch --]
[-- Type: text/x-diff, Size: 6458 bytes --]

From c62858bf888fec2f61febafcd6afe2dc8c3f679b Mon Sep 17 00:00:00 2001
From: Thomas Schwinge <thomas@codesourcery.com>
Date: Mon, 15 May 2023 20:00:07 +0200
Subject: [PATCH] Support parallel testing in libgomp: fallback Perl 'flock'
 [PR66005]

Follow-up to commit 6c3b30ef9e0578509bdaf59c13da4a212fe6c2ba
"Support parallel testing in libgomp, part II [PR66005]"
("..., and enable if 'flock' is available for serializing execution testing"),
where we saw:

> On my Dell Precision 7530 laptop:
>
>     $ uname -srvi
>     Linux 5.15.0-71-generic #78-Ubuntu SMP Tue Apr 18 09:00:29 UTC 2023 x86_64
>     $ grep '^model name' < /proc/cpuinfo | uniq -c
>          12 model name      : Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz
>     $ nvidia-smi -L
>     GPU 0: Quadro P1000 (UUID: GPU-e043973b-b52a-d02b-c066-a8fdbf64e8ea)
>
> ... [...]: case (c) standard configuration, no offloading
> configured, [...]

>     $ \time make check-target-libgomp
>
> Case (c), baseline; [...]:
>
>     1180.98user 110.80system 19:36.40elapsed 109%CPU (0avgtext+0avgdata 505148maxresident)k
>     1133.22user 111.08system 19:35.75elapsed 105%CPU (0avgtext+0avgdata 505212maxresident)k
>
> Case (c), parallelized [using 'flock']:
>
> [...]
>     -j12 GCC_TEST_PARALLEL_SLOTS=12
>     2591.04user 192.64system 4:44.98elapsed 976%CPU (0avgtext+0avgdata 505216maxresident)k
>     2581.23user 195.21system 4:47.51elapsed 965%CPU (0avgtext+0avgdata 505212maxresident)k

Quite the same when instead of 'flock' using this fallback Perl 'flock':

    2565.23user 194.35system 4:46.77elapsed 962%CPU (0avgtext+0avgdata 505216maxresident)k
    2549.38user 200.20system 4:46.08elapsed 961%CPU (0avgtext+0avgdata 505216maxresident)k

	PR testsuite/66005
	gcc/
	* doc/install.texi: Document (optional) Perl usage for parallel
	testing of libgomp.
	libgomp/
	* testsuite/lib/libgomp.exp: 'flock' through stdout.
	* testsuite/flock: New.
	* configure.ac (FLOCK): Point to that if no 'flock' available, but
	'perl' is.
	* configure: Regenerate.
---
 gcc/doc/install.texi              |  3 +++
 libgomp/configure                 | 42 +++++++++++++++++++++++++++++++
 libgomp/configure.ac              |  5 ++++
 libgomp/testsuite/flock           | 17 +++++++++++++
 libgomp/testsuite/lib/libgomp.exp |  4 ++-
 5 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100755 libgomp/testsuite/flock

diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index dfab47dac96..fe4a972980f 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -382,6 +382,9 @@ tables.
 
 Used by @command{automake}.
 
+If available, enables parallel testing of @samp{libgomp} in case that
+@command{flock} is not available.
+
 @end table
 
 Several support libraries are necessary to build GCC, some are required,
diff --git a/libgomp/configure b/libgomp/configure
index 2b45acd08c6..a280ca9238a 100755
--- a/libgomp/configure
+++ b/libgomp/configure
@@ -16457,6 +16457,8 @@ $as_echo "unable to detect (assuming 1)" >&6; }
 fi
 
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for flock implementation" >&5
+$as_echo "$as_me: checking for flock implementation" >&6;}
 for ac_prog in flock
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
@@ -16499,6 +16501,46 @@ fi
   test -n "$FLOCK" && break
 done
 
+# Fallback if 'perl' is available.
+if test -z "$FLOCK"; then
+  # Extract the first word of "perl", so it can be a program name with args.
+set dummy perl; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_FLOCK+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$FLOCK"; then
+  ac_cv_prog_FLOCK="$FLOCK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_FLOCK="$srcdir/testsuite/flock"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+FLOCK=$ac_cv_prog_FLOCK
+if test -n "$FLOCK"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FLOCK" >&5
+$as_echo "$FLOCK" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
 
 # Get target configury.
 . ${srcdir}/configure.tgt
diff --git a/libgomp/configure.ac b/libgomp/configure.ac
index 701573e3d99..2a7bf7c174a 100644
--- a/libgomp/configure.ac
+++ b/libgomp/configure.ac
@@ -349,7 +349,12 @@ fi
 AX_COUNT_CPUS
 AC_SUBST(CPU_COUNT)
 
+AC_MSG_NOTICE([checking for flock implementation])
 AC_CHECK_PROGS(FLOCK, flock)
+# Fallback if 'perl' is available.
+if test -z "$FLOCK"; then
+  AC_CHECK_PROG(FLOCK, perl, $srcdir/testsuite/flock)
+fi
 
 # Get target configury.
 . ${srcdir}/configure.tgt
diff --git a/libgomp/testsuite/flock b/libgomp/testsuite/flock
new file mode 100755
index 00000000000..71878b104f1
--- /dev/null
+++ b/libgomp/testsuite/flock
@@ -0,0 +1,17 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+# Only arguments '--exclusive 1' exactly are supported.
+(@ARGV == 2) or die;
+my $mode = shift;
+($mode eq "--exclusive") or die;
+my $fd = shift;
+($fd eq "1") or die;
+
+use Fcntl ':flock';
+
+open(my $fh, '>&=', 1) or die "open: $!";
+
+flock($fh, LOCK_EX) or die "flock: $!";
diff --git a/libgomp/testsuite/lib/libgomp.exp b/libgomp/testsuite/lib/libgomp.exp
index 3ad5503e287..2f9e538278f 100644
--- a/libgomp/testsuite/lib/libgomp.exp
+++ b/libgomp/testsuite/lib/libgomp.exp
@@ -338,12 +338,14 @@ if ![info exists ::env(GCC_RUNTEST_PARALLELIZE_DIR)] {
     rename libgomp_load standard_libgomp_load
     proc libgomp_load { program args } {
 	# ... in order to serialize execution testing via an exclusive lock.
+	# We use stdout, as per <https://perldoc.perl.org/functions/flock>
+	# "[...] FILEHANDLE [...] be open with write intent to use LOCK_EX".
 	set lock_file ../lock
 	set lock_kind --exclusive
 	set lock_fd [open $lock_file a+]
 	set lock_clock_begin [clock seconds]
 	global FLOCK
-	exec $FLOCK $lock_kind 0 <@ $lock_fd
+	exec $FLOCK $lock_kind 1 >@ $lock_fd
 	set lock_clock_end [clock seconds]
 	verbose -log "Got ${FLOCK}('$lock_file', '$lock_kind') at [clock format $lock_clock_end] after [expr $lock_clock_end - $lock_clock_begin] s" 2
 
-- 
2.34.1