From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from csb.redhat.com (gnu.wildebeest.org [45.83.234.184]) by sourceware.org (Postfix) with ESMTPS id 4BC4D3858D32 for ; Sun, 15 Jan 2023 21:46:12 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.2 sourceware.org 4BC4D3858D32 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=klomp.org Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=klomp.org Received: by csb.redhat.com (Postfix, from userid 10916) id 6D872CB9DF; Sun, 15 Jan 2023 22:46:11 +0100 (CET) From: Mark Wielaard To: buildbot@sourceware.org Cc: Mark Wielaard Subject: [PATCH] Add a make_output_sync_arg property for workers to override Date: Sun, 15 Jan 2023 22:46:09 +0100 Message-Id: <20230115214609.152613-1-mark@klomp.org> X-Mailer: git-send-email 2.31.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-3038.1 required=5.0 tests=BAYES_00,GIT_PATCH_0,JMQ_SPF_NEUTRAL,KAM_DMARC_STATUS,SPF_HELO_NONE,SPF_PASS,TXREP autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org List-Id: make --output-sync=line is really nice for parallel make stdout logs. But unfortunately old make doesn't support it. So check a make_output_sync_arg property to see if it can be used. A worker can set this to the empty string if it has a really old make. Luckily only centos 7 seems to have such an old make. --- builder/master.cfg | 61 +++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/builder/master.cfg b/builder/master.cfg index 600103b..aa55186 100644 --- a/builder/master.cfg +++ b/builder/master.cfg @@ -60,7 +60,8 @@ c['workers'].append(sourceware_worker) centos_x86_64_worker = worker.Worker("centos-x86_64", getpw("centos-x86_64"), max_builds=1, - properties={'ncpus': 8, 'maxcpus': 8}, + properties={'ncpus': 8, 'maxcpus': 8, + 'make_output_sync_arg': ''}, notify_on_missing='mark@klomp.org'); c['workers'].append(centos_x86_64_worker) debian_i386_worker = worker.Worker("debian-i386", @@ -962,27 +963,31 @@ getversion_step = steps.SetPropertyFromCommand( name="get version", command='grep PACKAGE_VERSION config.h | cut -f2 -d\\"', property="package_version") +# Allows a worker to override argument (with the empty string) +# if make doesn't support --output-sync=line +make_output_sync = util.Property('make_output_sync_arg', + default='--output-sync=line') make_step = steps.Compile( - command=["make", "--output-sync=line", util.Interpolate('-j%(prop:ncpus)s')], + command=["make", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name="make", haltOnFailure=True) make_check_step = steps.Test( - command=["make", "--output-sync=line", "check", util.Interpolate('-j%(prop:ncpus)s')], + command=["make", make_output_sync, "check", util.Interpolate('-j%(prop:ncpus)s')], name="make check", haltOnFailure=False, flunkOnFailure=True) # Same as make_check_step but with tests/test-suite.log recorded make_check_test_suite_step = steps.Test( - command=["make", "--output-sync=line", "check", util.Interpolate('-j%(prop:ncpus)s')], + command=["make", make_output_sync, "check", util.Interpolate('-j%(prop:ncpus)s')], name="make check", haltOnFailure=False, flunkOnFailure=True, logfiles={"test-suite.log": "tests/test-suite.log"}) make_distcheck_step = steps.Test( - command=["make", "--output-sync=line", "distcheck", + command=["make", make_output_sync, "distcheck", util.Interpolate('-j%(prop:ncpus)s')], name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600) # Same as make_distcheck_step but with tests/test-suite-log from _build # Needs package_version and package_name properties set make_distcheck_test_suite_step = steps.Test( - command=["make", "--output-sync=line", "distcheck", + command=["make", make_output_sync, "distcheck", util.Interpolate('-j%(prop:ncpus)s')], name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600, @@ -1944,19 +1949,19 @@ gccrust_configure_bootstrap_step = steps.Configure( gccrust_make_step = steps.Compile( workdir='gccrs-build', - command=['make', "--output-sync=line", util.Interpolate('-j%(prop:ncpus)s')], + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name='make', haltOnFailure=True) gccrust_make_bootstrap_step = steps.Compile( workdir='gccrs-build', - command=['make', "--output-sync=line", util.Interpolate('-j%(prop:maxcpus)s')], + command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], name='make', haltOnFailure=True) gccrust_check_step = steps.Compile( workdir='gccrs-build', - command=['make', "--output-sync=line", 'check-rust', util.Interpolate('-j%(prop:ncpus)s')], + command=['make', make_output_sync, 'check-rust', util.Interpolate('-j%(prop:ncpus)s')], name='make check', logfiles={ "rust.sum": "gcc/testsuite/rust/rust.sum", "rust.log": "gcc/testsuite/rust/rust.log" }, @@ -1964,7 +1969,7 @@ gccrust_check_step = steps.Compile( gccrust_check_bootstrap_step = steps.Compile( workdir='gccrs-build', - command=['make', "--output-sync=line", 'check-rust', util.Interpolate('-j%(prop:maxcpus)s')], + command=['make', make_output_sync, 'check-rust', util.Interpolate('-j%(prop:maxcpus)s')], name='make check', logfiles={ "rust.sum": "gcc/testsuite/rust/rust.sum", "rust.log": "gcc/testsuite/rust/rust.log" }, @@ -2130,14 +2135,14 @@ binutils_step_configure_libctf = steps.Configure( haltOnFailure=True) binutils_step_compile = steps.Compile( workdir='binutils-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'all-gas', 'all-ld', 'all-binutils', 'all-gold'], name='make', haltOnFailure=True) binutils_step_compile_libctf = steps.Compile( workdir='binutils-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'all-gas', 'all-ld', 'all-binutils', 'all-gold', 'all-libctf'], @@ -2145,7 +2150,7 @@ binutils_step_compile_libctf = steps.Compile( haltOnFailure=True) binutils_step_check = steps.Test( workdir='binutils-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'check-ld', 'check-gas', 'check-binutils', 'check-libsframe'], @@ -2165,7 +2170,7 @@ binutils_steps_bunsen = bunsen_logfile_upload_cpio_steps( binutils_step_check_libctf = steps.Test( workdir='binutils-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'check-ld', 'check-gas', 'check-binutils', 'check-libctf', 'check-libsframe'], @@ -2187,7 +2192,7 @@ binutils_steps_bunsen_libctf = bunsen_logfile_upload_cpio_steps( binutils_step_check_gas_binutils = steps.Test( workdir='binutils-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'check-gas', 'check-binutils'], name='make check', @@ -2492,7 +2497,7 @@ gdb_configure_32_step = steps.Configure( haltOnFailure=True) gdb_make_step = steps.Compile( workdir='gdb-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), 'all-gdb', 'all-gdbserver'], name='make', @@ -2893,7 +2898,7 @@ def binutils_gdb_factory_factory(runtestflags=None): name='configure', haltOnFailure=True)) binutils_gdb_factory.addStep(steps.Compile( - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), '-k', 'all'], # for better logs, keep going name='make', @@ -2988,7 +2993,7 @@ gcc_factory.addStep(steps.Configure( name='configure', haltOnFailure=True)) gcc_factory.addStep(steps.Compile( - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s'), '-k', 'all', 'install'], # for better logs, keep going name='make', @@ -3016,7 +3021,7 @@ gcc_build_factory.addStep(gcc_rm_build_step) gcc_build_factory.addStep(gcc_build_configure_step) gcc_build_factory.addStep(steps.Compile( workdir='gcc-build', - command=['make', "--output-sync=line", util.Interpolate('-j%(prop:ncpus)s')], + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name='make', haltOnFailure=True)) gcc_build_factory.addStep(gcc_rm_build_step) @@ -3137,14 +3142,14 @@ def gcc_full_build_factory_gen(extra_configure_arg=None): gcc_full_build_factory.addStep(gcc_full_build_configure_step) gcc_full_build_factory.addStep(steps.Compile( workdir='gcc-build', - command=['make', "--output-sync=line", util.Interpolate('-j%(prop:maxcpus)s')], + command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], name='make', haltOnFailure=True)) # We want parallelism to get through this as quickly as possible. # Even if that means bunsen gets some parallel/duplicate log files gcc_full_build_factory.addStep(steps.Test( workdir='gcc-build', - command=['make', 'check', "--output-sync=line", util.Interpolate('-j%(prop:maxcpus)s')], + command=['make', 'check', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], name='make check', haltOnFailure=False, flunkOnFailure=True)) gcc_full_build_factory.addSteps(bunsen_logfile_upload_cpio_steps( @@ -3205,21 +3210,21 @@ glibc_configure_step = steps.Configure( glibc_make_step = steps.Compile( workdir='glibc-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], name='make', haltOnFailure=False, flunkOnFailure=True) glibc_make_build_step = steps.Compile( workdir='glibc-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name='make', haltOnFailure=True) glibc_check_step = steps.Test( workdir='glibc-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s'), 'check'], name='make check', @@ -3227,7 +3232,7 @@ glibc_check_step = steps.Test( glibc_check_build_step = steps.Test( workdir='glibc-build', - command=['make', "--output-sync=line", + command=['make', make_output_sync, 'subdirs=elf', util.Interpolate('-j%(prop:ncpus)s'), 'check'], @@ -3364,11 +3369,11 @@ libabigail_git_step = steps.Git( name="git checkout", haltOnFailure=True) libabigail_make_check_test_suite_step = steps.Test( - command=["make", "check", "ENABLE_SLOW_TEST=yes", "--output-sync=line", util.Interpolate('-j%(prop:ncpus)s')], + command=["make", "check", "ENABLE_SLOW_TEST=yes", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name="make check", haltOnFailure=False, flunkOnFailure=True, logfiles={"test-suite.log": "tests/test-suite.log"}) libabigail_fast_distcheck = steps.Test( - command=["make", "distcheck-fast", "--output-sync=line", util.Interpolate('-j%(prop:ncpus)s')], + command=["make", "distcheck-fast", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600, logfiles={"test-suite.log": util.Interpolate('%(prop:package_name)s-%(prop:package_version)s/_build/sub/tests/test-suite.log')}) @@ -3826,7 +3831,7 @@ systemtap_factory.addStep(steps.Configure( haltOnFailure=True)) systemtap_factory.addStep(steps.Compile( workdir='stap-build', - command=['make', 'V=1', '--output-sync=line', util.Interpolate('-j%(prop:maxcpus)s')], + command=['make', 'V=1', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], name='make', haltOnFailure=True)) systemtap_factory.addStep(steps.Compile( -- 2.31.1