From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from csb.redhat.com (gnu.wildebeest.org [45.83.234.184]) by sourceware.org (Postfix) with ESMTPS id B9C6B3858D32 for ; Sun, 15 Jan 2023 23:31:00 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.2 sourceware.org B9C6B3858D32 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=klomp.org Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=klomp.org Received: by csb.redhat.com (Postfix, from userid 10916) id D9C1DCB9DF; Mon, 16 Jan 2023 00:30:59 +0100 (CET) From: Mark Wielaard To: buildbot@sourceware.org Cc: Mark Wielaard Subject: [PATCH] Add addOutputSync renderer to add --output-sync=line Date: Mon, 16 Jan 2023 00:30:52 +0100 Message-Id: <20230115233052.170809-1-mark@klomp.org> X-Mailer: git-send-email 2.31.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-3038.1 required=5.0 tests=BAYES_00,GIT_PATCH_0,JMQ_SPF_NEUTRAL,KAM_DMARC_STATUS,SPF_HELO_NONE,SPF_PASS,TXREP autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org List-Id: Unfortunately adding an empty '' argument to make will try to make the '' target. So add a addOutputSync renderer function that augments the make command array with --output-sync=line if the worker hasn't set the property to the empty string. Unfortunately this doesn't work together with add_make_V1. https://docs.buildbot.net/latest/manual/configuration/properties.html#renderer --- builder/master.cfg | 134 ++++++++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 57 deletions(-) diff --git a/builder/master.cfg b/builder/master.cfg index d623924..11af8a0 100644 --- a/builder/master.cfg +++ b/builder/master.cfg @@ -965,33 +965,44 @@ getversion_step = steps.SetPropertyFromCommand( name="get version", command='grep PACKAGE_VERSION config.h | cut -f2 -d\\"', property="package_version") + # Allows a worker to override argument (with the empty string) # if make doesn't support --output-sync=line -make_output_sync = util.Property('make_output_sync_arg', - default='--output-sync=line', - defaultWhenFalse=False) +@util.renderer +def addOutputSync(props,command): + make_output_sync = props.getProperty('make_output_sync_arg', + default='--output-sync=line', + defaultWhenFalse=False) + if make_output_sync: + command.extend(make_output_sync) + + return command + make_step = steps.Compile( - command=["make", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", + util.Interpolate('-j%(prop:ncpus)s')]), name="make", haltOnFailure=True) make_check_step = steps.Test( - command=["make", make_output_sync, "check", util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "check", + util.Interpolate('-j%(prop:ncpus)s')]), name="make check", haltOnFailure=False, flunkOnFailure=True) # Same as make_check_step but with tests/test-suite.log recorded make_check_test_suite_step = steps.Test( - command=["make", make_output_sync, "check", util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "check", + util.Interpolate('-j%(prop:ncpus)s')]), name="make check", haltOnFailure=False, flunkOnFailure=True, logfiles={"test-suite.log": "tests/test-suite.log"}) make_distcheck_step = steps.Test( - command=["make", make_output_sync, "distcheck", - util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "distcheck", + util.Interpolate('-j%(prop:ncpus)s')]), name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600) # Same as make_distcheck_step but with tests/test-suite-log from _build # Needs package_version and package_name properties set make_distcheck_test_suite_step = steps.Test( - command=["make", make_output_sync, "distcheck", - util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "distcheck", + util.Interpolate('-j%(prop:ncpus)s')]), name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600, logfiles={"test-suite.log": @@ -1591,8 +1602,8 @@ elfutils_factory.addStep(elfutils_git_step) elfutils_factory.addStep(autoreconf_step) elfutils_factory.addStep(configure_step) elfutils_factory.addStep(getversion_step) -elfutils_factory.addStep(add_make_V1(make_step)) -elfutils_factory.addStep(add_make_V1(make_check_test_suite_step)) +elfutils_factory.addStep(make_step) +elfutils_factory.addStep(make_check_test_suite_step) elfutils_factory.addSteps(elfutils_upload_to_bunsen) elfutils_factory.addStep(make_distclean_step) @@ -1602,9 +1613,9 @@ elfutils_factory_distcheck.addStep(elfutils_git_step) elfutils_factory_distcheck.addStep(autoreconf_step) elfutils_factory_distcheck.addStep(configure_step) elfutils_factory_distcheck.addStep(getversion_step) -elfutils_factory_distcheck.addStep(add_make_V1(make_step)) -elfutils_factory_distcheck.addStep(add_make_V1(make_check_test_suite_step)) -elfutils_factory_distcheck.addStep(add_make_V1(make_distcheck_test_suite_step)) +elfutils_factory_distcheck.addStep(make_step) +elfutils_factory_distcheck.addStep(make_check_test_suite_step) +elfutils_factory_distcheck.addStep(make_distcheck_test_suite_step) elfutils_factory_distcheck.addSteps(elfutils_upload_to_bunsen) elfutils_factory_distcheck.addStep(make_distclean_step) @@ -1614,8 +1625,8 @@ elfutils_factory_val.addStep(elfutils_git_step) elfutils_factory_val.addStep(autoreconf_step) elfutils_factory_val.addStep(elfutils_configure_valgrind_step) elfutils_factory_val.addStep(getversion_step) -elfutils_factory_val.addStep(add_make_V1(make_step)) -elfutils_factory_val.addStep(add_make_V1(make_check_test_suite_step)) +elfutils_factory_val.addStep(make_step) +elfutils_factory_val.addStep(make_check_test_suite_step) elfutils_factory_val.addSteps(elfutils_upload_to_bunsen) elfutils_factory_val.addStep(make_distclean_step) @@ -1625,8 +1636,8 @@ elfutils_factory_mvu.addStep(elfutils_git_step) elfutils_factory_mvu.addStep(autoreconf_step) elfutils_factory_mvu.addStep(elfutils_configure_valgrind_undefined_step) elfutils_factory_mvu.addStep(getversion_step) -elfutils_factory_mvu.addStep(add_make_V1(make_step)) -elfutils_factory_mvu.addStep(add_make_V1(make_check_test_suite_step)) +elfutils_factory_mvu.addStep(make_step) +elfutils_factory_mvu.addStep(make_check_test_suite_step) elfutils_factory_mvu.addSteps(elfutils_upload_to_bunsen) elfutils_factory_mvu.addStep(make_distclean_step) @@ -1636,8 +1647,8 @@ elfutils_factory_mu.addStep(elfutils_git_step) elfutils_factory_mu.addStep(autoreconf_step) elfutils_factory_mu.addStep(elfutils_configure_undefined_step) elfutils_factory_mu.addStep(getversion_step) -elfutils_factory_mu.addStep(add_make_V1(make_step)) -elfutils_factory_mu.addStep(add_make_V1(make_check_test_suite_step)) +elfutils_factory_mu.addStep(make_step) +elfutils_factory_mu.addStep(make_check_test_suite_step) elfutils_factory_mu.addSteps(elfutils_upload_to_bunsen) elfutils_factory_mu.addStep(make_distclean_step) @@ -1647,8 +1658,8 @@ elfutils_factory_mua.addStep(elfutils_git_step) elfutils_factory_mua.addStep(autoreconf_step) elfutils_factory_mua.addStep(elfutils_configure_address_undefined_step) elfutils_factory_mua.addStep(getversion_step) -elfutils_factory_mua.addStep(add_make_V1(make_step)) -elfutils_factory_mua.addStep(add_make_V1(make_check_test_suite_step)) +elfutils_factory_mua.addStep(make_step) +elfutils_factory_mua.addStep(make_check_test_suite_step) elfutils_factory_mua.addSteps(elfutils_upload_to_bunsen) elfutils_factory_mua.addStep(make_distclean_step) @@ -1952,19 +1963,22 @@ gccrust_configure_bootstrap_step = steps.Configure( gccrust_make_step = steps.Compile( workdir='gccrs-build', - command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:ncpus)s')]), name='make', haltOnFailure=True) gccrust_make_bootstrap_step = steps.Compile( workdir='gccrs-build', - command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make', haltOnFailure=True) gccrust_check_step = steps.Compile( workdir='gccrs-build', - command=['make', make_output_sync, 'check-rust', util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(['make', 'check-rust', + util.Interpolate('-j%(prop:ncpus)s')]), name='make check', logfiles={ "rust.sum": "gcc/testsuite/rust/rust.sum", "rust.log": "gcc/testsuite/rust/rust.log" }, @@ -1972,7 +1986,8 @@ gccrust_check_step = steps.Compile( gccrust_check_bootstrap_step = steps.Compile( workdir='gccrs-build', - command=['make', make_output_sync, 'check-rust', util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', 'check-rust', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make check', logfiles={ "rust.sum": "gcc/testsuite/rust/rust.sum", "rust.log": "gcc/testsuite/rust/rust.log" }, @@ -2138,25 +2153,25 @@ binutils_step_configure_libctf = steps.Configure( haltOnFailure=True) binutils_step_compile = steps.Compile( workdir='binutils-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), - 'all-gas', 'all-ld', 'all-binutils', 'all-gold'], + 'all-gas', 'all-ld', 'all-binutils', 'all-gold']), name='make', haltOnFailure=True) binutils_step_compile_libctf = steps.Compile( workdir='binutils-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), 'all-gas', 'all-ld', 'all-binutils', 'all-gold', - 'all-libctf'], + 'all-libctf']), name='make', haltOnFailure=True) binutils_step_check = steps.Test( workdir='binutils-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), 'check-ld', 'check-gas', 'check-binutils', - 'check-libsframe'], + 'check-libsframe']), name='make check', logfiles={ "ld.sum": "ld/ld.sum", "ld.log": "ld/ld.log", @@ -2173,10 +2188,10 @@ binutils_steps_bunsen = bunsen_logfile_upload_cpio_steps( binutils_step_check_libctf = steps.Test( workdir='binutils-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), 'check-ld', 'check-gas', 'check-binutils', 'check-libctf', - 'check-libsframe'], + 'check-libsframe']), name='make check', logfiles={ "ld.sum": "ld/ld.sum", "ld.log": "ld/ld.log", @@ -2195,9 +2210,9 @@ binutils_steps_bunsen_libctf = bunsen_logfile_upload_cpio_steps( binutils_step_check_gas_binutils = steps.Test( workdir='binutils-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), - 'check-gas', 'check-binutils'], + 'check-gas', 'check-binutils']), name='make check', logfiles={ "gas.sum": "gas/testsuite/gas.sum", "gas.log": "gas/testsuite/gas.log", @@ -2500,9 +2515,9 @@ gdb_configure_32_step = steps.Configure( haltOnFailure=True) gdb_make_step = steps.Compile( workdir='gdb-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), - 'all-gdb', 'all-gdbserver'], + 'all-gdb', 'all-gdbserver']), name='make', haltOnFailure=True) gdb_make_clean_step = steps.ShellCommand( @@ -2901,9 +2916,9 @@ def binutils_gdb_factory_factory(runtestflags=None): name='configure', haltOnFailure=True)) binutils_gdb_factory.addStep(steps.Compile( - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), - '-k', 'all'], # for better logs, keep going + '-k', 'all']), # for better logs, keep going name='make', haltOnFailure=False, flunkOnFailure=True)) # build failure = bad # run without parallelism, to eliminate duplicate .log/.sum file uploads (from before merging) @@ -2996,9 +3011,9 @@ gcc_factory.addStep(steps.Configure( name='configure', haltOnFailure=True)) gcc_factory.addStep(steps.Compile( - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:ncpus)s'), - '-k', 'all', 'install'], # for better logs, keep going + '-k', 'all', 'install']), # for better logs, keep going name='make', haltOnFailure=False, flunkOnFailure=True)) # build failure = bad gcc_factory.addStep(steps.Test( @@ -3024,7 +3039,8 @@ gcc_build_factory.addStep(gcc_rm_build_step) gcc_build_factory.addStep(gcc_build_configure_step) gcc_build_factory.addStep(steps.Compile( workdir='gcc-build', - command=['make', make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:ncpus)s')]), name='make', haltOnFailure=True)) gcc_build_factory.addStep(gcc_rm_build_step) @@ -3153,14 +3169,16 @@ def gcc_full_build_factory_gen(extra_configure_arg=None): gcc_full_build_factory.addStep(gcc_full_build_configure_step) gcc_full_build_factory.addStep(steps.Compile( workdir='gcc-build', - command=['make', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make', haltOnFailure=True)) # We want parallelism to get through this as quickly as possible. # Even if that means bunsen gets some parallel/duplicate log files gcc_full_build_factory.addStep(steps.Test( workdir='gcc-build', - command=['make', 'check', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', 'check', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make check', haltOnFailure=False, flunkOnFailure=True)) gcc_full_build_factory.addSteps(bunsen_logfile_upload_cpio_steps( @@ -3221,32 +3239,32 @@ glibc_configure_step = steps.Configure( glibc_make_step = steps.Compile( workdir='glibc-build', - command=['make', make_output_sync, - util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make', haltOnFailure=False, flunkOnFailure=True) glibc_make_build_step = steps.Compile( workdir='glibc-build', - command=['make', make_output_sync, - util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(['make', + util.Interpolate('-j%(prop:ncpus)s')]), name='make', haltOnFailure=True) glibc_check_step = steps.Test( workdir='glibc-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', util.Interpolate('-j%(prop:maxcpus)s'), - 'check'], + 'check']), name='make check', haltOnFailure=False, flunkOnFailure=True) glibc_check_build_step = steps.Test( workdir='glibc-build', - command=['make', make_output_sync, + command=addOutputSync.withArgs(['make', 'subdirs=elf', util.Interpolate('-j%(prop:ncpus)s'), - 'check'], + 'check']), name='make check', haltOnFailure=False, flunkOnFailure=True) @@ -3387,11 +3405,12 @@ libabigail_git_step = steps.Git( name="git checkout", haltOnFailure=True) libabigail_make_check_test_suite_step = steps.Test( - command=["make", "check", "ENABLE_SLOW_TEST=yes", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "check", "ENABLE_SLOW_TEST=yes", util.Interpolate('-j%(prop:ncpus)s')]), name="make check", haltOnFailure=False, flunkOnFailure=True, logfiles={"test-suite.log": "tests/test-suite.log"}) libabigail_fast_distcheck = steps.Test( - command=["make", "distcheck-fast", make_output_sync, util.Interpolate('-j%(prop:ncpus)s')], + command=addOutputSync.withArgs(["make", "distcheck-fast", + util.Interpolate('-j%(prop:ncpus)s')]), name="make distcheck", haltOnFailure=False, flunkOnFailure=True, timeout=3600, logfiles={"test-suite.log": util.Interpolate('%(prop:package_name)s-%(prop:package_version)s/_build/sub/tests/test-suite.log')}) @@ -3849,7 +3868,8 @@ systemtap_factory.addStep(steps.Configure( haltOnFailure=True)) systemtap_factory.addStep(steps.Compile( workdir='stap-build', - command=['make', 'V=1', make_output_sync, util.Interpolate('-j%(prop:maxcpus)s')], + command=addOutputSync.withArgs(['make', 'V=1', + util.Interpolate('-j%(prop:maxcpus)s')]), name='make', haltOnFailure=True)) systemtap_factory.addStep(steps.Compile( -- 2.31.1