From 7e67984ba110dad6bafea0b22a56584494f11c56 Mon Sep 17 00:00:00 2001 From: Mark Wielaard Date: Mon, 25 Apr 2022 11:02:30 +0200 Subject: [PATCH] Add binutils as project and documentation on adding a new project --- README | 90 +++++++++++++++++++++++++++++++++++++++++----- TODO | 7 ++-- builder/master.cfg | 90 ++++++++++++++++++++++++++++++++++++++++++++++ htdocs/index.html | 5 +++ 4 files changed, 180 insertions(+), 12 deletions(-) diff --git a/README b/README index 3dda795..3afc8f2 100644 --- a/README +++ b/README @@ -8,13 +8,87 @@ builder/master.cfg: buildbot configuration htdocs: document root of https://builder.sourceware.org/ - How to add a new project - ... Add a change_source - ... Add a scheduler - ... Add a builder, see below - ... Add a mail notifier - -- How to add a new builder - ... + - Define a repourl to be used in the GitPoller and any Build Steps + project_repourl = 'git://sourceware.org/git/project.git' + + - Define a change source, e.g. a GitPoller and add it + project_gitpoller = changes.GitPoller(repourl=project_repourl, + branches=['main'], + pollInterval=227, + project='project') + c['change_source'].append(project_gitpoller) + + You can make it listen to multiple branches, use a pollInterval + not used by any other poller. Note that one repourl can only be + used once, if projects share a git repo the schedulers for the + project can select which files in the repo belong to which project. + See for example the gccrs or binutils-gdb schedulers. + + - Add a scheduler + project_scheduler = schedulers.SingleBranchScheduler( + name="project", + change_filter=util.ChangeFilter(project="project", + branch="master"), + builderNames=["project-distro-arch", ...]) + c['schedulers'].append(project_scheduler) + + You might want to add a isImportantFile filter if not + all file changes must trigger a build (see gccrust or binutils + for examples). + + - Add a builder, factory and build steps see below... + + - Add a mail notifier + In general it is a good idea to only sent email for all + the builders from a scheduler combined based on the scheduler + project tag and only for changes (from good to bad or bad to good): + + generator_project = reporters.BuildSetStatusGenerator( + mode=('change',), tags=['project']) + mn_project = reporters.MailNotifier( + fromaddr="builder@sourceware.org", + sendToInterestedUsers=True, # The patch committers. + extraRecipients=['project-list@sourceware.org'], + generators=[generator_project]) + c['services'].append(mn_project) + +- How to add a new builder/fectory/steps + Asssuming there is already a project defined as above with a + repourl, change-source, scheduler and (mail) notifiers + + The scheduler lists one or more builder names, each builder + defines a factory that generates the build steps, one or more + workers to execute the steps and defines tags which can be used + by reporters. + + project_distro_arch_builder = util.BuilderConfig( + name="binutils-debian-amd64", + workernames=["distro-arch"], + tags=["project", "distro", "arch"], + factory=project_factory) + c['builders'].append(project__builder) + + There are a couple of common steps that might be useful to + reuse in you factory. A factory using common steps look something + like: + + project_factory = util.BuildFactory() + project_factory.addStep(steps.Git( + repourl=project_repourl, + mode='full', method='fresh', + name="git checkout", + haltOnFailure=True)) + project_factory.addStep(autoreconf_step) + project_factory.addStep(configure_step) + project_factory.addStep(make_step) + project_factory.addStep(make_check_test_suite_step) + + But often you need to tweak the steps a little. + + Finally add the new builder to htdocs/index.html under the right project + + distro-arch
+ - How to add a new worker ... @@ -118,4 +192,4 @@ Commit the changes. NOTE for actually upgrading the virtual-env on sourceware you will need a shell account and run /sourceware/builder/bin/buildbot-upgrade.sh as user builder. This will shutdown the builder, do the above updates based on the -buildbot.requirements file, upgrade-master and restart the builder. \ No newline at end of file +buildbot.requirements file, upgrade-master and restart the builder. diff --git a/TODO b/TODO index 495ac99..fe82827 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,10 @@ -- More documentation on adding builders and workers. - Can use the debian-ppc64/fedora-ppc64 and binutils/gdb builders projects - as examples. +- More documentation on adding workers. + Can use the debian-ppc64/fedora-ppc64 as examples. - We can deprecate the fedora-ppc64 worker if the debian-ppc64 works out. fedora-ppc64 is slow, no working ccache and runs unsupported Fedora 22. -- Add a binutils builder (start small). +- Generate the "dashboard" in htdocs/index.html automatically. - Integrate parts of the old gdb buildbot Old GDB buildbot documentation and config files diff --git a/builder/master.cfg b/builder/master.cfg index 5e8b544..e66e60a 100644 --- a/builder/master.cfg +++ b/builder/master.cfg @@ -201,6 +201,13 @@ gccrust_gitpoller = changes.GitPoller(repourl=gccrust_repourl, project='gccrust') c['change_source'].append(gccrust_gitpoller) +binutils_gdb_repourl='git://sourceware.org/git/binutils-gdb.git' +binutils_gdb_gitpoller = changes.GitPoller(repourl=binutils_gdb_repourl, + branches=['main'], + pollInterval=227, + project='binutils-gdb') +c['change_source'].append(binutils_gdb_gitpoller) + ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. @@ -338,6 +345,29 @@ gccrust_scheduler = schedulers.SingleBranchScheduler( "gccrust-debian-ppc64"]) c['schedulers'].append(gccrust_scheduler) +# Only trigger scheduler for changes to binutils (or deps) +binutils_files = ["bfd/", + "binutils/", "gas/", "ld/", + "gold/", "elfcpp/", + "include/", "libiberty/", "opcodes/", + "configure", "Makefile.in"] + +def binutilsImportant(change): + for file in change.files: + for pattern in binutils_files: + match = re.match(pattern, file) + if match: + return True + return False + +binutils_scheduler = schedulers.SingleBranchScheduler( + name="binutils", + change_filter=util.ChangeFilter(project="binutils", + branch="master"), + fileIsImportant=binutilsImportant, + builderNames=["binutils-debian-amd64"]) +c['schedulers'].append(binutils_scheduler) + ####### BUILDERS # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: @@ -1021,6 +1051,56 @@ gccrust_debian_ppc64_builder = util.BuilderConfig( factory=gccrust_factory) c['builders'].append(gccrust_debian_ppc64_builder) +# binutils build steps, factory and builders + +binutils_factory = util.BuildFactory() +binutils_factory.addStep(steps.Git( + workdir='binutils-gdb', + repourl=binutils_gdb_repourl, + mode='full', method='fresh', + name="git checkout", + haltOnFailure=True)) +binutils_factory.addStep(steps.ShellCommand( + command=["rm", "-rf", + util.Interpolate ("%(prop:builddir)s/binutils-build")], + name="rm -rf binutils-build", + haltOnFailure=True)) +binutils_factory.addStep(steps.Configure( + workdir='binutils-build', + command=['../binutils-gdb/configure', + '--enable-gold', + '--enable-shared', + '--enable-target=all'], + name='configure', + haltOnFailure=True)) +binutils_factory.addStep(steps.Compile( + workdir='binutils-build', + command=['make', + util.Interpolate('-j%(prop:ncpus)s'), + 'all-gas', 'all-ld', 'all-binutils', 'all-gold'], + name='make', + haltOnFailure=True)) +binutils_factory.addStep(steps.Compile( + workdir='binutils-build', + command=['make', + util.Interpolate('-j%(prop:ncpus)s'), + 'check-ld', 'check-gas', 'check-binutils'], + name='make check', + logfiles={ "ld.sum": "ld/ld.sum", + "ld.log": "ld/ld.log", + "gas.sum": "gas/testsuite/gas.sum", + "gas.log": "gas/testsuite/gas.log", + "binutils.sum": "binutils/binutils.sum", + "binutils.log": "binutils/binutils.log" }, + haltOnFailure=True)) + +binutils_debian_amd64_builder = util.BuilderConfig( + name="binutils-debian-amd64", + workernames=["debian-amd64"], + tags=["binutils", "debian", "x86_64"], + factory=binutils_factory) +c['builders'].append(binutils_debian_amd64_builder) + # libabigail build steps, factory and builders libabigail_package_name_step = steps.SetProperty( @@ -1309,6 +1389,16 @@ mn_gccrust = reporters.MailNotifier( generators=[generator_gccrust]) c['services'].append(mn_gccrust) +# Report for the whole binutils tagged builder set +generator_binutils = reporters.BuildSetStatusGenerator( + mode=('change',), tags=['binutils']) +mn_binutils = reporters.MailNotifier( + fromaddr="builder@sourceware.org", + sendToInterestedUsers=True, + extraRecipients=['binutils@sourceware.org'], + generators=[generator_binutils]) +c['services'].append(mn_binutils) + # Report for the whole libabigail tagged builder set generator_libabigail = reporters.BuildSetStatusGenerator( mode=('change',), tags=['libabigail']) diff --git a/htdocs/index.html b/htdocs/index.html index 9a2d64e..9d12fb9 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -30,6 +30,11 @@

Projects

+ + + +
binutilsdebian-amd64
+
bzip2 centos-x86_64
-- 2.30.2