[PATCH 1/2] Add snapshots worker and elfutils coverage builder

[PATCH 1/2] Add snapshots worker and elfutils coverage builder

[Mark Wielaard]

Document the snapshots in SETUP and README_workers.

Add publish-snapshots.sh script to be run on snapshots builder cron.
Add snapshots worker.
Add lcov to Containerfile-debian-stable.
Add elfutils snapshots scheduler, coverage factory and builder.
---
 README_workers                                | 10 +++
 SETUP                                         |  3 +
 .../containers/Containerfile-debian-stable    |  2 +-
 builder/master.cfg                            | 71 +++++++++++++++++++
 publish-snapshots.sh                          | 51 +++++++++++++
 5 files changed, 136 insertions(+), 1 deletion(-)
 create mode 100644 publish-snapshots.sh

diff --git a/README_workers b/README_workers
index 76ed9c3..eb190a1 100644
--- a/README_workers
+++ b/README_workers
@@ -180,3 +180,13 @@ passwd:
       groups: [docker]
       ssh_authorized_keys:
         - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnB//uE66TX+nmLuox3ETfZ7dvjUl2mlwN4Q+107EWCuP05eKxcahyzwenmzODzzC8/hUVeUtuCJJyMW8+CpqCluHd2bEUT9WIzP1C/T22jBUbhY7zXLufSYfKmZe9zHNUizEbqbGAfR01D3jv3wis8JNOq/yopGLKJmAPd/Ye8XN5lsfAUTREIF+xygjxSb6SZPU9UvKlJmnwZoCN2Fd1u4IhSWBcD3dyZyirD80ZLGsSdOUmzX2Z2NaSI5tG6MxjYtQZbzmsG/Z0HY1Y/KiSolX989+7dMKMy2V911SkS6Cx9aVuSY+ig9Rq6GpfjGkBnHrNCCeVFe8PlYJl6DXZ
+
+Adding a snapshots worker
+-------------------------
+
+This is exactly the same as adding a container (latent) worker with
+max_builds=1.  The only difference is that it should be setup to run a
+webserver hosting the content of /var/snapshots and that the builder
+user (outside the container) should have a cronjob installed that runs
+the publish-snapshots.sh script that picks up the output from the
+build job from the shared/output folder.
diff --git a/SETUP b/SETUP
index c9f7005..6a0e560 100644
--- a/SETUP
+++ b/SETUP
@@ -146,3 +146,6 @@ part going into the docker_host builder user .ssh/authorized_keys. So
 the DockerLatentWorker can run docker system dial-stdio on the
 docker_host. Since the buildbot will use the ~/.ssh config make sure
 that the hosts are in .known_hosts (by logging in by hand once).
+
+The snapshots builder is just like a DockerLatentWorker.  See also
+'Adding a snapshots worker' in README_workers.
diff --git a/builder/containers/Containerfile-debian-stable b/builder/containers/Containerfile-debian-stable
index 4dace49..37f11f7 100644
--- a/builder/containers/Containerfile-debian-stable
+++ b/builder/containers/Containerfile-debian-stable
@@ -7,7 +7,7 @@ RUN apt-get update && \
       libisl-dev libmpc-dev libgmp-dev libmpfr-dev libgmp-dev \
       patch util-linux diffutils iproute2 libarchive-tools cpio rpm2cpio procps \
       coreutils make git autotools-dev autoconf dejagnu automake gettext bison flex \
-      gawk m4 pkg-config gettext autopoint autogen \
+      gawk m4 pkg-config gettext autopoint autogen lcov \
       bzip2 libbz2-dev xz-utils liblzma-dev gzip zlib1g-dev zstd libzstd-dev \
       libexpat-dev libncurses-dev \
       findutils file tar curl libarchive-tools libarchive-dev libcurl4-gnutls-dev \
diff --git a/builder/master.cfg b/builder/master.cfg
index ed8e7f8..29d9d1c 100644
--- a/builder/master.cfg
+++ b/builder/master.cfg
@@ -235,6 +235,18 @@ bbo1_2_worker = worker.DockerLatentWorker("bbo1-2",
                            properties={'ncpus': 8, 'maxcpus': 16});
 c['workers'].append(bbo1_2_worker)
 
+# Another OSUOSL machine for producing coverage, doc, manual etc. snapshots
+snapshots_worker = worker.DockerLatentWorker("snapshots",
+                        None,
+                        masterFQDN="builder.sourceware.org",
+                        docker_host="ssh://builder@snapshots.sourceware.org",
+                        dockerfile=util.Interpolate('%(prop:container-file)s'),
+                        volumes=["/home/builder/shared:/home/builder/shared"],
+                        build_wait_timeout=0,
+                        max_builds=1,
+                        properties={'ncpus': 4, 'maxcpus': 4});
+c['workers'].append(snapshots_worker)
+
 vm_workers = ['bb1-1', 'bb1-2', 'bb2-1', 'bb2-2', 'bb3',
               'bbo1-1', 'bbo1-2']
 big_vm_workers = ['bb3', 'bbo1-2']
@@ -510,6 +522,16 @@ elfutils_try_scheduler = schedulers.AnyBranchScheduler(
                       "elfutils-try-opensuseleap-x86_64"])
 c['schedulers'].append(elfutils_try_scheduler)
 
+elfutils_snapshot_scheduler = schedulers.Periodic(
+        name="elfutils-snapshots",
+        change_filter=util.ChangeFilter(project="elfutils",
+                                        branch="master"),
+        branch="master",
+        periodicBuildTimer=15*60, # 15 minutes in seconds
+        onlyIfChanged=True,
+        reason="elfutils periodic project master branch snapshot",
+        builderNames=["elfutils-snapshots-coverage"])
+c['schedulers'].append(elfutils_snapshot_scheduler)
 
 libabigail_scheduler = schedulers.SingleBranchScheduler(
         name="libabigail",
@@ -1012,6 +1034,10 @@ make_check_step = steps.Test(
         command=addOutputSync.withArgs(["make", "check",
             util.Interpolate('-j%(prop:ncpus)s')]),
         name="make check", haltOnFailure=False, flunkOnFailure=True)
+# make check explicitly not using -j
+make_single_check_step = steps.Test(
+        command=["make", "check"],
+        name="make check")
 # Same as make_check_step but with tests/test-suite.log recorded
 make_check_test_suite_step = steps.Test(
         command=addOutputSync.withArgs(["make", "check",
@@ -1063,6 +1089,12 @@ def add_make_V1(step):
         newstep.command.append("V=1")
         return newstep
 
+# generic snapshot steps.
+# Wait up to 1.5 minutes for output dir to become free
+wait_snapshots_output_ready_step = steps.ShellCommand(
+        command='for s in 1 2 3; do if [ -d "/home/builder/shared/output" ]; then sleep 30; fi; done; [ ! -d "/home/builder/shared/output" ]',
+        haltOnFailure=True)
+
 # Bunsen steps: extract globby files and some build properties metadata, upload them.
 # files[]: which a list of ([glob, glob, glob], destdir)  tuples.
 
@@ -1643,6 +1675,24 @@ elfutils_configure_address_undefined_step = steps.Configure(
                  "--enable-sanitize-address"],
         name="configure", haltOnFailure=True,
         logfiles={"config.log": "config.log"})
+# configure for coverage (gcov)
+elfutils_configure_coverage_step = steps.Configure(
+        command=["./configure",
+                 "--enable-maintainer-mode",
+                 "--enable-gcov"],
+        name="configure", haltOnFailure=True,
+        logfiles={"config.log": "config.log"})
+elfutils_make_coverage_step = steps.Test(
+        command=["make", "coverage"],
+        name="make check")
+
+elfutils_create_output_step = steps.ShellCommand(
+        name="create output",
+        command=["mv", "coverage", "/home/builder/shared/output"])
+elfutils_create_publish_file_step = steps.ShellCommand(
+        name="create publish file",
+        command=["echo", "elfutils/coverage",
+                 ">", "/home/builder/shared/publish"])
 
 # Upload automake test results, autoconf config.log, in both build and dist-build trees
 elfutils_upload_to_bunsen = bunsen_logfile_upload_cpio_steps(
@@ -1716,6 +1766,17 @@ elfutils_factory_mua.addStep(make_check_test_suite_step)
 elfutils_factory_mua.addSteps(elfutils_upload_to_bunsen)
 elfutils_factory_mua.addStep(make_distclean_step)
 
+elfutils_factory_coverage = util.BuildFactory()
+elfutils_factory_coverage.addStep(elfutils_git_step)
+elfutils_factory_coverage.addStep(autoreconf_step)
+elfutils_factory_coverage.addStep(elfutils_configure_coverage_step)
+elfutils_factory_coverage.addStep(make_step)
+elfutils_factory_coverage.addStep(make_single_check_step)
+elfutils_factory_coverage.addStep(elfutils_make_coverage_step)
+elfutils_factory_coverage.addStep(wait_snapshots_output_ready_step)
+elfutils_factory_coverage.addStep(elfutils_create_output_step)
+elfutils_factory_coverage.addStep(elfutils_create_publish_file_step)
+
 elfutils_centos_x86_64_builder = util.BuilderConfig(
         name="elfutils-centos-x86_64",
         workernames=["centos-x86_64"],
@@ -1966,6 +2027,16 @@ elfutils_try_debian_testing_x86_64_builder = util.BuilderConfig(
         factory=elfutils_factory_distcheck)
 c['builders'].append(elfutils_try_debian_testing_x86_64_builder)
 
+elfutils_snapshots_coverage_builder = util.BuilderConfig(
+        name="elfutils-snapshots-coverage",
+        collapseRequests=True,
+        properties={'container-file':
+                    readContainerFile('debian-stable')},
+        workernames="snapshots",
+        tags=["elfutils-snapshots", "coverage"],
+        factory=elfutils_factory_coverage)
+c['builders'].append(elfutils_snapshots_coverage_builder)
+
 # gccrust build steps, factory and builders
 # All steps are custom because of the workdir settings
 
diff --git a/publish-snapshots.sh b/publish-snapshots.sh
new file mode 100644
index 0000000..2453723
--- /dev/null
+++ b/publish-snapshots.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -e
+
+SNAPSHOTSDIR=/var/snapshots
+
+PUBLISHDIR=$HOME/shared
+PUBLISHOUTPUT=output
+PUBLISHFILE=publish
+
+# unique date string
+PUBLISHDATE=$(date -u +%Y-%m-%d_%H-%M_%s)
+
+# where we keep the files while publishing
+PUBLISHINGDIR=$HOME/publishing
+
+# Make sure only one process is publishing
+LOCKFILE=$HOME/publish_lock
+exec 200>$LOCKFILE
+flock -n 200 || exit 1
+
+if [ -f $PUBLISHDIR/$PUBLISHFILE ]; then
+  # Make sure publishing dir is clean
+  rm -rf $PUBLISHINGDIR
+  mkdir -p $PUBLISHINGDIR
+
+  # Move output and publish file for processing
+  mv $PUBLISHDIR/$PUBLISHOUTPUT $PUBLISHDIR/$PUBLISHFILE $PUBLISHINGDIR
+
+  # Where to publish, sanity check given subdir
+  read PUBLISHSUBDIR < $PUBLISHINGDIR/$PUBLISHFILE
+  if echo $PUBLISHSUBDIR | grep -E '^[0-9a-zA-Z][0-9a-zA-Z/_-]{1,64}$'; then
+    mkdir -p $SNAPSHOTSDIR/$PUBLISHSUBDIR
+
+    # We need a copy to get the selinux context correct
+    cp -r $PUBLISHINGDIR/$PUBLISHOUTPUT $SNAPSHOTSDIR/$PUBLISHSUBDIR/$PUBLISHDATE
+    # Update latest link
+    cd $SNAPSHOTSDIR/$PUBLISHSUBDIR
+    rm -f latest
+    ln -s $PUBLISHDATE latest
+
+    # touch all parent dirs to update last modified time
+    pdirs=$PUBLISHSUBDIR
+    cd $SNAPSHOTSDIR
+    while [ $pdirs != "." ]; do touch $pdirs; pdirs=$(dirname $pdirs); done
+  else
+    echo BAD subdir
+  fi
+
+  rm -rf $PUBLISHINGDIR
+fi
-- 
2.31.1

[PATCH 2/2] Fix elfutils make_coverage and create_publish step typos

[Mark Wielaard]

---
 builder/master.cfg | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/builder/master.cfg b/builder/master.cfg
index 29d9d1c..ccf1745 100644
--- a/builder/master.cfg
+++ b/builder/master.cfg
@@ -1684,15 +1684,14 @@ elfutils_configure_coverage_step = steps.Configure(
         logfiles={"config.log": "config.log"})
 elfutils_make_coverage_step = steps.Test(
         command=["make", "coverage"],
-        name="make check")
+        name="make coverage")
 
 elfutils_create_output_step = steps.ShellCommand(
         name="create output",
         command=["mv", "coverage", "/home/builder/shared/output"])
 elfutils_create_publish_file_step = steps.ShellCommand(
         name="create publish file",
-        command=["echo", "elfutils/coverage",
-                 ">", "/home/builder/shared/publish"])
+        command="echo elfutils/coverage > /home/builder/shared/publish")
 
 # Upload automake test results, autoconf config.log, in both build and dist-build trees
 elfutils_upload_to_bunsen = bunsen_logfile_upload_cpio_steps(
-- 
2.31.1