[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20200220-9-g44c3b47

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20200220-9-g44c3b47

[Jon TURNEY]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=44c3b473d6365332a6a388f9c6da94e30e9c7dea

commit 44c3b473d6365332a6a388f9c6da94e30e9c7dea
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Mar 9 21:40:41 2020 +0000

    Don't include hostname in email subject if it's uninteresting

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4bdb8295f8c92128dbc65405eb71341866a0cd5d

commit 4bdb8295f8c92128dbc65405eb71341866a0cd5d
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Mar 2 12:09:50 2020 +0000

    Avoid an exception if no previous .ini file exists

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4fd321e9367aac6d49f8e3d344a1eab560c0fcf7

commit 4fd321e9367aac6d49f8e3d344a1eab560c0fcf7
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Feb 27 12:11:34 2020 +0000

    Add option to specify gpg key(s) to use for signing setup.ini
    
    If the option is absent, gpg will use it's default (the first key found
    in the secret keyring, in the absence of specific configuration).

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=77d127441c4ee2c3174be1a79e3f0f8d1523dab7

commit 77d127441c4ee2c3174be1a79e3f0f8d1523dab7
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Mar 1 20:03:21 2020 +0000

    Log output from compression and signing subprocesses

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9b473bd66e10194e0d59782362b1a7ba90bee083

commit 9b473bd66e10194e0d59782362b1a7ba90bee083
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Mar 1 19:58:21 2020 +0000

    Just compute the compressed filename once

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f411dd352a92d728f50626e4af5132a54bd6bf94

commit f411dd352a92d728f50626e4af5132a54bd6bf94
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Feb 24 17:49:38 2020 +0000

    Make --force work more sensibly when daemonized
    
    If daemonized, --force should force regeneration of static content in
    htdocs initially (in case the generation code has changed), but that
    static content should only by updated as needed on subsequent loops.
    
    Use open_amifc() so that mtimes aren't changed by forced regeneration of
    static content, if it hasn't actually changed.
    
    (This also atomically updates that content, just in case someone happens
    to read it while it's being updated)
    
    Regenerating package listing pages is expensive (since we have to read
    all of every tar archive to do so), so only do that with '--force --force'.


Diff:
---
 calm/calm.py     | 28 ++++++++++++++++++++-------
 calm/pkg2html.py | 34 +++++++++++++++-----------------
 calm/utils.py    | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 25 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index dd3e184..3fb0e62 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -385,6 +385,10 @@ def do_output(args, state):
     # update packages listings
     # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
     pkg2html.update_package_listings(args, state.packages)
+    # if we are daemonized, allow force regeneration of static content in htdocs
+    # initially (in case the generation code has changed), but update that
+    # static content only as needed on subsequent loops
+    args.force = 0
 
     update_json = False
 
@@ -434,7 +438,8 @@ def do_output(args, state):
                     os.remove(tmpfile.name)
                 else:
                     # make a backup of the current setup.ini
-                    shutil.copy2(inifile, inifile + '.bak')
+                    if os.path.exists(inifile):
+                        shutil.copy2(inifile, inifile + '.bak')
 
                     # replace setup.ini
                     logging.info("moving %s to %s" % (tmpfile.name, inifile))
@@ -443,17 +448,19 @@ def do_output(args, state):
 
                     # compress and re-sign
                     for ext in ['.ini', '.bz2', '.xz']:
+                        extfile = os.path.join(basedir, 'setup' + ext)
                         try:
-                            os.remove(os.path.join(basedir, 'setup' + ext + '.sig'))
+                            os.remove(extfile + '.sig')
                         except FileNotFoundError:
                             pass
 
                         if ext == '.bz2':
-                            os.system('/usr/bin/bzip2 <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext))
+                            utils.system('/usr/bin/bzip2 <%s >%s' % (inifile, extfile))
                         elif ext == '.xz':
-                            os.system('/usr/bin/xz -6e <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext))
+                            utils.system('/usr/bin/xz -6e <%s >%s' % (inifile, extfile))
 
-                        os.system('/usr/bin/gpg --batch --yes -b </dev/null ' + os.path.join(basedir, 'setup' + ext))
+                        keys = ' '.join(['-u' + k for k in args.keys])
+                        utils.system('/usr/bin/gpg ' + keys + ' --batch --yes -b ' + extfile)
 
                     # arrange for checksums to be recomputed
                     for sumfile in ['md5.sum', 'sha512.sum']:
@@ -658,9 +665,10 @@ def main():
     parser = argparse.ArgumentParser(description='Upset replacement')
     parser.add_argument('-d', '--daemon', action='store', nargs='?', const=pidfile_default, help="daemonize (PIDFILE defaults to " + pidfile_default + ")", metavar='PIDFILE')
     parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS')
-    parser.add_argument('--force', action='store_true', help="overwrite existing files")
+    parser.add_argument('--force', action='count', help="force regeneration of static htdocs content", default=0)
     parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
     parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default)
+    parser.add_argument('--key', action='append', metavar='KEYID', help="key to use to sign setup.ini", default=[], dest='keys')
     parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default)
     parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default)
     parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default)
@@ -679,7 +687,13 @@ def main():
         args.email = args.email.split(',')
 
     state = CalmState()
-    state.subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1])
+
+    host = os.uname()[1]
+    if 'sourceware.org' not in host:
+        host = ' from ' + host
+    else:
+        host = ''
+    state.subject = 'calm%s: cygwin package upload report%s' % (' [dry-run]' if args.dryrun else '', host)
 
     status = 0
     if args.daemon:
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 110f03a..b6c5490 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -163,9 +163,8 @@ def update_package_listings(args, packages):
         # if listing files were added or removed, or it doesn't already exist,
         # or force, update the summary
         if p in update_summary or not os.path.exists(summary) or args.force:
-            logging.debug('writing %s' % summary)
             if not args.dryrun:
-                with open(summary, 'w') as f:
+                with utils.open_amifc(summary) as f:
                     os.fchmod(f.fileno(), 0o755)
 
                     pos = arch_packages(packages, p)
@@ -305,9 +304,17 @@ def update_package_listings(args, packages):
 #
 def write_packages_inc(args, packages, name, kind, includer):
     packages_inc = os.path.join(args.htdocs, name)
-    logging.debug('writing %s' % packages_inc)
     if not args.dryrun:
-        with open(packages_inc, 'w') as index:
+
+        def touch_including(changed):
+            if changed:
+                # touch the including file for the benefit of 'XBitHack full'
+                package_list = os.path.join(args.htdocs, includer)
+                if os.path.exists(package_list):
+                    logging.info("touching %s for the benefit of 'XBitHack full'" % (package_list))
+                    utils.touch(package_list)
+
+        with utils.open_amifc(packages_inc, cb=touch_including) as index:
             os.fchmod(index.fileno(), 0o644)
 
             # This list contains all packages in any arch. Source packages
@@ -371,11 +378,6 @@ def write_packages_inc(args, packages, name, kind, includer):
 
             print('</table>', file=index)
 
-        # touch the including file for the benefit of 'XBitHack full'
-        package_list = os.path.join(args.htdocs, includer)
-        if os.path.exists(package_list):
-            utils.touch(package_list)
-
 
 def write_arch_listing(args, packages, arch):
     update_summary = set()
@@ -393,9 +395,8 @@ def write_arch_listing(args, packages, arch):
 
     htaccess = os.path.join(base, '.htaccess')
     if not os.path.exists(htaccess) or args.force:
-        logging.debug('writing %s' % htaccess)
         if not args.dryrun:
-            with open(htaccess, 'w') as f:
+            with utils.open_amifc(htaccess) as f:
 
                 print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (arch),
                       file=f)
@@ -413,9 +414,8 @@ def write_arch_listing(args, packages, arch):
 
         htaccess = os.path.join(dir, '.htaccess')
         if not os.path.exists(htaccess):
-            logging.debug('writing %s' % htaccess)
             if not args.dryrun or args.force:
-                with open(htaccess, 'w') as f:
+                with utils.open_amifc(htaccess) as f:
                     # We used to allow access to the directory listing as a
                     # crude way of listing the versions of the package available
                     # for which file lists were available. Redirect that index
@@ -444,16 +444,14 @@ def write_arch_listing(args, packages, arch):
             fver = re.sub(r'\.tar.*$', '', tn)
             listing = os.path.join(dir, fver)
 
-            # ... if it doesn't already exist, or force
-            if not os.path.exists(listing) or args.force:
-
-                logging.debug('writing %s' % listing)
+            # ... if it doesn't already exist, or --force --force
+            if not os.path.exists(listing) or (args.force > 1):
 
                 if not args.dryrun:
                     # versions are being added, so summary needs updating
                     update_summary.add(p)
 
-                    with open(listing, 'w') as f:
+                    with utils.open_amifc(listing) as f:
                         bv = packages[p].best_version
                         header = p + ": " + sdesc(packages[p], bv)
 
diff --git a/calm/utils.py b/calm/utils.py
index d6d6e1c..14bd517 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -25,8 +25,12 @@
 # utility functions
 #
 
+import filecmp
 import logging
 import os
+import subprocess
+
+from contextlib import contextmanager
 
 
 #
@@ -51,3 +55,58 @@ def makedirs(name):
         os.makedirs(name, exist_ok=True)
     except FileExistsError:
         pass
+
+
+#
+# a wrapper for open() which:
+#
+# - atomically changes the file contents (atomic)
+# - only touches the mtime if the file contents have changed (move-if-changed)
+#
+@contextmanager
+def open_amifc(filepath, mode='w', cb=None):
+    tmppath = filepath + '~'
+    while os.path.isfile(tmppath):
+        tmppath += '~'
+
+    try:
+        with open(tmppath, mode) as file:
+            logging.debug('writing %s for move-if-changed' % (tmppath))
+            yield file
+
+        changed = not os.path.exists(filepath) or not filecmp.cmp(tmppath, filepath, shallow=False)
+        if changed:
+            logging.info("writing %s" % (filepath))
+            os.rename(tmppath, filepath)
+        else:
+            logging.debug("unchanged %s" % (filepath))
+    finally:
+        try:
+            os.remove(tmppath)
+        except OSError:
+            pass
+
+    # notify callback if file was changed or not
+    if cb:
+        cb(changed)
+
+
+#
+# run a subprocess, logging it's output
+#
+# N.B. because we use shell=True, args should be a string to be supplied to 'sh
+# -c', not a list.
+#
+def system(args):
+    logging.debug(args)
+    try:
+        output = subprocess.check_output(args, shell=True,
+                                         stdin=subprocess.DEVNULL,
+                                         stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        for l in e.output.decode().splitlines():
+            logging.warning(l)
+        logging.warning('%s exited %d' % (args.split()[0], e.returncode))
+    else:
+        for l in output.decode().splitlines():
+            logging.info(l)