From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 6D504395307B; Mon, 9 Mar 2020 23:23:06 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 6D504395307B DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1583796186; bh=Eu4r979DPGvQK6rTl+8ZxPHvJkrRuOFfYCZ3Fe2zRVM=; h=To:Subject:Date:From:From; b=azmlrnp2vz/zjiNg16FQdbIX0ziuY0XkYjPCWnFGp2LfHY2Q1Iq4/n4uJjYsZTSdR 3CyGJDEaOFJkj64fb2wtsRPbd5kkE6vBx7mRcXuqXfoYtBBJg/ZVWh7eRdOkh/JCkB pi8Zj/ATsFzMPHdxF8F9AYsSKKVVC8urptdXax7g= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20200220-9-g44c3b47 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 3cc08991e70dc07e5e64cb4295b4328f991159b0 X-Git-Newrev: 44c3b473d6365332a6a388f9c6da94e30e9c7dea Message-Id: <20200309232306.6D504395307B@sourceware.org> Date: Mon, 9 Mar 2020 23:23:06 +0000 (GMT) From: Jon TURNEY X-BeenThere: cygwin-apps-cvs@cygwin.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Cygwin-apps-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 09 Mar 2020 23:23:06 -0000 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=44c3b473d6365332a6a388f9c6da94e30e9c7dea commit 44c3b473d6365332a6a388f9c6da94e30e9c7dea Author: Jon Turney Date: Mon Mar 9 21:40:41 2020 +0000 Don't include hostname in email subject if it's uninteresting https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4bdb8295f8c92128dbc65405eb71341866a0cd5d commit 4bdb8295f8c92128dbc65405eb71341866a0cd5d Author: Jon Turney Date: Mon Mar 2 12:09:50 2020 +0000 Avoid an exception if no previous .ini file exists https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4fd321e9367aac6d49f8e3d344a1eab560c0fcf7 commit 4fd321e9367aac6d49f8e3d344a1eab560c0fcf7 Author: Jon Turney Date: Thu Feb 27 12:11:34 2020 +0000 Add option to specify gpg key(s) to use for signing setup.ini If the option is absent, gpg will use it's default (the first key found in the secret keyring, in the absence of specific configuration). https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=77d127441c4ee2c3174be1a79e3f0f8d1523dab7 commit 77d127441c4ee2c3174be1a79e3f0f8d1523dab7 Author: Jon Turney Date: Sun Mar 1 20:03:21 2020 +0000 Log output from compression and signing subprocesses https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9b473bd66e10194e0d59782362b1a7ba90bee083 commit 9b473bd66e10194e0d59782362b1a7ba90bee083 Author: Jon Turney Date: Sun Mar 1 19:58:21 2020 +0000 Just compute the compressed filename once https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f411dd352a92d728f50626e4af5132a54bd6bf94 commit f411dd352a92d728f50626e4af5132a54bd6bf94 Author: Jon Turney Date: Mon Feb 24 17:49:38 2020 +0000 Make --force work more sensibly when daemonized If daemonized, --force should force regeneration of static content in htdocs initially (in case the generation code has changed), but that static content should only by updated as needed on subsequent loops. Use open_amifc() so that mtimes aren't changed by forced regeneration of static content, if it hasn't actually changed. (This also atomically updates that content, just in case someone happens to read it while it's being updated) Regenerating package listing pages is expensive (since we have to read all of every tar archive to do so), so only do that with '--force --force'. Diff: --- calm/calm.py | 28 ++++++++++++++++++++------- calm/pkg2html.py | 34 +++++++++++++++----------------- calm/utils.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 25 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index dd3e184..3fb0e62 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -385,6 +385,10 @@ def do_output(args, state): # update packages listings # XXX: perhaps we need a --[no]listing command line option to disable this from being run? pkg2html.update_package_listings(args, state.packages) + # if we are daemonized, allow force regeneration of static content in htdocs + # initially (in case the generation code has changed), but update that + # static content only as needed on subsequent loops + args.force = 0 update_json = False @@ -434,7 +438,8 @@ def do_output(args, state): os.remove(tmpfile.name) else: # make a backup of the current setup.ini - shutil.copy2(inifile, inifile + '.bak') + if os.path.exists(inifile): + shutil.copy2(inifile, inifile + '.bak') # replace setup.ini logging.info("moving %s to %s" % (tmpfile.name, inifile)) @@ -443,17 +448,19 @@ def do_output(args, state): # compress and re-sign for ext in ['.ini', '.bz2', '.xz']: + extfile = os.path.join(basedir, 'setup' + ext) try: - os.remove(os.path.join(basedir, 'setup' + ext + '.sig')) + os.remove(extfile + '.sig') except FileNotFoundError: pass if ext == '.bz2': - os.system('/usr/bin/bzip2 <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext)) + utils.system('/usr/bin/bzip2 <%s >%s' % (inifile, extfile)) elif ext == '.xz': - os.system('/usr/bin/xz -6e <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext)) + utils.system('/usr/bin/xz -6e <%s >%s' % (inifile, extfile)) - os.system('/usr/bin/gpg --batch --yes -b ', file=index) - # touch the including file for the benefit of 'XBitHack full' - package_list = os.path.join(args.htdocs, includer) - if os.path.exists(package_list): - utils.touch(package_list) - def write_arch_listing(args, packages, arch): update_summary = set() @@ -393,9 +395,8 @@ def write_arch_listing(args, packages, arch): htaccess = os.path.join(base, '.htaccess') if not os.path.exists(htaccess) or args.force: - logging.debug('writing %s' % htaccess) if not args.dryrun: - with open(htaccess, 'w') as f: + with utils.open_amifc(htaccess) as f: print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (arch), file=f) @@ -413,9 +414,8 @@ def write_arch_listing(args, packages, arch): htaccess = os.path.join(dir, '.htaccess') if not os.path.exists(htaccess): - logging.debug('writing %s' % htaccess) if not args.dryrun or args.force: - with open(htaccess, 'w') as f: + with utils.open_amifc(htaccess) as f: # We used to allow access to the directory listing as a # crude way of listing the versions of the package available # for which file lists were available. Redirect that index @@ -444,16 +444,14 @@ def write_arch_listing(args, packages, arch): fver = re.sub(r'\.tar.*$', '', tn) listing = os.path.join(dir, fver) - # ... if it doesn't already exist, or force - if not os.path.exists(listing) or args.force: - - logging.debug('writing %s' % listing) + # ... if it doesn't already exist, or --force --force + if not os.path.exists(listing) or (args.force > 1): if not args.dryrun: # versions are being added, so summary needs updating update_summary.add(p) - with open(listing, 'w') as f: + with utils.open_amifc(listing) as f: bv = packages[p].best_version header = p + ": " + sdesc(packages[p], bv) diff --git a/calm/utils.py b/calm/utils.py index d6d6e1c..14bd517 100644 --- a/calm/utils.py +++ b/calm/utils.py @@ -25,8 +25,12 @@ # utility functions # +import filecmp import logging import os +import subprocess + +from contextlib import contextmanager # @@ -51,3 +55,58 @@ def makedirs(name): os.makedirs(name, exist_ok=True) except FileExistsError: pass + + +# +# a wrapper for open() which: +# +# - atomically changes the file contents (atomic) +# - only touches the mtime if the file contents have changed (move-if-changed) +# +@contextmanager +def open_amifc(filepath, mode='w', cb=None): + tmppath = filepath + '~' + while os.path.isfile(tmppath): + tmppath += '~' + + try: + with open(tmppath, mode) as file: + logging.debug('writing %s for move-if-changed' % (tmppath)) + yield file + + changed = not os.path.exists(filepath) or not filecmp.cmp(tmppath, filepath, shallow=False) + if changed: + logging.info("writing %s" % (filepath)) + os.rename(tmppath, filepath) + else: + logging.debug("unchanged %s" % (filepath)) + finally: + try: + os.remove(tmppath) + except OSError: + pass + + # notify callback if file was changed or not + if cb: + cb(changed) + + +# +# run a subprocess, logging it's output +# +# N.B. because we use shell=True, args should be a string to be supplied to 'sh +# -c', not a list. +# +def system(args): + logging.debug(args) + try: + output = subprocess.check_output(args, shell=True, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + for l in e.output.decode().splitlines(): + logging.warning(l) + logging.warning('%s exited %d' % (args.split()[0], e.returncode)) + else: + for l in output.decode().splitlines(): + logging.info(l)