From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 114283 invoked by alias); 23 Mar 2016 11:28:04 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 113717 invoked by uid 9795); 23 Mar 2016 11:28:04 -0000 Date: Wed, 23 Mar 2016 11:28:00 -0000 Message-ID: <20160323112804.113610.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. b26ae5b83bad5b1c12ef7d59377278059f0ede5d X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: bedb9881c8d19b7a537c59ff10446f56f9abf3e4 X-Git-Newrev: b26ae5b83bad5b1c12ef7d59377278059f0ede5d X-SW-Source: 2016-q1/txt/msg00072.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b26ae5b83bad5b1c12ef7d59377278059f0ede5d commit b26ae5b83bad5b1c12ef7d59377278059f0ede5d Author: Jon Turney Date: Wed Mar 23 10:26:55 2016 +0000 Ignore packages marked as OBSOLETE in cygwin-pkg-maint Update tests appropriately https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=872274f5b91ad4cf309918d25b9c02ec7e5b08fc commit 872274f5b91ad4cf309918d25b9c02ec7e5b08fc Author: Jon Turney Date: Mon Mar 21 20:37:40 2016 +0000 Remove shell script, do everything in python Have I mentioned lately how much I hate shell scripts... v2: Backup existing .ini file, rather than overwriting it Add an option to control where we look for setup executable Diff: --- TODO | 1 - calm-cron-script | 36 --------------- calm.py | 86 ++++++++++++++++++++++++++++++++++--- extract-setup-version | 66 ---------------------------- maintainers.py | 11 +++- setup_exe.py | 73 +++++++++++++++++++++++++++++++ testdata/pkglist/cygwin-pkg-maint | 2 +- testdata/pkglist/expected | 2 +- tests.py | 2 +- 9 files changed, 164 insertions(+), 115 deletions(-) diff --git a/TODO b/TODO index d782f11..246aac2 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,3 @@ -* cronscript should be in python * per-package-version hint files (e.g. named .hint, or /.setup.hint inside tar file) * more than 2 versions possible (and automatically vault 'old' versions, where we have some mechanism to explicity say what is old) * enable .xz compressed setup.ini (setup.exe checks for it and presumably can handle it) diff --git a/calm-cron-script b/calm-cron-script deleted file mode 100755 index 554c868..0000000 --- a/calm-cron-script +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# - -# a unicode locale is required -export LANG=en_US.UTF-8 - -FTP=/var/ftp/pub/cygwin -SRCDIR=`/usr/bin/dirname "$0"` - -for arch in x86_64 x86 ; do - TMPFILE=`/bin/mktemp -t setup.ini.${arch}XXX` - INIFILE=${FTP}/${arch}/setup.ini - - SETUP_VERSION=`${SRCDIR}/extract-setup-version /www/sourceware/htdocs/cygwin/setup-${arch}.exe` - - ${SRCDIR}/calm --inifile ${TMPFILE} --setup-version ${SETUP_VERSION} --arch=${arch} --email - if [ $? -eq 0 ] - then - # if changed in more than timestamp - /usr/bin/diff -I^setup-timestamp -w -B -q ${INIFILE} ${TMPFILE} >/dev/null - if [ $? -eq 1 ] - then - # update setup.ini - /bin/mv ${TMPFILE} ${INIFILE} - # compress - /usr/bin/bzip2 <${INIFILE} >${INIFILE/%.ini/.bz2} - # re-sign - /bin/rm -f ${INIFILE}.sig ${INIFILE/%.ini/.bz2}.sig - /usr/bin/gpg --batch --yes -b ${INIFILE} - /usr/bin/gpg --batch --yes -b ${INIFILE/%.ini/.bz2} - # arrange for checksums to be recomputed - /bin/rm -f ${FTP}/${arch}/md5.sum ${FTP}/${arch}/sha512.sum - fi - fi - /bin/rm -f ${TMPFILE} -done diff --git a/calm.py b/calm.py index d4568d4..6cbc690 100755 --- a/calm.py +++ b/calm.py @@ -48,13 +48,16 @@ import argparse import logging import os +import shutil import sys +import tempfile from buffering_smtp_handler import mail_logs import common_constants import maintainers import package import pkg2html +import setup_exe import uploads @@ -62,7 +65,7 @@ import uploads # # -def main(args): +def process_arch(args): details = '%s%s' % (args.arch, ',dry-run' if args.dryrun else '') # send one email per run to leads @@ -76,7 +79,7 @@ def main(args): # validate the package set if not package.validate_packages(args, packages): logging.error("existing package set has errors, not processing uploads or writing setup.ini") - return 1 + return False # read maintainer list mlist = maintainers.Maintainer.read(args) @@ -133,7 +136,79 @@ def main(args): # XXX: perhaps we need a --[no]listing command line option to disable this from being run? pkg2html.update_package_listings(args, packages) - return 0 + return True + + +# +# +# + +def main(args): + # for each arch + for arch in common_constants.ARCHES: + logging.info("processing arch %s" % (arch)) + + args.arch = arch + args.setup_version = setup_exe.extract_version(os.path.join(args.setupdir, 'setup-' + args.arch + '.exe')) + logging.info("setup version is '%s'" % (args.setup_version)) + + basedir = os.path.join(args.rel_area, args.arch) + inifile = os.path.join(basedir, 'setup.ini') + + # write setup.ini to a temporary file + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + args.inifile = tmpfile.name + + changed = False + if not process_arch(args): + # generating setup.ini failed + pass + elif not os.path.exists(inifile): + # if the setup.ini file doesn't exist yet + logging.warning('no existing %s' % (inifile)) + changed = True + else: + # or, if it's changed in more than timestamp + status = os.system('/usr/bin/diff -I^setup-timestamp -w -B -q %s %s >/dev/null' % (inifile, tmpfile.name)) + logging.info('diff exit status %d' % (status)) + if (status >> 8) == 1: + changed = True + + # then update setup.ini + if changed: + if args.dryrun: + logging.warning("not moving %s to %s, due to --dry-run" % (tmpfile.name, inifile)) + os.remove(tmpfile.name) + else: + # make a backup of the current setup.ini + shutil.copy2(inifile, inifile + '.bak') + + # replace setup.ini + logging.warning("moving %s to %s" % (tmpfile.name, inifile)) + os.rename(tmpfile.name, inifile) + + # compress and re-sign + for ext in ['.ini', '.bz2']: + try: + os.remove(os.path.join(basedir, 'setup' + ext + '.sig')) + except FileNotFoundError: + pass + + if ext == '.bz2': + os.system('/usr/bin/bzip2 <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext)) + + os.system('/usr/bin/gpg --batch --yes -b ' + os.path.join(basedir, 'setup' + ext)) + + # arrange for checksums to be recomputed + for sumfile in ['md5.sum', 'sha512.sum']: + try: + os.remove(os.path.join(basedir, sumfile)) + except FileNotFoundError: + pass + else: + logging.info("removing %s, unchanged %s" % (tmpfile.name, inifile)) + os.remove(tmpfile.name) + # # @@ -145,22 +220,21 @@ if __name__ == "__main__": orphanmaint_default = common_constants.ORPHANMAINT pkglist_default = common_constants.PKGMAINT relarea_default = common_constants.FTP + setupdir_default = common_constants.HTDOCS vault_default = common_constants.VAULT logdir_default = '/sourceware/cygwin-staging/logs' parser = argparse.ArgumentParser(description='Upset replacement') - parser.add_argument('--arch', action='store', required=True, choices=common_constants.ARCHES) parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS') parser.add_argument('--force', action='store_true', help="overwrite existing files") parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default) parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default) - parser.add_argument('--inifile', '-u', action='store', help='output filename', required=True) parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default) parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default) parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') - parser.add_argument('--setup-version', action='store', metavar='VERSION', help='value for setup-version key') + parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default) parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', help="don't do anything") parser.add_argument('--vault', action='store', metavar='DIR', help="vault directory (default: " + vault_default + ")", default=vault_default, dest='vault') parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output') diff --git a/extract-setup-version b/extract-setup-version deleted file mode 100755 index 23eac1c..0000000 --- a/extract-setup-version +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2015 Jon Turney -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -# -# extract version from setup executable -# - -import argparse -import os -import re - - -# -# the setup binary contains a '%%% setup-version ' string identifying the -# version, but it's hard to read that when it's UPX packed. -# -# so instead, we expect the exe file to be a symlink to a file which encodes -# the version in it's name -# -# XXX: possibly we could work around this by placing the version string into -# a string resource and using upx's --keep-resource flag to keep that resource -# uncompressed -# - -def main(fn): - # canonicalize the pathname - fn = os.path.realpath(fn) - - match = re.search(r'setup-([\d\.]+).x86', fn) - if match: - print(match.group(1)) - return 0 - else: - return 1 - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Extract version from setup executable') - parser.add_argument('exe', action='store', nargs='?', metavar='filename', help='executable file') - (args) = parser.parse_args() - - if args.exe: - exit(main(args.exe)) - - parser.print_help() - -# XXX:maybe this could make the filename from /www/sourceware/htdocs/cygwin and arch? diff --git a/maintainers.py b/maintainers.py index d0afb0f..8bc9ec7 100644 --- a/maintainers.py +++ b/maintainers.py @@ -101,13 +101,18 @@ class Maintainer(object): pkg = match.group(1) m = match.group(2) + # if maintainer starts with a word in all caps, just use that + m = re.sub(r'^([A-Z]+)\b.*$', r'\1', m) + + # ignore packages marked as 'OBSOLETE' + if m == 'OBSOLETE': + continue + # orphaned packages get the default maintainer if we have # one, otherwise are assigned to 'ORPHANED' - if m.startswith('ORPHANED'): + if m == 'ORPHANED': if orphanMaint is not None: m = orphanMaint - else: - m = 'ORPHANED' # joint maintainers are separated by '/' for name in m.split('/'): diff --git a/setup_exe.py b/setup_exe.py new file mode 100755 index 0000000..d9d85be --- /dev/null +++ b/setup_exe.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2015 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +# +# extract version from setup executable +# + +import argparse +import os +import re + + +# +# the setup binary contains a '%%% setup-version ' string identifying the +# version, but it's hard to read that when it's UPX packed. +# +# so instead, we expect the exe file to be a symlink to a file which encodes +# the version in it's name +# +# XXX: possibly we could work around this by placing the version string into +# a string resource and using upx's --keep-resource flag to keep that resource +# uncompressed +# + +def extract_version(fn): + # check the file exists + if not os.path.exists(fn): + raise FileNotFoundError + + # canonicalize the pathname + fn = os.path.realpath(fn) + + match = re.search(r'setup-([\d\.]+).x86', fn) + if match: + return match.group(1) + else: + return None + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Extract version from setup executable') + parser.add_argument('exe', action='store', nargs='?', metavar='filename', help='executable file') + (args) = parser.parse_args() + + if args.exe: + v = extract_version(args.exe) + if v: + print(v) + exit(0) + exit(1) + + parser.print_help() + +# XXX:maybe this could make the filename from /www/sourceware/htdocs/cygwin and arch? diff --git a/testdata/pkglist/cygwin-pkg-maint b/testdata/pkglist/cygwin-pkg-maint index d2375af..41b4175 100644 --- a/testdata/pkglist/cygwin-pkg-maint +++ b/testdata/pkglist/cygwin-pkg-maint @@ -1744,7 +1744,7 @@ presentproto Yaakov Selkowitz prison Yaakov Selkowitz pristine-tar Jari Aalto procmail Daniel Boland -procps Corinna Vinschen +procps OBSOLETE (Corinna Vinschen) proj Marco Atzeri protobuf ORPHANED (Reini Urban) pscan Jari Aalto diff --git a/testdata/pkglist/expected b/testdata/pkglist/expected index 4e8db92..b340aa2 100644 --- a/testdata/pkglist/expected +++ b/testdata/pkglist/expected @@ -9,7 +9,7 @@ 'Chris LeBlanc': maintainers.Maintainer('Chris LeBlanc', [], ['python-h5py', 'python3-h5py']), 'Christian Franke': maintainers.Maintainer('Christian Franke', [], ['busybox', 'cdrkit', 'ddrescue', 'hostname', 'isomaster', 'ncdu', 'postfix', 'smartmontools']), 'Christian Kellermann': maintainers.Maintainer('Christian Kellermann', [], ['chicken']), - 'Corinna Vinschen': maintainers.Maintainer('Corinna Vinschen', [], ['attr', 'base-cygwin', 'catgets', 'cpio', 'crypt', 'csih', 'cygrunsrv', 'cygwin', 'cygwin-devel', 'editrights', 'email', 'eventlog', 'fetchmail', 'file', 'gawk', 'gdb', 'getent', 'ipc-utils', 'libedit', 'login', 'lynx', 'mt', 'nc', 'nc6', 'openssh', 'openssl', 'patch', 'pl', 'procps', 'psmisc', 'rebase', 'recode', 'rlwrap', 'rng-tools', 'robots', 'sed', 'ssmtp', 'syslog-ng', 'tcsh', 'tin', 'tzcode', 'windows-default-manifest', 'xinetd']), + 'Corinna Vinschen': maintainers.Maintainer('Corinna Vinschen', [], ['attr', 'base-cygwin', 'catgets', 'cpio', 'crypt', 'csih', 'cygrunsrv', 'cygwin', 'cygwin-devel', 'editrights', 'email', 'eventlog', 'fetchmail', 'file', 'gawk', 'gdb', 'getent', 'ipc-utils', 'libedit', 'login', 'lynx', 'mt', 'nc', 'nc6', 'openssh', 'openssl', 'patch', 'pl', 'psmisc', 'rebase', 'recode', 'rlwrap', 'rng-tools', 'robots', 'sed', 'ssmtp', 'syslog-ng', 'tcsh', 'tin', 'tzcode', 'windows-default-manifest', 'xinetd']), 'Damien Doligez': maintainers.Maintainer('Damien Doligez', [], ['ocaml']), 'Daniel Boland': maintainers.Maintainer('Daniel Boland', [], ['libfakesu', 'procmail', 'sendmail']), 'Dave Kilroy': maintainers.Maintainer('Dave Kilroy', [], ['chere']), diff --git a/tests.py b/tests.py index 6f0ff19..3252ac6 100755 --- a/tests.py +++ b/tests.py @@ -262,7 +262,7 @@ class TestMain(unittest.TestCase): for (f, t) in ready_fns: os.system('touch %s "%s"' % (t, f)) - self.assertEqual(calm.main(args), 0) + self.assertEqual(calm.process_arch(args), True) for d in ['rel_area', 'homedir', 'htdocs', 'vault']: with self.subTest(directory=d):