From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 129460 invoked by alias); 12 Apr 2016 13:21:06 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 129430 invoked by uid 9795); 12 Apr 2016 13:21:05 -0000 Date: Tue, 12 Apr 2016 13:21:00 -0000 Message-ID: <20160412132105.129394.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 26dbb50c4d88c58f52aa3464cd1eed3eb4160831 X-Git-Newrev: 2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12 X-SW-Source: 2016-q2/txt/msg00002.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12 commit 2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12 Author: Jon Turney Date: Tue Apr 12 11:07:52 2016 +0100 Add a queue interface to a package validator Place a copy of each package upload into a filesystem queue, to be processed by something else which does checks on the package https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=996236245f9fbaf4d665cd16dc413e6cc69ede2a commit 996236245f9fbaf4d665cd16dc413e6cc69ede2a Author: Jon Turney Date: Mon Apr 11 22:06:31 2016 +0100 In read_package(), log as an error if treated as an error due to strict Also fix typos of 'warning' for 'warnings' https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9ee4bb2c74cd77fdf444c709e3ab8afbfeadbf44 commit 9ee4bb2c74cd77fdf444c709e3ab8afbfeadbf44 Author: Jon Turney Date: Mon Apr 4 14:30:16 2016 +0100 Tolerate random files in the homes directory Tolerate and ignore random files in the homes directory Add appropriate test https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ba55e84fa5b660e292b16d310376485431cc424d commit ba55e84fa5b660e292b16d310376485431cc424d Author: Jon Turney Date: Mon Apr 11 22:13:01 2016 +0100 Extract version correctly from .lzma files ... if we had any Diff: --- .travis.yml | 2 +- calm.py | 6 +++ maintainers.py | 3 + package.py | 14 ++++--- queue.py | 70 ++++++++++++++++++++++++++++++++ testdata/process_arch/homedir.expected | 2 +- uploads.py | 23 ++++++++++ 7 files changed, 112 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72a7ef5..e6c1776 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python sudo: false python: - "3.4" -install: "pip install pep8" +install: "pip install pep8 dirq" script: - ./pep8 - ./tests.py diff --git a/calm.py b/calm.py index a383d46..38e8cec 100755 --- a/calm.py +++ b/calm.py @@ -59,6 +59,7 @@ import common_constants import maintainers import package import pkg2html +import queue import setup_exe import uploads @@ -111,6 +112,9 @@ def process_arch(args): logging.debug("nothing to do for maintainer %s" % (name)) continue + # queue for source package validator + queue.add(args, to_relarea, os.path.join(m.homedir(), args.arch)) + # merge package set merged_packages = package.merge(packages, mpackages) @@ -243,6 +247,7 @@ if __name__ == "__main__": setupdir_default = common_constants.HTDOCS vault_default = common_constants.VAULT logdir_default = '/sourceware/cygwin-staging/logs' + queuedir_default = '/sourceware/cygwin-staging/queue' parser = argparse.ArgumentParser(description='Upset replacement') parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS') @@ -252,6 +257,7 @@ if __name__ == "__main__": parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default) parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default) parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) + parser.add_argument('--queuedir', action='store', metavar='DIR', help="queue directory (default: '" + queuedir_default + "')", default=queuedir_default) parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default) diff --git a/maintainers.py b/maintainers.py index 8bc9ec7..a078376 100644 --- a/maintainers.py +++ b/maintainers.py @@ -71,6 +71,9 @@ class Maintainer(object): self._homedirs = homedirs for n in os.listdir(homedirs): + if not os.path.isdir(os.path.join(homedirs, n)): + continue + m = Maintainer._find(mlist, n) for e in ['!email', '!mail']: diff --git a/package.py b/package.py index 8e6a405..cddbdb5 100755 --- a/package.py +++ b/package.py @@ -101,6 +101,7 @@ def sha512_file(fn, block_size=256*128): # read a single package # def read_package(packages, basedir, dirpath, files, strict=False): + strict_lvl = logging.ERROR if strict else logging.WARNING relpath = os.path.relpath(dirpath, basedir) warnings = False @@ -161,8 +162,8 @@ def read_package(packages, basedir, dirpath, files, strict=False): # start with a number match = re.match(r'^' + re.escape(p) + '-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.tar\.(bz2|gz|lzma|xz)$', f) if not match: - logging.warning("tar file '%s' in package '%s' doesn't follow naming convention" % (f, p)) - warning = True + logging.lvl(strict_lvl, "tar file '%s' in package '%s' doesn't follow naming convention" % (f, p)) + warnings = True else: # historically, V can contain a '-' (since we can use the fact # we already know P to split unambiguously), but this is a bad @@ -187,8 +188,8 @@ def read_package(packages, basedir, dirpath, files, strict=False): # warn about unexpected files, including tarfiles which don't match the # package name if files: - logging.warning("unexpected files in %s: %s" % (p, ', '.join(files))) - warning = True + logging.log(strict_lvl, "unexpected files in %s: %s" % (p, ', '.join(files))) + warnings = True packages[p].hints = hints packages[p].tars = tars @@ -214,7 +215,7 @@ def read_package(packages, basedir, dirpath, files, strict=False): if colon: package_basename = re.sub(r'^lib(.*?)(|-devel|\d*)$', r'\1', p) if package_basename.upper().startswith(colon.group(1).upper()): - logging.warning("package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2)))) + logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2)))) warnings = True elif (len(files) > 0) and (relpath.count(os.path.sep) > 0): @@ -222,6 +223,7 @@ def read_package(packages, basedir, dirpath, files, strict=False): if strict: return warnings + return False @@ -301,7 +303,7 @@ def validate_packages(args, packages): # extract just the version part from tar filename v = re.sub(r'^' + re.escape(p) + '-', '', t) - v = re.sub(r'(-src|)\.tar\.(xz|bz2|gz)$', '', v) + v = re.sub(r'(-src|)\.tar\.(bz2|gz|lzma|xz)$', '', v) # for each version, a package can contain at most one source tar # file and at most one install tar file. warn if we have too many diff --git a/queue.py b/queue.py new file mode 100644 index 0000000..ed4f89a --- /dev/null +++ b/queue.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2016 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +from dirq.QueueSimple import QueueSimple +import logging +import os +import re + +import uploads + +QUEUE = 'package_queue' + + +# +# +# + +def add(args, movelist, fromdir): + if not hasattr(args, 'queuedir'): + return + + queue_root = os.path.join(args.queuedir, 'dirq') + upload_root = os.path.join(args.queuedir, 'uploads') + + dirq = QueueSimple(os.path.join(queue_root, QUEUE)) + + # clean up empty directories + dirq.purge() + os.system('find %s -depth -mindepth 1 -type d -empty -delete' % upload_root) + + # are there any source packages in the filelist? + srcpkgs = [] + for p in movelist: + for f in movelist[p]: + if re.search(r'-src.tar.(bz2|gz|lzma|xz)$', f): + srcpkgs.append(os.path.join(args.arch, p, f)) + + # if so... + # + # XXX: really this should break things up into the set of files for each + # source file + if len(srcpkgs) >= 1: + # keep all the files for comparison + uploads.copy(args, movelist, fromdir, os.path.join(upload_root, args.arch)) + + # queue any srcpkgs + for p in srcpkgs: + if not args.dryrun: + logging.debug("queuing source package %s for validation" % (p)) + dirq.add(p) diff --git a/testdata/homes/an_unexpected_file b/testdata/homes/an_unexpected_file new file mode 100644 index 0000000..e69de29 diff --git a/testdata/process_arch/homedir.expected b/testdata/process_arch/homedir.expected index 39fb742..091867f 100644 --- a/testdata/process_arch/homedir.expected +++ b/testdata/process_arch/homedir.expected @@ -1,4 +1,4 @@ -{'.': [], +{'.': ['an_unexpected_file'], 'Blooey McFooey': [], 'Blooey McFooey/x86': [], 'Blooey McFooey/x86/release': [], diff --git a/uploads.py b/uploads.py index 0df5d39..6909d76 100644 --- a/uploads.py +++ b/uploads.py @@ -30,6 +30,7 @@ import filecmp import os import logging import re +import shutil import time import package @@ -236,3 +237,25 @@ def move_to_relarea(m, args, movelist): def move_to_vault(args, movelist): move(args, movelist, os.path.join(args.rel_area, args.arch), os.path.join(args.vault, args.arch)) + + +# +# +# + +def copy(args, movelist, fromdir, todir): + for p in sorted(movelist): + logging.debug("mkdir %s" % os.path.join(todir, p)) + if not args.dryrun: + try: + os.makedirs(os.path.join(todir, p), exist_ok=True) + except FileExistsError: + pass + logging.debug("copy from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p))) + for f in sorted(movelist[p]): + if os.path.exists(os.path.join(fromdir, p, f)): + logging.debug("%s" % (f)) + if not args.dryrun: + shutil.copy2(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) + else: + logging.error("%s can't be copied as it doesn't exist" % (f))