From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id A6F033858C52; Thu, 19 Jan 2023 16:26:08 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org A6F033858C52 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1674145568; bh=T+/zjM2PwcM7uWo9t2KZ5hwfwjLERm/JNQ3qy3hNXeg=; h=To:Subject:Date:From:From; b=watQuxM3hNAlz1grgcuqSal1QAqkcpN/0z0qJUrHWuuGDun+0itv13jEsA3dv4BmP sm9VL0Qp7ddegDnAiwdsMmvRnspCeqKEs+lxSeZ3EnK5fGL1DRuH/SFbx9BhTTmRq7 7uHhOr+EMlaXPYC18F++Yo/TEOaV2IZ8FOFZfCaI= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20221205-28-g87009cd X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 99e4a07a5b8747bf7abf45367d9a3ea0c9aed97e X-Git-Newrev: 87009cdc1b35968c61317dfe5375f5126ec44918 Message-Id: <20230119162608.A6F033858C52@sourceware.org> Date: Thu, 19 Jan 2023 16:26:08 +0000 (GMT) From: Jon Turney List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=87009cdc1b35968c61317dfe5375f5126ec44918 commit 87009cdc1b35968c61317dfe5375f5126ec44918 Author: Jon Turney Date: Thu Jan 19 13:39:20 2023 +0000 Don't add duplicate license keys Don't add a custom keys to license_expression's database, if it's already present. Also drop unneeded 'XVIEW' license. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d77b94da78034219f9b5d89ecfe4fc24562cd344 commit d77b94da78034219f9b5d89ecfe4fc24562cd344 Author: Jon Turney Date: Thu Jan 19 13:13:17 2023 +0000 Propagate any error reading packages through mksetupini At the moment, calm.process_relarea() is expected to keep going, although we might later fail due to package set validation problemss. (In particular tests just assume that we ignore various problematic packages, although we don't actually check that they are being rejected with the epxected problem) https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ad6b47c27b1113abacdfebf7234d38b6524606cd commit ad6b47c27b1113abacdfebf7234d38b6524606cd Author: Jon Turney Date: Thu Jan 19 11:50:43 2023 +0000 Relax trusted maintainer restrictions Rename orphanmaint -> trustedmaint Also revise and relax logic so it's more consistent: trusted maintainers can do these things via a shell, so don't stop doing them more easily via calm. Drop convulted "add trusted maintainers as maintainers of orphaned packages so they can upload them", and just check directly against trusted maintainer list to determine if an upload is permitted. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0939d5bd86f4ad757dc230256e39246d35db1743 commit 0939d5bd86f4ad757dc230256e39246d35db1743 Author: Jon Turney Date: Mon Jan 16 14:03:58 2023 +0000 Add 'calm-tool vault' https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3280517c030ac2706deada7fad6d15f9b602ff80 commit 3280517c030ac2706deada7fad6d15f9b602ff80 Author: Jon Turney Date: Tue Jan 17 12:55:32 2023 +0000 Remove obsolete 'mkmaintdir' tool We removed all the !packages files some time ago. maintainer-keys/add now takes care of ensuring the appropriate directories exist when a maintainer is added. combine identifies defunct maintainers, using a different criteria, but similarly doesn't do any clean up. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=12f3fd73653116c5b8601e0dd0e44bb5ef36cdd7 commit 12f3fd73653116c5b8601e0dd0e44bb5ef36cdd7 Author: Jon Turney Date: Mon Jan 16 17:24:27 2023 +0000 Only allow requires: of packages which currently exist In f3a2daab817a, we accidentally allowed packages to depend on a package which has been removed, instead of just applying that to obsoletes. Fix that so we check that requires: contains packages which actually exist! Also clean up no longer required 'splitchar' Fixes: f3a2daab817a ("Persistently record all package names") https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bb40e56925ea39a75b03aa205f4a783ea3b1f7ab commit bb40e56925ea39a75b03aa205f4a783ea3b1f7ab Author: Jon Turney Date: Wed Jan 18 14:52:28 2023 +0000 Validate character set used by the package version, V Just as package name, only allow alphanumerics and '-._+' Warn about other characters, just as we already warn about '-', and add an exception for the one existing package which breaks these rules (with a ~). Also: '._+' should be allowed in R https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=96fdd81dca85221cc5b3b63e36f1f0fc3566bfc6 commit 96fdd81dca85221cc5b3b63e36f1f0fc3566bfc6 Author: Jon Turney Date: Fri Jan 13 13:23:49 2023 +0000 Check for forbidden hyphen-digit sequence in package name Diff: --- calm/calm.py | 17 ++++-- calm/common_constants.py | 11 +++- calm/db.py | 33 +++++++++++ calm/hint.py | 4 +- calm/maintainers.py | 14 ++--- calm/mkgitoliteconf.py | 4 +- calm/mkmaintdir | 141 ----------------------------------------------- calm/mksetupini.py | 6 +- calm/package.py | 65 +++++++++++++++------- calm/past_mistakes.py | 8 +-- calm/pkg2html.py | 2 +- calm/tool_util.py | 65 ++++++++++++++++++++++ calm/untest.py | 26 ++------- calm/uploads.py | 4 +- calm/vault.py | 62 +++++++++++++++++++++ test/test_calm.py | 6 +- 16 files changed, 255 insertions(+), 213 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index dc66926..b6e7e6e 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -103,7 +103,7 @@ def process_relarea(args, state): # read the package list for each arch for arch in common_constants.ARCHES: logging.debug("reading existing packages for arch %s" % (arch)) - packages[arch] = package.read_packages(args.rel_area, arch) + packages[arch], _ = package.read_packages(args.rel_area, arch) state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: @@ -144,7 +144,7 @@ def process_relarea(args, state): def process_uploads(args, state): # read maintainer list - mlist = maintainers.read(args, getattr(args, 'orphanmaint', None)) + mlist = maintainers.read(args) # make the list of all packages all_packages = maintainers.all_packages(mlist) @@ -314,11 +314,13 @@ def remove_stale_packages(args, packages, state): to_vault['noarch'] = MoveList() to_vault['src'] = MoveList() + vault_requests = db.vault_requests(args) + for arch in common_constants.ARCHES: logging.debug("checking for stale packages for arch %s" % (arch)) # find stale packages - to_vault[arch] = package.stale_packages(packages[arch]) + to_vault[arch] = package.stale_packages(packages[arch], vault_requests) # remove stale packages from package set to_vault[arch].map(lambda p, f: package.delete(packages[arch], p, f)) @@ -654,6 +656,11 @@ def mail_cb(state, loghandler): # send each maintainer mail containing log entries caused by their actions, # or pertaining to their packages + # + # XXX: prev_maint=False here is a kind of wrong: it prevents the previous + # maintainer of an orphaned package from getting mails about it being + # altered by a trusted maintainer, but also stops them getting mails if the + # do something themselves... mlist = maintainers.read(state.args, prev_maint=False) for m in mlist.values(): email = m.email @@ -711,7 +718,7 @@ def main(): htdocs_default = os.path.join(common_constants.HTDOCS, 'packages') homedir_default = common_constants.HOMEDIR stagingdir_default = common_constants.STAGINGDIR - orphanmaint_default = common_constants.ORPHANMAINT + trustedmaint_default = common_constants.TRUSTEDMAINT pidfile_default = '/sourceware/cygwin-staging/calm.pid' pkglist_default = common_constants.PKGMAINT relarea_default = common_constants.FTP @@ -727,7 +734,7 @@ def main(): parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default) parser.add_argument('--key', action='append', metavar='KEYID', help="key to use to sign setup.ini", default=[], dest='keys') parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default) - parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default) + parser.add_argument('--trustedmaint', action='store', metavar='NAMES', help="trusted package maintainers (default: '" + trustedmaint_default + "')", default=trustedmaint_default) parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') diff --git a/calm/common_constants.py b/calm/common_constants.py index cc43009..fb7cdc3 100644 --- a/calm/common_constants.py +++ b/calm/common_constants.py @@ -44,17 +44,22 @@ EMAILS = ','.join(list(map(lambda m: m[0] + '@' + m[1], zip(['corinna', 'Stromek # every email we send is bcc'd to these addresses ALWAYS_BCC = 'jturney@sourceware.org' -# these maintainers can upload orphaned packages as well +# these maintainers are 'trusted' +# +# they can: +# - git push to any package repo +# - upload any package +# - untest any package +# - vault any package # # (these people have sourceware shell access and cygwin group membership, so # they can do whatever they like directly, anyhow) -ORPHANMAINT = '/'.join([ +TRUSTEDMAINT = '/'.join([ 'Corinna Vinschen', 'Eric Blake', 'Jon Turney', 'Ken Brown', 'Marco Atzeri', - 'Yaakov Selkowitz', ]) # architectures we support diff --git a/calm/db.py b/calm/db.py index c1efbcf..b7f7040 100644 --- a/calm/db.py +++ b/calm/db.py @@ -42,6 +42,12 @@ def connect(args): conn.execute('''CREATE TABLE IF NOT EXISTS historic_package_names (name TEXT NOT NULL PRIMARY KEY )''') + + conn.execute('''CREATE TABLE IF NOT EXISTS vault_requests + (srcpackage TEXT NOT NULL, + vr TEXT NOT NULL + )''') + conn.commit() return conn @@ -71,3 +77,30 @@ def update_package_names(args, packages): # - names which the removed package provide:d # - other packages which might provide: the name of a removed package return (historic_names - current_names) + + +# +# vault requests made via 'calm-tool vault' +# +def vault_requests(args): + requests = {} + + with connect(args) as conn: + conn.row_factory = sqlite3.Row + + cur = conn.execute("SELECT * FROM vault_requests") + for row in cur.fetchall(): + spkg = row['srcpackage'] + if spkg not in requests: + requests[spkg] = set() + requests[spkg].add(row['vr']) + + # remove all rows + cur = conn.execute("DELETE FROM vault_requests") + + return requests + + +def vault_request_add(args, p, v): + with connect(args) as conn: + conn.execute('INSERT INTO vault_requests (srcpackage, vr) VALUES (?,?)', (p, v)) diff --git a/calm/hint.py b/calm/hint.py index 6034c1d..1ec86d9 100755 --- a/calm/hint.py +++ b/calm/hint.py @@ -39,10 +39,10 @@ else: extra_licenses = [ 'Linux-man-pages-copyleft', # requires SPDX license-list 3.15 'Public-Domain', - 'XVIEW', ] for l in extra_licenses: - json.append({"spdx_license_key": l}) + if not any(j["spdx_license_key"] == l for j in json): + json.append({"spdx_license_key": l}) licensing = license_expression.build_spdx_licensing(json) # types of key: diff --git a/calm/maintainers.py b/calm/maintainers.py index 0a8225a..7c1fc7d 100644 --- a/calm/maintainers.py +++ b/calm/maintainers.py @@ -136,7 +136,7 @@ def add_directories(mlist, homedirs): # add maintainers from the package maintainers list, with the packages they # maintain -def add_packages(mlist, pkglist, orphanMaint=None, prev_maint=True): +def add_packages(mlist, pkglist, prev_maint=True): with open(pkglist) as f: for (i, l) in enumerate(f): l = l.rstrip() @@ -156,13 +156,9 @@ def add_packages(mlist, pkglist, orphanMaint=None, prev_maint=True): if status == 'OBSOLETE': continue - # orphaned packages get the default maintainer(s) if we - # have one, otherwise they are assigned to 'ORPHANED' + # orphaned packages are assigned to 'ORPHANED' elif status == 'ORPHANED': - if orphanMaint is not None: - m = orphanMaint - else: - m = status + m = status if prev_maint: # also add any previous maintainer(s) listed @@ -199,10 +195,10 @@ def add_packages(mlist, pkglist, orphanMaint=None, prev_maint=True): # create maintainer list -def read(args, orphanmaint=None, prev_maint=True): +def read(args, prev_maint=True): mlist = {} mlist = add_directories(mlist, args.homedir) - mlist = add_packages(mlist, args.pkglist, orphanmaint, prev_maint) + mlist = add_packages(mlist, args.pkglist, prev_maint) return mlist diff --git a/calm/mkgitoliteconf.py b/calm/mkgitoliteconf.py index e90b4c8..c67277e 100755 --- a/calm/mkgitoliteconf.py +++ b/calm/mkgitoliteconf.py @@ -51,7 +51,7 @@ def transform_username(name): def do_main(args): # read maintainer list mlist = {} - mlist = maintainers.add_packages(mlist, args.pkglist, getattr(args, 'orphanmaint', None)) + mlist = maintainers.add_packages(mlist, args.pkglist) # make the list of all packages maintainers.all_packages(mlist) @@ -70,7 +70,7 @@ def do_main(args): # global configuration print('') - print('@leads = %s' % ' '.join(map(transform_username, common_constants.ORPHANMAINT.split('/')))) + print('@leads = %s' % ' '.join(map(transform_username, common_constants.TRUSTEDMAINT.split('/')))) print('') print('repo @all') print(' RW = @leads') diff --git a/calm/mkmaintdir b/calm/mkmaintdir deleted file mode 100755 index 88c385a..0000000 --- a/calm/mkmaintdir +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2015 Jon Turney -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -# -# create maintainer upload directories -# -# a re-implementation of the mkpkgdir perl script in python -# intended to be run from crontab every 5 minutes -# -# - Read existing maintainer directories, build a list of maintainer -# - Read cygwin-pkg-maint, add to list of maintainers, and build a list of -# packages for each maintainer -# - Assign orpahaned packages to the project lead(s) -# - For each maintainer, create a home directory, set permissions, and write a -# !packages file -# - Report if the maintainer has no packages and mark with !defunct -# - -import argparse -import grp -import logging -import os -import pwd -import re -import sys - -import common_constants -import maintainers - -# -# -# - -cygwin_uid = pwd.getpwnam('cygwin').pw_uid -cygstage_gid = grp.getgrnam('cygstage').gr_gid - -# different values to be used when we are not running on sourceware.org, but my -# test system... -if os.uname()[1] == 'tambora': - cygwin_uid = pwd.getpwnam('jon').pw_uid - cygstage_gid = grp.getgrnam('None').gr_gid - - -# -# -# - -def main(args): - # clear the umask in case it is set - os.umask(0) - - # create maintainer list - mlist = {} - mlist = maintainers.add_directories(mlist, args.homedir) - mlist = maintainers.add_packages(mlist, args.pkglist, args.orphanmaint) - - # create or suggest removal for each maintainer directory - for name in sorted(mlist.keys()): - m = mlist[name] - dirpath = m.homedir() - - # if the path exists, but isn't a directory - if os.path.exists(dirpath) and not os.path.isdir(dirpath): - logging.error("%s exists and isn't a directory!" % dirpath) - continue - - # ensure the upload directory exists, with appropriate permissions, owner and contents - logging.info('processing %s' % dirpath) - if not args.dryrun: - os.makedirs(dirpath, exist_ok=True) - os.chown(dirpath, cygwin_uid, cygstage_gid) - os.chmod(dirpath, 0o2775) - # write !packages file (we don't use this for anything anymore, but - # keep it around for information) - with open(os.path.join(dirpath, '!packages'), 'w') as fd: - os.fchown(fd.fileno(), cygwin_uid, cygstage_gid) - print('|'.join([re.escape(p) for p in m.pkgs]), file=fd) - # and create arch subdirectories, with appropriate owner - for subdir in common_constants.ARCHES: - os.makedirs(os.path.join(dirpath, subdir, 'release'), exist_ok=True) - os.chown(os.path.join(dirpath, subdir, 'release'), cygwin_uid, cygstage_gid) - - # create/remove !defunct as appropriate - defunct = os.path.join(dirpath, '!defunct') - if len(m.pkgs) == 0: - # if they have no packages, suggest removing their upload directory (once) - if not os.path.exists(defunct): - logging.warning("defunct maintainer %s, consider removing their directory?" % name) - if not args.dryrun: - open(defunct, 'w').close() - else: - # remove defunct marker if no longer defunct - if os.path.exists(defunct): - logging.info("maintainer %s no longer defunct" % name) - if not args.dryrun: - os.unlink(defunct) - - -# -# -# - -if __name__ == "__main__": - homedir_default = common_constants.HOMEDIR - orphanmaint_default = common_constants.ORPHANMAINT - pkglist_default = common_constants.PKGMAINT - - parser = argparse.ArgumentParser(description='Create maintainer upload directories') - parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default) - parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default) - parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) - parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', help="don't do anything") - parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output') - (args) = parser.parse_args() - - if args.verbose: - logging.getLogger().setLevel(logging.INFO) - - logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s') - - main(args) diff --git a/calm/mksetupini.py b/calm/mksetupini.py index 170311b..9e3a006 100755 --- a/calm/mksetupini.py +++ b/calm/mksetupini.py @@ -48,7 +48,11 @@ except ImportError: # def do_main(args): # build package list - packages = package.read_packages(args.rel_area, args.arch) + packages, error = package.read_packages(args.rel_area, args.arch) + + if not error: + logging.error("errors reading package set, not writing setup.ini") + return 1 # spellcheck text hints if args.spell: diff --git a/calm/package.py b/calm/package.py index d450ea9..81d07fe 100755 --- a/calm/package.py +++ b/calm/package.py @@ -141,6 +141,7 @@ class Hint(object): # read a packages from a directory hierarchy # def read_packages(rel_area, arch): + error = False packages = {} # / noarch/ and src/ directories are considered @@ -151,11 +152,11 @@ def read_packages(rel_area, arch): logging.debug('reading packages from %s' % releasedir) for (dirpath, _subdirs, files) in os.walk(releasedir, followlinks=True): - read_package_dir(packages[root], rel_area, dirpath, files) + error = read_package_dir(packages[root], rel_area, dirpath, files) or error logging.debug("%d packages read from %s" % (len(packages[root]), releasedir)) - return merge({}, *packages.values()) + return (merge({}, *packages.values()), error) # helper function to compute sha512 for a particular file @@ -362,6 +363,10 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): logging.error("package '%s' name contains illegal characters" % p) return True + if re.search(r'-\d', p): + logging.error("package '%s' name contains hyphen followed a digit" % p) + return True + # assumption: no real package names end with '-src' # # enforce this, because source and install package names exist in a @@ -400,8 +405,8 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): # warn if filename doesn't follow P-V-R naming convention # # P must match the package name, V can contain anything, R must - # start with a number - match = re.match(r'^' + re.escape(p) + r'-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.(tar' + common_constants.PACKAGE_COMPRESSIONS_RE + r'|hint)$', f) + # start with a number and can't include a hyphen + match = re.match(r'^' + re.escape(p) + r'-(.+)-(\d[0-9a-zA-Z._+]*)(-src|)\.(tar' + common_constants.PACKAGE_COMPRESSIONS_RE + r'|hint)$', f) if not match: logging.error("file '%s' in package '%s' doesn't follow naming convention" % (f, p)) return True @@ -413,7 +418,7 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): # we already know P to split unambiguously), but this is a bad # idea. if '-' in v: - if v in past_mistakes.hyphen_in_version.get(p, []): + if v in past_mistakes.illegal_char_in_version.get(p, []): lvl = logging.INFO else: lvl = logging.ERROR @@ -424,6 +429,14 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): logging.error("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p)) warnings = True + if not re.match(r'^[\w\-._+]*$', v): + if v in past_mistakes.illegal_char_in_version.get(p, []): + lvl = logging.INFO + else: + lvl = logging.ERROR + warnings = True + logging.log(lvl, "file '%s' in package '%s' has a version which contains illegal characters" % (f, p)) + # if not there already, add to version-release list vr = '%s-%s' % (v, r) vr_list.add(vr) @@ -653,7 +666,7 @@ def upgrade_oldstyle_obsoletes(packages): # # validate the package database # -def validate_packages(args, packages, valid_requires_extra=None, missing_obsolete_extra=None): +def validate_packages(args, packages, valid_provides_extra=None, missing_obsolete_extra=None): error = False if packages is None: @@ -662,12 +675,9 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet if missing_obsolete_extra is None: missing_obsolete_extra = {} - # build the set of valid things to requires: etc. + # build the set of valid things to depends: on valid_requires = set() - if valid_requires_extra: - valid_requires.update(valid_requires_extra) - for p in packages: valid_requires.add(p) for hints in packages[p].version_hints.values(): @@ -681,23 +691,27 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet packages[p].obsoleted_by = set() packages[p].orphaned = False + # it's also valid to obsoletes: packages which have been removed + valid_obsoletes = set(valid_requires) + if valid_provides_extra: + valid_obsoletes.update(valid_provides_extra) + # perform various package validations for p in sorted(packages.keys()): for (v, hints) in packages[p].version_hints.items(): - for (c, okmissing, splitchar) in [ - ('depends', 'missing-depended-package', ','), - ('obsoletes', 'missing-obsoleted-package', ',') + for (c, okmissing, valid) in [ + ('depends', 'missing-depended-package', valid_requires), + ('obsoletes', 'missing-obsoleted-package', valid_obsoletes) ]: # if c is in hints, and not the empty string if hints.get(c, ''): - for r in hints[c].split(splitchar): + for r in hints[c].split(','): # remove any extraneous whitespace r = r.strip() # strip off any version relation enclosed in '()' # following the package name - if splitchar: - r = re.sub(r'(.*) +\(.*\)', r'\1', r) + r = re.sub(r'(.*) +\(.*\)', r'\1', r) if c == 'depends': # don't count cygwin-debuginfo for the purpose of @@ -714,7 +728,7 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet # all packages listed in a hint must exist (unless the # disable-check option says that's ok) - if (r not in valid_requires) and (r not in past_mistakes.nonexistent_provides + past_mistakes.expired_provides): + if (r not in valid) and (r not in past_mistakes.nonexistent_provides + past_mistakes.expired_provides): if okmissing not in getattr(args, 'disable_check', []): logging.error("package '%s' version '%s' %s: '%s', but nothing satisfies that" % (p, v, c, r)) error = True @@ -1532,7 +1546,7 @@ def mark_package_fresh(packages, p, v, mark=Freshness.fresh): SO_AGE_THRESHOLD_YEARS = 5 -def stale_packages(packages): +def stale_packages(packages, vault_requests): certain_age = time.time() - (SO_AGE_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) logging.debug("cut-off date for soversion package to be considered old is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age)))) @@ -1604,6 +1618,19 @@ def stale_packages(packages): mark = noretain_hint_mark + # - marked via 'calm-tool vault' + # + es = po.srcpackage(bv, suffix=False) + if es in vault_requests: + def vault_requests_mark(v): + if v in vault_requests[es]: + logging.info("package '%s' version '%s' not retained due vault request" % (pn, v)) + return Freshness.conditional + else: + return Freshness.fresh + + mark = vault_requests_mark + # mark any versions explicitly listed in the keep: override hint (unconditionally) for v in po.override_hints.get('keep', '').split(): if v in po.versions(): @@ -1712,5 +1739,5 @@ def stale_packages(packages): if __name__ == "__main__": for arch in common_constants.ARCHES: - packages = read_packages(common_constants.FTP, arch) + packages, _ = read_packages(common_constants.FTP, arch) print("arch %s has %d packages" % (arch, len(packages))) diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 6678d86..fed6c49 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -27,14 +27,16 @@ # uses. # -# packages with historical versions containing a hyphen -hyphen_in_version = { +# packages with historical versions containing a hyphen, or other illegal +# character +illegal_char_in_version = { 'ctorrent': ['1.3.4-dnh3.2'], 'email': ['3.2.1-git', '3.2.3-git'], 'email-debuginfo': ['3.2.1-git', '3.2.3-git'], 'fdupes': ['1.50-PR2'], 'gendef': ['1.0-svn2931'], 'gendef-debuginfo': ['1.0-svn2931'], + 'gt5': ['1.5.0~20111220+bzr29'], 'hidapi': ['0.8.0-rc1'], 'hidapi-debuginfo': ['0.8.0-rc1'], 'libhidapi-devel': ['0.8.0-rc1'], @@ -48,8 +50,6 @@ hyphen_in_version = { 'mingw64-x86_64-hidapi-debuginfo': ['0.8.0-rc1'], 'recode': ['3.7-beta2'], 'recode-debuginfo': ['3.7-beta2'], - 'tack': ['1.07-20150606'], - 'tack-debuginfo': ['1.07-20150606'], } # cygport places this into the requires of every debuginfo package, including diff --git a/calm/pkg2html.py b/calm/pkg2html.py index 2194a05..2a0d25d 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -573,5 +573,5 @@ if __name__ == "__main__": logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s') - packages = package.read_packages(args.rel_area, args.arch) + packages, _ = package.read_packages(args.rel_area, args.arch) update_package_listings(args, packages, args.arch) diff --git a/calm/tool_util.py b/calm/tool_util.py new file mode 100644 index 0000000..43fe3ea --- /dev/null +++ b/calm/tool_util.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +import logging +import os +import re + +from . import common_constants +from . import maintainers + + +def split(pvr): + # split name and vr + match = re.match(r'^(.+?)-(\d.*)', pvr) + if not match: + logging.error("unable to determine package and version-release from '%s'" % (pvr)) + return (None, None) + + p = match.group(1) + vr = match.group(2) + + return (p, vr) + + +def permitted(p): + # check CYGNAME is a maintainer for package + cygname = os.environ.get('CYGNAME', None) + + mlist = {} + mlist = maintainers.add_packages(mlist, common_constants.PKGMAINT, trustedMaint=common_constants.TRUSTEDMAINT) + + # CYGNAME is a maintainer for package + if p in mlist[cygname].pkgs: + return True + + # CYGNAME is a trusted maintainer + if cygname in common_constants.TRUSTEDMAINT.split('/'): + return True + + if cygname not in mlist: + logging.error("'%s' is not a package maintainer" % (cygname)) + return False + + logging.error("package '%s' is not in the package list for maintainer '%s'" % (p, cygname)) + return False diff --git a/calm/untest.py b/calm/untest.py index f3995e7..7d6d774 100644 --- a/calm/untest.py +++ b/calm/untest.py @@ -28,31 +28,15 @@ import re import sys from . import common_constants -from . import maintainers +from . import tool_util def untest(pvr): - # split name and vr - match = re.match(r'^(.+?)-(\d.*)', pvr) - if not match: - logging.error("unable to determine package and version-release from '%s'" % (pvr)) + p, vr = tool_util.split(pvr) + if not p: return - p = match.group(1) - vr = match.group(2) - - # check CYGNAME is a maintainer for package - cygname = os.environ['CYGNAME'] - - mlist = {} - mlist = maintainers.add_packages(mlist, common_constants.PKGMAINT, orphanMaint=common_constants.ORPHANMAINT) - - if cygname not in mlist: - logging.error("'%s' is not a package maintainer" % (cygname)) - return - - if p not in mlist[cygname].pkgs: - logging.error("package '%s' is not in the package list for maintainer '%s'" % (p, cygname)) + if not tool_util.permitted(p): return # remove '^test:' lines from any package and subpackage hints @@ -85,7 +69,7 @@ def untest(pvr): def main(): parser = argparse.ArgumentParser(description='remove test: hint') - parser.add_argument('package', nargs='*', metavar='PVR') + parser.add_argument('package', nargs='+', metavar='SPVR') (args) = parser.parse_args() logging.getLogger().setLevel(logging.INFO) diff --git a/calm/uploads.py b/calm/uploads.py index ba18217..fcb23cd 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -143,8 +143,8 @@ def scan(scandir, m, all_packages, arch, args): logging.error("package '%s' is not in the package list" % relpath) continue - # only process packages for which we are listed as a maintainer - if not package.is_in_package_list(pkgpath, m.pkgs): + # only process packages for which we are listed as a maintainer, or we are a trusted maintainer + if not (package.is_in_package_list(pkgpath, m.pkgs) or (m.name in args.trustedmaint.split('/'))): logging.warning("package '%s' is not in the package list for maintainer '%s'" % (relpath, m.name)) continue diff --git a/calm/vault.py b/calm/vault.py new file mode 100644 index 0000000..76cb8ad --- /dev/null +++ b/calm/vault.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +import argparse +import logging +import os +import sys +import types + +from . import common_constants +from . import db +from . import tool_util + + +def vault(pvr): + p, vr = tool_util.split(pvr) + if not p: + return + + if not tool_util.permitted(p): + return + + args = types.SimpleNamespace() + args.htdocs = os.path.join(common_constants.HTDOCS, 'packages') + + db.vault_request_add(args, p, vr) + + +def main(): + parser = argparse.ArgumentParser(description='mark packages for vaulting') + parser.add_argument('package', nargs='+', metavar='SPVR') + (args) = parser.parse_args() + + logging.getLogger().setLevel(logging.INFO) + logging.basicConfig(format='vault: %(message)s') + + for p in args.package: + vault(p) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/test_calm.py b/test/test_calm.py index 0f1534c..bd8390d 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -180,7 +180,7 @@ class CalmTest(unittest.TestCase): packages = {} for arch in common_constants.ARCHES: packages[arch] = {} - packages[args.arch] = package.read_packages(args.rel_area, args.arch) + packages[args.arch], _ = package.read_packages(args.rel_area, args.arch) package.validate_packages(args, packages[args.arch]) pkg2html.update_package_listings(args, packages) @@ -319,7 +319,7 @@ class CalmTest(unittest.TestCase): mlist = {} mlist = maintainers.add_directories(mlist, 'testdata/homes') - mlist = maintainers.add_packages(mlist, 'testdata/pkglist/cygwin-pkg-maint', None) + mlist = maintainers.add_packages(mlist, 'testdata/pkglist/cygwin-pkg-maint') compare_with_expected_file(self, 'testdata/pkglist', mlist) @@ -379,7 +379,7 @@ class CalmTest(unittest.TestCase): args.release = 'testing' args.setup_version = '4.321' - packages = package.read_packages(args.rel_area, args.arch) + packages, _ = package.read_packages(args.rel_area, args.arch) package.delete(packages, 'x86_64/release/nonexistent', 'nosuchfile-1.0.0.tar.xz') self.assertEqual(package.validate_packages(args, packages), True) package.write_setup_ini(args, packages, args.arch)