From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 3EA18386F0C1; Tue, 7 Jun 2022 20:46:57 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 3EA18386F0C1 To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-51-g46d04fa X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 51b7563055729cf836da85ccac7d731583c26392 X-Git-Newrev: 46d04fa775587ed9ab22bd92d9211d5abb00b999 Message-Id: <20220607204657.3EA18386F0C1@sourceware.org> Date: Tue, 7 Jun 2022 20:46:57 +0000 (GMT) From: Jon TURNEY X-BeenThere: cygwin-apps-cvs@cygwin.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Cygwin-apps git logs List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 07 Jun 2022 20:46:57 -0000 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=46d04fa775587ed9ab22bd92d9211d5abb00b999 commit 46d04fa775587ed9ab22bd92d9211d5abb00b999 Author: Jon Turney Date: Tue Jun 7 11:11:09 2022 +0100 Avoid false-positive in warning about replace-version: Avoid warning about replace-version: listing an installable version when the replacable version is a test: version (and a non-test version exists), because the setup depsolver wil correctly downgrade in that case. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=867e172bf9c1095c0939551398b83ce47647d18f commit 867e172bf9c1095c0939551398b83ce47647d18f Author: Jon Turney Date: Tue Jun 7 10:59:18 2022 +0100 Only mail leads log records of ERROR or higher Emulate the behaviour of retainLevel=ERROR for the leads email, as we had prior to a1cb1581. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f2af73f6d9f220c3b3d211edfae4e69966f31beb commit f2af73f6d9f220c3b3d211edfae4e69966f31beb Author: Jon Turney Date: Sat Jun 4 14:00:40 2022 +0100 Improve default for EMAIL when testing https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d4888ac0292dfc801da586e0873e923925396382 commit d4888ac0292dfc801da586e0873e923925396382 Author: Jon Turney Date: Mon May 30 14:25:14 2022 +0100 CI: Drop python 3.5 and add python 3.10 license_expression doesn't support python 3.5 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dd860396f2fa459872360704626443ecab000d48 commit dd860396f2fa459872360704626443ecab000d48 Author: Jon Turney Date: Fri May 27 15:47:15 2022 +0100 Extend fix-missing-src-hint-info for license: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f4e788e17bce36b1b4658646728fb877463c32f4 commit f4e788e17bce36b1b4658646728fb877463c32f4 Author: Jon Turney Date: Fri May 27 15:34:49 2022 +0100 Allow 'license:' key in source hint Allow 'license:' key in source hint, and check it contains a valid SPDX license expression. Show the key value in source package summary page, and in repology JSON output. Update tests appropriately Future work: Add 'license:' to the set of mandatory keys for a source package. Diff: --- .flake8 | 2 + .github/workflows/calm.yaml | 2 +- calm/calm.py | 7 +- calm/common_constants.py | 2 +- calm/fix-missing-src-hint-info.py | 10 +- calm/fixes.py | 101 +++++++++++++++++---- calm/hint.py | 24 ++++- calm/package.py | 13 ++- calm/pkg2html.py | 5 + requirements.txt | 1 + .../x86/release/cygwin/cygwin-2.2.1-1-src.expected | 5 +- .../htdocs.expected/summary/cygwin-src.html | 3 + test/testdata/process_arch/packages.json.expected | 1 + .../x86/release/cygwin/cygwin-2.2.1-1-src.hint | 1 + 14 files changed, 145 insertions(+), 32 deletions(-) diff --git a/.flake8 b/.flake8 index 0da33f9..48e7f81 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,5 @@ [flake8] ignore=E741,E129,W504,A003,B020,B902 max-line-length=240 +per-file-ignores = + calm/fixes.py:E241,E127 diff --git a/.github/workflows/calm.yaml b/.github/workflows/calm.yaml index 69be9d2..90a532a 100644 --- a/.github/workflows/calm.yaml +++ b/.github/workflows/calm.yaml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 diff --git a/calm/calm.py b/calm/calm.py index 62dc902..63cdc60 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -630,12 +630,13 @@ def mail_cb(state, loghandler): if not state.args.email: return - # if there are any log records of ERROR level or higher, send all records to - # leads + # if there are any log records of ERROR level or higher, send those records + # to leads if any([record.levelno >= logging.ERROR for record in loghandler.buffer]): leads_email = BufferingSMTPHandler(state.args.email, subject='%s' % (state.subject)) for record in loghandler.buffer: - leads_email.handle(record) + if record.levelno >= logging.ERROR: + leads_email.handle(record) leads_email.close() # send each maintainer mail containing log entries caused by their actions, diff --git a/calm/common_constants.py b/calm/common_constants.py index b85c32c..1887681 100644 --- a/calm/common_constants.py +++ b/calm/common_constants.py @@ -74,7 +74,7 @@ DEFAULT_KEEP_DAYS = 0 # different values to be used when we are not running on sourceware.org, but my # test system... if os.uname()[1] == 'tambora': - EMAILS = 'jon.turney@dronecode.org.uk' + EMAILS = 'debug' ALWAYS_BCC = '' # package compressions diff --git a/calm/fix-missing-src-hint-info.py b/calm/fix-missing-src-hint-info.py index 88142a6..3185083 100644 --- a/calm/fix-missing-src-hint-info.py +++ b/calm/fix-missing-src-hint-info.py @@ -32,7 +32,7 @@ from . import common_constants from . import fixes -def fix_hints(relarea, packages): +def fix_hints(relarea, packages, fixids): for (dirpath, _subdirs, files) in os.walk(relarea): # only apply to listed packages, if specified @@ -50,7 +50,7 @@ def fix_hints(relarea, packages): logging.error('hint %s missing' % hf) continue - fixes.fix_hint(dirpath, hf, f, ['homepage']) + fixes.fix_hint(dirpath, hf, f, fixids) # @@ -62,6 +62,7 @@ def main(): parser = argparse.ArgumentParser(description='src hint improver') parser.add_argument('package', nargs='*', metavar='PACKAGE') + parser.add_argument('--fix', action='append', help='ids of fixes to perform') parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output', default=0) parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='relarea') (args) = parser.parse_args() @@ -69,9 +70,12 @@ def main(): if args.verbose: logging.getLogger().setLevel(logging.INFO) + if not args.fix: + args.fix = ['homepage', 'license'] + logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s') - fix_hints(args.relarea, args.package) + fix_hints(args.relarea, args.package, args.fix) # diff --git a/calm/fixes.py b/calm/fixes.py index 796805a..e7f8146 100644 --- a/calm/fixes.py +++ b/calm/fixes.py @@ -63,6 +63,29 @@ def read_cygport(dirpath, tf): return content +def _parse_cygport_var(dirpath, tf, var): + # crack open corresponding -src.tar and parse var out from .cygport + logging.debug('examining %s' % tf) + content = read_cygport(dirpath, tf) + + value = None + if content: + for l in content.splitlines(): + match = re.match(r'^\s*' + var + r'\s*=\s*("|)([^"].*)\1', l) + if match: + if value: + logging.warning('multiple %s lines in .cygport in srcpkg %s' % (var, tf)) + pn = os.path.basename(dirpath) + value = match.group(2) + value = re.sub(r'\$({|)(PN|ORIG_PN|NAME)(}|)', pn, value) + + if value and '$' in value: + logging.warning('unknown shell parameter expansions in %s="%s" in .cygport in srcpkg %s' % (var, value, tf)) + value = None + + return value + + class NoRedirection(urllib.request.HTTPErrorProcessor): def http_response(self, request, response): return response @@ -89,24 +112,7 @@ def _fix_homepage_src_hint(hints, dirpath, _hf, tf): if 'homepage' in hints: homepage = hints['homepage'] else: - # crack open corresponding -src.tar and parse homepage out from .cygport - logging.debug('examining %s' % tf) - content = read_cygport(dirpath, tf) - - homepage = None - if content: - for l in content.splitlines(): - match = re.match(r'^\s*HOMEPAGE\s*=\s*("|)([^"].*)\1', l) - if match: - if homepage: - logging.warning('multiple HOMEPAGE lines in .cygport in srcpkg %s', tf) - pn = os.path.basename(dirpath) - homepage = match.group(2) - homepage = re.sub(r'\$({|)(PN|ORIG_PN|NAME)(}|)', pn, homepage) - - if homepage and '$' in homepage: - logging.warning('unknown shell parameter expansions in HOMEPAGE="%s" in .cygport in srcpkg %s' % (homepage, tf)) - homepage = None + homepage = _parse_cygport_var(dirpath, tf, 'HOMEPAGE') if not homepage: logging.info('cannot determine homepage: from srcpkg %s' % tf) @@ -139,6 +145,63 @@ def _fix_homepage_src_hint(hints, dirpath, _hf, tf): return False +# for specific packages, map some human-readable license texts to SPDX expressions +licmap = [ + ('Apache License, Version 2', 'Apache-2.0', ['meson', 'ninja']), + ('BSD 3-Clause', 'BSD-3-Clause', ['libsolv', 'mingw64-i686-libsolv', 'mingw64-x86_64-libsolv']), + ('BSD3/GPLv2+', 'BSD-3-Clause AND GPL-2.0-or-later', ['dash']), + ('CC BY-SA 3.0', 'CC-BY-SA-3.0', ['dmalloc']), + ('GNU General Public License, Version 2', 'GPL-2.0-only', ['buildbot-slave', 'buildbot-worker']), + ('GNU General Public License, Version 3', 'GPL-3.0-or-later', ['osslsigncode']), + ('GPL', 'GPL-2.0-or-later', ['cpuid']), + ('GPL', 'GPL-3.0-or-later', ['units']), + ('GPLv2+', 'GPL-2.0-or-later', ['grep', 'gzip', 'readline']), + ('GPLv2+FontEmbExc/OFL-1.1', 'GPL-2.0-or-later WITH Font-exception-2.0 OR OFL-1.1', + ['unifont']), + ('GPLv3+', 'GPL-3.0-or-later', ['bison', 'wget']), + ('LGPLv2.1+/GPLv2+', 'LGPL-2.1-or-later AND GPL-2.0-or-later', ['libgcrypt']), + ('LGPLv3+/GPLv2+/GPLv3+/LGPLv2+', '(LGPL-3.0-or-later OR GPL-2.0-or-later) AND GPL-3.0-or-later', + ['libidn', 'mingw64-i686-libidn', 'mingw64-x86_64-libidn']), + ('LGPLv3+/GPLv2+/GPLv3+/Unicode2016', '(LGPL-3.0-or-later OR GPL-2.0-or-later) AND GPL-3.0-or-later AND Unicode-DFS-2016', + ['libidn2', 'mingw64-i686-libidn2', 'mingw64-x86_64-libidn2']), + ('MIT License', 'MIT', ['python-future']), + ('MIT-like', 'curl', ['curl', 'mingw64-i686-curl', 'mingw64-x86_64-curl']), + ('MIT-like', 'Linux-man-pages-copyleft', ['man-pages-linux', 'man-pages-posix']), + ('MIT-like', 'BSD-Source-Code', ['vttest']), + ('Public domain', 'BSD-3-Clause AND Public-Domain', ['tzdata', 'tzcode']), + ('SGI Free Software License B', 'SGI-B-2.0', ['khronos-opengl-registry']), + ('Sun OpenLook', 'XVIEW', ['xview']), +] + + +def _fix_license_src_hint(hints, dirpath, _hf, tf): + # already present? + if 'license' in hints: + lic = hints['license'] + else: + lic = _parse_cygport_var(dirpath, tf, 'LICENSE') + + if not lic: + logging.info('cannot determine license: from srcpkg %s' % tf) + return False + + pn = dirpath.split(os.path.sep)[-1] + for (human, spdx, pl) in licmap: + if (pn in pl) and (lic.lower() == human.lower()): + lic = spdx + logging.info("converted license text '%s' to SPDX license expression '%s'" % (human, spdx)) + break + + logging.info('adding license:%s to hints for srcpkg %s' % (lic, tf)) + + # changed? + if lic != hints.get('license', None): + hints['license'] = lic + return True + + return False + + def _fix_invalid_keys_hint(hints, _dirpath, hf, _tf): # eliminate keys that aren't appropriate to the package type if hf.endswith('-src.hint'): @@ -170,6 +233,8 @@ def fix_hint(dirpath, hf, tf, problems): changed = _fix_homepage_src_hint(hints, dirpath, hf, tf) if 'invalid_keys' in problems: changed = _fix_invalid_keys_hint(hints, dirpath, hf, tf) or changed + if 'license' in problems: + changed = _fix_license_src_hint(hints, dirpath, hf, tf) or changed # write updated hints if changed: diff --git a/calm/hint.py b/calm/hint.py index 15c468f..99408d7 100755 --- a/calm/hint.py +++ b/calm/hint.py @@ -26,8 +26,20 @@ # from collections import OrderedDict -import re import argparse +import license_expression +import re + +# reach inside license_expression to add custom license ids we permit +json = license_expression.get_license_index() +extra_licenses = [ + 'Linux-man-pages-copyleft', # requires SPDX license-list 3.15 + 'Public-Domain', + 'XVIEW', +] +for l in extra_licenses: + json.append({"spdx_license_key": l}) +licensing = license_expression.build_spdx_licensing(json) # types of key: # 'multilineval' - always have a value, which may be multiline @@ -67,6 +79,7 @@ hintkeys[spvr].update({ 'skip': 'noval', # in all spvr hints, but ignored 'homepage': 'val', 'build-depends': 'optval', + 'license': 'val', }) hintkeys[override] = { @@ -277,6 +290,15 @@ def hint_file_parse(fn, kind, strict=False): if not re.match(r'(\S+)\s+(\S.*)', value): errors.append('message value must have id and text') + # license must be a valid spdx license expression + if key == 'license': + try: + le = licensing.validate(value, strict=True) + except (license_expression.ExpressionParseError, license_expression.ExpressionError) as e: + errors.append('value for key %s not a valid license expression: %s' % (key, e)) + if le.original_expression != le.normalized_expression: + errors.append("license expression: '%s' normalizes to '%s'" % (value, le.normalized_expression)) + # warn if value starts with a quote followed by whitespace if re.match(r'^"[ \t]+', value): warnings.append('value for key %s starts with quoted whitespace' % (key)) diff --git a/calm/package.py b/calm/package.py index 0b3dec8..482d042 100755 --- a/calm/package.py +++ b/calm/package.py @@ -698,14 +698,16 @@ def validate_packages(args, packages, valid_requires_extra=None): # warn if replace-versions lists a version which is less than # the current version (which is pointless as the current version # will replace it anyhow) - if packages[p].best_version: - if SetupVersion(rv) <= SetupVersion(packages[p].best_version): - logging.warning("package '%s' replace-versions: uselessly lists version '%s', which is <= current version" % (p, rv)) + bv = packages[p].best_version + if bv: + if SetupVersion(rv) <= SetupVersion(bv): + logging.warning("package '%s' replace-versions: uselessly lists version '%s', which is <= current version '%s'" % (p, rv, bv)) # warn if replace-versions lists a version which is also # available to install (as this doesn't work as expected) if rv in packages[p].version_hints: - logging.warning("package '%s' replace-versions: lists version '%s', which is also available to install" % (p, rv)) + if ('test' in packages[p].version_hints[rv]) == ('test' in packages[p].version_hints[bv]): + logging.warning("package '%s' replace-versions: lists version '%s', which is also available to install" % (p, rv)) # If the install tarball is empty, we should probably either be marked # obsolete (if we have no dependencies) or virtual (if we do) @@ -1212,6 +1214,9 @@ def write_repo_json(args, packages, f): if 'homepage' in po.version_hints[bv]: d['homepage'] = po.version_hints[bv]['homepage'] + if 'license' in po.version_hints[bv]: + d['license'] = po.version_hints[bv]['license'] + if pkg_maintainers[po.orig_name] and ('ORPHANED' not in pkg_maintainers[po.orig_name]): d['maintainers'] = sorted(pkg_maintainers[po.orig_name]) diff --git a/calm/pkg2html.py b/calm/pkg2html.py index ae6a6a7..565e515 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -242,6 +242,11 @@ def update_package_listings(args, packages): homepage = po.version_hints[po.best_version].get('homepage', None) if homepage: print('homepage: %s

' % (homepage, homepage), file=f) + lic = po.version_hints[po.best_version].get('license', None) + if lic: + print('license: %s' % (lic), file=f) + print('(SPDX)', file=f) + print('

', file=f) else: es = po.srcpackage(bv) print('source package: %s

' % linkify_package(es), file=f) diff --git a/requirements.txt b/requirements.txt index bc5496c..1f4e625 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ flake8-blind-except flake8-bugbear ; python_version >= "3.5" flake8-builtins flake8-import-order == 0.14.1 +license_expression lockfile pycodestyle python-daemon diff --git a/test/testdata/hints/x86/release/cygwin/cygwin-2.2.1-1-src.expected b/test/testdata/hints/x86/release/cygwin/cygwin-2.2.1-1-src.expected index 9cf692b..3657b00 100644 --- a/test/testdata/hints/x86/release/cygwin/cygwin-2.2.1-1-src.expected +++ b/test/testdata/hints/x86/release/cygwin/cygwin-2.2.1-1-src.expected @@ -1 +1,4 @@ -OrderedDict([('sdesc', '"The UNIX emulation engine"'), ('ldesc', '"The UNIX emulation engine"'), ('category', 'Base')]) +{'sdesc': '"The UNIX emulation engine"', + 'ldesc': '"The UNIX emulation engine"', + 'category': 'Base', + 'license': 'LGPL-3.0-or-later WITH LGPL-3.0-linking-exception'} diff --git a/test/testdata/htdocs.expected/summary/cygwin-src.html b/test/testdata/htdocs.expected/summary/cygwin-src.html index a68217c..4720b17 100755 --- a/test/testdata/htdocs.expected/summary/cygwin-src.html +++ b/test/testdata/htdocs.expected/summary/cygwin-src.html @@ -14,6 +14,9 @@ description: The UNIX emulation engine

categories: Base

install package(s): cygwin, cygwin-debuginfo, cygwin-devel

+license: LGPL-3.0-or-later WITH LGPL-3.0-linking-exception +(SPDX) +

maintainer(s): Corinna Vinschen, Yaakov Selkowitz (Use the mailing list to report bugs or ask questions. Do not contact the maintainer(s) directly.) diff --git a/test/testdata/process_arch/packages.json.expected b/test/testdata/process_arch/packages.json.expected index 512fcf2..be21cfa 100644 --- a/test/testdata/process_arch/packages.json.expected +++ b/test/testdata/process_arch/packages.json.expected @@ -51,6 +51,7 @@ ' "arches": [\n' ' "x86"\n' ' ],\n' + ' "license": "LGPL-3.0-or-later WITH LGPL-3.0-linking-exception",\n' ' "maintainers": [\n' ' "Corinna Vinschen",\n' ' "Yaakov Selkowitz"\n' diff --git a/test/testdata/relarea/x86/release/cygwin/cygwin-2.2.1-1-src.hint b/test/testdata/relarea/x86/release/cygwin/cygwin-2.2.1-1-src.hint index d924e38..28bb602 100644 --- a/test/testdata/relarea/x86/release/cygwin/cygwin-2.2.1-1-src.hint +++ b/test/testdata/relarea/x86/release/cygwin/cygwin-2.2.1-1-src.hint @@ -1,3 +1,4 @@ sdesc: "The UNIX emulation engine" ldesc: "The UNIX emulation engine" category: Base +license: LGPL-3.0-or-later WITH LGPL-3.0-linking-exception