From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id DE7A73858D20; Sun, 11 Jun 2023 13:55:35 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org DE7A73858D20 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1686491735; bh=NKrOgTf0JgcoO+yXbrXKF4XCN//PxETo8wOyKDn+Af8=; h=To:Subject:Date:From:From; b=dDkUq9xatR+U30QjjYA3DeAVJMZEx2whdVSxKCNiPmf8/+5T0eZpU5bFjn491g++a H8+YlyoiRktnPgXyX3g8U4kqTzFK0vguipnqqSQr0B/pnCRvsZCNVEEw8rXta2iiwQ fJZZEL2yAf4DhFiO9FSyju7y5Y9s5ZcYJ6vmr2wA= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-33-g6012a2f X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 9658ffbf216528b5a2b306e9204b531942409c9a X-Git-Newrev: 6012a2fcb30796a2b729f817cb0898b0fd91bb65 Message-Id: <20230611135535.DE7A73858D20@sourceware.org> Date: Sun, 11 Jun 2023 13:55:35 +0000 (GMT) From: Jon Turney List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=6012a2fcb30796a2b729f817cb0898b0fd91bb65 commit 6012a2fcb30796a2b729f817cb0898b0fd91bb65 Author: Jon Turney Date: Sun Jun 11 11:32:55 2023 +0100 Remember package group information so we can show it on summary page https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=767b265ded753251e45217702098a235b7ea4b6f commit 767b265ded753251e45217702098a235b7ea4b6f Author: Jon Turney Date: Sun Jun 11 11:20:15 2023 +0100 Generate an includeable HTML fragment with the list of reports https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4fc6ab2fc3be623c7eedddfd71ab5da1fc4c4514 commit 4fc6ab2fc3be623c7eedddfd71ab5da1fc4c4514 Author: Jon Turney Date: Sat Jun 10 14:13:52 2023 +0100 Add gtksourceview to slotted packages in repology data https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=c5e4413e80202522d881fe1673f5823031ce79da commit c5e4413e80202522d881fe1673f5823031ce79da Author: Jon Turney Date: Sat Jun 10 14:12:24 2023 +0100 Reduce scope of scallwag_db transaction lock If multiple deploys are ready simultaneously, don't hold the lock over all of them. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2e62f47252fd09c954ce4b3adc110414ce816a4d commit 2e62f47252fd09c954ce4b3adc110414ce816a4d Author: Jon Turney Date: Sat May 20 18:01:22 2023 +0100 Add a simple way of grouping packages for maintainership Define a team with a line starting with '@', e.g.: @team Maintainer1/Maintainer2 Then @team can be referred in a packages maintainer list, as shorthand for that list of maintainers. Also ignore lines starting with '#' in maintainer list as comments Diff: --- calm/maintainers.py | 136 ++++++++++++++++++++--------- calm/pkg2html.py | 14 ++- calm/repology.py | 10 ++- calm/reports.py | 48 ++++++---- calm/scallywag_db.py | 3 +- test/testdata/process_arch/htdocs.expected | 2 +- 6 files changed, 146 insertions(+), 67 deletions(-) diff --git a/calm/maintainers.py b/calm/maintainers.py index d0497cb..e2c60c7 100644 --- a/calm/maintainers.py +++ b/calm/maintainers.py @@ -48,9 +48,10 @@ from . import utils # supports is_orphaned() and maintainers() methods # class MaintainerPackage(UserString): - def __init__(self, name, maintainers, orphaned): + def __init__(self, name, maintainers, groups, orphaned): super().__init__(name) self._maintainers = maintainers + self._groups = groups self._orphaned = orphaned # XXX: for historical reasons, 'ORPHANED' still appears in the maintainer @@ -61,6 +62,9 @@ class MaintainerPackage(UserString): def maintainers(self): return self._maintainers + def groups(self): + return self._groups + # # @@ -151,48 +155,16 @@ def add_directories(mlist, homedirs): @utils.mtime_cache def _read_pkglist(pkglist): mpkgs = {} + teams = {} with open(pkglist) as f: for (i, l) in enumerate(f): - orphaned = False - l = l.rstrip() - - # match lines of the form ' ' - match = re.match(r'^(\S+)\s+(.+)$', l) - if match: - pkg = match.group(1) - rest = match.group(2) - - # does rest starts with a status in all caps? - status_match = re.match(r'^([A-Z]{2,})\b.*$', rest) - if status_match: - status = status_match.group(1) - - # packages marked as 'OBSOLETE' are obsolete - if status == 'OBSOLETE': - # obsolete packages have no maintainer - # - # XXX: perhaps disallow even trusties to upload (or - # warn if they try?) - m = '' - - # orphaned packages are assigned to 'ORPHANED' - elif status == 'ORPHANED': - m = status - orphaned = True - - # also add any previous maintainer(s) listed - prevm = re.match(r'^ORPHANED\s\((.*)\)', rest) - if prevm: - m = m + '/' + prevm.group(1) - else: - logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l)) - continue - else: - m = rest + def _split_maintainer_names(m, teams=None): # joint maintainers are separated by '/' maintainers = list() + groups = list() + for name in m.split('/'): if not name: continue @@ -206,15 +178,85 @@ def _read_pkglist(pkglist): try: name.encode('ascii') except UnicodeError: - logging.error("non-ascii maintainer name '%s' in line %s:%d, skipped" % (rest, pkglist, i)) + logging.error("non-ascii maintainer name '%s' in %s:%d: '%s', skipped" % (name, pkglist, i, l)) continue - maintainers.append(name) - - mpkgs[pkg] = MaintainerPackage(pkg, maintainers, orphaned) + if name.startswith('@'): + groups.append(name[1:]) + if teams and name in teams: + for n in teams[name]: + if n not in maintainers: + maintainers.append(n) + else: + logging.error("unknown team '%s' in %s:%d: '%s', skipped" % (name, pkglist, i, l)) + else: + # avoid adding name if it's already in the list (a set + # is not appropriate here, as we have the concept of + # 'first named maintainer' + if name not in maintainers: + maintainers.append(name) + + return maintainers, groups + + if l.startswith('#'): + # comment + continue + elif l.startswith('@'): + # 'team' definition of the form '@ ' + match = re.match(r'^(\S+)\s+(.+)$', l) + if match: + team = match.group(1) + rest = match.group(2) + + teams[team], _ = _split_maintainer_names(rest) + continue else: - logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l)) + # package + orphaned = False + l = l.rstrip() + + # match lines of the form ' ' + match = re.match(r'^(\S+)\s+(.+)$', l) + if match: + pkg = match.group(1) + rest = match.group(2) + + # does rest starts with a status in all caps? + status_match = re.match(r'^([A-Z]{2,})\b.*$', rest) + if status_match: + status = status_match.group(1) + + # packages marked as 'OBSOLETE' are obsolete + if status == 'OBSOLETE': + # obsolete packages have no maintainer + # + # XXX: perhaps disallow even trusties to upload (or + # warn if they try?) + m = '' + + # orphaned packages are assigned to 'ORPHANED' + elif status == 'ORPHANED': + m = status + orphaned = True + + # also add any previous maintainer(s) listed + prevm = re.match(r'^ORPHANED\s\((.*)\)', rest) + if prevm: + m = m + '/' + prevm.group(1) + else: + logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l)) + continue + else: + m = rest + + maintainers, groups = _split_maintainer_names(m, teams) + + mpkgs[pkg] = MaintainerPackage(pkg, maintainers, groups, orphaned) + continue + + # couldn't handle the line + logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l)) return mpkgs @@ -259,3 +301,13 @@ def update_reminder_times(mlist): # a list of all packages def all_packages(pkglist): return pkg_list(pkglist).keys() + + +# +# +# + +if __name__ == "__main__": + from . import common_constants + p = pkg_list(common_constants.PKGMAINT) + print(p['xwininfo'].maintainers()) diff --git a/calm/pkg2html.py b/calm/pkg2html.py index a4cd59a..43591ff 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -271,10 +271,14 @@ def update_package_listings(args, packages): m_pn = es_po.orig_name if m_pn not in pkg_maintainers: m = None - elif pkg_maintainers[m_pn].is_orphaned(): - m = 'ORPHANED' + pkg_groups = None else: - m = ', '.join(sorted(pkg_maintainers[m_pn].maintainers())) + if pkg_maintainers[m_pn].is_orphaned(): + m = 'ORPHANED' + else: + m = ', '.join(sorted(pkg_maintainers[m_pn].maintainers())) + + pkg_groups = pkg_maintainers[m_pn].groups() if m: print('maintainer(s): %s ' % m, file=f) @@ -284,6 +288,10 @@ def update_package_listings(args, packages): Do not contact the maintainer(s) directly.)'''), file=f) print('

', file=f) + if pkg_groups: + print('group: %s ' % ','.join(pkg_groups), file=f) + print('

', file=f) + if po.kind == package.Kind.source: repo = '/git/cygwin-packages/%s.git' % pn if os.path.exists(repo): diff --git a/calm/repology.py b/calm/repology.py index 45a0da6..bb2595b 100644 --- a/calm/repology.py +++ b/calm/repology.py @@ -43,7 +43,13 @@ use_legacy = {'qt': [LegacyData('5', []), LegacyData('4', []), LegacyData('3', [])], 'gtk': [LegacyData('3', ['3.9', '+']), - LegacyData('2', [])]} + LegacyData('2', [])], + 'gtksourceview': [LegacyData('2', []), + LegacyData('3', []), + LegacyData('4', []), + LegacyData('5', []), + ] + } def repology_fetch_versions(): @@ -99,7 +105,7 @@ def repology_fetch_versions(): # packages # # (multiple cygwin source packages can correspond to a single - # canonical repology package name, e.g. foo and mingww64-arch-foo) + # canonical repology package name, e.g. foo and mingw64-arch-foo) for i in p: if i['repo'] == "cygwin": source_pn = i['srcname'] diff --git a/calm/reports.py b/calm/reports.py index a9a3f4b..b101ef4 100644 --- a/calm/reports.py +++ b/calm/reports.py @@ -57,6 +57,15 @@ def template(title, body, f): '''), file=f) +def write_report(args, title, body, fn, reportlist): + reportlist[title] = os.path.join('reports', fn) + + fn = os.path.join(args.htdocs, 'reports', fn) + + with utils.open_amifc(fn) as f: + template(title, body.getvalue(), f) + + def linkify(pn, po): return '{1}'.format(pn, po.orig_name) @@ -64,7 +73,7 @@ def linkify(pn, po): # # produce a report of unmaintained packages # -def unmaintained(args, packages, reportsdir): +def unmaintained(args, packages, reportlist): pkg_maintainers = maintainers.pkg_list(args.pkglist) um_list = [] @@ -123,14 +132,12 @@ def unmaintained(args, packages, reportsdir): print('', file=body) - unmaintained = os.path.join(reportsdir, 'unmaintained.html') - with utils.open_amifc(unmaintained) as f: - template('Unmaintained packages', body.getvalue(), f) + write_report(args, 'Unmaintained packages', body, 'unmaintained.html', reportlist) # produce a report of deprecated packages # -def deprecated(args, packages, reportsdir): +def deprecated(args, packages, reportlist): dep_list = [] arch = 'x86_64' @@ -183,15 +190,13 @@ def deprecated(args, packages, reportsdir): print('', file=body) - deprecated = os.path.join(reportsdir, 'deprecated_so.html') - with utils.open_amifc(deprecated) as f: - template('Deprecated shared library packages', body.getvalue(), f) + write_report(args, 'Deprecated shared library packages', body, 'deprecated_so.html', reportlist) # produce a report of packages which need rebuilding for the latest major # version version provides # -def provides_rebuild(args, packages, reportfile, provide_package): +def provides_rebuild(args, packages, fn, provide_package, reportlist): pr_list = [] arch = 'x86_64' @@ -246,8 +251,7 @@ def provides_rebuild(args, packages, reportfile, provide_package): print('', file=body) - with utils.open_amifc(reportfile) as f: - template('Packages needing rebuilds for latest %s' % provide_package, body.getvalue(), f) + write_report(args, 'Packages needing rebuilds for latest %s' % provide_package, body, fn, reportlist) # @@ -255,11 +259,21 @@ def do_reports(args, packages): if args.dryrun: return - reportsdir = os.path.join(args.htdocs, 'reports') - pkg2html.ensure_dir_exists(args, reportsdir) + reportlist = {} + + pkg2html.ensure_dir_exists(args, os.path.join(args.htdocs, 'reports')) + + unmaintained(args, packages, reportlist) + deprecated(args, packages, reportlist) - unmaintained(args, packages, reportsdir) - deprecated(args, packages, reportsdir) + provides_rebuild(args, packages, 'perl_rebuilds.html', 'perl_base', reportlist) + provides_rebuild(args, packages, 'ruby_rebuilds.html', 'ruby', reportlist) - provides_rebuild(args, packages, os.path.join(reportsdir, 'perl_rebuilds.html'), 'perl_base') - provides_rebuild(args, packages, os.path.join(reportsdir, 'ruby_rebuilds.html'), 'ruby') + fn = os.path.join(args.htdocs, 'reports_list.inc') + with utils.open_amifc(fn) as f: + print('
    ', file=f) + for r in reportlist: + print('
  • ', file=f) + print('%s' % (reportlist[r], r), file=f) + print('
    • ', file=f) + print('
', file=f) diff --git a/calm/scallywag_db.py b/calm/scallywag_db.py index a310c55..ceb20da 100644 --- a/calm/scallywag_db.py +++ b/calm/scallywag_db.py @@ -52,5 +52,4 @@ def do_deploys(cb): status = 'deploy failed' conn.execute("UPDATE jobs SET status = ? WHERE id = ?", (status, r.id)) - - conn.commit() + conn.commit() diff --git a/test/testdata/process_arch/htdocs.expected b/test/testdata/process_arch/htdocs.expected index 55585d6..d9afd07 100644 --- a/test/testdata/process_arch/htdocs.expected +++ b/test/testdata/process_arch/htdocs.expected @@ -1,4 +1,4 @@ -{'.': ['calm.db', 'packages.inc', 'src_packages.inc'], +{'.': ['calm.db', 'packages.inc', 'reports_list.inc', 'src_packages.inc'], 'reports': ['deprecated_so.html', 'perl_rebuilds.html', 'ruby_rebuilds.html', 'unmaintained.html'], 'summary': ['arc-src.html', 'arc.html',