public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-33-g6012a2f
@ 2023-06-11 13:55 Jon Turney
0 siblings, 0 replies; only message in thread
From: Jon Turney @ 2023-06-11 13:55 UTC (permalink / raw)
To: cygwin-apps-cvs
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=6012a2fcb30796a2b729f817cb0898b0fd91bb65
commit 6012a2fcb30796a2b729f817cb0898b0fd91bb65
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Jun 11 11:32:55 2023 +0100
Remember package group information so we can show it on summary page
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=767b265ded753251e45217702098a235b7ea4b6f
commit 767b265ded753251e45217702098a235b7ea4b6f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Jun 11 11:20:15 2023 +0100
Generate an includeable HTML fragment with the list of reports
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4fc6ab2fc3be623c7eedddfd71ab5da1fc4c4514
commit 4fc6ab2fc3be623c7eedddfd71ab5da1fc4c4514
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sat Jun 10 14:13:52 2023 +0100
Add gtksourceview to slotted packages in repology data
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=c5e4413e80202522d881fe1673f5823031ce79da
commit c5e4413e80202522d881fe1673f5823031ce79da
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sat Jun 10 14:12:24 2023 +0100
Reduce scope of scallwag_db transaction lock
If multiple deploys are ready simultaneously, don't hold the lock over
all of them.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2e62f47252fd09c954ce4b3adc110414ce816a4d
commit 2e62f47252fd09c954ce4b3adc110414ce816a4d
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sat May 20 18:01:22 2023 +0100
Add a simple way of grouping packages for maintainership
Define a team with a line starting with '@', e.g.:
@team Maintainer1/Maintainer2
Then @team can be referred in a packages maintainer list, as shorthand
for that list of maintainers.
Also ignore lines starting with '#' in maintainer list as comments
Diff:
---
calm/maintainers.py | 136 ++++++++++++++++++++---------
calm/pkg2html.py | 14 ++-
calm/repology.py | 10 ++-
calm/reports.py | 48 ++++++----
calm/scallywag_db.py | 3 +-
test/testdata/process_arch/htdocs.expected | 2 +-
6 files changed, 146 insertions(+), 67 deletions(-)
diff --git a/calm/maintainers.py b/calm/maintainers.py
index d0497cb..e2c60c7 100644
--- a/calm/maintainers.py
+++ b/calm/maintainers.py
@@ -48,9 +48,10 @@ from . import utils
# supports is_orphaned() and maintainers() methods
#
class MaintainerPackage(UserString):
- def __init__(self, name, maintainers, orphaned):
+ def __init__(self, name, maintainers, groups, orphaned):
super().__init__(name)
self._maintainers = maintainers
+ self._groups = groups
self._orphaned = orphaned
# XXX: for historical reasons, 'ORPHANED' still appears in the maintainer
@@ -61,6 +62,9 @@ class MaintainerPackage(UserString):
def maintainers(self):
return self._maintainers
+ def groups(self):
+ return self._groups
+
#
#
@@ -151,48 +155,16 @@ def add_directories(mlist, homedirs):
@utils.mtime_cache
def _read_pkglist(pkglist):
mpkgs = {}
+ teams = {}
with open(pkglist) as f:
for (i, l) in enumerate(f):
- orphaned = False
- l = l.rstrip()
-
- # match lines of the form '<package> <maintainer(s)|status>'
- match = re.match(r'^(\S+)\s+(.+)$', l)
- if match:
- pkg = match.group(1)
- rest = match.group(2)
-
- # does rest starts with a status in all caps?
- status_match = re.match(r'^([A-Z]{2,})\b.*$', rest)
- if status_match:
- status = status_match.group(1)
-
- # packages marked as 'OBSOLETE' are obsolete
- if status == 'OBSOLETE':
- # obsolete packages have no maintainer
- #
- # XXX: perhaps disallow even trusties to upload (or
- # warn if they try?)
- m = ''
-
- # orphaned packages are assigned to 'ORPHANED'
- elif status == 'ORPHANED':
- m = status
- orphaned = True
-
- # also add any previous maintainer(s) listed
- prevm = re.match(r'^ORPHANED\s\((.*)\)', rest)
- if prevm:
- m = m + '/' + prevm.group(1)
- else:
- logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l))
- continue
- else:
- m = rest
+ def _split_maintainer_names(m, teams=None):
# joint maintainers are separated by '/'
maintainers = list()
+ groups = list()
+
for name in m.split('/'):
if not name:
continue
@@ -206,15 +178,85 @@ def _read_pkglist(pkglist):
try:
name.encode('ascii')
except UnicodeError:
- logging.error("non-ascii maintainer name '%s' in line %s:%d, skipped" % (rest, pkglist, i))
+ logging.error("non-ascii maintainer name '%s' in %s:%d: '%s', skipped" % (name, pkglist, i, l))
continue
- maintainers.append(name)
-
- mpkgs[pkg] = MaintainerPackage(pkg, maintainers, orphaned)
+ if name.startswith('@'):
+ groups.append(name[1:])
+ if teams and name in teams:
+ for n in teams[name]:
+ if n not in maintainers:
+ maintainers.append(n)
+ else:
+ logging.error("unknown team '%s' in %s:%d: '%s', skipped" % (name, pkglist, i, l))
+ else:
+ # avoid adding name if it's already in the list (a set
+ # is not appropriate here, as we have the concept of
+ # 'first named maintainer'
+ if name not in maintainers:
+ maintainers.append(name)
+
+ return maintainers, groups
+
+ if l.startswith('#'):
+ # comment
+ continue
+ elif l.startswith('@'):
+ # 'team' definition of the form '@<team> <maintainer(s)>'
+ match = re.match(r'^(\S+)\s+(.+)$', l)
+ if match:
+ team = match.group(1)
+ rest = match.group(2)
+
+ teams[team], _ = _split_maintainer_names(rest)
+ continue
else:
- logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l))
+ # package
+ orphaned = False
+ l = l.rstrip()
+
+ # match lines of the form '<package> <maintainer(s)|status>'
+ match = re.match(r'^(\S+)\s+(.+)$', l)
+ if match:
+ pkg = match.group(1)
+ rest = match.group(2)
+
+ # does rest starts with a status in all caps?
+ status_match = re.match(r'^([A-Z]{2,})\b.*$', rest)
+ if status_match:
+ status = status_match.group(1)
+
+ # packages marked as 'OBSOLETE' are obsolete
+ if status == 'OBSOLETE':
+ # obsolete packages have no maintainer
+ #
+ # XXX: perhaps disallow even trusties to upload (or
+ # warn if they try?)
+ m = ''
+
+ # orphaned packages are assigned to 'ORPHANED'
+ elif status == 'ORPHANED':
+ m = status
+ orphaned = True
+
+ # also add any previous maintainer(s) listed
+ prevm = re.match(r'^ORPHANED\s\((.*)\)', rest)
+ if prevm:
+ m = m + '/' + prevm.group(1)
+ else:
+ logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l))
+ continue
+ else:
+ m = rest
+
+ maintainers, groups = _split_maintainer_names(m, teams)
+
+ mpkgs[pkg] = MaintainerPackage(pkg, maintainers, groups, orphaned)
+ continue
+
+ # couldn't handle the line
+ logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l))
return mpkgs
@@ -259,3 +301,13 @@ def update_reminder_times(mlist):
# a list of all packages
def all_packages(pkglist):
return pkg_list(pkglist).keys()
+
+
+#
+#
+#
+
+if __name__ == "__main__":
+ from . import common_constants
+ p = pkg_list(common_constants.PKGMAINT)
+ print(p['xwininfo'].maintainers())
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index a4cd59a..43591ff 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -271,10 +271,14 @@ def update_package_listings(args, packages):
m_pn = es_po.orig_name
if m_pn not in pkg_maintainers:
m = None
- elif pkg_maintainers[m_pn].is_orphaned():
- m = 'ORPHANED'
+ pkg_groups = None
else:
- m = ', '.join(sorted(pkg_maintainers[m_pn].maintainers()))
+ if pkg_maintainers[m_pn].is_orphaned():
+ m = 'ORPHANED'
+ else:
+ m = ', '.join(sorted(pkg_maintainers[m_pn].maintainers()))
+
+ pkg_groups = pkg_maintainers[m_pn].groups()
if m:
print('<span class="detail">maintainer(s)</span>: %s ' % m, file=f)
@@ -284,6 +288,10 @@ def update_package_listings(args, packages):
<a href="/problems.html#personal-email">Do not contact the maintainer(s) directly</a>.)</span>'''), file=f)
print('<br><br>', file=f)
+ if pkg_groups:
+ print('<span class="detail">group</span>: %s ' % ','.join(pkg_groups), file=f)
+ print('<br><br>', file=f)
+
if po.kind == package.Kind.source:
repo = '/git/cygwin-packages/%s.git' % pn
if os.path.exists(repo):
diff --git a/calm/repology.py b/calm/repology.py
index 45a0da6..bb2595b 100644
--- a/calm/repology.py
+++ b/calm/repology.py
@@ -43,7 +43,13 @@ use_legacy = {'qt': [LegacyData('5', []),
LegacyData('4', []),
LegacyData('3', [])],
'gtk': [LegacyData('3', ['3.9', '+']),
- LegacyData('2', [])]}
+ LegacyData('2', [])],
+ 'gtksourceview': [LegacyData('2', []),
+ LegacyData('3', []),
+ LegacyData('4', []),
+ LegacyData('5', []),
+ ]
+ }
def repology_fetch_versions():
@@ -99,7 +105,7 @@ def repology_fetch_versions():
# packages
#
# (multiple cygwin source packages can correspond to a single
- # canonical repology package name, e.g. foo and mingww64-arch-foo)
+ # canonical repology package name, e.g. foo and mingw64-arch-foo)
for i in p:
if i['repo'] == "cygwin":
source_pn = i['srcname']
diff --git a/calm/reports.py b/calm/reports.py
index a9a3f4b..b101ef4 100644
--- a/calm/reports.py
+++ b/calm/reports.py
@@ -57,6 +57,15 @@ def template(title, body, f):
</html>'''), file=f)
+def write_report(args, title, body, fn, reportlist):
+ reportlist[title] = os.path.join('reports', fn)
+
+ fn = os.path.join(args.htdocs, 'reports', fn)
+
+ with utils.open_amifc(fn) as f:
+ template(title, body.getvalue(), f)
+
+
def linkify(pn, po):
return '<a href="/packages/summary/{0}.html">{1}</a>'.format(pn, po.orig_name)
@@ -64,7 +73,7 @@ def linkify(pn, po):
#
# produce a report of unmaintained packages
#
-def unmaintained(args, packages, reportsdir):
+def unmaintained(args, packages, reportlist):
pkg_maintainers = maintainers.pkg_list(args.pkglist)
um_list = []
@@ -123,14 +132,12 @@ def unmaintained(args, packages, reportsdir):
print('</table>', file=body)
- unmaintained = os.path.join(reportsdir, 'unmaintained.html')
- with utils.open_amifc(unmaintained) as f:
- template('Unmaintained packages', body.getvalue(), f)
+ write_report(args, 'Unmaintained packages', body, 'unmaintained.html', reportlist)
# produce a report of deprecated packages
#
-def deprecated(args, packages, reportsdir):
+def deprecated(args, packages, reportlist):
dep_list = []
arch = 'x86_64'
@@ -183,15 +190,13 @@ def deprecated(args, packages, reportsdir):
print('</table>', file=body)
- deprecated = os.path.join(reportsdir, 'deprecated_so.html')
- with utils.open_amifc(deprecated) as f:
- template('Deprecated shared library packages', body.getvalue(), f)
+ write_report(args, 'Deprecated shared library packages', body, 'deprecated_so.html', reportlist)
# produce a report of packages which need rebuilding for the latest major
# version version provides
#
-def provides_rebuild(args, packages, reportfile, provide_package):
+def provides_rebuild(args, packages, fn, provide_package, reportlist):
pr_list = []
arch = 'x86_64'
@@ -246,8 +251,7 @@ def provides_rebuild(args, packages, reportfile, provide_package):
print('</table>', file=body)
- with utils.open_amifc(reportfile) as f:
- template('Packages needing rebuilds for latest %s' % provide_package, body.getvalue(), f)
+ write_report(args, 'Packages needing rebuilds for latest %s' % provide_package, body, fn, reportlist)
#
@@ -255,11 +259,21 @@ def do_reports(args, packages):
if args.dryrun:
return
- reportsdir = os.path.join(args.htdocs, 'reports')
- pkg2html.ensure_dir_exists(args, reportsdir)
+ reportlist = {}
+
+ pkg2html.ensure_dir_exists(args, os.path.join(args.htdocs, 'reports'))
+
+ unmaintained(args, packages, reportlist)
+ deprecated(args, packages, reportlist)
- unmaintained(args, packages, reportsdir)
- deprecated(args, packages, reportsdir)
+ provides_rebuild(args, packages, 'perl_rebuilds.html', 'perl_base', reportlist)
+ provides_rebuild(args, packages, 'ruby_rebuilds.html', 'ruby', reportlist)
- provides_rebuild(args, packages, os.path.join(reportsdir, 'perl_rebuilds.html'), 'perl_base')
- provides_rebuild(args, packages, os.path.join(reportsdir, 'ruby_rebuilds.html'), 'ruby')
+ fn = os.path.join(args.htdocs, 'reports_list.inc')
+ with utils.open_amifc(fn) as f:
+ print('<ul>', file=f)
+ for r in reportlist:
+ print('<li>', file=f)
+ print('<a href="%s">%s</a>' % (reportlist[r], r), file=f)
+ print('</li>', file=f)
+ print('</ul>', file=f)
diff --git a/calm/scallywag_db.py b/calm/scallywag_db.py
index a310c55..ceb20da 100644
--- a/calm/scallywag_db.py
+++ b/calm/scallywag_db.py
@@ -52,5 +52,4 @@ def do_deploys(cb):
status = 'deploy failed'
conn.execute("UPDATE jobs SET status = ? WHERE id = ?", (status, r.id))
-
- conn.commit()
+ conn.commit()
diff --git a/test/testdata/process_arch/htdocs.expected b/test/testdata/process_arch/htdocs.expected
index 55585d6..d9afd07 100644
--- a/test/testdata/process_arch/htdocs.expected
+++ b/test/testdata/process_arch/htdocs.expected
@@ -1,4 +1,4 @@
-{'.': ['calm.db', 'packages.inc', 'src_packages.inc'],
+{'.': ['calm.db', 'packages.inc', 'reports_list.inc', 'src_packages.inc'],
'reports': ['deprecated_so.html', 'perl_rebuilds.html', 'ruby_rebuilds.html', 'unmaintained.html'],
'summary': ['arc-src.html',
'arc.html',
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2023-06-11 13:55 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-06-11 13:55 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-33-g6012a2f Jon Turney
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).