[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-72-g4914ed4

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-72-g4914ed4

[Jon Turney]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4914ed4985b30b9a87a093b6fb8b8e3a12b71eaa

commit 4914ed4985b30b9a87a093b6fb8b8e3a12b71eaa
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Mar 23 21:40:03 2024 +0000

    Add a report of python packages which need rebuilding

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bde9b2159630c4251f0aeb0003a19c9a0b425134

commit bde9b2159630c4251f0aeb0003a19c9a0b425134
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Mar 23 15:49:26 2024 +0000

    Future proof the re for "python modules for old python versions"

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9d7f014e4004ab1723c84aa42728410bfd70d1b4

commit 9d7f014e4004ab1723c84aa42728410bfd70d1b4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Mar 24 14:10:14 2024 +0000

    Relocate lockfile
    
    Relocate lockfile somewhere writeable by cygwin-admin uid

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=91d07c8c3eee66cfcd2e7b382e42f7a52bfb6509

commit 91d07c8c3eee66cfcd2e7b382e42f7a52bfb6509
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Mar 23 15:42:26 2024 +0000

    Record reason for calm daemon stopping in logfile


Diff:
---
 calm/calm.py    |  11 +++---
 calm/package.py |   2 +-
 calm/reports.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 113 insertions(+), 7 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index f9baea7..65e75dc 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -784,17 +784,18 @@ def do_daemon(args, state):
             #
             # so we arrange for signals to raise an InterruptedError
             # exception, to pop out here
-            irk.irk("calm daemon stopped by SIGTERM")
+            stop_reason = "calm daemon stopped by SIGTERM"
 
         except Exception as e:
             with BufferingSMTPHandler(toaddrs=args.email, subject='calm stopping due to unhandled exception'):
                 logging.error("exception %s" % (type(e).__name__), exc_info=True)
-            irk.irk("calm daemon stopped due to unhandled exception")
+            stop_reason = "calm daemon stopped due to unhandled exception"
 
         else:
-            irk.irk("calm daemon stopped for unknown reason")
+            stop_reason = "calm daemon stopped for unknown reason"
 
-        logging.info("calm daemon stopped")
+        irk.irk(stop_reason)
+        logging.info(stop_reason)
 
 
 def mail_logs(state):
@@ -880,7 +881,7 @@ def main():
     homedir_default = common_constants.HOMEDIR
     stagingdir_default = common_constants.STAGINGDIR
     trustedmaint_default = common_constants.TRUSTEDMAINT
-    pidfile_default = '/sourceware/cygwin-staging/calm.pid'
+    pidfile_default = '/sourceware/cygwin-staging/lock/calm.pid'
     pkglist_default = common_constants.PKGMAINT
     relarea_default = common_constants.FTP
     repodir_default = '/git/cygwin-packages'
diff --git a/calm/package.py b/calm/package.py
index 39f3655..3b530cb 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -1046,7 +1046,7 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
 
             # ignore Python module packages, as we may keep old versions of
             # those
-            if re.match(r'^python[23][5678]?-.*', install_p):
+            if re.match(r'^python3\d+-.*', install_p):
                 continue
 
             # ignore packages where best_version is a test version (i.e doesn't
diff --git a/calm/reports.py b/calm/reports.py
index d204032..54a0d76 100644
--- a/calm/reports.py
+++ b/calm/reports.py
@@ -286,7 +286,7 @@ def provides_rebuild(args, packages, fn, provide_package, reportlist):
                 break
 
     body = io.StringIO()
-    print(' <p>Packages whose latest version depends on a version provides: other than %s.</p>' % pp_provide, file=body)
+    print('<p>Packages whose latest version depends on a version provides: other than %s.</p>' % pp_provide, file=body)
 
     print('<table class="grid sortable">', file=body)
     print('<tr><th>package</th><th>srcpackage</th><th>version</th><th>depends</th></tr>', file=body)
@@ -300,6 +300,110 @@ def provides_rebuild(args, packages, fn, provide_package, reportlist):
     write_report(args, 'Packages needing rebuilds for latest %s' % provide_package, body, fn, reportlist, bool(pr_list))
 
 
+# produce a report of python modules/bindings/linked packages which depend on
+# non-latest version of python
+def python_rebuild(args, packages, fn, reportlist):
+    pr_list = []
+
+    # XXX: look into how we can change this, after x86 is dropped
+    arch = 'x86_64'
+
+    # assume that python3 depends on the latest python3n package
+    py_package = packages[arch].get('python3', None)
+    if not py_package:
+        return
+
+    latest_py = py_package.version_hints[py_package.best_version]['depends'].split(', ')[0]
+
+    modules = {}
+
+    for p in packages[arch]:
+        po = packages[arch][p]
+        bv = po.best_version
+
+        if po.obsoleted_by:
+            continue
+
+        depends = packages[arch][p].version_hints[bv]['depends'].split(', ')
+        depends = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in depends]
+
+        for d in depends:
+            # scan for a 'pythonnn' dependency
+            if not re.match(r'python\d+$', d):
+                continue
+
+            # if it's a generic python dependency, it's ok
+            if d == 'python3':
+                continue
+
+            # if this package is called 'idlenn', it's ok
+            if p == d.replace('python', 'idle'):
+                break
+
+            # if this package is called 'pythonnn-foo', it's ok, and remember
+            # the module/binding name and version
+            if p.startswith(d + '-'):
+                name = p[len(d) + 1:]
+
+                if name not in modules:
+                    modules[name] = []
+
+                ver = int(d[6:])
+                modules[name].append(ver)
+
+                break
+
+            # if it depends on the latest python version, this package is ok
+            if d == latest_py:
+                break
+
+            # requires an old python version
+            pr = types.SimpleNamespace()
+            pr.pn = p
+            pr.po = po
+            pr.spn = po.srcpackage(bv)
+            pr.spo = packages[arch][pr.spn]
+            pr.depends = d
+            pr.bv = bv
+
+            pr_list.append(pr)
+
+            break
+
+    # now look at list of module/bindings we've made
+    latest_ver = int(latest_py[6:])
+    for m in modules:
+        highest_ver = sorted(modules[m], reverse=True)[0]
+        if highest_ver == latest_ver:
+            continue
+
+        # if module/binding doesn't exist for latest python version, indicate
+        # that it needs updating
+        pr = types.SimpleNamespace()
+        pr.pn = 'python' + str(highest_ver) + '-' + m
+        pr.po = packages[arch][pr.pn]
+        pr.spn = pr.po.srcpackage(pr.po.best_version)
+        pr.spo = packages[arch][pr.spn]
+        pr.depends = 'python' + str(highest_ver)
+        pr.bv = pr.po.best_version
+
+        pr_list.append(pr)
+
+    body = io.StringIO()
+    print('<p>Packages for python module or binding for, or linkage to, a python version other than %s.</p>' % latest_py, file=body)
+
+    print('<table class="grid sortable">', file=body)
+    print('<tr><th>package</th><th>srcpackage</th><th>version</th><th>depends</th></tr>', file=body)
+
+    for pr in sorted(pr_list, key=lambda i: (i.depends, i.pn)):
+        print('<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' %
+              (linkify(pr.pn, pr.po), linkify(pr.spn, pr.spo), pr.bv, pr.depends), file=body)
+
+    print('</table>', file=body)
+
+    write_report(args, 'Packages needing rebuilds for latest python', body, fn, reportlist, bool(pr_list))
+
+
 #
 def do_reports(args, packages):
     if args.dryrun:
@@ -315,6 +419,7 @@ def do_reports(args, packages):
 
     provides_rebuild(args, packages, 'perl_rebuilds.html', 'perl_base', reportlist)
     provides_rebuild(args, packages, 'ruby_rebuilds.html', 'ruby', reportlist)
+    python_rebuild(args, packages, 'python_rebuilds.html', reportlist)
 
     fn = os.path.join(args.htdocs, 'reports_list.inc')
     with utils.open_amifc(fn) as f: