[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-44-gcda9f05

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-44-gcda9f05

[Jon Turney]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cda9f0512545bf5374cbed851e58140537a38f39

commit cda9f0512545bf5374cbed851e58140537a38f39
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Oct 14 21:34:10 2023 +0100

    Expire python2 modules
    
    For the first time, we are expiring obsolete packages just because they
    are obsolete, so it is time for the previous commit, to ensure we retain
    any obsolete: hints they might have caused to be generated.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=17dc61e9f95c74366465d0a56121cef84b88d415

commit 17dc61e9f95c74366465d0a56121cef84b88d415
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Jun 20 16:52:08 2022 +0100

    Persist missing_obsolete
    
    Persist the missing obsolete: data generated from old-style obsoletion
    packages, so it is remembered, even if after the obsoleted package is
    removed.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=72608dc7f48988a3509538d0a01411c2586a513f

commit 72608dc7f48988a3509538d0a01411c2586a513f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Oct 14 15:02:27 2023 +0100

    Simplify 'conditionally retained' package marking
    
    Move the logic into a single function, rather than selectively defining
    a callback function depending on if the package meets some criteria.


Diff:
---
 calm/calm.py          |   2 +-
 calm/db.py            |  34 +++++++++++
 calm/package.py       | 157 +++++++++++++++++++++++---------------------------
 calm/past_mistakes.py |   4 ++
 4 files changed, 111 insertions(+), 86 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index 22b0a7a..73b0e05 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -111,7 +111,7 @@ def process_relarea(args, state):
 
     state.valid_provides = db.update_package_names(args, packages)
     for arch in common_constants.ARCHES:
-        state.missing_obsolete[arch] = package.upgrade_oldstyle_obsoletes(packages[arch])
+        state.missing_obsolete[arch] = db.update_missing_obsolete(args, packages, arch)
 
     # validate the package set for each arch
     for arch in common_constants.ARCHES:
diff --git a/calm/db.py b/calm/db.py
index b7f7040..0aef83a 100644
--- a/calm/db.py
+++ b/calm/db.py
@@ -30,6 +30,7 @@ import logging
 import os
 import sqlite3
 
+from . import package
 from . import utils
 
 
@@ -48,6 +49,12 @@ def connect(args):
                      vr TEXT NOT NULL
                     )''')
 
+    conn.execute('''CREATE TABLE IF NOT EXISTS missing_obsolete
+                    (name TEXT NOT NULL,
+                     arch TEXT NOT NULL,
+                     replaces TEXT NOT NULL,
+                     PRIMARY KEY (name, arch)
+                    )''')
     conn.commit()
 
     return conn
@@ -104,3 +111,30 @@ def vault_requests(args):
 def vault_request_add(args, p, v):
     with connect(args) as conn:
         conn.execute('INSERT INTO vault_requests (srcpackage, vr) VALUES (?,?)', (p, v))
+
+
+#
+# this accumulates missing_obsoletes data for packages, so we will remember it
+# even after the obsoleted package has been removed
+#
+def update_missing_obsolete(args, packages, arch):
+    data = {}
+    with connect(args) as conn:
+        conn.row_factory = sqlite3.Row
+
+        # read
+        cur = conn.execute("SELECT name, replaces FROM missing_obsolete WHERE arch = ?", (arch,))
+        for row in cur.fetchall():
+            data[row['name']] = set(row['replaces'].split())
+
+        # update missing obsoletes data
+        missing_obsolete = package.upgrade_oldstyle_obsoletes(packages[arch], data.copy())
+
+        # update
+        for n, r in missing_obsolete.items():
+            if n not in data:
+                conn.execute('INSERT INTO missing_obsolete (name, arch, replaces) VALUES (?, ? , ?)', (n, arch, ' '.join(r)))
+            else:
+                conn.execute('UPDATE missing_obsolete SET replaces = ? WHERE name = ? AND arch = ?', (' '.join(r), n, arch))
+
+    return missing_obsolete
diff --git a/calm/package.py b/calm/package.py
index f60c238..e45874c 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -584,11 +584,10 @@ def sort_key(k):
 # generate a record to add an obsoletes: header to the replacement package.
 #
 
-OBSOLETE_CONVERT_THRESHOLD_YEARS = 20
+OBSOLETE_CONVERT_THRESHOLD_YEARS = 2
 
 
-def upgrade_oldstyle_obsoletes(packages):
-    missing_obsolete = {}
+def upgrade_oldstyle_obsoletes(packages, missing_obsolete):
     certain_age = time.time() - (OBSOLETE_CONVERT_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60)
     logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age))))
 
@@ -1531,12 +1530,68 @@ class Freshness(IntEnum):
 
 
 def mark_package_fresh(packages, p, v, mark=Freshness.fresh):
-    if callable(mark):
-        mark = mark(v)
-
     packages[p].tar(v).fresh = mark
 
 
+#
+# helper function evaluate if package needs marking for conditional retention
+#
+
+def mark_fn(packages, po, v, certain_age, vault_requests):
+    pn = po.name
+    bv = po.best_version
+
+    # 'conditional' package retention means the package is weakly retained.
+    # This allows total expiry when a source package no longer provides
+    # anything useful:
+    #
+    # - if all we have is a source package and a debuginfo package, then we
+    # shouldn't retain anything.
+    #
+    if pn.endswith('-debuginfo'):
+        return Freshness.conditional
+
+    # - shared library packages which don't come from the current version of
+    # source (i.e. is superseded or removed), have no packages from a
+    # different source package which depend on them, and are over a certain
+    # age
+    #
+    es = po.version_hints[bv].get('external-source', None)
+    if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and
+        not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)):
+        if es and (packages[es].best_version != bv):
+            mtime = po.tar(v).mtime
+            if mtime < certain_age:
+                logging.debug("deprecated soversion package '%s' version '%s' mtime '%s' is over cut-off age" % (pn, v, time.strftime("%F %T %Z", time.localtime(mtime))))
+                return Freshness.conditional
+
+    # - if package depends on anything in expired_provides
+    #
+    requires = po.version_hints[v].get('depends', '').split(', ')
+    if re.match(r'^python(|2|27)-', pn):
+        if any(ep in requires for ep in past_mistakes.expired_provides) or po.obsolete:
+            logging.info("package '%s' version '%s' not retained as it requires a provide known to be expired" % (pn, v))
+            return Freshness.conditional
+
+    # - explicitly marked as 'noretain'
+    #
+    if 'noretain' in po.override_hints:
+        noretain_versions = po.override_hints.get('noretain', '').split()
+        if (v in noretain_versions) or ('all' in noretain_versions):
+            return Freshness.conditional
+
+    # - marked via 'calm-tool vault'
+    #
+    es = po.srcpackage(bv, suffix=False)
+    if es in vault_requests:
+        if v in vault_requests[es]:
+            logging.info("package '%s' version '%s' not retained due vault request" % (pn, v))
+            return Freshness.conditional
+
+    # otherwise, make no change
+    return Freshness.fresh
+
+
 #
 # construct a move list of stale packages
 #
@@ -1553,83 +1608,8 @@ def stale_packages(packages, vault_requests):
         if po.kind != Kind.binary:
             continue
 
-        mark = Freshness.fresh
-
-        # 'conditional' package retention means the package is weakly retained.
-        # This allows total expiry when a source package no longer provides
-        # anything useful:
-        #
-        # - if all we have is a source package and a debuginfo package, then we
-        # shouldn't retain anything.
-        #
-        if pn.endswith('-debuginfo'):
-            mark = Freshness.conditional
-
-        # - shared library packages which don't come from the current version of
-        # source (i.e. is superseded or removed), have no packages from a
-        # different source package which depend on them, and are over a certain
-        # age
-        #
-        bv = po.best_version
-        es = po.version_hints[bv].get('external-source', None)
-        if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and
-            not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)):
-            if es and (packages[es].best_version != bv):
-                def dep_so_age_mark(v):
-                    mtime = po.tar(v).mtime
-                    if mtime < certain_age:
-                        logging.debug("deprecated soversion package '%s' version '%s' mtime '%s' is over cut-off age" % (pn, v, time.strftime("%F %T %Z", time.localtime(mtime))))
-                        return Freshness.conditional
-                    else:
-                        return Freshness.fresh
-
-                mark = dep_so_age_mark
-
-        # - if package depends on anything in expired_provides
-        #
-        all_reqs = set.union(*(set(po.version_hints[v].get('depends', '').split(', ')) for v in po.versions()))
-        if all_reqs.intersection(set(past_mistakes.expired_provides)):
-            def expired_provides_mark(v):
-                requires = po.version_hints[v].get('depends', '').split(', ')
-                if any(ep in requires for ep in past_mistakes.expired_provides):
-                    # XXX: for the moment, don't allow this to expire the
-                    # current version, though!
-                    if v != po.best_version:
-                        logging.info("package '%s' version '%s' not retained as it requires a provide known to be expired" % (pn, v))
-                        return Freshness.conditional
-                    else:
-                        logging.info("package '%s' version '%s' requires a provide known to be expired, but not expired as it's the current version" % (pn, v))
-
-                return Freshness.fresh
-
-            mark = expired_provides_mark
-
-        # - explicitly marked as 'noretain'
-        #
-        if 'noretain' in po.override_hints:
-            def noretain_hint_mark(v):
-                noretain_versions = po.override_hints.get('noretain', '').split()
-                if (v in noretain_versions) or ('all' in noretain_versions):
-                    return Freshness.conditional
-                else:
-                    return Freshness.fresh
-
-            mark = noretain_hint_mark
-
-        # - marked via 'calm-tool vault'
-        #
-        es = po.srcpackage(bv, suffix=False)
-        if es in vault_requests:
-            def vault_requests_mark(v):
-                if v in vault_requests[es]:
-                    logging.info("package '%s' version '%s' not retained due vault request" % (pn, v))
-                    return Freshness.conditional
-                else:
-                    return Freshness.fresh
-
-            mark = vault_requests_mark
-
-        # mark any versions explicitly listed in the keep: override hint (unconditionally)
+        # mark as fresh any versions explicitly listed in the keep: override
+        # hint (unconditionally)
         for v in po.override_hints.get('keep', '').split():
             if v in po.versions():
                 mark_package_fresh(packages, pn, v)
@@ -1643,7 +1623,7 @@ def stale_packages(packages, vault_requests):
             if 'test' not in po.version_hints[v]:
                 if keep_count <= 0:
                     break
-                mark_package_fresh(packages, pn, v, mark)
+                mark_package_fresh(packages, pn, v)
                 keep_count = keep_count - 1
 
         # mark as fresh the highest n test versions, where n is given by the
@@ -1656,7 +1636,7 @@ def stale_packages(packages, vault_requests):
             if 'test' in po.version_hints[v]:
                 if keep_count <= 0:
                     break
-                mark_package_fresh(packages, pn, v, mark)
+                mark_package_fresh(packages, pn, v)
                 keep_count = keep_count - 1
             else:
                 if 'keep-superseded-test' not in po.override_hints:
@@ -1674,6 +1654,13 @@ def stale_packages(packages, vault_requests):
                     newer = True
 
             if newer:
+                mark_package_fresh(packages, pn, v)
+
+        # overwrite with 'conditional' package retention mark if it meets
+        # various criteria
+        for v in sorted(po.versions(), key=lambda v: SetupVersion(v)):
+            mark = mark_fn(packages, po, v, certain_age, vault_requests)
+            if mark != Freshness.fresh:
                 mark_package_fresh(packages, pn, v, mark)
 
     # mark source packages as fresh if any install package which uses it is fresh
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index 6561fde..13dfd50 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -171,6 +171,8 @@ nonexistent_provides = historical_provides + [
     'python2',
     'python2-devel',
     'python27',
+    'python-pygments',
+    'python-lxml',
     '_windows',
     r'perl5_\d+',
     r'ruby_\d+',
@@ -180,6 +182,8 @@ nonexistent_provides = historical_provides + [
 
 # provides: which don't exist and packages which require them should be expired
 expired_provides = [
+    'python2',
+    'python27',
 ]
 
 # empty source packages