[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20220627-2-g78dc005

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20220627-2-g78dc005

[Jon TURNEY]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=78dc0051e7f53830f72c219c437a5b175906dfd2

commit 78dc0051e7f53830f72c219c437a5b175906dfd2
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Jun 30 19:07:20 2022 +0100

    Improve deprecated soversion expiry (again)
    
    "unused" should mean "no packages from a different source package depend
    on it", not "no packages depend on it".
    
    e.g. 'bind' produces both 'libdns165', and 'libbind9_140' which depends
    on it.  Both are eligible for expiry when they have no other rdepends
    outside of 'bind'.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=eca3a88d174516051883b5c7404f6985939d4b4e

commit eca3a88d174516051883b5c7404f6985939d4b4e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed May 5 14:08:46 2021 +0100

    Improve handling of old-style obsoletion packages
    
    Identify old-style obsoletion packages (where the package is empty, has
    a category of _obsolete, and requires: it's replacement), and upgrade
    them to the new style (where the obsoleting package has an obsoletes:).
    
    This is done per-arch, simply because lots of these historic obsoletion
    packages only exist for x86, and it's redundant to obsolete: them on
    x86_64.
    
    Treating this similarly to past_mistakes.missing_obsoletes means that a
    warning occurs when new version of package is missing that expected
    obsolete:
    
    Initially only apply to packages over a certain age, so we can observe
    the effect on a small number of packages to make checking it's doing the
    right thing easier.


Diff:
---
 calm/calm.py          |  12 ++++--
 calm/package.py       | 104 +++++++++++++++++++++++++++++++++++++++++---------
 calm/past_mistakes.py |  36 +++++++++++++++++
 3 files changed, 131 insertions(+), 21 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index 49bdcfd..04db081 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -89,6 +89,7 @@ class CalmState(object):
         self.subject = ''
         self.packages = {}
         self.valid_provides = set()
+        self.missing_obsolete = {}
 
 
 #
@@ -104,10 +105,13 @@ def process_relarea(args, state):
         logging.debug("reading existing packages for arch %s" % (arch))
         packages[arch] = package.read_packages(args.rel_area, arch)
 
-    # validate the package set for each arch
     state.valid_provides = db.update_package_names(args, packages)
     for arch in common_constants.ARCHES:
-        if not package.validate_packages(args, packages[arch], state.valid_provides):
+        state.missing_obsolete[arch] = package.upgrade_oldstyle_obsoletes(packages[arch])
+
+    # validate the package set for each arch
+    for arch in common_constants.ARCHES:
+        if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete[arch]):
             logging.error("existing %s package set has errors" % (arch))
             error = True
 
@@ -208,7 +212,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc):
     state.valid_provides = db.update_package_names(args, merged_packages)
     for arch in common_constants.ARCHES:
         logging.debug("validating merged %s package set for maintainer %s" % (arch, name))
-        if not package.validate_packages(args, merged_packages[arch], state.valid_provides):
+        if not package.validate_packages(args, merged_packages[arch], state.valid_provides, state.missing_obsolete):
             logging.error("error while validating merged %s packages for %s" % (arch, name))
             valid = False
 
@@ -329,7 +333,7 @@ def remove_stale_packages(args, packages, state):
     error = False
     state.valid_provides = db.update_package_names(args, packages)
     for arch in common_constants.ARCHES:
-        if not package.validate_packages(args, packages[arch], state.valid_provides):
+        if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete):
             logging.error("%s package set has errors after removing stale packages" % arch)
             error = True
 
diff --git a/calm/package.py b/calm/package.py
index 482d042..32b4d31 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -496,15 +496,81 @@ def sort_key(k):
     return k
 
 
+#
+# generate missing_obsolete data for upgrading old-style obsoletion install
+# packages:
+#
+# they are empty, have the '_obsolete' category, and requires: exactly 1
+# package, their replacement.
+#
+# generate a record to add an obsoletes: header to the replacement package.
+#
+
+OBSOLETE_AGE_THRESHOLD_YEARS = 20
+
+
+def upgrade_oldstyle_obsoletes(packages):
+    missing_obsolete = {}
+    certain_age = time.time() - (OBSOLETE_AGE_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60)
+    logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age))))
+
+    for p in sorted(packages):
+        if packages[p].kind == Kind.binary:
+            for vr in packages[p].versions():
+                # initially apply to a subset over a certain age, to gradually
+                # introduce this change
+                mtime = packages[p].tar(vr).mtime
+                if mtime > certain_age:
+                    continue
+                logging.debug("_obsolete package '%s' version '%s' mtime '%s' is over cut-off age" % (p, vr, time.strftime("%F %T %Z", time.localtime(mtime))))
+
+                if packages[p].tar(vr).is_empty:
+                    if '_obsolete' in packages[p].version_hints[vr]['category']:
+                        requires = packages[p].version_hints[vr].get('requires', '').split()
+
+                        if p in past_mistakes.old_style_obsolete_by:
+                            o = past_mistakes.old_style_obsolete_by[p]
+
+                            # empty replacement means "ignore"
+                            if not o:
+                                continue
+
+                            logging.debug('%s is hardcoded as obsoleted by %s ' % (p, o))
+
+                        else:
+                            if len(requires) == 0:
+                                # obsolete but has no replacement
+                                logging.warning('%s is obsolete, but has no replacement' % (p))
+                                continue
+                            elif len(requires) == 1:
+                                o = requires[0]
+                            elif len(requires) >= 2:
+                                # obsolete with multiple replacements (pick one?)
+                                logging.warning('%s %s is obsoleted by %d packages (%s)' % (p, vr, len(requires), requires))
+                                continue
+
+                        if o in packages:
+                            if o not in missing_obsolete:
+                                missing_obsolete[o] = set()
+
+                            missing_obsolete[o].add(p)
+                            logging.info("converting from empty, _obsolete category package '%s' to 'obsoletes: %s' in package '%s'" % (p, p, o))
+
+    return missing_obsolete
+
+
 #
 # validate the package database
 #
-def validate_packages(args, packages, valid_requires_extra=None):
+def validate_packages(args, packages, valid_requires_extra=None, missing_obsolete_extra=None):
     error = False
 
     if packages is None:
         return False
 
+    if missing_obsolete_extra is None:
+        missing_obsolete_extra = {}
+
     # build the set of valid things to requires: etc.
     valid_requires = set()
 
@@ -573,17 +639,19 @@ def validate_packages(args, packages, valid_requires_extra=None):
             # some old packages are missing needed obsoletes:, add them where
             # needed, and make sure the uploader is warned if/when package is
             # updated
-            if p in past_mistakes.missing_obsolete:
-                obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',')
-                obsoletes = [o.strip() for o in obsoletes]
-                obsoletes = [o for o in obsoletes if o]
-
-                needed = past_mistakes.missing_obsolete[p]
-                for n in needed:
-                    if n not in obsoletes:
-                        obsoletes.append(n)
-                        packages[p].version_hints[v]['obsoletes'] = ','.join(obsoletes)
-                        logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v))
+            for mo in [past_mistakes.missing_obsolete, missing_obsolete_extra]:
+                if p in mo:
+                    obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',')
+                    obsoletes = [o.strip() for o in obsoletes]
+                    obsoletes = [o for o in obsoletes if o]
+
+                    # XXX: this needs to recurse so we don't drop transitive missing obsoletes?
+                    needed = mo[p]
+                    for n in sorted(needed):
+                        if n not in obsoletes:
+                            obsoletes.append(n)
+                            packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes)
+                            logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v))
 
             # if external-source is used, the package must exist
             if 'external-source' in hints:
@@ -1383,15 +1451,17 @@ def stale_packages(packages):
         # shouldn't retain anything.
         #
         # - shared library packages which don't come from the current version of
-        # source (i.e. is superseded or removed), have no packages which depend
-        # on them, and are over a certain age
+        # source (i.e. is superseded or removed), have no packages from a
+        # different source package which depend on them, and are over a certain
+        # age
         #
         mark = Freshness.fresh
         if pn.endswith('-debuginfo'):
             mark = Freshness.conditional
-        if (len(po.rdepends) == 0) and re.match(common_constants.SOVERSION_PACKAGE_RE, pn):
-            bv = po.best_version
-            es = po.version_hints[bv].get('external-source', None)
+        bv = po.best_version
+        es = po.version_hints[bv].get('external-source', None)
+        if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and
+            not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)):
             if es and (packages[es].best_version != bv):
                 def dep_so_age_mark(v):
                     mtime = po.tar(v).mtime
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index af6d650..c0e724f 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -220,3 +220,39 @@ empty_source = {
     'pinentry-qt3-src': ['0.7.6-3'],             # obsoleted by pinentry-qt
     'xerces-c-devel-src': ['2.8.0-1'],           # obsoleted by libxerces-c-devel
 }
+
+# additional data for the heuristic for upgrading old-style obsoletion packages
+old_style_obsolete_by = {
+    'at-spi2-atk': 'libatk-bridge2.0_0',
+    'qt-gstreamer': 'libQtGStreamer1_0_0',
+    'lighttpd-mod_trigger_b4_dl': 'lighttpd',
+    # these require: both epoch1 and epoch2 replacements, but epoch1 contains
+    # the same version
+    'gcc-tools-autoconf': 'gcc-tools-epoch1-autoconf',
+    'gcc-tools-automake': 'gcc-tools-epoch1-automake',
+    # these are odd and only exist to record an optional dependency on the
+    # language runtime (dynamically loaded at runtime), which is also noted in
+    # build-requires:
+    'vim-lua': 'vim',
+    'vim-perl': 'vim',
+    'vim-python': 'vim',
+    'vim-python3': 'vim',
+    'vim-ruby': 'vim',
+    # (An empty replacement means "don't apply this heuristic". We use that to
+    # not bother with some x86-only packages which have complex replacements,
+    # since they will be going away relatively soon anyhow...)
+    'SuiteSparse': '',
+    'libSuiteSparse-devel': '',
+    'libksba': '',
+    'libstdc++6-devel': '',
+    'octave-forge': '',
+    'plotutils-devel': '',
+    'rpm-doc': '',
+    'tetex-base': '',
+    'tetex-bin': '',
+    'tetex-extra': '',
+    'tetex-tiny': '',
+    'tetex-x11': '',
+    'texlive-collection-langtibetan': '',
+    'texlive-collection-texinfo': '',
+}