[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-89-g79e8308

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-89-g79e8308

[Jon Turney]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 21291 bytes --]




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=79e8308164e5b6d8dfcef05c68e492311552dc31

commit 79e8308164e5b6d8dfcef05c68e492311552dc31
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Apr 9 21:16:58 2024 +0100

    Add some more old-style obsoletion upgrade information
    
    These packages weren't being noticed before as being "obsolete, but no
    replacement" due to the way split() can produce a list containing an
    empty string, rather than an empty list.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ee039db724d8b555b2748bb9295e4ad0106babe4

commit ee039db724d8b555b2748bb9295e4ad0106babe4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Apr 9 19:43:08 2024 +0100

    Convert package lists in hints to actual lists
    
    Convert package lists in hints to actual lists in one place, rather than
    doing it every time we need to use the value.
    
    This fixes the bug that build-requires wasn't written to JSON as a list.
    
    Update test data appropriately.
    
    Future work: Factor out "strip out version constraints from a package
    list" into a utility function.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b768c25a36de53156cb11b129a0bac57bc5cac46

commit b768c25a36de53156cb11b129a0bac57bc5cac46
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Apr 8 16:34:04 2024 +0100

    Add repr for RepoPath objects to aid debugging


Diff:
---
 calm/package.py                                   | 81 +++++++++++++----------
 calm/past_mistakes.py                             |  7 ++
 calm/pkg2html.py                                  |  8 +--
 calm/reports.py                                   | 14 ++--
 calm/utils.py                                     |  7 +-
 test/testdata/process_arch/packages.json.expected |  4 +-
 test/testdata/uploads/pkglist.expected            | 14 ++--
 7 files changed, 74 insertions(+), 61 deletions(-)

diff --git a/calm/package.py b/calm/package.py
index efc01c4..abaec1b 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -84,6 +84,9 @@ class RepoPath():
     def move(self):
         return (os.path.join(self.arch, 'release', self.path), self.fn)
 
+    def __repr__(self):
+        return "RepoPath(%s, %s, %s)" % (self.arch, self.path, self.fn)
+
 
 # information we keep about a package
 class Package(object):
@@ -265,13 +268,30 @@ def read_hints(p, fn, kind, strict=False):
         for l in hints['parse-warnings']:
             logging.info("package '%s': %s" % (p, l))
 
-    # generate depends: from requires:
-    # XXX: store this as a list, rather than splitting it into one everywhere we
-    # use it
-    hints['depends'] = ', '.join(process_package_constraint_list(hints.get('requires', '')))
-    # erase requires:, to ensure there is nothing using it
+    # convert hint keys which have a value which is a list to an actual list (to
+    # avoid doing the splitting and whitespace handling everywhere)
+    #
+    # XXX: guarantee they exist and are an empty list if empty, so we don't need
+    # to check if they exist everywhere?)
+    for k in ['obsoletes', 'provides', 'conflicts', 'build-depends']:
+        if k in hints:
+            v = hints[k].strip()
+            if not v:
+                v = []
+            else:
+                # split on comma, remove any extraneous whitespace
+                v = [i.strip() for i in v.split(',')]
+            hints[k] = v
+
+    # 'depends' is special, generated from from requires:
+    hints['depends'] = process_package_constraint_list(hints.get('requires', ''))
+    # erase requires:, to ensure there is nothing still using it
     hints.pop('requires', None)
 
+    # disable check is just whitespace separated
+    if 'disable-check' in hints:
+        hints['disable-check'] = hints['disable-check'].split()
+
     return hints
 
 
@@ -624,7 +644,7 @@ def upgrade_oldstyle_obsoletes(packages, missing_obsolete):
                     continue
                 logging.debug("_obsolete package '%s' version '%s' mtime '%s' is over cut-off age" % (p, vr, time.strftime("%F %T %Z", time.localtime(mtime))))
 
-                requires = packages[p].version_hints[vr].get('depends', '').split(', ')
+                requires = packages[p].version_hints[vr].get('depends', [])
                 requires = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in requires]
 
                 o = None
@@ -642,7 +662,7 @@ def upgrade_oldstyle_obsoletes(packages, missing_obsolete):
 
                 else:
                     # ignore self-destruct packages
-                    provides = packages[p].version_hints[vr].get('provides', '')
+                    provides = packages[p].version_hints[vr].get('provides', [])
                     if '_self-destruct' in provides:
                         continue
 
@@ -690,7 +710,7 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
     for p in packages:
         valid_requires.add(p)
         for hints in packages[p].version_hints.values():
-            valid_requires.update(hints.get('provides', '').split())
+            valid_requires.update(hints.get('provides', []))
 
             # reset computed package state
             packages[p].has_requires = False
@@ -715,10 +735,7 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
             ]:
                 # if c is in hints, and not the empty string
                 if hints.get(c, ''):
-                    for r in hints[c].split(','):
-                        # remove any extraneous whitespace
-                        r = r.strip()
-
+                    for r in hints[c]:
                         # strip off any version relation enclosed in '()'
                         # following the package name
                         r = re.sub(r'(.*) +\(.*\)', r'\1', r)
@@ -764,15 +781,13 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
             if p in packages:
                 for v in packages[p].version_hints:
 
-                    obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',')
-                    obsoletes = [o.strip() for o in obsoletes]
-                    obsoletes = [o for o in obsoletes if o]
+                    obsoletes = packages[p].version_hints[v].get('obsoletes', [])
 
                     def add_needed_obsoletes(needed):
                         for n in sorted(needed):
                             if n not in obsoletes:
                                 obsoletes.append(n)
-                                packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes)
+                                packages[p].version_hints[v]['obsoletes'] = obsoletes
                                 logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v))
 
                             # recurse so we don't drop transitive missing obsoletes
@@ -791,10 +806,9 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
         # in read_hints(), so fix that up here.
     for p in sorted(packages):
         for hints in packages[p].version_hints.values():
-            obsoletes = hints.get('obsoletes', '')
+            obsoletes = hints.get('obsoletes', [])
             if obsoletes:
-                for o in obsoletes.split(','):
-                    o = o.strip()
+                for o in obsoletes:
                     o = re.sub(r'(.*) +\(.*\)', r'\1', o)
 
                     if o in packages:
@@ -802,10 +816,10 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
 
                         for (ov, ohints) in packages[o].version_hints.items():
                             if 'depends' in ohints:
-                                depends = ohints['depends'].split(', ')
+                                depends = ohints['depends']
                                 if p in depends:
                                     depends = [d for d in depends if d != p]
-                                    packages[o].version_hints[ov]['depends'] = ', '.join(depends)
+                                    packages[o].version_hints[ov]['depends'] = depends
                                     logging.debug("removed obsoleting '%s' from the depends: of package '%s'" % (p, o))
                     else:
                         logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p))
@@ -907,7 +921,7 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
                 if packages[p].tar(vr).is_empty:
                     # this classification relies on obsoleting packages
                     # not being present in depends
-                    if packages[p].version_hints[vr].get('depends', ''):
+                    if packages[p].version_hints[vr].get('depends', []):
                         # also allow '_obsolete' because old obsoletion
                         # packages depend on their replacement, but are not
                         # obsoleted by it
@@ -947,9 +961,7 @@ def validate_packages(args, packages, valid_provides_extra=None, missing_obsolet
                     ('obsoletes', 'obsoleted_by'),
             ]:
                 if k in hints:
-                    dpl = hints[k].split(',')
-                    for dp in dpl:
-                        dp = dp.strip()
+                    for dp in hints[k]:
                         dp = re.sub(r'(.*)\s+\(.*\)', r'\1', dp)
                         if dp in packages:
                             getattr(packages[dp], a).add(p)
@@ -1101,7 +1113,7 @@ def assign_importance(packages):
     # recursively give dependencies of base packages the basedep importance
     def recursive_basedep(p):
         bv = p.best_version
-        requires = p.version_hints[bv].get('depends', '').split(', ')
+        requires = p.version_hints[bv].get('depends', [])
         requires = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in requires]
         for r in requires:
             if r in packages:
@@ -1351,27 +1363,26 @@ def write_setup_ini(args, packages, arch):
 
                 if version in po.versions():
                     if hints.get('depends', ''):
-                        print("depends2: %s" % hints.get('depends', ''), file=f)
+                        print("depends2: %s" % ', '.join(hints.get('depends', [])), file=f)
 
                     if hints.get('obsoletes', ''):
-                        print("obsoletes: %s" % hints['obsoletes'], file=f)
+                        print("obsoletes: %s" % ', '.join(hints['obsoletes']), file=f)
 
                     if hints.get('provides', ''):
-                        print("provides: %s" % hints['provides'], file=f)
+                        print("provides: %s" % ', '.join(hints['provides']), file=f)
 
                     if hints.get('conflicts', ''):
-                        print("conflicts: %s" % hints['conflicts'], file=f)
+                        print("conflicts: %s" % ','.join(hints['conflicts']), file=f)
 
                 if s:
                     src_hints = packages[s].version_hints.get(version, {})
-                    bd = src_hints.get('build-depends', '')
+                    bd = src_hints.get('build-depends', [])
 
                     # Ideally, we'd transform dependency atoms which aren't
                     # cygwin package names into package names. For the moment,
                     # we don't have the information to do that, so filter them
                     # all out.
-                    if bd:
-                        bd = [atom for atom in bd.split(', ') if '(' not in atom]
+                    bd = [atom for atom in bd if '(' not in atom]
 
                     if bd:
                         print("build-depends: %s" % ', '.join(bd), file=f)
@@ -1457,7 +1468,7 @@ def write_repo_json(args, packages, f):
             sp = {'name': sp, 'categories': hints.get('category', '').split()}
             for k in ['depends', 'provides', 'obsoletes']:
                 if hints.get(k, None):
-                    sp[k] = [d.strip() for d in hints[k].split(',')]
+                    sp[k] = hints[k]
             spl.append(sp)
         d['subpackages'] = spl
 
@@ -1642,7 +1653,7 @@ def mark_fn(packages, po, v, certain_age, vault_requests):
 
     # - if package depends on anything in expired_provides
     #
-    requires = po.version_hints[v].get('depends', '').split(', ')
+    requires = po.version_hints[v].get('depends', [])
     if any(ep in requires for ep in past_mistakes.expired_provides):
         logging.debug("package '%s' version '%s' not retained as it requires a provide known to be expired" % (pn, v))
         return Freshness.conditional
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index 4ab53c0..b9ffec9 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -277,6 +277,8 @@ old_style_obsolete_by = {
     'at-spi2-atk': 'libatk-bridge2.0_0',
     'idle3': 'idle39',
     'lighttpd-mod_trigger_b4_dl': 'lighttpd',
+    'python-gi-common': 'python3-gi',
+    'python-pyatspi-common': 'python3-pyatspi',
     'qt-gstreamer': 'libQtGStreamer1_0_0',
     # these are odd and only exist to record an optional dependency on the
     # language runtime (dynamically loaded at runtime), which is also noted in
@@ -291,6 +293,11 @@ old_style_obsolete_by = {
     'python3-.*': '',
     # these packages probably should be marked as self-destruct?
     'mate-utils': '',
+    'octave-octcdf': '',
+    'python-twisted-debuginfo': '',
     'texlive-collection-htmlxml': '',
+    'vte2.91': '',
     'w32api': '',
+    # self-destruct, or need to start to exist
+    'cron-debuginfo': '',
 }
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 70b0aeb..c619bb9 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -248,12 +248,8 @@ def update_package_listings(args, packages):
                             if details[key].is_attr:
                                 value[arch] = getattr(pos[arch], key, set())
                             else:
-                                t = pos[arch].version_hints[pos[arch].best_version].get(key, None)
-
-                                if t:
-                                    value[arch] = set(t.split(', '))
-                                else:
-                                    value[arch] = set()
+                                t = pos[arch].version_hints[pos[arch].best_version].get(key, [])
+                                value[arch] = set(t)
                             values.update(value[arch])
 
                         if values:
diff --git a/calm/reports.py b/calm/reports.py
index eb1078b..0ff0b11 100644
--- a/calm/reports.py
+++ b/calm/reports.py
@@ -295,15 +295,15 @@ def provides_rebuild(args, packages, fn, provide_package, reportlist):
 
     if pp_package:
         pp_bv = pp_package.best_version
-        pp_provide = pp_package.version_hints[pp_bv]['provides']
+        pp_provide = pp_package.version_hints[pp_bv]['provides'][0]
         pp_provide_base = re.sub(r'\d+$', '', pp_provide)
 
         for p in packages[arch]:
             po = packages[arch][p]
             bv = po.best_version
 
-            depends = packages[arch][p].version_hints[bv]['depends'].split(', ')
-            depends = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in depends]
+            depends = packages[arch][p].version_hints[bv]['depends']
+            depends = utils.deplist_without_verrel(depends)
 
             for d in depends:
                 if not d.startswith(pp_provide_base):
@@ -350,12 +350,12 @@ def python_rebuild(args, packages, fn, reportlist):
     # XXX: look into how we can change this, after x86 is dropped
     arch = 'x86_64'
 
-    # assume that python3 depends on the latest python3n package
+    # assume that python3 depends only on the latest python3n package
     py_package = packages[arch].get('python3', None)
     if not py_package:
         return
 
-    latest_py = py_package.version_hints[py_package.best_version]['depends'].split(', ')[0]
+    latest_py = py_package.version_hints[py_package.best_version]['depends'][0]
 
     modules = {}
 
@@ -366,8 +366,8 @@ def python_rebuild(args, packages, fn, reportlist):
         if po.obsoleted_by:
             continue
 
-        depends = packages[arch][p].version_hints[bv]['depends'].split(', ')
-        depends = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in depends]
+        depends = packages[arch][p].version_hints[bv]['depends']
+        depends = utils.deplist_without_verrel(depends)
 
         for d in depends:
             # scan for a 'pythonnn' dependency
diff --git a/calm/utils.py b/calm/utils.py
index f6679fb..af2f5cb 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -204,8 +204,5 @@ def sendmail(hdr, msg):
 #
 # remove version-constrains from a list of dependencies
 #
-def deplist_without_verrel(dl):
-    dpl = dl.split(',')
-    dpl = [dp.strip() for dp in dpl]
-    dpl = [re.sub(r'(.*)\s+\(.*\)', r'\1', dp) for dp in dpl]
-    return dpl
+def deplist_without_verrel(dpl):
+    return [re.sub(r'(.*)\s+\(.*\)', r'\1', dp) for dp in dpl]
diff --git a/test/testdata/process_arch/packages.json.expected b/test/testdata/process_arch/packages.json.expected
index 2c41b14..bc50d46 100644
--- a/test/testdata/process_arch/packages.json.expected
+++ b/test/testdata/process_arch/packages.json.expected
@@ -411,7 +411,9 @@
  '            "arches": [\n'
  '                "x86_64"\n'
  '            ],\n'
- '            "build-depends": "cygwin-devel",\n'
+ '            "build-depends": [\n'
+ '                "cygwin-devel"\n'
+ '            ],\n'
  '            "name": "test-e",\n'
  '            "subpackages": [\n'
  '                {\n'
diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected
index 72489ac..224e4bf 100644
--- a/test/testdata/uploads/pkglist.expected
+++ b/test/testdata/uploads/pkglist.expected
@@ -4,7 +4,7 @@
                     'Like it’s you’re Markup Language™ Nokogiri’s tool―that '
                     'Bézier."',
            'category': 'Devel',
-           'depends': 'cygwin'}}, {}, False),
+           'depends': ['cygwin']}}, {}, False),
  'testpackage-src': Package('testpackage', {'1.0-1': Tar('testpackage-1.0-1-src.tar.bz2', 'x86_64/release/testpackage', 'acfd77df3347e6432ccf29c12989964bc680a158d574f85dfa7ef222759f411006c7bd2773e37c5abdee628bea769b2da9aae213db615cd91402fd385373933d', 266, False)}, {'1.0-1': {'sdesc': '"A test package"',
            'ldesc': '"A test package\n'
                     "It's description might contains some unicode junk\n"
@@ -13,25 +13,25 @@
            'category': 'Devel',
            'homepage': 'http://homepage.url',
            'parse-warnings': ["key 'license' missing"],
-           'depends': ''}}, {}, False),
+           'depends': []}}, {}, False),
  'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86_64/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"',
            'ldesc': '"A test subpackage"',
            'category': 'Devel',
            'external-source': 'testpackage-src',
-           'depends': ''}}, {}, False),
+           'depends': []}}, {}, False),
  'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86_64/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}, {'1.0-1': {'category': 'Base',
            'sdesc': '"test package (zstd compressed)"',
            'ldesc': '"test package (zstd compressed)"',
-           'depends': ''}}, {}, False),
+           'depends': []}}, {}, False),
  'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86_64/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}, {'1.0-1': {'category': 'Base',
-           'build-depends': 'cygport',
+           'build-depends': ['cygport'],
            'sdesc': '"test package (zstd compressed)"',
            'ldesc': '"test package (zstd compressed)"',
            'homepage': 'http://zstd.testpkg.invalid',
            'skip': '',
            'parse-warnings': ["key 'license' missing"],
-           'depends': ''}}, {}, False),
+           'depends': []}}, {}, False),
  'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86_64/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"',
            'ldesc': '"A test subpackage 2"',
            'category': 'Devel',
-           'depends': ''}}, {}, False)}
+           'depends': []}}, {}, False)}