From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 069803856DD9; Fri, 1 Jul 2022 12:38:25 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 069803856DD9 To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20220627-2-g78dc005 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 385eeb6a094e30aaaf224b9c5b4a025a09002726 X-Git-Newrev: 78dc0051e7f53830f72c219c437a5b175906dfd2 Message-Id: <20220701123825.069803856DD9@sourceware.org> Date: Fri, 1 Jul 2022 12:38:25 +0000 (GMT) From: Jon TURNEY X-BeenThere: cygwin-apps-cvs@cygwin.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Cygwin-apps git logs List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 01 Jul 2022 12:38:25 -0000 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=78dc0051e7f53830f72c219c437a5b175906dfd2 commit 78dc0051e7f53830f72c219c437a5b175906dfd2 Author: Jon Turney Date: Thu Jun 30 19:07:20 2022 +0100 Improve deprecated soversion expiry (again) "unused" should mean "no packages from a different source package depend on it", not "no packages depend on it". e.g. 'bind' produces both 'libdns165', and 'libbind9_140' which depends on it. Both are eligible for expiry when they have no other rdepends outside of 'bind'. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=eca3a88d174516051883b5c7404f6985939d4b4e commit eca3a88d174516051883b5c7404f6985939d4b4e Author: Jon Turney Date: Wed May 5 14:08:46 2021 +0100 Improve handling of old-style obsoletion packages Identify old-style obsoletion packages (where the package is empty, has a category of _obsolete, and requires: it's replacement), and upgrade them to the new style (where the obsoleting package has an obsoletes:). This is done per-arch, simply because lots of these historic obsoletion packages only exist for x86, and it's redundant to obsolete: them on x86_64. Treating this similarly to past_mistakes.missing_obsoletes means that a warning occurs when new version of package is missing that expected obsolete: Initially only apply to packages over a certain age, so we can observe the effect on a small number of packages to make checking it's doing the right thing easier. Diff: --- calm/calm.py | 12 ++++-- calm/package.py | 104 +++++++++++++++++++++++++++++++++++++++++--------- calm/past_mistakes.py | 36 +++++++++++++++++ 3 files changed, 131 insertions(+), 21 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 49bdcfd..04db081 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -89,6 +89,7 @@ class CalmState(object): self.subject = '' self.packages = {} self.valid_provides = set() + self.missing_obsolete = {} # @@ -104,10 +105,13 @@ def process_relarea(args, state): logging.debug("reading existing packages for arch %s" % (arch)) packages[arch] = package.read_packages(args.rel_area, arch) - # validate the package set for each arch state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: - if not package.validate_packages(args, packages[arch], state.valid_provides): + state.missing_obsolete[arch] = package.upgrade_oldstyle_obsoletes(packages[arch]) + + # validate the package set for each arch + for arch in common_constants.ARCHES: + if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete[arch]): logging.error("existing %s package set has errors" % (arch)) error = True @@ -208,7 +212,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): state.valid_provides = db.update_package_names(args, merged_packages) for arch in common_constants.ARCHES: logging.debug("validating merged %s package set for maintainer %s" % (arch, name)) - if not package.validate_packages(args, merged_packages[arch], state.valid_provides): + if not package.validate_packages(args, merged_packages[arch], state.valid_provides, state.missing_obsolete): logging.error("error while validating merged %s packages for %s" % (arch, name)) valid = False @@ -329,7 +333,7 @@ def remove_stale_packages(args, packages, state): error = False state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: - if not package.validate_packages(args, packages[arch], state.valid_provides): + if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete): logging.error("%s package set has errors after removing stale packages" % arch) error = True diff --git a/calm/package.py b/calm/package.py index 482d042..32b4d31 100755 --- a/calm/package.py +++ b/calm/package.py @@ -496,15 +496,81 @@ def sort_key(k): return k +# +# generate missing_obsolete data for upgrading old-style obsoletion install +# packages: +# +# they are empty, have the '_obsolete' category, and requires: exactly 1 +# package, their replacement. +# +# generate a record to add an obsoletes: header to the replacement package. +# + +OBSOLETE_AGE_THRESHOLD_YEARS = 20 + + +def upgrade_oldstyle_obsoletes(packages): + missing_obsolete = {} + certain_age = time.time() - (OBSOLETE_AGE_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) + logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age)))) + + for p in sorted(packages): + if packages[p].kind == Kind.binary: + for vr in packages[p].versions(): + # initially apply to a subset over a certain age, to gradually + # introduce this change + mtime = packages[p].tar(vr).mtime + if mtime > certain_age: + continue + logging.debug("_obsolete package '%s' version '%s' mtime '%s' is over cut-off age" % (p, vr, time.strftime("%F %T %Z", time.localtime(mtime)))) + + if packages[p].tar(vr).is_empty: + if '_obsolete' in packages[p].version_hints[vr]['category']: + requires = packages[p].version_hints[vr].get('requires', '').split() + + if p in past_mistakes.old_style_obsolete_by: + o = past_mistakes.old_style_obsolete_by[p] + + # empty replacement means "ignore" + if not o: + continue + + logging.debug('%s is hardcoded as obsoleted by %s ' % (p, o)) + + else: + if len(requires) == 0: + # obsolete but has no replacement + logging.warning('%s is obsolete, but has no replacement' % (p)) + continue + elif len(requires) == 1: + o = requires[0] + elif len(requires) >= 2: + # obsolete with multiple replacements (pick one?) + logging.warning('%s %s is obsoleted by %d packages (%s)' % (p, vr, len(requires), requires)) + continue + + if o in packages: + if o not in missing_obsolete: + missing_obsolete[o] = set() + + missing_obsolete[o].add(p) + logging.info("converting from empty, _obsolete category package '%s' to 'obsoletes: %s' in package '%s'" % (p, p, o)) + + return missing_obsolete + + # # validate the package database # -def validate_packages(args, packages, valid_requires_extra=None): +def validate_packages(args, packages, valid_requires_extra=None, missing_obsolete_extra=None): error = False if packages is None: return False + if missing_obsolete_extra is None: + missing_obsolete_extra = {} + # build the set of valid things to requires: etc. valid_requires = set() @@ -573,17 +639,19 @@ def validate_packages(args, packages, valid_requires_extra=None): # some old packages are missing needed obsoletes:, add them where # needed, and make sure the uploader is warned if/when package is # updated - if p in past_mistakes.missing_obsolete: - obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') - obsoletes = [o.strip() for o in obsoletes] - obsoletes = [o for o in obsoletes if o] - - needed = past_mistakes.missing_obsolete[p] - for n in needed: - if n not in obsoletes: - obsoletes.append(n) - packages[p].version_hints[v]['obsoletes'] = ','.join(obsoletes) - logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) + for mo in [past_mistakes.missing_obsolete, missing_obsolete_extra]: + if p in mo: + obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') + obsoletes = [o.strip() for o in obsoletes] + obsoletes = [o for o in obsoletes if o] + + # XXX: this needs to recurse so we don't drop transitive missing obsoletes? + needed = mo[p] + for n in sorted(needed): + if n not in obsoletes: + obsoletes.append(n) + packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes) + logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) # if external-source is used, the package must exist if 'external-source' in hints: @@ -1383,15 +1451,17 @@ def stale_packages(packages): # shouldn't retain anything. # # - shared library packages which don't come from the current version of - # source (i.e. is superseded or removed), have no packages which depend - # on them, and are over a certain age + # source (i.e. is superseded or removed), have no packages from a + # different source package which depend on them, and are over a certain + # age # mark = Freshness.fresh if pn.endswith('-debuginfo'): mark = Freshness.conditional - if (len(po.rdepends) == 0) and re.match(common_constants.SOVERSION_PACKAGE_RE, pn): - bv = po.best_version - es = po.version_hints[bv].get('external-source', None) + bv = po.best_version + es = po.version_hints[bv].get('external-source', None) + if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and + not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)): if es and (packages[es].best_version != bv): def dep_so_age_mark(v): mtime = po.tar(v).mtime diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index af6d650..c0e724f 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -220,3 +220,39 @@ empty_source = { 'pinentry-qt3-src': ['0.7.6-3'], # obsoleted by pinentry-qt 'xerces-c-devel-src': ['2.8.0-1'], # obsoleted by libxerces-c-devel } + +# additional data for the heuristic for upgrading old-style obsoletion packages +old_style_obsolete_by = { + 'at-spi2-atk': 'libatk-bridge2.0_0', + 'qt-gstreamer': 'libQtGStreamer1_0_0', + 'lighttpd-mod_trigger_b4_dl': 'lighttpd', + # these require: both epoch1 and epoch2 replacements, but epoch1 contains + # the same version + 'gcc-tools-autoconf': 'gcc-tools-epoch1-autoconf', + 'gcc-tools-automake': 'gcc-tools-epoch1-automake', + # these are odd and only exist to record an optional dependency on the + # language runtime (dynamically loaded at runtime), which is also noted in + # build-requires: + 'vim-lua': 'vim', + 'vim-perl': 'vim', + 'vim-python': 'vim', + 'vim-python3': 'vim', + 'vim-ruby': 'vim', + # (An empty replacement means "don't apply this heuristic". We use that to + # not bother with some x86-only packages which have complex replacements, + # since they will be going away relatively soon anyhow...) + 'SuiteSparse': '', + 'libSuiteSparse-devel': '', + 'libksba': '', + 'libstdc++6-devel': '', + 'octave-forge': '', + 'plotutils-devel': '', + 'rpm-doc': '', + 'tetex-base': '', + 'tetex-bin': '', + 'tetex-extra': '', + 'tetex-tiny': '', + 'tetex-x11': '', + 'texlive-collection-langtibetan': '', + 'texlive-collection-texinfo': '', +}