From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 6B62A3858C62; Wed, 11 Jan 2023 12:47:02 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 6B62A3858C62 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1673441222; bh=ND1A5ttnDUmjUkcSdfwGXtRnRDVEX886wwWwtP9yqqk=; h=To:Subject:Date:From:From; b=ouYq0cMpaUWH+RLCrxO1uYmSzOaUhdJk5vp/wQl4TRE8oFqO5u+GhizbRYT6dSo/L oLcf1cWLe7k2/OTN2dsz3sOph1LSMwlHs1xjbuTjoQ1lXW6UpFD79JVylYJvePyTWG 7S1wBFWpkU+zZpDR1DT8K1edwyxQbkzmhBu/RsI0= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20221205-16-gd8c6dd1 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: aec1dcf7af8df98df4c7ded84fd3a7a7750f9efd X-Git-Newrev: d8c6dd106db65c039295e2ec76ffbda50f6d0bd2 Message-Id: <20230111124702.6B62A3858C62@sourceware.org> Date: Wed, 11 Jan 2023 12:47:02 +0000 (GMT) From: Jon Turney List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d8c6dd106db65c039295e2ec76ffbda50f6d0bd2 commit d8c6dd106db65c039295e2ec76ffbda50f6d0bd2 Author: Jon Turney Date: Wed Jan 11 10:54:57 2023 +0000 Look for multiple obsoletes of the same package This probably isn't always wrong, but seems like it's something we don't want or handle well at the moment (the solver will pick exactly one replacement package to install, since that's all that needed to satisfy the provides). https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=36532b0908f702a4567a9599c08fddc9795f1022 commit 36532b0908f702a4567a9599c08fddc9795f1022 Author: Jon Turney Date: Fri Jan 6 17:50:49 2023 +0000 Recursively apply missing obsoletes https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b0ea56b7f82ba930d16304e0860f444689bd7d69 commit b0ea56b7f82ba930d16304e0860f444689bd7d69 Author: Jon Turney Date: Thu Jan 5 16:13:46 2023 +0000 Allow regex matching for old_style_obsolete_by data https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=31819b5ed1b2523cb272a1474cd00ce6bddca33c commit 31819b5ed1b2523cb272a1474cd00ce6bddca33c Author: Jon Turney Date: Wed Jan 4 19:25:56 2023 +0000 Don't synthesize a new-style obsolete when current version isn't obsolete https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1c38fe4217480f3d58969ba2cf071b14bd3583d4 commit 1c38fe4217480f3d58969ba2cf071b14bd3583d4 Author: Jon Turney Date: Mon Dec 12 16:20:55 2022 +0000 Don't convert blacklisted old-style obsoletes: to new-style ruby-atk-debuginfo is obsoleted by ruby, but depends on cygwin-debuginfo. The existing obsolete is the correct one. okular-odp has a strange anomaly where it's category _obsolete, but the oldest version depends on calligra-libs as a replacement, but more recent versions depend on okular-calligra, which is correct. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dac9781717443dc723a73f6faf0d8bcd2f42786f commit dac9781717443dc723a73f6faf0d8bcd2f42786f Author: Jon Turney Date: Sat Jan 7 16:22:24 2023 +0000 Don't warn about obsoletion with no replacement for packages marked self-destruct https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=891c464a15c6c80013142e5b254876e1776a764e commit 891c464a15c6c80013142e5b254876e1776a764e Author: Jon Turney Date: Sat Jan 7 16:43:58 2023 +0000 Rename old-style obsoletion conversion threshold for clarity https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f4efac3d1d0e03bfc44a090dfad61d154fbb97d4 commit f4efac3d1d0e03bfc44a090dfad61d154fbb97d4 Author: Jon Turney Date: Sun Jun 19 13:11:44 2022 +0100 Finish data for old_style_obsolete_by heuristic Diff: --- calm/package.py | 150 ++++++++++++++++++++++++++++++++------------------ calm/past_mistakes.py | 10 +++- 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/calm/package.py b/calm/package.py index 4538b8d..19e85d1 100755 --- a/calm/package.py +++ b/calm/package.py @@ -571,17 +571,30 @@ def sort_key(k): # generate a record to add an obsoletes: header to the replacement package. # -OBSOLETE_AGE_THRESHOLD_YEARS = 20 +OBSOLETE_CONVERT_THRESHOLD_YEARS = 20 def upgrade_oldstyle_obsoletes(packages): missing_obsolete = {} - certain_age = time.time() - (OBSOLETE_AGE_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) + certain_age = time.time() - (OBSOLETE_CONVERT_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age)))) for p in sorted(packages): if packages[p].kind == Kind.binary: - for vr in packages[p].versions(): + for vr in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): + # we only really want to consider packages where the current + # version is obsolete. + # + # (if older versions are obsolete and we were somehow + # un-obsoleted, we'd need to somehow infer version constraints + # on the obsoletions, or something) + # + # as a proxy for that, stop considering versions of this package + # when one isn't obsolete, don't consider older versions + if not (packages[p].tar(vr).is_empty and + '_obsolete' in packages[p].version_hints[vr]['category']): + break + # initially apply to a subset over a certain age, to gradually # introduce this change mtime = packages[p].tar(vr).mtime @@ -589,38 +602,50 @@ def upgrade_oldstyle_obsoletes(packages): continue logging.debug("_obsolete package '%s' version '%s' mtime '%s' is over cut-off age" % (p, vr, time.strftime("%F %T %Z", time.localtime(mtime)))) - if packages[p].tar(vr).is_empty: - if '_obsolete' in packages[p].version_hints[vr]['category']: - requires = packages[p].version_hints[vr].get('depends', '').split(', ') - requires = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in requires] + requires = packages[p].version_hints[vr].get('depends', '').split(', ') + requires = [re.sub(r'(.*) +\(.*\)', r'\1', r) for r in requires] - if p in past_mistakes.old_style_obsolete_by: - o = past_mistakes.old_style_obsolete_by[p] + o = None + for oso_re, oso_o in past_mistakes.old_style_obsolete_by.items(): + if re.match(r'^' + oso_re + r'$', p): + o = oso_o + break - # empty replacement means "ignore" - if not o: - continue + if o is not None: + # empty replacement means "ignore" + if not o: + continue - logging.debug('%s is hardcoded as obsoleted by %s ' % (p, o)) + logging.debug('%s is hardcoded as obsoleted by %s ' % (p, o)) - else: - if len(requires) == 0: - # obsolete but has no replacement - logging.warning('%s is obsolete, but has no replacement' % (p)) - continue - elif len(requires) == 1: - o = requires[0] - elif len(requires) >= 2: - # obsolete with multiple replacements (pick one?) - logging.warning('%s %s is obsoleted by %d packages (%s)' % (p, vr, len(requires), requires)) - continue - - if o in packages: - if o not in missing_obsolete: - missing_obsolete[o] = set() - - missing_obsolete[o].add(p) - logging.info("converting from empty, _obsolete category package '%s' to 'obsoletes: %s' in package '%s'" % (p, p, o)) + else: + # ignore self-destruct packages + provides = packages[p].version_hints[vr].get('provides', '') + if '_self-destruct' in provides: + continue + + if len(requires) == 0: + # obsolete but has no replacement + logging.warning('%s is obsolete, but has no replacement' % (p)) + continue + elif len(requires) == 1: + o = requires[0] + elif len(requires) >= 2: + # obsolete with multiple replacements (pick one?) + logging.warning('%s %s is obsoleted by %d packages (%s)' % (p, vr, len(requires), requires)) + continue + + # ignore if o it's blacklisted + if o in ['cygwin-debuginfo', 'calligra-libs']: + logging.debug("not adding 'obsoletes: %s' to '%s' as blacklisted" % (p, o)) + continue + + if o in packages: + if o not in missing_obsolete: + missing_obsolete[o] = set() + + missing_obsolete[o].add(p) + logging.info("converting from empty, _obsolete category package '%s' to 'obsoletes: %s' in package '%s'" % (p, p, o)) return missing_obsolete @@ -649,16 +674,15 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet valid_requires.update(hints.get('provides', '').split()) # reset computed package state + packages[p].has_requires = False packages[p].obsolete = False packages[p].rdepends = set() packages[p].build_rdepends = set() + packages[p].obsoleted_by = set() packages[p].orphaned = False # perform various package validations for p in sorted(packages.keys()): - logging.log(5, "validating package '%s'" % (p)) - has_requires = False - for (v, hints) in packages[p].version_hints.items(): for (c, okmissing, splitchar) in [ ('depends', 'missing-depended-package', ','), @@ -681,7 +705,7 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet # cygport always makes debuginfo packages require # that, even if they are empty if r != 'cygwin-debuginfo': - has_requires = True + packages[p].has_requires = True # a package should not appear in it's own hint if r == p: @@ -701,29 +725,38 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet logging.error("package '%s' version '%s' %s source package '%s'" % (p, v, c, r)) error = True + # if external-source is used, the package must exist + if 'external-source' in hints: + e = hints['external-source'] + if e not in packages: + logging.error("package '%s' version '%s' refers to non-existent or errored external-source '%s'" % (p, v, e)) + error = True + # some old packages are missing needed obsoletes:, add them where # needed, and make sure the uploader is warned if/when package is # updated - for mo in [past_mistakes.missing_obsolete, missing_obsolete_extra]: - if p in mo: + for mo in [past_mistakes.missing_obsolete, missing_obsolete_extra]: + for p in mo: + if p in packages: + for v in packages[p].version_hints: + obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') obsoletes = [o.strip() for o in obsoletes] obsoletes = [o for o in obsoletes if o] - # XXX: this needs to recurse so we don't drop transitive missing obsoletes? - needed = mo[p] - for n in sorted(needed): - if n not in obsoletes: - obsoletes.append(n) - packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes) - logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) + def add_needed_obsoletes(needed): + for n in sorted(needed): + if n not in obsoletes: + obsoletes.append(n) + packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes) + logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) - # if external-source is used, the package must exist - if 'external-source' in hints: - e = hints['external-source'] - if e not in packages: - logging.error("package '%s' version '%s' refers to non-existent or errored external-source '%s'" % (p, v, e)) - error = True + # recurse so we don't drop transitive missing obsoletes + if n in mo: + logging.debug("recursing to examine obsoletions of '%s' for adding to '%s'" % (n, p)) + add_needed_obsoletes(mo[n]) + + add_needed_obsoletes(mo[p]) # If package A is obsoleted by package B, B should appear in the # requires: for A (so the upgrade occurs with pre-depends: aware @@ -732,6 +765,7 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet # versions of setup, which should just install B). This condition can # occur since we might have synthesized the depends: from the requires: # in read_hints(), so fix that up here. + for p in sorted(packages): for hints in packages[p].version_hints.values(): obsoletes = hints.get('obsoletes', '') if obsoletes: @@ -766,7 +800,7 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet # list is just confusing), so if it's not obsolete, mark it as # 'not_for_output' if packages[p].kind == Kind.binary: - if not has_nonempty_install and not has_requires and not obsolete: + if not has_nonempty_install and not packages[p].has_requires and not obsolete: if not packages[p].not_for_output: packages[p].not_for_output = True logging.info("package '%s' has no non-empty install tarfiles and no dependencies, marking as 'not for output'" % (p)) @@ -877,13 +911,16 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet lvl = logging.ERROR logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr)) - # build the set of packages which depends: on this package (rdepends), and - # the set of packages which build-depends: on it (build_rdepends) + # build inverted relations: + # the set of packages which depends: on this package (rdepends), + # the set of packages which build-depends: on it (build_rdepends), and + # the set of packages which obsoletes: it (obsoleted_by) for p in packages: for hints in packages[p].version_hints.values(): for k, a in [ ('depends', 'rdepends'), - ('build-depends', 'build_rdepends') + ('build-depends', 'build_rdepends'), + ('obsoletes', 'obsoleted_by'), ]: if k in hints: dpl = hints[k].split(',') @@ -893,6 +930,11 @@ def validate_packages(args, packages, valid_requires_extra=None, missing_obsolet if dp in packages: getattr(packages[dp], a).add(p) + # warn about multiple obsoletes of same package + for p in sorted(packages.keys()): + if len(packages[p].obsoleted_by) >= 2: + logging.debug("package '%s' is obsoleted by more than one package: %s" % (p, ','.join(packages[p].obsoleted_by))) + # make another pass to verify a source tarfile exists for every install # tarfile version for p in packages.keys(): diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 5cd85a7..08885ec 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -176,8 +176,9 @@ empty_source = { # additional data for the heuristic for upgrading old-style obsoletion packages old_style_obsolete_by = { 'at-spi2-atk': 'libatk-bridge2.0_0', - 'qt-gstreamer': 'libQtGStreamer1_0_0', + 'idle3': 'idle39', 'lighttpd-mod_trigger_b4_dl': 'lighttpd', + 'qt-gstreamer': 'libQtGStreamer1_0_0', # these are odd and only exist to record an optional dependency on the # language runtime (dynamically loaded at runtime), which is also noted in # build-requires: @@ -187,5 +188,10 @@ old_style_obsolete_by = { 'vim-python3': 'vim', 'vim-ruby': 'vim', # (An empty replacement means "don't apply this heuristic") - 'libksba': '', + # we have other plans for 'python3-*' packages, they will become virtuals + 'python3-.*': '', + # these packages probably should be marked as self-destruct? + 'mate-utils': '', + 'texlive-collection-htmlxml': '', + 'w32api': '', }