From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id DA9023858418; Mon, 16 Oct 2023 10:58:14 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org DA9023858418 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1697453894; bh=WX1dhIuVjQxpvYp1U2xXpMRR6uds3RJKVMrOCpF5eug=; h=To:Subject:Date:From:From; b=fC5jesN6CsasNonvTXeiSeJrQDJ35T53Yi/BEwHkdpxy8QBqB7FeWDC4pJObggv16 1m+Uh1KDrNMDxY+pJCkdjk7YsDUztt2Kg4LQm+pusm6kZP8Qij4k3jYhM2pwnJlPta TlsQULxbAQKSAf88uVgpmUDjMOxaIgfU0gXowVJU= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-44-gcda9f05 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 68971589a695695851d45a50d7b201465b6564bd X-Git-Newrev: cda9f0512545bf5374cbed851e58140537a38f39 Message-Id: <20231016105814.DA9023858418@sourceware.org> Date: Mon, 16 Oct 2023 10:58:13 +0000 (GMT) From: Jon Turney List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cda9f0512545bf5374cbed851e58140537a38f39 commit cda9f0512545bf5374cbed851e58140537a38f39 Author: Jon Turney Date: Sat Oct 14 21:34:10 2023 +0100 Expire python2 modules For the first time, we are expiring obsolete packages just because they are obsolete, so it is time for the previous commit, to ensure we retain any obsolete: hints they might have caused to be generated. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=17dc61e9f95c74366465d0a56121cef84b88d415 commit 17dc61e9f95c74366465d0a56121cef84b88d415 Author: Jon Turney Date: Mon Jun 20 16:52:08 2022 +0100 Persist missing_obsolete Persist the missing obsolete: data generated from old-style obsoletion packages, so it is remembered, even if after the obsoleted package is removed. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=72608dc7f48988a3509538d0a01411c2586a513f commit 72608dc7f48988a3509538d0a01411c2586a513f Author: Jon Turney Date: Sat Oct 14 15:02:27 2023 +0100 Simplify 'conditionally retained' package marking Move the logic into a single function, rather than selectively defining a callback function depending on if the package meets some criteria. Diff: --- calm/calm.py | 2 +- calm/db.py | 34 +++++++++++ calm/package.py | 157 +++++++++++++++++++++++--------------------------- calm/past_mistakes.py | 4 ++ 4 files changed, 111 insertions(+), 86 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 22b0a7a..73b0e05 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -111,7 +111,7 @@ def process_relarea(args, state): state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: - state.missing_obsolete[arch] = package.upgrade_oldstyle_obsoletes(packages[arch]) + state.missing_obsolete[arch] = db.update_missing_obsolete(args, packages, arch) # validate the package set for each arch for arch in common_constants.ARCHES: diff --git a/calm/db.py b/calm/db.py index b7f7040..0aef83a 100644 --- a/calm/db.py +++ b/calm/db.py @@ -30,6 +30,7 @@ import logging import os import sqlite3 +from . import package from . import utils @@ -48,6 +49,12 @@ def connect(args): vr TEXT NOT NULL )''') + conn.execute('''CREATE TABLE IF NOT EXISTS missing_obsolete + (name TEXT NOT NULL, + arch TEXT NOT NULL, + replaces TEXT NOT NULL, + PRIMARY KEY (name, arch) + )''') conn.commit() return conn @@ -104,3 +111,30 @@ def vault_requests(args): def vault_request_add(args, p, v): with connect(args) as conn: conn.execute('INSERT INTO vault_requests (srcpackage, vr) VALUES (?,?)', (p, v)) + + +# +# this accumulates missing_obsoletes data for packages, so we will remember it +# even after the obsoleted package has been removed +# +def update_missing_obsolete(args, packages, arch): + data = {} + with connect(args) as conn: + conn.row_factory = sqlite3.Row + + # read + cur = conn.execute("SELECT name, replaces FROM missing_obsolete WHERE arch = ?", (arch,)) + for row in cur.fetchall(): + data[row['name']] = set(row['replaces'].split()) + + # update missing obsoletes data + missing_obsolete = package.upgrade_oldstyle_obsoletes(packages[arch], data.copy()) + + # update + for n, r in missing_obsolete.items(): + if n not in data: + conn.execute('INSERT INTO missing_obsolete (name, arch, replaces) VALUES (?, ? , ?)', (n, arch, ' '.join(r))) + else: + conn.execute('UPDATE missing_obsolete SET replaces = ? WHERE name = ? AND arch = ?', (' '.join(r), n, arch)) + + return missing_obsolete diff --git a/calm/package.py b/calm/package.py index f60c238..e45874c 100755 --- a/calm/package.py +++ b/calm/package.py @@ -584,11 +584,10 @@ def sort_key(k): # generate a record to add an obsoletes: header to the replacement package. # -OBSOLETE_CONVERT_THRESHOLD_YEARS = 20 +OBSOLETE_CONVERT_THRESHOLD_YEARS = 2 -def upgrade_oldstyle_obsoletes(packages): - missing_obsolete = {} +def upgrade_oldstyle_obsoletes(packages, missing_obsolete): certain_age = time.time() - (OBSOLETE_CONVERT_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age)))) @@ -1531,12 +1530,68 @@ class Freshness(IntEnum): def mark_package_fresh(packages, p, v, mark=Freshness.fresh): - if callable(mark): - mark = mark(v) - packages[p].tar(v).fresh = mark +# +# helper function evaluate if package needs marking for conditional retention +# + +def mark_fn(packages, po, v, certain_age, vault_requests): + pn = po.name + bv = po.best_version + + # 'conditional' package retention means the package is weakly retained. + # This allows total expiry when a source package no longer provides + # anything useful: + # + # - if all we have is a source package and a debuginfo package, then we + # shouldn't retain anything. + # + if pn.endswith('-debuginfo'): + return Freshness.conditional + + # - shared library packages which don't come from the current version of + # source (i.e. is superseded or removed), have no packages from a + # different source package which depend on them, and are over a certain + # age + # + es = po.version_hints[bv].get('external-source', None) + if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and + not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)): + if es and (packages[es].best_version != bv): + mtime = po.tar(v).mtime + if mtime < certain_age: + logging.debug("deprecated soversion package '%s' version '%s' mtime '%s' is over cut-off age" % (pn, v, time.strftime("%F %T %Z", time.localtime(mtime)))) + return Freshness.conditional + + # - if package depends on anything in expired_provides + # + requires = po.version_hints[v].get('depends', '').split(', ') + if re.match(r'^python(|2|27)-', pn): + if any(ep in requires for ep in past_mistakes.expired_provides) or po.obsolete: + logging.info("package '%s' version '%s' not retained as it requires a provide known to be expired" % (pn, v)) + return Freshness.conditional + + # - explicitly marked as 'noretain' + # + if 'noretain' in po.override_hints: + noretain_versions = po.override_hints.get('noretain', '').split() + if (v in noretain_versions) or ('all' in noretain_versions): + return Freshness.conditional + + # - marked via 'calm-tool vault' + # + es = po.srcpackage(bv, suffix=False) + if es in vault_requests: + if v in vault_requests[es]: + logging.info("package '%s' version '%s' not retained due vault request" % (pn, v)) + return Freshness.conditional + + # otherwise, make no change + return Freshness.fresh + + # # construct a move list of stale packages # @@ -1553,83 +1608,8 @@ def stale_packages(packages, vault_requests): if po.kind != Kind.binary: continue - mark = Freshness.fresh - - # 'conditional' package retention means the package is weakly retained. - # This allows total expiry when a source package no longer provides - # anything useful: - # - # - if all we have is a source package and a debuginfo package, then we - # shouldn't retain anything. - # - if pn.endswith('-debuginfo'): - mark = Freshness.conditional - - # - shared library packages which don't come from the current version of - # source (i.e. is superseded or removed), have no packages from a - # different source package which depend on them, and are over a certain - # age - # - bv = po.best_version - es = po.version_hints[bv].get('external-source', None) - if (re.match(common_constants.SOVERSION_PACKAGE_RE, pn) and - not any(packages[p].srcpackage(packages[p].best_version) != es for p in po.rdepends)): - if es and (packages[es].best_version != bv): - def dep_so_age_mark(v): - mtime = po.tar(v).mtime - if mtime < certain_age: - logging.debug("deprecated soversion package '%s' version '%s' mtime '%s' is over cut-off age" % (pn, v, time.strftime("%F %T %Z", time.localtime(mtime)))) - return Freshness.conditional - else: - return Freshness.fresh - - mark = dep_so_age_mark - - # - if package depends on anything in expired_provides - # - all_reqs = set.union(*(set(po.version_hints[v].get('depends', '').split(', ')) for v in po.versions())) - if all_reqs.intersection(set(past_mistakes.expired_provides)): - def expired_provides_mark(v): - requires = po.version_hints[v].get('depends', '').split(', ') - if any(ep in requires for ep in past_mistakes.expired_provides): - # XXX: for the moment, don't allow this to expire the - # current version, though! - if v != po.best_version: - logging.info("package '%s' version '%s' not retained as it requires a provide known to be expired" % (pn, v)) - return Freshness.conditional - else: - logging.info("package '%s' version '%s' requires a provide known to be expired, but not expired as it's the current version" % (pn, v)) - - return Freshness.fresh - - mark = expired_provides_mark - - # - explicitly marked as 'noretain' - # - if 'noretain' in po.override_hints: - def noretain_hint_mark(v): - noretain_versions = po.override_hints.get('noretain', '').split() - if (v in noretain_versions) or ('all' in noretain_versions): - return Freshness.conditional - else: - return Freshness.fresh - - mark = noretain_hint_mark - - # - marked via 'calm-tool vault' - # - es = po.srcpackage(bv, suffix=False) - if es in vault_requests: - def vault_requests_mark(v): - if v in vault_requests[es]: - logging.info("package '%s' version '%s' not retained due vault request" % (pn, v)) - return Freshness.conditional - else: - return Freshness.fresh - - mark = vault_requests_mark - - # mark any versions explicitly listed in the keep: override hint (unconditionally) + # mark as fresh any versions explicitly listed in the keep: override + # hint (unconditionally) for v in po.override_hints.get('keep', '').split(): if v in po.versions(): mark_package_fresh(packages, pn, v) @@ -1643,7 +1623,7 @@ def stale_packages(packages, vault_requests): if 'test' not in po.version_hints[v]: if keep_count <= 0: break - mark_package_fresh(packages, pn, v, mark) + mark_package_fresh(packages, pn, v) keep_count = keep_count - 1 # mark as fresh the highest n test versions, where n is given by the @@ -1656,7 +1636,7 @@ def stale_packages(packages, vault_requests): if 'test' in po.version_hints[v]: if keep_count <= 0: break - mark_package_fresh(packages, pn, v, mark) + mark_package_fresh(packages, pn, v) keep_count = keep_count - 1 else: if 'keep-superseded-test' not in po.override_hints: @@ -1674,6 +1654,13 @@ def stale_packages(packages, vault_requests): newer = True if newer: + mark_package_fresh(packages, pn, v) + + # overwrite with 'conditional' package retention mark if it meets + # various criteria + for v in sorted(po.versions(), key=lambda v: SetupVersion(v)): + mark = mark_fn(packages, po, v, certain_age, vault_requests) + if mark != Freshness.fresh: mark_package_fresh(packages, pn, v, mark) # mark source packages as fresh if any install package which uses it is fresh diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 6561fde..13dfd50 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -171,6 +171,8 @@ nonexistent_provides = historical_provides + [ 'python2', 'python2-devel', 'python27', + 'python-pygments', + 'python-lxml', '_windows', r'perl5_\d+', r'ruby_\d+', @@ -180,6 +182,8 @@ nonexistent_provides = historical_provides + [ # provides: which don't exist and packages which require them should be expired expired_provides = [ + 'python2', + 'python27', ] # empty source packages