From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 655FB385842C; Sat, 12 Feb 2022 19:26:29 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 655FB385842C To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-22-g0e8e44b X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 3e7c381a985a850284731a2b8f23d0368cff20a7 X-Git-Newrev: 0e8e44b3a39e9eff83b7c98c8aa4bca8541cb4b3 Message-Id: <20220212192629.655FB385842C@sourceware.org> Date: Sat, 12 Feb 2022 19:26:29 +0000 (GMT) From: Jon TURNEY X-BeenThere: cygwin-apps-cvs@cygwin.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Cygwin-apps git logs List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 12 Feb 2022 19:26:29 -0000 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0e8e44b3a39e9eff83b7c98c8aa4bca8541cb4b3 commit 0e8e44b3a39e9eff83b7c98c8aa4bca8541cb4b3 Author: Jon Turney Date: Fri Feb 11 22:59:17 2022 +0000 Add python 3.9 to GitHub test matrix Also drop unused .travis.yml https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7dc16f3dfe1531a4a9f09aea472e3e678e9f5517 commit 7dc16f3dfe1531a4a9f09aea472e3e678e9f5517 Author: Jon Turney Date: Fri Feb 11 18:48:03 2022 +0000 Remove 'stability level' computation and use Follow up to d8869d2e. Since d863c1a2, 3cc08991 and 27852b24 these 'stability levels' can't be set in override.hint, so they are just determined by sorting by version. Update tests as 'keep-count: 0' is now effective, rather than having an minimum of 1 (the curr: version, or 2 before prev: was also dropped). https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=da538458cb43b48613b1051fd87e3af01f62826b commit da538458cb43b48613b1051fd87e3af01f62826b Author: Jon Turney Date: Tue Feb 8 18:11:52 2022 +0000 Provide missing obsoletes on some debuginfo packages Diff: --- .github/workflows/calm.yaml | 2 +- .travis.yml | 10 -- calm/package.py | 171 ++++++++------------- calm/past_mistakes.py | 26 +++- .../x86/release/staleversion/override.expected | 2 +- .../relarea/x86/release/staleversion/override.hint | 2 +- 6 files changed, 84 insertions(+), 129 deletions(-) diff --git a/.github/workflows/calm.yaml b/.github/workflows/calm.yaml index 5fce459..69be9d2 100644 --- a/.github/workflows/calm.yaml +++ b/.github/workflows/calm.yaml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4459a22..0000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -python: - - "3.6" - - "3.7" - - "3.8" -install: - - pip install -r requirements.txt -script: - - flake8 --count --show-source - - python -m unittest discover diff --git a/calm/package.py b/calm/package.py index 37aa58c..e5452d7 100755 --- a/calm/package.py +++ b/calm/package.py @@ -548,6 +548,21 @@ def validate_packages(args, packages): logging.error("package '%s' version '%s' %s source package '%s'" % (p, v, c, r)) error = True + # some old packages are missing needed obsoletes:, add them where + # needed, and make sure the uploader is warned if/when package is + # updated + if p in past_mistakes.missing_obsolete: + obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') + obsoletes = [o.strip() for o in obsoletes] + obsoletes = [o for o in obsoletes if o] + + needed = past_mistakes.missing_obsolete[p] + for n in needed: + if n not in obsoletes: + obsoletes.append(n) + packages[p].version_hints[v]['obsoletes'] = ','.join(obsoletes) + logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) + # if external-source is used, the package must exist if 'external-source' in hints: e = hints['external-source'] @@ -603,99 +618,41 @@ def validate_packages(args, packages): else: packages[p].not_for_output = False - levels = ['test', 'curr'] - - # assign a version to each stability level - packages[p].stability = defaultdict() - - # sort in order from highest to lowest version - for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): - level_found = False - - while True: - # no stability levels left - if len(levels) == 0: - break - - l = levels[0] - - # if package is explicitly marked as having that stability level - # (only used for test, currently) - if (l == 'test') and ('test' in packages[p].version_hints[v]): - logging.debug("package '%s' version '%s' marked as stability '%s'" % (p, v, l)) - else: - # level 'test' must be explicitly assigned - if l == 'test': - levels.remove(l) - # go around again to check at the new level - continue - else: - # if version is explicitly marked test, it can't be - # assigned to any other stability level - if 'test' in packages[p].version_hints[v]: - logging.debug("package '%s' version '%s' can't be used for stability '%s' as it's marked test" % (p, v, l)) - break - - level_found = True - logging.log(5, "package '%s' stability '%s' assigned version '%s'" % (p, l, v)) - break - - if not level_found: - continue - - # assign version to level - packages[p].stability[l] = v - packages[p].version_hints[v][l] = '' - # and remove from list of unallocated levels - levels.remove(l) - - # lastly, fill in any levels which we skipped over because a higher - # stability level was overriden to a lower version - for l in levels: - if l in packages[p].override_hints: - if packages[p].override_hints[l] is not None: - packages[p].stability[l] = packages[p].override_hints[l] - - l = 'test' - if l not in packages[p].stability: - for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): - if 'test' in packages[p].version_hints[v]: - packages[p].stability[l] = v - packages[p].version_hints[v][l] = '' - break - # identify a 'best' version to take certain information from: this is # the curr version, if we have one, otherwise, the highest version. - if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].versions()): - packages[p].best_version = packages[p].stability['curr'] - elif len(packages[p].versions()): - packages[p].best_version = sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True)[0] + for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): + if 'test' not in packages[p].version_hints[v]: + packages[p].best_version = v + break else: - logging.error("package '%s' doesn't have any versions" % (p)) - packages[p].best_version = None - error = True + if len(packages[p].versions()): + packages[p].best_version = sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True)[0] - # the package must have some versions - if not packages[p].stability: - logging.error("no versions at any stability level for package '%s'" % (p)) - error = True - # it's also probably a really good idea if a curr version exists - elif (('curr' not in packages[p].stability) and - ('missing-curr' not in packages[p].version_hints[packages[p].best_version].get('disable-check', '')) and - ('missing-curr' not in getattr(args, 'disable_check', []))): - logging.warning("package '%s' doesn't have a curr version" % (p)) + # warn if no non-test ('curr') version exists + if (('missing-curr' not in packages[p].version_hints[packages[p].best_version].get('disable-check', '')) and + ('missing-curr' not in getattr(args, 'disable_check', []))): + logging.warning("package '%s' doesn't have any non-test versions (i.e. no curr: version)" % (p)) + + else: + # the package must have some versions + logging.error("package '%s' doesn't have any versions" % (p)) + packages[p].best_version = None + error = True # error if the curr: version isn't the most recent non-test: version mtimes = {} for vr in packages[p].versions(): mtimes[vr] = packages[p].tar(vr).mtime + cv = None + nontest_versions = [v for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True) if 'test' not in packages[p].version_hints.get(v, {})] + if len(nontest_versions) >= 1: + cv = nontest_versions[0] + for v in sorted(packages[p].versions(), key=lambda v: mtimes[v], reverse=True): if 'test' in packages[p].version_hints[v]: continue - cv = packages[p].stability['curr'] - if cv not in packages[p].versions(): continue @@ -867,9 +824,10 @@ def validate_packages(args, packages): if re.match(r'^python[23][5678]?-.*', install_p): continue - # ignore packages which don't have a current version (i.e. are test - # only) - if 'curr' not in packages[install_p].stability: + # ignore packages where best_version is a test version (i.e doesn't + # have a current version, is test only), since the check we are + # doing here is 'same current version' + if 'test' not in packages[install_p].version_hints[packages[install_p].best_version]: continue # ignore packages which have a different external-source: @@ -923,6 +881,8 @@ def validate_package_maintainers(args, packages): # validate that all packages are in the package list for p in sorted(packages): # ignore obsolete packages + if packages[p].obsolete: + continue if any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints]): continue # validate that the package is in a path which starts with something in the package list @@ -1036,18 +996,25 @@ def write_setup_ini(args, packages, arch): # due to a historic bug in setup (fixed in 78e4c7d7), we keep the # [curr] version first, to ensure that dependencies are used # correctly. - if 'curr' in po.stability: - version = po.stability['curr'] - vs.append((version, 'curr')) + nontest_versions = [v for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True) if 'test' not in po.version_hints.get(v, {})] + curr_version = None + if len(nontest_versions) >= 1: + curr_version = nontest_versions[0] + vs.append((curr_version, 'curr')) # purely for compatibility with previous ordering, identify the # 'prev' version (the non-test version before the current version), # if it exists, so we can put it last. prev_version = None - nontest_versions = [v for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True) if 'test' not in po.version_hints.get(v, {})] if len(nontest_versions) >= 2: prev_version = nontest_versions[1] + # ditto the 'test' version + test_version = None + test_versions = [v for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True) if 'test' in po.version_hints.get(v, {})] + if len(test_versions) >= 1: + test_version = test_versions[0] + # next put any other versions # # these [prev] or [test] sections are superseded by the final ones. @@ -1061,19 +1028,11 @@ def write_setup_ini(args, packages, arch): versions.update(packages[sibling_src].versions()) for version in sorted(versions, key=lambda v: SetupVersion(v), reverse=True): - # skip over versions assigned to stability level: 'curr' has - # already be done, and 'test' will be done later - skip = False - for level in ['curr', 'test']: - if level in po.stability: - if version == po.stability[level]: - skip = True - break - - if version == prev_version: - skip = True - - if skip: + # skip over versions which have a special place in the ordering: + # 'curr' has already been done, 'prev' and 'test' will be done + # later + if ((version == curr_version) or (version == prev_version) or + (version == test_version)): continue # test versions receive the test label @@ -1087,16 +1046,14 @@ def write_setup_ini(args, packages, arch): if prev_version: vs.append((prev_version, "prev")) - # finally, add 'test' version + # finally, add the 'test' version # # because setup processes version sections in order, these supersede # any previous [prev] and [test] sections (hopefully). i.e. the # version in the final [test] section is the one selected when test # packages are requested. - for level in ['test']: - if level in po.stability: - version = po.stability[level] - vs.append((version, level)) + if test_version: + vs.append((test_version, "test")) # write the section for each version for (version, tag) in vs: @@ -1373,12 +1330,6 @@ def mark_package_fresh(packages, p, v): def stale_packages(packages): for pn, po in packages.items(): - # mark any versions used by stability levels as fresh - for level in ['curr']: - if level in po.stability: - v = po.stability[level] - mark_package_fresh(packages, pn, v) - # mark any versions explicitly listed in the keep: override hint for v in po.override_hints.get('keep', '').split(): if v in po.versions(): diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 064ad27..27e5546 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -174,20 +174,34 @@ mtime_anomalies = [ # packages with maintainer anomalies # -# don't add to this list, fix the package (e.g. by obsoleting it) +# don't add to this list, fix the package maint_anomalies = { 'libelf0': ['0.8.13-2'], # libelf is called libelf0 in x86 arch 'libelf0-devel': ['0.8.13-2'], - 'ksnapshot-debuginfo': ['15.04.3-1'], # should be obsoleted by spectacle(-debuginfo) - 'man-debuginfo': ['1.6g-2'], # should be obsoleted by man-db(-debuginfo) - 'procps-debuginfo': ['3.2.8-5'], # should be obsoleted by procps-ng(-debuginfo) - 'python-debuginfo': ['2.7.12-1'], # should be obsoleted by python2(-debuginfo) - 'transfig-debuginfo': ['3.2.5e-2'], # should be obsoleted by xfig(-debuginfo) +} + +# packages missing obsoletions +# +# don't add to this list, fix the package (e.g. by adding the needed obsoletions) +# (an enhancement to cygport might be necessary to support doing that?) +missing_obsolete = { + 'filemanager-actions-debuginfo': ['caja-actions-debuginfo'], + 'librsync-debuginfo': ['rdiff-debuginfo'], + 'man-db-debuginfo': ['man-debuginfo'], # contain conflicting files + 'procps-ng-debuginfo': ['procps-debuginfo'], # contain conflicting files + 'python2-debuginfo': ['python-debuginfo'], # contain conflicting files + 'python-dbus-debuginfo': ['python3-dbus-debuginfo'], + 'rxvt-unicode-debuginfo': ['rxvt-unicode-X-debuginfo'], + 'spectacle-debuginfo': ['ksnapshot-debuginfo'], + 'xfce4-pulseaudio-plugin-debuginfo': ['xfce4-mixer-debuginfo'], + 'xfig-debuginfo': ['transfig-debuginfo'], # contain conflicting files } # provides: which don't exist nonexistent_provides = [ 'perl5_026', + 'rdiff-debuginfo', # not in x86 + 'rxvt-unicode-X-debuginfo', # not in x86_64 ] # empty source packages diff --git a/test/testdata/hints/x86/release/staleversion/override.expected b/test/testdata/hints/x86/release/staleversion/override.expected index 4506cc2..5348659 100644 --- a/test/testdata/hints/x86/release/staleversion/override.expected +++ b/test/testdata/hints/x86/release/staleversion/override.expected @@ -1 +1 @@ -OrderedDict([('keep', '243-0'), ('keep-count', '0'), ('keep-count-test', '1')]) +OrderedDict([('keep', '243-0'), ('keep-count', '1'), ('keep-count-test', '1')]) diff --git a/test/testdata/relarea/x86/release/staleversion/override.hint b/test/testdata/relarea/x86/release/staleversion/override.hint index e127171..1dee613 100644 --- a/test/testdata/relarea/x86/release/staleversion/override.hint +++ b/test/testdata/relarea/x86/release/staleversion/override.hint @@ -1,3 +1,3 @@ keep: 243-0 -keep-count: 0 +keep-count: 1 keep-count-test: 1