From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id B8C1A383E80F; Mon, 25 May 2020 15:53:08 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org B8C1A383E80F To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20200401-27-g85136ea X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: b11ef2ffff507e3e132891b29bfc83fc737d6981 X-Git-Newrev: 85136ea903139a4182799246bbc546729a830fed Message-Id: <20200525155308.B8C1A383E80F@sourceware.org> Date: Mon, 25 May 2020 15:53:08 +0000 (GMT) From: Jon TURNEY X-BeenThere: cygwin-apps-cvs@cygwin.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Cygwin-apps git logs List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 25 May 2020 15:53:08 -0000 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=85136ea903139a4182799246bbc546729a830fed commit 85136ea903139a4182799246bbc546729a830fed Author: Jon Turney Date: Mon May 25 16:28:59 2020 +0100 Follow any version: override when checking if a hint is stale When uploading a replacement hint with a version: override, the overriden version doesn't exist anymore, so the hint will be considered stale and removed. This causes the upload to fail as movelist validation notices that the hint is both uploaded and automatically vaulted. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=fb3a4d3cb501a22b3d55165fc3d6fd222ae47c51 commit fb3a4d3cb501a22b3d55165fc3d6fd222ae47c51 Author: Jon Turney Date: Fri May 22 15:09:42 2020 +0100 Improve unique-version check Ignore install packages which have a different current version where they've been adopted by a different source package. Future work: This still gets confused by minizip package. The best way to fix that is probably to simply always record a sibling source package as external-source: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=620236e1132a1ec6a619961278d02d38868f3655 commit 620236e1132a1ec6a619961278d02d38868f3655 Author: Jon Turney Date: Mon May 18 17:30:39 2020 +0100 Install additional flake8 plugins to use in CI https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7ab4a15f026a2478dd8ac0ac61cb5e0de36b9748 commit 7ab4a15f026a2478dd8ac0ac61cb5e0de36b9748 Author: Jon Turney Date: Wed May 20 14:42:41 2020 +0100 Avoid mutable argument default for read_package_dir() Avoid a potentially mutable argument default for read_package_dir() (hopefully not an issue presently as we don't modify that argument anywhere) B006 Do not use mutable data structures for argument defaults https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=aab58145c5aae11d1ebbd99eaf425901670e8a89 commit aab58145c5aae11d1ebbd99eaf425901670e8a89 Author: Jon Turney Date: Wed May 20 14:39:13 2020 +0100 Fix redundant exception types in irk.py B014 Redundant exception types in `except (ConnectionRefusedError, OSError):`. Write `except OSError:`, which catches exactly the same exceptions. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=c452b2cb5c8fbe0a7a50fd90f295806c8a029567 commit c452b2cb5c8fbe0a7a50fd90f295806c8a029567 Author: Jon Turney Date: Mon May 18 17:15:32 2020 +0100 Clean up unnecessary uses of get/setattr() in test_calm.py B009 Do not call getattr with a constant attribute value, it is not any safer than normal property access. B010 Do not call setattr with a constant attribute value, it is not any safer than normal property access. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a68bad0fefa1f98195b2be511d16b5cffaa3715f commit a68bad0fefa1f98195b2be511d16b5cffaa3715f Author: Jon Turney Date: Wed May 20 14:49:34 2020 +0100 Drop an unneeded iteration over key as well as values B007 Loop control variable 'v' not used within the loop body. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=513ad0fab28255496b9ba40803c87880edeb70b0 commit 513ad0fab28255496b9ba40803c87880edeb70b0 Author: Jon Turney Date: Mon May 18 17:05:05 2020 +0100 Mark unused variable in uses of os.walk() B007 Loop control variable 'subdirs' not used within the loop body. If this is intended, start the name with an underscore. Diff: --- .travis.yml | 3 +- calm/dedupsrc.py | 4 +-- calm/find-duplicates.py | 2 +- calm/fix-annotate-perl-hint.py | 2 +- calm/fix-skip-only-hint.py | 4 +-- calm/hint-migrate.py | 2 +- calm/irk.py | 2 +- calm/package.py | 18 ++++++++-- calm/past_mistakes.py | 4 --- requirements.txt | 8 +++++ test/test_calm.py | 80 +++++++++++++++++++++--------------------- 11 files changed, 73 insertions(+), 56 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c3b6f0..5bdc87a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ sudo: false python: - "3.4" - "3.6" -install: "pip install pycodestyle flake8 dirq" +install: + - pip install -r requirements.txt script: - flake8 --count --show-source - python -m unittest discover diff --git a/calm/dedupsrc.py b/calm/dedupsrc.py index 4bf48ea..5f84aea 100644 --- a/calm/dedupsrc.py +++ b/calm/dedupsrc.py @@ -55,7 +55,7 @@ def hint_file_write(fn, hints): # def invent_sdesc(path, vr): - for (dirpath, subdirs, files) in os.walk(path): + for (dirpath, _subdirs, files) in os.walk(path): # debuginfo packages never have a good sdesc if 'debuginfo' in dirpath: continue @@ -170,7 +170,7 @@ def dedup(archive, relarea): # adjust external-source in .hint for all subpackages for arch in ['x86', 'x86_64']: - for (dirpath, subdirs, files) in os.walk(os.path.join(relarea, arch, path)): + for (dirpath, _subdirs, files) in os.walk(os.path.join(relarea, arch, path)): subpkg = os.path.basename(dirpath) filename = subpkg + '-' + vr + '.hint' if filename in files: diff --git a/calm/find-duplicates.py b/calm/find-duplicates.py index ec850a4..7e49219 100644 --- a/calm/find-duplicates.py +++ b/calm/find-duplicates.py @@ -131,7 +131,7 @@ def compare_archives(f1, f2): def find_duplicates(args): basedir = os.path.join(args.rel_area, common_constants.ARCHES[0], 'release') - for (dirpath, subdirs, files) in os.walk(basedir): + for (dirpath, _subdirs, files) in os.walk(basedir): relpath = os.path.relpath(dirpath, basedir) otherdir = os.path.join(args.rel_area, common_constants.ARCHES[1], 'release', relpath) diff --git a/calm/fix-annotate-perl-hint.py b/calm/fix-annotate-perl-hint.py index d3cf1cf..0f2679a 100644 --- a/calm/fix-annotate-perl-hint.py +++ b/calm/fix-annotate-perl-hint.py @@ -64,7 +64,7 @@ def fix_one_hint(dirpath, hintfile): def fix_hints(relarea): - for (dirpath, subdirs, files) in os.walk(relarea): + for (dirpath, _subdirs, files) in os.walk(relarea): for f in files: if f.endswith('.hint'): fix_one_hint(dirpath, f) diff --git a/calm/fix-skip-only-hint.py b/calm/fix-skip-only-hint.py index 4ea0b1b..d25387a 100644 --- a/calm/fix-skip-only-hint.py +++ b/calm/fix-skip-only-hint.py @@ -77,7 +77,7 @@ def invent_from_subpackages(path, vr): sdesc_candidates = [] category_candidates = [] - for (dirpath, subdirs, files) in os.walk(path): + for (dirpath, _subdirs, files) in os.walk(path): # debuginfo packages never have a good information if 'debuginfo' in dirpath: continue @@ -180,7 +180,7 @@ def fix_hints(rel_area, mode): skip_only = 0 invented = 0 - for (dirpath, subdirs, files) in os.walk(rel_area): + for (dirpath, _subdirs, files) in os.walk(rel_area): vrs = {} for f in files: match = re.match(r'^.*?-(\d.*).hint$', f) diff --git a/calm/hint-migrate.py b/calm/hint-migrate.py index d4c166e..d7c6f41 100644 --- a/calm/hint-migrate.py +++ b/calm/hint-migrate.py @@ -42,7 +42,7 @@ def hint_migrate(args): for arch in common_constants.ARCHES + ['noarch']: basedir = os.path.join(args.rel_area, arch, 'release') - for (dirpath, subdirs, files) in os.walk(basedir): + for (dirpath, _subdirs, files) in os.walk(basedir): if 'setup.hint' not in files: continue diff --git a/calm/irk.py b/calm/irk.py index 0037a33..5dd195d 100755 --- a/calm/irk.py +++ b/calm/irk.py @@ -34,7 +34,7 @@ def irk(message, target=DEFAULT_TARGET, server=DEFAULT_SERVER): send(s, target, message) s.close() - except (ConnectionRefusedError, OSError): + except OSError: pass diff --git a/calm/package.py b/calm/package.py index 2790206..5d98f19 100755 --- a/calm/package.py +++ b/calm/package.py @@ -113,7 +113,7 @@ def read_packages(rel_area, arch): releasedir = os.path.join(rel_area, root) logging.debug('reading packages from %s' % releasedir) - for (dirpath, subdirs, files) in os.walk(releasedir, followlinks=True): + for (dirpath, _subdirs, files) in os.walk(releasedir, followlinks=True): read_package_dir(packages[root], rel_area, dirpath, files) logging.debug("%d packages read from %s" % (len(packages[root]), releasedir)) @@ -187,7 +187,10 @@ def clean_hints(p, hints, warnings): # (return True if problems, False otherwise) # -def read_package_dir(packages, basedir, dirpath, files, remove=[], upload=False): +def read_package_dir(packages, basedir, dirpath, files, remove=None, upload=False): + if remove is None: + remove = [] + relpath = os.path.relpath(dirpath, basedir) # the package name is always the directory name @@ -537,7 +540,7 @@ def validate_packages(args, packages): # versions of setup, which should just install B). This condition can # occur since we might have synthesized the depends: from the requires: # in read_hints(), so fix that up here. - for (v, hints) in packages[p].version_hints.items(): + for hints in packages[p].version_hints.values(): obsoletes = hints.get('obsoletes', '') if obsoletes: for o in obsoletes.split(','): @@ -867,6 +870,12 @@ def validate_packages(args, packages): if 'curr' not in packages[install_p].stability: continue + # ignore packages which have a different external-source: + # (e.g. where a different source package supersedes this one) + es = packages[install_p].version_hints[packages[install_p].best_version].get('external-source', source_p) + if es != source_p: + continue + # ignore specific packages we disable this check for if ((install_p in past_mistakes.nonunique_versions) or ('unique-version' in packages[install_p].version_hints[packages[install_p].best_version].get('disable-check', ''))): @@ -1401,6 +1410,9 @@ def stale_packages(packages): all_stale[v] = False for v in po.hints: + # apply any version override + v = po.hints[v].hints.get('version', v) + # if there's a pvr.hint without a fresh source or install of the # same version, move it as well if all_stale.get(v, True): diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 4ad1ee0..627411b 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -109,11 +109,9 @@ nonunique_versions = [ 'cgdb-debuginfo', # debuginfo from NMU needs to age out 'dolphin4', # dropped from kde-baseapps 'gcc-java', # dropped from gcc 6 - 'girepository-AppStream1.0', # moved from appstream-glib to appstream 'girepository-SpiceClientGtk2.0', # gtk2 dropped from spice-gtk 'gnome-panel-doc', 'gtk2.0-engines-svg', - 'guile-doc', 'kdepasswd', # dropped from split kde-baseapps 'kexi', # split out from calligra 'kfilereplace', # split out from kdewebdev @@ -125,7 +123,6 @@ nonunique_versions = [ 'libquota-devel', # no longer provided by e2fsprogs 'libturbojpeg', # no number means it isn't considered an old soversion 'libtxc_dxtn', # split out from s2tc - 'libtxc_dxtn-devel', # split out from s2tc 'mingw64-i686-poppler-qt4', # dropped since 0.62.0 'mingw64-i686-spice-gtk2.0', # gtk2 dropped from spice-gtk 'mingw64-x86_64-poppler-qt4', # dropped since 0.62.0 @@ -137,7 +134,6 @@ nonunique_versions = [ 'python-spiceclientgtk', # gtk2 dropped from spice-gtk 'sng-debuginfo', 'sqlite3-zlib', # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead - 'transfig-debuginfo', # after transfig 3.2.6 source is included in xfig 'w3m-img', ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f7e4f65 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +dirq +flake8 +flake8-blind-except +flake8-bugbear ; python_version >= "3.5" +flake8-import-order == 0.14.1 +lockfile +pycodestyle +python-daemon diff --git a/test/test_calm.py b/test/test_calm.py index 712c7bc..17aeb86 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -85,7 +85,7 @@ def compare_with_expected_file(test, dirpath, results, basename=None): def capture_dirtree(basedir): tree = {} - for dirpath, dirnames, filenames in os.walk(basedir): + for dirpath, _dirnames, filenames in os.walk(basedir): tree[os.path.relpath(dirpath, basedir)] = sorted(filenames) return tree @@ -131,7 +131,7 @@ class CalmTest(unittest.TestCase): self.maxDiff = None basedir = 'testdata/relarea' - for (dirpath, subdirs, files) in os.walk(basedir): + for (dirpath, _subdirs, files) in os.walk(basedir): relpath = os.path.relpath(dirpath, basedir) for f in files: expected = os.path.join('testdata/hints', relpath) @@ -162,13 +162,13 @@ class CalmTest(unittest.TestCase): htdocs = 'testdata/htdocs' args = types.SimpleNamespace() - setattr(args, 'arch', 'x86') - setattr(args, 'htdocs', htdocs) - setattr(args, 'rel_area', 'testdata/relarea') - setattr(args, 'homedir', 'testdata/homes') - setattr(args, 'dryrun', False) - setattr(args, 'force', True) - setattr(args, 'pkglist', 'testdata/pkglist/cygwin-pkg-maint') + args.arch = 'x86' + args.htdocs = htdocs + args.rel_area = 'testdata/relarea' + args.homedir = 'testdata/homes' + args.dryrun = False + args.force = True + args.pkglist = 'testdata/pkglist/cygwin-pkg-maint' packages = {} for arch in common_constants.ARCHES: @@ -183,7 +183,7 @@ class CalmTest(unittest.TestCase): compare_with_expected_file(self, 'testdata/htdocs.expected', dirlist, 'dirtree') # compare the output files with expected - for (dirpath, subdirs, files) in os.walk(htdocs): + for (dirpath, _subdirs, files) in os.walk(htdocs): relpath = os.path.relpath(dirpath, htdocs) for f in files: with self.subTest(file=os.path.join(relpath, f)): @@ -318,9 +318,9 @@ class CalmTest(unittest.TestCase): logging.info('test_root = %s', test_root) args = types.SimpleNamespace() - setattr(args, 'arch', 'x86') - setattr(args, 'rel_area', 'testdata/relarea') - setattr(args, 'dryrun', False) + args.arch = 'x86' + args.rel_area = 'testdata/relarea' + args.dryrun = False shutil.copytree('testdata/homes', os.path.join(test_root, 'testdata/homes')) oldcwd = os.getcwd() @@ -357,14 +357,14 @@ class CalmTest(unittest.TestCase): self.maxDiff = None args = types.SimpleNamespace() - setattr(args, 'arch', 'x86') - setattr(args, 'dryrun', False) - setattr(args, 'force', True) - setattr(args, 'inifile', 'testdata/inifile/setup.ini') - setattr(args, 'pkglist', 'testdata/pkglist/cygwin-pkg-maint') - setattr(args, 'rel_area', 'testdata/relarea') - setattr(args, 'release', 'testing') - setattr(args, 'setup_version', '4.321') + args.arch = 'x86' + args.dryrun = False + args.force = True + args.inifile = 'testdata/inifile/setup.ini' + args.pkglist = 'testdata/pkglist/cygwin-pkg-maint' + args.rel_area = 'testdata/relarea' + args.release = 'testing' + args.setup_version = '4.321' packages = package.read_packages(args.rel_area, args.arch) package.delete(packages, 'x86/release/nonexistent', 'nosuchfile-1.0.0.tar.xz') @@ -388,17 +388,17 @@ class CalmTest(unittest.TestCase): setattr(args, d, tempfile.mktemp()) logging.info('%s = %s', d, getattr(args, d)) - shutil.copytree('testdata/relarea', getattr(args, 'rel_area')) - shutil.copytree('testdata/homes.conflict', getattr(args, 'homedir')) + shutil.copytree('testdata/relarea', args.rel_area) + shutil.copytree('testdata/homes.conflict', args.homedir) - setattr(args, 'dryrun', False) - setattr(args, 'email', None) - setattr(args, 'force', False) - setattr(args, 'pkglist', 'testdata/pkglist/cygwin-pkg-maint') - setattr(args, 'stale', True) + args.dryrun = False + args.email = None + args.force = False + args.pkglist = 'testdata/pkglist/cygwin-pkg-maint' + args.stale = True # set appropriate !ready - m_homedir = os.path.join(getattr(args, 'homedir'), 'Blooey McFooey') + m_homedir = os.path.join(args.homedir, 'Blooey McFooey') os.system('touch "%s"' % (os.path.join(m_homedir, 'x86', 'release', 'staleversion', '!ready'))) state = calm.calm.CalmState() @@ -421,22 +421,22 @@ class CalmTest(unittest.TestCase): setattr(args, d, tempfile.mktemp()) logging.info('%s = %s', d, getattr(args, d)) - setattr(args, 'dryrun', False) - setattr(args, 'email', None) - setattr(args, 'force', False) - setattr(args, 'inifile', os.path.join(getattr(args, 'rel_area'), 'setup.ini')) - setattr(args, 'pkglist', 'testdata/pkglist/cygwin-pkg-maint') - setattr(args, 'release', 'trial') - setattr(args, 'setup_version', '3.1415') - setattr(args, 'stale', True) + args.dryrun = False + args.email = None + args.force = False + args.inifile = os.path.join(args.rel_area, 'setup.ini') + args.pkglist = 'testdata/pkglist/cygwin-pkg-maint' + args.release = 'trial' + args.setup_version = '3.1415' + args.stale = True state = calm.calm.CalmState() - shutil.copytree('testdata/relarea', getattr(args, 'rel_area')) - shutil.copytree('testdata/homes', getattr(args, 'homedir')) + shutil.copytree('testdata/relarea', args.rel_area) + shutil.copytree('testdata/homes', args.homedir) # set appropriate !readys - m_homedir = os.path.join(getattr(args, 'homedir'), 'Blooey McFooey') + m_homedir = os.path.join(args.homedir, 'Blooey McFooey') ready_fns = [(os.path.join(m_homedir, 'x86', 'release', 'testpackage', '!ready'), ''), (os.path.join(m_homedir, 'x86', 'release', 'testpackage2', 'testpackage2-subpackage', '!ready'), ''), (os.path.join(m_homedir, 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700'),