From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 43108 invoked by alias); 5 Jun 2019 10:08:41 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 43054 invoked by uid 9795); 5 Jun 2019 10:08:40 -0000 Date: Wed, 05 Jun 2019 10:08:00 -0000 Message-ID: <20190605100840.43032.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20190530-9-gbc05695 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 364bbc4fd6d43fda32ebc5711df4d584eec0d65d X-Git-Newrev: bc056955516aaaabfa70f02c77e87ddca474bf93 X-SW-Source: 2019-q2/txt/msg00017.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bc056955516aaaabfa70f02c77e87ddca474bf93 commit bc056955516aaaabfa70f02c77e87ddca474bf93 Author: Jon Turney Date: Wed Jun 5 10:58:30 2019 +0100 Fix ignored files warning I think this has been broken since 6b03c67d. Setting reminders_issued if the grace period has expired causes the timestamp file to get touched every time, so it's never been long enough after the first time to actually issue the warning. Also increase the reminder period to a week. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0720a3e4163072f48e114123a762e2c32f487b76 commit 0720a3e4163072f48e114123a762e2c32f487b76 Author: Jon Turney Date: Tue Jun 4 21:21:14 2019 +0100 Only update package metadata dump if it's contents will have changed Even if we were to drop the timestamp element from the dump, we can't write the json and then check if it's changed, as the order of python lists and dict keys is non-deterministic. So instead, just write it when it seems that something has changed. This allows fetching the dump with If-Modified-Since: to work as expected. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=673af6883fc5a22f6c3e730a81be3d6a0a32ad2f commit 673af6883fc5a22f6c3e730a81be3d6a0a32ad2f Author: Jon Turney Date: Tue Jun 4 19:50:09 2019 +0100 Fix and warn about some common, long-standing typos See also https://cygwin.com/ml/cygwin-apps/2015-10/msg00056.html https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a702bd3f07e94193a551e4311e543fc79e2f5e64 commit a702bd3f07e94193a551e4311e543fc79e2f5e64 Author: Jon Turney Date: Mon Jun 3 12:45:29 2019 +0100 Prevent !reminder-timestamp failing a test with an old checkout https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dded9ac8a584f7aa1ce7ca29a858fa3f05f5fd45 commit dded9ac8a584f7aa1ce7ca29a858fa3f05f5fd45 Author: Jon Turney Date: Sun Jun 2 21:27:08 2019 +0100 Only allow hyphen in version for specific historical versions Only allow hyphen in version for the specific historical package versions which have them, not for any version of those packages. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=92821169fad00b777923961e8d067bb9c0a0e748 commit 92821169fad00b777923961e8d067bb9c0a0e748 Author: Jon Turney Date: Sun Jun 2 16:57:00 2019 +0100 Drop unneeded things from past-mistakes Diff: --- calm/calm.py | 29 +++++---- calm/hint.py | 39 ++++++++++- calm/package.py | 7 +- calm/past_mistakes.py | 94 ++++++++++---------------- calm/uploads.py | 12 ++-- test/test_calm.py | 4 + test/testdata/process_arch/homedir.expected | 2 +- 7 files changed, 103 insertions(+), 84 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 768003e..8627d1a 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -378,6 +378,8 @@ def do_output(args, state): # XXX: perhaps we need a --[no]listing command line option to disable this from being run? pkg2html.update_package_listings(args, state.packages) + update_json = False + # for each arch for arch in common_constants.ARCHES: logging.debug("writing setup.ini for arch %s" % (arch)) @@ -417,6 +419,8 @@ def do_output(args, state): # then update setup.ini if changed: + update_json = True + if args.dryrun: logging.warning("not moving %s to %s, due to --dry-run" % (tmpfile.name, inifile)) os.remove(tmpfile.name) @@ -455,18 +459,19 @@ def do_output(args, state): # write packages.json jsonfile = os.path.join(args.htdocs, 'packages.json.xz') - with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmpfile: - logging.debug('writing %s' % (tmpfile.name)) - with lzma.open(tmpfile, 'wt') as lzf: - package.write_repo_json(args, state.packages, lzf) - logging.info("moving %s to %s" % (tmpfile.name, jsonfile)) - shutil.move(tmpfile.name, jsonfile) - - # make it world-readable, if we can - try: - os.chmod(jsonfile, 0o644) - except (OSError): - pass + if update_json or not os.path.exists(jsonfile): + with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmpfile: + logging.debug('writing %s' % (tmpfile.name)) + with lzma.open(tmpfile, 'wt') as lzf: + package.write_repo_json(args, state.packages, lzf) + logging.info("moving %s to %s" % (tmpfile.name, jsonfile)) + shutil.move(tmpfile.name, jsonfile) + + # make it world-readable, if we can + try: + os.chmod(jsonfile, 0o644) + except (OSError): + pass # diff --git a/calm/hint.py b/calm/hint.py index 6e1a948..c340b94 100755 --- a/calm/hint.py +++ b/calm/hint.py @@ -235,22 +235,28 @@ def hint_file_parse(fn, kind): if c.lower() not in categories: errors.append("unknown category '%s'" % (c)) - # verify that value for ldesc or sdesc is quoted - # (genini forces this) if key in ['sdesc', 'ldesc']: + # verify that value for ldesc or sdesc is quoted (genini + # forces this) if not (value.startswith('"') and value.endswith('"')): errors.append("%s value '%s' should be quoted" % (key, value)) + # warn about and fix common typos in ldesc/sdesc + value, msg = typofix(value) + if msg: + warnings.append("%s in %s" % (','.join(msg), key)) + # if sdesc ends with a '.', warn and fix it if key == 'sdesc': if re.search(r'\."$', value): warnings.append("sdesc ends with '.'") value = re.sub(r'\."$', '"', value) - # warn if sdesc contains ' ' + # if sdesc contains ' ', warn and fix it if key == 'sdesc': if ' ' in value: warnings.append("sdesc contains ' '") + value = value.replace(' ', ' ') # only 'ldesc' and 'message' are allowed a multi-line value if (type != 'multilineval') and (len(value.splitlines()) > 1): @@ -321,6 +327,33 @@ def hint_file_parse(fn, kind): # +# words that Cygwin package maintainers apparently can't spell correctly +# + +words = [ + (' accomodates ', ' accommodates '), + (' consistant ', ' consistent '), + (' examing ', ' examining '), + (' extremly ', ' extremely '), + (' interm ', ' interim '), + (' procesors ', ' processors '), + (' utilitzed ', ' utilized '), + (' utilties ', ' utilities '), +] + + +def typofix(v): + msg = [] + + for (wrong, right) in words: + if wrong in v: + v = v.replace(wrong, right) + msg.append('%s -> %s' % (wrong.strip(), right.strip())) + + return v, msg + + +# # # diff --git a/calm/package.py b/calm/package.py index 502547e..76ddb85 100755 --- a/calm/package.py +++ b/calm/package.py @@ -249,11 +249,11 @@ def read_package(packages, basedir, dirpath, files, remove=[]): # we already know P to split unambiguously), but this is a bad # idea. if '-' in v: - if p not in past_mistakes.hyphen_in_version: + if v in past_mistakes.hyphen_in_version.get(p, []): + lvl = logging.INFO + else: lvl = logging.ERROR warnings = True - else: - lvl = logging.INFO logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p)) if not v[0].isdigit(): @@ -1028,7 +1028,6 @@ def write_repo_json(args, packages, f): 'name': pn, 'versions': versions, 'summary': po.version_hints[bv].get('sdesc', '').strip('"'), - 'categories': po.version_hints[bv].get('category', '').split(), # to be removed 'subpackages': [{'name': sp, 'categories': package(sp).version_hints[package(sp).best_version].get('category', '').split()} for sp in po.is_used_by], 'arches': arches, } diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index dcb76d0..9a04f3d 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -27,44 +27,42 @@ # uses. # -# package names which have been used with versions containing a hyphen -hyphen_in_version = [ - 'ctorrent', - 'email', - 'email-debuginfo', - 'fdupes', - 'fftw3', - 'fftw3-debuginfo', - 'fftw3-doc', - 'gendef', - 'gendef-debuginfo', - 'gtk3-engines-unico', - 'gtk3-engines-unico-debuginfo', - 'hidapi', - 'hidapi-debuginfo', - 'libfftw3_3', - 'libfftw3-devel', - 'libfftw3-omp3', - 'libhidapi-devel', - 'libhidapi0', - 'libmangle', - 'libmangle-debuginfo', - 'man-pages-posix', - 'mingw64-i686-hidapi', - 'mingw64-i686-hidapi-debuginfo', - 'mingw64-x86_64-hidapi', - 'mingw64-x86_64-hidapi-debuginfo', - 'recode', - 'recode-debuginfo', - 'socat', - 'socat-debuginfo', - 'tack', - 'tack-debuginfo', - 'xemacs-mule-sumo', - 'xemacs-sumo', - 'xview', - 'xview-devel', -] +# packages with historical versions containing a hyphen +hyphen_in_version = { + 'ctorrent': ['1.3.4-dnh3.2'], + 'email': ['3.2.1-git', '3.2.3-git'], + 'email-debuginfo': ['3.2.1-git', '3.2.3-git'], + 'fdupes': ['1.50-PR2'], + 'fftw3': ['3.3.6-pl1'], + 'fftw3-debuginfo': ['3.3.6-pl1'], + 'fftw3-doc': ['3.3.6-pl1'], + 'gendef': ['1.0-svn2931'], + 'gendef-debuginfo': ['1.0-svn2931'], + 'hidapi': ['0.8.0-rc1'], + 'hidapi-debuginfo': ['0.8.0-rc1'], + 'libfftw3_3': ['3.3.6-pl1'], + 'libfftw3-devel': ['3.3.6-pl1'], + 'libfftw3-omp3': ['3.3.6-pl1'], + 'libhidapi-devel': ['0.8.0-rc1'], + 'libhidapi0': ['0.8.0-rc1'], + 'libmangle': ['1.0-svn2930'], + 'libmangle-debuginfo': ['1.0-svn2930'], + 'man-pages-posix': ['2013-a'], + 'mingw64-i686-hidapi': ['0.8.0-rc1'], + 'mingw64-i686-hidapi-debuginfo': ['0.8.0-rc1'], + 'mingw64-x86_64-hidapi': ['0.8.0-rc1'], + 'mingw64-x86_64-hidapi-debuginfo': ['0.8.0-rc1'], + 'recode': ['3.7-beta2'], + 'recode-debuginfo': ['3.7-beta2'], + 'socat': ['2.0.0-b8', '2.0.0-b9'], + 'socat-debuginfo': ['2.0.0-b8', '2.0.0-b9'], + 'tack': ['1.07-20130713', '1.07-20150606'], + 'tack-debuginfo': ['1.07-20130713', '1.07-20150606'], + 'xemacs-mule-sumo': ['2007-04-27'], + 'xemacs-sumo': ['2007-04-27'], + 'xview': ['3.2p1.4-28'], + 'xview-devel': ['3.2p1.4-28'], +} # cygport places this into the requires of every debuginfo package self_requires = [ @@ -109,7 +107,6 @@ self_source = [ nonunique_versions = [ 'bzr-debuginfo', # debuginfo from NMU needs to age out 'cgdb-debuginfo', # debuginfo from NMU needs to age out - 'dolphin', # split out from kde-baseapps 'dolphin4', # dropped from kde-baseapps 'gcc-java', # dropped from gcc 6 'girepository-AppStream1.0', # moved from appstream-glib to appstream @@ -118,33 +115,20 @@ nonunique_versions = [ 'gtk2.0-engines-svg', 'guile-doc', 'kdepasswd', # dropped from split kde-baseapps - 'kdialog', # split out from kde-baseapps - 'keditbookmarks', # split out from kde-baseapps 'kexi', # split out from calligra - 'kfind', # split out from kde-baseapps 'kfilereplace', # split out from kdewebdev - 'kimagemapeditor', # split out from kdewebdev - 'klinkstatus', # split out from kdewebdev - 'konqueror', # split out from kde-baseapps - 'libatomic_ops-devel', # separated out from libgc 'libcaca-doc', # dropped pending fix for current doxygen 'libflint', # no number means it isn't considered an old soversion 'libfltk-doc', 'libgcj-common', # dropped from gcc 6 'libical_cxx-devel', - 'libpoppler-qt4_4', # dropped since 0.62.0 - 'libpoppler-qt4-devel', # dropped since 0.62.0 'libquota-devel', # no longer provided by e2fsprogs 'libturbojpeg', # no number means it isn't considered an old soversion 'libtxc_dxtn', # split out from s2tc 'libtxc_dxtn-devel', # split out from s2tc 'mingw64-i686-poppler-qt4', # dropped since 0.62.0 - 'mingw64-i686-qt5-declarative-debuginfo', # dropped in 5.9 - 'mingw64-i686-qt5-tools-debuginfo', # dropped in 5.9 'mingw64-i686-spice-gtk2.0', # gtk2 dropped from spice-gtk 'mingw64-x86_64-poppler-qt4', # dropped since 0.62.0 - 'mingw64-x86_64-qt5-declarative-debuginfo', # dropped in 5.9 - 'mingw64-x86_64-qt5-tools-debuginfo', # dropped in 5.9 'mingw64-x86_64-spice-gtk2.0', # gtk2 dropped from spice-gtk 'minizip', 'mutter-doc', @@ -152,7 +136,6 @@ nonunique_versions = [ 'okular4-part', # changed to okular5-part in 17.04 'python-spiceclientgtk', # gtk2 dropped from spice-gtk 'sng-debuginfo', - 'socat-debuginfo', # debuginfo for test version when curr has no debuginfo 'sqlite3-zlib', # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead 'transfig-debuginfo', # after transfig 3.2.6 source is included in xfig 'w3m-img', @@ -172,14 +155,10 @@ empty_but_not_obsolete = [ mtime_anomalies = [ 'gcc-java', 'gcc-tools-epoch2-autoconf', - 'glproto', 'gv-debuginfo', 'libgcj-common', 'libgcj16', 'python-gtk2.0', - 'python-gtk2.0-demo', - 'python-gtk2.0-devel', - 'python-wx', # timestamps reset when split out from wxWidgets 'subversion', # 1.8 and 1.9 might be built in either order... 'subversion-debuginfo', 'subversion-devel', @@ -189,5 +168,4 @@ mtime_anomalies = [ 'subversion-python', 'subversion-ruby', 'subversion-tools', - 'xextproto', ] diff --git a/calm/uploads.py b/calm/uploads.py index 69951cf..5b765cb 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -37,8 +37,8 @@ import time from . import common_constants from . import package -# reminders will be issued daily -REMINDER_INTERVAL = 60 * 60 * 24 +# reminders will be issued weekly +REMINDER_INTERVAL = 60 * 60 * 24 * 7 # reminders don't start to be issued until an hour after upload REMINDER_GRACE = 60 * 60 @@ -116,12 +116,10 @@ def scan(m, all_packages, arch, args): m.reminders_timestamp_checked = True logging.debug("ignoring %s as there is no !ready" % fn) - ignored += 1 # don't warn until file is at least REMINDER_GRACE old if (file_mtime < (time.time() - REMINDER_GRACE)): - if not args.dryrun: - m.reminders_issued = True + ignored += 1 else: logging.warning("ignoring %s as it is newer than !ready" % fn) files.remove(f) @@ -267,9 +265,11 @@ def scan(m, all_packages, arch, args): # if files are being ignored, and more than REMINDER_INTERVAL has elapsed # since we warned about files being ignored, warn again - if ignored > 0 and m.reminders_issued: + if ignored > 0: if (time.time() > (m.reminder_time + REMINDER_INTERVAL)): logging.warning("ignored %d files in %s as there is no !ready" % (ignored, arch)) + if not args.dryrun: + m.reminders_issued = True return ScanResult(error, packages, move, vault, remove, remove_success) diff --git a/test/test_calm.py b/test/test_calm.py index 9fefa19..a44517b 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -468,6 +468,10 @@ class CalmTest(unittest.TestCase): for (f, t) in touches: os.system('touch %s -d %s' % (f, t)) + # ensure !reminder-timestamp is created for uploads + home = os.path.join('testdata', 'homes', 'Blooey McFooey') + os.system('find "%s" -type f -exec touch -d "12 hours ago" {} +' % (home)) + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) diff --git a/test/testdata/process_arch/homedir.expected b/test/testdata/process_arch/homedir.expected index 2f825f1..8b90c99 100644 --- a/test/testdata/process_arch/homedir.expected +++ b/test/testdata/process_arch/homedir.expected @@ -1,5 +1,5 @@ {'.': ['an_unexpected_file'], - 'Blooey McFooey': [], + 'Blooey McFooey': ['!reminder-timestamp'], 'Blooey McFooey/noarch': [], 'Blooey McFooey/noarch/release': [], 'Blooey McFooey/noarch/release/perl-Net-SMTP-SSL': [],