public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20190530-9-gbc05695
@ 2019-06-05 10:08 jturney
0 siblings, 0 replies; only message in thread
From: jturney @ 2019-06-05 10:08 UTC (permalink / raw)
To: cygwin-apps-cvs
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bc056955516aaaabfa70f02c77e87ddca474bf93
commit bc056955516aaaabfa70f02c77e87ddca474bf93
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Wed Jun 5 10:58:30 2019 +0100
Fix ignored files warning
I think this has been broken since 6b03c67d. Setting reminders_issued
if the grace period has expired causes the timestamp file to get touched
every time, so it's never been long enough after the first time to
actually issue the warning.
Also increase the reminder period to a week.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0720a3e4163072f48e114123a762e2c32f487b76
commit 0720a3e4163072f48e114123a762e2c32f487b76
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue Jun 4 21:21:14 2019 +0100
Only update package metadata dump if it's contents will have changed
Even if we were to drop the timestamp element from the dump, we can't
write the json and then check if it's changed, as the order of python
lists and dict keys is non-deterministic.
So instead, just write it when it seems that something has changed.
This allows fetching the dump with If-Modified-Since: to work as
expected.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=673af6883fc5a22f6c3e730a81be3d6a0a32ad2f
commit 673af6883fc5a22f6c3e730a81be3d6a0a32ad2f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue Jun 4 19:50:09 2019 +0100
Fix and warn about some common, long-standing typos
See also https://cygwin.com/ml/cygwin-apps/2015-10/msg00056.html
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a702bd3f07e94193a551e4311e543fc79e2f5e64
commit a702bd3f07e94193a551e4311e543fc79e2f5e64
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon Jun 3 12:45:29 2019 +0100
Prevent !reminder-timestamp failing a test with an old checkout
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dded9ac8a584f7aa1ce7ca29a858fa3f05f5fd45
commit dded9ac8a584f7aa1ce7ca29a858fa3f05f5fd45
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Jun 2 21:27:08 2019 +0100
Only allow hyphen in version for specific historical versions
Only allow hyphen in version for the specific historical package
versions which have them, not for any version of those packages.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=92821169fad00b777923961e8d067bb9c0a0e748
commit 92821169fad00b777923961e8d067bb9c0a0e748
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Jun 2 16:57:00 2019 +0100
Drop unneeded things from past-mistakes
Diff:
---
calm/calm.py | 29 +++++----
calm/hint.py | 39 ++++++++++-
calm/package.py | 7 +-
calm/past_mistakes.py | 94 ++++++++++----------------
calm/uploads.py | 12 ++--
test/test_calm.py | 4 +
test/testdata/process_arch/homedir.expected | 2 +-
7 files changed, 103 insertions(+), 84 deletions(-)
diff --git a/calm/calm.py b/calm/calm.py
index 768003e..8627d1a 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -378,6 +378,8 @@ def do_output(args, state):
# XXX: perhaps we need a --[no]listing command line option to disable this from being run?
pkg2html.update_package_listings(args, state.packages)
+ update_json = False
+
# for each arch
for arch in common_constants.ARCHES:
logging.debug("writing setup.ini for arch %s" % (arch))
@@ -417,6 +419,8 @@ def do_output(args, state):
# then update setup.ini
if changed:
+ update_json = True
+
if args.dryrun:
logging.warning("not moving %s to %s, due to --dry-run" % (tmpfile.name, inifile))
os.remove(tmpfile.name)
@@ -455,18 +459,19 @@ def do_output(args, state):
# write packages.json
jsonfile = os.path.join(args.htdocs, 'packages.json.xz')
- with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmpfile:
- logging.debug('writing %s' % (tmpfile.name))
- with lzma.open(tmpfile, 'wt') as lzf:
- package.write_repo_json(args, state.packages, lzf)
- logging.info("moving %s to %s" % (tmpfile.name, jsonfile))
- shutil.move(tmpfile.name, jsonfile)
-
- # make it world-readable, if we can
- try:
- os.chmod(jsonfile, 0o644)
- except (OSError):
- pass
+ if update_json or not os.path.exists(jsonfile):
+ with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmpfile:
+ logging.debug('writing %s' % (tmpfile.name))
+ with lzma.open(tmpfile, 'wt') as lzf:
+ package.write_repo_json(args, state.packages, lzf)
+ logging.info("moving %s to %s" % (tmpfile.name, jsonfile))
+ shutil.move(tmpfile.name, jsonfile)
+
+ # make it world-readable, if we can
+ try:
+ os.chmod(jsonfile, 0o644)
+ except (OSError):
+ pass
#
diff --git a/calm/hint.py b/calm/hint.py
index 6e1a948..c340b94 100755
--- a/calm/hint.py
+++ b/calm/hint.py
@@ -235,22 +235,28 @@ def hint_file_parse(fn, kind):
if c.lower() not in categories:
errors.append("unknown category '%s'" % (c))
- # verify that value for ldesc or sdesc is quoted
- # (genini forces this)
if key in ['sdesc', 'ldesc']:
+ # verify that value for ldesc or sdesc is quoted (genini
+ # forces this)
if not (value.startswith('"') and value.endswith('"')):
errors.append("%s value '%s' should be quoted" % (key, value))
+ # warn about and fix common typos in ldesc/sdesc
+ value, msg = typofix(value)
+ if msg:
+ warnings.append("%s in %s" % (','.join(msg), key))
+
# if sdesc ends with a '.', warn and fix it
if key == 'sdesc':
if re.search(r'\."$', value):
warnings.append("sdesc ends with '.'")
value = re.sub(r'\."$', '"', value)
- # warn if sdesc contains ' '
+ # if sdesc contains ' ', warn and fix it
if key == 'sdesc':
if ' ' in value:
warnings.append("sdesc contains ' '")
+ value = value.replace(' ', ' ')
# only 'ldesc' and 'message' are allowed a multi-line value
if (type != 'multilineval') and (len(value.splitlines()) > 1):
@@ -321,6 +327,33 @@ def hint_file_parse(fn, kind):
#
+# words that Cygwin package maintainers apparently can't spell correctly
+#
+
+words = [
+ (' accomodates ', ' accommodates '),
+ (' consistant ', ' consistent '),
+ (' examing ', ' examining '),
+ (' extremly ', ' extremely '),
+ (' interm ', ' interim '),
+ (' procesors ', ' processors '),
+ (' utilitzed ', ' utilized '),
+ (' utilties ', ' utilities '),
+]
+
+
+def typofix(v):
+ msg = []
+
+ for (wrong, right) in words:
+ if wrong in v:
+ v = v.replace(wrong, right)
+ msg.append('%s -> %s' % (wrong.strip(), right.strip()))
+
+ return v, msg
+
+
+#
#
#
diff --git a/calm/package.py b/calm/package.py
index 502547e..76ddb85 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -249,11 +249,11 @@ def read_package(packages, basedir, dirpath, files, remove=[]):
# we already know P to split unambiguously), but this is a bad
# idea.
if '-' in v:
- if p not in past_mistakes.hyphen_in_version:
+ if v in past_mistakes.hyphen_in_version.get(p, []):
+ lvl = logging.INFO
+ else:
lvl = logging.ERROR
warnings = True
- else:
- lvl = logging.INFO
logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p))
if not v[0].isdigit():
@@ -1028,7 +1028,6 @@ def write_repo_json(args, packages, f):
'name': pn,
'versions': versions,
'summary': po.version_hints[bv].get('sdesc', '').strip('"'),
- 'categories': po.version_hints[bv].get('category', '').split(), # to be removed
'subpackages': [{'name': sp, 'categories': package(sp).version_hints[package(sp).best_version].get('category', '').split()} for sp in po.is_used_by],
'arches': arches,
}
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index dcb76d0..9a04f3d 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -27,44 +27,42 @@
# uses.
#
-# package names which have been used with versions containing a hyphen
-hyphen_in_version = [
- 'ctorrent',
- 'email',
- 'email-debuginfo',
- 'fdupes',
- 'fftw3',
- 'fftw3-debuginfo',
- 'fftw3-doc',
- 'gendef',
- 'gendef-debuginfo',
- 'gtk3-engines-unico',
- 'gtk3-engines-unico-debuginfo',
- 'hidapi',
- 'hidapi-debuginfo',
- 'libfftw3_3',
- 'libfftw3-devel',
- 'libfftw3-omp3',
- 'libhidapi-devel',
- 'libhidapi0',
- 'libmangle',
- 'libmangle-debuginfo',
- 'man-pages-posix',
- 'mingw64-i686-hidapi',
- 'mingw64-i686-hidapi-debuginfo',
- 'mingw64-x86_64-hidapi',
- 'mingw64-x86_64-hidapi-debuginfo',
- 'recode',
- 'recode-debuginfo',
- 'socat',
- 'socat-debuginfo',
- 'tack',
- 'tack-debuginfo',
- 'xemacs-mule-sumo',
- 'xemacs-sumo',
- 'xview',
- 'xview-devel',
-]
+# packages with historical versions containing a hyphen
+hyphen_in_version = {
+ 'ctorrent': ['1.3.4-dnh3.2'],
+ 'email': ['3.2.1-git', '3.2.3-git'],
+ 'email-debuginfo': ['3.2.1-git', '3.2.3-git'],
+ 'fdupes': ['1.50-PR2'],
+ 'fftw3': ['3.3.6-pl1'],
+ 'fftw3-debuginfo': ['3.3.6-pl1'],
+ 'fftw3-doc': ['3.3.6-pl1'],
+ 'gendef': ['1.0-svn2931'],
+ 'gendef-debuginfo': ['1.0-svn2931'],
+ 'hidapi': ['0.8.0-rc1'],
+ 'hidapi-debuginfo': ['0.8.0-rc1'],
+ 'libfftw3_3': ['3.3.6-pl1'],
+ 'libfftw3-devel': ['3.3.6-pl1'],
+ 'libfftw3-omp3': ['3.3.6-pl1'],
+ 'libhidapi-devel': ['0.8.0-rc1'],
+ 'libhidapi0': ['0.8.0-rc1'],
+ 'libmangle': ['1.0-svn2930'],
+ 'libmangle-debuginfo': ['1.0-svn2930'],
+ 'man-pages-posix': ['2013-a'],
+ 'mingw64-i686-hidapi': ['0.8.0-rc1'],
+ 'mingw64-i686-hidapi-debuginfo': ['0.8.0-rc1'],
+ 'mingw64-x86_64-hidapi': ['0.8.0-rc1'],
+ 'mingw64-x86_64-hidapi-debuginfo': ['0.8.0-rc1'],
+ 'recode': ['3.7-beta2'],
+ 'recode-debuginfo': ['3.7-beta2'],
+ 'socat': ['2.0.0-b8', '2.0.0-b9'],
+ 'socat-debuginfo': ['2.0.0-b8', '2.0.0-b9'],
+ 'tack': ['1.07-20130713', '1.07-20150606'],
+ 'tack-debuginfo': ['1.07-20130713', '1.07-20150606'],
+ 'xemacs-mule-sumo': ['2007-04-27'],
+ 'xemacs-sumo': ['2007-04-27'],
+ 'xview': ['3.2p1.4-28'],
+ 'xview-devel': ['3.2p1.4-28'],
+}
# cygport places this into the requires of every debuginfo package
self_requires = [
@@ -109,7 +107,6 @@ self_source = [
nonunique_versions = [
'bzr-debuginfo', # debuginfo from NMU needs to age out
'cgdb-debuginfo', # debuginfo from NMU needs to age out
- 'dolphin', # split out from kde-baseapps
'dolphin4', # dropped from kde-baseapps
'gcc-java', # dropped from gcc 6
'girepository-AppStream1.0', # moved from appstream-glib to appstream
@@ -118,33 +115,20 @@ nonunique_versions = [
'gtk2.0-engines-svg',
'guile-doc',
'kdepasswd', # dropped from split kde-baseapps
- 'kdialog', # split out from kde-baseapps
- 'keditbookmarks', # split out from kde-baseapps
'kexi', # split out from calligra
- 'kfind', # split out from kde-baseapps
'kfilereplace', # split out from kdewebdev
- 'kimagemapeditor', # split out from kdewebdev
- 'klinkstatus', # split out from kdewebdev
- 'konqueror', # split out from kde-baseapps
- 'libatomic_ops-devel', # separated out from libgc
'libcaca-doc', # dropped pending fix for current doxygen
'libflint', # no number means it isn't considered an old soversion
'libfltk-doc',
'libgcj-common', # dropped from gcc 6
'libical_cxx-devel',
- 'libpoppler-qt4_4', # dropped since 0.62.0
- 'libpoppler-qt4-devel', # dropped since 0.62.0
'libquota-devel', # no longer provided by e2fsprogs
'libturbojpeg', # no number means it isn't considered an old soversion
'libtxc_dxtn', # split out from s2tc
'libtxc_dxtn-devel', # split out from s2tc
'mingw64-i686-poppler-qt4', # dropped since 0.62.0
- 'mingw64-i686-qt5-declarative-debuginfo', # dropped in 5.9
- 'mingw64-i686-qt5-tools-debuginfo', # dropped in 5.9
'mingw64-i686-spice-gtk2.0', # gtk2 dropped from spice-gtk
'mingw64-x86_64-poppler-qt4', # dropped since 0.62.0
- 'mingw64-x86_64-qt5-declarative-debuginfo', # dropped in 5.9
- 'mingw64-x86_64-qt5-tools-debuginfo', # dropped in 5.9
'mingw64-x86_64-spice-gtk2.0', # gtk2 dropped from spice-gtk
'minizip',
'mutter-doc',
@@ -152,7 +136,6 @@ nonunique_versions = [
'okular4-part', # changed to okular5-part in 17.04
'python-spiceclientgtk', # gtk2 dropped from spice-gtk
'sng-debuginfo',
- 'socat-debuginfo', # debuginfo for test version when curr has no debuginfo
'sqlite3-zlib', # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead
'transfig-debuginfo', # after transfig 3.2.6 source is included in xfig
'w3m-img',
@@ -172,14 +155,10 @@ empty_but_not_obsolete = [
mtime_anomalies = [
'gcc-java',
'gcc-tools-epoch2-autoconf',
- 'glproto',
'gv-debuginfo',
'libgcj-common',
'libgcj16',
'python-gtk2.0',
- 'python-gtk2.0-demo',
- 'python-gtk2.0-devel',
- 'python-wx', # timestamps reset when split out from wxWidgets
'subversion', # 1.8 and 1.9 might be built in either order...
'subversion-debuginfo',
'subversion-devel',
@@ -189,5 +168,4 @@ mtime_anomalies = [
'subversion-python',
'subversion-ruby',
'subversion-tools',
- 'xextproto',
]
diff --git a/calm/uploads.py b/calm/uploads.py
index 69951cf..5b765cb 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -37,8 +37,8 @@ import time
from . import common_constants
from . import package
-# reminders will be issued daily
-REMINDER_INTERVAL = 60 * 60 * 24
+# reminders will be issued weekly
+REMINDER_INTERVAL = 60 * 60 * 24 * 7
# reminders don't start to be issued until an hour after upload
REMINDER_GRACE = 60 * 60
@@ -116,12 +116,10 @@ def scan(m, all_packages, arch, args):
m.reminders_timestamp_checked = True
logging.debug("ignoring %s as there is no !ready" % fn)
- ignored += 1
# don't warn until file is at least REMINDER_GRACE old
if (file_mtime < (time.time() - REMINDER_GRACE)):
- if not args.dryrun:
- m.reminders_issued = True
+ ignored += 1
else:
logging.warning("ignoring %s as it is newer than !ready" % fn)
files.remove(f)
@@ -267,9 +265,11 @@ def scan(m, all_packages, arch, args):
# if files are being ignored, and more than REMINDER_INTERVAL has elapsed
# since we warned about files being ignored, warn again
- if ignored > 0 and m.reminders_issued:
+ if ignored > 0:
if (time.time() > (m.reminder_time + REMINDER_INTERVAL)):
logging.warning("ignored %d files in %s as there is no !ready" % (ignored, arch))
+ if not args.dryrun:
+ m.reminders_issued = True
return ScanResult(error, packages, move, vault, remove, remove_success)
diff --git a/test/test_calm.py b/test/test_calm.py
index 9fefa19..a44517b 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -468,6 +468,10 @@ class CalmTest(unittest.TestCase):
for (f, t) in touches:
os.system('touch %s -d %s' % (f, t))
+ # ensure !reminder-timestamp is created for uploads
+ home = os.path.join('testdata', 'homes', 'Blooey McFooey')
+ os.system('find "%s" -type f -exec touch -d "12 hours ago" {} +' % (home))
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
diff --git a/test/testdata/process_arch/homedir.expected b/test/testdata/process_arch/homedir.expected
index 2f825f1..8b90c99 100644
--- a/test/testdata/process_arch/homedir.expected
+++ b/test/testdata/process_arch/homedir.expected
@@ -1,5 +1,5 @@
{'.': ['an_unexpected_file'],
- 'Blooey McFooey': [],
+ 'Blooey McFooey': ['!reminder-timestamp'],
'Blooey McFooey/noarch': [],
'Blooey McFooey/noarch/release': [],
'Blooey McFooey/noarch/release/perl-Net-SMTP-SSL': [],
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2019-06-05 10:08 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-06-05 10:08 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20190530-9-gbc05695 jturney
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).