From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 26142 invoked by alias); 10 Jun 2016 13:01:06 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 26114 invoked by uid 9795); 10 Jun 2016 13:01:05 -0000 Date: Fri, 10 Jun 2016 13:01:00 -0000 Message-ID: <20160610130105.26074.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. a361ca472d8cc7adbc7e1376dd2ddb322ca339d7 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: d86f99a0ec043f09521e35658d9ab6364e107642 X-Git-Newrev: a361ca472d8cc7adbc7e1376dd2ddb322ca339d7 X-SW-Source: 2016-q2/txt/msg00023.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a361ca472d8cc7adbc7e1376dd2ddb322ca339d7 commit a361ca472d8cc7adbc7e1376dd2ddb322ca339d7 Author: Jon Turney Date: Wed Jun 8 13:03:48 2016 +0000 Fix reminder rate limiting after noarch changes For noarch, we moved to a single timestamp per maintainer, but still reset it if it wasn't checked for an individual arch Pull the timestamp updating out to the maintainer class, so we can only reset it if there were no reminders for any arch https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=833572dfc55c31f85c5295fdbc36c35a5eb2006d commit 833572dfc55c31f85c5295fdbc36c35a5eb2006d Author: Jon Turney Date: Thu May 19 12:13:37 2016 +0000 Update TODO https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=110c18bd83aeca38f40b582a8db1c99cb51d77af commit 110c18bd83aeca38f40b582a8db1c99cb51d77af Author: Jon Turney Date: Thu May 19 13:08:44 2016 +0000 Directly check package is in self-source list Rather than adding a hint to indicate that a package is on the self-source list and then later checking for that hint, directly check if it's on the self-source list. Diff: --- TODO | 5 ++--- calm/calm.py | 3 +++ calm/maintainers.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ calm/package.py | 7 +------ calm/uploads.py | 37 ++++++++++--------------------------- 5 files changed, 60 insertions(+), 36 deletions(-) diff --git a/TODO b/TODO index f2c7bc7..60015d1 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,7 @@ * per-package-version hint files (e.g. named .hint, or /.setup.hint inside tar file) +* automatically vault stuff which has dropped out of setup.ini * more than 2 versions possible (and automatically vault 'old' versions, where we have some mechanism to explicity say what is old) -* automatically suppress 'empty source' packages from setup.ini ? +* automatically suppress 'empty source' packages with no deps from setup.ini ? * run more often, option to not do anything if no uploads (to avoid reading the release area if we don't need to), lockfile to avoid colliding runs * work out some way to package it as a replacement for genini -* write 'move' lines in report more compactly, don't need the full path on every line -* handle .noarch packages better * use irkerd to report when calm failed due to an error? diff --git a/calm/calm.py b/calm/calm.py index c9e5bd9..dd3269a 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -181,6 +181,9 @@ def process(args): packages[arch] = merged_packages[arch] logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name)) + # record updated reminder times for maintainers + maintainers.Maintainer.update_reminder_times(mlist) + return packages diff --git a/calm/maintainers.py b/calm/maintainers.py index 3d6d9c1..fad1d0e 100644 --- a/calm/maintainers.py +++ b/calm/maintainers.py @@ -32,6 +32,7 @@ # - the list of packages they maintain (given by cygwin-pkg-list) # - an email address (in HOME/!email (or !mail), as we don't want to publish # it, and want to allow the maintainer to change it) +# - the timestamp when 'ignoring' warnings were last emitted # import itertools @@ -40,6 +41,19 @@ import os import re import sys +# +# +# + + +def touch(fn, times=None): + with open(fn, 'a'): + os.utime(fn, times) + +# +# +# + class Maintainer(object): _homedirs = '' @@ -54,12 +68,37 @@ class Maintainer(object): self.email = email self.pkgs = pkgs + # the mtime of this file records the timestamp + reminder_file = os.path.join(self.homedir(), '!reminder-timestamp') + if os.path.exists(reminder_file): + self.reminder_time = os.path.getmtime(reminder_file) + else: + self.reminder_time = 0 + self.reminders_issued = False + self.reminders_timestamp_checked = False + def __repr__(self): return "maintainers.Maintainer('%s', %s, %s)" % (self.name, self.email, self.pkgs) def homedir(self): return os.path.join(Maintainer._homedirs, self.name) + def _update_reminder_time(self): + reminder_file = os.path.join(self.homedir(), '!reminder-timestamp') + + if self.reminders_issued: + # if reminders were issued, update the timestamp + logging.debug("updating reminder time for %s" % self.name) + touch(reminder_file) + elif (not self.reminders_timestamp_checked) and (self.reminder_time != 0): + # if we didn't need to check the reminder timestamp, it can be + # reset + logging.debug("resetting reminder time for %s" % self.name) + try: + os.remove(reminder_file) + except FileNotFoundError: + pass + @staticmethod def _find(mlist, name): mlist.setdefault(name, Maintainer(name)) @@ -140,6 +179,11 @@ class Maintainer(object): return mlist + @staticmethod + def update_reminder_times(mlist): + for m in mlist.values(): + m._update_reminder_time() + # a list of all packages @staticmethod def all_packages(mlist): diff --git a/calm/package.py b/calm/package.py index bf3d680..27a28fb 100755 --- a/calm/package.py +++ b/calm/package.py @@ -206,11 +206,6 @@ def read_package(packages, basedir, dirpath, files, strict=False): # now we have read the package, fix some common defects in the hints # - # note if the package is self-source - # XXX: this should really be defined as a setup.hint key - if p in past_mistakes.self_source: - packages[p].hints['self-source'] = '' - # don't allow a redundant 'package:' or 'package - ' at start of sdesc # # match case-insensitively, and use a base package name (trim off any @@ -443,7 +438,7 @@ def validate_packages(args, packages): continue # unless this package is marked as 'self-source' - if 'self-source' in packages[p].hints: + if p in past_mistakes.self_source: continue logging.error("package '%s' version '%s' is missing source" % (p, v)) diff --git a/calm/uploads.py b/calm/uploads.py index 9c11031..b35d11e 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -68,15 +68,9 @@ def scan(m, all_packages, arch, args): logging.debug('processing files with mtime older than %d' % (mtime)) remove.append(ready) - # the mtime of this file indicates when 'ignoring as there is no !ready' - # warnings were last emitted - reminder_file = os.path.join(m.homedir(), '!reminder-timestamp') - if os.path.exists(reminder_file): - reminder_time = os.path.getmtime(reminder_file) - else: - reminder_time = 0 - reminders = False - logging.debug("reminder-timestamp %d, interval %d, next reminder %d, current time %d" % (reminder_time, REMINDER_INTERVAL, reminder_time + REMINDER_INTERVAL, time.time())) + # we record a timestamp when 'ignoring as there is no !ready' warnings were + # last emitted + logging.debug("reminder-timestamp %d, interval %d, next reminder %d, current time %d" % (m.reminder_time, REMINDER_INTERVAL, m.reminder_time + REMINDER_INTERVAL, time.time())) # scan package directories for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')): @@ -140,15 +134,15 @@ def scan(m, all_packages, arch, args): # only process files newer than !ready if os.path.getmtime(fn) > mtime: if mtime == 0: - reminders = True + m.reminders_timestamp_checked = True lvl = logging.DEBUG # if more than REMINDER_INTERVAL has elapsed since we warned # about files being ignored, warn again - if time.time() > (reminder_time + REMINDER_INTERVAL): + if time.time() > (m.reminder_time + REMINDER_INTERVAL): lvl = logging.WARNING if not args.dryrun: - touch(reminder_file) + m.reminders_issued = True logging.log(lvl, "ignoring %s as there is no !ready" % fn) else: @@ -193,12 +187,10 @@ def scan(m, all_packages, arch, args): if package.read_package(packages, m.homedir(), dirpath, files, strict=True): error = True - # if we didn't need to check the reminder timestamp, it can be reset - if not reminders and not args.dryrun: - try: - os.remove(reminder_file) - except FileNotFoundError: - pass + # always consider timestamp as checked during a dry-run, so it is never + # reset + if args.dryrun: + m.reminders_timestamp_checked = True return ScanResult(error, packages, move, vault, remove, remove_success) @@ -207,15 +199,6 @@ def scan(m, all_packages, arch, args): # # -def touch(fn, times=None): - with open(fn, 'a'): - os.utime(fn, times) - - -# -# -# - def remove(args, remove): for f in remove: logging.debug("rm %s", f)