From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 38353 invoked by alias); 30 Nov 2017 12:33:59 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 38248 invoked by uid 9795); 30 Nov 2017 12:33:59 -0000 Date: Thu, 30 Nov 2017 12:33:00 -0000 Message-ID: <20171130123358.38209.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20171113-20-ge5676d8 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 353e677e736a9ee8094c0c1eaa5da6a89eb6b59c X-Git-Newrev: e5676d815af7882d7ef9f27dbd29a5e584673c5b X-SW-Source: 2017-q4/txt/msg00038.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=e5676d815af7882d7ef9f27dbd29a5e584673c5b commit e5676d815af7882d7ef9f27dbd29a5e584673c5b Author: Jon Turney Date: Thu Nov 30 12:17:16 2017 +0000 Update TODO https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3d3799d841321c601dd089655fc2d3226e5723ae commit 3d3799d841321c601dd089655fc2d3226e5723ae Author: Jon Turney Date: Thu Nov 30 00:19:00 2017 +0000 Make --disable-check able to take comma separated values ... as well as being repeated https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5e90fe3bcba7e55876b9837f6cfecceef7e7f53e commit 5e90fe3bcba7e55876b9837f6cfecceef7e7f53e Author: Jon Turney Date: Wed Nov 15 14:05:32 2017 +0000 Add a disable-check: key in override.hint to disable certain checks Add a disable-check: key to hints to disable certain checks, rather than a static lists in past_mistakes (which needs calm restarting to re-read) disable-check: contains a list of tokens, similar to those for the --disable-check option, which indicates which checks are to be downgraded from errors to warnings for this package in override.hint, disable-check: can contain: curr-most-recent check that the curr: version is the most recent non-test version Also, don't consider equal mtime to be newer Also, allow curr-most-recent to be disabled by --disable-check https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cd23cb2d268a566d224785ae14f77ea4cc8e6061 commit cd23cb2d268a566d224785ae14f77ea4cc8e6061 Author: Jon Turney Date: Wed Nov 29 22:55:21 2017 +0000 Add mksetupini option --disable-check Add mksetupini option --disable-check, as a more general replacement for --okmissing https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a44390655e05567726d4f511a4ec1a890569ffe6 commit a44390655e05567726d4f511a4ec1a890569ffe6 Author: Jon Turney Date: Wed Nov 29 17:24:24 2017 +0000 Make calm and mksetupini return a non-zero exit status on error Diff: --- TODO | 4 +--- calm/calm.py | 9 +++++++-- calm/hint.py | 1 + calm/mksetupini.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- calm/package.py | 27 +++++++++++++++++---------- 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/TODO b/TODO index 07510d5..7bfcbbf 100644 --- a/TODO +++ b/TODO @@ -1,10 +1,8 @@ -* more than 2 versions possible * upload a hash at the same time as package, and pass that through to setup.ini * mksetupini should have an okmissing option for override.hint which names non-existent versions * use ./setup.hint inside the tar file, avoiding all the hint/tar coherence problems * mksetupini should write to stdout if --inifile not specified -* 'not in the _obsolete category' talks about 'all levels' rather than 'all versions' * check packages for path collisions * don't do upload authorization by path, then remove unique path constraint * mksetupini should be able to verify requires: contains valid package names using a provided list of packages (or a cygwin-pkg-maint file?) -* certain checks where we have past_mistakes to forgive should instead have a forgive: key in override.hint to disable? +* make override.hint (optionally?) apply recursively? diff --git a/calm/calm.py b/calm/calm.py index 120d9c8..a1d6ca0 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -359,12 +359,14 @@ def do_main(args, state): if not packages: logging.error("not processing uploads or writing setup.ini") - return + return 1 state.packages = packages do_output(args, state) + return 0 + # # @@ -641,11 +643,14 @@ def main(): state = CalmState() state.subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1]) + status = 0 if args.daemon: do_daemon(args, state) else: logging_setup(args) - do_main(args, state) + status = do_main(args, state) + + return status # diff --git a/calm/hint.py b/calm/hint.py index f454fa8..2f4b8f4 100755 --- a/calm/hint.py +++ b/calm/hint.py @@ -67,6 +67,7 @@ overridekeys = { 'keep': 'val', 'keep-count': 'val', 'keep-days': 'val', + 'disable-check': 'val', } hintkeys = {} diff --git a/calm/mksetupini.py b/calm/mksetupini.py index f24e4c5..10daa4e 100755 --- a/calm/mksetupini.py +++ b/calm/mksetupini.py @@ -60,7 +60,7 @@ def do_main(args): # validate the package set if not package.validate_packages(args, packages): logging.error("package set has errors, not writing setup.ini") - return + return 1 # write setup.ini package.write_setup_ini(args, packages, args.arch) @@ -68,6 +68,8 @@ def do_main(args): if args.stats: stats(packages) + return 0 + # # @@ -91,16 +93,48 @@ def stats(packages): # +# argparse helpers for an option which can take a comma separated list of +# choices, or can be repeated (e.g.: --option a --option b,c -> +# option:[a,b,c]) +# + +class flatten_append(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + curr = getattr(namespace, self.dest, self.default) + curr.extend(values) + setattr(namespace, self.dest, curr) + + +class choiceList(object): + def __init__(self, choices): + self.choices = choices + + def __call__(self, csv): + args = csv.split(',') + remainder = sorted(set(args) - set(self.choices)) + if remainder: + msg = "invalid choices: %r (choose from %r)" % (remainder, self.choices) + raise argparse.ArgumentTypeError(msg) + return args + + def help(self): + return '{%s}' % (','.join(self.choices)) + + +# # # def main(): pkglist_default = common_constants.PKGMAINT relarea_default = common_constants.FTP + disable_check_choices = choiceList(['missing-curr', 'missing-depended-package', 'missing-obsoleted-package', 'missing-required-package', 'curr-most-recent']) + parser = argparse.ArgumentParser(description='Make setup.ini') parser.add_argument('--arch', action='store', required=True, choices=common_constants.ARCHES) + parser.add_argument('--disable-check', action=flatten_append, help='checks to disable', type=disable_check_choices, default=[], metavar=disable_check_choices.help()) parser.add_argument('--inifile', '-u', action='store', help='output filename', required=True) - parser.add_argument('--okmissing', action='append', help='missing things which are ok', choices=['curr', 'depended-package', 'obsoleted-package', 'required-package']) + parser.add_argument('--okmissing', action='append', help='superseded by --disable-check', choices=['curr', 'depended-package', 'obsoleted-package', 'required-package']) parser.add_argument('--pkglist', action='store', nargs='?', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", const=pkglist_default) parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') @@ -115,7 +149,15 @@ def main(): logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s') - do_main(args) + # The option name 'okmissing' was inherited from genini. The more general + # option 'disable-check' is intended to supersede that, eventually. + # + # For the moment '--okmissing=foo' is silently transformed into it's + # equivalent '--disable-check=missing-foo' + if args.okmissing: + args.disable_check.extend(['missing-' + m for m in args.okmissing]) + + return do_main(args) # diff --git a/calm/package.py b/calm/package.py index 73ba0ed..45f752f 100755 --- a/calm/package.py +++ b/calm/package.py @@ -416,9 +416,9 @@ def validate_packages(args, packages): for (v, hints) in packages[p].version_hints.items(): for (c, okmissing, splitchar) in [ - ('requires', 'required-package', None), - ('depends', 'depended-package', ','), - ('obsoletes', 'obsoleted-package', ',') + ('requires', 'missing-required-package', None), + ('depends', 'missing-depended-package', ','), + ('obsoletes', 'missing-obsoleted-package', ',') ]: if c in hints: for r in hints[c].split(splitchar): @@ -438,10 +438,10 @@ def validate_packages(args, packages): lvl = logging.WARNING if p not in past_mistakes.self_requires else logging.DEBUG logging.log(lvl, "package '%s' version '%s' %s itself" % (p, v, c)) - # all packages listed in a hint must exist (unless - # okmissing says that's ok) + # all packages listed in a hint must exist (unless the + # disable-check option says that's ok) if r not in packages: - if okmissing not in getattr(args, 'okmissing', []): + if okmissing not in getattr(args, 'disable_check', []): logging.error("package '%s' version '%s' %s nonexistent package '%s'" % (p, v, c, r)) error = True continue @@ -597,7 +597,7 @@ def validate_packages(args, packages): logging.error("no versions at any stability level for package '%s'" % (p)) error = True # it's also probably a really good idea if a curr version exists - elif 'curr' not in packages[p].stability and 'curr' not in getattr(args, 'okmissing', []): + elif 'curr' not in packages[p].stability and 'missing-curr' not in getattr(args, 'disable_check', []): logging.warning("package '%s' doesn't have a curr version" % (p)) # error if the curr: version isn't the most recent non-test: version @@ -605,13 +605,20 @@ def validate_packages(args, packages): if 'test' in packages[p].version_hints[v]: continue - if packages[p].stability['curr'] != v: - if p in past_mistakes.mtime_anomalies: + cv = packages[p].stability['curr'] + if cv != v: + if packages[p].vermap[v]['mtime'] == packages[p].vermap[cv]['mtime']: + # don't consider an equal mtime to be more recent + continue + + if ((p in past_mistakes.mtime_anomalies) or + ('curr-most-recent' in packages[p].override_hints.get('disable-check', '')) or + ('curr-most-recent' in getattr(args, 'disable_check', []))): lvl = logging.DEBUG else: lvl = logging.ERROR error = True - logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, packages[p].stability['curr'])) + logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, cv)) break