[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20171113-20-ge5676d8

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20171113-20-ge5676d8

[jturney]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=e5676d815af7882d7ef9f27dbd29a5e584673c5b

commit e5676d815af7882d7ef9f27dbd29a5e584673c5b
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Nov 30 12:17:16 2017 +0000

    Update TODO

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3d3799d841321c601dd089655fc2d3226e5723ae

commit 3d3799d841321c601dd089655fc2d3226e5723ae
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Nov 30 00:19:00 2017 +0000

    Make --disable-check able to take comma separated values
    
    ... as well as being repeated

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5e90fe3bcba7e55876b9837f6cfecceef7e7f53e

commit 5e90fe3bcba7e55876b9837f6cfecceef7e7f53e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Nov 15 14:05:32 2017 +0000

    Add a disable-check: key in override.hint to disable certain checks
    
    Add a disable-check: key to hints to disable certain checks, rather than a
    static lists in past_mistakes (which needs calm restarting to re-read)
    
    disable-check: contains a list of tokens, similar to those for the
    --disable-check option, which indicates which checks are to be downgraded
    from errors to warnings for this package
    
    in override.hint, disable-check: can contain:
    
      curr-most-recent
         check that the curr: version is the most recent non-test version
    
    Also, don't consider equal mtime to be newer
    
    Also, allow curr-most-recent to be disabled by --disable-check

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cd23cb2d268a566d224785ae14f77ea4cc8e6061

commit cd23cb2d268a566d224785ae14f77ea4cc8e6061
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Nov 29 22:55:21 2017 +0000

    Add mksetupini option --disable-check
    
    Add mksetupini option --disable-check, as a more general replacement for
    --okmissing

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a44390655e05567726d4f511a4ec1a890569ffe6

commit a44390655e05567726d4f511a4ec1a890569ffe6
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Nov 29 17:24:24 2017 +0000

    Make calm and mksetupini return a non-zero exit status on error


Diff:
---
 TODO               |    4 +---
 calm/calm.py       |    9 +++++++--
 calm/hint.py       |    1 +
 calm/mksetupini.py |   48 +++++++++++++++++++++++++++++++++++++++++++++---
 calm/package.py    |   27 +++++++++++++++++----------
 5 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/TODO b/TODO
index 07510d5..7bfcbbf 100644
--- a/TODO
+++ b/TODO
@@ -1,10 +1,8 @@
-* more than 2 versions possible
 * upload a hash at the same time as package, and pass that through to setup.ini
 * mksetupini should have an okmissing option for override.hint which names non-existent versions
 * use ./setup.hint inside the tar file, avoiding all the hint/tar coherence problems
 * mksetupini should write to stdout if --inifile not specified
-* 'not in the _obsolete category' talks about 'all levels' rather than 'all versions'
 * check packages for path collisions
 * don't do upload authorization by path, then remove unique path constraint
 * mksetupini should be able to verify requires: contains valid package names using a provided list of packages (or a cygwin-pkg-maint file?)
-* certain checks where we have past_mistakes to forgive should instead have a forgive: key in override.hint to disable?
+* make override.hint (optionally?) apply recursively?
diff --git a/calm/calm.py b/calm/calm.py
index 120d9c8..a1d6ca0 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -359,12 +359,14 @@ def do_main(args, state):
 
     if not packages:
         logging.error("not processing uploads or writing setup.ini")
-        return
+        return 1
 
     state.packages = packages
 
     do_output(args, state)
 
+    return 0
+
 
 #
 #
@@ -641,11 +643,14 @@ def main():
     state = CalmState()
     state.subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1])
 
+    status = 0
     if args.daemon:
         do_daemon(args, state)
     else:
         logging_setup(args)
-        do_main(args, state)
+        status = do_main(args, state)
+
+    return status
 
 
 #
diff --git a/calm/hint.py b/calm/hint.py
index f454fa8..2f4b8f4 100755
--- a/calm/hint.py
+++ b/calm/hint.py
@@ -67,6 +67,7 @@ overridekeys = {
     'keep': 'val',
     'keep-count': 'val',
     'keep-days': 'val',
+    'disable-check': 'val',
 }
 
 hintkeys = {}
diff --git a/calm/mksetupini.py b/calm/mksetupini.py
index f24e4c5..10daa4e 100755
--- a/calm/mksetupini.py
+++ b/calm/mksetupini.py
@@ -60,7 +60,7 @@ def do_main(args):
     # validate the package set
     if not package.validate_packages(args, packages):
         logging.error("package set has errors, not writing setup.ini")
-        return
+        return 1
 
     # write setup.ini
     package.write_setup_ini(args, packages, args.arch)
@@ -68,6 +68,8 @@ def do_main(args):
     if args.stats:
         stats(packages)
 
+    return 0
+
 
 #
 #
@@ -91,16 +93,48 @@ def stats(packages):
 
 
 #
+# argparse helpers for an option which can take a comma separated list of
+# choices, or can be repeated (e.g.: --option a --option b,c ->
+# option:[a,b,c])
+#
+
+class flatten_append(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        curr = getattr(namespace, self.dest, self.default)
+        curr.extend(values)
+        setattr(namespace, self.dest, curr)
+
+
+class choiceList(object):
+    def __init__(self, choices):
+        self.choices = choices
+
+    def __call__(self, csv):
+        args = csv.split(',')
+        remainder = sorted(set(args) - set(self.choices))
+        if remainder:
+            msg = "invalid choices: %r (choose from %r)" % (remainder, self.choices)
+            raise argparse.ArgumentTypeError(msg)
+        return args
+
+    def help(self):
+        return '{%s}' % (','.join(self.choices))
+
+
+#
 #
 #
 def main():
     pkglist_default = common_constants.PKGMAINT
     relarea_default = common_constants.FTP
 
+    disable_check_choices = choiceList(['missing-curr', 'missing-depended-package', 'missing-obsoleted-package', 'missing-required-package', 'curr-most-recent'])
+
     parser = argparse.ArgumentParser(description='Make setup.ini')
     parser.add_argument('--arch', action='store', required=True, choices=common_constants.ARCHES)
+    parser.add_argument('--disable-check', action=flatten_append, help='checks to disable', type=disable_check_choices, default=[], metavar=disable_check_choices.help())
     parser.add_argument('--inifile', '-u', action='store', help='output filename', required=True)
-    parser.add_argument('--okmissing', action='append', help='missing things which are ok', choices=['curr', 'depended-package', 'obsoleted-package', 'required-package'])
+    parser.add_argument('--okmissing', action='append', help='superseded by --disable-check', choices=['curr', 'depended-package', 'obsoleted-package', 'required-package'])
     parser.add_argument('--pkglist', action='store', nargs='?', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", const=pkglist_default)
     parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin')
     parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
@@ -115,7 +149,15 @@ def main():
 
     logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s')
 
-    do_main(args)
+    # The option name 'okmissing' was inherited from genini.  The more general
+    # option 'disable-check' is intended to supersede that, eventually.
+    #
+    # For the moment '--okmissing=foo' is silently transformed into it's
+    # equivalent '--disable-check=missing-foo'
+    if args.okmissing:
+        args.disable_check.extend(['missing-' + m for m in args.okmissing])
+
+    return do_main(args)
 
 
 #
diff --git a/calm/package.py b/calm/package.py
index 73ba0ed..45f752f 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -416,9 +416,9 @@ def validate_packages(args, packages):
 
         for (v, hints) in packages[p].version_hints.items():
             for (c, okmissing, splitchar) in [
-                    ('requires', 'required-package', None),
-                    ('depends', 'depended-package', ','),
-                    ('obsoletes', 'obsoleted-package', ',')
+                    ('requires', 'missing-required-package', None),
+                    ('depends', 'missing-depended-package', ','),
+                    ('obsoletes', 'missing-obsoleted-package', ',')
             ]:
                 if c in hints:
                     for r in hints[c].split(splitchar):
@@ -438,10 +438,10 @@ def validate_packages(args, packages):
                             lvl = logging.WARNING if p not in past_mistakes.self_requires else logging.DEBUG
                             logging.log(lvl, "package '%s' version '%s' %s itself" % (p, v, c))
 
-                        # all packages listed in a hint must exist (unless
-                        # okmissing says that's ok)
+                        # all packages listed in a hint must exist (unless the
+                        # disable-check option says that's ok)
                         if r not in packages:
-                            if okmissing not in getattr(args, 'okmissing', []):
+                            if okmissing not in getattr(args, 'disable_check', []):
                                 logging.error("package '%s' version '%s' %s nonexistent package '%s'" % (p, v, c, r))
                                 error = True
                             continue
@@ -597,7 +597,7 @@ def validate_packages(args, packages):
             logging.error("no versions at any stability level for package '%s'" % (p))
             error = True
         # it's also probably a really good idea if a curr version exists
-        elif 'curr' not in packages[p].stability and 'curr' not in getattr(args, 'okmissing', []):
+        elif 'curr' not in packages[p].stability and 'missing-curr' not in getattr(args, 'disable_check', []):
             logging.warning("package '%s' doesn't have a curr version" % (p))
 
         # error if the curr: version isn't the most recent non-test: version
@@ -605,13 +605,20 @@ def validate_packages(args, packages):
             if 'test' in packages[p].version_hints[v]:
                 continue
 
-            if packages[p].stability['curr'] != v:
-                if p in past_mistakes.mtime_anomalies:
+            cv = packages[p].stability['curr']
+            if cv != v:
+                if packages[p].vermap[v]['mtime'] == packages[p].vermap[cv]['mtime']:
+                    # don't consider an equal mtime to be more recent
+                    continue
+
+                if ((p in past_mistakes.mtime_anomalies) or
+                    ('curr-most-recent' in packages[p].override_hints.get('disable-check', '')) or
+                    ('curr-most-recent' in getattr(args, 'disable_check', []))):
                     lvl = logging.DEBUG
                 else:
                     lvl = logging.ERROR
                     error = True
-                logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, packages[p].stability['curr']))
+                logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, cv))
 
             break