From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 87829 invoked by alias); 18 Mar 2016 17:13:30 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 87805 invoked by uid 9795); 18 Mar 2016 17:13:30 -0000 Date: Fri, 18 Mar 2016 17:13:00 -0000 Message-ID: <20160318171330.87770.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 0782de43170a1a81f0cb51ea83e60a0b53b28c6b X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: c118ad8400827f35f2839051c4bc77221c840ad0 X-Git-Newrev: 0782de43170a1a81f0cb51ea83e60a0b53b28c6b X-SW-Source: 2016-q1/txt/msg00044.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0782de43170a1a81f0cb51ea83e60a0b53b28c6b commit 0782de43170a1a81f0cb51ea83e60a0b53b28c6b Author: Jon Turney Date: Fri Mar 18 16:47:40 2016 +0000 Write detailed calm log output to a file Also write detailed calm log output to a file Rotate this log file on each run Send normal output to stdout, rather than stderr Downgrade a verycommon message from update_package_listings() to DEBUG https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=615aa0700618bd7d70f11854294e9823b7d420eb commit 615aa0700618bd7d70f11854294e9823b7d420eb Author: Jon Turney Date: Thu Mar 17 23:46:04 2016 +0000 Remove deletion cookies after files are vaulted In uploads.scan(), rename 'ready' to 'remove' Add 'remove_success' (again) to hold the list of deletion cookies Remove those files after the vaulting has occured https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5ad04e2302bff312cb5d72b520be1fad7448fafb commit 5ad04e2302bff312cb5d72b520be1fad7448fafb Author: Jon Turney Date: Thu Mar 17 23:46:21 2016 +0000 Tolerate requests to delete files which don't exist https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=14871cd9ee4bdbc4777adc80227bd530690ac7b1 commit 14871cd9ee4bdbc4777adc80227bd530690ac7b1 Author: Jon Turney Date: Thu Mar 17 23:09:52 2016 +0000 Sort requires: before showing differences This is a cosmetic improvement to reporting Diff: --- calm | 25 ++++++++++++++++++++++--- package.py | 9 +++++++-- pkg2html.py | 2 +- tests.py | 3 ++- upload-scan | 3 ++- uploads.py | 23 ++++++++++++++--------- 6 files changed, 48 insertions(+), 17 deletions(-) diff --git a/calm b/calm index fa095bd..92fbdcb 100755 --- a/calm +++ b/calm @@ -91,7 +91,7 @@ def main(args): # also send a mail to each maintainer about their packages with mail_logs(args.email, toaddrs=m.email, subject='calm messages for %s [%s]' % (name, details)) as maint_email: - (error, mpackages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args) + (error, mpackages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args) uploads.remove(args, remove_always) @@ -117,6 +117,7 @@ def main(args): if package.validate_packages(args, merged_packages): # process the move list uploads.move_to_vault(args, to_vault) + uploads.remove(args, remove_success) uploads.move_to_relarea(m, args, to_relarea) # use merged package list packages = merged_packages @@ -163,10 +164,28 @@ if __name__ == "__main__": parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output') (args) = parser.parse_args() + # set up logging to a file + try: + os.makedirs('/var/log/cygwin/', exist_ok=True) + except FileExistsError: + pass + rfh = logging.handlers.RotatingFileHandler('/var/log/cygwin/calm.log', backupCount=24) + rfh.doRollover() # force a rotate on every run + rfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s')) + rfh.setLevel(logging.INFO) + logging.getLogger().addHandler(rfh) + + # setup logging to stdout, of WARNING messages or higher (INFO if verbose) + ch = logging.StreamHandler(sys.stdout) + ch.setFormatter(logging.Formatter(os.path.basename(sys.argv[0])+': %(message)s')) if args.verbose: - logging.getLogger().setLevel(logging.INFO) + ch.setLevel(logging.INFO) + else: + ch.setLevel(logging.WARNING) + logging.getLogger().addHandler(ch) - logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s') + # change root logger level from the default of WARNING + logging.getLogger().setLevel(logging.INFO) if args.email: args.email = args.email.split(',') diff --git a/package.py b/package.py index 27fae03..1213a67 100755 --- a/package.py +++ b/package.py @@ -592,15 +592,20 @@ def merge(a, b): else: c[p].tars[t] = b[p].tars[t] - # use hints from b, but warn that they have changed + # use hints from b, but warn if they have changed if a[p].hints != b[p].hints: c[p].hints = b[p].hints + # sort requires: as differences in ordering are uninteresting + for hints in [a[p].hints, b[p].hints]: + if 'requires' in hints: + hints['requires'] = ' '.join(sorted(hints['requires'].split())) + diff = '\n'.join(difflib.ndiff( pprint.pformat(a[p].hints).splitlines(), pprint.pformat(b[p].hints).splitlines())) - logging.warning("package '%s' hints changed\n%s\n" % (p, diff)) + logging.warning("package '%s' hints changed\n%s" % (p, diff)) return c diff --git a/pkg2html.py b/pkg2html.py index 549ac8c..7f593c4 100755 --- a/pkg2html.py +++ b/pkg2html.py @@ -170,7 +170,7 @@ def update_package_listings(args, packages): '''), file=f) else: - logging.info('Not writing %s, already exists' % html) + logging.debug('Not writing %s, already exists' % html) # this file should exist, so remove from the toremove list if html in toremove: diff --git a/tests.py b/tests.py index 6453337..5b97297 100755 --- a/tests.py +++ b/tests.py @@ -184,11 +184,12 @@ class TestMain(unittest.TestCase): for (f, t) in ready_fns: os.system('touch %s "%s"' % (t, f)) - (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args) + (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args) self.assertEqual(error, False) compare_with_expected_file(self, 'testdata/uploads', to_relarea, 'move') self.assertCountEqual(remove_always, [f for (f, t) in ready_fns]) + self.assertEqual(remove_success, []) compare_with_expected_file(self, 'testdata/uploads', packages, 'pkglist') def test_package_set(self): diff --git a/upload-scan b/upload-scan index 033574e..08ab677 100755 --- a/upload-scan +++ b/upload-scan @@ -64,7 +64,7 @@ def main(args): with mail_logs(args.email, toaddrs=(args.email or []) + m.email, subject='upset messages') as maint_email: # search for and validate uploaded packages - (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args) + (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args) # always remove all the !ready files uploads.remove(args, remove_always) @@ -72,6 +72,7 @@ def main(args): # but only move something if there were no errors if not error: uploads.move_to_vault(args, to_vault) + uploads.remove(args, remove_success) uploads.move_to_relarea(m, args, to_relarea) diff --git a/uploads.py b/uploads.py index af784c5..33ae511 100644 --- a/uploads.py +++ b/uploads.py @@ -45,7 +45,8 @@ def scan(m, all_packages, args): packages = defaultdict(package.Package) move = defaultdict(list) vault = defaultdict(list) - readys = [] + remove = [] + remove_success = [] error = False mtimes = [('', 0)] @@ -57,7 +58,7 @@ def scan(m, all_packages, args): mtime = os.path.getmtime(ready) mtimes.append(('', mtime)) logging.info('processing files with mtime older than %d' % (mtime)) - readys.append(ready) + remove.append(ready) # scan package directories for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')): @@ -74,7 +75,7 @@ def scan(m, all_packages, args): ready = os.path.join(dirpath, '!ready') mtime = os.path.getmtime(ready) mtimes.append((relpath + '/', mtime)) - readys.append(ready) + remove.append(ready) files.remove('!ready') logging.info("processing files below '%s' with mtime older than %d" % (relpath, mtime)) else: @@ -124,6 +125,7 @@ def scan(m, all_packages, args): if f.startswith('-'): vault[relpath].append(f[1:]) files.remove(f) + remove_success.append(fn) else: dest = os.path.join(releasedir, relpath, f) if os.path.isfile(dest): @@ -150,15 +152,15 @@ def scan(m, all_packages, args): if package.read_package(packages, basedir, dirpath, files, strict=True): error = True - return (error, packages, move, vault, readys) + return (error, packages, move, vault, remove, remove_success) # # # -def remove(args, readys): - for f in readys: +def remove(args, remove): + for f in remove: logging.info("rm %s", f) if not args.dryrun: os.unlink(f) @@ -177,9 +179,12 @@ def move(args, movelist, fromdir, todir): except FileExistsError: pass for f in movelist[p]: - logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f))) - if not args.dryrun: - os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) + if os.path.exists(os.path.join(fromdir, p, f)): + logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f))) + if not args.dryrun: + os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) + else: + logging.error("%s can't be moved as it doesn't exist" % (f)) def move_to_relarea(m, args, movelist):