https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3a6f0d4b783476966161d0e34e1e10c02a2ad8f4 commit 3a6f0d4b783476966161d0e34e1e10c02a2ad8f4 Author: Jon Turney Date: Thu Nov 10 15:51:53 2016 +0000 Give counts of vaulted by request and added packages https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1f9188c7d2ae20778aad4cd19f8b653d906c34ca commit 1f9188c7d2ae20778aad4cd19f8b653d906c34ca Author: Jon Turney Date: Wed Nov 9 16:43:40 2016 +0000 Improve some logging of the reason why calm failed https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a431029ba0d42e62b28906d368693d7312c69cc2 commit a431029ba0d42e62b28906d368693d7312c69cc2 Author: Jon Turney Date: Wed Nov 9 16:42:40 2016 +0000 Handle uploads with absent and removed override.hint differently Track the removed files in a package upload, so we can take different action for absent and remove override.hint If override.hint is absent in an upload, the values in any existing override.hint apply If override.hint is being removed in an upload, the values in any existing override.hint should be ignored https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f6fc4fa0ba921a9333ae957ac8de00ef4cea74ea commit f6fc4fa0ba921a9333ae957ac8de00ef4cea74ea Author: Jon Turney Date: Wed Nov 9 13:18:34 2016 +0000 Only report changed legacy hints when they exist for both packages This prevents bogus reports of a change when the legacy hint for a package doens't exist, because a pvr.hint is used instead. Diff: --- calm/calm.py | 9 +++++++-- calm/package.py | 16 ++++++++++------ calm/uploads.py | 4 +++- test/testdata/uploads/pkglist.expected | 6 +++--- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index a9a5ed5..eaafe42 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -107,9 +107,10 @@ def process(args): stale_to_vault = remove_stale_packages(args, packages) if stale_to_vault: for arch in common_constants.ARCHES + ['noarch']: - logging.info("vaulting %d old packages for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) + logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) uploads.move_to_vault(args, stale_to_vault[arch]) else: + logging.error("error while evaluating stale packages") return None # read maintainer list @@ -191,6 +192,7 @@ def process(args): # if an error occurred ... if not stale_to_vault: # ... discard move list and merged_packages + logging.error("error while evaluating stale packages for %s" % (name)) continue # check for conflicting movelists @@ -203,6 +205,7 @@ def process(args): # if an error occurred ... if conflicts: # ... discard move list and merged_packages + logging.error("error while validating movelists for %s" % (name)) continue # for each arch and noarch @@ -210,15 +213,17 @@ def process(args): logging.debug("moving %s packages for maintainer %s" % (arch, name)) # process the move lists + logging.info("vaulting %d package(s) for arch %s, by request" % (len(scan_result[arch].to_vault), arch)) uploads.move_to_vault(args, scan_result[arch].to_vault) uploads.remove(args, scan_result[arch].remove_success) + logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch)) uploads.move_to_relarea(m, args, scan_result[arch].to_relarea) # for each arch if args.stale: for arch in common_constants.ARCHES + ['noarch']: if stale_to_vault[arch]: - logging.info("vaulting %d old package for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) + logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) uploads.move_to_vault(args, stale_to_vault[arch]) # for each arch diff --git a/calm/package.py b/calm/package.py index 7861d2d..84699b1 100755 --- a/calm/package.py +++ b/calm/package.py @@ -149,7 +149,7 @@ def clean_hints(p, hints, strict_lvl, warnings): # # read a single package # -def read_package(packages, basedir, dirpath, files, strict=False): +def read_package(packages, basedir, dirpath, files, strict=False, remove=[]): strict_lvl = logging.ERROR if strict else logging.WARNING relpath = os.path.relpath(dirpath, basedir) warnings = False @@ -180,6 +180,8 @@ def read_package(packages, basedir, dirpath, files, strict=False): hints = {} # determine version overrides + note_absent = ('override.hint' in remove) or ('override.hint' in files) + if 'override.hint' in files: # read override.hint override_hints = read_hints(p, os.path.join(dirpath, 'override.hint'), hint.override) @@ -193,10 +195,12 @@ def read_package(packages, basedir, dirpath, files, strict=False): if level in hints: override_hints[level] = hints[level] - # explicitly note any absent stability level hints - for level in ['test', 'curr', 'prev']: - if level not in override_hints: - override_hints[level] = None + # if override.hint exists or is being removed, explicitly note absent + # stability level hints + if note_absent: + for level in ['test', 'curr', 'prev']: + if level not in override_hints: + override_hints[level] = None # after we have migrated them to override hints, remove stability # level hints from legacy hints @@ -798,7 +802,7 @@ def merge(a, *l): # assign the legacy hints from b to all vr in a which didn't # have a pvr.hint. Instead, just report if it's going to # change and let things get sorted out later on... - if a[p].legacy_hints != b[p].legacy_hints: + if a[p].legacy_hints and b[p].legacy_hints and a[p].legacy_hints != b[p].legacy_hints: diff = '\n'.join(difflib.ndiff( pprint.pformat(a[p].legacy_hints).splitlines(), pprint.pformat(b[p].legacy_hints).splitlines())) diff --git a/calm/uploads.py b/calm/uploads.py index 9d44bfe..15a6c3c 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -75,6 +75,7 @@ def scan(m, all_packages, arch, args): # scan package directories for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')): relpath = os.path.relpath(dirpath, m.homedir()) + removed_files = [] # skip uninteresting directories if (not files) or (relpath == os.path.join(arch, 'release')): @@ -164,6 +165,7 @@ def scan(m, all_packages, arch, args): else: vault[relpath].append(f[1:]) remove_success.append(fn) + removed_files.append(f[1:]) files.remove(f) continue @@ -212,7 +214,7 @@ def scan(m, all_packages, arch, args): # read and validate package if files: # strict means we consider warnings as fatal for upload - if package.read_package(packages, m.homedir(), dirpath, files, strict=True): + if package.read_package(packages, m.homedir(), dirpath, files, strict=True, remove=removed_files): error = True # always consider timestamp as checked during a dry-run, so it is never diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected index 0e4c587..f36b2a4 100644 --- a/test/testdata/uploads/pkglist.expected +++ b/test/testdata/uploads/pkglist.expected @@ -6,11 +6,11 @@ 'Like it’s you’re Markup Language™ Nokogiri’s tool―that ' 'Bézier."', 'category': 'Devel', - 'requires': 'cygwin'}}, {'curr': None, 'prev': None, 'test': None}, False), + 'requires': 'cygwin'}}, {}, False), 'testpackage-subpackage': Package('x86/release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', - 'external-source': 'testpackage'}}, {'curr': None, 'prev': None, 'test': None}, False), + 'external-source': 'testpackage'}}, {}, False), 'testpackage2-subpackage': Package('x86/release/testpackage2/testpackage2-subpackage', {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('6de201dfed1d45412509c65deb34690dc2d09c6aafccfe491fd2f440f92842b9c755b61dc7bcdd4cc0c9f18cf46c2b3a1241e99c4c2a33fff5555e7b2f0b6348', 14, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"', 'ldesc': '"A test subpackage 2"', - 'category': 'Devel'}}, {'curr': None, 'prev': None, 'test': None}, False)} + 'category': 'Devel'}}, {}, False)}