https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cbefca991dc088e2a53022a2aa2f4d92872f9db9 commit cbefca991dc088e2a53022a2aa2f4d92872f9db9 Author: Jon Turney Date: Mon May 3 23:44:00 2021 +0100 Simplify package processing by removing vermap Since a package now only contains either all source or all install archives, we can drop the 'category' indirection that vermap provides. Drop Package.vermap Drop the category argument to Package.tar(), and drop checks that it's the expected value before calling that. Drop the filename index (which came from vermap) into Package.tars, and rename it to Package.tarfiles ensure we've caught all references. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7fa131ab49af8b0d6188142a4b059f6b68777dcd commit 7fa131ab49af8b0d6188142a4b059f6b68777dcd Author: Jon Turney Date: Mon May 3 23:22:05 2021 +0100 Use a temporary dict when validating curr: version mtime is latest Use a temporary dict to hold mtime data when validating curr: version mtime is latest, rather than bodging that into vermap. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1b5ec6d6b9349bb4da35063529d3587250748170 commit 1b5ec6d6b9349bb4da35063529d3587250748170 Author: Jon Turney Date: Mon May 3 22:45:22 2021 +0100 Move some validation of tar file set for a package Move some validation of tar file set for a package from validate_packages() to read_one_package(). This is safe because merge() rejects merging packages when the same version appears in the tar file list for both packages. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=dcb1b09ae09b208d0f80b087c19c444eb8955505 commit dcb1b09ae09b208d0f80b087c19c444eb8955505 Author: Jon Turney Date: Mon May 3 17:55:31 2021 +0100 Encapsulate access to package version list Encapsulate access to the package version list, as a first step to removing vermap. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d8869d2e727d4b10806fedca2c62f3105ce4f8b3 commit d8869d2e727d4b10806fedca2c62f3105ce4f8b3 Author: Jon Turney Date: Mon May 3 17:47:46 2021 +0100 Drop validating versions appearing in override.hint keys This is a followup to d863c1a2, 3cc08991 and 27852b24. Since the curr:, prev:, test: keys are no longer permitted in override.hint, there's no point in checking their value. Do some simplifcation of the code which assigns versions to stability levels, but there's probably more to do there. Diff: --- calm/package.py | 314 ++++++++++++++------------------- calm/pkg2html.py | 25 +-- test/testdata/uploads/pkglist.expected | 12 +- 3 files changed, 151 insertions(+), 200 deletions(-) diff --git a/calm/package.py b/calm/package.py index b223edf..9c47f47 100755 --- a/calm/package.py +++ b/calm/package.py @@ -58,24 +58,26 @@ class Kind(Enum): class Package(object): def __init__(self): self.pkgpath = '' # path to package, relative to arch - self.tars = {} + self.tarfiles = {} self.hints = {} self.is_used_by = set() self.version_hints = {} self.override_hints = {} self.skip = False - self.vermap = {} def __repr__(self): return "Package('%s', %s, %s, %s, %s)" % ( self.pkgpath, - pprint.pformat(self.tars), + pprint.pformat(self.tarfiles), pprint.pformat(self.version_hints), pprint.pformat(self.override_hints), self.skip) - def tar(self, vr, category): - return self.tars[vr][self.vermap[vr][category]] + def tar(self, vr): + return self.tarfiles[vr] + + def versions(self): + return self.tarfiles.keys() # information we keep about a tar file @@ -283,9 +285,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): (relpath, packages[p].pkgpath)) return True - # determine version overrides - note_absent = ('override.hint' in remove) or ('override.hint' in files) - if 'override.hint' in files: # read override.hint override_hints = read_hints(p, os.path.join(dirpath, 'override.hint'), hint.override) @@ -296,13 +295,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): else: override_hints = {} - # if override.hint exists or is being removed, explicitly note absent - # stability level hints - if note_absent: - for level in ['test', 'curr', 'prev']: - if level not in override_hints: - override_hints[level] = None - # read sha512.sum sha512 = {} if 'sha512.sum' not in files: @@ -321,7 +313,7 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): # build a list of version-releases (since replacement pvr.hint files are # allowed to be uploaded, we must consider both .tar and .hint files for # that), and collect the attributes for each tar file - tars = defaultdict(dict) + tars = {} vr_list = set() for f in list(files): @@ -357,6 +349,23 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): vr_list.add(vr) if not f.endswith('.hint'): + # a package can only contain tar archives of the appropriate type + if not re.search(r'-src.*\.tar', f): + if kind == Kind.source: + logging.error("source package '%s' has install archives" % (p)) + return True + else: + if kind == Kind.binary: + logging.error("package '%s' has source archives" % (p)) + return True + + # for each version, a package can contain at most one tar file (of + # the appropriate type). Warn if we have too many (for e.g. both a + # .xz and .bz2 install tar file). + if vr in tars: + logging.error("package '%s' has more than one tar file for version '%s'" % (p, vr)) + return True + # collect the attributes for each tar file t = Tar() t.path = relpath @@ -371,7 +380,7 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): t.sha512 = sha512_file(os.path.join(dirpath, f)) logging.debug("no sha512.sum line for file %s in package '%s', computed sha512 hash is %s" % (f, p, t.sha512)) - tars[vr][f] = t + tars[vr] = t # determine hints for each version we've encountered version_hints = {} @@ -410,12 +419,13 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): version_hints[ovr] = pvr_hint hints[ovr] = hintobj - actual_tars[ovr] = tars[vr] + if vr in tars: + actual_tars[ovr] = tars[vr] packages[pn] = Package() packages[pn].version_hints = version_hints packages[pn].override_hints = override_hints - packages[pn].tars = actual_tars + packages[pn].tarfiles = actual_tars packages[pn].hints = hints packages[pn].pkgpath = pkgpath if kind == Kind.source: @@ -573,46 +583,14 @@ def validate_packages(args, packages): else: logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p)) - packages[p].vermap = {} - has_source = False has_install = False has_nonempty_install = False - for vr in packages[p].tars: - for (t, tar) in packages[p].tars[vr].items(): - # categorize each tarfile as either 'source' or 'install' - if re.search(r'-src.*\.tar', t): - category = 'source' - has_source = True - else: - category = 'install' - has_install = True - if not packages[p].tars[vr][t].is_empty: - has_nonempty_install = True - - if vr not in packages[p].vermap: - packages[p].vermap[vr] = {} - - # for each version, a package can contain at most one source tar - # file and at most one install tar file. warn if we have too many - # (for e.g. both a .xz and .bz2 install tar file) - if category in packages[p].vermap[vr]: - logging.error("package '%s' has more than one %s tar file for version '%s'" % (p, category, vr)) - error = True - - # store tarfile corresponding to this version and category - packages[p].vermap[vr][category] = t - packages[p].vermap[vr]['mtime'] = tar.mtime - - # check that package only contains tar archives of the appropriate type - if packages[p].kind == Kind.source: - if has_install: - logging.error("source package '%s' has install archives" % (p)) - error = True - elif packages[p].kind == Kind.binary: - if has_source: - logging.error("package '%s' has source archives" % (p)) - error = True + if packages[p].kind == Kind.binary: + for vr in packages[p].versions(): + has_install = True + if not packages[p].tar(vr).is_empty: + has_nonempty_install = True obsolete = any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints]) @@ -637,53 +615,31 @@ def validate_packages(args, packages): packages[p].skip = True logging.info("package '%s' appears to be source-only as it has no non-empty install tarfiles and no dependencies, marking as 'skip'" % (p)) - # verify the versions specified for stability level exist levels = ['test', 'curr', 'prev'] - for l in levels: - if l in packages[p].override_hints: - # check that if a version was specified, it exists - v = packages[p].override_hints[l] - - if v is None: - continue - - if v not in packages[p].vermap: - logging.error("package '%s' stability '%s' selects non-existent version '%s'" % (p, l, v)) - error = True # assign a version to each stability level packages[p].stability = defaultdict() # sort in order from highest to lowest version - for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): + for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): level_found = False while True: # no stability levels left if len(levels) == 0: - # XXX: versions which don't correspond to any stability level - # should be reported, we might want to remove them at some point - logging.log(5, "package '%s' has no stability levels left for version '%s'" % (p, v)) break l = levels[0] - # if current stability level has an override - if (l in packages[p].override_hints) and (packages[p].override_hints[l] is not None): - # if we haven't reached that version yet - if v != packages[p].override_hints[l]: - break - else: - logging.debug("package '%s' stability '%s' overridden to version '%s'" % (p, l, v)) # if package is explicitly marked as having that stability level # (only used for test, currently) - elif (l == 'test') and ('test' in packages[p].version_hints[v]): + if (l == 'test') and ('test' in packages[p].version_hints[v]): logging.debug("package '%s' version '%s' marked as stability '%s'" % (p, v, l)) else: - # level 'test' must be assigned by override + # level 'test' must be explicitly assigned if l == 'test': levels.remove(l) - # go around again to check for override at the new level + # go around again to check at the new level continue else: # if version is explicitly marked test, it can't be @@ -714,7 +670,7 @@ def validate_packages(args, packages): l = 'test' if l not in packages[p].stability: - for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): + for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): if 'test' in packages[p].version_hints[v]: packages[p].stability[l] = v packages[p].version_hints[v][l] = '' @@ -722,10 +678,10 @@ def validate_packages(args, packages): # identify a 'best' version to take certain information from: this is # the curr version, if we have one, otherwise, the highest version. - if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap): + if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].versions()): packages[p].best_version = packages[p].stability['curr'] - elif len(packages[p].vermap): - packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0] + elif len(packages[p].versions()): + packages[p].best_version = sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True)[0] else: logging.error("package '%s' doesn't have any versions" % (p)) packages[p].best_version = None @@ -742,17 +698,21 @@ def validate_packages(args, packages): logging.warning("package '%s' doesn't have a curr version" % (p)) # error if the curr: version isn't the most recent non-test: version - for v in sorted(packages[p].vermap.keys(), key=lambda v: packages[p].vermap[v]['mtime'], reverse=True): + mtimes = {} + for vr in packages[p].versions(): + mtimes[vr] = packages[p].tar(vr).mtime + + for v in sorted(packages[p].versions(), key=lambda v: mtimes[v], reverse=True): if 'test' in packages[p].version_hints[v]: continue cv = packages[p].stability['curr'] - if cv not in packages[p].vermap: + if cv not in packages[p].versions(): continue if cv != v: - if packages[p].vermap[v]['mtime'] == packages[p].vermap[cv]['mtime']: + if mtimes[v] == mtimes[cv]: # don't consider an equal mtime to be more recent continue @@ -784,47 +744,45 @@ def validate_packages(args, packages): # If the install tarball is empty, we should probably either be marked # obsolete (if we have no dependencies) or virtual (if we do) if packages[p].kind == Kind.binary and not packages[p].skip: - for vr in packages[p].vermap: - if 'install' in packages[p].vermap[vr]: - if packages[p].tar(vr, 'install').is_empty: - # this classification relies on obsoleting packages - # not being present in depends - if packages[p].version_hints[vr].get('depends', ''): - # also allow '_obsolete' because old obsoletion - # packages depend on their replacement, but are not - # obsoleted by it - expected_categories = ['virtual', '_obsolete'] - else: - expected_categories = ['_obsolete'] - - if all(c not in packages[p].version_hints[vr].get('category', '').lower() for c in expected_categories): - if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or - ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))): - lvl = logging.DEBUG - else: - lvl = logging.ERROR - error = True - logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in %s category" % (p, vr, expected_categories)) - # If the source tarball is empty, that can't be right! - elif packages[p].kind == Kind.source: - for vr in packages[p].vermap: - if 'source' in packages[p].vermap[vr]: - if packages[p].tar(vr, 'source').is_empty: - if ((vr in past_mistakes.empty_source.get(p, [])) and - '_obsolete' in packages[p].version_hints[vr].get('category', '')): + for vr in packages[p].versions(): + if packages[p].tar(vr).is_empty: + # this classification relies on obsoleting packages + # not being present in depends + if packages[p].version_hints[vr].get('depends', ''): + # also allow '_obsolete' because old obsoletion + # packages depend on their replacement, but are not + # obsoleted by it + expected_categories = ['virtual', '_obsolete'] + else: + expected_categories = ['_obsolete'] + + if all(c not in packages[p].version_hints[vr].get('category', '').lower() for c in expected_categories): + if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or + ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))): lvl = logging.DEBUG else: - error = True lvl = logging.ERROR - logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr)) + error = True + logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in %s category" % (p, vr, expected_categories)) + # If the source tarball is empty, that can't be right! + elif packages[p].kind == Kind.source: + for vr in packages[p].versions(): + if packages[p].tar(vr).is_empty: + if ((vr in past_mistakes.empty_source.get(p, [])) and + '_obsolete' in packages[p].version_hints[vr].get('category', '')): + lvl = logging.DEBUG + else: + error = True + lvl = logging.ERROR + logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr)) # make another pass to verify a source tarfile exists for every install # tarfile version for p in sorted(packages.keys()): - for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): - if 'install' not in packages[p].vermap[v]: - continue + if not packages[p].kind == Kind.binary: + continue + for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): sourceless = False missing_source = True @@ -837,14 +795,14 @@ def validate_packages(args, packages): # mark the source tarfile as being used by an install tarfile if es_p in packages: - if v in packages[es_p].vermap and 'source' in packages[es_p].vermap[v]: - packages[es_p].tar(v, 'source').is_used = True + if v in packages[es_p].versions(): + packages[es_p].tar(v).is_used = True packages[es_p].is_used_by.add(p) missing_source = False if missing_source: # unless the install tarfile is empty - if packages[p].tar(v, 'install').is_empty: + if packages[p].tar(v).is_empty: sourceless = True missing_source = False @@ -854,7 +812,7 @@ def validate_packages(args, packages): missing_source = False # ... it's an error for this package to be missing source - packages[p].tar(v, 'install').sourceless = sourceless + packages[p].tar(v).sourceless = sourceless if missing_source: logging.error("package '%s' version '%s' is missing source" % (p, v)) error = True @@ -862,17 +820,17 @@ def validate_packages(args, packages): # make another pass to verify that each non-empty source tarfile version has # at least one corresponding non-empty install tarfile, in some package. for p in sorted(packages.keys()): - for v in sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): - if 'source' not in packages[p].vermap[v]: + for v in sorted(packages[p].versions(), key=lambda v: SetupVersion(v), reverse=True): + if not packages[p].kind == Kind.source: continue - if packages[p].tar(v, 'source').is_empty: + if packages[p].tar(v).is_empty: continue if '_obsolete' in packages[p].version_hints[v].get('category', ''): continue - if not packages[p].tar(v, 'source').is_used: + if not packages[p].tar(v).is_used: logging.error("package '%s' version '%s' source has no non-empty install tarfiles" % (p, v)) error = True @@ -1073,10 +1031,10 @@ def write_setup_ini(args, packages, arch): # (to maintain historical behaviour, include versions which only # exist as a source package) # - versions = set(po.vermap.keys()) + versions = set(po.versions()) sibling_src = pn + '-src' if sibling_src in packages: - versions.update(packages[sibling_src].vermap.keys()) + versions.update(packages[sibling_src].versions()) for version in sorted(versions, key=lambda v: SetupVersion(v), reverse=True): # skip over versions assigned to stability level: 'curr' has @@ -1117,9 +1075,9 @@ def write_setup_ini(args, packages, arch): print("version: %s" % version, file=f) is_empty = False - if 'install' in po.vermap.get(version, {}): + if version in po.versions(): tar_line(po, 'install', version, f) - is_empty = po.tar(version, 'install').is_empty + is_empty = po.tar(version).is_empty hints = po.version_hints.get(version, {}) @@ -1133,7 +1091,7 @@ def write_setup_ini(args, packages, arch): # external-source points to a source file in another package if s: - if 'source' in packages[s].vermap.get(version, {}): + if version in packages[s].versions(): tar_line(packages[s], 'source', version, f) else: if not (is_empty or packages[s].orig_name in past_mistakes.self_source): @@ -1173,7 +1131,7 @@ def write_setup_ini(args, packages, arch): # helper function to output details for a particular tar file def tar_line(p, category, v, f): - to = p.tar(v, category) + to = p.tar(v) fn = os.path.join(to.path, to.fn) sha512 = to.sha512 size = to.size @@ -1266,21 +1224,22 @@ def merge(a, *l): c[p] = b[p] # else, if the package is both in a and b, we have to do a merge else: + # package must be of the same kind + if c[p].kind != b[p].kind: + logging.error("package '%s' is of more than one kind" % (p)) + return None + # package must exist at same relative path if c[p].pkgpath != b[p].pkgpath: logging.error("package '%s' is at paths %s and %s" % (p, c[p].pkgpath, b[p].pkgpath)) return None else: - for vr in b[p].tars: - if vr in c[p].tars: - for t in b[p].tars[vr]: - if t in c[p].tars[vr]: - logging.error("package '%s' has duplicate tarfile %s for version %s" % (p, t, vr)) - return None - else: - c[p].tars[vr][t] = b[p].tars[vr][t] + for vr in b[p].tarfiles: + if vr in c[p].tarfiles: + logging.error("package '%s' has duplicate tarfile for version %s" % (p, vr)) + return None else: - c[p].tars[vr] = b[p].tars[vr] + c[p].tarfiles[vr] = b[p].tarfiles[vr] # hints from b override hints from a, but warn if they have # changed @@ -1316,15 +1275,10 @@ def delete(packages, path, fn): for p in packages: if packages[p].pkgpath == pkgpath: - for vr in packages[p].tars: - for t in packages[p].tars[vr]: - if t == fn: - del packages[p].tars[vr][t] - break - - # if no packages remain for this vr, also remove from vermap - if not packages[p].tars[vr]: - packages[p].vermap.pop(vr, None) + for vr in packages[p].tarfiles: + if packages[p].tarfiles[vr].fn == fn: + del packages[p].tarfiles[vr] + break for h in packages[p].hints: if packages[p].hints[h].fn == fn: @@ -1333,7 +1287,7 @@ def delete(packages, path, fn): break # if nothing remains, also remove from package set - if not packages[p].vermap and not packages[p].hints: + if not packages[p].tarfiles and not packages[p].hints: ex_packages.append(p) # (modify package set outside of iteration over it) @@ -1367,8 +1321,8 @@ def is_in_package_list(ppath, plist): # def mark_package_fresh(packages, p, v): - if 'install' in packages[p].vermap[v]: - packages[p].tar(v, 'install').fresh = True + if packages[p].kind == Kind.binary: + packages[p].tar(v).fresh = True # ... mark any corresponding sibling or external-source package version as also fresh if 'external-source' in packages[p].version_hints[v]: @@ -1377,9 +1331,9 @@ def mark_package_fresh(packages, p, v): es_p = p + '-src' if es_p in packages: - if v in packages[es_p].vermap: - if 'source' in packages[es_p].vermap[v]: - packages[es_p].tar(v, 'source').fresh = True + if v in packages[es_p].versions(): + if packages[es_p].kind == Kind.source: + packages[es_p].tar(v).fresh = True # @@ -1396,7 +1350,7 @@ def stale_packages(packages): # mark any versions explicitly listed in the keep: override hint for v in po.override_hints.get('keep', '').split(): - if v in po.vermap: + if v in po.versions(): mark_package_fresh(packages, pn, v) else: logging.error("package '%s' has non-existent keep: version '%s'" % (pn, v)) @@ -1404,7 +1358,7 @@ def stale_packages(packages): # mark as fresh the highest n non-test versions, where n is given by the # keep-count: override hint, (defaulting to DEFAULT_KEEP_COUNT) keep_count = int(po.override_hints.get('keep-count', common_constants.DEFAULT_KEEP_COUNT)) - for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): + for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True): if 'test' not in po.version_hints[v]: if keep_count <= 0: break @@ -1414,7 +1368,7 @@ def stale_packages(packages): # mark as fresh the highest n test versions, where n is given by the # keep-count-test: override hint, (defaulting to DEFAULT_KEEP_COUNT_TEST) keep_count = int(po.override_hints.get('keep-count-test', common_constants.DEFAULT_KEEP_COUNT_TEST)) - for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v), reverse=True): + for v in sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True): if 'test' in po.version_hints[v]: if keep_count <= 0: break @@ -1427,11 +1381,10 @@ def stale_packages(packages): # it is included) keep_days = po.override_hints.get('keep-days', common_constants.DEFAULT_KEEP_DAYS) newer = False - for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v)): + for v in sorted(po.versions(), key=lambda v: SetupVersion(v)): if not newer: - if 'install' in po.vermap[v]: - if po.tar(v, 'install').mtime > (time.time() - (keep_days * 24 * 60 * 60)): - newer = True + if po.tar(v).mtime > (time.time() - (keep_days * 24 * 60 * 60)): + newer = True if newer: mark_package_fresh(packages, pn, v) @@ -1441,16 +1394,14 @@ def stale_packages(packages): for pn, po in packages.items(): all_stale = {} - for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v)): + for v in sorted(po.versions(), key=lambda v: SetupVersion(v)): all_stale[v] = True - for category in ['source', 'install']: - if category in po.vermap[v]: - if not getattr(po.tar(v, category), 'fresh', False): - to = po.tar(v, category) - stale.add(to.path, to.fn) - logging.debug("package '%s' version '%s' %s is stale" % (pn, v, category)) - else: - all_stale[v] = False + if not getattr(po.tar(v), 'fresh', False): + to = po.tar(v) + stale.add(to.path, to.fn) + logging.debug("package '%s' version '%s' is stale" % (pn, v)) + else: + all_stale[v] = False for v in po.hints: # if there's a pvr.hint without a fresh source or install of the @@ -1461,12 +1412,11 @@ def stale_packages(packages): logging.debug("package '%s' version '%s' hint is stale" % (pn, v)) # clean up freshness mark - for v in po.vermap: - for c in ['source', 'install']: - try: - delattr(po.tar(v, c), 'fresh') - except (KeyError, AttributeError): - pass + for v in po.versions(): + try: + delattr(po.tar(v), 'fresh') + except AttributeError: + pass return stale diff --git a/calm/pkg2html.py b/calm/pkg2html.py index ffd16b0..80bc393 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -42,7 +42,6 @@ import argparse import glob import html -import itertools import logging import math import os @@ -266,19 +265,20 @@ def update_package_listings(args, packages): print('', file=f) print('', file=f) - def tar_line(pn, p, category, v, arch, f): - if category not in p.vermap[v]: - return - size = int(math.ceil(p.tar(v, category).size / 1024)) - name = v if category == 'install' else v + ' (source)' - target = "%s-%s" % (p.orig_name, v) + ('' if category == 'install' else '-src') + def tar_line(pn, p, v, arch, f): + size = int(math.ceil(p.tar(v).size / 1024)) + if p.kind == package.Kind.binary: + name = v + target = "%s-%s" % (p.orig_name, v) + else: + name = v + ' (source)' + target = "%s-%s-src" % (p.orig_name, v) test = 'test' if 'test' in p.version_hints[v] else 'stable' - ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v, category).mtime)) + ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v).mtime)) print('' % (name, size, ts, arch, pn, target, test), file=f) - for version in sorted(packages[arch][p].vermap.keys(), key=lambda v: SetupVersion(v)): - tar_line(p, packages[arch][p], 'install', version, arch, f) - tar_line(p, packages[arch][p], 'source', version, arch, f) + for version in sorted(packages[arch][p].versions(), key=lambda v: SetupVersion(v)): + tar_line(p, packages[arch][p], version, arch, f) print('
VersionPackage SizeDateFilesStatus
%s%d KiB%s[list of files]%s

', file=f) print('', file=f) @@ -444,7 +444,8 @@ def write_arch_listing(args, packages, arch): else: listings = [] - for tn, to in itertools.chain.from_iterable([packages[p].tars[vr].items() for vr in packages[p].tars]): + for to in packages[p].tarfiles.values(): + tn = to.fn fver = re.sub(r'\.tar.*$', '', tn) listing = os.path.join(dirpath, fver) diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected index e5cbbbf..3b61113 100644 --- a/test/testdata/uploads/pkglist.expected +++ b/test/testdata/uploads/pkglist.expected @@ -1,4 +1,4 @@ -{'testpackage': Package('testpackage', {'1.0-1': {'testpackage-1.0-1.tar.bz2': Tar('testpackage-1.0-1.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test package"', +{'testpackage': Package('testpackage', {'1.0-1': Tar('testpackage-1.0-1.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test package"', 'ldesc': '"A test package\n' "It's description might contains some unicode junk\n" 'Like it’s you’re Markup Language™ Nokogiri’s tool―that ' @@ -6,7 +6,7 @@ 'category': 'Devel', 'requires': 'cygwin', 'depends': 'cygwin'}}, {}, False), - 'testpackage-src': Package('testpackage', {'1.0-1': {'testpackage-1.0-1-src.tar.bz2': Tar('testpackage-1.0-1-src.tar.bz2', 'x86/release/testpackage', 'acfd77df3347e6432ccf29c12989964bc680a158d574f85dfa7ef222759f411006c7bd2773e37c5abdee628bea769b2da9aae213db615cd91402fd385373933d', 266, False)}}, {'1.0-1': {'sdesc': '"A test package"', + 'testpackage-src': Package('testpackage', {'1.0-1': Tar('testpackage-1.0-1-src.tar.bz2', 'x86/release/testpackage', 'acfd77df3347e6432ccf29c12989964bc680a158d574f85dfa7ef222759f411006c7bd2773e37c5abdee628bea769b2da9aae213db615cd91402fd385373933d', 266, False)}, {'1.0-1': {'sdesc': '"A test package"', 'ldesc': '"A test package\n' "It's description might contains some unicode junk\n" 'Like it’s you’re Markup Language™ Nokogiri’s tool―that ' @@ -15,20 +15,20 @@ 'requires': 'cygwin', 'homepage': 'http://homepage.url', 'depends': 'cygwin'}}, {}, True), - 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': {'testpackage-subpackage-1.0-1.tar.bz2': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test subpackage"', + 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', 'external-source': 'testpackage-src'}}, {}, False), - 'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1.tar.zst': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}}, {'1.0-1': {'category': 'Base', + 'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}, {'1.0-1': {'category': 'Base', 'requires': '', 'sdesc': '"test package (zstd compressed)"', 'ldesc': '"test package (zstd compressed)"', 'depends': ''}}, {}, False), - 'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1-src.tar.zst': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}}, {'1.0-1': {'category': 'Base', + 'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}, {'1.0-1': {'category': 'Base', 'build-depends': 'cygport', 'sdesc': '"test package (zstd compressed)"', 'ldesc': '"test package (zstd compressed)"', 'skip': ''}}, {}, True), - 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}}, {'1.0-1': {'sdesc': '"A test subpackage 2"', + 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"', 'ldesc': '"A test subpackage 2"', 'category': 'Devel'}}, {}, False)}