public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210408-19-ga5cf1fb
@ 2021-05-24 20:37 Jon TURNEY
0 siblings, 0 replies; only message in thread
From: Jon TURNEY @ 2021-05-24 20:37 UTC (permalink / raw)
To: cygwin-apps-cvs
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a5cf1fbb3bc0b4bf5ce602f03eb2c0c4b8a94774
commit a5cf1fbb3bc0b4bf5ce602f03eb2c0c4b8a94774
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue May 11 16:08:09 2021 +0100
Ignore 'skip' hint
Ignore the 'skip' hint, as it's meaningless, now we properly classify
packages as source or install.
Instead, if installing a non-obsolete package will do nothing, mark it
as not_for_output.
Future work: also allow the 'does nothing' check to apply to '_obsolete'
packages. At the moment this would catch a few more packages which
probably need to be obsoleted: by something before they can be safely
removed.
(It's not safe to simply vault them as we need to ensure any existing
installed files owned by that package are removed when a system with
them installed is updated.)
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=aece4eb9b30d107c1f30aed6e86f170c943c16a0
commit aece4eb9b30d107c1f30aed6e86f170c943c16a0
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Wed May 12 22:48:29 2021 +0100
Add a tool to fix hint files with invalid hints
Add a tool to fix .hint files containing keys which are no longer valid
for that type of package (source or install).
Also fix invalid keys in -src.hint created at upload.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0f66095073bb39eb8a2f28c41d65ddfa76e43f22
commit 0f66095073bb39eb8a2f28c41d65ddfa76e43f22
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu May 20 15:48:25 2021 +0100
Move 'packaging repository' up page to be with the rest of the details
Diff:
---
calm/fix-invalid-key-hint.py | 81 ++++++++++++++++++++++++++++++++++
calm/fix-missing-src-hint-info.py | 2 +-
calm/fixes.py | 58 +++++++++++++++++++-----
calm/hint.py | 39 ++++++++--------
calm/package.py | 52 +++++++---------------
calm/pkg2html.py | 17 ++++---
calm/uploads.py | 4 +-
test/testdata/uploads/pkglist.expected | 4 +-
8 files changed, 178 insertions(+), 79 deletions(-)
diff --git a/calm/fix-invalid-key-hint.py b/calm/fix-invalid-key-hint.py
new file mode 100644
index 0000000..5f3340c
--- /dev/null
+++ b/calm/fix-invalid-key-hint.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import logging
+import os
+import sys
+
+from . import common_constants
+from . import fixes
+
+
+#
+#
+#
+
+def fix_hints(relarea, packages):
+ for (dirpath, _subdirs, files) in os.walk(relarea):
+
+ # only apply to listed packages, if specified
+ if packages:
+ relpath = os.path.relpath(dirpath, relarea)
+ relpath = relpath.split(os.path.sep)
+ if (len(relpath) < 3) or (relpath[2] not in packages):
+ continue
+
+ for f in files:
+ if f.endswith('.hint') and f != 'override.hint':
+ if fixes.fix_hint(dirpath, f, '', ['invalid_keys']):
+ logging.warning('fixed hints %s' % f)
+
+
+#
+#
+#
+
+def main():
+ relarea_default = common_constants.FTP
+
+ parser = argparse.ArgumentParser(description='fix invalid keys in hint')
+
+ parser.add_argument('package', nargs='*', metavar='PACKAGE')
+ parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output', default=0)
+ parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='relarea')
+ # XXX: should take an argument listing fixes to apply
+
+ (args) = parser.parse_args()
+ if args.verbose:
+ logging.getLogger().setLevel(logging.INFO)
+
+ logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s')
+
+ fix_hints(args.relarea, args.package)
+
+
+#
+#
+#
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/calm/fix-missing-src-hint-info.py b/calm/fix-missing-src-hint-info.py
index c1c9d28..88142a6 100644
--- a/calm/fix-missing-src-hint-info.py
+++ b/calm/fix-missing-src-hint-info.py
@@ -50,7 +50,7 @@ def fix_hints(relarea, packages):
logging.error('hint %s missing' % hf)
continue
- fixes.fix_homepage_src_hint(dirpath, hf, f)
+ fixes.fix_hint(dirpath, hf, f, ['homepage'])
#
diff --git a/calm/fixes.py b/calm/fixes.py
index e7a9839..613e361 100644
--- a/calm/fixes.py
+++ b/calm/fixes.py
@@ -84,16 +84,7 @@ def follow_redirect(homepage):
return homepage
-def fix_homepage_src_hint(dirpath, hf, tf):
- pn = os.path.basename(dirpath)
- hintfile = os.path.join(dirpath, hf)
- hints = hint.hint_file_parse(hintfile, hint.spvr)
-
- hints.pop('parse-warnings', None)
- if 'parse-errors' in hints:
- logging.error('invalid hints %s' % hf)
- return
-
+def _fix_homepage_src_hint(hints, dirpath, _hf, tf):
# already present?
if 'homepage' in hints:
homepage = hints['homepage']
@@ -109,6 +100,7 @@ def fix_homepage_src_hint(dirpath, hf, tf):
if match:
if homepage:
logging.warning('multiple HOMEPAGE lines in .cygport in srcpkg %s', tf)
+ pn = os.path.basename(dirpath)
homepage = match.group(2)
homepage = re.sub(r'\$({|)(PN|ORIG_PN|NAME)(}|)', pn, homepage)
@@ -118,10 +110,11 @@ def fix_homepage_src_hint(dirpath, hf, tf):
if not homepage:
logging.info('cannot determine homepage: from srcpkg %s' % tf)
- return
+ return False
logging.info('adding homepage:%s to hints for srcpkg %s' % (homepage, tf))
+ # check if redirect?
redirect_homepage = follow_redirect(homepage)
# trivial URL transformations aren't interesting
@@ -138,8 +131,49 @@ def fix_homepage_src_hint(dirpath, hf, tf):
if not redirect_homepage.startswith(homepage):
logging.warning('homepage:%s permanently redirects to %s' % (homepage, redirect_homepage))
- # write updated hints
+ # changed?
if homepage != hints.get('homepage', None):
hints['homepage'] = homepage
+ return True
+
+ return False
+
+
+def _fix_invalid_keys_hint(hints, _dirpath, hf, _tf):
+ # eliminate keys that aren't appropriate to the package type
+ if hf.endswith('-src.hint'):
+ valid_keys = hint.hintkeys[hint.spvr]
+ else:
+ valid_keys = hint.hintkeys[hint.pvr]
+
+ changed = False
+ for k in list(hints.keys()):
+ if k not in valid_keys:
+ logging.debug("discarding invalid key '%s:' from hint '%s'" % (k, hf))
+ hints.pop(k, None)
+ changed = True
+
+ return changed
+
+
+def fix_hint(dirpath, hf, tf, problems):
+ hintfile = os.path.join(dirpath, hf)
+ hints = hint.hint_file_parse(hintfile, None)
+
+ hints.pop('parse-warnings', None)
+ if 'parse-errors' in hints:
+ logging.error('invalid hints %s' % hf)
+ return
+
+ changed = False
+ if 'homepage' in problems:
+ changed = _fix_homepage_src_hint(hints, dirpath, hf, tf)
+ if 'invalid_keys' in problems:
+ changed = _fix_invalid_keys_hint(hints, dirpath, hf, tf) or changed
+
+ # write updated hints
+ if changed:
shutil.copy2(hintfile, hintfile + '.bak')
hint.hint_file_write(hintfile, hints)
+
+ return changed
diff --git a/calm/hint.py b/calm/hint.py
index 2c31714..6cabea4 100755
--- a/calm/hint.py
+++ b/calm/hint.py
@@ -45,7 +45,6 @@ commonkeys = {
'ldesc': 'multilineval',
'category': 'val',
'sdesc': 'val',
- 'skip': 'noval',
'requires': 'optval',
'obsoletes': 'optval',
'test': 'noval', # mark the package as a test version
@@ -65,6 +64,7 @@ hintkeys[pvr].update({
hintkeys[spvr] = commonkeys.copy()
hintkeys[spvr].update({
+ 'skip': 'noval', # in all spvr hints, but ignored
'homepage': 'val',
'build-depends': 'optval',
})
@@ -197,7 +197,7 @@ def hint_file_parse(fn, kind):
errors = []
warnings = []
- assert(kind in hintkeys)
+ assert((kind in hintkeys) or (kind is None))
with open(fn, 'rb') as f:
c = f.read()
@@ -221,21 +221,26 @@ def hint_file_parse(fn, kind):
key = match.group(1)
value = match.group(2)
- if key not in hintkeys[kind]:
- errors.append('unknown key %s at line %d' % (key, i))
- continue
- valtype = hintkeys[kind][key]
+ if kind is not None:
+ if key not in hintkeys[kind]:
+ errors.append('unknown key %s at line %d' % (key, i))
+ continue
+ valtype = hintkeys[kind][key]
- # check if the key occurs more than once
- if key in hints:
- errors.append('duplicate key %s' % (key))
+ # check if the key occurs more than once
+ if key in hints:
+ errors.append('duplicate key %s' % (key))
- # check the value meets any key-specific constraints
- if (valtype == 'val') and (len(value) == 0):
- errors.append('%s has empty value' % (key))
+ # check the value meets any key-specific constraints
+ if (valtype == 'val') and (len(value) == 0):
+ errors.append('%s has empty value' % (key))
- if (valtype == 'noval') and (len(value) != 0):
- errors.append("%s has non-empty value '%s'" % (key, value))
+ if (valtype == 'noval') and (len(value) != 0):
+ errors.append("%s has non-empty value '%s'" % (key, value))
+
+ # only 'ldesc' and 'message' are allowed a multi-line value
+ if (valtype != 'multilineval') and (len(value.splitlines()) > 1):
+ errors.append("key %s has multi-line value" % (key))
# validate all categories are in the category list (case-insensitively)
if key == 'category':
@@ -266,10 +271,6 @@ def hint_file_parse(fn, kind):
warnings.append("sdesc contains ' '")
value = value.replace(' ', ' ')
- # only 'ldesc' and 'message' are allowed a multi-line value
- if (valtype != 'multilineval') and (len(value.splitlines()) > 1):
- errors.append("key %s has multi-line value" % (key))
-
# message must have an id and some text
if key == 'message':
if not re.match(r'(\S+)\s+(\S.*)', value):
@@ -289,7 +290,7 @@ def hint_file_parse(fn, kind):
# for the pvr kind, 'category' and 'sdesc' must be present
# XXX: genini also requires 'requires' but that seems wrong
- if kind != override:
+ if (kind == pvr) or (kind == spvr):
mandatory = ['category', 'sdesc']
for k in mandatory:
if k not in hints:
diff --git a/calm/package.py b/calm/package.py
index 1e5b0f1..75dfc4c 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -63,7 +63,7 @@ class Package(object):
self.is_used_by = set()
self.version_hints = {}
self.override_hints = {}
- self.skip = False
+ self.not_for_output = False
def __repr__(self):
return "Package('%s', %s, %s, %s, %s)" % (
@@ -71,7 +71,7 @@ class Package(object):
pprint.pformat(self.tarfiles),
pprint.pformat(self.version_hints),
pprint.pformat(self.override_hints),
- self.skip)
+ self.not_for_output)
def tar(self, vr):
return self.tarfiles[vr]
@@ -428,10 +428,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
packages[pn].tarfiles = actual_tars
packages[pn].hints = hints
packages[pn].pkgpath = pkgpath
- if kind == Kind.source:
- packages[pn].skip = True
- else:
- packages[pn].skip = any(['skip' in version_hints[vr] for vr in version_hints])
packages[pn].kind = kind
# since we are kind of inventing the source package names, and don't
# want to report them, keep track of the real name
@@ -546,9 +542,9 @@ def validate_packages(args, packages):
error = True
continue
- # hint referencing a source-only package makes no sense
- if r in packages and packages[r].skip:
- logging.error("package '%s' version '%s' %s source-only package '%s'" % (p, v, c, r))
+ # package relation hint referencing a source package makes no sense
+ if r in packages and packages[r].kind == Kind.source:
+ logging.error("package '%s' version '%s' %s source package '%s'" % (p, v, c, r))
error = True
# if external-source is used, the package must exist
@@ -583,37 +579,23 @@ def validate_packages(args, packages):
else:
logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p))
- has_install = False
has_nonempty_install = False
if packages[p].kind == Kind.binary:
for vr in packages[p].versions():
- has_install = True
if not packages[p].tar(vr).is_empty:
has_nonempty_install = True
obsolete = any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints])
- # if the package has no install tarfiles (i.e. is source only), make
- # sure it is marked as 'skip' (which really means 'source-only' at the
- # moment)
- #
# if the package has no non-empty install tarfiles, and no dependencies
# installing it will do nothing (and making it appear in the package
- # list is just confusing), so if it's not obsolete, mark it as 'skip'
- #
- # (this needs to take place after uploads have been merged into the
- # package set, so that an upload containing just a replacement
- # setup.hint is not considered a source-only package)
- #
- if not packages[p].skip:
- if not has_install:
- packages[p].skip = True
- logging.info("package '%s' appears to be source-only as it has no install tarfiles, marking as 'skip'" % (p))
-
- elif not has_nonempty_install and not has_requires and not obsolete:
- packages[p].skip = True
- logging.info("package '%s' appears to be source-only as it has no non-empty install tarfiles and no dependencies, marking as 'skip'" % (p))
+ # list is just confusing), so if it's not obsolete, mark it as
+ # 'not_for_output'
+ if packages[p].kind == Kind.binary:
+ if not has_nonempty_install and not has_requires and not obsolete:
+ packages[p].not_for_output = True
+ logging.info("package '%s' has no non-empty install tarfiles and no dependencies, marking as 'not_for_output'" % (p))
levels = ['test', 'curr', 'prev']
@@ -743,7 +725,7 @@ def validate_packages(args, packages):
# If the install tarball is empty, we should probably either be marked
# obsolete (if we have no dependencies) or virtual (if we do)
- if packages[p].kind == Kind.binary and not packages[p].skip:
+ if packages[p].kind == Kind.binary and not packages[p].not_for_output:
for vr in packages[p].versions():
if packages[p].tar(vr).is_empty:
# this classification relies on obsoleting packages
@@ -980,8 +962,11 @@ def write_setup_ini(args, packages, arch):
for pn in sorted(packages, key=sort_key):
po = packages[pn]
- # do nothing if 'skip'
- if po.skip:
+ if po.kind == Kind.source:
+ continue
+
+ # do nothing if not_for_output
+ if po.not_for_output:
continue
# write package data
@@ -1263,9 +1248,6 @@ def merge(a, *l):
# merge hint file lists
c[p].hints.update(b[p].hints)
- # skip if both a and b are skip
- c[p].skip = c[p].skip and b[p].skip
-
return c
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 80bc393..5113d36 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -256,6 +256,12 @@ def update_package_listings(args, packages):
<a href="/problems.html#personal-email">Do not contact the maintainer(s) directly</a>.)</span>'''), file=f)
print('<br><br>', file=f)
+ if po.kind == package.Kind.source:
+ repo = 'git/cygwin-packages/%s.git' % pn
+ if os.path.exists('/' + repo):
+ repo_browse_url = '/git-cygwin-packages/?p=%s' % repo
+ print('<span class="detail">packaging repository</span>: <a href="%s">%s.git</a>' % (repo_browse_url, pn), file=f)
+
print('<ul>', file=f)
for arch in sorted(packages):
if p in packages[arch]:
@@ -283,12 +289,6 @@ def update_package_listings(args, packages):
print('</table><br>', file=f)
print('</ul>', file=f)
- if po.kind == package.Kind.source:
- repo = 'git/cygwin-packages/%s.git' % pn
- if os.path.exists('/' + repo):
- repo_browse_url = '/git-cygwin-packages/?p=%s' % repo
- print('<span class="detail">packaging repository</span>: <a href="%s">%s.git</a>' % (repo_browse_url, pn), file=f)
-
print(textwrap.dedent('''\
</div>
</body>
@@ -329,9 +329,8 @@ def write_packages_inc(args, packages, name, kind, includer):
if p.endswith('-debuginfo'):
continue
- if packages[arch][p].kind == package.Kind.binary:
- if packages[arch][p].skip:
- continue
+ if packages[arch][p].not_for_output:
+ continue
if packages[arch][p].kind == kind:
package_list[packages[arch][p].orig_name] = p
diff --git a/calm/uploads.py b/calm/uploads.py
index 9998e46..961b397 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -193,8 +193,10 @@ def scan(m, all_packages, arch, args):
remove.append(os.path.join(dirpath, old))
# see if we can fix-up missing homepage: in -src.hint file
+ # check homepage: for liveliness and redirection
+ # discard any keys which are invalid in a -src.hint
if (new in files):
- fixes.fix_homepage_src_hint(dirpath, new, f)
+ fixes.fix_hint(dirpath, new, f, ['homepage', 'invalid_keys'])
# filter out files we don't need to consider
for f in sorted(files):
diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected
index 3b61113..fb3d59d 100644
--- a/test/testdata/uploads/pkglist.expected
+++ b/test/testdata/uploads/pkglist.expected
@@ -14,7 +14,7 @@
'category': 'Devel',
'requires': 'cygwin',
'homepage': 'http://homepage.url',
- 'depends': 'cygwin'}}, {}, True),
+ 'depends': 'cygwin'}}, {}, False),
'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"',
'ldesc': '"A test subpackage"',
'category': 'Devel',
@@ -28,7 +28,7 @@
'build-depends': 'cygport',
'sdesc': '"test package (zstd compressed)"',
'ldesc': '"test package (zstd compressed)"',
- 'skip': ''}}, {}, True),
+ 'skip': ''}}, {}, False),
'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"',
'ldesc': '"A test subpackage 2"',
'category': 'Devel'}}, {}, False)}
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2021-05-24 20:37 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-05-24 20:37 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210408-19-ga5cf1fb Jon TURNEY
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).