https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=50d26eb220847bc845893ce5a4a760ac6ff5ee02 commit 50d26eb220847bc845893ce5a4a760ac6ff5ee02 Author: Jon Turney Date: Mon Apr 18 12:17:40 2016 +0100 Ignore in-progress sftp uploads https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=254e1c2ac096c93141014998f93c1ddb77e21078 commit 254e1c2ac096c93141014998f93c1ddb77e21078 Author: Jon Turney Date: Mon Apr 4 10:21:03 2016 +0100 Add testpackage/setup.hint testpackage is supposedly a valid package, but doesn't have a setup.hint update tests appropriately add altering the setup.hint to upload test https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b0490cd0a333ff3a65ebef0b61a1a12bc4235cbf commit b0490cd0a333ff3a65ebef0b61a1a12bc4235cbf Author: Jon Turney Date: Fri Apr 15 12:20:43 2016 +0100 Add an option to mksetupini to disable checking that required packages exist Add an --okmissing option to mksetupini, modelled after genini Add the okmissing token require-package, which disables checking that required packages exist This makes mksetupini a little more generally useful https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=42eb702159aa211dfcc5d5f9610836d026f1e782 commit 42eb702159aa211dfcc5d5f9610836d026f1e782 Author: Jon Turney Date: Fri Apr 15 12:22:47 2016 +0100 Make --pkglist optional in mksetupini Make --pkglist optional in mksetupini, and don't check package names against the package list it is used. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bf9e0b0c476b67646236a1111bdeeaecaf6f9d7a commit bf9e0b0c476b67646236a1111bdeeaecaf6f9d7a Author: Jon Turney Date: Thu Apr 14 18:18:22 2016 +0100 Ignore any dotfiles in a package directory https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8e947925c1b2597d11e1d000e01b57a9a8981f79 commit 8e947925c1b2597d11e1d000e01b57a9a8981f79 Author: Jon Turney Date: Thu Apr 14 17:53:39 2016 +0100 Downgrade warning about replacing an existing setup.hint Saying 'replacing' confuses people into thinking that we are moving the file now, which isn't the case. We'll warn about any differences in setup.hint before we do that move, which is more informative, anyhow Diff: --- mksetupini | 4 ++- package.py | 31 +++++++++++++------ .../x86/release/testpackage/setup.hint | 6 ++- testdata/htdocs.expected/x86/packages.inc | 1 + testdata/htdocs.expected/x86/testpackage/.htaccess | 3 ++ .../x86/testpackage/testpackage-0.1-1 | 5 +++ testdata/inifile/setup.ini.expected | 10 ++++++- testdata/process_arch/homedir.expected | 3 +- testdata/process_arch/rel_area.expected | 3 +- testdata/uploads/pkglist.expected | 7 +++- testdata/x86.hints/release/testpackage/expected | 1 + testdata/x86/release/testpackage/setup.hint | 3 ++ uploads.py | 9 +++++- 13 files changed, 67 insertions(+), 19 deletions(-) diff --git a/mksetupini b/mksetupini index 41d81bf..4c3a3d5 100755 --- a/mksetupini +++ b/mksetupini @@ -25,6 +25,7 @@ # mksetupini # # Make a setup.ini file from a collection of tarfiles and setup.hints +# (this is intended to be a replacement for genini) # import argparse @@ -98,7 +99,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Make setup.ini') parser.add_argument('--arch', action='store', required=True, choices=common_constants.ARCHES) parser.add_argument('--inifile', '-u', action='store', help='output filename', required=True) - parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) + parser.add_argument('--okmissing', action='append', help='missing things are ok', choices=['required-package']) + parser.add_argument('--pkglist', action='store', nargs='?', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", const=pkglist_default) parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') parser.add_argument('--spell', action='store_true', help='spellcheck text hints') diff --git a/package.py b/package.py index 7640429..e04c06b 100755 --- a/package.py +++ b/package.py @@ -185,6 +185,11 @@ def read_package(packages, basedir, dirpath, files, strict=False): tars[f].sha512 = sha512_file(os.path.join(dirpath, f)) logging.debug("no sha512.sum line for file %s in package '%s', computed sha512 hash is %s" % (f, p, tars[f].sha512)) + # ignore dotfiles + for f in files: + if f.startswith('.'): + files.remove(f) + # warn about unexpected files, including tarfiles which don't match the # package name if files: @@ -269,17 +274,20 @@ def validate_packages(args, packages): error = False for p in sorted(packages.keys()): - # all packages listed in requires must exist - if 'requires' in packages[p].hints: - for r in packages[p].hints['requires'].split(): - if r not in packages: - logging.error("package '%s' requires nonexistent package '%s'" % (p, r)) - error = True + logging.debug("validating package '%s'" % (p)) - # a package is should not appear in it's own requires - if r == p: - lvl = logging.WARNING if p not in past_mistakes.self_requires else logging.INFO - logging.log(lvl, "package '%s' requires itself" % (p)) + if 'required-package' not in getattr(args, 'okmissing', []): + # all packages listed in requires must exist + if 'requires' in packages[p].hints: + for r in packages[p].hints['requires'].split(): + if r not in packages: + logging.error("package '%s' requires nonexistent package '%s'" % (p, r)) + error = True + + # a package is should not appear in it's own requires + if r == p: + lvl = logging.WARNING if p not in past_mistakes.self_requires else logging.INFO + logging.log(lvl, "package '%s' requires itself" % (p)) # if external-source is used, the package must exist if 'external-source' in packages[p].hints: @@ -458,6 +466,9 @@ def validate_packages(args, packages): # def validate_package_maintainers(args, packages): + if not args.pkglist: + return + # read maintainer list mlist = {} mlist = maintainers.Maintainer.add_packages(mlist, args.pkglist) diff --git a/testdata/homes/Blooey McFooey/x86/release/testpackage/setup.hint b/testdata/homes/Blooey McFooey/x86/release/testpackage/setup.hint index 7cf0247..290111b 100644 --- a/testdata/homes/Blooey McFooey/x86/release/testpackage/setup.hint +++ b/testdata/homes/Blooey McFooey/x86/release/testpackage/setup.hint @@ -1,4 +1,6 @@ sdesc: "A test package" -ldesc: "A test package" +ldesc: "A test package +It's description might contains some unicode gibberish +Like it’s you’re Markup Language™ Nokogiri’s tool―that Bézier." category: Devel - +requires: cygwin diff --git a/testdata/homes/Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage/inprogress.SftpXFR.1234 b/testdata/homes/Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage/inprogress.SftpXFR.1234 new file mode 100644 index 0000000..e69de29 diff --git a/testdata/htdocs.expected/x86/packages.inc b/testdata/htdocs.expected/x86/packages.inc index cd5e235..8f95160 100755 --- a/testdata/htdocs.expected/x86/packages.inc +++ b/testdata/htdocs.expected/x86/packages.inc @@ -17,5 +17,6 @@ mDNSResponderBonjour Zeroconf implementation opensshThe OpenSSH server and client programs rpm-docObsolete package for RPM package management system manual pages +testpackageA test package diff --git a/testdata/htdocs.expected/x86/testpackage/.htaccess b/testdata/htdocs.expected/x86/testpackage/.htaccess new file mode 100644 index 0000000..3196d64 --- /dev/null +++ b/testdata/htdocs.expected/x86/testpackage/.htaccess @@ -0,0 +1,3 @@ +Options Indexes +IndexOptions -FancyIndexing +AddType text/html 1 2 3 4 5 6 7 8 9 diff --git a/testdata/htdocs.expected/x86/testpackage/testpackage-0.1-1 b/testdata/htdocs.expected/x86/testpackage/testpackage-0.1-1 new file mode 100644 index 0000000..76208aa --- /dev/null +++ b/testdata/htdocs.expected/x86/testpackage/testpackage-0.1-1 @@ -0,0 +1,5 @@ + +

testpackage: A test package (installed binaries and support files)

+
+
 + diff --git a/testdata/inifile/setup.ini.expected b/testdata/inifile/setup.ini.expected index 6b241ef..a69302f 100644 --- a/testdata/inifile/setup.ini.expected +++ b/testdata/inifile/setup.ini.expected @@ -193,4 +193,12 @@ 'install: x86/release/rpm-doc/rpm-doc-4.1-2.tar.bz2 50941 ' '7cc9db802364252e3206ce9f75c8ca53813d8308a22a425b50ef695dd8e51568740b06739d3aa3399a83fb3d3e1345ab7e2ad03a1e9d47c02dded3363bf4f493\n' 'source: x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 42 ' - '28c70b843fe01d90a3eeab4a3617551d236cd0b7d69668d1b1b6c8b14a9fd050e4039c192894c93bdf31575771c58c1fea2a41c24c8da22d10080d8b032b6369\n',) + '28c70b843fe01d90a3eeab4a3617551d236cd0b7d69668d1b1b6c8b14a9fd050e4039c192894c93bdf31575771c58c1fea2a41c24c8da22d10080d8b032b6369\n' + '\n' + '@ testpackage\n' + 'sdesc: "A test package"\n' + 'ldesc: "A test package"\n' + 'category: Devel\n' + 'version: 0.1-1\n' + 'install: x86/release/testpackage/testpackage-0.1-1.tar.bz2 0 ' + 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e\n',) diff --git a/testdata/process_arch/homedir.expected b/testdata/process_arch/homedir.expected index 091867f..8dcf175 100644 --- a/testdata/process_arch/homedir.expected +++ b/testdata/process_arch/homedir.expected @@ -9,6 +9,7 @@ 'Blooey McFooey/x86/release/testpackage': [], 'Blooey McFooey/x86/release/testpackage/testpackage-subpackage': [], 'Blooey McFooey/x86/release/testpackage2': ['setup.hint', 'testpackage2-1.0-1.tar.bz2'], - 'Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage': ['setup.hint', + 'Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage': ['inprogress.SftpXFR.1234', + 'setup.hint', 'testpackage2-subpackage-1.0-1.tar.bz2'], 'Jon Turney': ['!email']} diff --git a/testdata/process_arch/rel_area.expected b/testdata/process_arch/rel_area.expected index 1770b4b..a0f8d1d 100644 --- a/testdata/process_arch/rel_area.expected +++ b/testdata/process_arch/rel_area.expected @@ -3,7 +3,8 @@ 'x86/release': ['.gitignore', 'sha512.sum'], 'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.tar.bz2', 'setup.hint', 'sha512.sum'], 'x86/release/base-cygwin': ['base-cygwin-3.6-1.tar.xz', 'base-cygwin-3.8-1.tar.xz', 'setup.hint', 'sha512.sum'], - 'x86/release/cygwin': ['cygwin-2.2.0-1-src.tar.xz', + 'x86/release/cygwin': ['.this-should-be-ignored', + 'cygwin-2.2.0-1-src.tar.xz', 'cygwin-2.2.0-1.tar.xz', 'cygwin-2.2.1-1-src.tar.xz', 'cygwin-2.2.1-1.tar.xz', diff --git a/testdata/uploads/pkglist.expected b/testdata/uploads/pkglist.expected index 8caa9aa..f98adb3 100644 --- a/testdata/uploads/pkglist.expected +++ b/testdata/uploads/pkglist.expected @@ -1,7 +1,10 @@ {'testpackage': Package('release/testpackage', {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False), 'testpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test package"', - 'ldesc': '"A test package"', - 'category': 'Devel'}), + 'ldesc': '"A test package\n' + "It's description might contains some unicode gibberish\n" + 'Like it’s you’re Markup Language™ Nokogiri’s tool―that Bézier."', + 'category': 'Devel', + 'requires': 'cygwin'}), 'testpackage-subpackage': Package('release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', diff --git a/testdata/x86.hints/release/testpackage/expected b/testdata/x86.hints/release/testpackage/expected new file mode 100644 index 0000000..de11656 --- /dev/null +++ b/testdata/x86.hints/release/testpackage/expected @@ -0,0 +1 @@ +OrderedDict([('sdesc', '"A test package"'), ('ldesc', '"A test package"'), ('category', 'Devel')]) diff --git a/testdata/x86/release/cygwin/.this-should-be-ignored b/testdata/x86/release/cygwin/.this-should-be-ignored new file mode 100644 index 0000000..e69de29 diff --git a/testdata/x86/release/testpackage/setup.hint b/testdata/x86/release/testpackage/setup.hint new file mode 100644 index 0000000..10ee390 --- /dev/null +++ b/testdata/x86/release/testpackage/setup.hint @@ -0,0 +1,3 @@ +sdesc: "A test package" +ldesc: "A test package" +category: Devel diff --git a/uploads.py b/uploads.py index 6909d76..f53a67c 100644 --- a/uploads.py +++ b/uploads.py @@ -127,6 +127,13 @@ def scan(m, all_packages, args): files.remove(f) continue + # ignore in-progress sftp uploads. Net::SFTP::SftpServer uses + # temporary upload filenames ending with '.SftpXFR.' + if re.search(r'\.SftpXFR\.\d*$', f): + logging.debug("ignoring temporary upload file %s" % fn) + files.remove(f) + continue + # only process files newer than !ready if os.path.getmtime(fn) > mtime: if mtime == 0: @@ -164,7 +171,7 @@ def scan(m, all_packages, args): if filecmp.cmp(dest, fn, shallow=False): logging.debug("identical %s is already in release area" % fn) else: - logging.warning("replacing, different %s is already in release area" % fn) + logging.debug("different %s is already in release area" % fn) # we always consider setup.hint, as we can't have a valid package without it move[relpath].append(f) else: