public inbox for cygwin-apps-cvs@sourceware.org help / color / mirror / Atom feed
From: Jon TURNEY <jturney@sourceware.org> To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-54-g39ee07e Date: Sat, 18 Jun 2022 11:24:09 +0000 (GMT) [thread overview] Message-ID: <20220618112409.4720A3853540@sourceware.org> (raw) https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=39ee07ed3713329bde426fd9efa53f1bed5df89e commit 39ee07ed3713329bde426fd9efa53f1bed5df89e Author: Jon Turney <jon.turney@dronecode.org.uk> Date: Sat Jun 11 15:25:25 2022 +0100 Use a separate staging dir for automated build uploads Look in a different directory for packages being deployed by scallywag. Future work: this will allow disabling the uploading of local builds (on a per-package basis), whilst still allowing automated build uploads. Update tests appropriately. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1a74d4efd3a4d56d8b42f605f2c4b48afeb2da57 commit 1a74d4efd3a4d56d8b42f605f2c4b48afeb2da57 Author: Jon Turney <jon.turney@dronecode.org.uk> Date: Thu Jun 9 16:20:14 2022 +0100 Remove empty subdirectories Remove empty subdirectories in release and upload areas. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=234c197298d83040db5ebd48628fbda7bd98668c commit 234c197298d83040db5ebd48628fbda7bd98668c Author: Jon Turney <jon.turney@dronecode.org.uk> Date: Thu Jun 9 22:02:13 2022 +0100 Add some more maintainer-less package anomalies Add some more maintainer-less package anomalies, revealed by clean up in cygwin-pkg-maint. Diff: --- calm/calm.py | 19 +++++++++++++++---- calm/common_constants.py | 3 +++ calm/movelist.py | 11 +++++++---- calm/past_mistakes.py | 2 ++ calm/uploads.py | 11 ++++++----- calm/utils.py | 16 ++++++++++++++++ test/test_calm.py | 16 ++++++++++------ test/testdata/conflict/stagingdir.expected | 1 + test/testdata/process_arch/homedir.expected | 2 -- test/testdata/process_arch/stagingdir.expected | 1 + 10 files changed, 61 insertions(+), 21 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 63cdc60..49bdcfd 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -126,6 +126,10 @@ def process_relarea(args, state): logging.error("error while evaluating stale packages") return None + # clean up any empty directories + if not args.dryrun: + utils.rmemptysubdirs(args.rel_area) + return packages @@ -146,7 +150,8 @@ def process_uploads(args, state): m = mlist[name] with logfilters.AttrFilter(maint=m.name): - process_maintainer_uploads(args, state, all_packages, m) + process_maintainer_uploads(args, state, all_packages, m, args.homedir, 'upload') + process_maintainer_uploads(args, state, all_packages, m, args.stagingdir, 'staging') # record updated reminder times for maintainers maintainers.update_reminder_times(mlist) @@ -154,7 +159,7 @@ def process_uploads(args, state): return state.packages -def process_maintainer_uploads(args, state, all_packages, m): +def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): name = m.name # for each arch and noarch @@ -164,7 +169,7 @@ def process_maintainer_uploads(args, state, all_packages, m): logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name)) # read uploads - scan_result[arch] = uploads.scan(m, all_packages, arch, args) + scan_result[arch] = uploads.scan(basedir, m, all_packages, arch, args) # remove triggers uploads.remove(args, scan_result[arch].remove_always) @@ -247,7 +252,7 @@ def process_maintainer_uploads(args, state, all_packages, m): uploads.remove(args, scan_result[arch].remove_success) if scan_result[arch].to_relarea: logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch)) - scan_result[arch].to_relarea.move_to_relarea(m, args) + scan_result[arch].to_relarea.move_to_relarea(m, args, desc) # XXX: Note that there seems to be a separate process, not run # from cygwin-admin's crontab, which changes the ownership of # files in the release area to cyguser:cygwin @@ -264,6 +269,10 @@ def process_maintainer_uploads(args, state, all_packages, m): # use merged package list state.packages[arch] = merged_packages[arch] + # clean up any empty directories + if not args.dryrun: + utils.rmemptysubdirs(os.path.join(basedir, m.name)) + # report what we've done added = [] for arch in common_constants.ARCHES + ['noarch', 'src']: @@ -697,6 +706,7 @@ def logging_setup(args): def main(): htdocs_default = os.path.join(common_constants.HTDOCS, 'packages') homedir_default = common_constants.HOMEDIR + stagingdir_default = common_constants.STAGINGDIR orphanmaint_default = common_constants.ORPHANMAINT pidfile_default = '/sourceware/cygwin-staging/calm.pid' pkglist_default = common_constants.PKGMAINT @@ -718,6 +728,7 @@ def main(): parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin') parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default) + parser.add_argument('--stagingdir', action='store', metavar='DIR', help="automated build staging directory (default: " + stagingdir_default + ")", default=stagingdir_default) parser.add_argument('--no-stale', action='store_false', dest='stale', help="don't vault stale packages") parser.set_defaults(stale=True) parser.add_argument('--reports', action='store_true', dest='reports', help="don't produce reports", default=None) diff --git a/calm/common_constants.py b/calm/common_constants.py index 1887681..906a56a 100644 --- a/calm/common_constants.py +++ b/calm/common_constants.py @@ -32,6 +32,9 @@ import os # base directory for maintainer upload directories HOMEDIR = '/sourceware/cygwin-staging/home' +# base directory for automated build staging directories +STAGINGDIR = '/sourceware/cygwin-staging/staging' + # the 'release area', contains all released files, which are rsync'ed to mirrors FTP = '/var/ftp/pub/cygwin' diff --git a/calm/movelist.py b/calm/movelist.py index 07b69d7..6101271 100644 --- a/calm/movelist.py +++ b/calm/movelist.py @@ -34,10 +34,13 @@ from . import utils # class MoveList(object): - def __init__(self): + def __init__(self, basedir=None): # a movelist is a dict with relative directory paths for keys and a list # of filenames for each value self.movelist = defaultdict(list) + # the directory the paths are relative to (by default the 'relarea') + if basedir: + self.basedir = basedir def __len__(self): return len(self.movelist) @@ -69,10 +72,10 @@ class MoveList(object): else: logging.error("can't %s %s, as it doesn't exist" % (verb, f)) - def move_to_relarea(self, m, args): + def move_to_relarea(self, m, args, desc): if self.movelist: - logging.info("move from %s's upload area to release area:" % (m.name)) - self._move(args, m.homedir(), args.rel_area, 'deploy') + logging.info("move from %s's %s area to release area:" % (m.name, desc)) + self._move(args, self.basedir, args.rel_area, 'deploy') def move_to_vault(self, args): if self.movelist: diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index 495d6eb..af6d650 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -168,6 +168,8 @@ mtime_anomalies = [ maint_anomalies = { 'libelf0': ['0.8.13-2'], # libelf is called libelf0 in x86 arch 'libelf0-devel': ['0.8.13-2'], + 'manlint': ['1.6g-2'], # unclear why this is under man + 'python3-h5py-debuginfo': ['2.9.0-1'], # superceded by python-h5py-debuginfo } # packages missing obsoletions diff --git a/calm/uploads.py b/calm/uploads.py index 961b397..3bb8faa 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -52,11 +52,12 @@ ScanResult = namedtuple('ScanResult', 'error,packages,to_relarea,to_vault,remove # # -def scan(m, all_packages, arch, args): - basedir = os.path.join(m.homedir(), arch) +def scan(scandir, m, all_packages, arch, args): + homedir = os.path.join(scandir, m.name) + basedir = os.path.join(homedir, arch) packages = defaultdict(package.Package) - move = MoveList() + move = MoveList(homedir) vault = MoveList() remove = [] remove_success = [] @@ -80,7 +81,7 @@ def scan(m, all_packages, arch, args): # scan package directories for (dirpath, _subdirs, files) in os.walk(os.path.join(basedir, 'release')): - relpath = os.path.relpath(dirpath, m.homedir()) + relpath = os.path.relpath(dirpath, homedir) removed_files = [] # filter out files we don't need to consider @@ -277,7 +278,7 @@ def scan(m, all_packages, arch, args): # read and validate package if files: - if package.read_package_dir(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True): + if package.read_package_dir(packages, homedir, dirpath, files, remove=removed_files, upload=True): error = True # always consider timestamp as checked during a dry-run, so it is never diff --git a/calm/utils.py b/calm/utils.py index 14bd517..f25e5f4 100644 --- a/calm/utils.py +++ b/calm/utils.py @@ -57,6 +57,22 @@ def makedirs(name): pass +# +# remove any empty subdirectories +# +def rmemptysubdirs(path): + for (dirpath, _subdirs, _files) in os.walk(path, topdown=False, followlinks=True): + # don't remove the given directory, only subdirectories + if os.path.relpath(dirpath, path) == '.': + continue + + # check whether the directory is now empty after processing any + # subdirectories, and if so, remove it + if len(os.listdir(dirpath)) == 0: + logging.debug('rmdir %s' % dirpath) + os.rmdir(dirpath) + + # # a wrapper for open() which: # diff --git a/test/test_calm.py b/test/test_calm.py index fe05f9a..e34ac99 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -48,6 +48,8 @@ import calm.package as package import calm.pkg2html as pkg2html import calm.uploads as uploads +ARGDIRS = ['rel_area', 'homedir', 'htdocs', 'stagingdir', 'vault'] + # # helper functions @@ -351,7 +353,7 @@ class CalmTest(unittest.TestCase): for (f, t) in ready_fns: os.system('touch %s "%s"' % (t, f)) - scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args.arch, args) + scan_result = uploads.scan('testdata/homes', m, pkglist + ['not-on-maintainer-list'], args.arch, args) os.chdir(oldcwd) shutil.rmtree(test_root) @@ -395,12 +397,13 @@ class CalmTest(unittest.TestCase): args = types.SimpleNamespace() - for d in ['rel_area', 'homedir', 'htdocs', 'vault']: + for d in ARGDIRS: setattr(args, d, tempfile.mktemp()) logging.info('%s = %s', d, getattr(args, d)) shutil.copytree('testdata/relarea', args.rel_area) shutil.copytree('testdata/homes.conflict', args.homedir) + os.mkdir(args.stagingdir) args.dryrun = False args.email = None @@ -417,7 +420,7 @@ class CalmTest(unittest.TestCase): state.packages = calm.calm.process_uploads(args, state) self.assertTrue(state.packages) - for d in ['rel_area', 'homedir', 'htdocs', 'vault']: + for d in ARGDIRS: with self.subTest(directory=d): dirlist = capture_dirtree(getattr(args, d)) compare_with_expected_file(self, 'testdata/conflict', dirlist, d) @@ -428,7 +431,7 @@ class CalmTest(unittest.TestCase): args = types.SimpleNamespace() - for d in ['rel_area', 'homedir', 'htdocs', 'vault']: + for d in ARGDIRS: setattr(args, d, tempfile.mktemp()) logging.info('%s = %s', d, getattr(args, d)) @@ -446,6 +449,7 @@ class CalmTest(unittest.TestCase): shutil.copytree('testdata/relarea', args.rel_area) shutil.copytree('testdata/homes', args.homedir) + os.mkdir(args.stagingdir) # set appropriate !readys m_homedir = os.path.join(args.homedir, 'Blooey McFooey') @@ -472,7 +476,7 @@ class CalmTest(unittest.TestCase): results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1) compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini') - for d in ['rel_area', 'homedir', 'htdocs', 'vault']: + for d in ARGDIRS: with self.subTest(directory=d): dirlist = capture_dirtree(getattr(args, d)) compare_with_expected_file(self, 'testdata/process_arch', dirlist, d) @@ -483,7 +487,7 @@ class CalmTest(unittest.TestCase): del j['timestamp'] compare_with_expected_file(self, 'testdata/process_arch', json.dumps(j, sort_keys=True, indent=4), 'packages.json') - for d in ['rel_area', 'homedir', 'htdocs', 'vault']: + for d in ARGDIRS: shutil.rmtree(getattr(args, d)) @classmethod diff --git a/test/testdata/conflict/stagingdir.expected b/test/testdata/conflict/stagingdir.expected new file mode 100644 index 0000000..e82d3c8 --- /dev/null +++ b/test/testdata/conflict/stagingdir.expected @@ -0,0 +1 @@ +{'.': []} diff --git a/test/testdata/process_arch/homedir.expected b/test/testdata/process_arch/homedir.expected index 5804575..0e6408e 100644 --- a/test/testdata/process_arch/homedir.expected +++ b/test/testdata/process_arch/homedir.expected @@ -11,13 +11,11 @@ 'Blooey McFooey/x86/release/not-on-package-list': ['not-on-package-list-1.0-1.tar.bz2', 'setup.hint'], 'Blooey McFooey/x86/release/not-ready': ['-not-ready-0.9-1.tar.bz2', 'not-ready-1.0-1.tar.bz2', 'setup.hint'], 'Blooey McFooey/x86/release/per-version': ['per-version-5.0-1-src.hint.bak'], - 'Blooey McFooey/x86/release/per-version-replacement-hint-only': [], 'Blooey McFooey/x86/release/testpackage': ['testpackage-1.0-1-src.hint.bak'], 'Blooey McFooey/x86/release/testpackage-zstd': ['testpackage-zstd-1.0-1-src.hint', 'testpackage-zstd-1.0-1-src.tar.zst', 'testpackage-zstd-1.0-1.hint', 'testpackage-zstd-1.0-1.tar.zst'], - 'Blooey McFooey/x86/release/testpackage/testpackage-subpackage': [], 'Blooey McFooey/x86/release/testpackage2': ['setup.hint', 'testpackage2-1.0-1.tar.bz2'], 'Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage': ['inprogress.SftpXFR.1234', 'setup.hint', diff --git a/test/testdata/process_arch/stagingdir.expected b/test/testdata/process_arch/stagingdir.expected new file mode 100644 index 0000000..e82d3c8 --- /dev/null +++ b/test/testdata/process_arch/stagingdir.expected @@ -0,0 +1 @@ +{'.': []}
reply other threads:[~2022-06-18 11:24 UTC|newest] Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20220618112409.4720A3853540@sourceware.org \ --to=jturney@sourceware.org \ --cc=cygwin-apps-cvs@sourceware.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).