public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-54-g39ee07e
@ 2022-06-18 11:24 Jon TURNEY
  0 siblings, 0 replies; only message in thread
From: Jon TURNEY @ 2022-06-18 11:24 UTC (permalink / raw)
  To: cygwin-apps-cvs




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=39ee07ed3713329bde426fd9efa53f1bed5df89e

commit 39ee07ed3713329bde426fd9efa53f1bed5df89e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Jun 11 15:25:25 2022 +0100

    Use a separate staging dir for automated build uploads
    
    Look in a different directory for packages being deployed by scallywag.
    
    Future work: this will allow disabling the uploading of local builds (on
    a per-package basis), whilst still allowing automated build uploads.
    
    Update tests appropriately.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1a74d4efd3a4d56d8b42f605f2c4b48afeb2da57

commit 1a74d4efd3a4d56d8b42f605f2c4b48afeb2da57
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Jun 9 16:20:14 2022 +0100

    Remove empty subdirectories
    
    Remove empty subdirectories in release and upload areas.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=234c197298d83040db5ebd48628fbda7bd98668c

commit 234c197298d83040db5ebd48628fbda7bd98668c
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Jun 9 22:02:13 2022 +0100

    Add some more maintainer-less package anomalies
    
    Add some more maintainer-less package anomalies, revealed by clean up in
    cygwin-pkg-maint.


Diff:
---
 calm/calm.py                                   | 19 +++++++++++++++----
 calm/common_constants.py                       |  3 +++
 calm/movelist.py                               | 11 +++++++----
 calm/past_mistakes.py                          |  2 ++
 calm/uploads.py                                | 11 ++++++-----
 calm/utils.py                                  | 16 ++++++++++++++++
 test/test_calm.py                              | 16 ++++++++++------
 test/testdata/conflict/stagingdir.expected     |  1 +
 test/testdata/process_arch/homedir.expected    |  2 --
 test/testdata/process_arch/stagingdir.expected |  1 +
 10 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index 63cdc60..49bdcfd 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -126,6 +126,10 @@ def process_relarea(args, state):
             logging.error("error while evaluating stale packages")
             return None
 
+    # clean up any empty directories
+    if not args.dryrun:
+        utils.rmemptysubdirs(args.rel_area)
+
     return packages
 
 
@@ -146,7 +150,8 @@ def process_uploads(args, state):
         m = mlist[name]
 
         with logfilters.AttrFilter(maint=m.name):
-            process_maintainer_uploads(args, state, all_packages, m)
+            process_maintainer_uploads(args, state, all_packages, m, args.homedir, 'upload')
+            process_maintainer_uploads(args, state, all_packages, m, args.stagingdir, 'staging')
 
     # record updated reminder times for maintainers
     maintainers.update_reminder_times(mlist)
@@ -154,7 +159,7 @@ def process_uploads(args, state):
     return state.packages
 
 
-def process_maintainer_uploads(args, state, all_packages, m):
+def process_maintainer_uploads(args, state, all_packages, m, basedir, desc):
     name = m.name
 
     # for each arch and noarch
@@ -164,7 +169,7 @@ def process_maintainer_uploads(args, state, all_packages, m):
         logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name))
 
         # read uploads
-        scan_result[arch] = uploads.scan(m, all_packages, arch, args)
+        scan_result[arch] = uploads.scan(basedir, m, all_packages, arch, args)
 
         # remove triggers
         uploads.remove(args, scan_result[arch].remove_always)
@@ -247,7 +252,7 @@ def process_maintainer_uploads(args, state, all_packages, m):
         uploads.remove(args, scan_result[arch].remove_success)
         if scan_result[arch].to_relarea:
             logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch))
-        scan_result[arch].to_relarea.move_to_relarea(m, args)
+        scan_result[arch].to_relarea.move_to_relarea(m, args, desc)
         # XXX: Note that there seems to be a separate process, not run
         # from cygwin-admin's crontab, which changes the ownership of
         # files in the release area to cyguser:cygwin
@@ -264,6 +269,10 @@ def process_maintainer_uploads(args, state, all_packages, m):
         # use merged package list
         state.packages[arch] = merged_packages[arch]
 
+    # clean up any empty directories
+    if not args.dryrun:
+        utils.rmemptysubdirs(os.path.join(basedir, m.name))
+
     # report what we've done
     added = []
     for arch in common_constants.ARCHES + ['noarch', 'src']:
@@ -697,6 +706,7 @@ def logging_setup(args):
 def main():
     htdocs_default = os.path.join(common_constants.HTDOCS, 'packages')
     homedir_default = common_constants.HOMEDIR
+    stagingdir_default = common_constants.STAGINGDIR
     orphanmaint_default = common_constants.ORPHANMAINT
     pidfile_default = '/sourceware/cygwin-staging/calm.pid'
     pkglist_default = common_constants.PKGMAINT
@@ -718,6 +728,7 @@ def main():
     parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin')
     parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
     parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default)
+    parser.add_argument('--stagingdir', action='store', metavar='DIR', help="automated build staging directory (default: " + stagingdir_default + ")", default=stagingdir_default)
     parser.add_argument('--no-stale', action='store_false', dest='stale', help="don't vault stale packages")
     parser.set_defaults(stale=True)
     parser.add_argument('--reports', action='store_true', dest='reports', help="don't produce reports", default=None)
diff --git a/calm/common_constants.py b/calm/common_constants.py
index 1887681..906a56a 100644
--- a/calm/common_constants.py
+++ b/calm/common_constants.py
@@ -32,6 +32,9 @@ import os
 # base directory for maintainer upload directories
 HOMEDIR = '/sourceware/cygwin-staging/home'
 
+# base directory for automated build staging directories
+STAGINGDIR = '/sourceware/cygwin-staging/staging'
+
 # the 'release area', contains all released files, which are rsync'ed to mirrors
 FTP = '/var/ftp/pub/cygwin'
 
diff --git a/calm/movelist.py b/calm/movelist.py
index 07b69d7..6101271 100644
--- a/calm/movelist.py
+++ b/calm/movelist.py
@@ -34,10 +34,13 @@ from . import utils
 #
 
 class MoveList(object):
-    def __init__(self):
+    def __init__(self, basedir=None):
         # a movelist is a dict with relative directory paths for keys and a list
         # of filenames for each value
         self.movelist = defaultdict(list)
+        # the directory the paths are relative to (by default the 'relarea')
+        if basedir:
+            self.basedir = basedir
 
     def __len__(self):
         return len(self.movelist)
@@ -69,10 +72,10 @@ class MoveList(object):
                     else:
                         logging.error("can't %s %s, as it doesn't exist" % (verb, f))
 
-    def move_to_relarea(self, m, args):
+    def move_to_relarea(self, m, args, desc):
         if self.movelist:
-            logging.info("move from %s's upload area to release area:" % (m.name))
-        self._move(args, m.homedir(), args.rel_area, 'deploy')
+            logging.info("move from %s's %s area to release area:" % (m.name, desc))
+        self._move(args, self.basedir, args.rel_area, 'deploy')
 
     def move_to_vault(self, args):
         if self.movelist:
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index 495d6eb..af6d650 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -168,6 +168,8 @@ mtime_anomalies = [
 maint_anomalies = {
     'libelf0': ['0.8.13-2'],  # libelf is called libelf0 in x86 arch
     'libelf0-devel': ['0.8.13-2'],
+    'manlint': ['1.6g-2'],  # unclear why this is under man
+    'python3-h5py-debuginfo': ['2.9.0-1'],  # superceded by python-h5py-debuginfo
 }
 
 # packages missing obsoletions
diff --git a/calm/uploads.py b/calm/uploads.py
index 961b397..3bb8faa 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -52,11 +52,12 @@ ScanResult = namedtuple('ScanResult', 'error,packages,to_relarea,to_vault,remove
 #
 #
 
-def scan(m, all_packages, arch, args):
-    basedir = os.path.join(m.homedir(), arch)
+def scan(scandir, m, all_packages, arch, args):
+    homedir = os.path.join(scandir, m.name)
+    basedir = os.path.join(homedir, arch)
 
     packages = defaultdict(package.Package)
-    move = MoveList()
+    move = MoveList(homedir)
     vault = MoveList()
     remove = []
     remove_success = []
@@ -80,7 +81,7 @@ def scan(m, all_packages, arch, args):
 
     # scan package directories
     for (dirpath, _subdirs, files) in os.walk(os.path.join(basedir, 'release')):
-        relpath = os.path.relpath(dirpath, m.homedir())
+        relpath = os.path.relpath(dirpath, homedir)
         removed_files = []
 
         # filter out files we don't need to consider
@@ -277,7 +278,7 @@ def scan(m, all_packages, arch, args):
 
         # read and validate package
         if files:
-            if package.read_package_dir(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True):
+            if package.read_package_dir(packages, homedir, dirpath, files, remove=removed_files, upload=True):
                 error = True
 
     # always consider timestamp as checked during a dry-run, so it is never
diff --git a/calm/utils.py b/calm/utils.py
index 14bd517..f25e5f4 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -57,6 +57,22 @@ def makedirs(name):
         pass
 
 
+#
+# remove any empty subdirectories
+#
+def rmemptysubdirs(path):
+    for (dirpath, _subdirs, _files) in os.walk(path, topdown=False, followlinks=True):
+        # don't remove the given directory, only subdirectories
+        if os.path.relpath(dirpath, path) == '.':
+            continue
+
+        # check whether the directory is now empty after processing any
+        # subdirectories, and if so, remove it
+        if len(os.listdir(dirpath)) == 0:
+            logging.debug('rmdir %s' % dirpath)
+            os.rmdir(dirpath)
+
+
 #
 # a wrapper for open() which:
 #
diff --git a/test/test_calm.py b/test/test_calm.py
index fe05f9a..e34ac99 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -48,6 +48,8 @@ import calm.package as package
 import calm.pkg2html as pkg2html
 import calm.uploads as uploads
 
+ARGDIRS = ['rel_area', 'homedir', 'htdocs', 'stagingdir', 'vault']
+
 
 #
 # helper functions
@@ -351,7 +353,7 @@ class CalmTest(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args.arch, args)
+        scan_result = uploads.scan('testdata/homes', m, pkglist + ['not-on-maintainer-list'], args.arch, args)
 
         os.chdir(oldcwd)
         shutil.rmtree(test_root)
@@ -395,12 +397,13 @@ class CalmTest(unittest.TestCase):
 
         args = types.SimpleNamespace()
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
         shutil.copytree('testdata/relarea', args.rel_area)
         shutil.copytree('testdata/homes.conflict', args.homedir)
+        os.mkdir(args.stagingdir)
 
         args.dryrun = False
         args.email = None
@@ -417,7 +420,7 @@ class CalmTest(unittest.TestCase):
         state.packages = calm.calm.process_uploads(args, state)
         self.assertTrue(state.packages)
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             with self.subTest(directory=d):
                 dirlist = capture_dirtree(getattr(args, d))
                 compare_with_expected_file(self, 'testdata/conflict', dirlist, d)
@@ -428,7 +431,7 @@ class CalmTest(unittest.TestCase):
 
         args = types.SimpleNamespace()
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
@@ -446,6 +449,7 @@ class CalmTest(unittest.TestCase):
 
         shutil.copytree('testdata/relarea', args.rel_area)
         shutil.copytree('testdata/homes', args.homedir)
+        os.mkdir(args.stagingdir)
 
         # set appropriate !readys
         m_homedir = os.path.join(args.homedir, 'Blooey McFooey')
@@ -472,7 +476,7 @@ class CalmTest(unittest.TestCase):
             results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1)
             compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini')
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             with self.subTest(directory=d):
                 dirlist = capture_dirtree(getattr(args, d))
                 compare_with_expected_file(self, 'testdata/process_arch', dirlist, d)
@@ -483,7 +487,7 @@ class CalmTest(unittest.TestCase):
             del j['timestamp']
             compare_with_expected_file(self, 'testdata/process_arch', json.dumps(j, sort_keys=True, indent=4), 'packages.json')
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             shutil.rmtree(getattr(args, d))
 
     @classmethod
diff --git a/test/testdata/conflict/stagingdir.expected b/test/testdata/conflict/stagingdir.expected
new file mode 100644
index 0000000..e82d3c8
--- /dev/null
+++ b/test/testdata/conflict/stagingdir.expected
@@ -0,0 +1 @@
+{'.': []}
diff --git a/test/testdata/process_arch/homedir.expected b/test/testdata/process_arch/homedir.expected
index 5804575..0e6408e 100644
--- a/test/testdata/process_arch/homedir.expected
+++ b/test/testdata/process_arch/homedir.expected
@@ -11,13 +11,11 @@
  'Blooey McFooey/x86/release/not-on-package-list': ['not-on-package-list-1.0-1.tar.bz2', 'setup.hint'],
  'Blooey McFooey/x86/release/not-ready': ['-not-ready-0.9-1.tar.bz2', 'not-ready-1.0-1.tar.bz2', 'setup.hint'],
  'Blooey McFooey/x86/release/per-version': ['per-version-5.0-1-src.hint.bak'],
- 'Blooey McFooey/x86/release/per-version-replacement-hint-only': [],
  'Blooey McFooey/x86/release/testpackage': ['testpackage-1.0-1-src.hint.bak'],
  'Blooey McFooey/x86/release/testpackage-zstd': ['testpackage-zstd-1.0-1-src.hint',
                                                  'testpackage-zstd-1.0-1-src.tar.zst',
                                                  'testpackage-zstd-1.0-1.hint',
                                                  'testpackage-zstd-1.0-1.tar.zst'],
- 'Blooey McFooey/x86/release/testpackage/testpackage-subpackage': [],
  'Blooey McFooey/x86/release/testpackage2': ['setup.hint', 'testpackage2-1.0-1.tar.bz2'],
  'Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage': ['inprogress.SftpXFR.1234',
                                                                      'setup.hint',
diff --git a/test/testdata/process_arch/stagingdir.expected b/test/testdata/process_arch/stagingdir.expected
new file mode 100644
index 0000000..e82d3c8
--- /dev/null
+++ b/test/testdata/process_arch/stagingdir.expected
@@ -0,0 +1 @@
+{'.': []}



^ permalink raw reply	[flat|nested] only message in thread

only message in thread, other threads:[~2022-06-18 11:24 UTC | newest]

Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-06-18 11:24 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-54-g39ee07e Jon TURNEY

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).