From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 9604 invoked by alias); 21 Oct 2018 20:17:18 -0000 Mailing-List: contact cygwin-apps-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cygwin-apps-cvs-owner@sourceware.org Received: (qmail 9575 invoked by uid 9795); 21 Oct 2018 20:17:17 -0000 Date: Sun, 21 Oct 2018 20:17:00 -0000 Message-ID: <20181021201717.9546.qmail@sourceware.org> From: jturney@sourceware.org To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20181020-2-g0f05f12 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: c4b5da2c091f9d2f14e4cba8544d63d68369f47e X-Git-Newrev: 0f05f121774fe05aab0243d4eed30ff5dc398a56 X-SW-Source: 2018-q4/txt/msg00012.txt.bz2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0f05f121774fe05aab0243d4eed30ff5dc398a56 commit 0f05f121774fe05aab0243d4eed30ff5dc398a56 Author: Jon Turney Date: Sun Oct 21 18:31:22 2018 +0100 Warn about non-archive uploads Warn if a compressed empty file is uploaded in place of a compressed archive Also error if an impossibly small compressed archive is uploaded https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=def4b22ee4c24e0e3b72d4e67389c0908dc40630 commit def4b22ee4c24e0e3b72d4e67389c0908dc40630 Author: Jon Turney Date: Sun Oct 21 18:28:55 2018 +0100 Report 0-byte archives setup doesn't consider these valid, so report if they appear Diff: --- calm/common_constants.py | 8 ++++++++ calm/package.py | 12 ++++++++++-- calm/uploads.py | 17 +++++++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/calm/common_constants.py b/calm/common_constants.py index 8b17ee2..9db93b2 100644 --- a/calm/common_constants.py +++ b/calm/common_constants.py @@ -70,3 +70,11 @@ if os.uname()[1] == 'tambora': EMAILS = 'jon.turney@dronecode.org.uk' ALWAYS_BCC = '' MAILHOST = 'allegra' + +# size of a 0-byte file compressed +COMPRESSION_MINSIZE = { + 'bz2': 14, + 'gz': 24, + 'lzma': 23, + 'xz': 32, +} diff --git a/calm/package.py b/calm/package.py index e9bbe70..5e43f24 100755 --- a/calm/package.py +++ b/calm/package.py @@ -377,15 +377,23 @@ def read_package(packages, basedir, dirpath, files, remove=[], upload=False): # utility to determine if a tar file is empty # def tarfile_is_empty(tf): + size = os.path.getsize(tf) + + # report invalid files (smaller than the smallest possible compressed file + # for any of the compressions we support) + if size < 14: + logging.error("tar archive %s is too small (%d bytes)" % (tf, size)) + return True + # sometimes compressed empty files are used rather than a compressed empty # tar archive - if os.path.getsize(tf) <= 32: + if size <= 32: return True # parsing the tar archive just to determine if it contains at least one # archive member is relatively expensive, so we just assume it contains # something if it's over a certain size threshold - if os.path.getsize(tf) > 1024: + if size > 1024: return False # if it's really a tar file, does it contain zero files? diff --git a/calm/uploads.py b/calm/uploads.py index 79a9e7f..0861388 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -34,6 +34,7 @@ import shutil import tarfile import time +from . import common_constants from . import package # reminders will be issued daily @@ -202,12 +203,13 @@ def scan(m, all_packages, arch, args): continue # verify compressed archive files are valid - if re.search(r'\.tar\.(bz2|gz|lzma|xz)$', f): + match = re.search(r'\.tar\.(bz2|gz|lzma|xz)$', f) + if match: valid = True - # accept a compressed empty file, even though it isn't a valid - # compressed archive - if os.path.getsize(fn) > 32: + size = os.path.getsize(fn) + minsize = common_constants.COMPRESSION_MINSIZE[match.group(1)] + if size > minsize: try: # we need to extract all of an archive contents to validate # it @@ -217,6 +219,13 @@ def scan(m, all_packages, arch, args): valid = False logging.error("exception %s while reading %s" % (type(e).__name__, fn)) logging.debug('', exc_info=True) + elif size == minsize: + # accept a compressed empty file, even though it isn't a + # valid compressed archive + logging.warning("%s is a compressed empty file, not a compressed archive, please update to cygport >= 0.23.1" % f) + else: + logging.error("compressed archive %s is too small to be valid (%d bytes)" % (f, size)) + valid = False if not valid: files.remove(f)