From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 9CF7B38432D6; Wed, 14 Dec 2022 13:24:44 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 9CF7B38432D6 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1671024284; bh=j1EDNija4Z//KxjMIVS00njRH4Ult57KRM8uC2acAQ0=; h=To:Subject:Date:From:From; b=mKFFsyJbsyHQerMOwaPnyHzfBEyVYUcWWAruGapZiPtdY8uGFyWycgMiI3rzEb8ds 5rGMB6iWL4FokN6Mky8BJOQb2TmUEQBoi1idGcAkvQ20eLlUGV9Wav4IJRlUUUbl1t RRFpssT8W7X49TGjCnhdxYbXCLicc4xbdEi49ahU= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20221205-2-g8579fcf X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: e9a1fac4b4a36a292516b26ea2f8f44e5fd4614f X-Git-Newrev: 8579fcf737b93dd4b60cb42f2f1cb782a3886bd6 Message-Id: <20221214132444.9CF7B38432D6@sourceware.org> Date: Wed, 14 Dec 2022 13:24:44 +0000 (GMT) From: Jon TURNEY List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8579fcf737b93dd4b60cb42f2f1cb782a3886bd6 commit 8579fcf737b93dd4b60cb42f2f1cb782a3886bd6 Author: Jon Turney Date: Fri Sep 16 17:56:56 2022 +0100 Treat sha512.sum as a cache for sha512 computation Rather than treating the sha512.sum file as (kind of) part of the package, just treat it as a cache for sha512 computation of files in the same directory. Also use the mtime_cache decorator to avoid parsing sha512.sum files which haven't changed. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=56f16534dd1a0a63f3a5069ca13fe5d5aa7df691 commit 56f16534dd1a0a63f3a5069ca13fe5d5aa7df691 Author: Jon Turney Date: Fri Dec 9 14:35:37 2022 +0000 Add a simple wrapper for caching a parsed file until mtime changes Diff: --- calm/package.py | 76 +++++++++++++++++++++++++++++++++------------------------ calm/utils.py | 33 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 32 deletions(-) diff --git a/calm/package.py b/calm/package.py index 43fc4cc..d3b5ab0 100755 --- a/calm/package.py +++ b/calm/package.py @@ -44,6 +44,7 @@ from . import common_constants from . import hint from . import maintainers from . import past_mistakes +from . import utils from .movelist import MoveList from .version import SetupVersion @@ -159,7 +160,7 @@ def read_packages(rel_area, arch): # helper function to compute sha512 for a particular file # (block_size should be some multiple of sha512 block size which can be efficiently read) -def sha512_file(fn, block_size=256 * 128): +def sha512_file_hash(fn, block_size=256 * 128): sha512 = hashlib.sha512() with open(fn, 'rb') as f: @@ -169,6 +170,41 @@ def sha512_file(fn, block_size=256 * 128): return sha512.hexdigest() +# helper function to parse a sha512.sum file +@utils.mtime_cache +def sha512sum_file_read(sum_fn): + sha512 = {} + with open(sum_fn) as fo: + for l in fo: + match = re.match(r'^(\S+)\s+(?:\*|)(\S+)$', l) + if match: + sha512[match.group(2)] = match.group(1) + else: + logging.warning("bad line '%s' in checksum file %s" % (l.strip(), sum_fn)) + + return sha512 + + +# helper function to determine sha512 for a particular file +# +# read sha512 checksum from a sha512.sum file, if present, otherwise compute it +def sha512_file(fn): + (dirname, basename) = os.path.split(fn) + sum_fn = os.path.join(dirname, 'sha512.sum') + if os.path.exists(sum_fn): + sha512 = sha512sum_file_read(sum_fn) + if basename in sha512: + return sha512[basename] + else: + logging.debug("no line for file %s in checksum file %s" % (basename, sum_fn)) + else: + logging.debug("no sha512.sum in %s" % dirname) + + sha512 = sha512_file_hash(fn) + logging.debug("computed sha512 hash for %s is %s" % (basename, sha512)) + return sha512 + + # process a list of package version-constraints def process_package_constraint_list(pcl): # split, keeping optional version-relation, trim and sort @@ -264,31 +300,27 @@ def read_package_dir(packages, basedir, dirpath, files, upload=False): # the package name is always the directory name p = os.path.basename(dirpath) + # ignore dotfiles, backup files and checksum files + for f in files[:]: + if f.startswith('.') or f.endswith('.bak') or f == 'sha512.sum': + files.remove(f) + # no .hint files if not any([f.endswith('.hint') for f in files]): if (relpath.count(os.path.sep) > 1): - for s in ['sha512.sum']: - if s in files: - files.remove(s) - if len(files) > 0: logging.error("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files))) return True return False - # ignore dotfiles and backup files - for f in files[:]: - if f.startswith('.') or f.endswith('.bak'): - files.remove(f) - # classify files for which kind of package they belong to fl = {} for kind in list(Kind) + ['all']: fl[kind] = [] for f in files[:]: - if f == 'sha512.sum' or f == 'override.hint': + if f == 'override.hint': fl['all'].append(f) files.remove(f) elif re.match(r'^' + re.escape(p) + r'.*\.hint$', f): @@ -358,21 +390,6 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): else: override_hints = {} - # read sha512.sum - sha512 = {} - if 'sha512.sum' not in files: - logging.debug("no sha512.sum for package '%s'" % p) - else: - files.remove('sha512.sum') - - with open(os.path.join(dirpath, 'sha512.sum')) as fo: - for l in fo: - match = re.match(r'^(\S+)\s+(?:\*|)(\S+)$', l) - if match: - sha512[match.group(2)] = match.group(1) - else: - logging.warning("bad line '%s' in sha512.sum for package '%s'" % (l.strip(), p)) - # build a list of version-releases (since replacement pvr.hint files are # allowed to be uploaded, we must consider both .tar and .hint files for # that), and collect the attributes for each tar file @@ -437,12 +454,7 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict): t.size = os.path.getsize(os.path.join(dirpath, f)) t.is_empty = tarfile_is_empty(os.path.join(dirpath, f)) t.mtime = os.path.getmtime(os.path.join(dirpath, f)) - - if f in sha512: - t.sha512 = sha512[f] - else: - t.sha512 = sha512_file(os.path.join(dirpath, f)) - logging.debug("no sha512.sum line for file %s in package '%s', computed sha512 hash is %s" % (f, p, t.sha512)) + t.sha512 = sha512_file(os.path.join(dirpath, f)) tars[vr] = t diff --git a/calm/utils.py b/calm/utils.py index fb95cc6..9f75813 100644 --- a/calm/utils.py +++ b/calm/utils.py @@ -125,3 +125,36 @@ def system(args): else: for l in output.decode().splitlines(): logging.info(l) + + +# +# This provides a simple wrapper around a function which takes a pathname as +# it's only parameter. The result is cached as long as the mtime of the +# pathname is unchanged. +# +def mtime_cache(user_function): + sentinel = object() # unique object used to signal cache misses + cache = {} + + def wrapper(key): + # make sure path is absolute + key = os.path.abspath(key) + + (result, mtime) = cache.get(key, (sentinel, 0)) + + new_mtime = os.path.getmtime(key) + + # cache hit + if result is not sentinel: + # cache valid + if new_mtime == mtime: + return result + else: + logging.debug('%s cache invalidated by mtime change' % key) + + # cache miss + result = user_function(key) + cache[key] = (result, new_mtime) + return result + + return wrapper