[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20221205-2-g8579fcf

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20221205-2-g8579fcf

[Jon TURNEY]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8579fcf737b93dd4b60cb42f2f1cb782a3886bd6

commit 8579fcf737b93dd4b60cb42f2f1cb782a3886bd6
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Fri Sep 16 17:56:56 2022 +0100

    Treat sha512.sum as a cache for sha512 computation
    
    Rather than treating the sha512.sum file as (kind of) part of the
    package, just treat it as a cache for sha512 computation of files in
    the same directory.
    
    Also use the mtime_cache decorator to avoid parsing sha512.sum files
    which haven't changed.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=56f16534dd1a0a63f3a5069ca13fe5d5aa7df691

commit 56f16534dd1a0a63f3a5069ca13fe5d5aa7df691
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Fri Dec 9 14:35:37 2022 +0000

    Add a simple wrapper for caching a parsed file until mtime changes


Diff:
---
 calm/package.py | 76 +++++++++++++++++++++++++++++++++------------------------
 calm/utils.py   | 33 +++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 32 deletions(-)

diff --git a/calm/package.py b/calm/package.py
index 43fc4cc..d3b5ab0 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -44,6 +44,7 @@ from . import common_constants
 from . import hint
 from . import maintainers
 from . import past_mistakes
+from . import utils
 from .movelist import MoveList
 from .version import SetupVersion
 
@@ -159,7 +160,7 @@ def read_packages(rel_area, arch):
 
 # helper function to compute sha512 for a particular file
 # (block_size should be some multiple of sha512 block size which can be efficiently read)
-def sha512_file(fn, block_size=256 * 128):
+def sha512_file_hash(fn, block_size=256 * 128):
     sha512 = hashlib.sha512()
 
     with open(fn, 'rb') as f:
@@ -169,6 +170,41 @@ def sha512_file(fn, block_size=256 * 128):
     return sha512.hexdigest()
 
 
+# helper function to parse a sha512.sum file
+@utils.mtime_cache
+def sha512sum_file_read(sum_fn):
+    sha512 = {}
+    with open(sum_fn) as fo:
+        for l in fo:
+            match = re.match(r'^(\S+)\s+(?:\*|)(\S+)$', l)
+            if match:
+                sha512[match.group(2)] = match.group(1)
+            else:
+                logging.warning("bad line '%s' in checksum file %s" % (l.strip(), sum_fn))
+
+    return sha512
+
+
+# helper function to determine sha512 for a particular file
+#
+# read sha512 checksum from a sha512.sum file, if present, otherwise compute it
+def sha512_file(fn):
+    (dirname, basename) = os.path.split(fn)
+    sum_fn = os.path.join(dirname, 'sha512.sum')
+    if os.path.exists(sum_fn):
+        sha512 = sha512sum_file_read(sum_fn)
+        if basename in sha512:
+            return sha512[basename]
+        else:
+            logging.debug("no line for file %s in checksum file %s" % (basename, sum_fn))
+    else:
+        logging.debug("no sha512.sum in %s" % dirname)
+
+    sha512 = sha512_file_hash(fn)
+    logging.debug("computed sha512 hash for %s is %s" % (basename, sha512))
+    return sha512
+
+
 # process a list of package version-constraints
 def process_package_constraint_list(pcl):
     # split, keeping optional version-relation, trim and sort
@@ -264,31 +300,27 @@ def read_package_dir(packages, basedir, dirpath, files, upload=False):
     # the package name is always the directory name
     p = os.path.basename(dirpath)
 
+    # ignore dotfiles, backup files and checksum files
+    for f in files[:]:
+        if f.startswith('.') or f.endswith('.bak') or f == 'sha512.sum':
+            files.remove(f)
+
     # no .hint files
     if not any([f.endswith('.hint') for f in files]):
         if (relpath.count(os.path.sep) > 1):
-            for s in ['sha512.sum']:
-                if s in files:
-                    files.remove(s)
-
             if len(files) > 0:
                 logging.error("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
                 return True
 
         return False
 
-    # ignore dotfiles and backup files
-    for f in files[:]:
-        if f.startswith('.') or f.endswith('.bak'):
-            files.remove(f)
-
     # classify files for which kind of package they belong to
     fl = {}
     for kind in list(Kind) + ['all']:
         fl[kind] = []
 
     for f in files[:]:
-        if f == 'sha512.sum' or f == 'override.hint':
+        if f == 'override.hint':
             fl['all'].append(f)
             files.remove(f)
         elif re.match(r'^' + re.escape(p) + r'.*\.hint$', f):
@@ -358,21 +390,6 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict):
     else:
         override_hints = {}
 
-    # read sha512.sum
-    sha512 = {}
-    if 'sha512.sum' not in files:
-        logging.debug("no sha512.sum for package '%s'" % p)
-    else:
-        files.remove('sha512.sum')
-
-        with open(os.path.join(dirpath, 'sha512.sum')) as fo:
-            for l in fo:
-                match = re.match(r'^(\S+)\s+(?:\*|)(\S+)$', l)
-                if match:
-                    sha512[match.group(2)] = match.group(1)
-                else:
-                    logging.warning("bad line '%s' in sha512.sum for package '%s'" % (l.strip(), p))
-
     # build a list of version-releases (since replacement pvr.hint files are
     # allowed to be uploaded, we must consider both .tar and .hint files for
     # that), and collect the attributes for each tar file
@@ -437,12 +454,7 @@ def read_one_package(packages, p, relpath, dirpath, files, kind, strict):
             t.size = os.path.getsize(os.path.join(dirpath, f))
             t.is_empty = tarfile_is_empty(os.path.join(dirpath, f))
             t.mtime = os.path.getmtime(os.path.join(dirpath, f))
-
-            if f in sha512:
-                t.sha512 = sha512[f]
-            else:
-                t.sha512 = sha512_file(os.path.join(dirpath, f))
-                logging.debug("no sha512.sum line for file %s in package '%s', computed sha512 hash is %s" % (f, p, t.sha512))
+            t.sha512 = sha512_file(os.path.join(dirpath, f))
 
             tars[vr] = t
 
diff --git a/calm/utils.py b/calm/utils.py
index fb95cc6..9f75813 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -125,3 +125,36 @@ def system(args):
     else:
         for l in output.decode().splitlines():
             logging.info(l)
+
+
+#
+# This provides a simple wrapper around a function which takes a pathname as
+# it's only parameter.  The result is cached as long as the mtime of the
+# pathname is unchanged.
+#
+def mtime_cache(user_function):
+    sentinel = object()          # unique object used to signal cache misses
+    cache = {}
+
+    def wrapper(key):
+        # make sure path is absolute
+        key = os.path.abspath(key)
+
+        (result, mtime) = cache.get(key, (sentinel, 0))
+
+        new_mtime = os.path.getmtime(key)
+
+        # cache hit
+        if result is not sentinel:
+            # cache valid
+            if new_mtime == mtime:
+                return result
+            else:
+                logging.debug('%s cache invalidated by mtime change' % key)
+
+        # cache miss
+        result = user_function(key)
+        cache[key] = (result, new_mtime)
+        return result
+
+    return wrapper