[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-27-g0c139f8

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-27-g0c139f8

[Jon TURNEY]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0c139f81907b572da7a87652ea414a089080b640

commit 0c139f81907b572da7a87652ea414a089080b640
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Feb 17 12:29:48 2022 +0000

    Extend relaxed package retention to obsolete packages

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f3a2daab817aa876e3e23aeaf92bf493233d8963

commit f3a2daab817aa876e3e23aeaf92bf493233d8963
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Nov 23 14:59:37 2019 +0000

    Persistently record all package names
    
    Record all package names persistently, to determine the set of names of
    packages which have been removed.
    
    This allows removing packages whilst still validating the names in an
    obsoletes: hint, without having to manually maintain a list of those
    names.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=bfa6da62336781086b93b575b9490fabe58b1db6

commit bfa6da62336781086b93b575b9490fabe58b1db6
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Feb 17 22:51:13 2022 +0000

    Update perl annotation tool
    
    Simply add a dependency on the current perl version provide, where
    appropriate. We can do this now, as all the historical cases where it's
    missing have been fixed.
    
    Future work: Rather than hard-coding the current perl version provide,
    in the case of vendored perl modules, we could extract the relevant
    version from the /usr/share/perl5/vendor_perl/5.nn/ path that files are
    installed into. Likewise, for modules linked with cygperl5_nn, we could
    determine the relevant version by extracting them and examining their
    dependencies.


Diff:
---
 calm/calm.py                               | 33 ++++++++------
 calm/db.py                                 | 73 ++++++++++++++++++++++++++++++
 calm/fix-annotate-perl-hint.py             | 34 ++++++--------
 calm/package.py                            | 11 ++++-
 calm/past_mistakes.py                      |  4 --
 test/test_calm.py                          |  6 +--
 test/testdata/conflict/htdocs.expected     |  1 +
 test/testdata/process_arch/htdocs.expected |  2 +-
 8 files changed, 120 insertions(+), 44 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index 3244d2b..5d106e0 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -67,6 +67,7 @@ from .abeyance_handler import AbeyanceHandler
 from .buffering_smtp_handler import BufferingSMTPHandler
 from .movelist import MoveList
 from . import common_constants
+from . import db
 from . import irk
 from . import maintainers
 from . import package
@@ -86,25 +87,26 @@ class CalmState(object):
     def __init__(self):
         self.subject = ''
         self.packages = {}
+        self.valid_provides = set()
 
 
 #
 #
 #
 
-def process_relarea(args):
+def process_relarea(args, state):
     packages = {}
     error = False
 
-    # for each arch
+    # read the package list for each arch
     for arch in common_constants.ARCHES:
         logging.debug("reading existing packages for arch %s" % (arch))
-
-        # build package list
         packages[arch] = package.read_packages(args.rel_area, arch)
 
-        # validate the package set
-        if not package.validate_packages(args, packages[arch]):
+    # validate the package set for each arch
+    state.valid_provides = db.update_package_names(args, packages)
+    for arch in common_constants.ARCHES:
+        if not package.validate_packages(args, packages[arch], state.valid_provides):
             logging.error("existing %s package set has errors" % (arch))
             error = True
 
@@ -114,7 +116,7 @@ def process_relarea(args):
     # packages can be stale due to changes made directly in the release
     # area, so first check here if there are any stale packages to vault
     if args.stale:
-        stale_to_vault = remove_stale_packages(args, packages)
+        stale_to_vault = remove_stale_packages(args, packages, state)
         if stale_to_vault:
             for arch in common_constants.ARCHES + ['noarch', 'src']:
                 logging.info("vaulting %d old package(s) for arch %s" % (len(stale_to_vault[arch]), arch))
@@ -187,9 +189,11 @@ def process_uploads(args, state):
                 # remove files which are to be removed
                 scan_result[arch].to_vault.map(lambda p, f: package.delete(merged_packages[arch], p, f))
 
-                # validate the package set
+            # validate the package set
+            state.valid_provides = db.update_package_names(args, merged_packages)
+            for arch in common_constants.ARCHES:
                 logging.debug("validating merged %s package set for maintainer %s" % (arch, name))
-                if not package.validate_packages(args, merged_packages[arch]):
+                if not package.validate_packages(args, merged_packages[arch], state.valid_provides):
                     logging.error("error while validating merged %s packages for %s" % (arch, name))
                     valid = False
 
@@ -201,7 +205,7 @@ def process_uploads(args, state):
             # check for packages which are stale as a result of this upload,
             # which we will want in the same report
             if args.stale:
-                stale_to_vault = remove_stale_packages(args, merged_packages)
+                stale_to_vault = remove_stale_packages(args, merged_packages, state)
 
                 # if an error occurred ...
                 if not stale_to_vault:
@@ -274,7 +278,7 @@ def process(args, state):
         if args.dryrun:
             logging.warning("--dry-run is in effect, nothing will really be done")
 
-        state.packages = process_relarea(args)
+        state.packages = process_relarea(args, state)
         if not state.packages:
             return None
 
@@ -287,7 +291,7 @@ def process(args, state):
 # remove stale packages
 #
 
-def remove_stale_packages(args, packages):
+def remove_stale_packages(args, packages, state):
     to_vault = {}
     to_vault['noarch'] = MoveList()
     to_vault['src'] = MoveList()
@@ -309,8 +313,9 @@ def remove_stale_packages(args, packages):
     # re-validate package sets
     # (this shouldn't fail, but we check just to sure...)
     error = False
+    state.valid_provides = db.update_package_names(args, packages)
     for arch in common_constants.ARCHES:
-        if not package.validate_packages(args, packages[arch]):
+        if not package.validate_packages(args, packages[arch], state.valid_provides):
             logging.error("%s package set has errors after removing stale packages" % arch)
             error = True
 
@@ -561,7 +566,7 @@ def do_daemon(args, state):
                         if last_signal != signal.SIGALRM:
                             irk.irk("calm processing release area")
                         read_relarea = False
-                        state.packages = process_relarea(args)
+                        state.packages = process_relarea(args, state)
 
                     if not state.packages:
                         logging.error("errors in relarea, not processing uploads or writing setup.ini")
diff --git a/calm/db.py b/calm/db.py
new file mode 100644
index 0000000..c1efbcf
--- /dev/null
+++ b/calm/db.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+#
+# package db
+#
+
+
+import logging
+import os
+import sqlite3
+
+from . import utils
+
+
+def connect(args):
+    utils.makedirs(args.htdocs)
+    dbfn = os.path.join(args.htdocs, 'calm.db')
+    logging.debug("sqlite3 database %s" % (dbfn))
+
+    conn = sqlite3.connect(dbfn, detect_types=sqlite3.PARSE_DECLTYPES)
+    conn.execute('''CREATE TABLE IF NOT EXISTS historic_package_names
+                    (name TEXT NOT NULL PRIMARY KEY
+                    )''')
+    conn.commit()
+
+    return conn
+
+
+#
+# this tracks the set of all names we have ever had for packages, and returns
+# ones which aren't in the set of names for current package
+#
+def update_package_names(args, packages):
+    current_names = set()
+    for arch in packages:
+        current_names.update(packages[arch])
+
+    with connect(args) as conn:
+        conn.row_factory = sqlite3.Row
+
+        cur = conn.execute("SELECT name FROM historic_package_names")
+        historic_names = set([row['name'] for row in cur.fetchall()])
+
+        # add newly appearing names to current_names
+        for n in (current_names - historic_names):
+            conn.execute('INSERT INTO historic_package_names (name) VALUES (?)', (n,))
+            logging.debug("package '%s' name is added" % (n))
+
+    # this is data isn't quite perfect for this purpose: it doesn't know about:
+    # - names which the removed package provide:d
+    # - other packages which might provide: the name of a removed package
+    return (historic_names - current_names)
diff --git a/calm/fix-annotate-perl-hint.py b/calm/fix-annotate-perl-hint.py
index 126dd51..6959535 100644
--- a/calm/fix-annotate-perl-hint.py
+++ b/calm/fix-annotate-perl-hint.py
@@ -61,29 +61,23 @@ def fix_one_hint(dirpath, hintfile, tf):
         logging.error('invalid hints %s' % hintfile)
         return
 
+    annotation = ''
     modified = False
 
-    # if no annotation yet ...
-    if 'notes' not in hints:
-        requires = hints.get('requires', '').split()
-        if requires:
-            # is a perl provide is already present in requires?
-            if any(r.startswith('perl5_') for r in requires):
-                return
-
-            # ... otherwise, add a perl annotation
-            if ('perl_base' in requires) or ('perl' in requires):
-                logging.info("%s has perl in requires and no annotations" % (hintfile))
-                hints['notes'] = 'perl5_032'
-                modified = True
+    requires = hints.get('requires', '').split()
+    if requires:
+        # is a perl provide is already present in requires?
+        if any(r.startswith('perl5_') for r in requires):
+            return
 
-    # fix spelling mistake in 5_26 annotation
-    if hints.get('notes', '') == 'perl5_26':
-        hints['notes'] = 'perl5_026'
+        # ... otherwise, add a perl annotation
+        if ('perl_base' in requires) or ('perl' in requires):
+            logging.info("%s has perl but no perl5_nnn in requires" % (hintfile))
+            annotation = 'perl5_032'
 
     # if annotated, check if this package installs into vendor_perl, and if so,
     # add the annotated perl version to requires, if not already present
-    if hints.get('notes', '').startswith('perl5_0'):
+    if annotation:
         ivp = False
         exe = False
 
@@ -98,11 +92,11 @@ def fix_one_hint(dirpath, hintfile, tf):
 
         if ivp or knwn:
             requires = hints.get('requires', '').split()
-            if hints['notes'] not in requires:
-                requires.append(hints['notes'])
+            if annotation not in requires:
+                requires.append(annotation)
                 requires = sorted(requires)
                 modified = True
-                logging.warning("adding %s to requires in %s" % (hints['notes'], hintfile))
+                logging.warning("adding %s to requires in %s" % (annotation, hintfile))
             hints['requires'] = ' '.join(requires)
         else:
             if exe:
diff --git a/calm/package.py b/calm/package.py
index 275967a..6d9901b 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -482,14 +482,18 @@ def sort_key(k):
 #
 # validate the package database
 #
-def validate_packages(args, packages):
+def validate_packages(args, packages, valid_requires_extra=None):
     error = False
 
     if packages is None:
         return False
 
-    # build the set of valid things to requires:
+    # build the set of valid things to requires: etc.
     valid_requires = set()
+
+    if valid_requires_extra:
+        valid_requires.update(valid_requires_extra)
+
     for p in packages:
         valid_requires.add(p)
         for hints in packages[p].version_hints.values():
@@ -1347,6 +1351,9 @@ def stale_packages(packages):
                 if po.tar(v).sourceless:
                     mark = Freshness.conditional
                     break
+            # XXX: initially only apply to obsolete packages
+            if po.obsolete:
+                mark = Freshness.conditional
 
         # mark any versions explicitly listed in the keep: override hint (unconditionally)
         for v in po.override_hints.get('keep', '').split():
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index 0e50046..d8de78d 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -200,10 +200,6 @@ missing_obsolete = {
 }
 
 # provides: which don't exist
-#
-# we need statefullness to remember packages which have been totally expired,
-# rather than having to list them here. (not keeping this list means we can't
-# check for 'obsoletes: typoed-package-name').
 nonexistent_provides = [
     'perl5_026',
     'rdiff-debuginfo',           # not in x86
diff --git a/test/test_calm.py b/test/test_calm.py
index 1703e39..d859008 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -395,7 +395,7 @@ class CalmTest(unittest.TestCase):
 
         args = types.SimpleNamespace()
 
-        for d in ['rel_area', 'homedir', 'vault']:
+        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
@@ -413,11 +413,11 @@ class CalmTest(unittest.TestCase):
         os.system('touch "%s"' % (os.path.join(m_homedir, 'x86', 'release', 'staleversion', '!ready')))
 
         state = calm.calm.CalmState()
-        state.packages = calm.calm.process_relarea(args)
+        state.packages = calm.calm.process_relarea(args, state)
         state.packages = calm.calm.process_uploads(args, state)
         self.assertTrue(state.packages)
 
-        for d in ['rel_area', 'homedir', 'vault']:
+        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
             with self.subTest(directory=d):
                 dirlist = capture_dirtree(getattr(args, d))
                 compare_with_expected_file(self, 'testdata/conflict', dirlist, d)
diff --git a/test/testdata/conflict/htdocs.expected b/test/testdata/conflict/htdocs.expected
new file mode 100644
index 0000000..6d3735d
--- /dev/null
+++ b/test/testdata/conflict/htdocs.expected
@@ -0,0 +1 @@
+{'.': ['calm.db']}
diff --git a/test/testdata/process_arch/htdocs.expected b/test/testdata/process_arch/htdocs.expected
index 947fb4c..c1cb232 100644
--- a/test/testdata/process_arch/htdocs.expected
+++ b/test/testdata/process_arch/htdocs.expected
@@ -1,4 +1,4 @@
-{'.': ['packages.inc', 'src_packages.inc'],
+{'.': ['calm.db', 'packages.inc', 'src_packages.inc'],
  'summary': ['arc-src.html',
              'arc.html',
              'base-cygwin.html',