public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-15-g8b4211b
@ 2022-02-02 15:59 Jon TURNEY
  0 siblings, 0 replies; only message in thread
From: Jon TURNEY @ 2022-02-02 15:59 UTC (permalink / raw)
  To: cygwin-apps-cvs




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8b4211b3a103128ef6743f205418457745a4fc02

commit 8b4211b3a103128ef6743f205418457745a4fc02
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Jan 31 18:59:52 2022 +0000

    Include upstream version in 'unmaintained packages' report
    
    Fetch the upstream version from repology, and in include it in the
    'unmaintained packages' report.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=e2f6ae6c8e084e4eb77ec55f96b32409314dd563

commit e2f6ae6c8e084e4eb77ec55f96b32409314dd563
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Jan 31 15:36:57 2022 +0000

    Add 'deprecated shared libraries' report

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f81674c2f385bb6803dea509a8463bcb0abd789e

commit f81674c2f385bb6803dea509a8463bcb0abd789e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Jan 29 18:11:46 2022 +0000

    Add 'unmaintained packages' report

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=88b57728790ae1556d545697c15f315dfa182e75

commit 88b57728790ae1556d545697c15f315dfa182e75
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Jan 29 15:56:21 2022 +0000

    Don't emit depends: etc. when there's no install package
    
    Don't emit useless depends:, obsoletes:, provides: or conflicts: lines
    when there's no install package for this version, only a source package.


Diff:
---
 calm/calm.py     |  17 +++--
 calm/package.py  |  41 ++++++++----
 calm/pkg2html.py |   9 ++-
 calm/repology.py |  97 +++++++++++++++++++++++++++
 calm/reports.py  | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 344 insertions(+), 16 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index cfc05dd..c76ed41 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -71,6 +71,8 @@ from . import irk
 from . import maintainers
 from . import package
 from . import pkg2html
+from . import repology
+from . import reports
 from . import setup_exe
 from . import uploads
 from . import utils
@@ -278,6 +280,8 @@ def process(args, state):
 
         state.packages = process_uploads(args, state)
 
+    repology.annotate_packages(args, state.packages)
+
     return state.packages
 
 
@@ -382,10 +386,6 @@ def do_output(args, state):
     # update packages listings
     # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
     pkg2html.update_package_listings(args, state.packages)
-    # if we are daemonized, allow force regeneration of static content in htdocs
-    # initially (in case the generation code has changed), but update that
-    # static content only as needed on subsequent loops
-    args.force = 0
 
     update_json = False
 
@@ -482,6 +482,15 @@ def do_output(args, state):
         except (OSError):
             pass
 
+    # write reports
+    if update_json or args.force:
+        reports.do_reports(args, state.packages)
+
+    # if we are daemonized, allow force regeneration of static content in htdocs
+    # initially (in case the generation code has changed), but update that
+    # static content only as needed on subsequent loops
+    args.force = 0
+
 
 #
 # daemonization loop
diff --git a/calm/package.py b/calm/package.py
index 947d06b..9fd734c 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -495,8 +495,10 @@ def validate_packages(args, packages):
         for hints in packages[p].version_hints.values():
             valid_requires.update(hints.get('provides', '').split())
 
-            # reset obsolete:d by some other package state
+            # reset computed package state
             packages[p].obsolete = False
+            packages[p].rdepends = set()
+            packages[p].build_rdepends = set()
 
     # perform various package validations
     for p in sorted(packages.keys()):
@@ -762,6 +764,22 @@ def validate_packages(args, packages):
                         lvl = logging.ERROR
                     logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr))
 
+    # build the set of packages which depends: on this package (rdepends), and
+    # the set of packages which build-depends: on it (build_rdepends)
+    for p in packages:
+        for hints in packages[p].version_hints.values():
+            for k, a in [
+                    ('depends', 'rdepends'),
+                    ('build-depends', 'build_rdepends')
+            ]:
+                if k in hints:
+                    dpl = hints[k].split(',')
+                    for dp in dpl:
+                        dp = dp.strip()
+                        dp = re.sub(r'(.*)\s+\(.*\)', r'\1', dp)
+                        if dp in packages:
+                            getattr(packages[dp], a).add(p)
+
     # make another pass to verify a source tarfile exists for every install
     # tarfile version
     for p in packages.keys():
@@ -1100,11 +1118,18 @@ def write_setup_ini(args, packages, arch):
                 # which case we should emit a 'Source:' line, and the package is
                 # also itself emitted.
 
-                if hints.get('depends', '') or requires:
-                    print("depends2: %s" % hints.get('depends', ''), file=f)
+                if version in po.versions():
+                    if hints.get('depends', '') or requires:
+                        print("depends2: %s" % hints.get('depends', ''), file=f)
 
-                if hints.get('obsoletes', ''):
-                    print("obsoletes: %s" % hints['obsoletes'], file=f)
+                    if hints.get('obsoletes', ''):
+                        print("obsoletes: %s" % hints['obsoletes'], file=f)
+
+                    if hints.get('provides', ''):
+                        print("provides: %s" % hints['provides'], file=f)
+
+                    if hints.get('conflicts', ''):
+                        print("conflicts: %s" % hints['conflicts'], file=f)
 
                 if s:
                     src_hints = packages[s].version_hints.get(version, {})
@@ -1120,12 +1145,6 @@ def write_setup_ini(args, packages, arch):
                     if bd:
                         print("build-depends: %s" % ', '.join(bd), file=f)
 
-                if hints.get('provides', ''):
-                    print("provides: %s" % hints['provides'], file=f)
-
-                if hints.get('conflicts', ''):
-                    print("conflicts: %s" % hints['conflicts'], file=f)
-
 
 # helper function to output details for a particular tar file
 def tar_line(p, category, v, f):
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index b43ed71..b2080dc 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -119,6 +119,13 @@ def ensure_dir_exists(args, path):
         os.chmod(path, 0o755)
 
 
+#
+# format a unix epoch time (UTC)
+#
+def tsformat(ts):
+    return time.strftime('%Y-%m-%d %H:%M', time.gmtime(ts))
+
+
 #
 #
 #
@@ -280,7 +287,7 @@ def update_package_listings(args, packages):
                                     name = v + ' (source)'
                                     target = "%s-%s-src" % (p.orig_name, v)
                                 test = 'test' if 'test' in p.version_hints[v] else 'stable'
-                                ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v).mtime))
+                                ts = tsformat(p.tar(v).mtime)
                                 print('<tr><td>%s</td><td class="right">%d KiB</td><td>%s</td><td>[<a href="../%s/%s/%s">list of files</a>]</td><td>%s</td></tr>' % (name, size, ts, arch, pn, target, test), file=f)
 
                             for version in sorted(packages[arch][p].versions(), key=lambda v: SetupVersion(v)):
diff --git a/calm/repology.py b/calm/repology.py
new file mode 100644
index 0000000..2e19aba
--- /dev/null
+++ b/calm/repology.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+#
+# get 'latest upstream version' information from repology.org (note that this
+# may not exist for some packages, e.g. where upstream doesn't do releases)
+#
+
+import json
+import logging
+import time
+import urllib.request
+
+REPOLOGY_API_URL = 'https://repology.org/api/v1/projects/'
+last_check = 0
+
+
+def repology_fetch_versions():
+    upstream_versions = {}
+    last_pn = ''
+
+    while True:
+        url = REPOLOGY_API_URL
+        if last_pn:
+            url = url + last_pn + '/'
+        url += '?inrepo=cygwin'
+
+        r = urllib.request.urlopen(url)
+        j = json.loads(r.read().decode('utf-8'))
+
+        for pn in sorted(j.keys()):
+            p = j[pn]
+
+            # first, pick out the version which repology has called newest
+            newest_version = None
+            for i in p:
+                if i['status'] == 'newest':
+                    newest_version = i['version']
+                    break
+            else:
+                continue
+
+            # next, assign that version to all the corresponding cygwin source
+            # packages
+            #
+            # (multiple cygwin source packages can correspond to a single
+            # canonical repology package name, e.g. foo and mingww64-arch-foo)
+            for i in p:
+                if i['repo'] == "cygwin":
+                    source_pn = i['srcname']
+                    upstream_versions[source_pn] = newest_version
+
+        if pn == last_pn:
+            break
+        else:
+            last_pn = pn
+
+    return upstream_versions
+
+
+def annotate_packages(args, packages):
+    # rate limit to daily
+    global last_check
+    if (time.time() - last_check) < (24 * 60 * 60):
+        logging.info("not consulting %s due to ratelimit" % (REPOLOGY_API_URL))
+        return
+
+    logging.info("consulting %s" % (REPOLOGY_API_URL))
+    uv = repology_fetch_versions()
+
+    for pn in uv:
+        spn = pn + '-src'
+        for arch in packages:
+            if spn in packages[arch]:
+                packages[arch][spn].upstream_version = uv[pn]
+
+    last_check = time.time()
diff --git a/calm/reports.py b/calm/reports.py
new file mode 100644
index 0000000..bf77f3f
--- /dev/null
+++ b/calm/reports.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import io
+import os
+import re
+import textwrap
+import types
+
+from . import maintainers
+from . import package
+from . import pkg2html
+from . import utils
+from .version import SetupVersion
+
+
+def template(title, body, f):
+    os.fchmod(f.fileno(), 0o755)
+
+    print(textwrap.dedent('''\
+    <!DOCTYPE html>
+    <html>
+    <head>
+    <link rel="stylesheet" type="text/css" href="/static/builds/style.css"/>
+    <title>{0}</title>
+    </head>
+    <body>
+    <div id="main">
+    <h1>{0}</h1>''').format(title), file=f)
+
+    print(body, file=f)
+
+    print(textwrap.dedent('''\
+    </div>
+    </body>
+    </html>'''), file=f)
+
+
+def linkify(pn):
+    return '<a href="/packages/summary/{0}-src.html">{0}</a>'.format(pn)
+
+
+#
+# produce a report of unmaintained packages
+#
+def unmaintained(args, packages, reportsdir):
+    mlist = maintainers.read(args, None)
+    pkg_maintainers = maintainers.invert(mlist)
+
+    um_list = []
+
+    arch = 'x86_64'
+    # XXX: look into how we can make this 'src', after x86 is dropped
+    for p in packages[arch]:
+        po = packages[arch][p]
+
+        if po.kind != package.Kind.source:
+            continue
+
+        if 'ORPHANED' not in pkg_maintainers[po.orig_name]:
+            continue
+
+        # the highest version we have
+        v = sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True)[0]
+
+        # determine the number of unique rdepends over all subpackages (and
+        # likewise build_rdepends)
+        #
+        # zero rdepends makes this package a candidate for removal, whereas lots
+        # means it's important to update it.
+        rdepends = set()
+        build_rdepends = set()
+        for subp in po.is_used_by:
+            rdepends.update(packages[arch][subp].rdepends)
+            build_rdepends.update(packages[arch][subp].build_rdepends)
+
+        up = types.SimpleNamespace()
+        up.po = po
+        up.v = v
+        up.upstream_v = getattr(po, 'upstream_version', 'unknown')
+        up.ts = po.tar(v).mtime
+        up.rdepends = len(rdepends)
+        up.build_rdepends = len(build_rdepends)
+
+        # some packages are mature. If 'v' is still latest upstream version,
+        # then maybe we don't need to worry about this package quite as much...
+        if SetupVersion(v)._V == SetupVersion(up.upstream_v)._V:
+            up.upstream_v += " (unchanged)"
+
+        um_list.append(up)
+
+    body = io.StringIO()
+    print('<p>Packages without a maintainer.</p>', file=body)
+
+    print('<table class="grid">', file=body)
+    print('<tr><th>last updated</th><th>package</th><th>version</th><th>upstream version</th><th>rdepends</th><th>build_rdepends</th></tr>', file=body)
+
+    for up in sorted(um_list, key=lambda i: (i.rdepends + i.build_rdepends, i.ts), reverse=True):
+        po = up.po
+        print('<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' %
+              (pkg2html.tsformat(up.ts), linkify(po.orig_name), up.v, up.upstream_v, up.rdepends, up.build_rdepends), file=body)
+
+    print('</table>', file=body)
+
+    unmaintained = os.path.join(reportsdir, 'unmaintained.html')
+    with utils.open_amifc(unmaintained) as f:
+        template('Unmaintained packages', body.getvalue(), f)
+
+
+# produce a report of deprecated packages
+#
+def deprecated(args, packages, reportsdir):
+    dep_list = []
+
+    arch = 'x86_64'
+    # XXX: look into how we can make this 'src', after x86 is dropped
+    for p in packages[arch]:
+        po = packages[arch][p]
+
+        if po.kind != package.Kind.binary:
+            continue
+
+        if not re.match(r'^lib.*\d', p):
+            continue
+
+        bv = po.best_version
+        es = po.version_hints[bv].get('external-source', None)
+        if not es:
+            continue
+
+        if packages[arch][es].best_version == bv:
+            continue
+
+        if po.tar(bv).is_empty:
+            continue
+
+        # an old version of a shared library
+        depp = types.SimpleNamespace()
+        depp.pn = p
+        depp.po = po
+        depp.v = bv
+        depp.ts = po.tar(bv).mtime
+        depp.rdepends = len(po.rdepends)
+
+        dep_list.append(depp)
+
+    body = io.StringIO()
+    print(textwrap.dedent('''\
+    <p>Packages for old soversions. (The corresponding source package produces a
+    newer soversion, or has stopped producing this soversion).</p>'''), file=body)
+
+    print('<table class="grid">', file=body)
+    print('<tr><th>package</th><th>version</th><th>rdepends</th></tr>', file=body)
+
+    for depp in sorted(dep_list, key=lambda i: (i.rdepends, i.ts), reverse=True):
+        po = depp.po
+        print('<tr><td>%s</td><td>%s</td><td>%s</td></tr>' %
+              (linkify(depp.pn), depp.v, depp.rdepends), file=body)
+
+    print('</table>', file=body)
+
+    deprecated = os.path.join(reportsdir, 'deprecated_so.html')
+    with utils.open_amifc(deprecated) as f:
+        template('Deprecated shared library packages', body.getvalue(), f)
+
+
+#
+def do_reports(args, packages):
+    if args.dryrun:
+        return
+
+    reportsdir = os.path.join(args.htdocs, 'reports')
+    pkg2html.ensure_dir_exists(args, reportsdir)
+
+    unmaintained(args, packages, reportsdir)
+    deprecated(args, packages, reportsdir)



^ permalink raw reply	[flat|nested] only message in thread

only message in thread, other threads:[~2022-02-02 15:59 UTC | newest]

Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-02-02 15:59 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-15-g8b4211b Jon TURNEY

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).