From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 2201) id 7D6673857359; Sun, 20 Aug 2023 17:44:48 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 7D6673857359 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1692553488; bh=XhBfauOqCRiwEYoN9Dt/6af/nbRsbGeejOSg/NmDiY4=; h=To:Subject:Date:From:From; b=OPwfgWlu92/ETUnI1h04Q5pwmKXs0mm2GBJklXnJk3iRu5GtKP00lZKtoqPLh3IxK ZuheAIVTd7LN/YQgDWHRuY+tWAK8jh4v32rfHnoeumPpCMWMalMHZV+AAB5bNLm2IT dIFomvlYBkPLsnSiStS0u/KUc6zgdfghxGxNdx9k= To: cygwin-apps-cvs@sourceware.org Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-39-gf5a68e0 X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 68e2201f28238209dc58f81a82ba5e3ccdf4d0f5 X-Git-Newrev: f5a68e0073ccfa5e7b339d77d77376b134961f99 Message-Id: <20230820174448.7D6673857359@sourceware.org> Date: Sun, 20 Aug 2023 17:44:48 +0000 (GMT) From: Jon Turney List-Id: https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f5a68e0073ccfa5e7b339d77d77376b134961f99 commit f5a68e0073ccfa5e7b339d77d77376b134961f99 Author: Jon Turney Date: Wed Aug 16 13:51:37 2023 +0100 Also allow announce message to be determined by cygport https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0e2738cbfd177fcb4d0d99f330ea3469e3e052d4 commit 0e2738cbfd177fcb4d0d99f330ea3469e3e052d4 Author: Jon Turney Date: Sun Aug 13 15:20:44 2023 +0100 Try to add relevant changelog excerpt to announce message Look for a relevant section of changelog in README, between '----' delimiters, starting with one also containing the version, to add to the generated announce message. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ac9d311b31da6e64f470a52982e62af825c97c91 commit ac9d311b31da6e64f470a52982e62af825c97c91 Author: Jon Turney Date: Sun Aug 13 14:19:32 2023 +0100 Deploys can now automatically generate an announce email This is controlled by the 'announce' token. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3d778791f91c1440fb6a5a7418d9549071ee7814 commit 3d778791f91c1440fb6a5a7418d9549071ee7814 Author: Jon Turney Date: Sat Jul 8 13:23:11 2023 +0100 Factor out email sending to utils Smooth out some issues when --email isn't specified, making args.email an empty list rather than None, which can't be iterated. When the To: address is 'debug', dump email to log, rather than stdout. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f2764f165b23b2e9ae03849c84fb7ed310ed12b7 commit f2764f165b23b2e9ae03849c84fb7ed310ed12b7 Author: Jon Turney Date: Tue Aug 15 15:13:17 2023 +0100 Explicitly use count keyword argument to re.sub() This avoids flake B034: sub should pass `count` and `flags` as keyword arguments to avoid confusion due to unintuitive argument positions. Diff: --- calm/buffering_smtp_handler.py | 44 ++++------------- calm/calm.py | 105 +++++++++++++++++++++++++++++++++++++++-- calm/untest.py | 2 +- calm/utils.py | 38 +++++++++++++++ calm/version.py | 2 +- test/test_calm.py | 8 ++-- test/test_entrypoints.py | 4 +- 7 files changed, 157 insertions(+), 46 deletions(-) diff --git a/calm/buffering_smtp_handler.py b/calm/buffering_smtp_handler.py index f6d5f34..d0a8119 100644 --- a/calm/buffering_smtp_handler.py +++ b/calm/buffering_smtp_handler.py @@ -21,13 +21,11 @@ # -import email.message -import email.utils import logging import logging.handlers -import subprocess from . import common_constants +from . import utils class BufferingSMTPHandler(logging.handlers.BufferingHandler): @@ -60,37 +58,15 @@ class BufferingSMTPHandler(logging.handlers.BufferingHandler): msg = msg + 'SUMMARY: ' + ', '.join(['%d %s(s)' % (v, k) for (k, v) in summary.items()]) + "\r\n" - # build the email - m = email.message.Message() - m['From'] = self.fromaddr - m['To'] = ','.join(self.toaddrs) - m['Reply-To'] = self.replytoaddr - m['Bcc'] = common_constants.ALWAYS_BCC - m['Subject'] = self.subject - m['Message-Id'] = email.utils.make_msgid() - m['Date'] = email.utils.formatdate() - m['X-Calm'] = '1' - - # use utf-8 only if the message can't be ascii encoded - charset = 'ascii' - try: - msg.encode('ascii') - except UnicodeError: - charset = 'utf-8' - m.set_payload(msg, charset=charset) - - # if toaddrs consists of the single address 'debug', just dump the mail we would have sent - if self.toaddrs == ['debug']: - print('-' * 40) - for k in m: - print('%s: %s' % (k, m[k])) - print('-' * 40) - print(msg) - print('-' * 40) - else: - with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', self.fromaddr], stdin=subprocess.PIPE) as p: - p.communicate(m.as_bytes()) - logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode)) + hdr = {} + hdr['From'] = self.fromaddr + hdr['To'] = ','.join(self.toaddrs) + hdr['Reply-To'] = self.replytoaddr + hdr['Bcc'] = common_constants.ALWAYS_BCC + hdr['Subject'] = self.subject + hdr['X-Calm-Report'] = '1' + + utils.sendmail(hdr, msg) self.buffer = [] diff --git a/calm/calm.py b/calm/calm.py index 7ea4739..45fa09b 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -53,6 +53,7 @@ # import argparse +import codecs import functools import logging import lzma @@ -63,6 +64,8 @@ import sys import tempfile import time +import xtarfile + from . import common_constants from . import db from . import irk @@ -167,7 +170,11 @@ def process_uploads(args, state): def deploy_upload(r): m = mlist[r.user] with logfilters.AttrFilter(maint=m.name): - return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True) + announce = ('announce' in r.tokens) and ('noannounce' not in r.tokens) + if announce and r.announce: + announce = r.announce + + return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True, announce=announce) scallywag_db.do_deploys(deploy_upload) @@ -177,7 +184,7 @@ def process_uploads(args, state): return state.packages -def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False): +def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False, announce=False): # for each arch and noarch scan_result = {} success = True @@ -198,6 +205,10 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru if success: success = _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc) + # automatically generate announce email if requested + if announce and success and any([scan_result[a].to_relarea for a in scan_result]): + _announce_upload(args, scan_result, m, announce) + # remove upload files on success in homedir, always in stagingdir for arch in common_constants.ARCHES + ['noarch', 'src']: if scrub or success: @@ -210,6 +221,92 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru return success +def _announce_upload(args, scan_result, maintainer, announce): + srcpkg = None + pkglist = set() + for arch in common_constants.ARCHES + ['noarch', 'src']: + for po in scan_result[arch].packages.values(): + if po.kind == package.Kind.source: + srcpkg = po + assert len(po.versions()) == 1 + version = list(po.versions())[0] + ldesc = po.version_hints[version]['ldesc'].strip('"') + test = 'test' in po.version_hints[version] + + pkglist.add(po.orig_name) + + if not srcpkg: + logging.error("could not locate source package in upload") + return + logging.debug("source package is %s, version %s, test %s", srcpkg.orig_name, version, test) + + # find source tarfile for this particular package version + to = srcpkg.tar(version) + tf = to.repopath.abspath(args.rel_area) + + if isinstance(announce, str): + # use announce message extracted from cygport, if present + cl = announce + else: + # otherwise, look in the source tar file for one of the files we know + # contains an announce message + cl = '' + with xtarfile.open(tf, mode='r') as a: + files = a.getnames() + for readme in ['README', srcpkg.orig_name + '.README', 'ANNOUNCE']: + fn = srcpkg.orig_name + '-' + version + '.src/' + readme + if fn in files: + logging.debug("extracting %s from archive for changelog" % readme) + + f = codecs.getreader("utf-8")(a.extractfile(fn)) + + # use the contents of an ANNOUNCE file verbatim + if readme == 'ANNOUNCE': + cl = f.read() + break + + # otherwise, extract relevant part of ChangeLog from README + # (between one '---- .* ----' and the next '----' line) + found = False + for l in f: + if not found: + if l.startswith('----') and (version in l): + cl = l + found = True + else: + if l.startswith('----'): + break + cl = cl + '\n' + l + + break + + # TODO: maybe other mechanisms for getting package ChangeLog? + # NEWS inside upstream source tarball? + + # build the email + hdr = {} + hdr['From'] = maintainer.name + ' via Cygwin package uploader ' + hdr['To'] = 'cygwin-announce@cygwin.com' + hdr['Reply-To'] = 'cygwin@cygwin.com' + hdr['Bcc'] = ','.join(maintainer.email) + hdr['Subject'] = srcpkg.orig_name + ' ' + version + (' (TEST)' if test else '') + hdr['X-Calm-Announce'] = '1' + + msg = ''' +The following packages have been uploaded to the Cygwin distribution: + +%s + +%s + +%s +''' % ('\n'.join('* ' + p + '-' + version for p in sorted(pkglist)), ldesc, cl) + + # TODO: add an attachment: sha512 hashes of packages, gpg signed? + + utils.sendmail(hdr, msg) + + def _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc): name = m.name @@ -301,7 +398,7 @@ def _process_maintainer_uploads(scan_result, args, state, all_packages, m, based # use merged package list state.packages[arch] = merged_packages[arch] - # report what we've done + # report what we've done to irc added = [] for arch in common_constants.ARCHES + ['noarch', 'src']: added.append('%d (%s)' % (len(scan_result[arch].packages), arch)) @@ -754,7 +851,7 @@ def main(): parser = argparse.ArgumentParser(description='Upset replacement') parser.add_argument('-d', '--daemon', action='store', nargs='?', const=pidfile_default, help="daemonize (PIDFILE defaults to " + pidfile_default + ")", metavar='PIDFILE') - parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS') + parser.add_argument('--email', action='store', dest='email', nargs='?', default='', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS') parser.add_argument('--force', action='count', help="force regeneration of static htdocs content", default=0) parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default) parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default) diff --git a/calm/untest.py b/calm/untest.py index 7d6d774..7744108 100644 --- a/calm/untest.py +++ b/calm/untest.py @@ -53,7 +53,7 @@ def untest(pvr): content = fh.read() if re.search(r'^test:', content, re.MULTILINE): - content = re.sub(r'^test:\s*$', '', content, 0, re.MULTILINE) + content = re.sub(r'^test:\s*$', '', content, count=0, flags=re.MULTILINE) with open(fn, 'w') as fh: fh.write(content) diff --git a/calm/utils.py b/calm/utils.py index 9f75813..47085fd 100644 --- a/calm/utils.py +++ b/calm/utils.py @@ -25,6 +25,8 @@ # utility functions # +import email.message +import email.utils import filecmp import logging import os @@ -158,3 +160,39 @@ def mtime_cache(user_function): return result return wrapper + + +def sendmail(hdr, msg): + # sending email not enabled + if not hdr['To']: + return + + # build the email + m = email.message.Message() + + for h in hdr: + m[h] = hdr[h] + m['Message-Id'] = email.utils.make_msgid() + m['Date'] = email.utils.formatdate() + m['X-Calm'] = '1' + + # use utf-8 only if the message can't be ascii encoded + charset = 'ascii' + try: + msg.encode('ascii') + except UnicodeError: + charset = 'utf-8' + m.set_payload(msg, charset=charset) + + # if To: header consists of the single address 'debug', just dump the mail we would have sent + if m['To'] == 'debug': + logging.debug('-' * 40) + for k in m: + logging.debug('%s: %s' % (k, m[k])) + logging.debug('-' * 40) + logging.debug(msg) + logging.debug('-' * 40) + else: + with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', hdr['From']], stdin=subprocess.PIPE) as p: + p.communicate(m.as_bytes()) + logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode)) diff --git a/calm/version.py b/calm/version.py index c61e8fc..5a4ce84 100644 --- a/calm/version.py +++ b/calm/version.py @@ -58,7 +58,7 @@ class SetupVersion: setattr(self, i, split[j]) sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j]) sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))] - sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences] + sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), count=1) for m in sequences] setattr(self, '_' + i, sequences) def __str__(self): diff --git a/test/test_calm.py b/test/test_calm.py index 11697dc..819ce64 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -366,8 +366,8 @@ class CalmTest(unittest.TestCase): with open(args.inifile) as inifile: results = inifile.read() # fix the timestamp to match expected - results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, 1) - results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, 1) + results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, count=1) + results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, count=1) compare_with_expected_file(self, 'testdata/inifile', (results,), 'setup.ini') # XXX: delete a needed package, and check validate fails @@ -453,8 +453,8 @@ class CalmTest(unittest.TestCase): with open(os.path.join(args.rel_area, 'setup.ini')) as inifile: results = inifile.read() # fix the timestamp to match expected - results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, 1) - results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1) + results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, count=1) + results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, count=1) compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini') for d in ARGDIRS: diff --git a/test/test_entrypoints.py b/test/test_entrypoints.py index 82b1f23..030ef96 100644 --- a/test/test_entrypoints.py +++ b/test/test_entrypoints.py @@ -67,8 +67,8 @@ class EntryPointsTest(unittest.TestCase): results = inifile.read() # fix the timestamp to match expected - results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, 1) - results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, 1) + results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, count=1) + results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, count=1) compare_with_expected_file(self, 'testdata/mksetupini', results, 'setup.ini')