public inbox for cygwin-apps-cvs@sourceware.org
help / color / mirror / Atom feed
* [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-39-gf5a68e0
@ 2023-08-20 17:44 Jon Turney
0 siblings, 0 replies; only message in thread
From: Jon Turney @ 2023-08-20 17:44 UTC (permalink / raw)
To: cygwin-apps-cvs
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f5a68e0073ccfa5e7b339d77d77376b134961f99
commit f5a68e0073ccfa5e7b339d77d77376b134961f99
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Wed Aug 16 13:51:37 2023 +0100
Also allow announce message to be determined by cygport
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0e2738cbfd177fcb4d0d99f330ea3469e3e052d4
commit 0e2738cbfd177fcb4d0d99f330ea3469e3e052d4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Aug 13 15:20:44 2023 +0100
Try to add relevant changelog excerpt to announce message
Look for a relevant section of changelog in README, between '----'
delimiters, starting with one also containing the version, to add to the
generated announce message.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ac9d311b31da6e64f470a52982e62af825c97c91
commit ac9d311b31da6e64f470a52982e62af825c97c91
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sun Aug 13 14:19:32 2023 +0100
Deploys can now automatically generate an announce email
This is controlled by the 'announce' token.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3d778791f91c1440fb6a5a7418d9549071ee7814
commit 3d778791f91c1440fb6a5a7418d9549071ee7814
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sat Jul 8 13:23:11 2023 +0100
Factor out email sending to utils
Smooth out some issues when --email isn't specified, making args.email
an empty list rather than None, which can't be iterated.
When the To: address is 'debug', dump email to log, rather than stdout.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f2764f165b23b2e9ae03849c84fb7ed310ed12b7
commit f2764f165b23b2e9ae03849c84fb7ed310ed12b7
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue Aug 15 15:13:17 2023 +0100
Explicitly use count keyword argument to re.sub()
This avoids flake B034: sub should pass `count` and `flags` as keyword
arguments to avoid confusion due to unintuitive argument positions.
Diff:
---
calm/buffering_smtp_handler.py | 44 ++++-------------
calm/calm.py | 105 +++++++++++++++++++++++++++++++++++++++--
calm/untest.py | 2 +-
calm/utils.py | 38 +++++++++++++++
calm/version.py | 2 +-
test/test_calm.py | 8 ++--
test/test_entrypoints.py | 4 +-
7 files changed, 157 insertions(+), 46 deletions(-)
diff --git a/calm/buffering_smtp_handler.py b/calm/buffering_smtp_handler.py
index f6d5f34..d0a8119 100644
--- a/calm/buffering_smtp_handler.py
+++ b/calm/buffering_smtp_handler.py
@@ -21,13 +21,11 @@
#
-import email.message
-import email.utils
import logging
import logging.handlers
-import subprocess
from . import common_constants
+from . import utils
class BufferingSMTPHandler(logging.handlers.BufferingHandler):
@@ -60,37 +58,15 @@ class BufferingSMTPHandler(logging.handlers.BufferingHandler):
msg = msg + 'SUMMARY: ' + ', '.join(['%d %s(s)' % (v, k) for (k, v) in summary.items()]) + "\r\n"
- # build the email
- m = email.message.Message()
- m['From'] = self.fromaddr
- m['To'] = ','.join(self.toaddrs)
- m['Reply-To'] = self.replytoaddr
- m['Bcc'] = common_constants.ALWAYS_BCC
- m['Subject'] = self.subject
- m['Message-Id'] = email.utils.make_msgid()
- m['Date'] = email.utils.formatdate()
- m['X-Calm'] = '1'
-
- # use utf-8 only if the message can't be ascii encoded
- charset = 'ascii'
- try:
- msg.encode('ascii')
- except UnicodeError:
- charset = 'utf-8'
- m.set_payload(msg, charset=charset)
-
- # if toaddrs consists of the single address 'debug', just dump the mail we would have sent
- if self.toaddrs == ['debug']:
- print('-' * 40)
- for k in m:
- print('%s: %s' % (k, m[k]))
- print('-' * 40)
- print(msg)
- print('-' * 40)
- else:
- with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', self.fromaddr], stdin=subprocess.PIPE) as p:
- p.communicate(m.as_bytes())
- logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode))
+ hdr = {}
+ hdr['From'] = self.fromaddr
+ hdr['To'] = ','.join(self.toaddrs)
+ hdr['Reply-To'] = self.replytoaddr
+ hdr['Bcc'] = common_constants.ALWAYS_BCC
+ hdr['Subject'] = self.subject
+ hdr['X-Calm-Report'] = '1'
+
+ utils.sendmail(hdr, msg)
self.buffer = []
diff --git a/calm/calm.py b/calm/calm.py
index 7ea4739..45fa09b 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -53,6 +53,7 @@
#
import argparse
+import codecs
import functools
import logging
import lzma
@@ -63,6 +64,8 @@ import sys
import tempfile
import time
+import xtarfile
+
from . import common_constants
from . import db
from . import irk
@@ -167,7 +170,11 @@ def process_uploads(args, state):
def deploy_upload(r):
m = mlist[r.user]
with logfilters.AttrFilter(maint=m.name):
- return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True)
+ announce = ('announce' in r.tokens) and ('noannounce' not in r.tokens)
+ if announce and r.announce:
+ announce = r.announce
+
+ return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True, announce=announce)
scallywag_db.do_deploys(deploy_upload)
@@ -177,7 +184,7 @@ def process_uploads(args, state):
return state.packages
-def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False):
+def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False, announce=False):
# for each arch and noarch
scan_result = {}
success = True
@@ -198,6 +205,10 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru
if success:
success = _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc)
+ # automatically generate announce email if requested
+ if announce and success and any([scan_result[a].to_relarea for a in scan_result]):
+ _announce_upload(args, scan_result, m, announce)
+
# remove upload files on success in homedir, always in stagingdir
for arch in common_constants.ARCHES + ['noarch', 'src']:
if scrub or success:
@@ -210,6 +221,92 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru
return success
+def _announce_upload(args, scan_result, maintainer, announce):
+ srcpkg = None
+ pkglist = set()
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
+ for po in scan_result[arch].packages.values():
+ if po.kind == package.Kind.source:
+ srcpkg = po
+ assert len(po.versions()) == 1
+ version = list(po.versions())[0]
+ ldesc = po.version_hints[version]['ldesc'].strip('"')
+ test = 'test' in po.version_hints[version]
+
+ pkglist.add(po.orig_name)
+
+ if not srcpkg:
+ logging.error("could not locate source package in upload")
+ return
+ logging.debug("source package is %s, version %s, test %s", srcpkg.orig_name, version, test)
+
+ # find source tarfile for this particular package version
+ to = srcpkg.tar(version)
+ tf = to.repopath.abspath(args.rel_area)
+
+ if isinstance(announce, str):
+ # use announce message extracted from cygport, if present
+ cl = announce
+ else:
+ # otherwise, look in the source tar file for one of the files we know
+ # contains an announce message
+ cl = ''
+ with xtarfile.open(tf, mode='r') as a:
+ files = a.getnames()
+ for readme in ['README', srcpkg.orig_name + '.README', 'ANNOUNCE']:
+ fn = srcpkg.orig_name + '-' + version + '.src/' + readme
+ if fn in files:
+ logging.debug("extracting %s from archive for changelog" % readme)
+
+ f = codecs.getreader("utf-8")(a.extractfile(fn))
+
+ # use the contents of an ANNOUNCE file verbatim
+ if readme == 'ANNOUNCE':
+ cl = f.read()
+ break
+
+ # otherwise, extract relevant part of ChangeLog from README
+ # (between one '---- .* <version> ----' and the next '----' line)
+ found = False
+ for l in f:
+ if not found:
+ if l.startswith('----') and (version in l):
+ cl = l
+ found = True
+ else:
+ if l.startswith('----'):
+ break
+ cl = cl + '\n' + l
+
+ break
+
+ # TODO: maybe other mechanisms for getting package ChangeLog?
+ # NEWS inside upstream source tarball?
+
+ # build the email
+ hdr = {}
+ hdr['From'] = maintainer.name + ' via Cygwin package uploader <cygwin-no-reply@cygwin.com>'
+ hdr['To'] = 'cygwin-announce@cygwin.com'
+ hdr['Reply-To'] = 'cygwin@cygwin.com'
+ hdr['Bcc'] = ','.join(maintainer.email)
+ hdr['Subject'] = srcpkg.orig_name + ' ' + version + (' (TEST)' if test else '')
+ hdr['X-Calm-Announce'] = '1'
+
+ msg = '''
+The following packages have been uploaded to the Cygwin distribution:
+
+%s
+
+%s
+
+%s
+''' % ('\n'.join('* ' + p + '-' + version for p in sorted(pkglist)), ldesc, cl)
+
+ # TODO: add an attachment: sha512 hashes of packages, gpg signed?
+
+ utils.sendmail(hdr, msg)
+
+
def _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc):
name = m.name
@@ -301,7 +398,7 @@ def _process_maintainer_uploads(scan_result, args, state, all_packages, m, based
# use merged package list
state.packages[arch] = merged_packages[arch]
- # report what we've done
+ # report what we've done to irc
added = []
for arch in common_constants.ARCHES + ['noarch', 'src']:
added.append('%d (%s)' % (len(scan_result[arch].packages), arch))
@@ -754,7 +851,7 @@ def main():
parser = argparse.ArgumentParser(description='Upset replacement')
parser.add_argument('-d', '--daemon', action='store', nargs='?', const=pidfile_default, help="daemonize (PIDFILE defaults to " + pidfile_default + ")", metavar='PIDFILE')
- parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS')
+ parser.add_argument('--email', action='store', dest='email', nargs='?', default='', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS')
parser.add_argument('--force', action='count', help="force regeneration of static htdocs content", default=0)
parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default)
diff --git a/calm/untest.py b/calm/untest.py
index 7d6d774..7744108 100644
--- a/calm/untest.py
+++ b/calm/untest.py
@@ -53,7 +53,7 @@ def untest(pvr):
content = fh.read()
if re.search(r'^test:', content, re.MULTILINE):
- content = re.sub(r'^test:\s*$', '', content, 0, re.MULTILINE)
+ content = re.sub(r'^test:\s*$', '', content, count=0, flags=re.MULTILINE)
with open(fn, 'w') as fh:
fh.write(content)
diff --git a/calm/utils.py b/calm/utils.py
index 9f75813..47085fd 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -25,6 +25,8 @@
# utility functions
#
+import email.message
+import email.utils
import filecmp
import logging
import os
@@ -158,3 +160,39 @@ def mtime_cache(user_function):
return result
return wrapper
+
+
+def sendmail(hdr, msg):
+ # sending email not enabled
+ if not hdr['To']:
+ return
+
+ # build the email
+ m = email.message.Message()
+
+ for h in hdr:
+ m[h] = hdr[h]
+ m['Message-Id'] = email.utils.make_msgid()
+ m['Date'] = email.utils.formatdate()
+ m['X-Calm'] = '1'
+
+ # use utf-8 only if the message can't be ascii encoded
+ charset = 'ascii'
+ try:
+ msg.encode('ascii')
+ except UnicodeError:
+ charset = 'utf-8'
+ m.set_payload(msg, charset=charset)
+
+ # if To: header consists of the single address 'debug', just dump the mail we would have sent
+ if m['To'] == 'debug':
+ logging.debug('-' * 40)
+ for k in m:
+ logging.debug('%s: %s' % (k, m[k]))
+ logging.debug('-' * 40)
+ logging.debug(msg)
+ logging.debug('-' * 40)
+ else:
+ with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', hdr['From']], stdin=subprocess.PIPE) as p:
+ p.communicate(m.as_bytes())
+ logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode))
diff --git a/calm/version.py b/calm/version.py
index c61e8fc..5a4ce84 100644
--- a/calm/version.py
+++ b/calm/version.py
@@ -58,7 +58,7 @@ class SetupVersion:
setattr(self, i, split[j])
sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j])
sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))]
- sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences]
+ sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), count=1) for m in sequences]
setattr(self, '_' + i, sequences)
def __str__(self):
diff --git a/test/test_calm.py b/test/test_calm.py
index 11697dc..819ce64 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -366,8 +366,8 @@ class CalmTest(unittest.TestCase):
with open(args.inifile) as inifile:
results = inifile.read()
# fix the timestamp to match expected
- results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, 1)
- results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, 1)
+ results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, count=1)
+ results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, count=1)
compare_with_expected_file(self, 'testdata/inifile', (results,), 'setup.ini')
# XXX: delete a needed package, and check validate fails
@@ -453,8 +453,8 @@ class CalmTest(unittest.TestCase):
with open(os.path.join(args.rel_area, 'setup.ini')) as inifile:
results = inifile.read()
# fix the timestamp to match expected
- results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, 1)
- results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1)
+ results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, count=1)
+ results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, count=1)
compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini')
for d in ARGDIRS:
diff --git a/test/test_entrypoints.py b/test/test_entrypoints.py
index 82b1f23..030ef96 100644
--- a/test/test_entrypoints.py
+++ b/test/test_entrypoints.py
@@ -67,8 +67,8 @@ class EntryPointsTest(unittest.TestCase):
results = inifile.read()
# fix the timestamp to match expected
- results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, 1)
- results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, 1)
+ results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, count=1)
+ results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, count=1)
compare_with_expected_file(self, 'testdata/mksetupini', results, 'setup.ini')
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2023-08-20 17:44 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-08-20 17:44 [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-39-gf5a68e0 Jon Turney
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).