From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [216.205.24.124]) by sourceware.org (Postfix) with ESMTP id C09E63857C52 for ; Thu, 24 Sep 2020 18:54:34 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.3.2 sourceware.org C09E63857C52 Received: from mimecast-mx01.redhat.com (mimecast-mx01.redhat.com [209.132.183.4]) (Using TLS) by relay.mimecast.com with ESMTP id us-mta-99-DGCDeEPoN9yVSVlCA0wPJA-1; Thu, 24 Sep 2020 14:54:27 -0400 X-MC-Unique: DGCDeEPoN9yVSVlCA0wPJA-1 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.phx2.redhat.com [10.5.11.16]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx01.redhat.com (Postfix) with ESMTPS id BD6D7801AC8 for ; Thu, 24 Sep 2020 18:54:26 +0000 (UTC) Received: from theo.uglyboxes.com.com (ovpn-113-168.phx2.redhat.com [10.3.113.168]) by smtp.corp.redhat.com (Postfix) with ESMTP id 901A85C1C7 for ; Thu, 24 Sep 2020 18:54:26 +0000 (UTC) From: Keith Seitz To: bunsen@sourceware.org Subject: [RFC PATCH] New script `summarize' Date: Thu, 24 Sep 2020 11:54:24 -0700 Message-Id: <20200924185424.1496842-1-keiths@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.79 on 10.5.11.16 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII" X-Spam-Status: No, score=-13.5 required=5.0 tests=BAYES_00, DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H5, RCVD_IN_MSPIKE_WL, SPF_HELO_NONE, SPF_PASS, TXREP autolearn=ham autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on server2.sourceware.org X-BeenThere: bunsen@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Bunsen mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 24 Sep 2020 18:54:36 -0000 This patch proposes to add a new `summarize' script which attempts to reproduce the summary which appears at the end of DejaGNU .sum files. While this is largely only useful for test case verification, perhaps others might find it useful, too. The script may take an optional argument to limit the output. This is a comma-separated list of glob expressions, modeled on GDB's TESTS="..." argument to "make check". Example (this is with consolidate_pass=False): $ ./bunsen.py +summarize 969d2e7 Using branch index, checkout name wd-summarize Running summarize at PATH/bunsen/.bunsen/wd-summarize from PATH/bunsen/scripts-master/summarize.py with ['969d2e7'] === Summary for commit 969d2e7 of gdb version 10.0.50.20200826-git from branch master on x86_64 using \# of expected passes 67853 \# of unexpected failures 181 \# of expected failures 78 \# of known failures 111 \# of untested testcases 8 \# of unsupported tests 106 To facilitate these printing-type operations of Testruns, I've added a helper method to Testrun to gather common properties into an "info" dictionary that can be used by callers. This dictionary contains a new "board" property for recording the DejaGNU test board. This property is currently not used and will always be "". Patch coming. Comments? Keith --- bunsen.py | 28 +++++++++++++++ scripts-master/summarize.py | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100755 scripts-master/summarize.py diff --git a/bunsen.py b/bunsen.py index b089d80..99c4cc0 100755 --- a/bunsen.py +++ b/bunsen.py @@ -622,6 +622,34 @@ class Testrun(dict): # XXX Set summary=False if JSON was missing testcases. self.summary = self.summary and 'testcases' in json_data + # Return properties of this Testrun as printable strings, or + # "" if unknown. Returns a dictionary containing + # keys for architecture, board, branch, version. + + def get_info_strings(self): + info = dict() + if 'arch' in self: + info['architecture'] = self.arch + else: + info['architecture'] = '' + + if 'board' in self: + info['board'] = self.board + else: + info['board'] = '' + + if 'source_branch' in self: + info['branch'] = self.source_branch + else: + info['branch'] = '' + + if 'version' in self: + info['version'] = self.version + else: + info['version'] = '' + + return info + def add_testcase(self, name, outcome, **kwargs): ''' Append a testcase result to the Testrun data. diff --git a/scripts-master/summarize.py b/scripts-master/summarize.py new file mode 100755 index 0000000..e46db18 --- /dev/null +++ b/scripts-master/summarize.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Display a DejaGNU-like test summary, given the bunsen commit +# of the desired test run. Optionally also takes a comma-separated +# list of glob expressions to limit results. + +info = "summarize.py [tests]" +cmdline_args = [ + ('commit', None, '', + "commit to fetch results for"), + ('tests', None, '', + "comma-separated list of glob expressions of tests to summarize") +] + +import sys +import bunsen +from collections import Counter +from pathlib import PurePath + +# A list of test outcomes in output order. +outcome_labels = { + 'PASS' : 'expected passes', + 'FAIL' : 'unexpected failures', + 'XPASS' : 'unexpected successes', + 'XFAIL' : 'expected failures', + 'KPASS' : 'unknown successes', + 'KFAIL' : 'known failures', + 'UNTESTED' : 'untested testcases', + 'UNRESOLVED' : 'unresolved testcases', + 'UNSUPPORTED' : 'unsupported tests', + 'ERROR' : 'errors', + 'WARNING' : 'warnings' +} + +if __name__ == '__main__': + b = bunsen.Bunsen() + opts = b.cmdline_args(sys.argv, info=info, args=cmdline_args, + required_args=['commit'], optional_args=['tests']) + + testrun = b.testrun(opts.commit) + all_tests = testrun.testcases + found_tests = [] + if opts.tests is not None: + for glob in opts.tests.split(','): + found_tests.extend([t for t in all_tests if PurePath(t['name']).match(glob)]) + else: + found_tests = all_tests + + if found_tests: + info = testrun.get_info_strings() + + project = b.tags[0] if len(b.tags) == 1 else '' + print(f'Summary for commit {opts.commit} of {project} version {info["version"]}') + print(f'from branch {info["branch"]} on {info["architecture"]} using {info["board"]}') + if opts.tests is not None: + print(f'limiting results to tests matching: {opts.tests}') + print() + + # Collate results for outcomes + c = Counter(t['outcome'] for t in found_tests) + + # We could simply loop over the keys of the Counter, but that would not necessarily give + # us the same output order as DejaGNU itself. + for l in outcome_labels: + if c[l] != 0: + print('# of %-26s %d' % (outcome_labels[l], c[l])) + else: + print(f'found no tests matching \"{opts.tests}\"') -- 2.26.2