From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) by sourceware.org (Postfix) with ESMTPS id 5A12D3850237 for ; Wed, 29 Jun 2022 16:03:20 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 5A12D3850237 Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-454-uwQk35gKMAOtz8aRIIJvDw-1; Wed, 29 Jun 2022 12:03:18 -0400 X-MC-Unique: uwQk35gKMAOtz8aRIIJvDw-1 Received: from smtp.corp.redhat.com (int-mx10.intmail.prod.int.rdu2.redhat.com [10.11.54.10]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 95E961019C88 for ; Wed, 29 Jun 2022 16:03:18 +0000 (UTC) Received: from guittard.redhat.com (unknown [10.2.17.6]) by smtp.corp.redhat.com (Postfix) with ESMTP id 51EC240F06D for ; Wed, 29 Jun 2022 16:03:18 +0000 (UTC) From: Keith Seitz To: bunsen@sourceware.org Subject: [PATCH] bunsenql: Add r-dejagnu-summary Date: Wed, 29 Jun 2022 09:03:17 -0700 Message-Id: <20220629160317.16735-1-keiths@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.85 on 10.11.54.10 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII"; x-default=true X-Spam-Status: No, score=-11.2 required=5.0 tests=BAYES_00, DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, RCVD_IN_DNSWL_NONE, SPF_HELO_NONE, SPF_NONE, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: bunsen@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Bunsen mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 29 Jun 2022 16:03:22 -0000 This patch adds a bunsenql equivalent of my previous "bunsen +summarize" script. This can be used to output DejaGNU-like summaries for tests. This script supports limiting tests based on glob expression (although it does not support multiple glob expressions like +summarize) and "verbose" output which will output all (sub)test results. It supports both text and JSON output templates. The script does not use any of the pipeline-created summaries. It computes results directly from the data. This is useful to verify that results were properly imported and could be used to compare against the DejaGNU .sum file's summary section. Example output: $ # Output text summary of results for "gdb.gdb/selftext.exp": $ r-dejagnu-summary --git $gitrepo --db $gitrepo/bunsen.sqlite3 \ --expfile-glob gdb.gdb/selftest.exp unpatched # of expected passes 6 # of unexpected failures 6 $ # Same -- JSON version $ r-dejagnu-summary --git $gitrepo --db $gitrepo/bunsen.sqlite3 \ --expfile-glob gdb.gdb/selftest.exp --template json unpatched {"FAIL": 6, "PASS": 6} $ r-dejagnu-summary -git $gitrepo --db $gitrepo/bunsen.sqlite3 \ --expfile-glob gdb.gdb/selftest.exp -v unpatched PASS: gdb.gdb/selftest.exp: Set xgdb_prompt FAIL: gdb.gdb/selftest.exp: backtrace through signal handler (timeout) PASS: gdb.gdb/selftest.exp: disassemble main PASS: gdb.gdb/selftest.exp: printed version as pointer PASS: gdb.gdb/selftest.exp: run until breakpoint at captured_main FAIL: gdb.gdb/selftest.exp: send SIGINT signal to child process (timeout) FAIL: gdb.gdb/selftest.exp: send ^C to child process (timeout) FAIL: gdb.gdb/selftest.exp: send ^C to child process again (timeout) PASS: gdb.gdb/selftest.exp: set interrupt character in test_with_self PASS: gdb.gdb/selftest.exp: set listsize to 1 FAIL: gdb.gdb/selftest.exp: thread 1 (timeout) FAIL: gdb.gdb/selftest.exp: xgdb is at prompt === gdb Summary === # of expected passes 6 # of unexpected failures 6 $ # Same -- JSON version $ r-dejagnu-summary --git $gitrepo --db $gitrepo/bunsen.sqlite3 \ --expfile-glob gdb.gdb/selftest.exp -v --template json unpatched|jq { "project": "gdb", "summary": { "FAIL": 6, "PASS": 6 }, "tests": [ { "expfile": "gdb.gdb/selftest.exp", "outcome": "PASS", "subtest": "gdb.gdb/selftest.exp: Set xgdb_prompt" }, { "expfile": "gdb.gdb/selftest.exp", "outcome": "FAIL", "subtest": "gdb.gdb/selftest.exp: backtrace through signal handler (timeout)" }, ... ] } --- bin/r-dejagnu-summary | 159 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100755 bin/r-dejagnu-summary diff --git a/bin/r-dejagnu-summary b/bin/r-dejagnu-summary new file mode 100755 index 0000000..cc56765 --- /dev/null +++ b/bin/r-dejagnu-summary @@ -0,0 +1,159 @@ +#! /usr/bin/python3 + +# This script reads in results from the Bunsen database and outputs a +# DejaGNU-like summary of results. + +import argparse +from tabnanny import verbose +import git +import os +import sqlite3 as lite +import jinja2 +from collections import Counter + +# Query for the database for .exp filename, subtest name, and result for the +# given testrun ID and expfile glob. These are sorted for use by the "verbose" +# case so that two test suite runs may be diff'd. +query = ''' +SELECT + exs.name AS expfile, + sts.name AS subtest, + rs.name AS result +FROM + dejagnu_testcase tc, + dejagnu_testsuite ts +JOIN + dejagnu_string exs ON (expfile = exs.id) +JOIN + dejagnu_string sts ON (subtest = sts.id) +JOIN + dejagnu_string rs ON (result = rs.id) +WHERE + ts.tr IN (?) + AND tc.testsuite = ts.id + AND exs.name GLOB ? +ORDER BY subtest +''' + +# Jinja output templates. +summary_templates = { + 'text' : '''{% for l in outcomes %}{% if counter[l] != 0 %}{{ counter[l]|output_format(l) }}{% endif %}{% endfor %}''', + 'json' : '''{{ counter|tojson|safe }}''' +} +verbose_templates = { + 'text' : '''{% for t in output.tests %}{{t.outcome}}: {{t.subtest}}\n{% endfor %}\n\t\t=== {{ output.project }} Summary ===\n\n{% for l in outcomes %}{% if counter[l] != 0 %}{{ counter[l]|output_format(l) }}{% endif %}{% endfor %}''', + 'json' : '''{{ output|tojson|safe }}''' +} + +# A list of test outcomes in output order. +outcome_labels = { + 'PASS' : 'expected passes', + 'FAIL' : 'unexpected failures', + 'XPASS' : 'unexpected successes', + 'XFAIL' : 'expected failures', + 'KPASS' : 'unknown successes', + 'KFAIL' : 'known failures', + 'UNTESTED' : 'untested testcases', + 'UNRESOLVED' : 'unresolved testcases', + 'UNSUPPORTED' : 'unsupported tests', + 'PATH' : "paths in test names", + # bunsenql does not support DUPLICATE + 'DUPLICATE' : "duplicate test names", + 'ERROR' : 'errors', + 'WARNING' : 'warnings' +} + +# Format the test result labeled LABEL (key of `outcome_labels') with the given +# numerical number of results, COUNT. The jinja template never calls this +# filter with COUNT == 0. +def output_format(count, label): + return '# of %-26s %d\n' % (outcome_labels[label], count) + +# Get the name of the project represented by testrun TRID in the database DB. +# This data can be retrieved from the "TOOL_version" column in the `testrun_kv' table. +def get_project(db, trid): + project = db.execute( + 'SELECT key FROM testrun_kv WHERE tr = (?) AND key LIKE "%_version"', + (trid,)).fetchone()[0] + return project[:-8] + +def main(): + # Create argument parser and add our options. + parser = argparse.ArgumentParser( + description='Summarize results of DejaGNU tests', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--git', type=str, help='git testrun archive', + default='.') + parser.add_argument('--db', type=str, help='sqlite database file', + default='bunsen.sqlite3') + parser.add_argument('--template', type=str, help='jinja template', + default='text') + parser.add_argument('--expfile-glob', type=str, + help='limit results to DejaGNU expfile(s) matching glob pattern', + default='*') + parser.add_argument('-v', '--verbose', action=argparse.BooleanOptionalAction, + help='output verbose test results', default=False) + parser.add_argument('commitish', metavar='COMMITSH', type=str, + help='testrun commit or tag') + + # Parse and verify the command line arguments. + global args + args = parser.parse_args() + if len(args.commitish) < 1: + parser.error("require COMMITISH") + + # Verify output template. + if args.template not in summary_templates: + parser.error(f'Unknown output template `{args.template}\'. Must be one of: {", ".join(jinja_templates.keys())})') + + # Open GIT repo and get the commit of COMMITISH. + try: + testruns = git.Repo(args.git) + except Exception as e: + parser.error(f'Invalid GIT repository `{args.git}\'. Need --git?') + + try: + commit = testruns.commit(args.commitish) + except Exception as e: + parser.error(f'Unknown commitish {args.commitish} in git repo `{args.git}\'') + + # Connect to the database and read in the data from the Bunsen database. + if not os.path.isfile(args.db): + parser.error(f'Database `{args.db}\' not found. Need --db?') + + con = lite.connect(args.db, uri=True) + + # Get the testrun ID that corresponds to COMMITISH. + trid = con.execute( + 'SELECT id FROM testrun WHERE gitcommit = ?', (commit.hexsha,)).fetchone()[0] + + # Grab results and print them out. RESULTS is a list of tuples: (expfile, subtest, outcome) + results = con.execute(query, (trid, args.expfile_glob)).fetchall() + c = Counter(t[2] for t in results) + + project = get_project(con, trid) + jinja_params = { + 'counter': c, + 'outcomes': outcome_labels + } + if args.verbose: + # Output all test results, summary, and project. + tests = [{'expfile' : t[0], 'subtest' : t[1], 'outcome' : t[2]} for t in results] + output_dict = {'project' : project, 'tests' : tests, 'summary' : c} + jinja_params.update({'output' : output_dict}) + output_templates = verbose_templates + else: + # Output only a summary of results. + output_templates = summary_templates + + env = jinja2.Environment(loader=jinja2.DictLoader(output_templates)) + env.filters['output_format'] = output_format + template = env.get_template(args.template) + try: + print(template.render(jinja_params), end='') + except BrokenPipeError: + pass + + +if __name__ == '__main__': + main() -- 2.36.1