From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.129.124]) by sourceware.org (Postfix) with ESMTPS id 585BB3858D3C for ; Tue, 19 Apr 2022 19:01:22 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 585BB3858D3C Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-515-evgfFbuOOUG5UcYcuh3Y3w-1; Tue, 19 Apr 2022 15:01:18 -0400 X-MC-Unique: evgfFbuOOUG5UcYcuh3Y3w-1 Received: from smtp.corp.redhat.com (int-mx08.intmail.prod.int.rdu2.redhat.com [10.11.54.8]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 0A0F286B8A8 for ; Tue, 19 Apr 2022 19:01:18 +0000 (UTC) Received: from guittard.uglyboxes.com (unknown [10.2.16.83]) by smtp.corp.redhat.com (Postfix) with ESMTP id A8CD0C27E90; Tue, 19 Apr 2022 19:01:17 +0000 (UTC) From: Keith Seitz To: bunsen@sourceware.org Subject: [RFC] GDB sanity test Date: Tue, 19 Apr 2022 12:01:17 -0700 Message-Id: <20220419190117.903678-1-keiths@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.85 on 10.11.54.8 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII"; x-default=true X-Spam-Status: No, score=-11.4 required=5.0 tests=BAYES_00, DKIMWL_WL_HIGH, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, RCVD_IN_DNSWL_LOW, RCVD_IN_MSPIKE_H4, RCVD_IN_MSPIKE_WL, SPF_HELO_NONE, SPF_NONE, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.4 X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on server2.sourceware.org X-BeenThere: bunsen@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Bunsen mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 19 Apr 2022 19:01:24 -0000 This RFC is meant to start a discussion about testing. As bunsen starts to mature and more people rely on it, it is imperative that contributions don't randomly break other projects. In that vein, this is a basic sanity test that I've written for GDB which imports a single gdb.{log,sum} from a test fixture (.tar.xz). It double-checks all results and test counts. To do this, I've written a custom gdb test script that outputs every valid test outcome and run this on a fedora 33 VM. I'm only including the actual test file, not the glue code (tests/core.py) necessary to make it work. Does this look like an acceptable approach? Keith --- tests/bunsen/test_init.py | 11 +++ tests/core.py | 43 +++++++++ .../test_gdb_sanity/gdb-sanity.tar.xz | Bin 0 -> 2120 bytes .../fixtures/test_gdb_sanity/test_outcomes.c | 24 +++++ .../test_gdb_sanity/test_outcomes.exp | 50 +++++++++++ tests/gdb/test_gdb_sanity.py | 82 ++++++++++++++++++ 6 files changed, 210 insertions(+) create mode 100644 tests/bunsen/test_init.py create mode 100644 tests/core.py create mode 100644 tests/gdb/fixtures/test_gdb_sanity/gdb-sanity.tar.xz create mode 100644 tests/gdb/fixtures/test_gdb_sanity/test_outcomes.c create mode 100644 tests/gdb/fixtures/test_gdb_sanity/test_outcomes.exp create mode 100644 tests/gdb/test_gdb_sanity.py diff --git a/tests/gdb/test_gdb_sanity.py b/tests/gdb/test_gdb_sanity.py new file mode 100644 index 0000000..1ce65be --- /dev/null +++ b/tests/gdb/test_gdb_sanity.py @@ -0,0 +1,82 @@ +# A quick sanity test that the ensures that bunsen can import a really +# basic log/sum test run which contains all test outcomes. + +import os.path +from tests.core import setup_bunsen_repo, import_results + +TEST_FIXTURE = 'tests/gdb/fixtures/test_gdb_sanity/gdb-sanity.tar.xz' +PATH_STRING = '/home/fedora33/work/gdb/fsf/virgin/linux/gdb/testsuite' + +def assert_test_name(test, subtest): + assert test['name'] == 'gdb.gdb/test_outcomes.exp' + assert test['subtest'] == subtest + +def print_test(test): + print(f'{test["outcome"]}: {test["name"]}: {test["subtest"]}') + +def test_gdb_sanity(tmp_path): + # Initialize a new bunsen instance. + instance = setup_bunsen_repo(tmp_path) + assert instance is not None + + # Make sure the fixture for this test exists. + fixture = os.path.join(os.getcwd(), TEST_FIXTURE) + assert os.path.exists(fixture) + + # Import the results. + commits = import_results(fixture, instance) + assert commits is not None + assert len(commits) == 1 + + # Print out the commit info. + canonical, commit = commits[0] + print(f'successfully imported test fixture, commit {commit}') + print(f'(canonically {canonical}') + + # Get all test results and info strings. + testrun = instance.testrun(commit) + all_tests = testrun.testcases + info = testrun.get_info_strings() + + # Verify metadata. + assert info["branch"] == 'f33' + assert info["architecture"] == 'x86_64' + assert info['target_board'] == 'native-gdbserver/-m32' + assert testrun.source_commit == '1234567890123456789012345678901234567890' + assert info['version'] == '13.0.50.20220412-git' + + # Verify the tests. First the easy ones with one result each. + for outcome in {'FAIL', 'XPASS', 'XFAIL', 'KPASS', 'KFAIL', + 'UNTESTED', 'UNRESOLVED', 'UNSUPPORTED'}: + test = [t for t in all_tests if t['outcome'] == outcome] + assert len(test) == 1 + test = test[0] + print_test(test) + assert_test_name(test, 'expect ' + outcome) + + # PATH should also have one result. + test = [t for t in all_tests if t['outcome'] == 'PATH'] + assert len(test) == 1 + print_test(test[0]) + #assert_test_name(test[0], PATH_STRING) + + # PASS should have 3 results. One of these will be a duplicate, one will + # contain only a path name (which elicits GDB's 'PATH' outcome). + passes = [t for t in all_tests if t['outcome'] == 'PASS'] + assert len(passes) == 3 + + # Verify the PASS result resulting in a PATH outcome. + path = [t for t in passes if t['subtest'].startswith('expect PATH')] + assert len(path) == 1 + path = path[0] + print_test(path) + assert_test_name(path, 'expect PATH ' + PATH_STRING) + + # Now verify the remaining two PASS results, which have duplicate "subtest" + # names. The parser will append " <<2>>" for the duplicate to keep subtest + # names unique. + passes.remove(path) + print_test(passes[0]) + assert_test_name(passes[0], 'expect PASS') + print_test(passes[1]) + assert_test_name(passes[1], 'expect PASS <<2>>') \ No newline at end of file -- 2.35.1