From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-lj1-x22e.google.com (mail-lj1-x22e.google.com [IPv6:2a00:1450:4864:20::22e]) by sourceware.org (Postfix) with ESMTPS id EC4F6385772A for ; Fri, 2 Jun 2023 15:21:16 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.2 sourceware.org EC4F6385772A Authentication-Results: sourceware.org; dmarc=pass (p=none dis=none) header.from=linaro.org Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=linaro.org Received: by mail-lj1-x22e.google.com with SMTP id 38308e7fff4ca-2b072753301so2343851fa.1 for ; Fri, 02 Jun 2023 08:21:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; t=1685719275; x=1688311275; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=muTDMilMZzhrMEa98ml7GPANVoRgx7x3w3gZEa4UzVk=; b=XQuh3H6+J/DnVDv0MbmaD6UhK0xWwmXYh6SyarbklZBKeQAovNmRr0KqGtgMm+SSyU DnIl0ycl5x8U9z0fLT/ZTP7NLx/XNhOk+ttWTgnq9PwoiYn3cOcrmjst28wGj2ZLtqlg pqbcDGydTPNMsdatBDhYwwfbC1KOOl9eP6klGGx/itZhjATk2Ql+x5zVvDte2hbrbgdB eMDkd/0jL5HG8hhP4+A4fgaNIEoEtBxS8UOHmDfyCwLu7fXK7OeESw6HfokNTbzGoR88 o50Cd5F2e7CK2RBtwUaeGHkVsauLjpy2UL8vWXy4wlrD8JGWD2mrKw/eh/RGFryz9TZ3 H99A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1685719275; x=1688311275; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=muTDMilMZzhrMEa98ml7GPANVoRgx7x3w3gZEa4UzVk=; b=U1tuhtfGmIvbj6idJPTXdkSmZOypKTQDUvv2tuwUyQ/JINUKAWBW3edDV35JzyIaC7 LuZMH+jlEyoRl/Kv7FR3pjyjCdiOdX5PBVe/u2prIERSl8ckmt+VlBw36b6a8iJhwn0W /qJ77k9T7fxzqVxvckj0R5vOQ4PzFS2qa/VtUDThGsPiVm1q+p/CxG6aTRsaPUacDkDQ 1O9LO6AVg/ew44jcO5P2ekGHYILWGjKCSp1CofYOYisr6EzeuGX26A2V38bWr4xtVlCv H5hBk+9ZBODvOE4yI0KdKl3Mws3YzafL0X836+Q5JJRrOI0Mr1ggY2rD7qQeoZka9ax7 LNjg== X-Gm-Message-State: AC+VfDxlGxlsvDaT7K9urwGyn4DQmJbtkFV1s8hjLcO5arBrWJCP9wlh 1grOKOSKQF416aGX6JZMePNAih3J3dGiBwZleQx9 X-Google-Smtp-Source: ACHHUZ4c2Juh7wX7niJWYNuoxDs4fLJ3QcPIdN0mFeDIjZJz3+ubkRh1gOAsrFV6M3B9uiNRYS1B0w== X-Received: by 2002:a2e:a4ac:0:b0:2b1:a69e:6a8e with SMTP id g12-20020a2ea4ac000000b002b1a69e6a8emr2215637ljm.3.1685719274840; Fri, 02 Jun 2023 08:21:14 -0700 (PDT) Received: from localhost.localdomain (static.225.72.216.95.clients.your-server.de. [95.216.72.225]) by smtp.gmail.com with ESMTPSA id a18-20020a2eb172000000b002a8ac166e55sm255122ljm.44.2023.06.02.08.21.13 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Jun 2023 08:21:13 -0700 (PDT) From: Maxim Kuvyrkov To: gcc-patches@gcc.gnu.org Cc: Diego Novillo , Doug Evans , Maxim Kuvyrkov Subject: [PATCH 01/12] [contrib] validate_failures.py: Avoid testsuite aliasing Date: Fri, 2 Jun 2023 15:20:41 +0000 Message-Id: <20230602152052.1874860-2-maxim.kuvyrkov@linaro.org> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230602152052.1874860-1-maxim.kuvyrkov@linaro.org> References: <20230602152052.1874860-1-maxim.kuvyrkov@linaro.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-11.7 required=5.0 tests=BAYES_00,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,GIT_PATCH_0,RCVD_IN_DNSWL_NONE,SPF_HELO_NONE,SPF_PASS,TXREP,T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org List-Id: This patch adds tracking of current testsuite "tool" and "exp" to the processing of .sum files. This avoids aliasing between tests from different testsuites with same name+description. E.g., this is necessary for testsuite/c-c++-common, which is ran for both gcc and g++ "tools". This patch changes manifest format from ... FAIL: gcc_test FAIL: g++_test ... to ... === gcc tests === Running gcc/foo.exp ... FAIL: gcc_test === gcc Summary == === g++ tests === Running g++/bar.exp ... FAIL: g++_test === g++ Summary == . The new format uses same formatting as DejaGnu's .sum files to specify which "tool" and "exp" the test belongs to. --- .../testsuite-management/validate_failures.py | 137 +++++++++++++++--- 1 file changed, 115 insertions(+), 22 deletions(-) diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py index 43d9d50af8d..94ba2e58b51 100755 --- a/contrib/testsuite-management/validate_failures.py +++ b/contrib/testsuite-management/validate_failures.py @@ -64,6 +64,16 @@ import sys _VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ] _VALID_TEST_RESULTS_REX = re.compile("%s" % "|".join(_VALID_TEST_RESULTS)) +# Formats of .sum file sections +_TOOL_LINE_FORMAT = '\t\t=== %s tests ===\n' +_EXP_LINE_FORMAT = '\nRunning %s ...\n' +_SUMMARY_LINE_FORMAT = '\n\t\t=== %s Summary ===\n' + +# ... and their compiled regexs. +_TOOL_LINE_REX = re.compile('^\t\t=== (.*) tests ===\n') +_EXP_LINE_REX = re.compile('^Running (.*\.exp) \.\.\.\n') +_SUMMARY_LINE_REX = re.compile('^\t\t=== (.*) Summary ===\n') + # Subdirectory of srcdir in which to find the manifest file. _MANIFEST_SUBDIR = 'contrib/testsuite-management' @@ -111,9 +121,11 @@ class TestResult(object): ordinal: Monotonically increasing integer. It is used to keep results for one .exp file sorted by the order the tests were run. + tool: Top-level testsuite name (aka "tool" in DejaGnu parlance) of the test. + exp: Name of .exp testsuite file. """ - def __init__(self, summary_line, ordinal=-1): + def __init__(self, summary_line, ordinal, tool, exp): try: (self.attrs, summary_line) = SplitAttributesFromSummaryLine(summary_line) try: @@ -125,6 +137,12 @@ class TestResult(object): print('Failed to parse summary line: "%s"' % summary_line) raise self.ordinal = ordinal + if tool == None or exp == None: + # .sum file seem to be broken. There was no "tool" and/or "exp" + # lines preceding this result. + raise + self.tool = tool + self.exp = exp except ValueError: Error('Cannot parse summary line "%s"' % summary_line) @@ -133,14 +151,27 @@ class TestResult(object): self.state, summary_line, self)) def __lt__(self, other): - return (self.name < other.name or - (self.name == other.name and self.ordinal < other.ordinal)) + if (self.tool != other.tool): + return self.tool < other.tool + if (self.exp != other.exp): + return self.exp < other.exp + if (self.name != other.name): + return self.name < other.name + return self.ordinal < other.ordinal def __hash__(self): - return hash(self.state) ^ hash(self.name) ^ hash(self.description) - + return (hash(self.state) ^ hash(self.tool) ^ hash(self.exp) + ^ hash(self.name) ^ hash(self.description)) + + # Note that we don't include "attrs" in this comparison. This means that + # result entries "FAIL: test" and "flaky | FAIL: test" are considered + # the same. Therefore the ResultSet will preserve only the first occurence. + # In practice this means that flaky entries should preceed expected fails + # entries. def __eq__(self, other): return (self.state == other.state and + self.tool == other.tool and + self.exp == other.exp and self.name == other.name and self.description == other.description) @@ -174,6 +205,43 @@ class TestResult(object): return now > expiration_date +class ResultSet(set): + """Describes a set of DejaGNU test results. + This set can be read in from .sum files or emitted as a manifest. + + Attributes: + current_tool: Name of the current top-level DejaGnu testsuite. + current_exp: Name of the current .exp testsuite file. + """ + + def __init__(self): + super().__init__() + self.ResetToolExp() + + def ResetToolExp(self): + self.current_tool = None + self.current_exp = None + + def MakeTestResult(self, summary_line, ordinal=-1): + return TestResult(summary_line, ordinal, + self.current_tool, self.current_exp) + + def Print(self, outfile=sys.stdout): + current_tool = None + current_exp = None + + for result in sorted(self): + if current_tool != result.tool: + current_tool = result.tool + outfile.write(_TOOL_LINE_FORMAT % current_tool) + if current_exp != result.exp: + current_exp = result.exp + outfile.write(_EXP_LINE_FORMAT % current_exp) + outfile.write('%s\n' % result) + + outfile.write(_SUMMARY_LINE_FORMAT % 'Results') + + def GetMakefileValue(makefile_name, value_name): if os.path.exists(makefile_name): makefile = open(makefile_name, encoding='latin-1', mode='r') @@ -216,6 +284,21 @@ def IsInterestingResult(line): return bool(_VALID_TEST_RESULTS_REX.match(line)) +def IsToolLine(line): + """Return True if line mentions the tool (in DejaGnu terms) for the following tests.""" + return bool(_TOOL_LINE_REX.match(line)) + + +def IsExpLine(line): + """Return True if line mentions the .exp file for the following tests.""" + return bool(_EXP_LINE_REX.match(line)) + + +def IsSummaryLine(line): + """Return True if line starts .sum footer.""" + return bool(_SUMMARY_LINE_REX.match(line)) + + def IsInclude(line): """Return True if line is an include of another file.""" return line.startswith("@include ") @@ -244,18 +327,24 @@ def ParseManifestWorker(result_set, manifest_path): if _OPTIONS.verbosity >= 1: print('Parsing manifest file %s.' % manifest_path) manifest_file = open(manifest_path, encoding='latin-1', mode='r') - for line in manifest_file: - line = line.strip() + for orig_line in manifest_file: + line = orig_line.strip() if line == "": pass elif IsComment(line): pass elif IsNegativeResult(line): - result_set.remove(TestResult(GetNegativeResult(line))) + result_set.remove(result_set.MakeTestResult(GetNegativeResult(line))) elif IsInclude(line): ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path)) elif IsInterestingResult(line): - result_set.add(TestResult(line)) + result_set.add(result_set.MakeTestResult(line)) + elif IsExpLine(orig_line): + result_set.current_exp = _EXP_LINE_REX.match(orig_line).groups()[0] + elif IsToolLine(orig_line): + result_set.current_tool = _TOOL_LINE_REX.match(orig_line).groups()[0] + elif IsSummaryLine(orig_line): + result_set.ResetToolExp() else: Error('Unrecognized line in manifest file: %s' % line) manifest_file.close() @@ -263,21 +352,21 @@ def ParseManifestWorker(result_set, manifest_path): def ParseManifest(manifest_path): """Create a set of TestResult instances from the given manifest file.""" - result_set = set() + result_set = ResultSet() ParseManifestWorker(result_set, manifest_path) return result_set def ParseSummary(sum_fname): """Create a set of TestResult instances from the given summary file.""" - result_set = set() + result_set = ResultSet() # ordinal is used when sorting the results so that tests within each # .exp file are kept sorted. ordinal=0 sum_file = open(sum_fname, encoding='latin-1', mode='r') for line in sum_file: if IsInterestingResult(line): - result = TestResult(line, ordinal) + result = result_set.MakeTestResult(line, ordinal) ordinal += 1 if result.HasExpired(): # Tests that have expired are not added to the set of expected @@ -286,6 +375,13 @@ def ParseSummary(sum_fname): print('WARNING: Expected failure "%s" has expired.' % line.strip()) continue result_set.add(result) + elif IsExpLine(line): + result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0] + elif IsToolLine(line): + result_set.current_tool = _TOOL_LINE_REX.match(line).groups()[0] + result_set.current_exp = None + elif IsSummaryLine(line): + result_set.ResetToolExp() sum_file.close() return result_set @@ -301,7 +397,7 @@ def GetManifest(manifest_path): if os.path.exists(manifest_path): return ParseManifest(manifest_path) else: - return set() + return ResultSet() def CollectSumFiles(builddir): @@ -318,7 +414,7 @@ def CollectSumFiles(builddir): def GetResults(sum_files): """Collect all the test results from the given .sum files.""" - build_results = set() + build_results = ResultSet() for sum_fname in sum_files: print('\t%s' % sum_fname) build_results |= ParseSummary(sum_fname) @@ -332,7 +428,7 @@ def CompareResults(manifest, actual): """ # Collect all the actual results not present in the manifest. # Results in this set will be reported as errors. - actual_vs_manifest = set() + actual_vs_manifest = ResultSet() for actual_result in actual: if actual_result not in manifest: actual_vs_manifest.add(actual_result) @@ -341,7 +437,7 @@ def CompareResults(manifest, actual): # in the actual results. # Results in this set will be reported as warnings (since # they are expected failures that are not failing anymore). - manifest_vs_actual = set() + manifest_vs_actual = ResultSet() for expected_result in manifest: # Ignore tests marked flaky. if 'flaky' in expected_result.attrs: @@ -390,9 +486,7 @@ def GetBuildData(): def PrintSummary(msg, summary): print('\n\n%s' % msg) - for result in sorted(summary): - print(result) - + summary.Print() def GetSumFiles(results, build_dir): if not results: @@ -452,9 +546,8 @@ def ProduceManifest(): sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) actual = GetResults(sum_files) manifest_file = open(manifest_path, encoding='latin-1', mode='w') - for result in sorted(actual): - print(result) - manifest_file.write('%s\n' % result) + actual.Print(manifest_file) + actual.Print() manifest_file.close() return True -- 2.34.1