From: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
To: gcc-patches@gcc.gnu.org
Cc: Diego Novillo <dnovillo@google.com>, Doug Evans <dje@google.com>,
Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Subject: [PATCH 05/12] [contrib] validate_failures.py: Add more verbosity levels
Date: Fri, 2 Jun 2023 15:20:45 +0000 [thread overview]
Message-ID: <20230602152052.1874860-6-maxim.kuvyrkov@linaro.org> (raw)
In-Reply-To: <20230602152052.1874860-1-maxim.kuvyrkov@linaro.org>
... to control validate_failures.py output
---
.../testsuite-management/validate_failures.py | 82 +++++++++++--------
1 file changed, 46 insertions(+), 36 deletions(-)
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index 1bd09e0c20c..26ea1d6f53b 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -324,7 +324,7 @@ def GetNegativeResult(line):
def ParseManifestWorker(result_set, manifest_path):
"""Read manifest_path, adding the contents to result_set."""
- if _OPTIONS.verbosity >= 1:
+ if _OPTIONS.verbosity >= 5:
print('Parsing manifest file %s.' % manifest_path)
manifest_file = open(manifest_path, encoding='latin-1', mode='r')
for orig_line in manifest_file:
@@ -380,7 +380,8 @@ def ParseSummary(sum_fname):
# Tests that have expired are not added to the set of expected
# results. If they are still present in the set of actual results,
# they will cause an error to be reported.
- print('WARNING: Expected failure "%s" has expired.' % line.strip())
+ if _OPTIONS.verbosity >= 4:
+ print('WARNING: Expected failure "%s" has expired.' % line.strip())
continue
result_set.add(result)
elif IsExpLine(line):
@@ -425,7 +426,8 @@ def GetResults(sum_files, build_results = None):
if build_results == None:
build_results = ResultSet()
for sum_fname in sum_files:
- print('\t%s' % sum_fname)
+ if _OPTIONS.verbosity >= 3:
+ print('\t%s' % sum_fname)
build_results |= ParseSummary(sum_fname)
return build_results
@@ -489,42 +491,46 @@ def GetBuildData():
return None, None
srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =')
target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=')
- print('Source directory: %s' % srcdir)
- print('Build target: %s' % target)
+ if _OPTIONS.verbosity >= 3:
+ print('Source directory: %s' % srcdir)
+ print('Build target: %s' % target)
return srcdir, target
-def PrintSummary(msg, summary):
- print('\n\n%s' % msg)
+def PrintSummary(summary):
summary.Print()
def GetSumFiles(results, build_dir):
if not results:
- print('Getting actual results from build directory %s' % build_dir)
+ if _OPTIONS.verbosity >= 3:
+ print('Getting actual results from build directory %s' % build_dir)
sum_files = CollectSumFiles(build_dir)
else:
- print('Getting actual results from user-provided results')
+ if _OPTIONS.verbosity >= 3:
+ print('Getting actual results from user-provided results')
sum_files = results.split()
return sum_files
-def PerformComparison(expected, actual, ignore_missing_failures):
+def PerformComparison(expected, actual):
actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)
tests_ok = True
if len(actual_vs_expected) > 0:
- PrintSummary('Unexpected results in this build (new failures)',
- actual_vs_expected)
+ if _OPTIONS.verbosity >= 3:
+ print('\n\nUnexpected results in this build (new failures)')
+ if _OPTIONS.verbosity >= 1:
+ PrintSummary(actual_vs_expected)
tests_ok = False
- if not ignore_missing_failures and len(expected_vs_actual) > 0:
- PrintSummary('Expected results not present in this build (fixed tests)'
- '\n\nNOTE: This is not a failure. It just means that these '
- 'tests were expected\nto fail, but either they worked in '
- 'this configuration or they were not\npresent at all.\n',
- expected_vs_actual)
+ if _OPTIONS.verbosity >= 2 and len(expected_vs_actual) > 0:
+ print('\n\nExpected results not present in this build (fixed tests)'
+ '\n\nNOTE: This is not a failure. It just means that these '
+ 'tests were expected\nto fail, but either they worked in '
+ 'this configuration or they were not\npresent at all.\n')
+ PrintSummary(expected_vs_actual)
- if tests_ok:
+ if tests_ok and _OPTIONS.verbosity >= 3:
print('\nSUCCESS: No unexpected failures.')
return tests_ok
@@ -532,21 +538,25 @@ def PerformComparison(expected, actual, ignore_missing_failures):
def CheckExpectedResults():
manifest_path = GetManifestPath(True)
- print('Manifest: %s' % manifest_path)
+ if _OPTIONS.verbosity >= 3:
+ print('Manifest: %s' % manifest_path)
manifest = GetManifest(manifest_path)
sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
actual = GetResults(sum_files)
- if _OPTIONS.verbosity >= 1:
- PrintSummary('Tests expected to fail', manifest)
- PrintSummary('\nActual test results', actual)
+ if _OPTIONS.verbosity >= 5:
+ print('\n\nTests expected to fail')
+ PrintSummary(manifest)
+ print('\n\nActual test results')
+ PrintSummary(actual)
- return PerformComparison(manifest, actual, _OPTIONS.ignore_missing_failures)
+ return PerformComparison(manifest, actual)
def ProduceManifest():
manifest_path = GetManifestPath(False)
- print('Manifest: %s' % manifest_path)
+ if _OPTIONS.verbosity >= 3:
+ print('Manifest: %s' % manifest_path)
if os.path.exists(manifest_path) and not _OPTIONS.force:
Error('Manifest file %s already exists.\nUse --force to overwrite.' %
manifest_path)
@@ -569,13 +579,14 @@ def CompareBuilds():
if _OPTIONS.manifest:
manifest_path = GetManifestPath(True)
- print('Manifest: %s' % manifest_path)
+ if _OPTIONS.verbosity >= 3:
+ print('Manifest: %s' % manifest_path)
clean = GetManifest(manifest_path)
clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build)
clean = GetResults(clean_sum_files, clean)
- return PerformComparison(clean, actual, _OPTIONS.ignore_missing_failures)
+ return PerformComparison(clean, actual)
def Main(argv):
@@ -597,14 +608,6 @@ def Main(argv):
default=False, help='When used with --produce_manifest, '
'it will overwrite an existing manifest file '
'(default = False)')
- parser.add_option('--ignore_missing_failures', action='store_true',
- dest='ignore_missing_failures', default=False,
- help='When a failure is expected in the manifest but '
- 'it is not found in the actual results, the script '
- 'produces a note alerting to this fact. This means '
- 'that the expected failure has been fixed, or '
- 'it did not run, or it may simply be flaky '
- '(default = False)')
parser.add_option('--manifest', action='store', type='string',
dest='manifest', default=None,
help='Name of the manifest file to use (default = '
@@ -621,7 +624,14 @@ def Main(argv):
'starting with FAIL, XPASS or UNRESOLVED (default = '
'.sum files collected from the build directory).')
parser.add_option('--verbosity', action='store', dest='verbosity',
- type='int', default=0, help='Verbosity level (default = 0)')
+ type='int', default=3, help='Verbosity level '
+ '(default = 3). Level 0: only error output, this is '
+ 'useful in scripting when only the exit code is used. '
+ 'Level 1: output unexpected failures. '
+ 'Level 2: output unexpected passes. '
+ 'Level 3: output helpful information. '
+ 'Level 4: output notification on expired entries. '
+ 'Level 5: output debug information.')
global _OPTIONS
(_OPTIONS, _) = parser.parse_args(argv[1:])
--
2.34.1
next prev parent reply other threads:[~2023-06-02 15:21 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-02 15:20 [contrib] Extend and improve validate_failures.py Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 01/12] [contrib] validate_failures.py: Avoid testsuite aliasing Maxim Kuvyrkov
2023-06-03 15:17 ` Jeff Law
2023-06-05 14:06 ` Maxim Kuvyrkov
2023-09-26 15:46 ` Bernhard Reutner-Fischer
2023-09-27 14:47 ` Maxim Kuvyrkov
2023-10-03 10:37 ` rep.dot.nop
2023-11-02 13:31 ` Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 02/12] [contrib] validate_failures.py: Support expiry attributes in manifests Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 03/12] [contrib] validate_failures.py: Read in manifest when comparing build dirs Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 04/12] [contrib] validate_failures.py: Simplify GetManifestPath() Maxim Kuvyrkov
2023-06-02 15:20 ` Maxim Kuvyrkov [this message]
2023-06-02 15:20 ` [PATCH 06/12] [contrib] validate_failures.py: Be more stringent in parsing result lines Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 07/12] [contrib] validate_failures.py: Use exit code "2" to indicate regression Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 08/12] [contrib] validate_failures.py: Support "$tool:" prefix in exp names Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 09/12] [contrib] validate_failures.py: Improve error output Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 10/12] [contrib] validate_failures.py: Add new option --invert_match Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 11/12] [contrib] validate_failures.py: Add "--expiry_date YYYYMMDD" option Maxim Kuvyrkov
2023-06-02 15:20 ` [PATCH 12/12] [contrib] validate_failures.py: Ignore stray filesystem paths in results Maxim Kuvyrkov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230602152052.1874860-6-maxim.kuvyrkov@linaro.org \
--to=maxim.kuvyrkov@linaro.org \
--cc=dje@google.com \
--cc=dnovillo@google.com \
--cc=gcc-patches@gcc.gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).