From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-lf1-x130.google.com (mail-lf1-x130.google.com [IPv6:2a00:1450:4864:20::130]) by sourceware.org (Postfix) with ESMTPS id D9D9A385C6C8 for ; Fri, 2 Jun 2023 15:21:55 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.2 sourceware.org D9D9A385C6C8 Authentication-Results: sourceware.org; dmarc=pass (p=none dis=none) header.from=linaro.org Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=linaro.org Received: by mail-lf1-x130.google.com with SMTP id 2adb3069b0e04-4f3af42459bso226852e87.0 for ; Fri, 02 Jun 2023 08:21:55 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; t=1685719314; x=1688311314; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=xQFTZoYDemdqdSTQllOBrlgnGkqEbAZxOG6pYMRIkM4=; b=qc9CIrRqnhiq41zds7TaecbvpyKjcbIQ6dEmCQfv5j+1O7vYmFQr76CuC4f3ePGGxL 3chbBB6MKlF3uurutxr0ftQyYWZWaLX1T0x1bCsNwrlCvL4xgWv+XTM4RJ6pYlMtxstv +q0xRxxtBmDZdFd9DPv+k5nT5rjy9A8Nyuy/8lf5M0h5p3iRmpygPkUuuWI5heTC2mI+ Kls1xDHnUOkWOfkALnsJ2/FWl6LvZH1hKz/FYU9WHsy7XrdRFDBFWD7+fvKMXcyTkebw qSukpONnYeQJnecz30OOno9GzdU4NBK97SMQbO6QSSjupDudknUgVBazpvDwMDt43SFG SELw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1685719314; x=1688311314; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=xQFTZoYDemdqdSTQllOBrlgnGkqEbAZxOG6pYMRIkM4=; b=MmcVLskkVdpoJKFbyZHRtPMlSpIW/BxJX4jYJzmb5Tos25rY4vc7mMJ1OqB0PC/ySt nGjNbMnwzQZs+f011lIC3iCZjyxHoYC27WXRW52HMplFFi/sb97y0kaKHZemwyogGTjH r5OsZUvhzz2Jb4CJIjqyJ2InwiN0TbxcjXoItjcYxZZvx/jSkVreEY9ge9vBEgU1OvCv L3BUrhkTS35uCdfaT1+Yflei6nGZZcVqQ7yz4fIqPShBcgg/+O32iqccTHPfLCnI/5JW 0f3woP2fb8REzsKtYUrJDYH89ty88UYMHcu/5jGz6kk//rRfyGEww3nCLG04cwO1OAvU 1qGA== X-Gm-Message-State: AC+VfDwrcH+YSG5wC8MRjKfEl6wr/Mwuw6xjmy337JXFPYeQAuXHcCYq NcG+u4pz+B5My/R625dO7SRt528wGK0kQmdGkrrqoPY= X-Google-Smtp-Source: ACHHUZ6eNINpANSOqyWKeev1JCZoSJQyMhdzbrYfVdMOs/RP66cPwHwHJdI+vz86gLkQ1Hgvtqu8qA== X-Received: by 2002:a2e:b55a:0:b0:2b1:5a32:7bef with SMTP id a26-20020a2eb55a000000b002b15a327befmr2723472ljn.5.1685719313852; Fri, 02 Jun 2023 08:21:53 -0700 (PDT) Received: from localhost.localdomain (static.225.72.216.95.clients.your-server.de. [95.216.72.225]) by smtp.gmail.com with ESMTPSA id a18-20020a2eb172000000b002a8ac166e55sm255122ljm.44.2023.06.02.08.21.51 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Jun 2023 08:21:52 -0700 (PDT) From: Maxim Kuvyrkov To: gcc-patches@gcc.gnu.org Cc: Diego Novillo , Doug Evans , Maxim Kuvyrkov Subject: [PATCH 10/12] [contrib] validate_failures.py: Add new option --invert_match Date: Fri, 2 Jun 2023 15:20:50 +0000 Message-Id: <20230602152052.1874860-11-maxim.kuvyrkov@linaro.org> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230602152052.1874860-1-maxim.kuvyrkov@linaro.org> References: <20230602152052.1874860-1-maxim.kuvyrkov@linaro.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-12.2 required=5.0 tests=BAYES_00,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,GIT_PATCH_0,RCVD_IN_DNSWL_NONE,SPF_HELO_NONE,SPF_PASS,TXREP,T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org List-Id: This option is used to detect flaky tests that FAILed in the clean build (or manifest), but PASSed in the current build (or manifest). The option inverts output logic similar to what "-v/--invert-match" does for grep. --- .../testsuite-management/validate_failures.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py index 1919935cf53..6eb1acd473f 100755 --- a/contrib/testsuite-management/validate_failures.py +++ b/contrib/testsuite-management/validate_failures.py @@ -217,11 +217,17 @@ class ResultSet(set): Attributes: current_tool: Name of the current top-level DejaGnu testsuite. current_exp: Name of the current .exp testsuite file. + testsuites: A set of (tool, exp) tuples representing encountered testsuites. """ def __init__(self): super().__init__() self.ResetToolExp() + self.testsuites=set() + + def update(self, other): + super().update(other) + self.testsuites.update(other.testsuites) def ResetToolExp(self): self.current_tool = None @@ -246,6 +252,10 @@ class ResultSet(set): outfile.write(_SUMMARY_LINE_FORMAT % 'Results') + # Check if testsuite of expected_result is present in current results. + # This is used to compare partial test results against a full manifest. + def HasTestsuite(self, expected_result): + return (expected_result.tool, expected_result.exp) in self.testsuites def GetMakefileValue(makefile_name, value_name): if os.path.exists(makefile_name): @@ -391,6 +401,8 @@ def ParseSummary(sum_fname): result_set.add(result) elif IsExpLine(line): result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0] + result_set.testsuites.add((result_set.current_tool, + result_set.current_exp)) elif IsToolLine(line): result_set.current_tool = _TOOL_LINE_REX.match(line).groups()[0] result_set.current_exp = None @@ -433,7 +445,7 @@ def GetResults(sum_files, build_results = None): for sum_fname in sum_files: if _OPTIONS.verbosity >= 3: print('\t%s' % sum_fname) - build_results |= ParseSummary(sum_fname) + build_results.update(ParseSummary(sum_fname)) return build_results @@ -458,7 +470,11 @@ def CompareResults(manifest, actual): # Ignore tests marked flaky. if 'flaky' in expected_result.attrs: continue - if expected_result not in actual: + # We try to support comparing partial results vs full manifest + # (e.g., manifest has failures for gcc, g++, gfortran, but we ran only + # g++ testsuite). To achieve this we record encountered testsuites in + # actual.testsuites set, and then we check it here using HasTestsuite(). + if expected_result not in actual and actual.HasTestsuite(expected_result): manifest_vs_actual.add(expected_result) return actual_vs_manifest, manifest_vs_actual @@ -520,6 +536,13 @@ def GetSumFiles(results, build_dir): def PerformComparison(expected, actual): actual_vs_expected, expected_vs_actual = CompareResults(expected, actual) + if _OPTIONS.inverse_match: + # Switch results if inverse comparison is requested. + # This is useful in detecting flaky tests that FAILed in expected set, + # but PASSed in actual set. + actual_vs_expected, expected_vs_actual \ + = expected_vs_actual, actual_vs_expected + tests_ok = True if len(actual_vs_expected) > 0: if _OPTIONS.verbosity >= 3: @@ -613,6 +636,13 @@ def Main(argv): default=False, help='When used with --produce_manifest, ' 'it will overwrite an existing manifest file ' '(default = False)') + parser.add_option('--inverse_match', action='store_true', + dest='inverse_match', default=False, + help='Inverse result sets in comparison. ' + 'Output unexpected passes as unexpected failures and ' + 'unexpected failures as unexpected passes. ' + 'This is used to catch FAIL->PASS flaky tests. ' + '(default = False)') parser.add_option('--manifest', action='store', type='string', dest='manifest', default=None, help='Name of the manifest file to use (default = ' -- 2.34.1