[gcc(refs/users/marxin/heads/fix-pytest-usage)] Pytest in tests: improve

[gcc(refs/users/marxin/heads/fix-pytest-usage)] Pytest in tests: improve

[Martin Liska]

https://gcc.gnu.org/g:4d0b835ad5e61bb5ac3152bc40535e34724bc64a

commit 4d0b835ad5e61bb5ac3152bc40535e34724bc64a
Author: Martin Liska <mliska@suse.cz>
Date:   Thu Jan 14 09:09:32 2021 +0100

    Pytest in tests: improve
    
    gcc/ChangeLog:
    
            * doc/install.texi: Document that some tests need pytest module.
            * doc/sourcebuild.texi: Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * lib/gcov.exp: Use 'env python3' for execution of pytests.
            Check that pytest accepts all needed options first.
            Improve formatting of PASS/FAIL lines.

Diff:
---
 gcc/doc/install.texi       |  2 +-
 gcc/doc/sourcebuild.texi   |  4 ++++
 gcc/testsuite/lib/gcov.exp | 31 ++++++++++++++++++++++++-------
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 17b5382010e..958da8bc350 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -2990,7 +2990,7 @@ Second, you must have the testing tools installed.  This includes
 the DejaGnu site has links to these. For running the BRIG frontend
 tests, a tool to assemble the binary BRIGs from HSAIL text,
 @uref{https://github.com/HSAFoundation/HSAIL-Tools/,,HSAILasm} must
-be installed.
+be installed. Some optional tests also require Python3 and pytest module.
 
 If the directories where @command{runtest} and @command{expect} were
 installed are not in the @env{PATH}, you may need to set the following
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 3d0873dd074..b9cbe21a4bb 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -3092,6 +3092,10 @@ Check line counts in @command{gcov} tests.
 @item run-gcov [branches] [calls] @{ @var{opts} @var{sourcefile} @}
 Check branch and/or call counts, in addition to line counts, in
 @command{gcov} tests.
+
+@item run-gcov-pytest @{ @var{sourcefile} @var{pytest_file} @}
+Check output of @command{gcov} intermediate format with a pytest
+script.
 @end table
 
 @subsubsection Clean up generated test files
diff --git a/gcc/testsuite/lib/gcov.exp b/gcc/testsuite/lib/gcov.exp
index 4bcab1d3f1d..a43497b5a2b 100644
--- a/gcc/testsuite/lib/gcov.exp
+++ b/gcc/testsuite/lib/gcov.exp
@@ -247,6 +247,19 @@ proc verify-calls { testname testcase file } {
     return $failed
 }
 
+proc gcov-pytest-format-line { args } {
+    global subdir
+
+    set testcase [lindex $args 0]
+    set pytest_script [lindex $args 1]
+    set output_line [lindex $args 2]
+
+    set index [string first "::" $output_line]
+    set test_output [string range $output_line [expr $index + 2] [string length $output_line]]
+
+    return "$subdir/$testcase ${pytest_script}::${test_output}"
+}
+
 # Call by dg-final to run gcov --json-format which produces a JSON file
 # that is later analysed by a pytest Python script.
 # We pass filename of a test via GCOV_PATH environment variable.
@@ -261,30 +274,34 @@ proc run-gcov-pytest { args } {
     set testcase [remote_download host $testcase]
     set result [remote_exec host $GCOV "$testcase -i"]
 
-    set result [remote_exec host "pytest -m pytest --version"]
+    set pytest_cmd "env python3 -m pytest --color=no -rA -s --tb=no"
+    set result [remote_exec host "$pytest_cmd --version"]
     set status [lindex $result 0]
     if { $status != 0 } then {
-      unresolved "could not find Python interpreter and (or) pytest module for $testcase"
+      unsupported "could not find Python interpreter and (or) pytest module for $testcase"
       return
     }
 
     set pytest_script [lindex $args 1]
     setenv GCOV_PATH $testcase
-    verbose "pytest_script: $pytest_script" 2
-    spawn -noecho python3 -m pytest --color=no -rA -s --tb=no $srcdir/$subdir/$pytest_script
+    verbose "pytest_script: $srcdir $subdir $pytest_script" 2
+    spawn -noecho env python3 -m pytest --color=no -rA -s --tb=no $srcdir/$subdir/$pytest_script
 
     set prefix "\[^\r\n\]*"
     expect {
       -re "FAILED($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "ERROR($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "PASSED($prefix)\[^\r\n\]+\r\n" {
-       pass "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       pass $output
        exp_continue
       }
     }

[gcc(refs/users/marxin/heads/fix-pytest-usage)] Pytest in tests: improve

[Martin Liska]

https://gcc.gnu.org/g:22862617f972109a46b944f90d9ff5e4979960a3

commit 22862617f972109a46b944f90d9ff5e4979960a3
Author: Martin Liska <mliska@suse.cz>
Date:   Thu Jan 14 09:09:32 2021 +0100

    Pytest in tests: improve
    
    gcc/ChangeLog:
    
            * doc/install.texi: Document that some tests need pytest module.
            * doc/sourcebuild.texi: Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * lib/gcov.exp: Use 'env python3' for execution of pytests.
            Check that pytest accepts all needed options first.
            Improve formatting of PASS/FAIL lines.

Diff:
---
 gcc/doc/install.texi       |  2 +-
 gcc/doc/sourcebuild.texi   |  4 ++++
 gcc/testsuite/lib/gcov.exp | 31 ++++++++++++++++++++++++-------
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 17b5382010e..4c38244ae58 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -2990,7 +2990,7 @@ Second, you must have the testing tools installed.  This includes
 the DejaGnu site has links to these. For running the BRIG frontend
 tests, a tool to assemble the binary BRIGs from HSAIL text,
 @uref{https://github.com/HSAFoundation/HSAIL-Tools/,,HSAILasm} must
-be installed.
+be installed.  Some optional tests also require Python3 and pytest module.
 
 If the directories where @command{runtest} and @command{expect} were
 installed are not in the @env{PATH}, you may need to set the following
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 3d0873dd074..b9cbe21a4bb 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -3092,6 +3092,10 @@ Check line counts in @command{gcov} tests.
 @item run-gcov [branches] [calls] @{ @var{opts} @var{sourcefile} @}
 Check branch and/or call counts, in addition to line counts, in
 @command{gcov} tests.
+
+@item run-gcov-pytest @{ @var{sourcefile} @var{pytest_file} @}
+Check output of @command{gcov} intermediate format with a pytest
+script.
 @end table
 
 @subsubsection Clean up generated test files
diff --git a/gcc/testsuite/lib/gcov.exp b/gcc/testsuite/lib/gcov.exp
index 4bcab1d3f1d..f3309653b77 100644
--- a/gcc/testsuite/lib/gcov.exp
+++ b/gcc/testsuite/lib/gcov.exp
@@ -247,6 +247,19 @@ proc verify-calls { testname testcase file } {
     return $failed
 }
 
+proc gcov-pytest-format-line { args } {
+    global subdir
+
+    set testcase [lindex $args 0]
+    set pytest_script [lindex $args 1]
+    set output_line [lindex $args 2]
+
+    set index [string first "::" $output_line]
+    set test_output [string range $output_line [expr $index + 2] [string length $output_line]]
+
+    return "$subdir/$testcase ${pytest_script}::${test_output}"
+}
+
 # Call by dg-final to run gcov --json-format which produces a JSON file
 # that is later analysed by a pytest Python script.
 # We pass filename of a test via GCOV_PATH environment variable.
@@ -261,30 +274,34 @@ proc run-gcov-pytest { args } {
     set testcase [remote_download host $testcase]
     set result [remote_exec host $GCOV "$testcase -i"]
 
-    set result [remote_exec host "pytest -m pytest --version"]
+    set pytest_cmd "python3 -m pytest --color=no -rap -s --tb=no"
+    set result [remote_exec host "$pytest_cmd --version"]
     set status [lindex $result 0]
     if { $status != 0 } then {
-      unresolved "could not find Python interpreter and (or) pytest module for $testcase"
+      unsupported "$subdir/$testcase run-gcov-pytest python3 pytest missing"
       return
     }
 
     set pytest_script [lindex $args 1]
     setenv GCOV_PATH $testcase
-    verbose "pytest_script: $pytest_script" 2
-    spawn -noecho python3 -m pytest --color=no -rA -s --tb=no $srcdir/$subdir/$pytest_script
+    verbose "pytest_script: $srcdir $subdir $pytest_script" 2
+    spawn -noecho python3 -m pytest --color=no -rap -s --tb=no $srcdir/$subdir/$pytest_script
 
     set prefix "\[^\r\n\]*"
     expect {
       -re "FAILED($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "ERROR($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "PASSED($prefix)\[^\r\n\]+\r\n" {
-       pass "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       pass $output
        exp_continue
       }
     }

[gcc(refs/users/marxin/heads/fix-pytest-usage)] Pytest in tests: improve

[Martin Liska]

https://gcc.gnu.org/g:914421c59671bc7fc06457f3a59b05e363edd5f9

commit 914421c59671bc7fc06457f3a59b05e363edd5f9
Author: Martin Liska <mliska@suse.cz>
Date:   Thu Jan 14 09:09:32 2021 +0100

    Pytest in tests: improve
    
    gcc/ChangeLog:
    
            * doc/install.texi: Document that some tests need pytest module.
            * doc/sourcebuild.texi: Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * lib/gcov.exp: Use 'env python3' for execution of pytests.
            Check that pytest accepts all needed options first.
            Improve formatting of PASS/FAIL lines.

Diff:
---
 gcc/doc/install.texi       |  2 +-
 gcc/doc/sourcebuild.texi   |  4 ++++
 gcc/testsuite/lib/gcov.exp | 31 ++++++++++++++++++++++++-------
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 17b5382010e..958da8bc350 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -2990,7 +2990,7 @@ Second, you must have the testing tools installed.  This includes
 the DejaGnu site has links to these. For running the BRIG frontend
 tests, a tool to assemble the binary BRIGs from HSAIL text,
 @uref{https://github.com/HSAFoundation/HSAIL-Tools/,,HSAILasm} must
-be installed.
+be installed. Some optional tests also require Python3 and pytest module.
 
 If the directories where @command{runtest} and @command{expect} were
 installed are not in the @env{PATH}, you may need to set the following
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 3d0873dd074..5fbc4d49e4c 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -3092,6 +3092,10 @@ Check line counts in @command{gcov} tests.
 @item run-gcov [branches] [calls] @{ @var{opts} @var{sourcefile} @}
 Check branch and/or call counts, in addition to line counts, in
 @command{gcov} tests.
+
+@item run-gcov-pytest @{ @{sourcefile} @{pytest_file} @}
+Check output of @command{gcov} intermediate format with a pytest
+script.
 @end table
 
 @subsubsection Clean up generated test files
diff --git a/gcc/testsuite/lib/gcov.exp b/gcc/testsuite/lib/gcov.exp
index 4bcab1d3f1d..a43497b5a2b 100644
--- a/gcc/testsuite/lib/gcov.exp
+++ b/gcc/testsuite/lib/gcov.exp
@@ -247,6 +247,19 @@ proc verify-calls { testname testcase file } {
     return $failed
 }
 
+proc gcov-pytest-format-line { args } {
+    global subdir
+
+    set testcase [lindex $args 0]
+    set pytest_script [lindex $args 1]
+    set output_line [lindex $args 2]
+
+    set index [string first "::" $output_line]
+    set test_output [string range $output_line [expr $index + 2] [string length $output_line]]
+
+    return "$subdir/$testcase ${pytest_script}::${test_output}"
+}
+
 # Call by dg-final to run gcov --json-format which produces a JSON file
 # that is later analysed by a pytest Python script.
 # We pass filename of a test via GCOV_PATH environment variable.
@@ -261,30 +274,34 @@ proc run-gcov-pytest { args } {
     set testcase [remote_download host $testcase]
     set result [remote_exec host $GCOV "$testcase -i"]
 
-    set result [remote_exec host "pytest -m pytest --version"]
+    set pytest_cmd "env python3 -m pytest --color=no -rA -s --tb=no"
+    set result [remote_exec host "$pytest_cmd --version"]
     set status [lindex $result 0]
     if { $status != 0 } then {
-      unresolved "could not find Python interpreter and (or) pytest module for $testcase"
+      unsupported "could not find Python interpreter and (or) pytest module for $testcase"
       return
     }
 
     set pytest_script [lindex $args 1]
     setenv GCOV_PATH $testcase
-    verbose "pytest_script: $pytest_script" 2
-    spawn -noecho python3 -m pytest --color=no -rA -s --tb=no $srcdir/$subdir/$pytest_script
+    verbose "pytest_script: $srcdir $subdir $pytest_script" 2
+    spawn -noecho env python3 -m pytest --color=no -rA -s --tb=no $srcdir/$subdir/$pytest_script
 
     set prefix "\[^\r\n\]*"
     expect {
       -re "FAILED($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "ERROR($prefix)\[^\r\n\]+\r\n" {
-       fail "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       fail $output
        exp_continue
       }
       -re "PASSED($prefix)\[^\r\n\]+\r\n" {
-       pass "$expect_out(1,string)"
+       set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
+       pass $output
        exp_continue
       }
     }