public inbox for gdb-patches@sourceware.org
 help / color / mirror / Atom feed
* [PATCH 1/2] sim: testsuite: allow tests to declare expected exit status
@ 2021-01-05  6:55 Mike Frysinger
  2021-01-05  6:55 ` [PATCH 2/2] sim: m32r: clean up redundant test coverage Mike Frysinger
                   ` (5 more replies)
  0 siblings, 6 replies; 7+ messages in thread
From: Mike Frysinger @ 2021-01-05  6:55 UTC (permalink / raw)
  To: gdb-patches

Some tests want to verify they can control the exit status, and
allowing any non-zero value would allow tests to silently fail:
if it crashed & exited 1, or forced all non-zero to 1, then we
wouldn't be able to differentiate with a test exiting with a
status like 47.

Extend the test harness to allow tests to declare their expected
exit status that would be defined as a "pass".  This requires a
small tweak to the sim_run API to return the status directly, but
that shouldn't be a big deal as it's only used by sim code.
---
 sim/testsuite/lib/sim-defs.exp              | 23 +++++++++++----------
 sim/testsuite/sim/cris/c/c.exp              |  7 ++++++-
 sim/testsuite/sim/cris/hw/rv-n-cris/rvc.exp |  2 +-
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/sim/testsuite/lib/sim-defs.exp b/sim/testsuite/lib/sim-defs.exp
index b8ce230f13c3..43a07050f508 100644
--- a/sim/testsuite/lib/sim-defs.exp
+++ b/sim/testsuite/lib/sim-defs.exp
@@ -62,7 +62,7 @@ proc sim_compile { source dest type options } {
 #	timeout=val	- set the timeout to val for this run
 #
 # The result is a list of two elements.
-# The first is one of pass/fail/etc.
+# The first is the program's exit status (0/1/etc...).
 # The second is the program's output.
 #
 # This is different than the sim_load routine provided by
@@ -156,15 +156,7 @@ proc sim_run { prog sim_opts prog_opts redir options } {
 	remote_file host delete $prog
     }
 
-    # ??? Not sure the test for pass/fail is right.
-    # We just care that the simulator ran correctly, not whether the simulated
-    # program return 0 or non-zero from `main'.
-    set status fail
-    if { $return_code == 0 } {
-	set status pass
-    }
-
-    return [list $status $output]
+    return [list $return_code $output]
 }
 
 # Run testcase NAME.
@@ -183,6 +175,7 @@ proc sim_run { prog sim_opts prog_opts redir options } {
 # cc[(mach-list)]: <compiler options>
 # sim[(mach-list)]: <simulator options>
 # progopts: <arguments to the program being simulated>
+# status: program exit status to treat as "pass"
 # output: program output pattern to match with string-match
 # xerror: program is expected to return with a "failure" exit code
 # xfail: <PRMS-opt> <target-triplets-where-test-fails>
@@ -223,6 +216,7 @@ proc run_sim_test { name requested_machs } {
     set opts(cc) ""
     set opts(progopts) ""
     set opts(sim) ""
+    set opts(status) "0"
     set opts(output) ""
     set opts(mach) ""
     set opts(timeout) ""
@@ -385,15 +379,21 @@ proc run_sim_test { name requested_machs } {
 	}
 
 	set result [sim_run ${name}.x "$opts(sim,$mach) $global_sim_options" "$opts(progopts)" "" "$options"]
-	set status [lindex $result 0]
+	set return_code [lindex $result 0]
 	set output [lindex $result 1]
 
+	set status fail
+	if { $return_code == $opts(status) } {
+	    set status pass
+	}
+
 	if { "$status" == "pass" } {
 	    if { "$opts(xerror)" == "no" } {
 		if [string match $opts(output) $output] {
 		    pass "$mach $testname"
 		    file delete ${name}.o ${name}.x
 		} else {
+		    verbose -log "status:  $return_code" 3
 		    verbose -log "output:  $output" 3
 		    verbose -log "pattern: $opts(output)" 3
 		    fail "$mach $testname (execution)"
@@ -410,6 +410,7 @@ proc run_sim_test { name requested_machs } {
 		    pass "$mach $testname"
 		    file delete ${name}.o ${name}.x
 		} else {
+		    verbose -log "status:  $return_code" 3
 		    verbose -log "output:  $output" 3
 		    verbose -log "pattern: $opts(output)" 3
 		    fail "$mach $testname (execution)"
diff --git a/sim/testsuite/sim/cris/c/c.exp b/sim/testsuite/sim/cris/c/c.exp
index c9df98381e64..c872fcb97d5b 100644
--- a/sim/testsuite/sim/cris/c/c.exp
+++ b/sim/testsuite/sim/cris/c/c.exp
@@ -210,9 +210,14 @@ foreach src [lsort [glob -nocomplain $srcdir/$subdir/*.c]] {
 
     set result [sim_run $dest "$opts(sim,$mach)" "$opts(progoptions)" \
 	"" "$opts(simenv)"]
-    set status [lindex $result 0]
+    set return_code [lindex $result 0]
     set output [lindex $result 1]
 
+    set status fail
+    if { $return_code == 0 } {
+	set status pass
+    }
+
     if { "$status" == "pass" } {
 	if { "$opts(xerror)" == "no" } {
 	    if [string match $opts(output) $output] {
diff --git a/sim/testsuite/sim/cris/hw/rv-n-cris/rvc.exp b/sim/testsuite/sim/cris/hw/rv-n-cris/rvc.exp
index 98fc717ed761..0f9ecec32a26 100644
--- a/sim/testsuite/sim/cris/hw/rv-n-cris/rvc.exp
+++ b/sim/testsuite/sim/cris/hw/rv-n-cris/rvc.exp
@@ -67,7 +67,7 @@ proc sim_has_rv_and_cris {} {
     set return_code [lindex $result 0]
     set output [lindex $result 1]
 
-    if { "$return_code" == "pass" } {
+    if { $return_code == 0 } {
 	return 1
     }
 
-- 
2.28.0


^ permalink raw reply	[flat|nested] 7+ messages in thread
end of thread, other threads:[~2021-01-11 23:34 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-01-05  6:55 [PATCH 1/2] sim: testsuite: allow tests to declare expected exit status Mike Frysinger
2021-01-05  6:55 ` [PATCH 2/2] sim: m32r: clean up redundant test coverage Mike Frysinger
2021-01-05  7:30 ` [PATCH] sim: frv: " Mike Frysinger
2021-01-05  7:34 ` [PATCH] sim: mips: delete empty stub test dir Mike Frysinger
2021-01-05  8:17 ` [PATCH] sim: d10v: relocate tests & clean up test harness Mike Frysinger
2021-01-06  0:31 ` [PATCH] sim: testsuite: delete configure script Mike Frysinger
2021-01-11 23:34 ` [PATCH 1/2] sim: testsuite: allow tests to declare expected exit status Mike Frysinger

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).