[PATCH v3] math: Add more details to the test driver output.

[PATCH v3] math: Add more details to the test driver output.

[Joe Simmons-Talbott]

Add start and end indicators that identify the test being run in the
verbose output.  Better identify the tests for max errors in the
summary output.  Count each exception checked for each test. Remove
double counting of tests for the check_<type> functions other than
check_float_internal. Rename print_max_error and
print_complex_max_error to check_max_error and check_complex_max_error
respectively since they have side effects.

Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
---
Chnages to v2:
* Fix gnu style issues: space before '('

Changes to v1:
* Also check that the divide-by-zero exception check ran if required.
* Only increment noErrnoTests in one place within the actual check.
* Rename "builtin" test references to "maxerror".

 math/libm-test-driver.c  |   4 +-
 math/libm-test-support.c | 203 ++++++++++++++++++++++++++++++---------
 math/libm-test-support.h |   4 +-
 3 files changed, 162 insertions(+), 49 deletions(-)

diff --git a/math/libm-test-driver.c b/math/libm-test-driver.c
index 3356f9b10d..dfb56e8cde 100644
--- a/math/libm-test-driver.c
+++ b/math/libm-test-driver.c
@@ -1083,9 +1083,9 @@ struct test_Ff_b1_data
     = STR_CON3 (FUN, SUFF, TEST_SUFF) TEST_SUFF_STR;		\
   init_max_error (this_func, EXACT, TEST_COND_any_ibm128)
 #define END					\
-  print_max_error (this_func)
+  check_max_error (this_func)
 #define END_COMPLEX				\
-  print_complex_max_error (this_func)
+  check_complex_max_error (this_func)
 
 /* Run tests for a given function in all rounding modes.  */
 #define ALL_RM_TEST(FUNC, EXACT, ARRAY, LOOP_MACRO, END_MACRO, ...)	\
diff --git a/math/libm-test-support.c b/math/libm-test-support.c
index 0cae545f86..0796f9d495 100644
--- a/math/libm-test-support.c
+++ b/math/libm-test-support.c
@@ -112,6 +112,7 @@
 #include <argp.h>
 #include <errno.h>
 #include <string.h>
+#include <assert.h>
 
 /* This header defines func_ulps, func_real_ulps and func_imag_ulps
    arrays.  */
@@ -125,10 +126,13 @@ static FILE *ulps_file;		/* File to document difference.  */
 static int output_ulps;		/* Should ulps printed?  */
 static char *output_dir;	/* Directory where generated files will be written.  */
 
-static int noErrors;	/* number of errors */
-static int noTests;	/* number of tests (without testing exceptions) */
-static int noExcTests;	/* number of tests for exception flags */
-static int noErrnoTests;/* number of tests for errno values */
+#define TEST_INPUT 1
+#define TEST_MAXERROR 2
+static int noErrors;	    /* number of errors */
+static int noTests; 	    /* number of tests (without testing exceptions) */
+static int noMaxErrorTests; /* number of max error tests  */
+static int noExcTests;	    /* number of tests for exception flags */
+static int noErrnoTests;    /* number of tests for errno values */
 
 static unsigned int verbose;
 static int output_max_error;	/* Should the maximal errors printed?  */
@@ -299,9 +303,19 @@ print_screen_max_error (int ok)
 
 /* Update statistic counters.  */
 static void
-update_stats (int ok)
+update_stats (int ok, int testType)
 {
-  ++noTests;
+  switch (testType)
+    {
+	case TEST_INPUT:
+	  ++noTests;
+	  break;
+	case TEST_MAXERROR:
+	  ++noMaxErrorTests;
+	  break;
+	default:
+	  abort();
+    }
   if (!ok)
     ++noErrors;
 }
@@ -367,11 +381,30 @@ fpstack_test (const char *test_name)
 #endif
 }
 
+static void
+print_test_start (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen (1))
+    printf ("--- Start of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_MAXERROR ? " max error" : "", test_num, test_name);
+}
 
+static void
+print_test_end (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen (1))
+    printf ("--- End of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_MAXERROR ? " max error" : "", test_num, test_name);
+}
+
+/* This is a builtin test of overall max error.  */
 void
-print_max_error (const char *func_name)
+check_max_error (const char *func_name)
 {
   int ok = 0;
+  int thisTest = noMaxErrorTests;
+
+  print_test_start (thisTest, func_name, TEST_MAXERROR);
 
   if (max_error == 0.0 || (max_error <= prev_max_error && !ignore_max_ulp))
     {
@@ -392,14 +425,19 @@ print_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pmestr);
     }
 
-  update_stats (ok);
-}
+  update_stats (ok, TEST_MAXERROR);
 
+  print_test_end (thisTest, func_name, TEST_MAXERROR);
+}
 
+/* This is a builtin test of overall max error.  */
 void
-print_complex_max_error (const char *func_name)
+check_complex_max_error (const char *func_name)
 {
   int real_ok = 0, imag_ok = 0, ok;
+  int thisTest = noMaxErrorTests;
+
+  print_test_start (thisTest, func_name, TEST_MAXERROR);
 
   if (real_max_error == 0
       || (real_max_error <= prev_real_max_error && !ignore_max_ulp))
@@ -436,7 +474,8 @@ print_complex_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pimestr);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_MAXERROR);
+  print_test_end (thisTest, func_name, TEST_MAXERROR);
 }
 
 
@@ -477,12 +516,13 @@ test_single_exception (const char *test_name,
       else
 	{
 	  if (print_screen (1))
-	    printf ("%s: Exception \"%s\" not set\n", test_name,
+	    printf ("Pass: %s: Exception \"%s\" not set\n", test_name,
 		    flag_name);
 	}
     }
   if (!ok)
     ++noErrors;
+  ++noExcTests;
 }
 #endif
 
@@ -494,23 +534,32 @@ test_exceptions (const char *test_name, int exception)
 {
   if (flag_test_exceptions && EXCEPTION_TESTS (FLOAT))
     {
-      ++noExcTests;
+      int ran = 0;
 #ifdef FE_DIVBYZERO
       if ((exception & DIVIDE_BY_ZERO_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception,
-			       DIVIDE_BY_ZERO_EXCEPTION, FE_DIVBYZERO,
-			       "Divide by zero");
+	{
+	  test_single_exception (test_name, exception,
+			         DIVIDE_BY_ZERO_EXCEPTION, FE_DIVBYZERO,
+			         "Divide by zero");
+	  ran = 1;
+	}
 #endif
 #ifdef FE_INVALID
       if ((exception & INVALID_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception,
-			       INVALID_EXCEPTION, FE_INVALID,
-			       "Invalid operation");
+	{
+	  test_single_exception (test_name, exception,
+			         INVALID_EXCEPTION, FE_INVALID,
+			         "Invalid operation");
+	  ran = 1;
+	}
 #endif
 #ifdef FE_OVERFLOW
       if ((exception & OVERFLOW_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
-			       FE_OVERFLOW, "Overflow");
+	{
+	  test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
+			         FE_OVERFLOW, "Overflow");
+	  ran = 1;
+	}
 #endif
       /* Spurious "underflow" and "inexact" exceptions are always
 	 allowed for IBM long double, in line with the underlying
@@ -519,17 +568,30 @@ test_exceptions (const char *test_name, int exception)
       if ((exception & UNDERFLOW_EXCEPTION_OK) == 0
 	  && !(test_ibm128
 	       && (exception & UNDERFLOW_EXCEPTION) == 0))
-	test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
-			       FE_UNDERFLOW, "Underflow");
+	{
+	  test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
+			         FE_UNDERFLOW, "Underflow");
+	  ran = 1;
+	}
+
 #endif
 #ifdef FE_INEXACT
       if ((exception & (INEXACT_EXCEPTION | NO_INEXACT_EXCEPTION)) != 0
 	  && !(test_ibm128
 	       && (exception & NO_INEXACT_EXCEPTION) != 0))
-	test_single_exception (test_name, exception, INEXACT_EXCEPTION,
-			       FE_INEXACT, "Inexact");
+	{
+	  test_single_exception (test_name, exception, INEXACT_EXCEPTION,
+			         FE_INEXACT, "Inexact");
+	  ran = 1;
+	}
 #endif
+      assert (ran == 1);
     }
+   else
+     {
+	if (print_screen (1))
+	  printf ("Info: %s: No exceptions tested\n", test_name);
+     }
   feclearexcept (FE_ALL_EXCEPT);
 }
 
@@ -552,6 +614,7 @@ test_single_errno (const char *test_name, int errno_value,
 	printf ("Failure: %s: errno set to %d, expected %d (%s)\n",
 		test_name, errno_value, expected_value, expected_name);
     }
+  ++noErrnoTests;
 }
 
 /* Test whether errno (value ERRNO_VALUE) has been for TEST_NAME set
@@ -561,13 +624,39 @@ test_errno (const char *test_name, int errno_value, int exceptions)
 {
   if (flag_test_errno)
     {
-      ++noErrnoTests;
+      int ran = 0;
+
+      if ((exceptions & (ERRNO_UNCHANGED|ERRNO_EDOM|ERRNO_ERANGE)) == 0)
+	{
+	  if (print_screen (1))
+	    printf ("Info: %s: The value of errno was not tested\n",
+		    test_name);
+	  return;
+	}
+
+
       if (exceptions & ERRNO_UNCHANGED)
-	test_single_errno (test_name, errno_value, 0, "unchanged");
+	{
+	  test_single_errno (test_name, errno_value, 0, "unchanged");
+	  ran = 1;
+	}
       if (exceptions & ERRNO_EDOM)
-	test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	{
+	  test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	  ran = 1;
+	}
       if (exceptions & ERRNO_ERANGE)
-	test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	{
+	  test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	  ran = 1;
+	}
+
+      assert (ran == 1);
+    }
+  else
+    {
+	if (print_screen (1))
+	  printf ("Info: %s: No errno tests\n", test_name);
     }
 }
 
@@ -619,6 +708,9 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
   FLOAT diff = 0;
   FLOAT ulps = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
@@ -716,12 +808,13 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
 	  printf (" max.ulp   : %s\n", mustrn);
 	}
     }
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
 
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -776,12 +869,14 @@ check_int (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -795,11 +890,12 @@ check_int (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -810,12 +906,14 @@ check_long (const char *test_name, long int computed, long int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -829,11 +927,12 @@ check_long (const char *test_name, long int computed, long int expected,
       printf (" should be:  %ld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -844,12 +943,14 @@ check_bool (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if ((computed == 0) == (expected == 0))
     ok = 1;
 
@@ -863,11 +964,12 @@ check_bool (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -879,12 +981,14 @@ check_longlong (const char *test_name, long long int computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -898,11 +1002,12 @@ check_longlong (const char *test_name, long long int computed,
       printf (" should be:  %lld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -913,12 +1018,14 @@ check_intmax_t (const char *test_name, intmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -932,11 +1039,12 @@ check_intmax_t (const char *test_name, intmax_t computed,
       printf (" should be:  %jd\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -947,12 +1055,14 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -966,11 +1076,12 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
       printf (" should be:  %ju\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 /* Return whether a test with flags EXCEPTIONS should be run.  */
@@ -1211,9 +1322,11 @@ libm_test_finish (void)
     fclose (ulps_file);
 
   printf ("\nTest suite completed:\n");
-  printf ("  %d test cases plus %d tests for exception flags and\n"
-	  "    %d tests for errno executed.\n",
-	  noTests, noExcTests, noErrnoTests);
+  printf ("  %d max error test cases,\n", noMaxErrorTests);
+  printf ("  %d input tests,\n", noTests);
+  printf ("  - with %d tests for exception flags,\n", noExcTests);
+  printf ("  - with %d tests for errno executed.\n", noErrnoTests);
+
   if (noErrors)
     {
       printf ("  %d errors occurred.\n", noErrors);
diff --git a/math/libm-test-support.h b/math/libm-test-support.h
index 8baf7e1817..efb9523e9e 100644
--- a/math/libm-test-support.h
+++ b/math/libm-test-support.h
@@ -170,8 +170,8 @@ extern const char doc[];
 
 int enable_test (int);
 void init_max_error (const char *, int, int);
-void print_max_error (const char *);
-void print_complex_max_error (const char *);
+void check_max_error (const char *);
+void check_complex_max_error (const char *);
 void check_float (const char *, FLOAT, FLOAT, int);
 void check_complex (const char *, CFLOAT, CFLOAT, int);
 void check_int (const char *, int, int, int);
-- 
2.44.0

Re: [PATCH v3] math: Add more details to the test driver output.

[Joseph Myers]

On Thu, 16 May 2024, Joe Simmons-Talbott wrote:

> Add start and end indicators that identify the test being run in the
> verbose output.  Better identify the tests for max errors in the
> summary output.  Count each exception checked for each test. Remove
> double counting of tests for the check_<type> functions other than
> check_float_internal. Rename print_max_error and
> print_complex_max_error to check_max_error and check_complex_max_error
> respectively since they have side effects.
> 
> Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
> ---
> Chnages to v2:
> * Fix gnu style issues: space before '('

As noted, ths fixed version is OK.

-- 
Joseph S. Myers
josmyers@redhat.com

Re: [PATCH v3] math: Add more details to the test driver output.

[Joe Simmons-Talbott]

On Thu, May 16, 2024 at 5:54 PM Joseph Myers <josmyers@redhat.com> wrote:
>
> On Thu, 16 May 2024, Joe Simmons-Talbott wrote:
>
> > Add start and end indicators that identify the test being run in the
> > verbose output.  Better identify the tests for max errors in the
> > summary output.  Count each exception checked for each test. Remove
> > double counting of tests for the check_<type> functions other than
> > check_float_internal. Rename print_max_error and
> > print_complex_max_error to check_max_error and check_complex_max_error
> > respectively since they have side effects.
> >
> > Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
> > ---
> > Chnages to v2:
> > * Fix gnu style issues: space before '('
>
> As noted, ths fixed version is OK.

Thanks.  Would you please give me an explicit Reviewed-By: ?

Joe
>
> --
> Joseph S. Myers
> josmyers@redhat.com
>


-- 
Joe Simmons-Talbott

Re: [PATCH v3] math: Add more details to the test driver output.

[Joseph Myers]

[-- Attachment #1: Type: text/plain, Size: 1001 bytes --]

On Fri, 17 May 2024, Joe Simmons-Talbott wrote:

> On Thu, May 16, 2024 at 5:54 PM Joseph Myers <josmyers@redhat.com> wrote:
> >
> > On Thu, 16 May 2024, Joe Simmons-Talbott wrote:
> >
> > > Add start and end indicators that identify the test being run in the
> > > verbose output.  Better identify the tests for max errors in the
> > > summary output.  Count each exception checked for each test. Remove
> > > double counting of tests for the check_<type> functions other than
> > > check_float_internal. Rename print_max_error and
> > > print_complex_max_error to check_max_error and check_complex_max_error
> > > respectively since they have side effects.
> > >
> > > Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
> > > ---
> > > Chnages to v2:
> > > * Fix gnu style issues: space before '('
> >
> > As noted, ths fixed version is OK.
> 
> Thanks.  Would you please give me an explicit Reviewed-By: ?

Reviewed-By: Joseph Myers <josmyers@redhat.com>

-- 
Joseph S. Myers
josmyers@redhat.com