[PATCH] math: Add more details to the test driver output.

[PATCH] math: Add more details to the test driver output.

[Joe Simmons-Talbott]

Add start and end indicators that identify the test being run in the
verbose output.  Better identify the builtin tests for max errors in the
summary output.  Count each exception checked for each test. Remove
double counting of tests for the check_<type> functions other than
check_float_internal. Rename print_max_error and
print_complex_max_error to check_max_error and check_complex_max_error
respectively since they have side effects.

Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
---
 math/libm-test-driver.c  |   4 +-
 math/libm-test-support.c | 196 ++++++++++++++++++++++++++++++---------
 math/libm-test-support.h |   4 +-
 3 files changed, 158 insertions(+), 46 deletions(-)

diff --git a/math/libm-test-driver.c b/math/libm-test-driver.c
index 3356f9b10d..dfb56e8cde 100644
--- a/math/libm-test-driver.c
+++ b/math/libm-test-driver.c
@@ -1083,9 +1083,9 @@ struct test_Ff_b1_data
     = STR_CON3 (FUN, SUFF, TEST_SUFF) TEST_SUFF_STR;		\
   init_max_error (this_func, EXACT, TEST_COND_any_ibm128)
 #define END					\
-  print_max_error (this_func)
+  check_max_error (this_func)
 #define END_COMPLEX				\
-  print_complex_max_error (this_func)
+  check_complex_max_error (this_func)
 
 /* Run tests for a given function in all rounding modes.  */
 #define ALL_RM_TEST(FUNC, EXACT, ARRAY, LOOP_MACRO, END_MACRO, ...)	\
diff --git a/math/libm-test-support.c b/math/libm-test-support.c
index 1d60ac783b..073560ae0c 100644
--- a/math/libm-test-support.c
+++ b/math/libm-test-support.c
@@ -112,6 +112,7 @@
 #include <argp.h>
 #include <errno.h>
 #include <string.h>
+#include <assert.h>
 
 /* This header defines func_ulps, func_real_ulps and func_imag_ulps
    arrays.  */
@@ -125,10 +126,13 @@ static FILE *ulps_file;		/* File to document difference.  */
 static int output_ulps;		/* Should ulps printed?  */
 static char *output_dir;	/* Directory where generated files will be written.  */
 
-static int noErrors;	/* number of errors */
-static int noTests;	/* number of tests (without testing exceptions) */
-static int noExcTests;	/* number of tests for exception flags */
-static int noErrnoTests;/* number of tests for errno values */
+#define TEST_INPUT 1
+#define TEST_BUILTIN 2
+static int noErrors;	   /* number of errors */
+static int noTests; 	   /* number of tests (without testing exceptions) */
+static int noBuiltinTests; /* number of builtin tests  */
+static int noExcTests;	   /* number of tests for exception flags */
+static int noErrnoTests;   /* number of tests for errno values */
 
 static int verbose;
 static int output_max_error;	/* Should the maximal errors printed?  */
@@ -299,9 +303,19 @@ print_screen_max_error (int ok)
 
 /* Update statistic counters.  */
 static void
-update_stats (int ok)
+update_stats (int ok, int testType)
 {
-  ++noTests;
+  switch (testType)
+    {
+	case TEST_INPUT:
+	  ++noTests;
+	  break;
+	case TEST_BUILTIN:
+	  ++noBuiltinTests;
+	  break;
+	default:
+	  abort();
+    }
   if (!ok)
     ++noErrors;
 }
@@ -367,11 +381,30 @@ fpstack_test (const char *test_name)
 #endif
 }
 
+static void
+print_test_start (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen(1))
+    printf ("--- Start of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_BUILTIN ? " builtin" : "", test_num, test_name);
+}
 
+static void
+print_test_end (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen(1))
+    printf ("--- End of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_BUILTIN ? " builtin" : "", test_num, test_name);
+}
+
+/* This is a builtin test of overall max error.  */
 void
-print_max_error (const char *func_name)
+check_max_error (const char *func_name)
 {
   int ok = 0;
+  int thisTest = noBuiltinTests;
+
+  print_test_start (thisTest, func_name, TEST_BUILTIN);
 
   if (max_error == 0.0 || (max_error <= prev_max_error && !ignore_max_ulp))
     {
@@ -392,14 +425,19 @@ print_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pmestr);
     }
 
-  update_stats (ok);
-}
+  update_stats (ok, TEST_BUILTIN);
 
+  print_test_end (thisTest, func_name, TEST_BUILTIN);
+}
 
+/* This is a builtin test of overall max error.  */
 void
-print_complex_max_error (const char *func_name)
+check_complex_max_error (const char *func_name)
 {
   int real_ok = 0, imag_ok = 0, ok;
+  int thisTest = noBuiltinTests;
+
+  print_test_start (thisTest, func_name, TEST_BUILTIN);
 
   if (real_max_error == 0
       || (real_max_error <= prev_real_max_error && !ignore_max_ulp))
@@ -436,7 +474,8 @@ print_complex_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pimestr);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_BUILTIN);
+  print_test_end (thisTest, func_name, TEST_BUILTIN);
 }
 
 
@@ -477,12 +516,13 @@ test_single_exception (const char *test_name,
       else
 	{
 	  if (print_screen (1))
-	    printf ("%s: Exception \"%s\" not set\n", test_name,
+	    printf ("Pass: %s: Exception \"%s\" not set\n", test_name,
 		    flag_name);
 	}
     }
   if (!ok)
     ++noErrors;
+  ++noExcTests;
 }
 #endif
 
@@ -494,7 +534,7 @@ test_exceptions (const char *test_name, int exception)
 {
   if (flag_test_exceptions && EXCEPTION_TESTS (FLOAT))
     {
-      ++noExcTests;
+      int ran = 0;
 #ifdef FE_DIVBYZERO
       if ((exception & DIVIDE_BY_ZERO_EXCEPTION_OK) == 0)
 	test_single_exception (test_name, exception,
@@ -503,14 +543,20 @@ test_exceptions (const char *test_name, int exception)
 #endif
 #ifdef FE_INVALID
       if ((exception & INVALID_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception,
-			       INVALID_EXCEPTION, FE_INVALID,
-			       "Invalid operation");
+	{
+	  test_single_exception (test_name, exception,
+			         INVALID_EXCEPTION, FE_INVALID,
+			         "Invalid operation");
+	  ran = 1;
+	}
 #endif
 #ifdef FE_OVERFLOW
       if ((exception & OVERFLOW_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
-			       FE_OVERFLOW, "Overflow");
+	{
+	  test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
+			         FE_OVERFLOW, "Overflow");
+	  ran = 1;
+	}
 #endif
       /* Spurious "underflow" and "inexact" exceptions are always
 	 allowed for IBM long double, in line with the underlying
@@ -519,17 +565,30 @@ test_exceptions (const char *test_name, int exception)
       if ((exception & UNDERFLOW_EXCEPTION_OK) == 0
 	  && !(test_ibm128
 	       && (exception & UNDERFLOW_EXCEPTION) == 0))
-	test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
-			       FE_UNDERFLOW, "Underflow");
+	{
+	  test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
+			         FE_UNDERFLOW, "Underflow");
+	  ran = 1;
+	}
+
 #endif
 #ifdef FE_INEXACT
       if ((exception & (INEXACT_EXCEPTION | NO_INEXACT_EXCEPTION)) != 0
 	  && !(test_ibm128
 	       && (exception & NO_INEXACT_EXCEPTION) != 0))
-	test_single_exception (test_name, exception, INEXACT_EXCEPTION,
-			       FE_INEXACT, "Inexact");
+	{
+	  test_single_exception (test_name, exception, INEXACT_EXCEPTION,
+			         FE_INEXACT, "Inexact");
+	  ran = 1;
+	}
 #endif
+      assert (ran == 1);
     }
+   else
+     {
+	if (print_screen (1))
+	  printf ("Info: %s: No exceptions tested\n", test_name);
+     }
   feclearexcept (FE_ALL_EXCEPT);
 }
 
@@ -561,13 +620,42 @@ test_errno (const char *test_name, int errno_value, int exceptions)
 {
   if (flag_test_errno)
     {
-      ++noErrnoTests;
+      int ran = 0;
+
+      if ((exceptions & (ERRNO_UNCHANGED|ERRNO_EDOM|ERRNO_ERANGE)) == 0)
+	{
+	  if (print_screen (1))
+	    printf ("Info: %s: The value of errno was not tested\n",
+		    test_name);
+	  return;
+	}
+
+
       if (exceptions & ERRNO_UNCHANGED)
-	test_single_errno (test_name, errno_value, 0, "unchanged");
+	{
+	  test_single_errno (test_name, errno_value, 0, "unchanged");
+	  ran = 1;
+          ++noErrnoTests;
+	}
       if (exceptions & ERRNO_EDOM)
-	test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	{
+	  test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	  ran = 1;
+          ++noErrnoTests;
+	}
       if (exceptions & ERRNO_ERANGE)
-	test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	{
+	  test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	  ran = 1;
+          ++noErrnoTests;
+	}
+
+      assert (ran == 1);
+    }
+  else
+    {
+	if (print_screen (1))
+	  printf ("Info: %s: No errno tests\n", test_name);
     }
 }
 
@@ -619,6 +707,9 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
   FLOAT diff = 0;
   FLOAT ulps = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
@@ -716,12 +807,13 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
 	  printf (" max.ulp   : %s\n", mustrn);
 	}
     }
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
 
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -776,12 +868,14 @@ check_int (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -795,11 +889,12 @@ check_int (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -810,12 +905,14 @@ check_long (const char *test_name, long int computed, long int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -829,11 +926,12 @@ check_long (const char *test_name, long int computed, long int expected,
       printf (" should be:  %ld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -844,12 +942,14 @@ check_bool (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if ((computed == 0) == (expected == 0))
     ok = 1;
 
@@ -863,11 +963,12 @@ check_bool (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -879,12 +980,14 @@ check_longlong (const char *test_name, long long int computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -898,11 +1001,12 @@ check_longlong (const char *test_name, long long int computed,
       printf (" should be:  %lld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -913,12 +1017,14 @@ check_intmax_t (const char *test_name, intmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -932,11 +1038,12 @@ check_intmax_t (const char *test_name, intmax_t computed,
       printf (" should be:  %jd\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -947,12 +1054,14 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -966,11 +1075,12 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
       printf (" should be:  %ju\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 /* Return whether a test with flags EXCEPTIONS should be run.  */
@@ -1192,9 +1302,11 @@ libm_test_finish (void)
     fclose (ulps_file);
 
   printf ("\nTest suite completed:\n");
-  printf ("  %d test cases plus %d tests for exception flags and\n"
-	  "    %d tests for errno executed.\n",
-	  noTests, noExcTests, noErrnoTests);
+  printf ("  %d builtin test cases,\n", noBuiltinTests);
+  printf ("  %d input tests,\n", noTests);
+  printf ("  - with %d tests for exception flags,\n", noExcTests);
+  printf ("  - with %d tests for errno executed.\n", noErrnoTests);
+
   if (noErrors)
     {
       printf ("  %d errors occurred.\n", noErrors);
diff --git a/math/libm-test-support.h b/math/libm-test-support.h
index 8baf7e1817..efb9523e9e 100644
--- a/math/libm-test-support.h
+++ b/math/libm-test-support.h
@@ -170,8 +170,8 @@ extern const char doc[];
 
 int enable_test (int);
 void init_max_error (const char *, int, int);
-void print_max_error (const char *);
-void print_complex_max_error (const char *);
+void check_max_error (const char *);
+void check_complex_max_error (const char *);
 void check_float (const char *, FLOAT, FLOAT, int);
 void check_complex (const char *, CFLOAT, CFLOAT, int);
 void check_int (const char *, int, int, int);
-- 
2.44.0

Re: [PATCH] math: Add more details to the test driver output.

[Joseph Myers]

On Wed, 8 May 2024, Joe Simmons-Talbott wrote:

> +#define TEST_INPUT 1
> +#define TEST_BUILTIN 2
> +static int noErrors;	   /* number of errors */
> +static int noTests; 	   /* number of tests (without testing exceptions) */
> +static int noBuiltinTests; /* number of builtin tests  */

The concept of "builtin" tests seems obscure; there are lots of things it 
could mean.  It doesn't refer to tests for built-in functions; it doesn't 
refer to all the auxiliary tests for exceptiosn and errno that are run as 
part of testing a given input value.  So I think the comment needs to 
explain what's meant by a builtin test (or, even better, find a name that 
makes it clearer, and use that name in the output that refers to the 
number of such tests as well as in variable / macro naming).

-- 
Joseph S. Myers
josmyers@redhat.com

Re: [PATCH] math: Add more details to the test driver output.

[Joe Simmons-Talbott]

On Thu, May 16, 2024 at 12:46 PM Joseph Myers <josmyers@redhat.com> wrote:
>
> On Wed, 8 May 2024, Joe Simmons-Talbott wrote:
>
> > +#define TEST_INPUT 1
> > +#define TEST_BUILTIN 2
> > +static int noErrors;    /* number of errors */
> > +static int noTests;     /* number of tests (without testing exceptions) */
> > +static int noBuiltinTests; /* number of builtin tests  */
>
> The concept of "builtin" tests seems obscure; there are lots of things it
> could mean.  It doesn't refer to tests for built-in functions; it doesn't
> refer to all the auxiliary tests for exceptiosn and errno that are run as
> part of testing a given input value.  So I think the comment needs to
> explain what's meant by a builtin test (or, even better, find a name that
> makes it clearer, and use that name in the output that refers to the
> number of such tests as well as in variable / macro naming).
>
> --
> Joseph S. Myers
> josmyers@redhat.com
>

Thank you for your review. I agree that "builtin" is probably not the
best option.  I'm working on a v2 patch that uses "maxerror" instead
since that is what is being counted here.

Thanks,
Joe

-- 
Joe Simmons-Talbott