[PATCH] benchtests: Improve benchtests for strstr

[PATCH] benchtests: Improve benchtests for strstr

[Adhemerval Zanella]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=yes, Size: 13004 bytes --]

Use same strategy as bench-strstr.c (93eebae5168e5cf2 and 80b2bfb53504)
and use json_ctx for output to help standardize format across all
benchtests.
---
 benchtests/bench-strcasestr.c | 349 ++++++++++++++++++++++++++--------
 1 file changed, 273 insertions(+), 76 deletions(-)

diff --git a/benchtests/bench-strcasestr.c b/benchtests/bench-strcasestr.c
index f6d1a78fba..84a0bef38f 100644
--- a/benchtests/bench-strcasestr.c
+++ b/benchtests/bench-strcasestr.c
@@ -16,10 +16,36 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define MIN_PAGE_SIZE 131072
 #define TEST_MAIN
 #define TEST_NAME "strcasestr"
 #include "bench-string.h"
 
+#include "json-lib.h"
+
+static const char input[] =
+"This manual is written with the assumption that you are at least "
+"somewhat familiar with the C programming language and basic programming "
+"concepts.  Specifically, familiarity with ISO standard C (*note ISO "
+"C::), rather than “traditional” pre-ISO C dialects, is assumed.\n"
+
+"   The GNU C Library includes several “header files”, each of which "
+"provides definitions and declarations for a group of related facilities; "
+"this information is used by the C compiler when processing your program. "
+"For example, the header file ‘stdio.h’ declares facilities for "
+"performing input and output, and the header file ‘string.h’ declares "
+"string processing utilities.  The organization of this manual generally "
+"follows the same division as the header files.\n"
+
+"   If you are reading this manual for the first time, you should read "
+"all of the introductory material and skim the remaining chapters.  There "
+"are a _lot_ of functions in the GNU C Library and it’s not realistic to "
+"expect that you will be able to remember exactly _how_ to use each and "
+"every one of them.  It’s more important to become generally familiar "
+"with the kinds of facilities that the library provides, so that when you "
+"are writing your programs you can recognize _when_ to make use of "
+"library functions, and _where_ in this manual you can find more specific "
+"information about them.\n";
 
 #define STRCASESTR simple_strcasestr
 #define NO_ALIAS
@@ -32,123 +58,294 @@ typedef char *(*proto_t) (const char *, const char *);
 IMPL (simple_strcasestr, 0)
 IMPL (strcasestr, 1)
 
-
 static void
-do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
+	     const char *s2, char *exp_result)
 {
-  size_t i, iters = INNER_LOOP_ITERS_SMALL;
+  size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
   timing_t start, stop, cur;
+  char *res;
 
   TIMING_NOW (start);
   for (i = 0; i < iters; ++i)
-    {
-      CALL (impl, s1, s2);
-    }
+    res = CALL (impl, s1, s2);
   TIMING_NOW (stop);
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
-}
+  json_element_double (json_ctx, (double) cur / (double) iters);
 
+  if (res != exp_result)
+    {
+      error (0, 0, "Wrong result in function %s %s %s", impl->name,
+	     (res == NULL) ? "(null)" : res,
+	     (exp_result == NULL) ? "(null)" : exp_result);
+      ret = 1;
+    }
+}
 
 static void
-do_test (size_t align1, size_t align2, size_t len1, size_t len2,
-	 int fail)
+do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
+	 size_t len2, int fail)
 {
   char *s1 = (char *) (buf1 + align1);
   char *s2 = (char *) (buf2 + align2);
 
-  static const char d[] = "1234567890abcxyz";
-#define dl (sizeof (d) - 1)
-  char *ss2 = s2;
-  for (size_t l = len2; l > 0; l = l > dl ? l - dl : 0)
-    {
-      size_t t = l > dl ? dl : l;
-      ss2 = mempcpy (ss2, d, t);
-    }
-  s2[len2] = '\0';
+  size_t size = sizeof (input) - 1;
+  size_t pos = (len1 + len2) % size;
 
-  if (fail)
+  char *ss2 = s2;
+  for (size_t l = len2; l > 0; l = l > size ? l - size : 0)
     {
-      char *ss1 = s1;
-      for (size_t l = len1; l > 0; l = l > dl ? l - dl : 0)
+      size_t t = l > size ? size : l;
+      if (pos + t <= size)
+	ss2 = mempcpy (ss2, input + pos, t);
+      else
 	{
-	  size_t t = l > dl ? dl : l;
-	  memcpy (ss1, d, t);
-	  ++ss1[len2 > 7 ? 7 : len2 - 1];
-	  ss1 += t;
+	  ss2 = mempcpy (ss2, input + pos, size - pos);
+	  ss2 = mempcpy (ss2, input, t - (size - pos));
 	}
     }
-  else
+  s2[len2] = '\0';
+
+  char *ss1 = s1;
+  for (size_t l = len1; l > 0; l = l > size ? l - size : 0)
     {
-      memset (s1, '0', len1);
-      for (size_t i = 0; i < len2; ++i)
-	s1[len1 - len2 + i] = toupper (s2[i]);
+      size_t t = l > size ? size : l;
+      memcpy (ss1, input, t);
+      ss1 += t;
     }
+
+  if (!fail)
+    memcpy (s1 + len1 - len2, s2, len2);
   s1[len1] = '\0';
 
-  printf ("Length %4zd/%zd, alignment %2zd/%2zd, %s:",
-	  len1, len2, align1, align2, fail ? "fail" : "found");
+  /* Remove any accidental matches except for the last if !fail.  */
+  for (ss1 = simple_strcasestr (s1, s2);
+       ss1 != NULL;
+       ss1 = simple_strcasestr (ss1 + 1, s2))
+    if (fail || ss1 != s1 + len1 - len2)
+      ++ss1[len2 / 2];
+
+  json_element_object_begin (json_ctx);
+  json_attr_uint (json_ctx, "len_haystack", len1);
+  json_attr_uint (json_ctx, "len_needle", len2);
+  json_attr_uint (json_ctx, "align_haystack", align1);
+  json_attr_uint (json_ctx, "align_needle", align2);
+  json_attr_uint (json_ctx, "fail", fail);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 
-  putchar ('\n');
+}
+
+/* Test needles which exhibit worst-case performance for naive quadradic
+   implementations.  */
+
+static void
+test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
+{
+  char *ne = (char *) buf1;
+  char *hs = (char *) buf2;
+
+  /* Hard needle for strstr algorithm using skip table.  This results in many
+     memcmp calls comparing most of the needle.  */
+  {
+    memset (ne, 'a', ne_len);
+    ne[ne_len] = '\0';
+    ne[ne_len - 14] = 'b';
+
+    memset (hs, 'a', hs_len);
+    for (size_t i = ne_len; i <= hs_len; i += ne_len)
+      {
+	hs[i - 5] = 'b';
+	hs[i - 62] = 'b';
+      }
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
+
+  /* 2nd hard needle for strstr algorithm using skip table.  This results in
+     many memcmp calls comparing most of the needle.  */
+  {
+    memset (ne, 'a', ne_len);
+    ne[ne_len] = '\0';
+    ne[ne_len - 6] = 'b';
+
+    memset (hs, 'a', hs_len);
+    for (size_t i = ne_len; i <= hs_len; i += ne_len)
+      {
+	hs[i - 5] = 'b';
+	hs[i - 6] = 'b';
+      }
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
+
+  /* Hard needle for Two-way algorithm - the random input causes a large number
+     of branch mispredictions which significantly reduces performance on modern
+     micro architectures.  */
+  {
+    for (int i = 0; i < hs_len; i++)
+      hs[i] = (rand () & 255) > 155 ? 'a' : 'b';
+    hs[hs_len] = 0;
+
+    memset (ne, 'a', ne_len);
+    ne[ne_len - 2] = 'b';
+    ne[0] = 'b';
+    ne[ne_len] = 0;
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult 2-way");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
+
+  /* Hard needle for standard algorithm testing first few characters of
+   * needle.  */
+  {
+    for (int i = 0; i < hs_len; i++)
+      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
+    hs[hs_len] = 0;
+
+    for (int i = 0; i < ne_len; i++)
+      {
+	if (i % 3 == 0)
+	  ne[i] = 'a';
+	else if (i % 3 == 1)
+	  ne[i] = 'b';
+	else
+	  ne[i] = 'c';
+      }
+    ne[ne_len] = 0;
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
 }
 
 static int
 test_main (void)
 {
+  json_ctx_t json_ctx;
   test_init ();
 
-  printf ("%23s", "");
+  json_init (&json_ctx, 0, stdout);
+
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
-  putchar ('\n');
+    json_element_string (&json_ctx, impl->name);
+  json_array_end (&json_ctx);
 
-  for (size_t klen = 2; klen < 32; ++klen)
-    for (size_t hlen = 2 * klen; hlen < 16 * klen; hlen += klen)
+  json_array_begin (&json_ctx, "results");
+
+  for (size_t hlen = 8; hlen <= 256;)
+    for (size_t klen = 1; klen <= 16; klen++)
       {
-	do_test (0, 0, hlen, klen, 0);
-	do_test (0, 0, hlen, klen, 1);
-	do_test (0, 3, hlen, klen, 0);
-	do_test (0, 3, hlen, klen, 1);
-	do_test (0, 9, hlen, klen, 0);
-	do_test (0, 9, hlen, klen, 1);
-	do_test (0, 15, hlen, klen, 0);
-	do_test (0, 15, hlen, klen, 1);
-
-	do_test (3, 0, hlen, klen, 0);
-	do_test (3, 0, hlen, klen, 1);
-	do_test (3, 3, hlen, klen, 0);
-	do_test (3, 3, hlen, klen, 1);
-	do_test (3, 9, hlen, klen, 0);
-	do_test (3, 9, hlen, klen, 1);
-	do_test (3, 15, hlen, klen, 0);
-	do_test (3, 15, hlen, klen, 1);
-
-	do_test (9, 0, hlen, klen, 0);
-	do_test (9, 0, hlen, klen, 1);
-	do_test (9, 3, hlen, klen, 0);
-	do_test (9, 3, hlen, klen, 1);
-	do_test (9, 9, hlen, klen, 0);
-	do_test (9, 9, hlen, klen, 1);
-	do_test (9, 15, hlen, klen, 0);
-	do_test (9, 15, hlen, klen, 1);
-
-	do_test (15, 0, hlen, klen, 0);
-	do_test (15, 0, hlen, klen, 1);
-	do_test (15, 3, hlen, klen, 0);
-	do_test (15, 3, hlen, klen, 1);
-	do_test (15, 9, hlen, klen, 0);
-	do_test (15, 9, hlen, klen, 1);
-	do_test (15, 15, hlen, klen, 0);
-	do_test (15, 15, hlen, klen, 1);
+	do_test (&json_ctx, 1, 3, hlen, klen, 0);
+	do_test (&json_ctx, 0, 9, hlen, klen, 1);
+
+	do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
+	do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
+	if (hlen < 64)
+	  {
+	    hlen += 8;
+	  }
+	else
+	  {
+	    hlen += 32;
+	  }
+      }
+
+  for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
+    for (size_t klen = 4; klen <= 256; klen *= 2)
+      {
+	do_test (&json_ctx, 1, 11, hlen, klen, 0);
+	do_test (&json_ctx, 14, 5, hlen, klen, 1);
+
+    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
+    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
+
+	do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
+	do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
       }
 
-  do_test (0, 0, page_size - 1, 16, 0);
-  do_test (0, 0, page_size - 1, 16, 1);
+  test_hard_needle (&json_ctx, 64, 65536);
+  test_hard_needle (&json_ctx, 256, 65536);
+  test_hard_needle (&json_ctx, 1024, 65536);
+
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
 
   return ret;
 }
-- 
2.34.1

Re: [PATCH] benchtests: Improve benchtests for strstr

[Andreas Schwab]

> Content-Type: text/plain; charset=yes

yes is not a valid charset.

-- 
Andreas Schwab, SUSE Labs, schwab@suse.de
GPG Key fingerprint = 0196 BAD8 1CE9 1970 F4BE  1748 E4D4 88E3 0EEA B9D7
"And now for something completely different."

Re: [PATCH] benchtests: Improve benchtests for strstr

[Adhemerval Zanella Netto]

On 18/03/24 09:44, Andreas Schwab wrote:
>> Content-Type: text/plain; charset=yes
> 
> yes is not a valid charset.
> 

Hum, I am not sure why I getting this.

Re: [PATCH] benchtests: Improve benchtests for strstr

[Andreas Schwab]

git config sendemail.assume8bitEncoding UTF-8

-- 
Andreas Schwab, SUSE Labs, schwab@suse.de
GPG Key fingerprint = 0196 BAD8 1CE9 1970 F4BE  1748 E4D4 88E3 0EEA B9D7
"And now for something completely different."

Re: [PATCH] benchtests: Improve benchtests for strstr

[Adhemerval Zanella Netto]

On 18/03/24 10:05, Andreas Schwab wrote:
> git config sendemail.assume8bitEncoding UTF-8
> 

Thanks!

Re: [PATCH] benchtests: Improve benchtests for strstr

[Noah Goldstein]

On Mon, Mar 18, 2024 at 7:28 AM Adhemerval Zanella
<adhemerval.zanella@linaro.org> wrote:
>
> Use same strategy as bench-strstr.c (93eebae5168e5cf2 and 80b2bfb53504)
> and use json_ctx for output to help standardize format across all
> benchtests.
> ---
>  benchtests/bench-strcasestr.c | 349 ++++++++++++++++++++++++++--------
>  1 file changed, 273 insertions(+), 76 deletions(-)
>
> diff --git a/benchtests/bench-strcasestr.c b/benchtests/bench-strcasestr.c
> index f6d1a78fba..84a0bef38f 100644
> --- a/benchtests/bench-strcasestr.c
> +++ b/benchtests/bench-strcasestr.c
> @@ -16,10 +16,36 @@
>     License along with the GNU C Library; if not, see
>     <https://www.gnu.org/licenses/>.  */
>
> +#define MIN_PAGE_SIZE 131072
>  #define TEST_MAIN
>  #define TEST_NAME "strcasestr"
>  #include "bench-string.h"
>
> +#include "json-lib.h"
> +
> +static const char input[] =
> +"This manual is written with the assumption that you are at least "
> +"somewhat familiar with the C programming language and basic programming "
> +"concepts.  Specifically, familiarity with ISO standard C (*note ISO "
> +"C::), rather than “traditional” pre-ISO C dialects, is assumed.\n"
> +
> +"   The GNU C Library includes several “header files”, each of which "
> +"provides definitions and declarations for a group of related facilities; "
> +"this information is used by the C compiler when processing your program. "
> +"For example, the header file ‘stdio.h’ declares facilities for "
> +"performing input and output, and the header file ‘string.h’ declares "
> +"string processing utilities.  The organization of this manual generally "
> +"follows the same division as the header files.\n"
> +
> +"   If you are reading this manual for the first time, you should read "
> +"all of the introductory material and skim the remaining chapters.  There "
> +"are a _lot_ of functions in the GNU C Library and it’s not realistic to "
> +"expect that you will be able to remember exactly _how_ to use each and "
> +"every one of them.  It’s more important to become generally familiar "
> +"with the kinds of facilities that the library provides, so that when you "
> +"are writing your programs you can recognize _when_ to make use of "
> +"library functions, and _where_ in this manual you can find more specific "
> +"information about them.\n";
>
>  #define STRCASESTR simple_strcasestr
>  #define NO_ALIAS
> @@ -32,123 +58,294 @@ typedef char *(*proto_t) (const char *, const char *);
>  IMPL (simple_strcasestr, 0)
>  IMPL (strcasestr, 1)
>
> -
>  static void
> -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> +            const char *s2, char *exp_result)
>  {
> -  size_t i, iters = INNER_LOOP_ITERS_SMALL;
> +  size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
was it taking to long? 256 doesn't seem like so much for this.
>    timing_t start, stop, cur;
> +  char *res;
>
>    TIMING_NOW (start);
>    for (i = 0; i < iters; ++i)
> -    {
> -      CALL (impl, s1, s2);
> -    }
> +    res = CALL (impl, s1, s2);
>    TIMING_NOW (stop);
>
>    TIMING_DIFF (cur, start, stop);
>
> -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> -}
> +  json_element_double (json_ctx, (double) cur / (double) iters);
>
> +  if (res != exp_result)
> +    {
> +      error (0, 0, "Wrong result in function %s %s %s", impl->name,
> +            (res == NULL) ? "(null)" : res,
> +            (exp_result == NULL) ? "(null)" : exp_result);
> +      ret = 1;
> +    }
> +}
>
>  static void
> -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> -        int fail)
> +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> +        size_t len2, int fail)
>  {
>    char *s1 = (char *) (buf1 + align1);
>    char *s2 = (char *) (buf2 + align2);
>
> -  static const char d[] = "1234567890abcxyz";
> -#define dl (sizeof (d) - 1)
> -  char *ss2 = s2;
> -  for (size_t l = len2; l > 0; l = l > dl ? l - dl : 0)
> -    {
> -      size_t t = l > dl ? dl : l;
> -      ss2 = mempcpy (ss2, d, t);
> -    }
> -  s2[len2] = '\0';
> +  size_t size = sizeof (input) - 1;
> +  size_t pos = (len1 + len2) % size;
>
> -  if (fail)
> +  char *ss2 = s2;
> +  for (size_t l = len2; l > 0; l = l > size ? l - size : 0)
>      {
> -      char *ss1 = s1;
> -      for (size_t l = len1; l > 0; l = l > dl ? l - dl : 0)
> +      size_t t = l > size ? size : l;
> +      if (pos + t <= size)
> +       ss2 = mempcpy (ss2, input + pos, t);
> +      else
>         {
> -         size_t t = l > dl ? dl : l;
> -         memcpy (ss1, d, t);
> -         ++ss1[len2 > 7 ? 7 : len2 - 1];
> -         ss1 += t;
> +         ss2 = mempcpy (ss2, input + pos, size - pos);
> +         ss2 = mempcpy (ss2, input, t - (size - pos));
>         }
>      }
> -  else
> +  s2[len2] = '\0';
> +
> +  char *ss1 = s1;
> +  for (size_t l = len1; l > 0; l = l > size ? l - size : 0)
>      {
> -      memset (s1, '0', len1);
> -      for (size_t i = 0; i < len2; ++i)
> -       s1[len1 - len2 + i] = toupper (s2[i]);
> +      size_t t = l > size ? size : l;
> +      memcpy (ss1, input, t);
> +      ss1 += t;
>      }
> +
> +  if (!fail)
> +    memcpy (s1 + len1 - len2, s2, len2);
>    s1[len1] = '\0';
>
> -  printf ("Length %4zd/%zd, alignment %2zd/%2zd, %s:",
> -         len1, len2, align1, align2, fail ? "fail" : "found");
> +  /* Remove any accidental matches except for the last if !fail.  */
> +  for (ss1 = simple_strcasestr (s1, s2);
> +       ss1 != NULL;
> +       ss1 = simple_strcasestr (ss1 + 1, s2))
> +    if (fail || ss1 != s1 + len1 - len2)
> +      ++ss1[len2 / 2];
> +
> +  json_element_object_begin (json_ctx);
> +  json_attr_uint (json_ctx, "len_haystack", len1);
> +  json_attr_uint (json_ctx, "len_needle", len2);
> +  json_attr_uint (json_ctx, "align_haystack", align1);
> +  json_attr_uint (json_ctx, "align_needle", align2);
> +  json_attr_uint (json_ctx, "fail", fail);
> +
> +  json_array_begin (json_ctx, "timings");
>
>    FOR_EACH_IMPL (impl, 0)
> -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +
> +  json_array_end (json_ctx);
> +  json_element_object_end (json_ctx);
>
> -  putchar ('\n');
> +}
> +
> +/* Test needles which exhibit worst-case performance for naive quadradic
> +   implementations.  */
> +
> +static void
> +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
> +{
> +  char *ne = (char *) buf1;
> +  char *hs = (char *) buf2;
> +
> +  /* Hard needle for strstr algorithm using skip table.  This results in many
> +     memcmp calls comparing most of the needle.  */
> +  {
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len] = '\0';
> +    ne[ne_len - 14] = 'b';
> +
> +    memset (hs, 'a', hs_len);
> +    for (size_t i = ne_len; i <= hs_len; i += ne_len)
> +      {
> +       hs[i - 5] = 'b';
> +       hs[i - 62] = 'b';
> +      }
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* 2nd hard needle for strstr algorithm using skip table.  This results in
> +     many memcmp calls comparing most of the needle.  */
> +  {
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len] = '\0';
> +    ne[ne_len - 6] = 'b';
> +
> +    memset (hs, 'a', hs_len);
> +    for (size_t i = ne_len; i <= hs_len; i += ne_len)
> +      {
> +       hs[i - 5] = 'b';
> +       hs[i - 6] = 'b';
> +      }
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* Hard needle for Two-way algorithm - the random input causes a large number
> +     of branch mispredictions which significantly reduces performance on modern
> +     micro architectures.  */
> +  {
> +    for (int i = 0; i < hs_len; i++)
> +      hs[i] = (rand () & 255) > 155 ? 'a' : 'b';
> +    hs[hs_len] = 0;
> +
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len - 2] = 'b';
> +    ne[0] = 'b';
> +    ne[ne_len] = 0;
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* Hard needle for standard algorithm testing first few characters of
> +   * needle.  */
> +  {
> +    for (int i = 0; i < hs_len; i++)
> +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> +    hs[hs_len] = 0;
> +
> +    for (int i = 0; i < ne_len; i++)
> +      {
> +       if (i % 3 == 0)
> +         ne[i] = 'a';
> +       else if (i % 3 == 1)
> +         ne[i] = 'b';
> +       else
> +         ne[i] = 'c';
> +      }
> +    ne[ne_len] = 0;
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
>  }
>
>  static int
>  test_main (void)
>  {
> +  json_ctx_t json_ctx;
>    test_init ();
>
> -  printf ("%23s", "");
> +  json_init (&json_ctx, 0, stdout);
> +
> +  json_document_begin (&json_ctx);
> +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> +
> +  json_attr_object_begin (&json_ctx, "functions");
> +  json_attr_object_begin (&json_ctx, TEST_NAME);
> +  json_attr_string (&json_ctx, "bench-variant", "");
> +
> +  json_array_begin (&json_ctx, "ifuncs");
>    FOR_EACH_IMPL (impl, 0)
> -    printf ("\t%s", impl->name);
> -  putchar ('\n');
> +    json_element_string (&json_ctx, impl->name);
> +  json_array_end (&json_ctx);
>
> -  for (size_t klen = 2; klen < 32; ++klen)
> -    for (size_t hlen = 2 * klen; hlen < 16 * klen; hlen += klen)
> +  json_array_begin (&json_ctx, "results");
> +
> +  for (size_t hlen = 8; hlen <= 256;)
> +    for (size_t klen = 1; klen <= 16; klen++)
>        {
> -       do_test (0, 0, hlen, klen, 0);
> -       do_test (0, 0, hlen, klen, 1);
> -       do_test (0, 3, hlen, klen, 0);
> -       do_test (0, 3, hlen, klen, 1);
> -       do_test (0, 9, hlen, klen, 0);
> -       do_test (0, 9, hlen, klen, 1);
> -       do_test (0, 15, hlen, klen, 0);
> -       do_test (0, 15, hlen, klen, 1);
> -
> -       do_test (3, 0, hlen, klen, 0);
> -       do_test (3, 0, hlen, klen, 1);
> -       do_test (3, 3, hlen, klen, 0);
> -       do_test (3, 3, hlen, klen, 1);
> -       do_test (3, 9, hlen, klen, 0);
> -       do_test (3, 9, hlen, klen, 1);
> -       do_test (3, 15, hlen, klen, 0);
> -       do_test (3, 15, hlen, klen, 1);
> -
> -       do_test (9, 0, hlen, klen, 0);
> -       do_test (9, 0, hlen, klen, 1);
> -       do_test (9, 3, hlen, klen, 0);
> -       do_test (9, 3, hlen, klen, 1);
> -       do_test (9, 9, hlen, klen, 0);
> -       do_test (9, 9, hlen, klen, 1);
> -       do_test (9, 15, hlen, klen, 0);
> -       do_test (9, 15, hlen, klen, 1);
> -
> -       do_test (15, 0, hlen, klen, 0);
> -       do_test (15, 0, hlen, klen, 1);
> -       do_test (15, 3, hlen, klen, 0);
> -       do_test (15, 3, hlen, klen, 1);
> -       do_test (15, 9, hlen, klen, 0);
> -       do_test (15, 9, hlen, klen, 1);
> -       do_test (15, 15, hlen, klen, 0);
> -       do_test (15, 15, hlen, klen, 1);
> +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> +
> +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> +       if (hlen < 64)
> +         {
> +           hlen += 8;
> +         }
> +       else
> +         {
> +           hlen += 32;
> +         }
> +      }
> +
> +  for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> +    for (size_t klen = 4; klen <= 256; klen *= 2)
> +      {
> +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> +
> +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> +
> +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
>        }
>
> -  do_test (0, 0, page_size - 1, 16, 0);
> -  do_test (0, 0, page_size - 1, 16, 1);
> +  test_hard_needle (&json_ctx, 64, 65536);
> +  test_hard_needle (&json_ctx, 256, 65536);
> +  test_hard_needle (&json_ctx, 1024, 65536);
> +
> +  json_array_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_document_end (&json_ctx);
>
>    return ret;
>  }
> --
> 2.34.1
>

Re: [PATCH] benchtests: Improve benchtests for strstr

[Arjun Shankar]

Hi Adhemerval,

Overall, this looks good to me. I double checked the JSON output and
it is well formed. Subject has a typo (strcasestr). With the typo
fixed:

Reviewed-by: Arjun Shankar <arjun@redhat.com>

I have commented on the iteration count below, but it's not an
essential change for my Reviewed-by to apply.

> Use same strategy as bench-strstr.c (93eebae5168e5cf2 and 80b2bfb53504)
> and use json_ctx for output to help standardize format across all
> benchtests.

OK.

> --- a/benchtests/bench-strcasestr.c
> +++ b/benchtests/bench-strcasestr.c
> @@ -16,10 +16,36 @@
>     License along with the GNU C Library; if not, see
>     <https://www.gnu.org/licenses/>.  */
>
> +#define MIN_PAGE_SIZE 131072

OK. Used by bench-string.h. Aligns with the strstr benchtest.

>  #define TEST_MAIN
>  #define TEST_NAME "strcasestr"
>  #include "bench-string.h"
>
> +#include "json-lib.h"

OK. We will use JSON for output.

> +
> +static const char input[] =
> +"This manual is written with the assumption that you are at least "
> +"somewhat familiar with the C programming language and basic programming "
> +"concepts.  Specifically, familiarity with ISO standard C (*note ISO "
> +"C::), rather than “traditional�€? pre-ISO C dialects, is assumed.\n"
> +
> +"   The GNU C Library includes several “header files�€?, each of which "
> +"provides definitions and declarations for a group of related facilities; "
> +"this information is used by the C compiler when processing your program. "
> +"For example, the header file ‘stdio.h’ declares facilities for "
> +"performing input and output, and the header file ‘string.h’ declares "
> +"string processing utilities.  The organization of this manual generally "
> +"follows the same division as the header files.\n"
> +
> +"   If you are reading this manual for the first time, you should read "
> +"all of the introductory material and skim the remaining chapters.  There "
> +"are a _lot_ of functions in the GNU C Library and it’s not realistic to "
> +"expect that you will be able to remember exactly _how_ to use each and "
> +"every one of them.  It’s more important to become generally familiar "
> +"with the kinds of facilities that the library provides, so that when you "
> +"are writing your programs you can recognize _when_ to make use of "
> +"library functions, and _where_ in this manual you can find more specific "
> +"information about them.\n";

OK. New input, same as strstr.

>
>  #define STRCASESTR simple_strcasestr
>  #define NO_ALIAS
> @@ -32,123 +58,294 @@ typedef char *(*proto_t) (const char *, const char *);
>  IMPL (simple_strcasestr, 0)
>  IMPL (strcasestr, 1)
>
> -
>  static void
> -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> +            const char *s2, char *exp_result)

OK. New argument because we will do JSON output.

>  {
> -  size_t i, iters = INNER_LOOP_ITERS_SMALL;
> +  size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;

I saw Noah's comment on this so I compared timings with the previous
version. Looks like (on my laptop) the benchmark used to take ~0.7s
earlier, vs ~0.3s now -- which goes up to over 2.2s if we stuck with
the old iteration count here. I'm neutral about this change. Just
wanted to address Noah's question. Without any personal preference:
maybe "/ 4" instead of "/ 8" will bring the runtime in line with the
previous (at least based on my runs)?

>    timing_t start, stop, cur;
> +  char *res;
>
>    TIMING_NOW (start);
>    for (i = 0; i < iters; ++i)
> -    {
> -      CALL (impl, s1, s2);
> -    }
> +    res = CALL (impl, s1, s2);
>    TIMING_NOW (stop);
>
>    TIMING_DIFF (cur, start, stop);
>
> -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> -}
> +  json_element_double (json_ctx, (double) cur / (double) iters);
>
> +  if (res != exp_result)
> +    {
> +      error (0, 0, "Wrong result in function %s %s %s", impl->name,
> +            (res == NULL) ? "(null)" : res,
> +            (exp_result == NULL) ? "(null)" : exp_result);
> +      ret = 1;
> +    }
> +}

OK. After changes, this bit matches bench-strstr line for line. The
functions are very similar, so having these benchmarks line up makes
sense.

>
>  static void
> -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> -        int fail)
> +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> +        size_t len2, int fail)
>  {
>    char *s1 = (char *) (buf1 + align1);
>    char *s2 = (char *) (buf2 + align2);
>
> -  static const char d[] = "1234567890abcxyz";
> -#define dl (sizeof (d) - 1)
> -  char *ss2 = s2;
> -  for (size_t l = len2; l > 0; l = l > dl ? l - dl : 0)
> -    {
> -      size_t t = l > dl ? dl : l;
> -      ss2 = mempcpy (ss2, d, t);
> -    }
> -  s2[len2] = '\0';
> +  size_t size = sizeof (input) - 1;
> +  size_t pos = (len1 + len2) % size;
>
> -  if (fail)
> +  char *ss2 = s2;
> +  for (size_t l = len2; l > 0; l = l > size ? l - size : 0)
>      {
> -      char *ss1 = s1;
> -      for (size_t l = len1; l > 0; l = l > dl ? l - dl : 0)
> +      size_t t = l > size ? size : l;
> +      if (pos + t <= size)
> +       ss2 = mempcpy (ss2, input + pos, t);
> +      else
>         {
> -         size_t t = l > dl ? dl : l;
> -         memcpy (ss1, d, t);
> -         ++ss1[len2 > 7 ? 7 : len2 - 1];
> -         ss1 += t;
> +         ss2 = mempcpy (ss2, input + pos, size - pos);
> +         ss2 = mempcpy (ss2, input, t - (size - pos));
>         }
>      }
> -  else
> +  s2[len2] = '\0';
> +
> +  char *ss1 = s1;
> +  for (size_t l = len1; l > 0; l = l > size ? l - size : 0)
>      {
> -      memset (s1, '0', len1);
> -      for (size_t i = 0; i < len2; ++i)
> -       s1[len1 - len2 + i] = toupper (s2[i]);
> +      size_t t = l > size ? size : l;
> +      memcpy (ss1, input, t);
> +      ss1 += t;
>      }
> +
> +  if (!fail)
> +    memcpy (s1 + len1 - len2, s2, len2);
>    s1[len1] = '\0';
>
> -  printf ("Length %4zd/%zd, alignment %2zd/%2zd, %s:",
> -         len1, len2, align1, align2, fail ? "fail" : "found");
> +  /* Remove any accidental matches except for the last if !fail.  */
> +  for (ss1 = simple_strcasestr (s1, s2);
> +       ss1 != NULL;
> +       ss1 = simple_strcasestr (ss1 + 1, s2))
> +    if (fail || ss1 != s1 + len1 - len2)
> +      ++ss1[len2 / 2];
> +
> +  json_element_object_begin (json_ctx);
> +  json_attr_uint (json_ctx, "len_haystack", len1);
> +  json_attr_uint (json_ctx, "len_needle", len2);
> +  json_attr_uint (json_ctx, "align_haystack", align1);
> +  json_attr_uint (json_ctx, "align_needle", align2);
> +  json_attr_uint (json_ctx, "fail", fail);
> +
> +  json_array_begin (json_ctx, "timings");
>
>    FOR_EACH_IMPL (impl, 0)
> -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +
> +  json_array_end (json_ctx);
> +  json_element_object_end (json_ctx);
>
> -  putchar ('\n');
> +}
> +

OK. Again, the result matches bench-strstr minus the differences in
the for loop.

> +/* Test needles which exhibit worst-case performance for naive quadradic
> +   implementations.  */
> +

OK.

> +static void
> +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
> +{
> +  char *ne = (char *) buf1;
> +  char *hs = (char *) buf2;
> +
> +  /* Hard needle for strstr algorithm using skip table.  This results in many
> +     memcmp calls comparing most of the needle.  */
> +  {
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len] = '\0';
> +    ne[ne_len - 14] = 'b';
> +
> +    memset (hs, 'a', hs_len);
> +    for (size_t i = ne_len; i <= hs_len; i += ne_len)
> +      {
> +       hs[i - 5] = 'b';
> +       hs[i - 62] = 'b';
> +      }
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* 2nd hard needle for strstr algorithm using skip table.  This results in
> +     many memcmp calls comparing most of the needle.  */
> +  {
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len] = '\0';
> +    ne[ne_len - 6] = 'b';
> +
> +    memset (hs, 'a', hs_len);
> +    for (size_t i = ne_len; i <= hs_len; i += ne_len)
> +      {
> +       hs[i - 5] = 'b';
> +       hs[i - 6] = 'b';
> +      }
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* Hard needle for Two-way algorithm - the random input causes a large number
> +     of branch mispredictions which significantly reduces performance on modern
> +     micro architectures.  */
> +  {
> +    for (int i = 0; i < hs_len; i++)
> +      hs[i] = (rand () & 255) > 155 ? 'a' : 'b';
> +    hs[hs_len] = 0;
> +
> +    memset (ne, 'a', ne_len);
> +    ne[ne_len - 2] = 'b';
> +    ne[0] = 'b';
> +    ne[ne_len] = 0;
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* Hard needle for standard algorithm testing first few characters of
> +   * needle.  */
> +  {
> +    for (int i = 0; i < hs_len; i++)
> +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> +    hs[hs_len] = 0;
> +
> +    for (int i = 0; i < ne_len; i++)
> +      {
> +       if (i % 3 == 0)
> +         ne[i] = 'a';
> +       else if (i % 3 == 1)
> +         ne[i] = 'b';
> +       else
> +         ne[i] = 'c';
> +      }
> +    ne[ne_len] = 0;
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
>  }
>
>  static int
>  test_main (void)
>  {
> +  json_ctx_t json_ctx;
>    test_init ();
>
> -  printf ("%23s", "");
> +  json_init (&json_ctx, 0, stdout);
> +
> +  json_document_begin (&json_ctx);
> +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> +
> +  json_attr_object_begin (&json_ctx, "functions");
> +  json_attr_object_begin (&json_ctx, TEST_NAME);
> +  json_attr_string (&json_ctx, "bench-variant", "");
> +
> +  json_array_begin (&json_ctx, "ifuncs");
>    FOR_EACH_IMPL (impl, 0)
> -    printf ("\t%s", impl->name);
> -  putchar ('\n');
> +    json_element_string (&json_ctx, impl->name);
> +  json_array_end (&json_ctx);
>
> -  for (size_t klen = 2; klen < 32; ++klen)
> -    for (size_t hlen = 2 * klen; hlen < 16 * klen; hlen += klen)
> +  json_array_begin (&json_ctx, "results");
> +
> +  for (size_t hlen = 8; hlen <= 256;)
> +    for (size_t klen = 1; klen <= 16; klen++)
>        {
> -       do_test (0, 0, hlen, klen, 0);
> -       do_test (0, 0, hlen, klen, 1);
> -       do_test (0, 3, hlen, klen, 0);
> -       do_test (0, 3, hlen, klen, 1);
> -       do_test (0, 9, hlen, klen, 0);
> -       do_test (0, 9, hlen, klen, 1);
> -       do_test (0, 15, hlen, klen, 0);
> -       do_test (0, 15, hlen, klen, 1);
> -
> -       do_test (3, 0, hlen, klen, 0);
> -       do_test (3, 0, hlen, klen, 1);
> -       do_test (3, 3, hlen, klen, 0);
> -       do_test (3, 3, hlen, klen, 1);
> -       do_test (3, 9, hlen, klen, 0);
> -       do_test (3, 9, hlen, klen, 1);
> -       do_test (3, 15, hlen, klen, 0);
> -       do_test (3, 15, hlen, klen, 1);
> -
> -       do_test (9, 0, hlen, klen, 0);
> -       do_test (9, 0, hlen, klen, 1);
> -       do_test (9, 3, hlen, klen, 0);
> -       do_test (9, 3, hlen, klen, 1);
> -       do_test (9, 9, hlen, klen, 0);
> -       do_test (9, 9, hlen, klen, 1);
> -       do_test (9, 15, hlen, klen, 0);
> -       do_test (9, 15, hlen, klen, 1);
> -
> -       do_test (15, 0, hlen, klen, 0);
> -       do_test (15, 0, hlen, klen, 1);
> -       do_test (15, 3, hlen, klen, 0);
> -       do_test (15, 3, hlen, klen, 1);
> -       do_test (15, 9, hlen, klen, 0);
> -       do_test (15, 9, hlen, klen, 1);
> -       do_test (15, 15, hlen, klen, 0);
> -       do_test (15, 15, hlen, klen, 1);
> +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> +
> +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> +       if (hlen < 64)
> +         {
> +           hlen += 8;
> +         }
> +       else
> +         {
> +           hlen += 32;
> +         }
> +      }
> +
> +  for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> +    for (size_t klen = 4; klen <= 256; klen *= 2)
> +      {
> +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> +
> +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> +
> +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
>        }
>
> -  do_test (0, 0, page_size - 1, 16, 0);
> -  do_test (0, 0, page_size - 1, 16, 1);
> +  test_hard_needle (&json_ctx, 64, 65536);
> +  test_hard_needle (&json_ctx, 256, 65536);
> +  test_hard_needle (&json_ctx, 1024, 65536);
> +
> +  json_array_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_document_end (&json_ctx);
>
>    return ret;
>  }

OK. All of this ends up being identical to bench-strstr.