[Bug gcov-profile/97461] New: allocate_gcov_kvp() deadlocks in firefox LTO+PGO build (overridden malloc() recursion)

["slyfox at gcc dot gnu.org" <gcc-bugzilla@gcc.gnu.org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97461

            Bug ID: 97461
           Summary: allocate_gcov_kvp() deadlocks in firefox LTO+PGO build
                    (overridden malloc() recursion)
           Product: gcc
           Version: 11.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: gcov-profile
          Assignee: unassigned at gcc dot gnu.org
          Reporter: slyfox at gcc dot gnu.org
                CC: marxin at gcc dot gnu.org
  Target Milestone: ---

Single-file example is extracted from firefox-81 build hangup (LTO+PGO
flavour).

Here is the single-file reproducer that converts hangup to a crash:

// gcc-11.0.0 a.c -o a -fprofile-generate -ggdb3 && ./a

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

static int malloc_depth = 0;

static char memory[128* 1024];
static size_t memory_p = 0;

void f1(void) {}
void f2(void) {}

typedef void (*fun_t)(void);
static const fun_t funs[2] = { f1, f2, };

static void * malloc_impl(size_t size) {
    void * r = &memory[memory_p];
    memory_p += size;

    // force TOPN profile
    funs[size % 2]();
    return r;
}

// Override default malloc, check it it get s called recursively
void * malloc(size_t size) {
    // Must not be called recursively. Malloc implementation does not support
it.
    if (malloc_depth != 0) __builtin_trap();

    ++malloc_depth;
      void * r = malloc_impl(size);
    --malloc_depth;
    return r;
}

// Called from gcov
void *calloc(size_t nmemb, size_t size) {
    // Must not be called recursively.  Malloc implementation does not support
it.
    if (malloc_depth != 0) __builtin_trap();

    ++malloc_depth;
      void * r = malloc_impl(size * nmemb);
      memset(r, 0, size * nmemb);
    --malloc_depth;
    return r;
}

void free(void *ptr){}

int main() {
    void * p = malloc(8);
    return p != 0;
}

How to crash:

$ gcc-11.0.0 a.c -o a -ggdb3 && ./a
$ gcc-11.0.0 a.c -o a -fprofile-generate -ggdb3 && ./a
Illegal instruction (core dumped)

Here we have a malloc recursion of
    malloc()->malloc_internals()->gcov->calloc()->malloc_internals().

malloc() is re-entered twice:

Program received signal SIGILL, Illegal instruction.
0x00005555555565e7 in calloc (nmemb=1, size=24) at a.c:103
103         if (malloc_depth != 0) __builtin_trap();
(gdb) bt
#0  0x00005555555565e7 in calloc (nmemb=1, size=24) at a.c:103
#1  0x0000555555556cf3 in allocate_gcov_kvp () at
/var/tmp/portage/sys-devel/gcc-11.0.0_pre9999/work/gcc-11.0.0_pre9999/libgcc/libgcov.h:441
#2  gcov_topn_add_value (count=1, increment_total=1, use_atomic=0,
value=721827547, counters=0x55555557b660 <__gcov4.malloc_impl>) at
/var/tmp/portage/sys-devel/gcc-11.0.0_pre9999/work/gcc-11.0.0_pre9999/libgcc/libgcov.h:489
#3  __gcov_topn_values_profiler_body (use_atomic=0, value=721827547,
counters=0x55555557b660 <__gcov4.malloc_impl>) at
/var/tmp/portage/sys-devel/gcc-11.0.0_pre9999/work/gcc-11.0.0_pre9999/libgcc/libgcov-profiler.c:103
#4  __gcov_indirect_call_profiler_body (use_atomic=0, cur_func=<optimized out>,
value=721827547) at
/var/tmp/portage/sys-devel/gcc-11.0.0_pre9999/work/gcc-11.0.0_pre9999/libgcc/libgcov-profiler.c:163
#5  __gcov_indirect_call_profiler_v4 (value=721827547, cur_func=<optimized
out>) at
/var/tmp/portage/sys-devel/gcc-11.0.0_pre9999/work/gcc-11.0.0_pre9999/libgcc/libgcov-profiler.c:172
#6  0x000055555555631e in f1 () at a.c:74
#7  0x0000555555556482 in malloc_impl (size=8) at a.c:85
#8  0x0000555555556537 in malloc (size=8) at a.c:95
#9  0x0000555555556760 in main () at a.c:115