Context: Linux ubuntucluster 4.15.0-55-generic #60-Ubuntu SMP Tue Jul 2 18:22:20 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux # /lib/x86_64-linux-gnu/libc.so.6 GNU C Library (Ubuntu GLIBC 2.27-3ubuntu1) stable release version 2.27. Lua 5.1 Use PRELOAD Jemalloc 5.3. Description: Similar issue based on https://sourceware.org/bugzilla/show_bug.cgi?id=19431 BT Threads: Thread 377 (Thread 0x7fbcab75b700 (LWP 4403)): $6 = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with waiters, Owner ID = 3156, Robust = No, Shared = No, Protocol = None} Thread 368 (Thread 0x7fbcac964700 (LWP 4394)): $7 = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with waiters, Owner ID = 3156, Robust = No, Shared = No, Protocol = None} Thread 360 (Thread 0x7fbcad96c700 (LWP 4386)): $8 = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with waiters, Owner ID = 3156, Robust = No, Shared = No, Protocol = None} Thread 355 (Thread 0x7fbcae371700 (LWP 4381)): $2 = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with waiters, Owner ID = 3156, Robust = No, Shared = No, Protocol = None} [Switching to thread 7 (Thread 0x7fbd273fa700 (LWP 3156))] #0  __lll_lock_wait_private () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:95 95      ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: No such file or directory. (gdb) bt #0  __lll_lock_wait_private () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:95 #1  0x00007fbd286c0e2a in __GI__IO_list_lock () at genops.c:1188 #2  0x00007fbd28715965 in __libc_fork () at ../sysdeps/nptl/fork.c:125 #3  0x00007fbd286b16b4 in _IO_new_proc_open (fp=fp@entry=0x7fbc9860ae00, command=command@entry=0x7fbc019251f8 "lsb_release -i", mode=, mode@entry=0x7fbc37e03df8 "r") at iopopen.c:122 #4  0x00007fbd286b1938 in _IO_new_popen (command=command@entry=0x7fbc019251f8 "lsb_release -i", mode=mode@entry=0x7fbc37e03df8 "r") at iopopen.c:206 #5  0x000056156400b4f2 in io_popen (L=0x7fbc019f8000) at liolib.c:178 #6  0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc019f8000, func=func@entry=0x7fbba9e04490, nresults=nresults@entry=-1) at ldo.c:320 #7  0x0000561564003602 in luaV_execute (L=L@entry=0x7fbc019f8000, nexeccalls=6, nexeccalls@entry=1) at lvm.c:591 #8  0x0000561563ffa2ad in luaD_call (L=0x7fbc019f8000, func=0x7fbba9e04380, nResults=) at ldo.c:378 #9  0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc019f8000, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd273f8490) at ldo.c:116 #10 0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc019f8000, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd273f8490, old_top=896, ef=) at ldo.c:464 #11 0x0000561563ff79d5 in lua_pcall (L=L@entry=0x7fbc019f8000, nargs=nargs@entry=0, nresults=nresults@entry=-1, errfunc=errfunc@entry=1) at lapi.c:821 #12 0x000056156400893c in luaB_xpcall (L=0x7fbc019f8000) at lbaselib.c:389 #13 0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc019f8000, func=func@entry=0x7fbba9e04360, nresults=nresults@entry=2) at ldo.c:320 #14 0x0000561564003602 in luaV_execute (L=L@entry=0x7fbc019f8000, nexeccalls=5, nexeccalls@entry=1) at lvm.c:591 #15 0x0000561563ffa2ad in luaD_call (L=0x7fbc019f8000, func=0x7fbba9e041b0, nResults=) at ldo.c:378 #16 0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc019f8000, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd273f8740) at ldo.c:116 #17 0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc019f8000, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd273f8740, old_top=432, ef=) at ldo.c:464 #18 0x0000561563ff79d5 in lua_pcall (L=L@entry=0x7fbc019f8000, nargs=nargs@entry=0, nresults=nresults@entry=-1, errfunc=errfunc@entry=1) at lapi.c:821 #19 0x000056156400893c in luaB_xpcall (L=0x7fbc019f8000) at lbaselib.c:389 #20 0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc019f8000, func=func@entry=0x7fbba9e04190, nresults=nresults@entry=-1) at ldo.c:320 #21 0x000056156400359f in luaV_execute (L=L@entry=0x7fbc019f8000, nexeccalls=3, nexeccalls@entry=1) at lvm.c:612 #22 0x0000561563ffa2ad in luaD_call (L=0x7fbc019f8000, func=0x7fbba9e04040, nResults=) at ldo.c:378 #23 0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc019f8000, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd273f89f0) at ldo.c:116 #24 0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc019f8000, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd273f89f0, old_top=64, ef=) at ldo.c:464 #25 0x0000561563ff79d5 in lua_pcall (L=0x7fbc019f8000, nargs=, nresults=-1, errfunc=) at lapi.c:821 (gdb)f 1 (gdb)p list_all_lock $6 = {lock = 2, cnt = 1, owner = 0x7fbd223d2700} [Switching to thread 47 (Thread 0x7fbd223d2700 (LWP 3196))] #0  __lll_lock_wait_private () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:95 #1  0x00007fbd286c0274 in _IO_flush_all_lockp (do_lock=do_lock@entry=1) at genops.c:762 #2  0x00007fbd286c06fa in __GI__IO_flush_all () at genops.c:790 #3  0x00007fbd286af895 in __GI__IO_fflush (fp=fp@entry=0x0) at iofflush.c:34 #4  0x000056156400b4e7 in io_popen (L=0x7fbc2905e780) at liolib.c:178 #5  0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc2905e780, func=func@entry=0x7fbc08418c90, nresults=nresults@entry=-1) at ldo.c:320 #6  0x0000561564003602 in luaV_execute (L=L@entry=0x7fbc2905e780, nexeccalls=6, nexeccalls@entry=1) at lvm.c:591 #7  0x0000561563ffa2ad in luaD_call (L=0x7fbc2905e780, func=0x7fbc08418b80, nResults=) at ldo.c:378 #8  0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc2905e780, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd223d0490) at ldo.c:116 #9  0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc2905e780, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd223d0490, old_top=896, ef=) at ldo.c:464 #10 0x0000561563ff79d5 in lua_pcall (L=L@entry=0x7fbc2905e780, nargs=nargs@entry=0, nresults=nresults@entry=-1, errfunc=errfunc@entry=1) at lapi.c:821 #11 0x000056156400893c in luaB_xpcall (L=0x7fbc2905e780) at lbaselib.c:389 #12 0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc2905e780, func=func@entry=0x7fbc08418b60, nresults=nresults@entry=2) at ldo.c:320 #13 0x0000561564003602 in luaV_execute (L=L@entry=0x7fbc2905e780, nexeccalls=5, nexeccalls@entry=1) at lvm.c:591 #14 0x0000561563ffa2ad in luaD_call (L=0x7fbc2905e780, func=0x7fbc084189b0, nResults=) at ldo.c:378 #15 0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc2905e780, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd223d0740) at ldo.c:116 #16 0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc2905e780, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd223d0740, old_top=432, ef=) at ldo.c:464 #17 0x0000561563ff79d5 in lua_pcall (L=L@entry=0x7fbc2905e780, nargs=nargs@entry=0, nresults=nresults@entry=-1, errfunc=errfunc@entry=1) at lapi.c:821 #18 0x000056156400893c in luaB_xpcall (L=0x7fbc2905e780) at lbaselib.c:389 #19 0x0000561563ff9f48 in luaD_precall (L=L@entry=0x7fbc2905e780, func=func@entry=0x7fbc08418990, nresults=nresults@entry=-1) at ldo.c:320 #20 0x000056156400359f in luaV_execute (L=L@entry=0x7fbc2905e780, nexeccalls=3, nexeccalls@entry=1) at lvm.c:612 #21 0x0000561563ffa2ad in luaD_call (L=0x7fbc2905e780, func=0x7fbc08418840, nResults=) at ldo.c:378 #22 0x0000561563ff971e in luaD_rawrunprotected (L=L@entry=0x7fbc2905e780, f=f@entry=0x561563ff5b80 , ud=ud@entry=0x7fbd223d09f0) at ldo.c:116 #23 0x0000561563ffa41b in luaD_pcall (L=L@entry=0x7fbc2905e780, func=func@entry=0x561563ff5b80 , u=u@entry=0x7fbd223d09f0, old_top=64, ef=) at ldo.c:464 #24 0x0000561563ff79d5 in lua_pcall (L=0x7fbc2905e780, nargs=, nresults=-1, errfunc=) at lapi.c:821 (gdb) f 1 #1  0x00007fbd286c0274 in _IO_flush_all_lockp (do_lock=do_lock@entry=1) at genops.c:762 762     genops.c: No such file or directory. (gdb) p list_all_lock $7 = {lock = 2, cnt = 1, owner = 0x7fbd223d2700} (gdb) p *fp $8 = {_flags = -72539000, _IO_read_ptr = 0x0, _IO_read_end = 0x0, _IO_read_base = 0x0, _IO_write_base = 0x0, _IO_write_ptr = 0x0, _IO_write_end = 0x0, _IO_buf_base = 0x0, _IO_buf_end = 0x0, _IO_save_base = 0x0, _IO_backup_base = 0x0,   _IO_save_end = 0x0, _markers = 0x0, _chain = 0x7fbc8ce14f80, _fileno = 1331, _flags2 = 128, _old_offset = 5788344089832285558, _cur_column = 0, _vtable_offset = 46 '.', _shortbuf = "1", _lock = 0x7fbca8219160, _offset = -1,   _codecvt = 0x722d302e33207465, _wide_data = 0x7fbca8219170, _freeres_list = 0x0, _freeres_buf = 0x313930353d544e45, __pad5 = 3979268027835692085, _mode = -1, _unused2 = "bba22cd111cb426c\" ti"} (gdb) p *(fp._lock) $9 = {lock = 2, cnt = 1, owner = 0x7fbd219cd700} 0x7fbd219cd700 (thread 52) #0  __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1  0x00007fbd29836025 in __GI___pthread_mutex_lock (mutex=mutex@entry=0x7fbca8415d68) at ../nptl/pthread_mutex_lock.c:80 #2  0x00007fbd2a7c4fde in malloc_mutex_lock_final (mutex=0x7fbca8415d28) at include/jemalloc/internal/mutex.h:151 #3  je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7fbca8415d28) at src/mutex.c:90 #4  0x00007fbd2a764b66 in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #5  je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7fbd219ccca8, arena=arena@entry=0x7fbca8401040, cache_bin=cache_bin@entry=0x7fbd219cd2a8, cache_bin_info=, binind=binind@entry=28, nfill=)     at src/arena.c:972 #6  0x00007fbd2a7d1329 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7fbd219ccca8, arena=arena@entry=0x7fbca8401040, tcache=tcache@entry=0x7fbd219cd000, cache_bin=cache_bin@entry=0x7fbd219cd2a8, binind=binind@entry=28,     tcache_success=tcache_success@entry=0x7fbd219c8d00) at src/tcache.c:238 #7  0x00007fbd2a75a9df in tcache_alloc_small (slow_path=, zero=false, binind=28, size=, tcache=0x7fbd219cd000, arena=, tsd=) at include/jemalloc/internal/tcache_inlines.h:68 #8  arena_malloc (slow_path=, tcache=0x7fbd219cd000, zero=false, ind=28, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #9  iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7fbd219cd000, zero=false, ind=28, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #10 imalloc_no_sample (ind=28, usize=4096, size=, tsd=0x7fbd219ccca8, dopts=, sopts=) at src/jemalloc.c:2398 #11 imalloc_body (tsd=0x7fbd219ccca8, dopts=, sopts=) at src/jemalloc.c:2573 #12 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #13 je_malloc_default (size=) at src/jemalloc.c:2722 #14 0x00007fbd2a75b14d in imalloc_fastpath (fallback_alloc=0x7fbd2a75a820 , size=size@entry=4096) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:266 #15 malloc (size=size@entry=4096) at src/jemalloc.c:2746 #16 0x00007fbd286af13c in __GI__IO_file_doallocate (fp=0x7fbca8219080) at filedoalloc.c:101 #17 0x00007fbd286bf329 in __GI__IO_doallocbuf (fp=fp@entry=0x7fbca8219080) at genops.c:365 #18 0x00007fbd286be204 in _IO_new_file_underflow (fp=0x7fbca8219080) at fileops.c:495 #19 0x00007fbd286bf3a2 in __GI__IO_default_uflow (fp=0x7fbca8219080) at genops.c:380 #20 0x00007fbd286b0e12 in __GI__IO_getline_info (eof=0x0, extract_delim=, delim=10, n=1023, buf=0x7fbd219c8f80 "", fp=0x7fbca8219080, fp@entry=0x0) at iogetline.c:60 #21 __GI__IO_getline (fp=fp@entry=0x7fbca8219080, buf=buf@entry=0x7fbd219c8f80 "", n=, delim=delim@entry=10, extract_delim=extract_delim@entry=1) at iogetline.c:34 #22 0x00007fbd286afb7d in _IO_fgets (buf=0x7fbd219c8f80 "", n=, fp=0x7fbca8219080) at iofgets.c:53 $5 = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with waiters, Owner ID = 3156, Robust = No, Shared = No, Protocol = None} [Switching to thread 7 (Thread 0x7fbd273fa700 (LWP 3156))] The initial Thread.... ==> genops.c int _IO_flush_all_lockp (int do_lock) {   int result = 0;   struct _IO_FILE *fp; #ifdef _IO_MTSAFE_IO   _IO_cleanup_region_start_noarg (flush_cleanup);   _IO_lock_lock (list_all_lock); #endif   for (fp = (_IO_FILE *) _IO_list_all; fp != NULL; fp = fp->_chain)     {       run_fp = fp;       if (do_lock)     _IO_flockfile (fp);       if (((fp->_mode <= 0 && fp->_IO_write_ptr > fp->_IO_write_base)        || (_IO_vtable_offset (fp) == 0            && fp->_mode > 0 && (fp->_wide_data->_IO_write_ptr                     > fp->_wide_data->_IO_write_base))        )       && _IO_OVERFLOW (fp, EOF) == EOF)     result = EOF;       if (do_lock)     _IO_funlockfile (fp);       run_fp = NULL;     } #ifdef _IO_MTSAFE_IO   _IO_lock_unlock (list_all_lock);   _IO_cleanup_region_end (0); #endif   return result; } Regards @7i77an -- This email has been checked for viruses by Avast antivirus software. www.avast.com