#!/usr/bin/stap

# This script tries to identify contended mutex locks.

global thread_blocktime		  # time of latest lock call by thread
global lock_waits # long-lived stats on (pid,lock) blockage elapsed time
global process_names		  # long-lived pid-to-execname mapping

global mutex_locks;		  # number of locks in progress
global mutex_stack;		  # flag if we already printed a stack

probe process("/lib64/libpthread.so.0").function("pthread_mutex_lock") {
  if (mutex_locks[$mutex]++ > 0) {	  # Contended
      process_names[pid()] = execname();
      thread_blocktime[tid()] = gettimeofday_us();
      if (!mutex_stack[pid(), $mutex]) {
	mutex_stack[pid(), $mutex] = 1;
	printf("%s[%d] mutex %p stack trace:\n", process_names[pid()], pid(), $mutex)
	print_ubacktrace();
      }
  }
}

probe process("/lib64/libpthread.so.0").function("pthread_mutex_lock").return {
  --mutex_locks[$mutex];
  t = tid();
  ts = thread_blocktime[t];
  if (ts) {
    elapsed = gettimeofday_us() - ts;
    lock_waits[pid(), $mutex] <<< elapsed;
    delete thread_blocktime[tid()]
  }
}

probe end {
  // TODO: sort output by sum of wait times, rather than count.
  foreach ([pid, m] in lock_waits-) {
    printf ("%s[%d] mutex %p contended %d times, %dus total, %dus avg\n",
            process_names[pid], pid, m, 
	    @count(lock_waits[pid,m]),
	    @sum(lock_waits[pid,m]),
	    @avg(lock_waits[pid,m]));
  }
}