Re: question on locking of pthread structures

Re: question on locking of pthread structures

[Stanley Gambarin]

correction to previous email... the scheduling policy
in use is SCHED_RR, not SCHED_FIFO.  Updated
source code below.

-- stanley

> #include <stdio.h>
> #include <stdlib.h>
> #include <signal.h>
> #include <time.h>
> #include <pthread.h>
> #include <sched.h>
> #include <errno.h>
> 
> 
> pthread_t t1; /* prio 1 */
> pthread_t t2; /* prio 6 */
> pthread_t t3; /* prio 11 */
> static volatile int state = 0;
> 
> void *
> run1(void *arg)
> {
>   struct sched_param sparam;
>   int policy, i, j=0, count=0, y=0;
>   long start, end, diff;
>   
>   fprintf(stderr, "thread 1 - start\n");
>   start = diff = time(NULL);
>   state = 1; /* allow thread 2 to start */
>   do {
>     if (!pthread_getschedparam (pthread_self(), &policy, &sparam) 
>         && (sparam.sched_priority != 1)) { /* got bumped by t3 */
>         if (count == 0) {
>           fprintf(stderr, "thread 1 - lowering priority after being bumped by t3\n");
>           sparam.sched_priority = 1;
>           if (pthread_setschedparam (pthread_self(), policy, &sparam)) {
>             perror("pthread_setschedparam"); return (void *)-2;
>           }
>           ++count;
>         }
>     }
>     for (i=0; i<1000; i++) {
>       j += j*i;
>     }
>     end = time(NULL);
>     if ((end-diff)>=2) {
>         fprintf(stderr, "thread 1 - continue busy loop\n");
>         diff = end;
>         if (!y) { ++y; pthread_yield(); /* once */}
>     }
>   } while ((end-start) < 10);
>   fprintf(stderr, "thread 1 - done\n");
>   return NULL;
> }
> 
> void *
> run2(void *arg)
> {
>   long start, end, diff;
>   int i, j=0, y=0;
> 
>   fprintf(stderr, "thread 2 - start\n");
>   start = diff = time(NULL);
>   state = 2;
>   do {
>     for (i=0; i<10000; i++) j+=i;
>     end = time(NULL);
>     if ((end-diff)>=2) {
>       fprintf(stderr, "thread 2 - continue busy loop\n");
>       diff = end;
>       if (!y) { ++y; pthread_yield(); /* once */}
>     }
>   } while ((end-start) < 10);
>   fprintf(stderr, "thread 2 - done\n");
>   return NULL;
> }
> 
> void *
> run3(void *arg)
> {
>   struct sched_param sp1, sp2;
>   int policy, i;
>   long start, end, diff, j=0;
>  
>   fprintf(stderr, "thread 3 - start\n");
>   i = 0; 
>   start = diff = time(NULL);
>   do {
>     fprintf(stderr, "thread 3 - checking for priority of t1\n");
>     if (!pthread_getschedparam (t1, &policy, &sp1) &&
>         !pthread_getschedparam (t3, &policy, &sp2) &&
>         (sp1.sched_priority < sp2.sched_priority)) {
>       fprintf(stderr, "thread 3 - bumping prio of t1 from %d to %d\n", sp1.sched_priority, sp2.sched_priority);
>       if (pthread_setschedparam (t1, policy, &sp2)) {
>         perror("pthread_setschedparam"); return (void *)-2;
>       }
>     }
>     for (i=0; i<10000; i++) {
>         j += j*i;
>     }
>     end = time(NULL);
>     if ((end-diff)>=2) {
>         fprintf(stderr, "thread 3 - continue busy loop\n");
>         diff = end;
>     }
>   } while ((end-start) < 10);
>   fprintf(stderr, "thread 3 - done\n");
>   return NULL;
> }
> 
> int
> main()
> {
>   pthread_attr_t attr;
>   struct sched_param sched_param;
> 
>   if (pthread_attr_init(&attr) != 0) return -1;
>   if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED) != 0) return -1;
> 
>   {
>      int policy = SCHED_RR;
>      sched_param.sched_priority = 12;
>      if (pthread_setschedparam(pthread_self(), policy, &sched_param) != 0) return -1;
>      if (pthread_getschedparam(pthread_self(), &policy, &sched_param) != 0) return -1;
>   }
>   if (pthread_attr_setschedpolicy(&attr, SCHED_RR) != 0) return -1;
> 
>   
>   /* create t1 prio 1 */
>   fprintf(stderr, "creating thread 1\n");
>   sched_param.sched_priority = 1;
>   if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
>   if (0 != pthread_create(&t1, &attr, run1, NULL)) {
>     perror("pthread_create"); return -4;
>   }
>   while (state != 1) usleep(3);
> 
>   /* create t2 prio 6 */
>   fprintf(stderr, "creating thread 2\n");
>   sched_param.sched_priority = 6;
>   if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
>   if (0 != pthread_create(&t2, &attr, run2, NULL)) {
>     perror("pthread_create"); return -5;
>   }
>   while (state != 2) usleep(3);
> 
>   /* create t3 prio 11 */
>   fprintf(stderr, "creating thread 3\n");
>   sched_param.sched_priority = 11;
>   if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
>   if (0 != pthread_create(&t3, &attr, run3, NULL)) {
>     perror("pthread_create"); return -6;
>   }
> 
>   /* let them run */
>   pthread_join(t1, NULL);
>   pthread_join(t2, NULL);
>   pthread_join(t3, NULL);
>   return 0;
> }

question on locking of pthread structures

[Stanley Gambarin]

[-- Attachment #1: Type: text/plain, Size: 2191 bytes --]

Hello,

   i've ran into this using large application.  I was able to reproduce
the problem using a small testcase (attached).  It occurs (for me) on
my linux (single-processor) laptop, running vanilla 2.6.11.11
kernel compiled with SMP (i'm not sure whether it matters or not).
Machine is running gentoo, using glibc 2.3.5, compiled with gcc 3.3.4
This problem is reproducible for me on both LinuxThreads and NPTL.

% gcc prio.c -o prio.x -lpthread
% su root  # required for running SCHED_RR
% ./prio.x  # your output may vary
creating thread 1
thread 1 - start
creating thread 2
thread 2 - start
creating thread 3
thread 3 - start
thread 3 - checking for priority of t1
thread 3 - bumping prio of t1 from 1 to 11
thread 3 - checking for priority of t1
thread 3 - checking for priority of t1
thread 3 - checking for priority of t1
thread 3 - checking for priority of t1
thread 3 - checking for priority of t1
thread 1 - lowering priority after being bumped by t3
thread 3 - checking for priority of t1          <=== t3 hangs
thread 2 - continue busy loop
thread 2 - continue busy loop
thread 2 - continue busy loop
thread 2 - continue busy loop
thread 2 - continue busy loop
thread 2 - done
thread 3 - bumping prio of t1 from 1 to 11
thread 3 - continue busy loop
thread 3 - done
thread 1 - continue busy loop
thread 1 - done

   what's happening (i think) is that t1 got scheduled out (when we
lowered it's priority) while still holding lock for it's internal data
structure.  t3 will attempt to access that structure (via 
pthread_getschedparams)
and since it can not get a hold of the lock, t2 will run.  Running this 
test case
on 3 other RTOSs did not cause t3 to be scheduled out.  The questions are:

a) is this a bug or a feature of pthread implementation on linux
b) can i somehow tell pthread library to not schedule out threads until
internal mutexes have been freed (if my assumption above is correct)
c) is there an easy configuration fix for the above problem (i don't have
a restriction on compiler/kernel/etc).

Any help in getting this problem resolved would be highly appreciated.
Please CC: me on the replies, since I am not part of the mailing list.

Thanks,
-- stanley

[-- Attachment #2: prio.c --]
[-- Type: text/x-csrc, Size: 4192 bytes --]

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <time.h>
#include <pthread.h>
#include <sched.h>
#include <errno.h>


pthread_t t1; /* prio 1 */
pthread_t t2; /* prio 6 */
pthread_t t3; /* prio 11 */
static volatile int state = 0;

void *
run1(void *arg)
{
  struct sched_param sparam;
  int policy, i, j=0, count=0, y=0;
  long start, end, diff;
  
  fprintf(stderr, "thread 1 - start\n");
  start = diff = time(NULL);
  state = 1; /* allow thread 2 to start */
  do {
    if (!pthread_getschedparam (pthread_self(), &policy, &sparam) 
        && (sparam.sched_priority != 1)) { /* got bumped by t3 */
        if (count == 0) {
          fprintf(stderr, "thread 1 - lowering priority after being bumped by t3\n");
          sparam.sched_priority = 1;
          if (pthread_setschedparam (pthread_self(), policy, &sparam)) {
            perror("pthread_setschedparam"); return (void *)-2;
          }
          ++count;
        }
    }
    for (i=0; i<1000; i++) {
      j += j*i;
    }
    end = time(NULL);
    if ((end-diff)>=2) {
        fprintf(stderr, "thread 1 - continue busy loop\n");
        diff = end;
        if (!y) { ++y; pthread_yield(); /* once */}
    }
  } while ((end-start) < 10);
  fprintf(stderr, "thread 1 - done\n");
  return NULL;
}

void *
run2(void *arg)
{
  long start, end, diff;
  int i, j=0, y=0;

  fprintf(stderr, "thread 2 - start\n");
  start = diff = time(NULL);
  state = 2;
  do {
    for (i=0; i<10000; i++) j+=i;
    end = time(NULL);
    if ((end-diff)>=2) {
      fprintf(stderr, "thread 2 - continue busy loop\n");
      diff = end;
      if (!y) { ++y; pthread_yield(); /* once */}
    }
  } while ((end-start) < 10);
  fprintf(stderr, "thread 2 - done\n");
  return NULL;
}

void *
run3(void *arg)
{
  struct sched_param sp1, sp2;
  int policy, i;
  long start, end, diff, j=0;
 
  fprintf(stderr, "thread 3 - start\n");
  i = 0; 
  start = diff = time(NULL);
  do {
    fprintf(stderr, "thread 3 - checking for priority of t1\n");
    if (!pthread_getschedparam (t1, &policy, &sp1) &&
        !pthread_getschedparam (t3, &policy, &sp2) &&
        (sp1.sched_priority < sp2.sched_priority)) {
      fprintf(stderr, "thread 3 - bumping prio of t1 from %d to %d\n", sp1.sched_priority, sp2.sched_priority);
      if (pthread_setschedparam (t1, policy, &sp2)) {
        perror("pthread_setschedparam"); return (void *)-2;
      }
    }
    for (i=0; i<10000; i++) {
        j += j*i;
    }
    end = time(NULL);
    if ((end-diff)>=2) {
        fprintf(stderr, "thread 3 - continue busy loop\n");
        diff = end;
    }
  } while ((end-start) < 10);
  fprintf(stderr, "thread 3 - done\n");
  return NULL;
}

int
main()
{
  pthread_attr_t attr;
  struct sched_param sched_param;

  if (pthread_attr_init(&attr) != 0) return -1;
  if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED) != 0) return -1;

  {
     int policy = SCHED_FIFO;
     sched_param.sched_priority = 12;
     if (pthread_setschedparam(pthread_self(), policy, &sched_param) != 0) return -1;
     if (pthread_getschedparam(pthread_self(), &policy, &sched_param) != 0) return -1;
  }
  if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO) != 0) return -1;

  
  /* create t1 prio 1 */
  fprintf(stderr, "creating thread 1\n");
  sched_param.sched_priority = 1;
  if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
  if (0 != pthread_create(&t1, &attr, run1, NULL)) {
    perror("pthread_create"); return -4;
  }
  while (state != 1) usleep(3);

  /* create t2 prio 6 */
  fprintf(stderr, "creating thread 2\n");
  sched_param.sched_priority = 6;
  if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
  if (0 != pthread_create(&t2, &attr, run2, NULL)) {
    perror("pthread_create"); return -5;
  }
  while (state != 2) usleep(3);

  /* create t3 prio 11 */
  fprintf(stderr, "creating thread 3\n");
  sched_param.sched_priority = 11;
  if (pthread_attr_setschedparam(&attr, &sched_param) != 0) return -1;
  if (0 != pthread_create(&t3, &attr, run3, NULL)) {
    perror("pthread_create"); return -6;
  }

  /* let them run */
  pthread_join(t1, NULL);
  pthread_join(t2, NULL);
  pthread_join(t3, NULL);
  return 0;
}