[PATCH 2/3] [gomp] Thread pool management

[PATCH 2/3] [gomp] Thread pool management

[Sebastian Huber]

In RTEMS we may have multiple scheduler instances with different
scheduling algorithms.  In addition we have a single process environment
so all threads run in one address space.  In order to support work
stealing applications it is important to limit the number of thread
pools used for OpenMP since otherwise we may end up in an explosion of
OpenMP worker threads.

libgomp/ChangeLog
2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>

	* config/posix/pool.h: New.
	* config/rtems/pool.h: Likewise.
	* config/rtems/proc.c: Likewise.
	* libgomp.h (gomp_thread_destructor): Declare.
	* team.c: Include configuration provided <pool.h>.
	(gomp_get_thread_pool): Define in configuration.
	(gomp_team_end): Call configuration defined
	gomp_release_thread_pool().
---
 libgomp/config/posix/pool.h |  60 ++++++++++++++++++
 libgomp/config/rtems/pool.h | 128 ++++++++++++++++++++++++++++++++++++++
 libgomp/config/rtems/proc.c | 145 ++++++++++++++++++++++++++++++++++++++++++++
 libgomp/libgomp.h           |   2 +
 libgomp/team.c              |  22 +------
 5 files changed, 337 insertions(+), 20 deletions(-)
 create mode 100644 libgomp/config/posix/pool.h
 create mode 100644 libgomp/config/rtems/pool.h
 create mode 100644 libgomp/config/rtems/proc.c

diff --git a/libgomp/config/posix/pool.h b/libgomp/config/posix/pool.h
new file mode 100644
index 0000000..0d127a0
--- /dev/null
+++ b/libgomp/config/posix/pool.h
@@ -0,0 +1,60 @@
+/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the default implementation of the thread pool management
+   for libgomp.  This type is private to the library.  */
+
+#ifndef GOMP_POOL_H
+#define GOMP_POOL_H 1
+
+#include <libgomp.h>
+
+/* Get the thread pool, allocate and initialize it on demand.  */
+
+static inline struct gomp_thread_pool *
+gomp_get_thread_pool (struct gomp_thread *thr, unsigned nthreads)
+{
+  struct gomp_thread_pool *pool = thr->thread_pool;
+  if (__builtin_expect (pool == NULL, 0))
+    {
+      pool = gomp_malloc (sizeof (*pool));
+      pool->threads = NULL;
+      pool->threads_size = 0;
+      pool->threads_used = 0;
+      pool->last_team = NULL;
+      pool->threads_busy = nthreads;
+      thr->thread_pool = pool;
+      pthread_setspecific (gomp_thread_destructor, thr);
+    }
+  return pool;
+}
+
+static inline void
+gomp_release_thread_pool (struct gomp_thread_pool *pool)
+{
+  /* Do nothing in the default implementation.  */
+}
+
+#endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/pool.h b/libgomp/config/rtems/pool.h
new file mode 100644
index 0000000..5c989d0
--- /dev/null
+++ b/libgomp/config/rtems/pool.h
@@ -0,0 +1,128 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the RTEMS implementation of the thread pool management
+   for libgomp.  This type is private to the library.  */
+
+#ifndef GOMP_POOL_H
+#define GOMP_POOL_H 1
+
+#include <libgomp.h>
+#include <sys/lock.h>
+#include <string.h>
+
+/* For each scheduler instance there may be a thread pool reservoir
+   to limit the number of thread pools used by the OpenMP master threads of this
+   scheduler instance.  The reservoirs are configured via the
+   GOMP_RTEMS_THREAD_POOLS environment variable.  */
+struct gomp_thread_pool_reservoir {
+  gomp_sem_t available;
+  gomp_mutex_t lock;
+  size_t index;
+  struct gomp_thread_pool *pools[];
+};
+
+struct gomp_tls_rtems_data {
+  struct gomp_thread_pool_reservoir *thread_pool_reservoir;
+};
+
+extern struct gomp_thread_pool_reservoir **gomp_thread_pool_reservoirs;
+
+extern __thread struct gomp_tls_rtems_data gomp_tls_rtems_data;
+
+static inline struct gomp_thread_pool_reservoir *
+gomp_get_thread_pool_reservoir (void)
+{
+  struct gomp_thread_pool_reservoir *res =
+    gomp_tls_rtems_data.thread_pool_reservoir;
+
+  if (res == NULL && gomp_thread_pool_reservoirs != NULL)
+    {
+      struct gomp_thread *thr = gomp_thread ();
+      thr->thread_pool = gomp_malloc_cleared (sizeof (*thr->thread_pool));
+      res = gomp_thread_pool_reservoirs[_Sched_Index ()];
+      gomp_tls_rtems_data.thread_pool_reservoir = res;
+    }
+
+  return res;
+}
+
+static inline struct gomp_thread_pool *
+gomp_get_own_thread_pool (struct gomp_thread *thr, unsigned nthreads)
+{
+  struct gomp_thread_pool *pool = thr->thread_pool;
+  if (__builtin_expect (pool == NULL, 0))
+    {
+      pool = gomp_malloc_cleared (sizeof (*pool));
+      pool->threads_busy = nthreads;
+      thr->thread_pool = pool;
+    }
+  return pool;
+}
+
+static inline struct gomp_thread_pool *
+gomp_get_thread_pool (struct gomp_thread *thr, unsigned nthreads)
+{
+  struct gomp_thread_pool *pool;
+
+  if (__builtin_expect (thr->thread_pool == NULL, 0))
+    pthread_setspecific (gomp_thread_destructor, thr);
+
+  if (nthreads != 1)
+    {
+      struct gomp_thread_pool_reservoir *res =
+	gomp_get_thread_pool_reservoir ();
+      if (res != NULL)
+	{
+	  gomp_sem_wait (&res->available);
+	  gomp_mutex_lock (&res->lock);
+	  pool = res->pools[--res->index];
+	  gomp_mutex_unlock (&res->lock);
+	  pool->threads_busy = nthreads;
+	  thr->thread_pool = pool;
+	}
+      else
+	pool = gomp_get_own_thread_pool (thr, nthreads);
+    }
+  else
+    pool = NULL;
+  return pool;
+}
+
+static inline void
+gomp_release_thread_pool (struct gomp_thread_pool *pool)
+{
+  struct gomp_thread_pool_reservoir *res =
+    gomp_tls_rtems_data.thread_pool_reservoir;
+  if (res != NULL)
+    {
+      gomp_mutex_lock (&res->lock);
+      res->pools[res->index++] = pool;
+      gomp_mutex_unlock (&res->lock);
+      gomp_sem_post (&res->available);
+    }
+}
+
+#endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/proc.c b/libgomp/config/rtems/proc.c
new file mode 100644
index 0000000..9c36dcb
--- /dev/null
+++ b/libgomp/config/rtems/proc.c
@@ -0,0 +1,145 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This file contains RTEMS specific routines related to counting
+   online processors and dynamic load balancing.  */
+
+#include "libgomp.h"
+#include <errno.h>
+#include <pool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+struct gomp_thread_pool_reservoir **gomp_thread_pool_reservoirs;
+
+__thread struct gomp_tls_rtems_data gomp_tls_rtems_data;
+
+static void
+allocate_thread_pool_reservoirs (void)
+{
+  struct gomp_thread_pool_reservoir **reservoirs;
+  size_t size = _Sched_Count () * sizeof (*reservoirs);
+  reservoirs = gomp_malloc (size);
+  gomp_thread_pool_reservoirs = reservoirs;
+  memset (reservoirs, 0, size);
+}
+
+static void
+allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
+{
+  struct gomp_thread_pool_reservoir *res;
+  struct gomp_thread_pool *pools;
+  unsigned long i;
+  size_t size;
+
+  res = gomp_thread_pool_reservoirs[scheduler];
+  if (res != NULL)
+    gomp_fatal ("Multiple thread pool reservoir initialization");
+  size = sizeof (*res) + count * (sizeof(pools) + sizeof(*pools));
+  pools = gomp_malloc (size);
+  memset (pools, 0, size);
+  res = (struct gomp_thread_pool_reservoir *) (pools + count);
+  res->index = count;
+  gomp_sem_init (&res->available, count);
+  gomp_mutex_init (&res->lock);
+  for (i = 0; i < count; ++i)
+    res->pools[i] = &pools[i];
+  gomp_thread_pool_reservoirs[scheduler] = res;
+}
+
+static char *
+parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
+{
+  size_t len;
+  int i;
+
+  if (*env == ':')
+    ++env;
+
+  errno = 0;
+  *count = strtoul (env, &env, 10);
+  if (errno != 0)
+    gomp_fatal ("Invalid thread pool count");
+
+  if (*env != '@')
+    gomp_fatal ("Invalid thread pool scheduler prefix");
+  ++env;
+
+  len = 0;
+  while (env[len] != '\0' && env[len] != ':')
+    ++len;
+  i = _Sched_Name_to_index (env, len);
+  if (i < 0)
+    gomp_fatal ("Invalid thread pool scheduler");
+  *scheduler = i;
+  env += len;
+
+  return env;
+}
+
+static void
+init_thread_pool_reservoirs (void)
+{
+  char *env = getenv ("GOMP_RTEMS_THREAD_POOLS");
+  if (env != NULL)
+    {
+      allocate_thread_pool_reservoirs ();
+      while (*env != '\0')
+	{
+	  unsigned long count;
+	  unsigned long scheduler;
+	  env = parse_thread_pools (env, &count, &scheduler);
+	  allocate_thread_pool_reservoir (count, scheduler);
+	}
+    }
+}
+
+void
+gomp_init_num_threads (void)
+{
+  gomp_global_icv.nthreads_var = omp_get_num_procs();
+  init_thread_pool_reservoirs ();
+}
+
+unsigned
+gomp_dynamic_max_threads (void)
+{
+  unsigned n_onln = (unsigned) omp_get_num_procs();
+  unsigned nthreads_var = gomp_icv (false)->nthreads_var;
+
+  if (n_onln > nthreads_var)
+    return nthreads_var;
+  else
+    return n_onln;
+}
+
+int
+omp_get_num_procs (void)
+{
+  return sysconf (_SC_NPROCESSORS_ONLN);
+}
+
+ialias (omp_get_num_procs)
diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h
index ac40e2a..9b803d6 100644
--- a/libgomp/libgomp.h
+++ b/libgomp/libgomp.h
@@ -513,6 +513,8 @@ static inline struct gomp_task_icv *gomp_icv (bool write)
 /* The attributes to be used during thread creation.  */
 extern pthread_attr_t gomp_thread_attr;
 
+extern pthread_key_t gomp_thread_destructor;
+
 /* Function prototypes.  */
 
 /* affinity.c */
diff --git a/libgomp/team.c b/libgomp/team.c
index 5c56182..5edae07 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -27,6 +27,7 @@
    creation and termination.  */
 
 #include "libgomp.h"
+#include <pool.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -134,26 +135,6 @@ gomp_thread_start (void *xdata)
   return NULL;
 }
 
-/* Get the thread pool, allocate and initialize it on demand.  */
-
-static struct gomp_thread_pool *
-gomp_get_thread_pool (struct gomp_thread *thr, unsigned nthreads)
-{
-  struct gomp_thread_pool *pool = thr->thread_pool;
-  if (__builtin_expect (pool == NULL, 0))
-    {
-      pool = gomp_malloc (sizeof (*pool));
-      pool->threads = NULL;
-      pool->threads_size = 0;
-      pool->threads_used = 0;
-      pool->last_team = NULL;
-      pool->threads_busy = nthreads;
-      thr->thread_pool = pool;
-      pthread_setspecific (gomp_thread_destructor, thr);
-    }
-  return pool;
-}
-
 static inline struct gomp_team *
 get_last_team (unsigned nthreads)
 {
@@ -930,6 +911,7 @@ gomp_team_end (void)
       if (pool->last_team)
 	free_team (pool->last_team);
       pool->last_team = team;
+      gomp_release_thread_pool (pool);
     }
 }
 
-- 
1.8.4.5

[PATCH 1/3] [gomp] Add RTEMS configuration

[Sebastian Huber]

libgomp/ChangeLog
2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>

	* config/rtems/bar.c: New.
	* config/rtems/bar.h: Likewise.
	* config/rtems/mutex.c: Likewise.
	* config/rtems/mutex.h: Likewise.
	* config/rtems/sem.c: Likewise.
	* config/rtems/sem.h: Likewise.
	* configure.ac (*-*-rtems*): Check that Newlib provides a proper
	<sys/lock.h> header file.
	* configure.tgt (*-*-rtems*): Enable RTEMS configuration if
	supported by Newlib.
	* configure: Regenerate.
---
 libgomp/config/rtems/bar.c   | 255 +++++++++++++++++++++++++++++++++++++++++++
 libgomp/config/rtems/bar.h   | 170 +++++++++++++++++++++++++++++
 libgomp/config/rtems/mutex.c |   1 +
 libgomp/config/rtems/mutex.h |  57 ++++++++++
 libgomp/config/rtems/sem.c   |   1 +
 libgomp/config/rtems/sem.h   |  55 ++++++++++
 libgomp/configure            |  17 +++
 libgomp/configure.ac         |   7 ++
 libgomp/configure.tgt        |   7 ++
 9 files changed, 570 insertions(+)
 create mode 100644 libgomp/config/rtems/bar.c
 create mode 100644 libgomp/config/rtems/bar.h
 create mode 100644 libgomp/config/rtems/mutex.c
 create mode 100644 libgomp/config/rtems/mutex.h
 create mode 100644 libgomp/config/rtems/sem.c
 create mode 100644 libgomp/config/rtems/sem.h

diff --git a/libgomp/config/rtems/bar.c b/libgomp/config/rtems/bar.c
new file mode 100644
index 0000000..05bb320
--- /dev/null
+++ b/libgomp/config/rtems/bar.c
@@ -0,0 +1,255 @@
+/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU OpenMP Library (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the RTEMS implementation of a barrier synchronization
+   mechanism for libgomp.  It is identical to the Linux implementation, except
+   that the futex API is slightly different.  This type is private to the
+   library.  */
+
+#include <libgomp.h>
+#include <bar.h>
+#include <limits.h>
+
+static gomp_barrier_t *
+generation_to_barrier(int *addr)
+{
+  return (gomp_barrier_t *)
+	 ((char *) addr - __builtin_offsetof (gomp_barrier_t, generation));
+}
+
+static void
+futex_wait (int *addr, int val)
+{
+  gomp_barrier_t *bar = generation_to_barrier (addr);
+  _Futex_Wait (&bar->futex, addr, val);
+}
+
+static void
+futex_wake (int *addr, int count)
+{
+  gomp_barrier_t *bar = generation_to_barrier (addr);
+  _Futex_Wake (&bar->futex, count);
+}
+
+static int
+do_spin (int *addr, int val)
+{
+  unsigned long long i, count = gomp_spin_count_var;
+
+  if (__builtin_expect (gomp_managed_threads > gomp_available_cpus, 0))
+    count = gomp_throttled_spin_count_var;
+  for (i = 0; i < count; i++)
+    if (__builtin_expect (__atomic_load_n (addr, MEMMODEL_RELAXED) != val, 0))
+      return 0;
+  return 1;
+}
+
+static void
+do_wait (int *addr, int val)
+{
+  if (do_spin (addr, val))
+    futex_wait (addr, val);
+}
+
+/* Everything below this point should be identical to the Linux
+   implementation.  */
+
+void
+gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
+{
+  if (__builtin_expect (state & BAR_WAS_LAST, 0))
+    {
+      /* Next time we'll be awaiting TOTAL threads again.  */
+      bar->awaited = bar->total;
+      __atomic_store_n (&bar->generation, bar->generation + BAR_INCR,
+			MEMMODEL_RELEASE);
+      futex_wake ((int *) &bar->generation, INT_MAX);
+    }
+  else
+    {
+      do
+	do_wait ((int *) &bar->generation, state);
+      while (__atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE) == state);
+    }
+}
+
+void
+gomp_barrier_wait (gomp_barrier_t *bar)
+{
+  gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar));
+}
+
+/* Like gomp_barrier_wait, except that if the encountering thread
+   is not the last one to hit the barrier, it returns immediately.
+   The intended usage is that a thread which intends to gomp_barrier_destroy
+   this barrier calls gomp_barrier_wait, while all other threads
+   call gomp_barrier_wait_last.  When gomp_barrier_wait returns,
+   the barrier can be safely destroyed.  */
+
+void
+gomp_barrier_wait_last (gomp_barrier_t *bar)
+{
+  gomp_barrier_state_t state = gomp_barrier_wait_start (bar);
+  if (state & BAR_WAS_LAST)
+    gomp_barrier_wait_end (bar, state);
+}
+
+void
+gomp_team_barrier_wake (gomp_barrier_t *bar, int count)
+{
+  futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count);
+}
+
+void
+gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
+{
+  unsigned int generation, gen;
+
+  if (__builtin_expect (state & BAR_WAS_LAST, 0))
+    {
+      /* Next time we'll be awaiting TOTAL threads again.  */
+      struct gomp_thread *thr = gomp_thread ();
+      struct gomp_team *team = thr->ts.team;
+
+      bar->awaited = bar->total;
+      team->work_share_cancelled = 0;
+      if (__builtin_expect (team->task_count, 0))
+	{
+	  gomp_barrier_handle_tasks (state);
+	  state &= ~BAR_WAS_LAST;
+	}
+      else
+	{
+	  state &= ~BAR_CANCELLED;
+	  state += BAR_INCR - BAR_WAS_LAST;
+	  __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE);
+	  futex_wake ((int *) &bar->generation, INT_MAX);
+	  return;
+	}
+    }
+
+  generation = state;
+  state &= ~BAR_CANCELLED;
+  do
+    {
+      do_wait ((int *) &bar->generation, generation);
+      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+      if (__builtin_expect (gen & BAR_TASK_PENDING, 0))
+	{
+	  gomp_barrier_handle_tasks (state);
+	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+	}
+      generation |= gen & BAR_WAITING_FOR_TASK;
+    }
+  while (gen != state + BAR_INCR);
+}
+
+void
+gomp_team_barrier_wait (gomp_barrier_t *bar)
+{
+  gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar));
+}
+
+void
+gomp_team_barrier_wait_final (gomp_barrier_t *bar)
+{
+  gomp_barrier_state_t state = gomp_barrier_wait_final_start (bar);
+  if (__builtin_expect (state & BAR_WAS_LAST, 0))
+    bar->awaited_final = bar->total;
+  gomp_team_barrier_wait_end (bar, state);
+}
+
+bool
+gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
+				   gomp_barrier_state_t state)
+{
+  unsigned int generation, gen;
+
+  if (__builtin_expect (state & BAR_WAS_LAST, 0))
+    {
+      /* Next time we'll be awaiting TOTAL threads again.  */
+      /* BAR_CANCELLED should never be set in state here, because
+	 cancellation means that at least one of the threads has been
+	 cancelled, thus on a cancellable barrier we should never see
+	 all threads to arrive.  */
+      struct gomp_thread *thr = gomp_thread ();
+      struct gomp_team *team = thr->ts.team;
+
+      bar->awaited = bar->total;
+      team->work_share_cancelled = 0;
+      if (__builtin_expect (team->task_count, 0))
+	{
+	  gomp_barrier_handle_tasks (state);
+	  state &= ~BAR_WAS_LAST;
+	}
+      else
+	{
+	  state += BAR_INCR - BAR_WAS_LAST;
+	  __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE);
+	  futex_wake ((int *) &bar->generation, INT_MAX);
+	  return false;
+	}
+    }
+
+  if (__builtin_expect (state & BAR_CANCELLED, 0))
+    return true;
+
+  generation = state;
+  do
+    {
+      do_wait ((int *) &bar->generation, generation);
+      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+      if (__builtin_expect (gen & BAR_CANCELLED, 0))
+	return true;
+      if (__builtin_expect (gen & BAR_TASK_PENDING, 0))
+	{
+	  gomp_barrier_handle_tasks (state);
+	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+	}
+      generation |= gen & BAR_WAITING_FOR_TASK;
+    }
+  while (gen != state + BAR_INCR);
+
+  return false;
+}
+
+bool
+gomp_team_barrier_wait_cancel (gomp_barrier_t *bar)
+{
+  return gomp_team_barrier_wait_cancel_end (bar, gomp_barrier_wait_start (bar));
+}
+
+void
+gomp_team_barrier_cancel (struct gomp_team *team)
+{
+  gomp_mutex_lock (&team->task_lock);
+  if (team->barrier.generation & BAR_CANCELLED)
+    {
+      gomp_mutex_unlock (&team->task_lock);
+      return;
+    }
+  team->barrier.generation |= BAR_CANCELLED;
+  gomp_mutex_unlock (&team->task_lock);
+  futex_wake ((int *) &team->barrier.generation, INT_MAX);
+}
diff --git a/libgomp/config/rtems/bar.h b/libgomp/config/rtems/bar.h
new file mode 100644
index 0000000..c85dce3
--- /dev/null
+++ b/libgomp/config/rtems/bar.h
@@ -0,0 +1,170 @@
+/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU OpenMP Library (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the RTEMS implementation of a barrier synchronization
+   mechanism for libgomp.  It is identical to the Linux implementation, except
+   that the futex API is slightly different.  This type is private to the
+   library.  */
+
+#ifndef GOMP_BARRIER_H
+#define GOMP_BARRIER_H 1
+
+#include <sys/lock.h>
+
+typedef struct
+{
+  /* Make sure total/generation is in a mostly read cacheline, while
+     awaited in a separate cacheline.  */
+  unsigned total __attribute__((aligned (64)));
+  unsigned generation;
+  struct _Futex_Control futex;
+  unsigned awaited __attribute__((aligned (64)));
+  unsigned awaited_final;
+} gomp_barrier_t;
+
+typedef unsigned int gomp_barrier_state_t;
+
+/* The generation field contains a counter in the high bits, with a few
+   low bits dedicated to flags.  Note that TASK_PENDING and WAS_LAST can
+   share space because WAS_LAST is never stored back to generation.  */
+#define BAR_TASK_PENDING	1
+#define BAR_WAS_LAST		1
+#define BAR_WAITING_FOR_TASK	2
+#define BAR_CANCELLED		4
+#define BAR_INCR		8
+
+static inline void gomp_barrier_init (gomp_barrier_t *bar, unsigned count)
+{
+  bar->total = count;
+  bar->awaited = count;
+  bar->awaited_final = count;
+  bar->generation = 0;
+  _Futex_Initialize (&bar->futex);
+}
+
+static inline void gomp_barrier_reinit (gomp_barrier_t *bar, unsigned count)
+{
+  __atomic_add_fetch (&bar->awaited, count - bar->total, MEMMODEL_ACQ_REL);
+  bar->total = count;
+}
+
+static inline void gomp_barrier_destroy (gomp_barrier_t *bar)
+{
+}
+
+extern void gomp_barrier_wait (gomp_barrier_t *);
+extern void gomp_barrier_wait_last (gomp_barrier_t *);
+extern void gomp_barrier_wait_end (gomp_barrier_t *, gomp_barrier_state_t);
+extern void gomp_team_barrier_wait (gomp_barrier_t *);
+extern void gomp_team_barrier_wait_final (gomp_barrier_t *);
+extern void gomp_team_barrier_wait_end (gomp_barrier_t *,
+					gomp_barrier_state_t);
+extern bool gomp_team_barrier_wait_cancel (gomp_barrier_t *);
+extern bool gomp_team_barrier_wait_cancel_end (gomp_barrier_t *,
+					       gomp_barrier_state_t);
+extern void gomp_team_barrier_wake (gomp_barrier_t *, int);
+struct gomp_team;
+extern void gomp_team_barrier_cancel (struct gomp_team *);
+
+static inline gomp_barrier_state_t
+gomp_barrier_wait_start (gomp_barrier_t *bar)
+{
+  unsigned int ret = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+  ret &= -BAR_INCR | BAR_CANCELLED;
+  /* A memory barrier is needed before exiting from the various forms
+     of gomp_barrier_wait, to satisfy OpenMP API version 3.1 section
+     2.8.6 flush Construct, which says there is an implicit flush during
+     a barrier region.  This is a convenient place to add the barrier,
+     so we use MEMMODEL_ACQ_REL here rather than MEMMODEL_ACQUIRE.  */
+  if (__atomic_add_fetch (&bar->awaited, -1, MEMMODEL_ACQ_REL) == 0)
+    ret |= BAR_WAS_LAST;
+  return ret;
+}
+
+static inline gomp_barrier_state_t
+gomp_barrier_wait_cancel_start (gomp_barrier_t *bar)
+{
+  return gomp_barrier_wait_start (bar);
+}
+
+/* This is like gomp_barrier_wait_start, except it decrements
+   bar->awaited_final rather than bar->awaited and should be used
+   for the gomp_team_end barrier only.  */
+static inline gomp_barrier_state_t
+gomp_barrier_wait_final_start (gomp_barrier_t *bar)
+{
+  unsigned int ret = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
+  ret &= -BAR_INCR | BAR_CANCELLED;
+  /* See above gomp_barrier_wait_start comment.  */
+  if (__atomic_add_fetch (&bar->awaited_final, -1, MEMMODEL_ACQ_REL) == 0)
+    ret |= BAR_WAS_LAST;
+  return ret;
+}
+
+static inline bool
+gomp_barrier_last_thread (gomp_barrier_state_t state)
+{
+  return state & BAR_WAS_LAST;
+}
+
+/* All the inlines below must be called with team->task_lock
+   held.  */
+
+static inline void
+gomp_team_barrier_set_task_pending (gomp_barrier_t *bar)
+{
+  bar->generation |= BAR_TASK_PENDING;
+}
+
+static inline void
+gomp_team_barrier_clear_task_pending (gomp_barrier_t *bar)
+{
+  bar->generation &= ~BAR_TASK_PENDING;
+}
+
+static inline void
+gomp_team_barrier_set_waiting_for_tasks (gomp_barrier_t *bar)
+{
+  bar->generation |= BAR_WAITING_FOR_TASK;
+}
+
+static inline bool
+gomp_team_barrier_waiting_for_tasks (gomp_barrier_t *bar)
+{
+  return (bar->generation & BAR_WAITING_FOR_TASK) != 0;
+}
+
+static inline bool
+gomp_team_barrier_cancelled (gomp_barrier_t *bar)
+{
+  return __builtin_expect ((bar->generation & BAR_CANCELLED) != 0, 0);
+}
+
+static inline void
+gomp_team_barrier_done (gomp_barrier_t *bar, gomp_barrier_state_t state)
+{
+  bar->generation = (state & -BAR_INCR) + BAR_INCR;
+}
+
+#endif /* GOMP_BARRIER_H */
diff --git a/libgomp/config/rtems/mutex.c b/libgomp/config/rtems/mutex.c
new file mode 100644
index 0000000..39bb64d
--- /dev/null
+++ b/libgomp/config/rtems/mutex.c
@@ -0,0 +1 @@
+/* Everything is in the header.  */
diff --git a/libgomp/config/rtems/mutex.h b/libgomp/config/rtems/mutex.h
new file mode 100644
index 0000000..39901ee
--- /dev/null
+++ b/libgomp/config/rtems/mutex.h
@@ -0,0 +1,57 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU OpenMP Library (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the RTEMS implementation of a mutex synchronization
+   mechanism for libgomp.  This type is private to the library.  */
+
+#ifndef GOMP_MUTEX_H
+#define GOMP_MUTEX_H 1
+
+#include <sys/lock.h>
+
+typedef struct _Mutex_Control gomp_mutex_t;
+
+#define GOMP_MUTEX_INIT_0 1
+
+static inline void gomp_mutex_init (gomp_mutex_t *mutex)
+{
+  _Mutex_Initialize (mutex);
+}
+
+static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
+{
+  _Mutex_Acquire (mutex);
+}
+
+static inline void gomp_mutex_unlock (gomp_mutex_t *mutex)
+{
+  _Mutex_Release (mutex);
+}
+
+static inline void gomp_mutex_destroy (gomp_mutex_t *mutex)
+{
+  _Mutex_Destroy (mutex);
+}
+
+#endif /* GOMP_MUTEX_H */
diff --git a/libgomp/config/rtems/sem.c b/libgomp/config/rtems/sem.c
new file mode 100644
index 0000000..39bb64d
--- /dev/null
+++ b/libgomp/config/rtems/sem.c
@@ -0,0 +1 @@
+/* Everything is in the header.  */
diff --git a/libgomp/config/rtems/sem.h b/libgomp/config/rtems/sem.h
new file mode 100644
index 0000000..0d6765b
--- /dev/null
+++ b/libgomp/config/rtems/sem.h
@@ -0,0 +1,55 @@
+/* Copyright (C) 2015 Free Software Foundation, Inc.
+   Contributed by Sebastian Huber <sebastian.huber@embedded-brains.de>.
+
+   This file is part of the GNU OpenMP Library (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This is the RTEMS implementation of a semaphore synchronization
+   mechanism for libgomp.  This type is private to the library.  */
+
+#ifndef GOMP_SEM_H
+#define GOMP_SEM_H 1
+
+#include <sys/lock.h>
+
+typedef struct _Semaphore_Control gomp_sem_t;
+
+static inline void gomp_sem_init (gomp_sem_t *sem, int value)
+{
+  _Semaphore_Initialize (sem, (unsigned int) value);
+}
+
+static inline void gomp_sem_wait (gomp_sem_t *sem)
+{
+  _Semaphore_Wait (sem);
+}
+
+static inline void gomp_sem_post (gomp_sem_t *sem)
+{
+  _Semaphore_Post (sem);
+}
+
+static inline void gomp_sem_destroy (gomp_sem_t *sem)
+{
+  _Semaphore_Destroy (sem);
+}
+
+#endif /* GOMP_SEM_H  */
diff --git a/libgomp/configure b/libgomp/configure
index f1a92ba..a42e6bb 100755
--- a/libgomp/configure
+++ b/libgomp/configure
@@ -15338,6 +15338,23 @@ $as_echo "#define HAVE_BROKEN_POSIX_SEMAPHORES 1" >>confdefs.h
     ;;
 esac
 
+# RTEMS specific checks
+case "$host" in
+  *-*-rtems*)
+    ac_fn_c_check_type "$LINENO" "struct _Mutex_Control" "ac_cv_type_struct__Mutex_Control" "#include <sys/lock.h>
+"
+if test "x$ac_cv_type_struct__Mutex_Control" = x""yes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT__MUTEX_CONTROL 1
+_ACEOF
+
+
+fi
+
+    ;;
+esac
+
  # Check whether --enable-linux-futex was given.
 if test "${enable_linux_futex+set}" = set; then :
   enableval=$enable_linux_futex;
diff --git a/libgomp/configure.ac b/libgomp/configure.ac
index 9cf0218..461845e 100644
--- a/libgomp/configure.ac
+++ b/libgomp/configure.ac
@@ -213,6 +213,13 @@ case "$host" in
     ;;
 esac
 
+# RTEMS specific checks
+case "$host" in
+  *-*-rtems*)
+    AC_CHECK_TYPES([struct _Mutex_Control],[],[],[#include <sys/lock.h>])
+    ;;
+esac
+
 GCC_LINUX_FUTEX(:)
 
 # Check for pthread_{,attr_}[sg]etaffinity_np.
diff --git a/libgomp/configure.tgt b/libgomp/configure.tgt
index 2970f6f..25ec7fc 100644
--- a/libgomp/configure.tgt
+++ b/libgomp/configure.tgt
@@ -151,6 +151,13 @@ case "${target}" in
 	XLDFLAGS="${XLDFLAGS} -lpthread"
 	;;
 
+  *-*-rtems*)
+	# Use self-contained synchronization objects if provided by Newlib
+	if test "x$ac_cv_type_struct__Mutex_Control" = xyes ; then
+	    config_path="rtems posix"
+	fi
+	;;
+
   *)
 	;;
 
-- 
1.8.4.5

[PATCH 3/3] [gomp] Add thread attribute customization

[Sebastian Huber]

libgomp/ChangeLog
2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>

	* config/posix/pool.h (gomp_adjust_thread_attr): New.
	* config/rtems/pool.h (gomp_adjust_thread_attr): Likewise.
	(gomp_thread_pool_reservoir): Add priority member.
	* confi/rtems/proc.c (allocate_thread_pool_reservoir): Add
	priority.
	(parse_thread_pools): Likewise.
	* team.c (gomp_team_start): Rename thread_attr to mutable_attr.
	Call configuration provided gomp_adjust_thread_attr(). Destroy
	mutable attributes if necessary.
	* libgomp.texi: Document GOMP_RTEMS_THREAD_POOLS.
---
 libgomp/config/posix/pool.h |  7 +++++
 libgomp/config/rtems/pool.h | 29 ++++++++++++++++++
 libgomp/config/rtems/proc.c | 23 +++++++++++---
 libgomp/libgomp.texi        | 75 +++++++++++++++++++++++++++++++++++----------
 libgomp/team.c              | 15 ++++-----
 5 files changed, 121 insertions(+), 28 deletions(-)

diff --git a/libgomp/config/posix/pool.h b/libgomp/config/posix/pool.h
index 0d127a0..a8e2eec 100644
--- a/libgomp/config/posix/pool.h
+++ b/libgomp/config/posix/pool.h
@@ -57,4 +57,11 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
   /* Do nothing in the default implementation.  */
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  /* Do nothing in the default implementation.  */
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/pool.h b/libgomp/config/rtems/pool.h
index 5c989d0..facac05 100644
--- a/libgomp/config/rtems/pool.h
+++ b/libgomp/config/rtems/pool.h
@@ -41,6 +41,7 @@ struct gomp_thread_pool_reservoir {
   gomp_sem_t available;
   gomp_mutex_t lock;
   size_t index;
+  int priority;
   struct gomp_thread_pool *pools[];
 };
 
@@ -125,4 +126,32 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
     }
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  struct gomp_thread_pool_reservoir *res = gomp_get_thread_pool_reservoir ();
+  if (res != NULL && res->priority > 0)
+    {
+      struct sched_param param;
+      int err;
+      if (attr != mutable_attr)
+	{
+	  attr = mutable_attr;
+	  pthread_attr_init (attr);
+	}
+      memset (&param, 0, sizeof (param));
+      param.sched_priority = res->priority;
+      err = pthread_attr_setschedparam (attr, &param);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler parameters failed: %s", strerror (err));
+      err = pthread_attr_setschedpolicy (attr, SCHED_FIFO);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler policy failed: %s", strerror (err));
+      err = pthread_attr_setinheritsched (attr, PTHREAD_EXPLICIT_SCHED);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set explicit scheduler failed: %s", strerror (err));
+    }
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/proc.c b/libgomp/config/rtems/proc.c
index 9c36dcb..2939928 100644
--- a/libgomp/config/rtems/proc.c
+++ b/libgomp/config/rtems/proc.c
@@ -48,7 +48,8 @@ allocate_thread_pool_reservoirs (void)
 }
 
 static void
-allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
+allocate_thread_pool_reservoir (unsigned long count, unsigned long priority,
+				unsigned long scheduler)
 {
   struct gomp_thread_pool_reservoir *res;
   struct gomp_thread_pool *pools;
@@ -63,6 +64,7 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
   memset (pools, 0, size);
   res = (struct gomp_thread_pool_reservoir *) (pools + count);
   res->index = count;
+  res->priority = priority;
   gomp_sem_init (&res->available, count);
   gomp_mutex_init (&res->lock);
   for (i = 0; i < count; ++i)
@@ -71,7 +73,8 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
 }
 
 static char *
-parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
+parse_thread_pools (char *env, unsigned long *count, unsigned long *priority,
+		    unsigned long *scheduler)
 {
   size_t len;
   int i;
@@ -84,6 +87,17 @@ parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
   if (errno != 0)
     gomp_fatal ("Invalid thread pool count");
 
+  if (*env == '$')
+    {
+      ++env;
+      errno = 0;
+      *priority = strtoul (env, &env, 10);
+      if (errno != 0)
+	gomp_fatal ("Invalid thread pool priority");
+    }
+  else
+    *priority = -1;
+
   if (*env != '@')
     gomp_fatal ("Invalid thread pool scheduler prefix");
   ++env;
@@ -110,9 +124,10 @@ init_thread_pool_reservoirs (void)
       while (*env != '\0')
 	{
 	  unsigned long count;
+	  unsigned long priority;
 	  unsigned long scheduler;
-	  env = parse_thread_pools (env, &count, &scheduler);
-	  allocate_thread_pool_reservoir (count, scheduler);
+	  env = parse_thread_pools (env, &count, &priority, &scheduler);
+	  allocate_thread_pool_reservoir (count, priority, scheduler);
 	}
     }
 }
diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi
index 6c7f1ae..06b1c67 100644
--- a/libgomp/libgomp.texi
+++ b/libgomp/libgomp.texi
@@ -1306,23 +1306,24 @@ section 4 of the OpenMP specification in version 4.0, while those
 beginning with @env{GOMP_} are GNU extensions.
 
 @menu
-* OMP_CANCELLATION::      Set whether cancellation is activated
-* OMP_DISPLAY_ENV::       Show OpenMP version and environment variables
-* OMP_DEFAULT_DEVICE::    Set the device used in target regions
-* OMP_DYNAMIC::           Dynamic adjustment of threads
-* OMP_MAX_ACTIVE_LEVELS:: Set the maximum number of nested parallel regions
-* OMP_NESTED::            Nested parallel regions
-* OMP_NUM_THREADS::       Specifies the number of threads to use
-* OMP_PROC_BIND::         Whether theads may be moved between CPUs
-* OMP_PLACES::            Specifies on which CPUs the theads should be placed
-* OMP_STACKSIZE::         Set default thread stack size
-* OMP_SCHEDULE::          How threads are scheduled
-* OMP_THREAD_LIMIT::      Set the maximum number of threads
-* OMP_WAIT_POLICY::       How waiting threads are handled
-* GOMP_CPU_AFFINITY::     Bind threads to specific CPUs
-* GOMP_DEBUG::            Enable debugging output
-* GOMP_STACKSIZE::        Set default thread stack size
-* GOMP_SPINCOUNT::        Set the busy-wait spin count
+* OMP_CANCELLATION::        Set whether cancellation is activated
+* OMP_DISPLAY_ENV::         Show OpenMP version and environment variables
+* OMP_DEFAULT_DEVICE::      Set the device used in target regions
+* OMP_DYNAMIC::             Dynamic adjustment of threads
+* OMP_MAX_ACTIVE_LEVELS::   Set the maximum number of nested parallel regions
+* OMP_NESTED::              Nested parallel regions
+* OMP_NUM_THREADS::         Specifies the number of threads to use
+* OMP_PROC_BIND::           Whether theads may be moved between CPUs
+* OMP_PLACES::              Specifies on which CPUs the theads should be placed
+* OMP_STACKSIZE::           Set default thread stack size
+* OMP_SCHEDULE::            How threads are scheduled
+* OMP_THREAD_LIMIT::        Set the maximum number of threads
+* OMP_WAIT_POLICY::         How waiting threads are handled
+* GOMP_CPU_AFFINITY::       Bind threads to specific CPUs
+* GOMP_DEBUG::              Enable debugging output
+* GOMP_STACKSIZE::          Set default thread stack size
+* GOMP_SPINCOUNT::          Set the busy-wait spin count
+* GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools
 @end menu
 
 
@@ -1705,6 +1706,46 @@ or @env{OMP_WAIT_POLICY} is @code{PASSIVE}.
 
 
 
+@node GOMP_RTEMS_THREAD_POOLS
+@section @env{GOMP_RTEMS_THREAD_POOLS} -- Set the RTEMS specific thread pools
+@cindex Environment Variable
+@cindex Implementation specific setting
+@table @asis
+@item @emph{Description}:
+This environment variable is only used on the RTEMS real-time operating system.
+It determines the scheduler instance specific thread pools.  The format for
+@env{GOMP_RTEMS_THREAD_POOLS} is a list of optional
+@code{<thread-pool-count>[$<priority>]@@<scheduler-name>} configurations
+separated by @code{:} where:
+@itemize @bullet
+@item @code{<thread-pool-count>} is the thread pool count for this scheduler
+instance.
+@item @code{$<priority>} is an optional priority for the worker threads of a
+thread pool according to @code{pthread_setschedparam}.  In case a priority
+value is omitted, then a worker thread will inherit the priority of the OpenMP
+master thread that created it.  The priority of the worker thread is not
+changed after creation, even if a new OpenMP master thread using the worker has
+a different priority.
+@item @code{@@<scheduler-name>} is the scheduler instance name according to the
+RTEMS application configuration.
+@end itemize
+In case no thread pool configuration is specified for a scheduler instance,
+then each OpenMP master thread of this scheduler instance will use its own
+dynamically allocated thread pool.  To limit the worker thread count of the
+thread pools, each OpenMP master thread must call @code{omp_set_num_threads}.
+@item @emph{Example}:
+Lets suppose we have three scheduler instances @code{IO}, @code{WRK0}, and
+@code{WRK1} with @env{GOMP_RTEMS_THREAD_POOLS} set to
+@code{"1@@WRK0:3$4@@WRK1"}.  Then there are no thread pool restrictions for
+scheduler instance @code{IO}.  In the scheduler instance @code{WRK0} there is
+one thread pool available.  Since no priority is specified for this scheduler
+instance, the worker thread inherits the priority of the OpenMP master thread
+that created it.  In the scheduler instance @code{WRK1} there are three thread
+pools available and their worker threads run at priority four.
+@end table
+
+
+
 @c ---------------------------------------------------------------------
 @c The libgomp ABI
 @c ---------------------------------------------------------------------
diff --git a/libgomp/team.c b/libgomp/team.c
index 5edae07..2a51b66 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
   bool nested;
   struct gomp_thread_pool *pool;
   unsigned i, n, old_threads_used = 0;
-  pthread_attr_t thread_attr, *attr;
+  pthread_attr_t mutable_attr, *attr;
   unsigned long nthreads_var;
   char bind, bind_var;
   unsigned int s = 0, rest = 0, p = 0, k = 0;
@@ -697,11 +697,11 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
   if (__builtin_expect (gomp_places_list != NULL, 0))
     {
       size_t stacksize;
-      pthread_attr_init (&thread_attr);
-      pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED);
+      attr = &mutable_attr;
+      pthread_attr_init (attr);
+      pthread_attr_setdetachstate (attr, PTHREAD_CREATE_DETACHED);
       if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
-	pthread_attr_setstacksize (&thread_attr, stacksize);
-      attr = &thread_attr;
+	pthread_attr_setstacksize (attr, stacksize);
     }
 
   start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
@@ -799,13 +799,14 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
       start_data->thread_pool = pool;
       start_data->nested = nested;
 
+      attr = gomp_adjust_thread_attr (attr, &mutable_attr);
       err = pthread_create (&pt, attr, gomp_thread_start, start_data++);
       if (err != 0)
 	gomp_fatal ("Thread creation failed: %s", strerror (err));
     }
 
-  if (__builtin_expect (gomp_places_list != NULL, 0))
-    pthread_attr_destroy (&thread_attr);
+  if (__builtin_expect (attr == &mutable_attr, 0))
+    pthread_attr_destroy (attr);
 
  do_release:
   gomp_barrier_wait (nested ? &team->barrier : &pool->threads_dock);
-- 
1.8.4.5

Re: [PATCH 1/3] [gomp] Add RTEMS configuration

[Jakub Jelinek]

On Tue, Jul 28, 2015 at 01:04:57PM +0200, Sebastian Huber wrote:
> libgomp/ChangeLog
> 2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>
> 
> 	* config/rtems/bar.c: New.
> 	* config/rtems/bar.h: Likewise.
> 	* config/rtems/mutex.c: Likewise.
> 	* config/rtems/mutex.h: Likewise.
> 	* config/rtems/sem.c: Likewise.
> 	* config/rtems/sem.h: Likewise.
> 	* configure.ac (*-*-rtems*): Check that Newlib provides a proper
> 	<sys/lock.h> header file.
> 	* configure.tgt (*-*-rtems*): Enable RTEMS configuration if
> 	supported by Newlib.
> 	* configure: Regenerate.

> --- /dev/null
> +++ b/libgomp/config/rtems/bar.c
> @@ -0,0 +1,255 @@
> +
> +static gomp_barrier_t *
> +generation_to_barrier(int *addr)

Missing space before (.

Ok with that fixed.

	Jakub

Re: [PATCH 1/3] [gomp] Add RTEMS configuration

[Jakub Jelinek]

On Thu, Sep 03, 2015 at 11:46:49AM +0200, Jakub Jelinek wrote:
> On Tue, Jul 28, 2015 at 01:04:57PM +0200, Sebastian Huber wrote:
> > libgomp/ChangeLog
> > 2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>
> > 
> > 	* config/rtems/bar.c: New.
> > 	* config/rtems/bar.h: Likewise.
> > 	* config/rtems/mutex.c: Likewise.
> > 	* config/rtems/mutex.h: Likewise.
> > 	* config/rtems/sem.c: Likewise.
> > 	* config/rtems/sem.h: Likewise.
> > 	* configure.ac (*-*-rtems*): Check that Newlib provides a proper
> > 	<sys/lock.h> header file.
> > 	* configure.tgt (*-*-rtems*): Enable RTEMS configuration if
> > 	supported by Newlib.
> > 	* configure: Regenerate.
> 
> > --- /dev/null
> > +++ b/libgomp/config/rtems/bar.c
> > @@ -0,0 +1,255 @@
> > +
> > +static gomp_barrier_t *
> > +generation_to_barrier(int *addr)
> 
> Missing space before (.

Oh, and please use
#include "libgomp.h"
#include "bar.h"
instead of
#include <libgomp.h>
#include <bar.h>
(to help differentiate system headers from libgomp specific headers).

	Jakub

Re: [PATCH 2/3] [gomp] Thread pool management

[Jakub Jelinek]

On Tue, Jul 28, 2015 at 01:04:58PM +0200, Sebastian Huber wrote:
> +#ifndef GOMP_POOL_H
> +#define GOMP_POOL_H 1
> +
> +#include <libgomp.h>

Please use #include "libgomp.h" here.

> +#include <libgomp.h>

Similarly.

> +
> +#include "libgomp.h"
> +#include <errno.h>
> +#include <pool.h>

#include "pool.h" (and perhaps if possible move after libgomp.h).

> --- a/libgomp/team.c
> +++ b/libgomp/team.c
> @@ -27,6 +27,7 @@
>     creation and termination.  */
>  
>  #include "libgomp.h"
> +#include <pool.h>

Likewise.

Ok with those changes.

	Jakub

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Jakub Jelinek]

On Tue, Jul 28, 2015 at 01:04:59PM +0200, Sebastian Huber wrote:
> libgomp/ChangeLog
> 2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>
> 
> 	* config/posix/pool.h (gomp_adjust_thread_attr): New.
> 	* config/rtems/pool.h (gomp_adjust_thread_attr): Likewise.
> 	(gomp_thread_pool_reservoir): Add priority member.
> 	* confi/rtems/proc.c (allocate_thread_pool_reservoir): Add
> 	priority.
> 	(parse_thread_pools): Likewise.
> 	* team.c (gomp_team_start): Rename thread_attr to mutable_attr.
> 	Call configuration provided gomp_adjust_thread_attr(). Destroy
> 	mutable attributes if necessary.
> 	* libgomp.texi: Document GOMP_RTEMS_THREAD_POOLS.

Wonder if the RTEMS specific bits in libgomp.texi shouldn't be guarded
with
@ifset RTEMS
...
@end ifset
and --texinfo='@set RTEMS' be used when compiling it.
But perhaps it can be done incrementally, so the patch is ok.

> @@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
>    bool nested;
>    struct gomp_thread_pool *pool;
>    unsigned i, n, old_threads_used = 0;
> -  pthread_attr_t thread_attr, *attr;
> +  pthread_attr_t mutable_attr, *attr;

Just wonder why have you renamed this variable.  It is a thread attribute
after all, even after your changes.  mutable_attr doesn't make much sense to
me.

	Jakub

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Sebastian Huber]

On 03/09/15 12:19, Jakub Jelinek wrote:
>> @@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
>> >    bool nested;
>> >    struct gomp_thread_pool *pool;
>> >    unsigned i, n, old_threads_used = 0;
>> >-  pthread_attr_t thread_attr, *attr;
>> >+  pthread_attr_t mutable_attr, *attr;
> Just wonder why have you renamed this variable.  It is a thread attribute
> after all, even after your changes.  mutable_attr doesn't make much sense to
> me.

We have only thread attributes in this function: mutable_attr and attr. 
The attr is initialized with &gomp_thread_attr and gomp_thread_attr is 
supposed to be read-only by this function. Under certain conditions we 
have to modify the initial attributes. Since gomp_thread_attr is 
read-only, we have to copy it and then modify the copy. For this we need 
some storage: mutable_attr.

-- 
Sebastian Huber, embedded brains GmbH

Address : Dornierstr. 4, D-82178 Puchheim, Germany
Phone   : +49 89 189 47 41-16
Fax     : +49 89 189 47 41-09
E-Mail  : sebastian.huber@embedded-brains.de
PGP     : Public key available on request.

Diese Nachricht ist keine geschäftliche Mitteilung im Sinne des EHUG.

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Jakub Jelinek]

On Thu, Sep 03, 2015 at 12:57:53PM +0200, Sebastian Huber wrote:
> On 03/09/15 12:19, Jakub Jelinek wrote:
> >>@@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
> >>>    bool nested;
> >>>    struct gomp_thread_pool *pool;
> >>>    unsigned i, n, old_threads_used = 0;
> >>>-  pthread_attr_t thread_attr, *attr;
> >>>+  pthread_attr_t mutable_attr, *attr;
> >Just wonder why have you renamed this variable.  It is a thread attribute
> >after all, even after your changes.  mutable_attr doesn't make much sense to
> >me.
> 
> We have only thread attributes in this function: mutable_attr and attr. The
> attr is initialized with &gomp_thread_attr and gomp_thread_attr is supposed
> to be read-only by this function. Under certain conditions we have to modify
> the initial attributes. Since gomp_thread_attr is read-only, we have to copy
> it and then modify the copy. For this we need some storage: mutable_attr.

So use local_thread_attr if you want to stress it, but IMHO thread_attr
just just fine.  I really don't like mutable_attr.

	Jakub

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Sebastian Huber]

On 03/09/15 13:05, Jakub Jelinek wrote:
> On Thu, Sep 03, 2015 at 12:57:53PM +0200, Sebastian Huber wrote:
>> >On 03/09/15 12:19, Jakub Jelinek wrote:
>>>> > >>@@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
>>>>> > >>>    bool nested;
>>>>> > >>>    struct gomp_thread_pool *pool;
>>>>> > >>>    unsigned i, n, old_threads_used = 0;
>>>>> > >>>-  pthread_attr_t thread_attr, *attr;
>>>>> > >>>+  pthread_attr_t mutable_attr, *attr;
>>> > >Just wonder why have you renamed this variable.  It is a thread attribute
>>> > >after all, even after your changes.  mutable_attr doesn't make much sense to
>>> > >me.
>> >
>> >We have only thread attributes in this function: mutable_attr and attr. The
>> >attr is initialized with &gomp_thread_attr and gomp_thread_attr is supposed
>> >to be read-only by this function. Under certain conditions we have to modify
>> >the initial attributes. Since gomp_thread_attr is read-only, we have to copy
>> >it and then modify the copy. For this we need some storage: mutable_attr.
> So use local_thread_attr if you want to stress it, but IMHO thread_attr
> just just fine.  I really don't like mutable_attr.

Ok, if I don't rename thread_attr, is the patch ok?

-- 
Sebastian Huber, embedded brains GmbH

Address : Dornierstr. 4, D-82178 Puchheim, Germany
Phone   : +49 89 189 47 41-16
Fax     : +49 89 189 47 41-09
E-Mail  : sebastian.huber@embedded-brains.de
PGP     : Public key available on request.

Diese Nachricht ist keine geschäftliche Mitteilung im Sinne des EHUG.

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Jakub Jelinek]

On Thu, Sep 03, 2015 at 01:09:23PM +0200, Sebastian Huber wrote:
> >>>We have only thread attributes in this function: mutable_attr and attr. The
> >>>attr is initialized with &gomp_thread_attr and gomp_thread_attr is supposed
> >>>to be read-only by this function. Under certain conditions we have to modify
> >>>the initial attributes. Since gomp_thread_attr is read-only, we have to copy
> >>>it and then modify the copy. For this we need some storage: mutable_attr.
> >So use local_thread_attr if you want to stress it, but IMHO thread_attr
> >just just fine.  I really don't like mutable_attr.
> 
> Ok, if I don't rename thread_attr, is the patch ok?

Yes.

	Jakub

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Sebastian Huber]

On 03/09/15 13:10, Jakub Jelinek wrote:
> On Thu, Sep 03, 2015 at 01:09:23PM +0200, Sebastian Huber wrote:
>>>>> We have only thread attributes in this function: mutable_attr and attr. The
>>>>> attr is initialized with &gomp_thread_attr and gomp_thread_attr is supposed
>>>>> to be read-only by this function. Under certain conditions we have to modify
>>>>> the initial attributes. Since gomp_thread_attr is read-only, we have to copy
>>>>> it and then modify the copy. For this we need some storage: mutable_attr.
>>> So use local_thread_attr if you want to stress it, but IMHO thread_attr
>>> just just fine.  I really don't like mutable_attr.
>> Ok, if I don't rename thread_attr, is the patch ok?
> Yes.

Thanks a lot for your kind review.

I committed the patches as:

https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227439
https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227440
https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227441
https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227442

-- 
Sebastian Huber, embedded brains GmbH

Address : Dornierstr. 4, D-82178 Puchheim, Germany
Phone   : +49 89 189 47 41-16
Fax     : +49 89 189 47 41-09
E-Mail  : sebastian.huber@embedded-brains.de
PGP     : Public key available on request.

Diese Nachricht ist keine geschäftliche Mitteilung im Sinne des EHUG.

Re: [PATCH 3/3] [gomp] Add thread attribute customization

[Jakub Jelinek]

On Thu, Sep 03, 2015 at 01:36:35PM +0200, Sebastian Huber wrote:
> On 03/09/15 13:10, Jakub Jelinek wrote:
> >On Thu, Sep 03, 2015 at 01:09:23PM +0200, Sebastian Huber wrote:
> >>>>>We have only thread attributes in this function: mutable_attr and attr. The
> >>>>>attr is initialized with &gomp_thread_attr and gomp_thread_attr is supposed
> >>>>>to be read-only by this function. Under certain conditions we have to modify
> >>>>>the initial attributes. Since gomp_thread_attr is read-only, we have to copy
> >>>>>it and then modify the copy. For this we need some storage: mutable_attr.
> >>>So use local_thread_attr if you want to stress it, but IMHO thread_attr
> >>>just just fine.  I really don't like mutable_attr.
> >>Ok, if I don't rename thread_attr, is the patch ok?
> >Yes.
> 
> Thanks a lot for your kind review.
> 
> I committed the patches as:
> 
> https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227439
> https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227440
> https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227441
> https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=227442

Unfortunately it broke stuff, here is a fix I've committed:

2015-09-03  Jakub Jelinek  <jakub@redhat.com>

	* configure.tgt: Add missing ;; in between nvptx and rtems
	snippets.

--- libgomp/configure.tgt	(revision 227456)
+++ libgomp/configure.tgt	(working copy)
@@ -153,6 +153,7 @@ case "${target}" in
 
   nvptx*-*-*)
 	config_path="nvptx"
+	;;
 
   *-*-rtems*)
 	# Use self-contained synchronization objects if provided by Newlib


	Jakub