[PATCH] libgomp(OMPD PROJECT): add ICVs debugging information.

[PATCH] libgomp(OMPD PROJECT): add ICVs debugging information.

[Mohamed Atef]

[-- Attachment #1: Type: text/plain, Size: 1796 bytes --]

Hi,
   This Patch contains the function that gets all global ICVs information,
and prototypes of local ICVs.

Notes:
1) gomp_affinity_format_len doesn't have a value so I assumed that
gomp_affinity_format has length 100 for now.

2) I didn't have any knowledge in OpenMP before this project,
so if any of the ICVs scopes is wrong please let me know.

I hope to hear from you soon, as we are running out of time.
Thanks



2022-03-12  Mohamed Atef  <mohamedatef1698@gmail.com>

        *Makefile.am: add ompd-icv to libgompd_la_SOURCES.
        *Makefile.in: Regenerate.
        *parallel.c: fixed the call of ompd_bp_parallel_begin, and
        ompd_bp_parallel_begin.
        *ompd-icv.c: New file.
        *omp-tools.h.in: fix soe writing formats.
        *ompd-helper.h: (struct ompd_thread_handle_t, struct
ompd_parallel_handle_t,
        struct ompd_task_handle_t, GET_VALUE macro, CHECK macro,
FOREACH_OMPD_ICV macro,
        enum ompd_icv,
        ): Defined
        (prototypes of ompd_get_nthread, ompd_get_thread_limit,
ompd_get_run_sched, ompd_get_run_sched_chunk_size,
        ompd_get_default_device, ompd_get_dynamic,
ompd_get_max_active_levels, ompd_get_proc_bind,
ompd_is_final,ompd_is_implicit,
        ompd_get_team_size, ompd_get_cancellation,
ompd_get_max_task_priority, ompd_get_stacksize, ompd_get_debug,
ompd_get_display_affinity,
        ompd_get_affinity_format, ompd_get_affinity_format_len,
ompd_get_wait_policy, ompd_get_num_teams, ompd_get_teams_thread_limit,
ompd_get_spin_count,
        ompd_get_available_cpus, ompd_get_throttled_spin_count, and
ompd_get_managed_threads): Added.
        *ompd-init.c: GET_VALUE is used instead of a ton of lines.
        *libgompd.map: (ompd_enumerate_icvs, ompd_get_icv_from_scope, and
ompd_get_icv_string_from_scope): exported.

[-- Attachment #2: diff.txt --]
[-- Type: text/plain, Size: 28537 bytes --]


 
diff --git a/libgomp/Makefile.am b/libgomp/Makefile.am
index 22a27df105e..20d0d62f473 100644
--- a/libgomp/Makefile.am
+++ b/libgomp/Makefile.am
@@ -93,7 +93,7 @@ libgomp_la_SOURCES = alloc.c atomic.c barrier.c critical.c env.c error.c \
 	priority_queue.c affinity-fmt.c teams.c allocator.c oacc-profiling.c \
 	oacc-target.c ompd-support.c
 
-libgompd_la_SOURCES = ompd-init.c ompd-helper.c
+libgompd_la_SOURCES = ompd-init.c ompd-helper.c ompd-icv.c
 include $(top_srcdir)/plugin/Makefrag.am
 
 if USE_FORTRAN
diff --git a/libgomp/Makefile.in b/libgomp/Makefile.in
index 8ecf6dcf192..df0986ee5c2 100644
--- a/libgomp/Makefile.in
+++ b/libgomp/Makefile.in
@@ -223,7 +223,7 @@ am_libgomp_la_OBJECTS = alloc.lo atomic.lo barrier.lo critical.lo \
 	oacc-target.lo ompd-support.lo $(am__objects_1)
 libgomp_la_OBJECTS = $(am_libgomp_la_OBJECTS)
 libgompd_la_LIBADD =
-am_libgompd_la_OBJECTS = ompd-init.lo ompd-helper.lo
+am_libgompd_la_OBJECTS = ompd-init.lo ompd-helper.lo ompd-icv.lo
 libgompd_la_OBJECTS = $(am_libgompd_la_OBJECTS)
 AM_V_P = $(am__v_P_@AM_V@)
 am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
@@ -258,15 +258,16 @@ am__depfiles_remade = ./$(DEPDIR)/affinity-fmt.Plo \
 	./$(DEPDIR)/oacc-mem.Plo ./$(DEPDIR)/oacc-parallel.Plo \
 	./$(DEPDIR)/oacc-plugin.Plo ./$(DEPDIR)/oacc-profiling.Plo \
 	./$(DEPDIR)/oacc-target.Plo ./$(DEPDIR)/ompd-helper.Plo \
-	./$(DEPDIR)/ompd-init.Plo ./$(DEPDIR)/ompd-support.Plo \
-	./$(DEPDIR)/ordered.Plo ./$(DEPDIR)/parallel.Plo \
-	./$(DEPDIR)/priority_queue.Plo ./$(DEPDIR)/proc.Plo \
-	./$(DEPDIR)/ptrlock.Plo ./$(DEPDIR)/scope.Plo \
-	./$(DEPDIR)/sections.Plo ./$(DEPDIR)/sem.Plo \
-	./$(DEPDIR)/single.Plo ./$(DEPDIR)/splay-tree.Plo \
-	./$(DEPDIR)/target.Plo ./$(DEPDIR)/task.Plo \
-	./$(DEPDIR)/team.Plo ./$(DEPDIR)/teams.Plo \
-	./$(DEPDIR)/time.Plo ./$(DEPDIR)/work.Plo
+	./$(DEPDIR)/ompd-icv.Plo ./$(DEPDIR)/ompd-init.Plo \
+	./$(DEPDIR)/ompd-support.Plo ./$(DEPDIR)/ordered.Plo \
+	./$(DEPDIR)/parallel.Plo ./$(DEPDIR)/priority_queue.Plo \
+	./$(DEPDIR)/proc.Plo ./$(DEPDIR)/ptrlock.Plo \
+	./$(DEPDIR)/scope.Plo ./$(DEPDIR)/sections.Plo \
+	./$(DEPDIR)/sem.Plo ./$(DEPDIR)/single.Plo \
+	./$(DEPDIR)/splay-tree.Plo ./$(DEPDIR)/target.Plo \
+	./$(DEPDIR)/task.Plo ./$(DEPDIR)/team.Plo \
+	./$(DEPDIR)/teams.Plo ./$(DEPDIR)/time.Plo \
+	./$(DEPDIR)/work.Plo
 am__mv = mv -f
 COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
@@ -605,7 +606,7 @@ libgomp_la_SOURCES = alloc.c atomic.c barrier.c critical.c env.c \
 	oacc-async.c oacc-plugin.c oacc-cuda.c priority_queue.c \
 	affinity-fmt.c teams.c allocator.c oacc-profiling.c \
 	oacc-target.c ompd-support.c $(am__append_3)
-libgompd_la_SOURCES = ompd-init.c ompd-helper.c
+libgompd_la_SOURCES = ompd-init.c ompd-helper.c ompd-icv.c
 
 # Nvidia PTX OpenACC plugin.
 @PLUGIN_NVPTX_TRUE@libgomp_plugin_nvptx_version_info = -version-info $(libtool_VERSION)
@@ -818,6 +819,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/oacc-profiling.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/oacc-target.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ompd-helper.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ompd-icv.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ompd-init.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ompd-support.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ordered.Plo@am__quote@ # am--include-marker
@@ -1259,6 +1261,7 @@ distclean: distclean-recursive
 	-rm -f ./$(DEPDIR)/oacc-profiling.Plo
 	-rm -f ./$(DEPDIR)/oacc-target.Plo
 	-rm -f ./$(DEPDIR)/ompd-helper.Plo
+	-rm -f ./$(DEPDIR)/ompd-icv.Plo
 	-rm -f ./$(DEPDIR)/ompd-init.Plo
 	-rm -f ./$(DEPDIR)/ompd-support.Plo
 	-rm -f ./$(DEPDIR)/ordered.Plo
@@ -1449,6 +1452,7 @@ maintainer-clean: maintainer-clean-recursive
 	-rm -f ./$(DEPDIR)/oacc-profiling.Plo
 	-rm -f ./$(DEPDIR)/oacc-target.Plo
 	-rm -f ./$(DEPDIR)/ompd-helper.Plo
+	-rm -f ./$(DEPDIR)/ompd-icv.Plo
 	-rm -f ./$(DEPDIR)/ompd-init.Plo
 	-rm -f ./$(DEPDIR)/ompd-support.Plo
 	-rm -f ./$(DEPDIR)/ordered.Plo
diff --git a/libgomp/libgompd.map b/libgomp/libgompd.map
index ef29a0ea5a8..74dafcb097e 100644
--- a/libgomp/libgompd.map
+++ b/libgomp/libgompd.map
@@ -7,6 +7,9 @@ OMPD_5.1 {
 	ompd_device_initialize;
 	ompd_rel_address_space_handle;
 	ompd_finalize;
+	ompd_enumerate_icvs;
+	ompd_get_icv_from_scope;
+	ompd_get_icv_string_from_scope;
   local:
   	*;
 };
diff --git a/libgomp/omp-tools.h.in b/libgomp/omp-tools.h.in
index bfab48a0a63..1c0c69abb75 100644
--- a/libgomp/omp-tools.h.in
+++ b/libgomp/omp-tools.h.in
@@ -65,12 +65,12 @@ typedef struct _ompd_task_handle ompd_task_handle_t;
 
 typedef enum ompd_scope_t
 {
-	ompd_scope_global		    = 1,
-	ompd_scope_address_space = 2,
-	ompd_scope_thread		    = 3,
-	ompd_scope_parallel		 = 4,
-	ompd_scope_implicit_task = 5,
-	ompd_scope_task          = 6
+	ompd_scope_global		    	= 1,
+	ompd_scope_address_space 	= 2,
+	ompd_scope_thread		    	= 3,
+	ompd_scope_parallel		 		= 4,
+	ompd_scope_implicit_task 	= 5,
+	ompd_scope_task          	= 6
 } ompd_scope_t;
 
 typedef __UINT64_TYPE__ ompd_icv_id_t;
@@ -110,7 +110,7 @@ typedef struct ompd_device_type_sizes_t
 typedef ompd_rc_t (*ompd_callback_memory_alloc_fn_t) (ompd_size_t, void **);
 typedef ompd_rc_t (*ompd_callback_memory_free_fn_t) (void *);
 
-typedef ompd_rc_t (*ompd_callback_get_thread_context_for_thread_id_fn_t) 
+typedef ompd_rc_t (*ompd_callback_get_thread_context_for_thread_id_fn_t)
 	(ompd_address_space_context_t *, ompd_thread_id_t,
 	ompd_size_t, const void *, ompd_thread_context_t **);
 
@@ -119,7 +119,7 @@ typedef ompd_rc_t (*ompd_callback_sizeof_fn_t)
 	(ompd_address_space_context_t *, ompd_device_type_sizes_t *);
 
 
-typedef ompd_rc_t (*ompd_callback_symbol_addr_fn_t) 
+typedef ompd_rc_t (*ompd_callback_symbol_addr_fn_t)
 	(ompd_address_space_context_t *, ompd_thread_context_t *,
 	const char *, ompd_address_t *, const char *);
 
@@ -139,7 +139,7 @@ typedef ompd_rc_t (*ompd_callback_device_host_fn_t)
 
 typedef ompd_rc_t (*ompd_callback_print_string_fn_t) (const char *, int);
 
-typedef struct ompd_callbacks_t 
+typedef struct ompd_callbacks_t
 {
 	ompd_callback_memory_alloc_fn_t alloc_memory;
 	ompd_callback_memory_free_fn_t free_memory;
@@ -163,11 +163,11 @@ ompd_rc_t ompd_get_version_string(const char **);
 
 ompd_rc_t ompd_finalize(void);
 
-ompd_rc_t ompd_process_initialize 
+ompd_rc_t ompd_process_initialize
 	(ompd_address_space_context_t *, ompd_address_space_handle_t **);
 
 
-ompd_rc_t ompd_device_initialize 
+ompd_rc_t ompd_device_initialize
 	(ompd_address_space_handle_t *, ompd_address_space_context_t *,
 	ompd_device_t, ompd_size_t, void *, ompd_address_space_handle_t **);
 
@@ -175,7 +175,7 @@ ompd_rc_t ompd_rel_address_space_handle(ompd_address_space_handle_t *);
 
 ompd_rc_t ompd_get_omp_version (ompd_address_space_handle_t *, ompd_word_t *);
 
-ompd_rc_t ompd_get_omp_version_string (ompd_address_space_handle_t *, 
+ompd_rc_t ompd_get_omp_version_string (ompd_address_space_handle_t *,
 	const char **);
 
 
@@ -246,7 +246,7 @@ ompd_rc_t ompd_get_display_control_vars (ompd_address_space_handle_t *,
 ompd_rc_t ompd_rel_display_control_vars (const char * const **);
 
 ompd_rc_t ompd_enumerate_icvs (ompd_address_space_handle_t *, ompd_icv_id_t,
-	ompd_icv_id_t *, const char **, ompd_scope_t *,int *);
+	ompd_icv_id_t *, const char **, ompd_scope_t *, int *);
 
 ompd_rc_t ompd_get_icv_from_scope (void *, ompd_scope_t, ompd_icv_id_t,
 	ompd_word_t *);
@@ -279,4 +279,4 @@ void ompd_bp_device_end(void);
 }; // extern "C"
 #endif
 
-#endif /* _OMP_TOOLS_H */
\ No newline at end of file
+#endif /* _OMP_TOOLS_H */
diff --git a/libgomp/ompd-helper.c b/libgomp/ompd-helper.c
index 033990073a5..e6e80188bb9 100644
--- a/libgomp/ompd-helper.c
+++ b/libgomp/ompd-helper.c
@@ -19,13 +19,210 @@
    <http://www.gnu.org/licenses/>.  */
 
 
-
+/* This file contains the source code of functions
+	declared in ompd-helper.h.  */
 
 
 #include "ompd-helper.h"
 
 ompd_device_type_sizes_t target_sizes;
 
+
+/* Get global ICVs.  */
+ompd_rc_t
+ompd_get_cancellation(ompd_address_space_handle_t *ah, ompd_word_t *cancel_var)
+{
+	CHECK(ah);
+	ompd_word_t cancel = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_cancel_var", cancel, cancel,
+		target_sizes.sizeof_char, 1, ret);
+	*cancel_var = cancel;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_max_task_priority(ompd_address_space_handle_t *ah, ompd_word_t *task_p)
+{
+	CHECK(ah);
+	ompd_word_t task_priority = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_max_task_priority_var", task_priority,
+		task_priority, target_sizes.sizeof_int, 1, ret);
+	*task_p = task_priority;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_stacksize(ompd_address_space_handle_t *ah, ompd_word_t *stacksize)
+{
+	CHECK(ah);
+	ompd_word_t stack_var = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "stacksize", stack_var, stack_var,
+		target_sizes.sizeof_long, 1, ret);
+	*stacksize = stack_var;
+	return ret;
+}
+
+
+ompd_rc_t
+ompd_get_debug(ompd_address_space_handle_t *ah, ompd_word_t *debug_var)
+{
+	CHECK(ah);
+	ompd_word_t debug = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_debug_var", debug, debug,
+		target_sizes.sizeof_int, 1, ret);
+	*debug_var = debug;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_display_affinity(ompd_address_space_handle_t *ah, ompd_word_t *aff)
+{
+	CHECK(ah);
+	ompd_word_t affin = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_display_affinity_var", affin, affin,
+		target_sizes.sizeof_char, 1, ret);
+	*aff = affin;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_affinity_format_len(ompd_address_space_handle_t *ah, ompd_word_t *len)
+{
+	CHECK(ah);
+	ompd_word_t len_var = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_affinity_format_len", len_var, len_var,
+		target_sizes.sizeof_int, 1, ret);
+	*len = len_var;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_affinity_format(ompd_address_space_handle_t *ah, const char **string)
+{
+	CHECK(ah);
+	ompd_word_t len = 100;
+	ompd_rc_t ret;
+	char *temp_str;
+	ompd_word_t addr;
+	ret = callbacks->alloc_memory(len + 1, (void **)&temp_str);
+	temp_str[len] = '\0';
+	ompd_address_t symbol_addr = {OMPD_SEGMENT_UNSPECIFIED, 0};
+	ret = callbacks->symbol_addr_lookup(ah->context, NULL,
+		"gomp_affinity_format_var", &symbol_addr, NULL);
+	ret = callbacks->read_memory(ah->context, NULL, &symbol_addr,
+		target_sizes.sizeof_pointer, &addr);
+	symbol_addr.address = addr;
+	ret = callbacks->read_string(ah->context, NULL, &symbol_addr, len,
+		(void *)temp_str);
+	ret = callbacks->device_to_host(ah->context, &temp_str,
+		target_sizes.sizeof_char, len, &temp_str);
+	*string = temp_str;
+	return ret;
+}
+
+
+ompd_rc_t
+ompd_get_wait_policy(ompd_address_space_handle_t *ah, ompd_word_t *wait_policy)
+{
+	CHECK(ah);
+	ompd_word_t wait_p = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "wait_policy", wait_p, wait_p,
+		target_sizes.sizeof_int, 1, ret);
+	*wait_policy = wait_p;
+	return ret;
+}
+
+
+ompd_rc_t
+ompd_get_num_teams(ompd_address_space_handle_t *ah, ompd_word_t *num_teams)
+{
+	CHECK(ah);
+	ompd_word_t num_t = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_num_teams_var", num_t, num_t,
+		target_sizes.sizeof_int, 1, ret);
+	*num_teams = num_t;
+	return ret;
+}
+
+
+ompd_rc_t
+ompd_get_teams_thread_limit(ompd_address_space_handle_t *ah,
+	ompd_word_t *thread_limit)
+{
+	CHECK(ah);
+	ompd_word_t thr_lim = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_teams_thread_limit_var", thr_lim, thr_lim,
+		target_sizes.sizeof_int, 1, ret);
+	*thread_limit = thr_lim;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_spin_count(ompd_address_space_handle_t *ah, ompd_word_t *spin_count)
+{
+	CHECK(ah);
+	ompd_word_t spins = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_spin_count_var", spins, spins,
+		target_sizes.sizeof_long_long, 1, ret);
+	*spin_count = spins;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_available_cpus(ompd_address_space_handle_t *ah, ompd_word_t *procs)
+{
+	CHECK(ah);
+	ompd_word_t cpus = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_available_cpus", cpus, cpus,
+		target_sizes.sizeof_long, 1, ret);
+	*procs = cpus;
+	return ret;
+}
+
+
+ompd_rc_t
+ompd_get_throttled_spin_count(ompd_address_space_handle_t *ah,
+	ompd_word_t *throt)
+{
+	CHECK(ah);
+	ompd_word_t temp = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_throttled_spin_count_var", temp, temp,
+		target_sizes.sizeof_long_long, 1, ret);
+	*throt = temp;
+	return ret;
+}
+
+ompd_rc_t
+ompd_get_managed_threads(ompd_address_space_handle_t *ah, ompd_word_t *man_th)
+{
+	CHECK(ah);
+	ompd_word_t temp = 0;
+	ompd_rc_t ret;
+	GET_VALUE(ah->context, NULL, "gomp_managed_threads", temp, temp,
+		target_sizes.sizeof_long, 1, ret);
+	*man_th = temp;
+	return ret;
+}
+
+
+
+/* End of global ICVs functions.  */
+
+
+
+
 ompd_rc_t
 get_sizes(ompd_address_space_context_t *context)
 {
diff --git a/libgomp/ompd-helper.h b/libgomp/ompd-helper.h
index 427833e703a..14ab63d8c6b 100644
--- a/libgomp/ompd-helper.h
+++ b/libgomp/ompd-helper.h
@@ -20,7 +20,8 @@
 
 
 
-/*This file contains the helper functions for OMPD.  */
+/* This file contains data types and declarations of functions that are not
+	provided by the book but we find them necessary.  */
 
 #ifndef _OMPD_HELPER_H
 #define _OMPD_HELPER_H
@@ -34,6 +35,7 @@ extern "C" {
 #include "ompd-types.h"
 #include <stdio.h>
 #include <stddef.h>
+#include <string.h>
 
 #define stringize(x)  stringize1(x)
 #define stringize1(x) #x
@@ -44,19 +46,139 @@ extern const ompd_callbacks_t *callbacks;
 extern __UINT64_TYPE__ ompd_state;
 extern ompd_device_type_sizes_t target_sizes;
 
+typedef struct _ompd_aspace_handle
+{
+	ompd_address_space_context_t *context;
+	ompd_device_t kind;
+}ompd_address_space_handle_t;
 
-ompd_rc_t get_sizes(ompd_address_space_context_t *);
-
+typedef struct _ompd_thread_handle
+{
+	ompd_address_space_handle_t *ah;
+	ompd_thread_context_t *thread_context;
+	ompd_address_t *th;
+}ompd_thread_handle_t;
 
+typedef struct _ompd_parallel_handle
+{
+	ompd_address_space_handle_t *ah;
+	ompd_address_t *th;
+}ompd_parallel_handle_t;
 
-struct _ompd_aspace_handle
+typedef struct _ompd_task_handle
 {
-	ompd_address_space_context_t *context;
-	ompd_device_t kind;
+	ompd_address_space_handle_t *ah;
+	ompd_address_t *th;
+}ompd_task_handle_t;
+
+#define GET_VALUE(context, thread_context, name, output, in_buf, size, count,	\
+	 ret) 																																			\
+	do {																											 		  						\
+		ompd_address_t symbol_addr = {OMPD_SEGMENT_UNSPECIFIED, 0};								\
+		ret = callbacks->symbol_addr_lookup(context, thread_context,	  					\
+			name, &symbol_addr, NULL);																							\
+		ret = callbacks->read_memory(context, thread_context, &symbol_addr,	size,	\
+			&in_buf);																																\
+		ret = callbacks->device_to_host(context, &in_buf, size, count, &output);	\
+	} while(0)																																	\
+
+
+#define CHECK(ah)													\
+	do{																			\
+		if(ah == NULL || ah->context == NULL)	\
+			return ompd_rc_stale_handle;				\
+		if(callbacks == NULL)									\
+			return ompd_rc_callback_error;			\
+	}while(0)
+
+
+
+/* (var_name, string_name, scope).  */
+#define FOREACH_OMPD_ICV(ompd_icv) \
+	ompd_icv(nthread_var, "nthread var", ompd_scope_thread) \
+	ompd_icv(thread_limit_var, "thread limit var", ompd_scope_task) \
+	ompd_icv(run_sched_var, "run sched limit var", ompd_scope_task) \
+	ompd_icv(run_sched_chunk_size, "run sched chunk size var", ompd_scope_task) \
+	ompd_icv(default_device_var, "default device var", ompd_scope_thread) \
+	ompd_icv(dyn_var, "dynamic var", ompd_scope_thread) \
+	ompd_icv(max_active_levels_var, "max active level var", ompd_scope_task) \
+	ompd_icv(bind_var, "proc bind var", ompd_scope_task) \
+	ompd_icv(cancellation_var, "cancel var", ompd_scope_address_space) \
+	ompd_icv(max_task_priority_var, "max task priority var", \
+		ompd_scope_address_space) \
+	ompd_icv(stacksize_var, "stack size var", ompd_scope_address_space) \
+	ompd_icv(debug_var, "debug var", ompd_scope_address_space) \
+	ompd_icv(display_affinity_var, "display affinity var", \
+		ompd_scope_address_space) \
+	ompd_icv(affinity_format_var, "affinity format var", \
+		ompd_scope_address_space) \
+	ompd_icv(affinity_format_len_var, "affinity format len var", \
+		ompd_scope_address_space) \
+	ompd_icv(wait_policy_var, "wait policy var", ompd_scope_address_space) \
+	ompd_icv(num_teams_var, "num teams var", ompd_scope_address_space) \
+	ompd_icv(teams_thread_limit_var, "teams thread limit var", \
+		ompd_scope_address_space) \
+	ompd_icv(spin_count_var, "spin count var", ompd_scope_address_space) \
+	ompd_icv(num_proc_var, "num proc var", ompd_scope_address_space) \
+	ompd_icv(throttled_spin_count_var, "throttled spin count var", \
+		ompd_scope_address_space)	\
+	ompd_icv(managed_threads_var, "managed threads var", \
+		ompd_scope_address_space) \
+	ompd_icv(thread_num_var, "thread num var", ompd_scope_thread) \
+	ompd_icv(final_task_var, "final task var", ompd_scope_task) \
+	ompd_icv(implicit_task_var, "implicit task var", ompd_scope_task) \
+	ompd_icv(team_size_var, "team size var", ompd_scope_parallel)
+
+enum ompd_icv
+{
+	ompd_icv_undefined_var = 0,
+	#define ompd_icv_iterator(var_name, string_name, scope) ompd_icv_##var_name,
+		FOREACH_OMPD_ICV(ompd_icv_iterator)
+	#undef ompd_icv_iterator
+	ompd_last_icv_var
 };
 
 
+ompd_rc_t get_sizes(ompd_address_space_context_t *);
+
 
+/* Get per task internal control variables.  */
+ompd_rc_t ompd_get_nthread(ompd_thread_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_thread_limit(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_run_sched(ompd_task_handle_t *,  ompd_word_t *);
+ompd_rc_t ompd_get_run_sched_chunk_size(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_default_device(ompd_thread_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_dynamic(ompd_thread_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_max_active_levels(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_proc_bind(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_is_final(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_is_implicit(ompd_task_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_team_size(ompd_parallel_handle_t *, ompd_word_t *);
+
+
+/* Get Global ICVs.  */
+ompd_rc_t ompd_get_cancellation(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_max_task_priority(ompd_address_space_handle_t *,
+	ompd_word_t *);
+ompd_rc_t ompd_get_stacksize(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_debug(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_display_affinity(ompd_address_space_handle_t *,
+	ompd_word_t *);
+ompd_rc_t ompd_get_affinity_format(ompd_address_space_handle_t *,
+	const char **);
+ompd_rc_t ompd_get_affinity_format_len(ompd_address_space_handle_t *,
+	ompd_word_t *);
+ompd_rc_t ompd_get_wait_policy(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_num_teams(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_teams_thread_limit(ompd_address_space_handle_t *,
+	ompd_word_t *);
+ompd_rc_t ompd_get_spin_count(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_available_cpus(ompd_address_space_handle_t *, ompd_word_t *);
+ompd_rc_t ompd_get_throttled_spin_count(ompd_address_space_handle_t *,
+	ompd_word_t *);
+ompd_rc_t ompd_get_managed_threads(ompd_address_space_handle_t *,
+	ompd_word_t *);
+/*End of Global ICVs.  */
 
 #ifdef __cplusplus
 } // extern C
diff --git a/libgomp/ompd-icv.c b/libgomp/ompd-icv.c
new file mode 100644
index 00000000000..8dd9bca8837
--- /dev/null
+++ b/libgomp/ompd-icv.c
@@ -0,0 +1,200 @@
+/* Copyright (C) 2021 Free Software Foundation, Inc.
+   Contributed by Mohamed Atef <mohamedatef1698@gmail.com>.
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+
+/* This file contains implementation of functions defined in 5.5.8 and 5.5.9
+  in OpenMP Application Programming Interface v5.1. */
+
+#include "ompd-helper.h"
+
+
+static const char *ompd_icv_string[] = { "undefined",
+	#define ompd_icv_iterator(var_name, string_name, scope) string_name,
+		FOREACH_OMPD_ICV(ompd_icv_iterator)
+	#undef ompd_icv_iterator
+};
+
+static const ompd_scope_t ompd_icv_scope[] = {ompd_scope_global,
+	#define ompd_icv_iterator(var_name, string_name, scope) scope,
+		FOREACH_OMPD_ICV(ompd_icv_iterator)
+	#undef ompd_icv_iterator
+};
+
+
+ompd_rc_t
+ompd_enumerate_icvs(ompd_address_space_handle_t *ah,
+	ompd_icv_id_t current, ompd_icv_id_t *next_id, const char **next_icv_name,
+	ompd_scope_t *next_scope, int *more)
+{
+	if(ah == NULL)
+		return ompd_rc_stale_handle;
+	if(current + 1 >= ompd_last_icv_var || next_id == NULL
+		|| next_icv_name == NULL || next_scope == NULL || more == NULL)
+			return ompd_rc_bad_input;
+	if(callbacks == NULL)
+		return ompd_rc_callback_error;
+	*next_id = current + 1;
+	char *temp_name = NULL;
+	ompd_rc_t ret = callbacks->alloc_memory(strlen(ompd_icv_string[*next_id]) + 1,
+		(void **)&temp_name);
+	if(ret)
+		return ret;
+	strcpy(temp_name, ompd_icv_string[*next_id]);
+	*next_icv_name = temp_name;
+	*next_scope = ompd_icv_scope[*next_id];
+	if((*next_id) + 1 < ompd_last_icv_var)
+		*more = 1;
+	else
+		*more = 0;
+	return ompd_rc_ok;
+}
+
+
+ompd_rc_t
+ompd_get_icv_from_scope(void *handle, ompd_scope_t scope, ompd_icv_id_t icv_id,
+	ompd_word_t *icv_value)
+{
+	if(handle == NULL)
+		return ompd_rc_stale_handle;
+	if(icv_value == NULL || !icv_id || icv_id >= ompd_last_icv_var)
+		return ompd_rc_bad_input;
+	if(callbacks == NULL)
+		return ompd_rc_callback_error;
+	ompd_device_t device;
+	switch(scope)
+		{
+			case ompd_scope_address_space:
+				device = ((ompd_address_space_handle_t *)handle)->kind;
+				break;
+			case ompd_scope_thread:
+				device = ((ompd_thread_handle_t *)handle)->ah->kind;
+				break;
+			case ompd_scope_parallel:
+				device = ((ompd_parallel_handle_t *)handle)->ah->kind;
+				break;
+			case ompd_scope_task:
+				device = ((ompd_task_handle_t *)handle)->ah->kind;
+				break;
+			default:
+				return ompd_rc_bad_input;
+		}
+	/* NO cuda for now.  */
+	if(device == OMPD_DEVICE_KIND_HOST)
+		{
+			switch(icv_id)
+				{
+					case ompd_icv_cancellation_var:
+						return ompd_get_cancellation((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_max_task_priority_var:
+						return ompd_get_max_task_priority(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_stacksize_var:
+						return ompd_get_stacksize((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_debug_var:
+						return ompd_get_debug((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_display_affinity_var:
+						return ompd_get_display_affinity(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_affinity_format_var:
+						return ompd_rc_incompatible;
+					case ompd_icv_affinity_format_len_var:
+						return ompd_get_affinity_format_len(
+								(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_wait_policy_var:
+						return ompd_get_wait_policy((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_num_teams_var:
+						return ompd_get_num_teams((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_teams_thread_limit_var:
+						return ompd_get_teams_thread_limit(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_spin_count_var:
+						return ompd_get_spin_count((ompd_address_space_handle_t *)handle,
+							icv_value);
+					case ompd_icv_num_proc_var:
+						return ompd_get_available_cpus(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_throttled_spin_count_var:
+						return ompd_get_throttled_spin_count(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					case ompd_icv_managed_threads_var:
+						return ompd_get_managed_threads(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					default:
+						return ompd_rc_unsupported;
+				}
+		}
+		return ompd_rc_error;
+}
+
+ompd_rc_t
+ompd_get_icv_string_from_scope(void *handle, ompd_scope_t scope,
+	ompd_icv_id_t icv_id, const char **icv_value)
+{
+	if(handle == NULL)
+		return ompd_rc_stale_handle;
+	if(icv_value == NULL || !icv_id || icv_id >= ompd_last_icv_var)
+		return ompd_rc_bad_input;
+	if(callbacks == NULL)
+		return ompd_rc_callback_error;
+	ompd_device_t device;
+	switch(scope)
+		{
+			case ompd_scope_address_space:
+				device = ((ompd_address_space_handle_t *)handle)->kind;
+				break;
+			case ompd_scope_thread:
+				device = ((ompd_thread_handle_t *)handle)->ah->kind;
+				break;
+			case ompd_scope_parallel:
+				device = ((ompd_parallel_handle_t *)handle)->ah->kind;
+				break;
+			case ompd_scope_task:
+				device = ((ompd_task_handle_t *)handle)->ah->kind;
+				break;
+			default:
+				return ompd_rc_bad_input;
+		}
+	/* No cuda.  */
+	if(device == OMPD_DEVICE_KIND_HOST)
+		{
+			switch(icv_id)
+				{
+					case ompd_icv_affinity_format_var:
+						return ompd_get_affinity_format(
+							(ompd_address_space_handle_t *)handle, icv_value
+						);
+					default:
+						return ompd_rc_unsupported;
+				}
+		}
+	return ompd_rc_error;
+}
diff --git a/libgomp/ompd-init.c b/libgomp/ompd-init.c
index 80e8e5ce62a..e65f0c89e86 100644
--- a/libgomp/ompd-init.c
+++ b/libgomp/ompd-init.c
@@ -20,7 +20,7 @@
 
 
 
-/*This file contains the implementation of functions defined in
+/* This file contains the implementation of functions defined in
 	section 5.5.1, 5.5.2.  */
 
 
@@ -69,7 +69,7 @@ ompd_get_version_string(const char **string)
 {
 	if(string == NULL)
 		return ompd_rc_bad_input;
-	static const char tmp_string[] = 
+	static const char tmp_string[] =
 		"GNU OpenMP runtime implementing OMPD version "
 			stringize(VERSION) " Debugging library.";
 	*string = tmp_string;
@@ -99,22 +99,12 @@ ompd_process_initialize(ompd_address_space_context_t *context,
 	if(ret != ompd_rc_ok)
 		return ret;
 
-	ompd_address_t symbol_addr = {OMPD_SEGMENT_UNSPECIFIED, 0};
-
-
 	//naive way to read from memory
-	ret = callbacks->symbol_addr_lookup(context, NULL, "ompd_state",
-			&symbol_addr, NULL);
-
-	ret = callbacks->read_memory(context, NULL, &symbol_addr,
-			target_sizes.sizeof_long_long, &ompd_state);
-
-	ret = callbacks->device_to_host(context, &ompd_state,
-			target_sizes.sizeof_long_long, 1, &ompd_state);
-
+	GET_VALUE(context, NULL, "ompd_state", ompd_state, ompd_state,
+		target_sizes.sizeof_long_long, 1, ret);
+	
 	ret = callbacks->alloc_memory(sizeof(ompd_address_space_handle_t),
 			(void **)(handle));
-	
 
 	if(ret != ompd_rc_ok)
 		return ret;
diff --git a/libgomp/parallel.c b/libgomp/parallel.c
index f1c8a36868c..1701eda3938 100644
--- a/libgomp/parallel.c
+++ b/libgomp/parallel.c
@@ -173,9 +173,9 @@ void
 GOMP_parallel (void (*fn) (void *), void *data, unsigned num_threads,
 	       unsigned int flags)
 {
-  num_threads = gomp_resolve_num_threads (num_threads, 0);
   if(ompd_state)
     ompd_bp_parallel_begin();
+  num_threads = gomp_resolve_num_threads (num_threads, 0);
   gomp_team_start (fn, data, num_threads, flags, gomp_new_team (num_threads),
 		   NULL);
   fn (data);