[PATCH 0/2] Offloading from dlopened libraries

[PATCH 0/2] Offloading from dlopened libraries

[Ilya Verbin]

Hi,

This patch fixes offloading from dlopened libraries, part 1 is for libgomp and
part 2 is for intelmic plugin (it's not finished yet).

How it works:
When a library is loaded it calls GOMP_offload_register as usually.
At this time some devices may already be initialized, and some may be not.
Therefore libgomp goes through all devices and for the initialized devices calls
GOMP_OFFLOAD_load_image, then receives corresponding addresses and inserts them
into splay tree.  Also it fills offload_images array for lazy Initialization.

When the library is unloaded it calls GOMP_offload_unregister.
This function also need to go through all devices and to call
GOMP_OFFLOAD_unload_image for all initialized devices.  Also it removes mapped
addresses from corresponding splay trees and pending images from the array.

Any thoughts on that?

Thomas, Julian,
Will this approach work for OpenACC+PTX?  I hope that it is general enough.
Yeah, I understand that this change will require some efforts on your part to
rebase the patches, but it would be good to define a common libgomp<->plugin
interface as early as possible.

Thanks,
  -- Ilya

[PATCH 1/2] Offloading from dlopened libraries: libgomp

[Ilya Verbin]

libgomp/
	* libgomp.map (GOMP_4.0.1): Add GOMP_offload_unregister.
	* libgomp_target.h (struct mapping_table): Replace with addr_pair.
	* target.c (struct gomp_device_descr): Remove register_image_func,
	get_table_func.  Add load_image_func, unload_image_func.
	(gomp_splay_tree_insert_mapping): New static function.
	(gomp_offload_image_to_device): Ditto.
	(GOMP_offload_register): Call gomp_offload_image_to_device for all
	initialized devices.
	(GOMP_offload_unregister): New function.
	(gomp_init_device): Replace a call to get_table_func with a call to
	gomp_offload_image_to_device.
	(gomp_load_plugin_for_device): Replace register_image and get_table
	with load_image and unload_image in DLSYM ().
	(gomp_register_images_for_device): Remove function.
	(gomp_target_init): Remove call to gomp_register_images_for_device.


diff --git a/libgomp/libgomp.map b/libgomp/libgomp.map
index f36df23..4feba32 100644
--- a/libgomp/libgomp.map
+++ b/libgomp/libgomp.map
@@ -231,4 +231,5 @@ GOMP_4.0 {
 GOMP_4.0.1 {
   global:
 	GOMP_offload_register;
+	GOMP_offload_unregister;
 } GOMP_4.0;
diff --git a/libgomp/libgomp_target.h b/libgomp/libgomp_target.h
index f7d19d0..348f29e 100644
--- a/libgomp/libgomp_target.h
+++ b/libgomp/libgomp_target.h
@@ -31,14 +31,12 @@ enum offload_target_type
   OFFLOAD_TARGET_TYPE_INTEL_MIC
 };
 
-/* Auxiliary struct, used for transferring a host-target address range mapping
-   from plugin to libgomp.  */
-struct mapping_table
+/* Auxiliary struct, used for transferring pairs of addresses from plugin
+   to libgomp.  */
+struct addr_pair
 {
-  uintptr_t host_start;
-  uintptr_t host_end;
-  uintptr_t tgt_start;
-  uintptr_t tgt_end;
+  uintptr_t start;
+  uintptr_t end;
 };
 
 #endif /* LIBGOMP_TARGET_H */
diff --git a/libgomp/target.c b/libgomp/target.c
index 5b4873b..855ea5b 100644
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -143,9 +143,9 @@ struct gomp_device_descr
   /* Function handlers.  */
   int (*get_type_func) (void);
   int (*get_num_devices_func) (void);
-  void (*register_image_func) (void *, void *);
   void (*init_device_func) (int);
-  int (*get_table_func) (int, void *);
+  int (*load_image_func) (int, void *, struct addr_pair **);
+  void (*unload_image_func) (int, void *);
   void *(*alloc_func) (int, size_t);
   void (*free_func) (int, void *);
   void *(*host2dev_func) (int, void *, const void *, size_t);
@@ -600,7 +600,82 @@ gomp_update (struct gomp_device_descr *devicep, size_t mapnum,
   gomp_mutex_unlock (&devicep->dev_env_lock);
 }
 
-/* This function should be called from every offload image.
+/* Insert mapping of host -> target address pairs to splay tree.  */
+
+static void
+gomp_splay_tree_insert_mapping (struct gomp_device_descr *devicep,
+				struct addr_pair *host_addr,
+				struct addr_pair *tgt_addr)
+{
+  struct target_mem_desc *tgt = gomp_malloc (sizeof (*tgt));
+  tgt->refcount = 1;
+  tgt->array = gomp_malloc (sizeof (*tgt->array));
+  tgt->tgt_start = tgt_addr->start;
+  tgt->tgt_end = tgt_addr->end;
+  tgt->to_free = NULL;
+  tgt->list_count = 0;
+  tgt->device_descr = devicep;
+  splay_tree_node node = tgt->array;
+  splay_tree_key k = &node->key;
+  k->host_start = host_addr->start;
+  k->host_end = host_addr->end;
+  k->tgt_offset = 0;
+  k->refcount = 1;
+  k->copy_from = false;
+  k->tgt = tgt;
+  node->left = NULL;
+  node->right = NULL;
+  splay_tree_insert (&devicep->dev_splay_tree, node);
+}
+
+/* Load image pointed by TARGET_DATA to the device, specified by DEVICEP.
+   And insert to splay tree the mapping between addresses from HOST_TABLE and
+   from loaded target image.  */
+
+static void
+gomp_offload_image_to_device (struct gomp_device_descr *devicep,
+			      void *host_table, void *target_data)
+{
+  void **host_func_table = ((void ***) host_table)[0];
+  void **host_funcs_end  = ((void ***) host_table)[1];
+  void **host_var_table  = ((void ***) host_table)[2];
+  void **host_vars_end   = ((void ***) host_table)[3];
+
+  /* The func table contains only addresses, the var table contains addresses
+     and corresponding sizes.  */
+  int num_funcs = host_funcs_end - host_func_table;
+  int num_vars  = (host_vars_end - host_var_table) / 2;
+
+  /* Load image to device and get target addresses for the image.  */
+  struct addr_pair *target_table = NULL;
+  int i, num_target_entries
+    = devicep->load_image_func (devicep->target_id, target_data, &target_table);
+
+  if (num_target_entries != num_funcs + num_vars)
+    gomp_fatal ("Can't map target functions or variables");
+
+  /* Insert host-target address mapping into devicep->dev_splay_tree.  */
+  for (i = 0; i < num_funcs; i++)
+    {
+      struct addr_pair host_addr;
+      host_addr.start = (uintptr_t) host_func_table[i];
+      host_addr.end = host_addr.start + 1;
+      gomp_splay_tree_insert_mapping (devicep, &host_addr, &target_table[i]);
+    }
+
+  for (i = 0; i < num_vars; i++)
+    {
+      struct addr_pair host_addr;
+      host_addr.start = (uintptr_t) host_var_table[i*2];
+      host_addr.end = host_addr.start + (uintptr_t) host_var_table[i*2+1];
+      gomp_splay_tree_insert_mapping (devicep, &host_addr,
+				      &target_table[num_funcs+i]);
+    }
+
+  free (target_table);
+}
+
+/* This function should be called from every offload image while loading.
    It gets the descriptor of the host func and var tables HOST_TABLE, TYPE of
    the target, and TARGET_DATA needed by target plugin.  */
 
@@ -608,6 +683,17 @@ void
 GOMP_offload_register (void *host_table, enum offload_target_type target_type,
 		       void *target_data)
 {
+  int i;
+
+  /* Load image to all initialized devices.  */
+  for (i = 0; i < num_devices; i++)
+    {
+      struct gomp_device_descr *devicep = &devices[i];
+      if (devicep->type == target_type && devicep->is_initialized)
+	gomp_offload_image_to_device (devicep, host_table, target_data);
+    }
+
+  /* Insert image to array of pending images.  */
   offload_images = gomp_realloc (offload_images,
 				 (num_offload_images + 1)
 				 * sizeof (struct offload_image_descr));
@@ -619,43 +705,81 @@ GOMP_offload_register (void *host_table, enum offload_target_type target_type,
   num_offload_images++;
 }
 
+/* This function should be called from every offload image while unloading.
+   It gets the descriptor of the host func and var tables HOST_TABLE, TYPE of
+   the target, and TARGET_DATA needed by target plugin.  */
+
+void
+GOMP_offload_unregister (void *host_table, enum offload_target_type target_type,
+			 void *target_data)
+{
+  void **host_func_table = ((void ***) host_table)[0];
+  void **host_funcs_end  = ((void ***) host_table)[1];
+  void **host_var_table  = ((void ***) host_table)[2];
+  void **host_vars_end   = ((void ***) host_table)[3];
+  int i;
+
+  /* The func table contains only addresses, the var table contains addresses
+     and corresponding sizes.  */
+  int num_funcs = host_funcs_end - host_func_table;
+  int num_vars  = (host_vars_end - host_var_table) / 2;
+
+  /* Unload image from all initialized devices.  */
+  for (i = 0; i < num_devices; i++)
+    {
+      int j;
+      struct gomp_device_descr *devicep = &devices[i];
+
+      if (devicep->type != target_type || !devicep->is_initialized)
+	continue;
+
+      devicep->unload_image_func (devicep->target_id, target_data);
+
+      /* Remove mapping from splay tree.  */
+      for (j = 0; j < num_funcs; j++)
+	{
+	  struct splay_tree_key_s k;
+	  k.host_start = (uintptr_t) host_func_table[j];
+	  k.host_end = k.host_start + 1;
+	  splay_tree_remove (&devicep->dev_splay_tree, &k);
+	}
+
+      for (j = 0; j < num_vars; j++)
+	{
+	  struct splay_tree_key_s k;
+	  k.host_start = (uintptr_t) host_var_table[j*2];
+	  k.host_end = k.host_start + (uintptr_t) host_var_table[j*2+1];
+	  splay_tree_remove (&devicep->dev_splay_tree, &k);
+	}
+    }
+
+  /* Remove image from array of pending images.  */
+  for (i = 0; i < num_offload_images; i++)
+    if (offload_images[i].target_data == target_data)
+      {
+	offload_images[i] = offload_images[--num_offload_images];
+	break;
+      }
+}
+
 /* This function initializes the target device, specified by DEVICEP.  */
 
 static void
 gomp_init_device (struct gomp_device_descr *devicep)
 {
-  devicep->init_device_func (devicep->target_id);
+  int i;
 
-  /* Get address mapping table for device.  */
-  struct mapping_table *table = NULL;
-  int num_entries = devicep->get_table_func (devicep->target_id, &table);
+  devicep->init_device_func (devicep->target_id);
 
-  /* Insert host-target address mapping into dev_splay_tree.  */
-  int i;
-  for (i = 0; i < num_entries; i++)
+  /* Load to device all images registered by the moment.  */
+  for (i = 0; i < num_offload_images; i++)
     {
-      struct target_mem_desc *tgt = gomp_malloc (sizeof (*tgt));
-      tgt->refcount = 1;
-      tgt->array = gomp_malloc (sizeof (*tgt->array));
-      tgt->tgt_start = table[i].tgt_start;
-      tgt->tgt_end = table[i].tgt_end;
-      tgt->to_free = NULL;
-      tgt->list_count = 0;
-      tgt->device_descr = devicep;
-      splay_tree_node node = tgt->array;
-      splay_tree_key k = &node->key;
-      k->host_start = table[i].host_start;
-      k->host_end = table[i].host_end;
-      k->tgt_offset = 0;
-      k->refcount = 1;
-      k->copy_from = false;
-      k->tgt = tgt;
-      node->left = NULL;
-      node->right = NULL;
-      splay_tree_insert (&devicep->dev_splay_tree, node);
+      struct offload_image_descr *image = &offload_images[i];
+      if (image->type == devicep->type)
+	gomp_offload_image_to_device (devicep, image->host_table,
+				      image->target_data);
     }
 
-  free (table);
   devicep->is_initialized = true;
 }
 
@@ -823,9 +947,9 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device,
   while (0)
   DLSYM (get_type);
   DLSYM (get_num_devices);
-  DLSYM (register_image);
   DLSYM (init_device);
-  DLSYM (get_table);
+  DLSYM (load_image);
+  DLSYM (unload_image);
   DLSYM (alloc);
   DLSYM (free);
   DLSYM (dev2host);
@@ -836,21 +960,6 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device,
   return true;
 }
 
-/* This function finds OFFLOAD_IMAGES corresponding to DEVICE type, and
-   registers them in the plugin.  */
-
-static void
-gomp_register_images_for_device (struct gomp_device_descr *device)
-{
-  int i;
-  for (i = 0; i < num_offload_images; i++)
-    {
-      struct offload_image_descr *image = &offload_images[i];
-      if (image->type == device->type)
-	device->register_image_func (image->host_table, image->target_data);
-    }
-}
-
 /* This function initializes the runtime needed for offloading.
    It parses the list of offload targets and tries to load the plugins for these
    targets.  Result of the function is properly initialized variable NUM_DEVICES
@@ -905,7 +1014,6 @@ gomp_target_init (void)
 		current_device.type = current_device.get_type_func ();
 		current_device.is_initialized = false;
 		current_device.dev_splay_tree.root = NULL;
-		gomp_register_images_for_device (&current_device);
 		for (i = 0; i < new_num_devices; i++)
 		  {
 		    current_device.id = num_devices + 1;
@@ -921,10 +1029,6 @@ gomp_target_init (void)
 	cur = next + 1;
       }
     while (next);
-
-  free (offload_images);
-  offload_images = NULL;
-  num_offload_images = 0;
 }
 
 #else /* PLUGIN_SUPPORT */

[PATCH 2/2] Offloading from dlopened libraries: intelmic plugin

[Ilya Verbin]

And here is the second part, which implements GOMP_OFFLOAD_[un]load_image in
the plugin.
Unfortunately so far unloading for libs is not implemented in liboffloadmic, but
I believe that this patch is better than the current code.

Tested with multiple dso's and on multiple emulated devices (as described here:
https://gcc.gnu.org/wiki/Offloading#How_to_try_offloading_enabled_GCC )

  -- Ilya


gcc/
	* config/i386/intelmic-mkoffload.c (generate_host_descr_file): Call
	GOMP_offload_unregister from the destructor.

liboffloadmic/
	* plugin/libgomp-plugin-intelmic.cpp: Include map.
	(AddrVect, DevAddrVect, ImgDevAddrMap): New typedefs.
	(num_devices, num_images, address_table): New static vars.
	(num_libraries, lib_descrs): Remove static vars.
	(set_mic_lib_path): Rename to ...
	(init): ... this.  Allocate address_table and get num_devices.
	(GOMP_OFFLOAD_get_num_devices): return num_devices.
	(load_lib_and_get_table): Remove static function.
	(offload_image): New static function.
	(GOMP_OFFLOAD_get_table): Remove function.
	(GOMP_OFFLOAD_load_image, GOMP_OFFLOAD_unload_image): New functions.


diff --git a/gcc/config/i386/intelmic-mkoffload.c b/gcc/config/i386/intelmic-mkoffload.c
index c972f56..b739b9b 100644
--- a/gcc/config/i386/intelmic-mkoffload.c
+++ b/gcc/config/i386/intelmic-mkoffload.c
@@ -337,13 +337,22 @@ generate_host_descr_file (const char *host_compiler)
 	   "#ifdef __cplusplus\n"
 	   "extern \"C\"\n"
 	   "#endif\n"
-	   "void GOMP_offload_register (void *, int, void *);\n\n"
+	   "void GOMP_offload_register (void *, int, void *);\n"
+	   "void GOMP_offload_unregister (void *, int, void *);\n\n"
 
 	   "__attribute__((constructor))\n"
 	   "static void\n"
 	   "init (void)\n"
 	   "{\n"
 	   "  GOMP_offload_register (&__OFFLOAD_TABLE__, %d, __offload_target_data);\n"
+	   "}\n\n", OFFLOAD_TARGET_TYPE_INTEL_MIC);
+
+  fprintf (src_file,
+	   "__attribute__((destructor))\n"
+	   "static void\n"
+	   "fini (void)\n"
+	   "{\n"
+	   "  GOMP_offload_unregister (&__OFFLOAD_TABLE__, %d, __offload_target_data);\n"
 	   "}\n", OFFLOAD_TARGET_TYPE_INTEL_MIC);
   fclose (src_file);
 
diff --git a/liboffloadmic/plugin/libgomp-plugin-intelmic.cpp b/liboffloadmic/plugin/libgomp-plugin-intelmic.cpp
index 28ddbc3..39cf74d 100644
--- a/liboffloadmic/plugin/libgomp-plugin-intelmic.cpp
+++ b/liboffloadmic/plugin/libgomp-plugin-intelmic.cpp
@@ -33,6 +33,7 @@
 #include <string.h>
 #include <utility>
 #include <vector>
+#include <map>
 #include <libgomp_target.h>
 #include "compiler_if_host.h"
 #include "main_target_image.h"
@@ -52,6 +53,29 @@ fprintf (stderr, "\n");					    \
 #endif
 
 
+/* Start/end addresses of functions and global variables on a device.  */
+typedef std::vector<addr_pair> AddrVect;
+
+/* Addresses for one image and all devices.  */
+typedef std::vector<AddrVect> DevAddrVect;
+
+/* Addresses for all images and all devices.  */
+typedef std::map<void *, DevAddrVect> ImgDevAddrMap;
+
+
+/* Total number of available devices.  */
+static int num_devices;
+
+/* Total number of shared libraries with offloading to Intel MIC.  */
+static int num_images;
+
+/* Two dimensional array: one key is a pointer to image,
+   second key is number of device.  Contains a vector of pointer pairs.  */
+static ImgDevAddrMap *address_table;
+
+/* Thread-safe registration of the main image.  */
+static pthread_once_t main_image_is_registered = PTHREAD_ONCE_INIT;
+
 static VarDesc vd_host2tgt = {
   { 1, 1 },		      /* dst, src			      */
   { 1, 0 },		      /* in, out			      */
@@ -89,28 +113,17 @@ static VarDesc vd_tgt2host = {
 };
 
 
-/* Total number of shared libraries with offloading to Intel MIC.  */
-static int num_libraries;
-
-/* Pointers to the descriptors, containing pointers to host-side tables and to
-   target images.  */
-static std::vector< std::pair<void *, void *> > lib_descrs;
-
-/* Thread-safe registration of the main image.  */
-static pthread_once_t main_image_is_registered = PTHREAD_ONCE_INIT;
-
-
 /* Add path specified in LD_LIBRARY_PATH to MIC_LD_LIBRARY_PATH, which is
    required by liboffloadmic.  */
 __attribute__((constructor))
 static void
-set_mic_lib_path (void)
+init (void)
 {
   const char *ld_lib_path = getenv (LD_LIBRARY_PATH_ENV);
   const char *mic_lib_path = getenv (MIC_LD_LIBRARY_PATH_ENV);
 
   if (!ld_lib_path)
-    return;
+    goto out;
 
   if (!mic_lib_path)
     setenv (MIC_LD_LIBRARY_PATH_ENV, ld_lib_path, 1);
@@ -132,6 +145,10 @@ set_mic_lib_path (void)
       if (!use_alloca)
 	free (mic_lib_path_new);
     }
+
+out:
+  address_table = new ImgDevAddrMap;
+  num_devices = _Offload_number_of_devices ();
 }
 
 extern "C" enum offload_target_type
@@ -145,18 +162,8 @@ GOMP_OFFLOAD_get_type (void)
 extern "C" int
 GOMP_OFFLOAD_get_num_devices (void)
 {
-  int res = _Offload_number_of_devices ();
-  TRACE ("(): return %d", res);
-  return res;
-}
-
-/* This should be called from every shared library with offloading.  */
-extern "C" void
-GOMP_OFFLOAD_register_image (void *host_table, void *target_image)
-{
-  TRACE ("(host_table = %p, target_image = %p)", host_table, target_image);
-  lib_descrs.push_back (std::make_pair (host_table, target_image));
-  num_libraries++;
+  TRACE ("(): return %d", num_devices);
+  return num_devices;
 }
 
 static void
@@ -179,7 +186,8 @@ register_main_image ()
   __offload_register_image (&main_target_image);
 }
 
-/* Load offload_target_main on target.  */
+/* liboffloadmic loads and runs offload_target_main on all available devices
+   during a first call to offload ().  */
 extern "C" void
 GOMP_OFFLOAD_init_device (int device)
 {
@@ -218,9 +226,11 @@ get_target_table (int device, int &num_funcs, int &num_vars, void **&table)
     }
 }
 
+/* Offload TARGET_IMAGE to all available devices and fill address_table with
+   corresponding target addresses.  */
+
 static void
-load_lib_and_get_table (int device, int lib_num, mapping_table *&table,
-			int &table_size)
+offload_image (void *target_image)
 {
   struct TargetImage {
     int64_t size;
@@ -229,19 +239,11 @@ load_lib_and_get_table (int device, int lib_num, mapping_table *&table,
     char data[];
   } __attribute__ ((packed));
 
-  void ***host_table_descr = (void ***) lib_descrs[lib_num].first;
-  void **host_func_start = host_table_descr[0];
-  void **host_func_end   = host_table_descr[1];
-  void **host_var_start  = host_table_descr[2];
-  void **host_var_end    = host_table_descr[3];
+  void *image_start = ((void **) target_image)[0];
+  void *image_end   = ((void **) target_image)[1];
 
-  void **target_image_descr = (void **) lib_descrs[lib_num].second;
-  void *image_start = target_image_descr[0];
-  void *image_end   = target_image_descr[1];
-
-  TRACE ("() host_table_descr { %p, %p, %p, %p }", host_func_start,
-	 host_func_end, host_var_start, host_var_end);
-  TRACE ("() target_image_descr { %p, %p }", image_start, image_end);
+  TRACE ("(target_image = %p { %p, %p })",
+	 target_image, image_start, image_end);
 
   int64_t image_size = (uintptr_t) image_end - (uintptr_t) image_start;
   TargetImage *image
@@ -254,94 +256,87 @@ load_lib_and_get_table (int device, int lib_num, mapping_table *&table,
     }
 
   image->size = image_size;
-  sprintf (image->name, "lib%010d.so", lib_num);
+  sprintf (image->name, "lib%010d.so", num_images++);
   memcpy (image->data, image_start, image->size);
 
   TRACE ("() __offload_register_image %s { %p, %d }",
 	 image->name, image_start, image->size);
   __offload_register_image (image);
 
-  int tgt_num_funcs = 0;
-  int tgt_num_vars = 0;
-  void **tgt_table = NULL;
-  get_target_table (device, tgt_num_funcs, tgt_num_vars, tgt_table);
-  free (image);
-
-  /* The func table contains only addresses, the var table contains addresses
-     and corresponding sizes.  */
-  int host_num_funcs = host_func_end - host_func_start;
-  int host_num_vars  = (host_var_end - host_var_start) / 2;
-  TRACE ("() host_num_funcs = %d, tgt_num_funcs = %d",
-	 host_num_funcs, tgt_num_funcs);
-  TRACE ("() host_num_vars = %d, tgt_num_vars = %d",
-	 host_num_vars, tgt_num_vars);
-  if (host_num_funcs != tgt_num_funcs)
+  /* Receive tables for target_image from all devices.  */
+  DevAddrVect dev_table;
+  for (int dev = 0; dev < num_devices; dev++)
     {
-      fprintf (stderr, "%s: Can't map target functions\n", __FILE__);
-      exit (1);
-    }
-  if (host_num_vars != tgt_num_vars)
-    {
-      fprintf (stderr, "%s: Can't map target variables\n", __FILE__);
-      exit (1);
-    }
+      int num_funcs = 0;
+      int num_vars = 0;
+      void **table = NULL;
 
-  table = (mapping_table *) realloc (table, (table_size + host_num_funcs
-					     + host_num_vars)
-					    * sizeof (mapping_table));
-  if (table == NULL)
-    {
-      fprintf (stderr, "%s: Can't allocate memory\n", __FILE__);
-      exit (1);
-    }
+      get_target_table (dev, num_funcs, num_vars, table);
 
-  for (int i = 0; i < host_num_funcs; i++)
-    {
-      mapping_table t;
-      t.host_start = (uintptr_t) host_func_start[i];
-      t.host_end = t.host_start + 1;
-      t.tgt_start = (uintptr_t) tgt_table[i];
-      t.tgt_end = t.tgt_start + 1;
-
-      TRACE ("() lib %d, func %d:\t0x%llx -- 0x%llx",
-	     lib_num, i, t.host_start, t.tgt_start);
-
-      table[table_size++] = t;
-    }
+      AddrVect curr_dev_table;
 
-  for (int i = 0; i < host_num_vars * 2; i += 2)
-    {
-      mapping_table t;
-      t.host_start = (uintptr_t) host_var_start[i];
-      t.host_end = t.host_start + (uintptr_t) host_var_start[i+1];
-      t.tgt_start = (uintptr_t) tgt_table[tgt_num_funcs+i];
-      t.tgt_end = t.tgt_start + (uintptr_t) tgt_table[tgt_num_funcs+i+1];
+      for (int i = 0; i < num_funcs; i++)
+	{
+	  addr_pair tgt_addr;
+	  tgt_addr.start = (uintptr_t) table[i];
+	  tgt_addr.end = tgt_addr.start + 1;
+	  TRACE ("() func %d:\t0x%llx..0x%llx", i,
+		 tgt_addr.start, tgt_addr.end);
+	  curr_dev_table.push_back (tgt_addr);
+	}
 
-      TRACE ("() lib %d, var %d:\t0x%llx (%d) -- 0x%llx (%d)", lib_num, i/2,
-	     t.host_start, t.host_end - t.host_start,
-	     t.tgt_start, t.tgt_end - t.tgt_start);
+      for (int i = 0; i < num_vars; i++)
+	{
+	  addr_pair tgt_addr;
+	  tgt_addr.start = (uintptr_t) table[num_funcs+i*2];
+	  tgt_addr.end = tgt_addr.start + (uintptr_t) table[num_funcs+i*2+1];
+	  TRACE ("() var %d:\t0x%llx..0x%llx", i, tgt_addr.start, tgt_addr.end);
+	  curr_dev_table.push_back (tgt_addr);
+	}
 
-      table[table_size++] = t;
+      dev_table.push_back (curr_dev_table);
     }
 
-  delete [] tgt_table;
+  address_table->insert (std::make_pair (target_image, dev_table));
+
+  free (image);
 }
 
 extern "C" int
-GOMP_OFFLOAD_get_table (int device, void *result)
+GOMP_OFFLOAD_load_image (int device, void *target_image, addr_pair **result)
 {
-  TRACE ("(num_libraries = %d)", num_libraries);
+  TRACE ("(device = %d, target_image = %p)", device, target_image);
 
-  mapping_table *table = NULL;
-  int table_size = 0;
+  /* If target_image is already present in address_table, then there is no need
+     to offload it.  */
+  if (address_table->count (target_image) == 0)
+    offload_image (target_image);
 
-  for (int i = 0; i < num_libraries; i++)
-    load_lib_and_get_table (device, i, table, table_size);
+  AddrVect *curr_dev_table = &(*address_table)[target_image][device];
+  int table_size = curr_dev_table->size ();
+  addr_pair *table = (addr_pair *) malloc (table_size * sizeof (addr_pair));
+  if (table == NULL)
+    {
+      fprintf (stderr, "%s: Can't allocate memory\n", __FILE__);
+      exit (1);
+    }
 
-  *(void **) result = table;
+  std::copy (curr_dev_table->begin (), curr_dev_table->end (), table);
+  *result = table;
   return table_size;
 }
 
+extern "C" void
+GOMP_OFFLOAD_unload_image (int device, void *target_image)
+{
+  TRACE ("(device = %d, target_image = %p)", device, target_image);
+
+  /* TODO: Currently liboffloadmic doesn't support __offload_unregister_image
+     for libraries.  */
+
+  address_table->erase (target_image);
+}
+
 extern "C" void *
 GOMP_OFFLOAD_alloc (int device, size_t size)
 {

Re: [PATCH 0/2] Offloading from dlopened libraries

[Ilya Verbin]

Hi!

How about this patch?  It adds a new symbol into GOMP_4.0.1 symver, so it would
be nice to include it into GCC 5 release.

On 14 Nov 02:53, Ilya Verbin wrote:
> This patch fixes offloading from dlopened libraries, part 1 is for libgomp and
> part 2 is for intelmic plugin.
> 
> How it works:
> When a library is loaded it calls GOMP_offload_register as usually.
> At this time some devices may already be initialized, and some may be not.
> Therefore libgomp goes through all devices and for the initialized devices calls
> GOMP_OFFLOAD_load_image, then receives corresponding addresses and inserts them
> into splay tree.  Also it fills offload_images array for lazy Initialization.
> 
> When the library is unloaded it calls GOMP_offload_unregister.
> This function also need to go through all devices and to call
> GOMP_OFFLOAD_unload_image for all initialized devices.  Also it removes mapped
> addresses from corresponding splay trees and pending images from the array.
> 
> Any thoughts on that?
> 
> Thomas, Julian,
> Will this approach work for OpenACC+PTX?  I hope that it is general enough.
> Yeah, I understand that this change will require some efforts on your part to
> rebase the patches, but it would be good to define a common libgomp<->plugin
> interface as early as possible.

  -- Ilya