Re: [PATCH 0/6, OpenACC, libgomp] Async re-work

[Chung-Lin Tang <chunglin_tang@mentor.com>]

[-- Attachment #1: Type: text/plain, Size: 2133 bytes --]

On 2018/12/13 11:51 PM, Thomas Schwinge wrote:
> On Thu, 13 Dec 2018 23:28:49 +0800, Chung-Lin Tang<chunglin_tang@mentor.com>  wrote:
>> On 2018/12/7 6:26 AM, Julian Brown wrote:
>>> On Thu, 6 Dec 2018 22:22:46 +0000
>>> Julian Brown<julian@codesourcery.com>  wrote:
>>>
>>>> On Thu, 6 Dec 2018 21:42:14 +0100
>>>> Thomas Schwinge<thomas@codesourcery.com>  wrote:
>>>>
>>>>> [...]
>>>>> ..., where the "Invalid read of size 8" happens, and which
>>>>> eventually would try to "free (tgt)" again, via
>>>>> libgomp/target.c:gomp_unmap_tgt:
>>>>>
>>>>>       attribute_hidden void
>>>>>       gomp_unmap_tgt (struct target_mem_desc *tgt)
>>>>>       {
>>>>>         /* Deallocate on target the tgt->tgt_start .. tgt->tgt_end
>>>>> region.  */ if (tgt->tgt_end)
>>>>>           gomp_free_device_memory (tgt->device_descr, tgt->to_free);
>>>>>       
>>>>>         free (tgt->array);
>>>>>         free (tgt);
>>>>>       }
>>>>>
>>>>> Is the "free (tgt)" in libgomp/target.c:gomp_unmap_vars_async wrong,
>>>>> or something else?

I think I understand the problem now. In gomp_unmap_vars_async(), in the case of
tgt->list_count == 0 (i.e. no map arguments at all) the code should simply free the tgt
and return, while the code in goacc_async_copyout_unmap_vars() didn't handle this case
and always scheduled an asynchronous free of the tgt later, causing that valgrind error
you see.

I am still testing the attached patch, but I think it is the right fix: I reviewed what I
wrote and it seemed the way I organized things into a goacc_async_copyout_unmap_vars() routine,
including the hackish refcount++, etc. is simply unneeded. I have deleted those stuff
and consolidated things back into gomp_unmap_vars_async().

I'll update the whole patches later after complete testing, the attached is the patch atop
of the prior async patches. (the small program you gave above does pass valgrind now)

Julian, I didn't try the OG8 refcount changes, it's just too large a set of changes to
reason about in so short time, maybe later when we are prepared to fix things completely as
you noted what those patches were capable of.

Chung-Lin







[-- Attachment #2: async-unmap-fix.diff --]
[-- Type: text/plain, Size: 3482 bytes --]

diff -ru trunk-orig/libgomp/oacc-async.c trunk-work/libgomp/oacc-async.c
--- trunk-orig/libgomp/oacc-async.c	2018-12-14 21:06:06.649794724 +0800
+++ trunk-work/libgomp/oacc-async.c	2018-12-14 22:11:29.252251925 +0800
@@ -238,31 +238,6 @@
   thr->default_async = async;
 }
 
-static void
-goacc_async_unmap_tgt (void *ptr)
-{
-  struct target_mem_desc *tgt = (struct target_mem_desc *) ptr;
-
-  if (tgt->refcount > 1)
-    tgt->refcount--;
-  else
-    gomp_unmap_tgt (tgt);
-}
-
-attribute_hidden void
-goacc_async_copyout_unmap_vars (struct target_mem_desc *tgt,
-				struct goacc_asyncqueue *aq)
-{
-  struct gomp_device_descr *devicep = tgt->device_descr;
-
-  /* Increment reference to delay freeing of device memory until callback
-     has triggered.  */
-  tgt->refcount++;
-  gomp_unmap_vars_async (tgt, true, aq);
-  devicep->openacc.async.queue_callback_func (aq, goacc_async_unmap_tgt,
-					      (void *) tgt);
-}
-
 attribute_hidden void
 goacc_async_free (struct gomp_device_descr *devicep,
 		  struct goacc_asyncqueue *aq, void *ptr)
diff -ru trunk-orig/libgomp/oacc-int.h trunk-work/libgomp/oacc-int.h
--- trunk-orig/libgomp/oacc-int.h	2018-12-14 21:06:06.649794724 +0800
+++ trunk-work/libgomp/oacc-int.h	2018-12-14 22:11:43.379947915 +0800
@@ -104,8 +104,6 @@
 
 void goacc_init_asyncqueues (struct gomp_device_descr *);
 bool goacc_fini_asyncqueues (struct gomp_device_descr *);
-void goacc_async_copyout_unmap_vars (struct target_mem_desc *,
-				     struct goacc_asyncqueue *);
 void goacc_async_free (struct gomp_device_descr *, struct goacc_asyncqueue *,
 		       void *);
 struct goacc_asyncqueue *get_goacc_asyncqueue (int);
diff -ru trunk-orig/libgomp/oacc-mem.c trunk-work/libgomp/oacc-mem.c
--- trunk-orig/libgomp/oacc-mem.c	2018-12-14 21:06:06.649794724 +0800
+++ trunk-work/libgomp/oacc-mem.c	2018-12-14 22:10:08.325998369 +0800
@@ -911,7 +911,7 @@
       else
 	{
 	  goacc_aq aq = get_goacc_asyncqueue (async);
-	  goacc_async_copyout_unmap_vars (t, aq);
+	  gomp_unmap_vars_async (t, true, aq);
 	}
     }
 
diff -ru trunk-orig/libgomp/oacc-parallel.c trunk-work/libgomp/oacc-parallel.c
--- trunk-orig/libgomp/oacc-parallel.c	2018-12-14 21:06:06.649794724 +0800
+++ trunk-work/libgomp/oacc-parallel.c	2018-12-14 22:09:51.918353575 +0800
@@ -245,7 +245,7 @@
     {
       acc_dev->openacc.async.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
 					dims, tgt, aq);
-      goacc_async_copyout_unmap_vars (tgt, aq);
+      gomp_unmap_vars_async (tgt, true, aq);
     }
 }
 
diff -ru trunk-orig/libgomp/target.c trunk-work/libgomp/target.c
--- trunk-orig/libgomp/target.c	2018-12-14 21:06:06.653794622 +0800
+++ trunk-work/libgomp/target.c	2018-12-14 20:42:03.629154346 +0800
@@ -1072,6 +1072,17 @@
   return is_tgt_unmapped;
 }
 
+static void
+gomp_unref_tgt (void *ptr)
+{
+  struct target_mem_desc *tgt = (struct target_mem_desc *) ptr;
+
+  if (tgt->refcount > 1)
+    tgt->refcount--;
+  else
+    gomp_unmap_tgt (tgt);
+}
+
 /* Unmap variables described by TGT.  If DO_COPYFROM is true, copy relevant
    variables back from device to host: if it is false, it is assumed that this
    has been done already.  */
@@ -1130,10 +1141,11 @@
 	gomp_remove_var (devicep, k);
     }
 
-  if (tgt->refcount > 1)
-    tgt->refcount--;
+  if (aq)
+    devicep->openacc.async.queue_callback_func (aq, gomp_unref_tgt,
+						(void *) tgt);
   else
-    gomp_unmap_tgt (tgt);
+    gomp_unref_tgt ((void *) tgt);
 
   gomp_mutex_unlock (&devicep->lock);
 }