plugin/plugin-nvptx.c: Fix fini_device call when already shutdown [PR113513] The following issue was found when running libgomp.c/target-52.c with nvptx offloading when the dg-set-target-env-var was honored. The issue occurred for both -foffload=disable and with offloading configured when an nvidia device is available. At the end of the program, the offloading parts are shutdown via two means: The callback registered via 'atexit (gomp_target_fini)' and - via code generated in mkoffload, the '__attribute__((destructor)) fini' function that calls GOMP_offload_unregister_ver. In normal processing, first gomp_target_fini is called - which then sets GOMP_DEVICE_FINALIZED for the device - and later GOMP_offload_unregister_ver, but that's then because the state is GOMP_DEVICE_FINALIZED. If both OMP_DISPLAY_ENV=true and OMP_TARGET_OFFLOAD="mandatory" are set, the call omp_display_env already invokes gomp_init_targets_once, i.e. it occurs earlier than usual and is invoked via __attribute__((constructor)) initialize_env. For some unknown reasons, while this does not have an effect on the order of the called plugin functions for initialization, it changes the order of function calls for shutting down. Namely, when the two environment variables are set, GOMP_offload_unregister_ver is called now before gomp_target_fini. - And it seems as if CUDA regards a call to cuModuleUnload (or unloading the last module?) as indication that the device context should be destroyed - or, at least, afterwards calling cuCtxGetDevice will return CUDA_ERROR_DEINITIALIZED. As the previous code in nvptx_attach_host_thread_to_device wasn't expecting that result, it called GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r)); causing a fatal error of the program. This commit handles now CUDA_ERROR_DEINITIALIZED in a special way such that GOMP_OFFLOAD_fini_device just works. When reading the code, the following was observed in addition: When gomp_fini_device is called, it invokes goacc_fini_asyncqueues to ensure that the queue is emptied. It seems to make sense to do likewise for GOMP_offload_unregister_ver, which this commit does in addition. libgomp/ChangeLog: PR libgomp/113513 * target.c (GOMP_offload_unregister_ver): Call goacc_fini_asyncqueues before invoking GOMP_offload_unregister_ver. * plugin/plugin-nvptx.c (nvptx_attach_host_thread_to_device): Change return type to int and new bool arg, it true, return -1 for CUDA_ERROR_DEINITIALIZED. (GOMP_OFFLOAD_fini_device): Handle the deinitialized gracefully. (nvptx_init, GOMP_OFFLOAD_load_image, GOMP_OFFLOAD_alloc, GOMP_OFFLOAD_host2dev, GOMP_OFFLOAD_dev2host, GOMP_OFFLOAD_memcpy2d, GOMP_OFFLOAD_memcpy3d, GOMP_OFFLOAD_openacc_async_host2dev, GOMP_OFFLOAD_openacc_async_dev2host): Update calls Signed-off-by: Tobias Burnus libgomp/plugin/plugin-nvptx.c | 46 ++++++++++++++++++++++++++----------------- libgomp/target.c | 7 +++++-- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/libgomp/plugin/plugin-nvptx.c b/libgomp/plugin/plugin-nvptx.c index c04c3acd679..318d3d2aca6 100644 --- a/libgomp/plugin/plugin-nvptx.c +++ b/libgomp/plugin/plugin-nvptx.c @@ -382,10 +382,13 @@ nvptx_init (void) } /* Select the N'th PTX device for the current host thread. The device must - have been previously opened before calling this function. */ + have been previously opened before calling this function. + Returns 1 if successful, 0 if an error occurred and a message has been + issued; if fini_okay, -1 is returned for CUDA_ERROR_DEINITIALIZED and + no error message is printed in that case. */ -static bool -nvptx_attach_host_thread_to_device (int n) +static int +nvptx_attach_host_thread_to_device (int n, bool fini_okay) { CUdevice dev; CUresult r; @@ -393,15 +396,17 @@ nvptx_attach_host_thread_to_device (int n) CUcontext thd_ctx; r = CUDA_CALL_NOCHECK (cuCtxGetDevice, &dev); + if (fini_okay && r == CUDA_ERROR_DEINITIALIZED) + return -1; if (r == CUDA_ERROR_NOT_PERMITTED) { /* Assume we're in a CUDA callback, just return true. */ - return true; + return 1; } if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT) { GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r)); - return false; + return 0; } if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n) @@ -414,7 +419,7 @@ nvptx_attach_host_thread_to_device (int n) if (!ptx_dev) { GOMP_PLUGIN_error ("device %d not found", n); - return false; + return 0; } CUDA_CALL (cuCtxGetCurrent, &thd_ctx); @@ -426,7 +431,7 @@ nvptx_attach_host_thread_to_device (int n) CUDA_CALL (cuCtxPushCurrent, ptx_dev->ctx); } - return true; + return 1; } static struct ptx_device * @@ -1252,8 +1257,11 @@ GOMP_OFFLOAD_fini_device (int n) if (ptx_devices[n] != NULL) { - if (!nvptx_attach_host_thread_to_device (n) - || !nvptx_close_device (ptx_devices[n])) + /* Returns 1 if successful, 0 if an error occurred, and -1 for + CUDA_ERROR_DEINITIALIZED. */ + int r = nvptx_attach_host_thread_to_device (n, true); + if (r == 0 + || (r == 1 && !nvptx_close_device (ptx_devices[n]))) { pthread_mutex_unlock (&ptx_dev_lock); return false; @@ -1329,7 +1337,7 @@ GOMP_OFFLOAD_load_image (int ord, unsigned version, const void *target_data, return -1; } - if (!nvptx_attach_host_thread_to_device (ord) + if (!nvptx_attach_host_thread_to_device (ord, false) || !link_ptx (&module, img_header->ptx_objs, img_header->ptx_num)) return -1; @@ -1568,7 +1576,7 @@ GOMP_OFFLOAD_unload_image (int ord, unsigned version, const void *target_data) void * GOMP_OFFLOAD_alloc (int ord, size_t size) { - if (!nvptx_attach_host_thread_to_device (ord)) + if (!nvptx_attach_host_thread_to_device (ord, false)) return NULL; struct ptx_device *ptx_dev = ptx_devices[ord]; @@ -1604,7 +1612,7 @@ GOMP_OFFLOAD_alloc (int ord, size_t size) bool GOMP_OFFLOAD_free (int ord, void *ptr) { - return (nvptx_attach_host_thread_to_device (ord) + return (nvptx_attach_host_thread_to_device (ord, false) && nvptx_free (ptr, ptx_devices[ord])); } @@ -1837,7 +1845,7 @@ cuda_memcpy_sanity_check (const void *h, const void *d, size_t s) bool GOMP_OFFLOAD_host2dev (int ord, void *dst, const void *src, size_t n) { - if (!nvptx_attach_host_thread_to_device (ord) + if (!nvptx_attach_host_thread_to_device (ord, false) || !cuda_memcpy_sanity_check (src, dst, n)) return false; CUDA_CALL (cuMemcpyHtoD, (CUdeviceptr) dst, src, n); @@ -1847,7 +1855,7 @@ GOMP_OFFLOAD_host2dev (int ord, void *dst, const void *src, size_t n) bool GOMP_OFFLOAD_dev2host (int ord, void *dst, const void *src, size_t n) { - if (!nvptx_attach_host_thread_to_device (ord) + if (!nvptx_attach_host_thread_to_device (ord, false) || !cuda_memcpy_sanity_check (dst, src, n)) return false; CUDA_CALL (cuMemcpyDtoH, dst, (CUdeviceptr) src, n); @@ -1868,7 +1876,8 @@ GOMP_OFFLOAD_memcpy2d (int dst_ord, int src_ord, size_t dim1_size, const void *src, size_t src_offset1_size, size_t src_offset0_len, size_t src_dim1_size) { - if (!nvptx_attach_host_thread_to_device (src_ord != -1 ? src_ord : dst_ord)) + if (!nvptx_attach_host_thread_to_device (src_ord != -1 ? src_ord : dst_ord, + false)) return false; /* TODO: Consider using CU_MEMORYTYPE_UNIFIED if supported. */ @@ -1960,7 +1969,8 @@ GOMP_OFFLOAD_memcpy3d (int dst_ord, int src_ord, size_t dim2_size, size_t src_offset0_len, size_t src_dim2_size, size_t src_dim1_len) { - if (!nvptx_attach_host_thread_to_device (src_ord != -1 ? src_ord : dst_ord)) + if (!nvptx_attach_host_thread_to_device (src_ord != -1 ? src_ord : dst_ord, + false)) return false; /* TODO: Consider using CU_MEMORYTYPE_UNIFIED if supported. */ @@ -2050,7 +2060,7 @@ bool GOMP_OFFLOAD_openacc_async_host2dev (int ord, void *dst, const void *src, size_t n, struct goacc_asyncqueue *aq) { - if (!nvptx_attach_host_thread_to_device (ord) + if (!nvptx_attach_host_thread_to_device (ord, false) || !cuda_memcpy_sanity_check (src, dst, n)) return false; CUDA_CALL (cuMemcpyHtoDAsync, (CUdeviceptr) dst, src, n, aq->cuda_stream); @@ -2061,7 +2071,7 @@ bool GOMP_OFFLOAD_openacc_async_dev2host (int ord, void *dst, const void *src, size_t n, struct goacc_asyncqueue *aq) { - if (!nvptx_attach_host_thread_to_device (ord) + if (!nvptx_attach_host_thread_to_device (ord, false) || !cuda_memcpy_sanity_check (dst, src, n)) return false; CUDA_CALL (cuMemcpyDtoHAsync, dst, (CUdeviceptr) src, n, aq->cuda_stream); diff --git a/libgomp/target.c b/libgomp/target.c index 1367e9cce6c..8d05877deb7 100644 --- a/libgomp/target.c +++ b/libgomp/target.c @@ -2706,8 +2706,11 @@ GOMP_offload_unregister_ver (unsigned version, const void *host_table, gomp_mutex_lock (&devicep->lock); if (devicep->type == target_type && devicep->state == GOMP_DEVICE_INITIALIZED) - gomp_unload_image_from_device (devicep, version, - host_table, target_data); + { + goacc_fini_asyncqueues (devicep); + gomp_unload_image_from_device (devicep, version, + host_table, target_data); + } gomp_mutex_unlock (&devicep->lock); }