public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc/devel/omp/gcc-12] Attempt to register OpenMP pinned memory using a device instead of 'mlock'
@ 2023-02-16 21:49 Thomas Schwinge
0 siblings, 0 replies; only message in thread
From: Thomas Schwinge @ 2023-02-16 21:49 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:a5a4800e92773da7126c00a9c79b172494d58ab5
commit a5a4800e92773da7126c00a9c79b172494d58ab5
Author: Thomas Schwinge <thomas@codesourcery.com>
Date: Thu Feb 16 15:57:37 2023 +0100
Attempt to register OpenMP pinned memory using a device instead of 'mlock'
Implemented for nvptx offloading via 'cuMemHostRegister'. This means: (a) not
running into 'mlock' limitations, and (b) the device is aware of this and may
optimize host <-> device memory transfers.
This re-works og12 commit ab7520b3b4cd9fdabfd63652badde478955bd3b5
"libgomp: pinned memory".
include/
* cuda/cuda.h (cuMemHostRegister, cuMemHostUnregister): New.
libgomp/
* config/linux/allocator.c (linux_memspace_alloc)
(linux_memspace_free, linux_memspace_realloc): Attempt to register
OpenMP pinned memory using a device instead of 'mlock'.
* libgomp-plugin.h (GOMP_OFFLOAD_register_page_locked)
(GOMP_OFFLOAD_unregister_page_locked): New.
* libgomp.h (gomp_register_page_locked)
(gomp_unregister_page_locked): New
(struct gomp_device_descr): Add 'register_page_locked_func',
'unregister_page_locked_func'.
* plugin/cuda-lib.def (cuMemHostRegister_v2, cuMemHostRegister)
(cuMemHostUnregister): New.
* plugin/plugin-nvptx.c (GOMP_OFFLOAD_register_page_locked)
(GOMP_OFFLOAD_unregister_page_locked): New.
* target.c (gomp_register_page_locked)
(gomp_unregister_page_locked): New.
(gomp_load_plugin_for_device): Handle 'register_page_locked',
'unregister_page_locked'.
* testsuite/libgomp.c/alloc-pinned-1.c: Adjust.
* testsuite/libgomp.c/alloc-pinned-2.c: Likewise.
* testsuite/libgomp.c/alloc-pinned-3.c: Likewise.
* testsuite/libgomp.c/alloc-pinned-4.c: Likewise.
* testsuite/libgomp.c/alloc-pinned-5.c: Likewise.
* testsuite/libgomp.c/alloc-pinned-6.c: Likewise.
Diff:
---
include/ChangeLog.omp | 4 +
include/cuda/cuda.h | 3 +
libgomp/ChangeLog.omp | 24 +++++
libgomp/config/linux/allocator.c | 74 ++++++++++++++-
libgomp/libgomp-plugin.h | 2 +
libgomp/libgomp.h | 4 +
libgomp/plugin/cuda-lib.def | 3 +
libgomp/plugin/plugin-nvptx.c | 33 +++++++
libgomp/target.c | 137 +++++++++++++++++++++++++++
libgomp/testsuite/libgomp.c/alloc-pinned-1.c | 25 +++++
libgomp/testsuite/libgomp.c/alloc-pinned-2.c | 25 +++++
libgomp/testsuite/libgomp.c/alloc-pinned-3.c | 43 ++++++++-
libgomp/testsuite/libgomp.c/alloc-pinned-4.c | 43 ++++++++-
libgomp/testsuite/libgomp.c/alloc-pinned-5.c | 25 +++++
libgomp/testsuite/libgomp.c/alloc-pinned-6.c | 34 ++++++-
15 files changed, 460 insertions(+), 19 deletions(-)
diff --git a/include/ChangeLog.omp b/include/ChangeLog.omp
index 190ae67a321..c8ef7a83c58 100644
--- a/include/ChangeLog.omp
+++ b/include/ChangeLog.omp
@@ -1,3 +1,7 @@
+2023-02-16 Thomas Schwinge <thomas@codesourcery.com>
+
+ * cuda/cuda.h (cuMemHostRegister, cuMemHostUnregister): New.
+
2023-02-09 Kwok Cheung Yeung <kcy@codesourcery.com>
* gomp-constants.h (GOMP_MAP_FLAG_SPECIAL_5): New.
diff --git a/include/cuda/cuda.h b/include/cuda/cuda.h
index 062d394b95f..b0c7636d318 100644
--- a/include/cuda/cuda.h
+++ b/include/cuda/cuda.h
@@ -183,6 +183,9 @@ CUresult cuMemAlloc (CUdeviceptr *, size_t);
CUresult cuMemAllocHost (void **, size_t);
CUresult cuMemAllocManaged(CUdeviceptr *, size_t, unsigned int);
CUresult cuMemHostAlloc (void **, size_t, unsigned int);
+#define cuMemHostRegister cuMemHostRegister_v2
+CUresult cuMemHostRegister(void *, size_t, unsigned int);
+CUresult cuMemHostUnregister(void *);
CUresult cuMemcpy (CUdeviceptr, CUdeviceptr, size_t);
#define cuMemcpyDtoDAsync cuMemcpyDtoDAsync_v2
CUresult cuMemcpyDtoDAsync (CUdeviceptr, CUdeviceptr, size_t, CUstream);
diff --git a/libgomp/ChangeLog.omp b/libgomp/ChangeLog.omp
index 819a5333907..7e464566a21 100644
--- a/libgomp/ChangeLog.omp
+++ b/libgomp/ChangeLog.omp
@@ -1,5 +1,29 @@
2023-02-16 Thomas Schwinge <thomas@codesourcery.com>
+ * config/linux/allocator.c (linux_memspace_alloc)
+ (linux_memspace_free, linux_memspace_realloc): Attempt to register
+ OpenMP pinned memory using a device instead of 'mlock'.
+ * libgomp-plugin.h (GOMP_OFFLOAD_register_page_locked)
+ (GOMP_OFFLOAD_unregister_page_locked): New.
+ * libgomp.h (gomp_register_page_locked)
+ (gomp_unregister_page_locked): New
+ (struct gomp_device_descr): Add 'register_page_locked_func',
+ 'unregister_page_locked_func'.
+ * plugin/cuda-lib.def (cuMemHostRegister_v2, cuMemHostRegister)
+ (cuMemHostUnregister): New.
+ * plugin/plugin-nvptx.c (GOMP_OFFLOAD_register_page_locked)
+ (GOMP_OFFLOAD_unregister_page_locked): New.
+ * target.c (gomp_register_page_locked)
+ (gomp_unregister_page_locked): New.
+ (gomp_load_plugin_for_device): Handle 'register_page_locked',
+ 'unregister_page_locked'.
+ * testsuite/libgomp.c/alloc-pinned-1.c: Adjust.
+ * testsuite/libgomp.c/alloc-pinned-2.c: Likewise.
+ * testsuite/libgomp.c/alloc-pinned-3.c: Likewise.
+ * testsuite/libgomp.c/alloc-pinned-4.c: Likewise.
+ * testsuite/libgomp.c/alloc-pinned-5.c: Likewise.
+ * testsuite/libgomp.c/alloc-pinned-6.c: Likewise.
+
* allocator.c (omp_realloc): Route 'free' through 'MEMSPACE_FREE'.
* config/linux/allocator.c (linux_memspace_alloc)
diff --git a/libgomp/config/linux/allocator.c b/libgomp/config/linux/allocator.c
index f278e5cdf14..81e64b268e9 100644
--- a/libgomp/config/linux/allocator.c
+++ b/libgomp/config/linux/allocator.c
@@ -24,6 +24,10 @@
/* Implement malloc routines that can handle pinned memory on Linux.
+ Given that pinned memory is typically used to help host <-> device memory
+ transfers, we attempt to register such using a device (really: libgomp
+ plugin), but fall back to mlock if no suitable device is available.
+
It's possible to use mlock on any heap memory, but using munlock is
problematic if there are multiple pinned allocations on the same page.
Tracking all that manually would be possible, but adds overhead. This may
@@ -37,6 +41,7 @@
#define _GNU_SOURCE
#include <sys/mman.h>
#include <string.h>
+#include <assert.h>
#include "libgomp.h"
static bool always_pinned_mode = false;
@@ -53,9 +58,15 @@ GOMP_enable_pinned_mode ()
always_pinned_mode = true;
}
+static int using_device_for_register_page_locked
+ = /* uninitialized */ -1;
+
static void *
linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
{
+ gomp_debug (0, "%s: memspace=%llu, size=%llu, pin=%d\n",
+ __FUNCTION__, (unsigned long long) memspace, (unsigned long long) size, pin);
+
/* Explicit pinning may not be required. */
pin = pin && !always_pinned_mode;
@@ -71,11 +82,32 @@ linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
if (addr == MAP_FAILED)
return NULL;
- if (mlock (addr, size))
+ int using_device
+ = __atomic_load_n (&using_device_for_register_page_locked,
+ MEMMODEL_RELAXED);
+ gomp_debug (0, " using_device=%d\n",
+ using_device);
+ if (using_device != 0)
+ {
+ using_device = gomp_register_page_locked (addr, size);
+ int using_device_old
+ = __atomic_exchange_n (&using_device_for_register_page_locked,
+ using_device, MEMMODEL_RELAXED);
+ gomp_debug (0, " using_device=%d, using_device_old=%d\n",
+ using_device, using_device_old);
+ assert (using_device_old == -1
+ /* We shouldn't have concurrently changed our mind. */
+ || using_device_old == using_device);
+ }
+ if (using_device == 0)
{
- gomp_debug (0, "libgomp: failed to pin memory (ulimit too low?)\n");
- munmap (addr, size);
- return NULL;
+ gomp_debug (0, " mlock\n");
+ if (mlock (addr, size))
+ {
+ gomp_debug (0, "libgomp: failed to pin memory (ulimit too low?)\n");
+ munmap (addr, size);
+ return NULL;
+ }
}
return addr;
@@ -87,6 +119,9 @@ linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
static void *
linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin)
{
+ gomp_debug (0, "%s: memspace=%llu, size=%llu, pin=%d\n",
+ __FUNCTION__, (unsigned long long) memspace, (unsigned long long) size, pin);
+
/* Explicit pinning may not be required. */
pin = pin && !always_pinned_mode;
@@ -107,13 +142,28 @@ static void
linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size,
int pin)
{
+ gomp_debug (0, "%s: memspace=%llu, addr=%p, size=%llu, pin=%d\n",
+ __FUNCTION__, (unsigned long long) memspace, addr, (unsigned long long) size, pin);
+
/* Explicit pinning may not be required. */
pin = pin && !always_pinned_mode;
if (memspace == ompx_unified_shared_mem_space)
gomp_usm_free (addr, GOMP_DEVICE_ICV);
else if (pin)
- munmap (addr, size);
+ {
+ int using_device
+ = __atomic_load_n (&using_device_for_register_page_locked,
+ MEMMODEL_RELAXED);
+ gomp_debug (0, " using_device=%d\n",
+ using_device);
+ if (using_device == 1)
+ gomp_unregister_page_locked (addr, size);
+ else
+ /* 'munlock'ing is implicit with following 'munmap'. */
+ ;
+ munmap (addr, size);
+ }
else
free (addr);
}
@@ -122,6 +172,9 @@ static void *
linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
size_t oldsize, size_t size, int oldpin, int pin)
{
+ gomp_debug (0, "%s: memspace=%llu, addr=%p, oldsize=%llu, size=%llu, oldpin=%d, pin=%d\n",
+ __FUNCTION__, (unsigned long long) memspace, addr, (unsigned long long) oldsize, (unsigned long long) size, oldpin, pin);
+
/* Explicit pinning may not be required. */
pin = pin && !always_pinned_mode;
@@ -129,6 +182,17 @@ linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
goto manual_realloc;
else if (oldpin && pin)
{
+ /* We can only expect to be able to just 'mremap' if not using a device
+ for registering page-locked memory. */
+ int using_device
+ = __atomic_load_n (&using_device_for_register_page_locked,
+ MEMMODEL_RELAXED);
+ gomp_debug (0, " using_device=%d\n",
+ using_device);
+ if (using_device != 0)
+ goto manual_realloc;
+
+ gomp_debug (0, " mremap\n");
void *newaddr = mremap (addr, oldsize, size, MREMAP_MAYMOVE);
if (newaddr == MAP_FAILED)
return NULL;
diff --git a/libgomp/libgomp-plugin.h b/libgomp/libgomp-plugin.h
index bb79ef8d9d7..345fc62d4f5 100644
--- a/libgomp/libgomp-plugin.h
+++ b/libgomp/libgomp-plugin.h
@@ -144,6 +144,8 @@ extern bool GOMP_OFFLOAD_free (int, void *);
extern void *GOMP_OFFLOAD_usm_alloc (int, size_t);
extern bool GOMP_OFFLOAD_usm_free (int, void *);
extern bool GOMP_OFFLOAD_is_usm_ptr (void *);
+extern bool GOMP_OFFLOAD_register_page_locked (void *, size_t);
+extern bool GOMP_OFFLOAD_unregister_page_locked (void *, size_t);
extern bool GOMP_OFFLOAD_dev2host (int, void *, const void *, size_t);
extern bool GOMP_OFFLOAD_host2dev (int, void *, const void *, size_t);
extern bool GOMP_OFFLOAD_dev2dev (int, void *, const void *, size_t);
diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h
index c001b468252..a5fa3f9daab 100644
--- a/libgomp/libgomp.h
+++ b/libgomp/libgomp.h
@@ -1133,6 +1133,8 @@ extern void gomp_target_rev (uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
void *);
extern void * gomp_usm_alloc (size_t size, int device_num);
extern void gomp_usm_free (void *device_ptr, int device_num);
+extern bool gomp_register_page_locked (void *, size_t);
+extern void gomp_unregister_page_locked (void *, size_t);
/* Splay tree definitions. */
typedef struct splay_tree_node_s *splay_tree_node;
@@ -1392,6 +1394,8 @@ struct gomp_device_descr
__typeof (GOMP_OFFLOAD_usm_alloc) *usm_alloc_func;
__typeof (GOMP_OFFLOAD_usm_free) *usm_free_func;
__typeof (GOMP_OFFLOAD_is_usm_ptr) *is_usm_ptr_func;
+ __typeof (GOMP_OFFLOAD_register_page_locked) *register_page_locked_func;
+ __typeof (GOMP_OFFLOAD_unregister_page_locked) *unregister_page_locked_func;
__typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
__typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
__typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
diff --git a/libgomp/plugin/cuda-lib.def b/libgomp/plugin/cuda-lib.def
index 9b786c9f2f6..8dbaadf848e 100644
--- a/libgomp/plugin/cuda-lib.def
+++ b/libgomp/plugin/cuda-lib.def
@@ -31,6 +31,9 @@ CUDA_ONE_CALL (cuMemAlloc)
CUDA_ONE_CALL (cuMemAllocHost)
CUDA_ONE_CALL (cuMemAllocManaged)
CUDA_ONE_CALL (cuMemHostAlloc)
+CUDA_ONE_CALL_MAYBE_NULL (cuMemHostRegister_v2)
+CUDA_ONE_CALL (cuMemHostRegister)
+CUDA_ONE_CALL (cuMemHostUnregister)
CUDA_ONE_CALL (cuMemcpy)
CUDA_ONE_CALL (cuMemcpyDtoDAsync)
CUDA_ONE_CALL (cuMemcpyDtoH)
diff --git a/libgomp/plugin/plugin-nvptx.c b/libgomp/plugin/plugin-nvptx.c
index 8e7b63bd637..698317f37ac 100644
--- a/libgomp/plugin/plugin-nvptx.c
+++ b/libgomp/plugin/plugin-nvptx.c
@@ -77,11 +77,14 @@ extern CUresult cuGetErrorString (CUresult, const char **);
CUresult cuLinkAddData (CUlinkState, CUjitInputType, void *, size_t,
const char *, unsigned, CUjit_option *, void **);
CUresult cuLinkCreate (unsigned, CUjit_option *, void **, CUlinkState *);
+#undef cuMemHostRegister
+CUresult cuMemHostRegister (void *, size_t, unsigned int);
#else
typedef size_t (*CUoccupancyB2DSize)(int);
CUresult cuLinkAddData_v2 (CUlinkState, CUjitInputType, void *, size_t,
const char *, unsigned, CUjit_option *, void **);
CUresult cuLinkCreate_v2 (unsigned, CUjit_option *, void **, CUlinkState *);
+CUresult cuMemHostRegister_v2 (void *, size_t, unsigned int);
CUresult cuOccupancyMaxPotentialBlockSize(int *, int *, CUfunction,
CUoccupancyB2DSize, size_t, int);
#endif
@@ -1704,6 +1707,36 @@ GOMP_OFFLOAD_is_usm_ptr (void *ptr)
return managed;
}
+
+bool
+GOMP_OFFLOAD_register_page_locked (void *ptr, size_t size)
+{
+ GOMP_PLUGIN_debug (0, "nvptx %s: ptr=%p, size=%llu\n",
+ __FUNCTION__, ptr, (unsigned long long) size);
+
+ unsigned int flags = 0;
+ /* Given 'CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING', we don't need
+ 'flags |= CU_MEMHOSTREGISTER_PORTABLE;' here. */
+ if (CUDA_CALL_EXISTS (cuMemHostRegister_v2))
+ CUDA_CALL (cuMemHostRegister_v2, ptr, size, flags);
+ else
+ CUDA_CALL (cuMemHostRegister, ptr, size, flags);
+
+ return true;
+}
+
+bool
+GOMP_OFFLOAD_unregister_page_locked (void *ptr, size_t size)
+{
+ GOMP_PLUGIN_debug (0, "nvptx %s: ptr=%p, size=%llu\n",
+ __FUNCTION__, ptr, (unsigned long long) size);
+
+ CUDA_CALL (cuMemHostUnregister, ptr);
+
+ return true;
+}
+
+
void
GOMP_OFFLOAD_openacc_exec (void (*fn) (void *), size_t mapnum,
void **hostaddrs, void **devaddrs,
diff --git a/libgomp/target.c b/libgomp/target.c
index 1b911c9bdb9..e7285188d1e 100644
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -4584,6 +4584,141 @@ gomp_usm_free (void *device_ptr, int device_num)
gomp_mutex_unlock (&devicep->lock);
}
+
+/* Device (really: libgomp plugin) for registering paged-locked memory. We
+ assume there is either none or exactly one such device for the lifetime of
+ the process. */
+
+static struct gomp_device_descr *device_for_register_page_locked
+ = /* uninitialized */ (void *) -1;
+
+static struct gomp_device_descr *
+get_device_for_register_page_locked (void)
+{
+ gomp_debug (0, "%s\n",
+ __FUNCTION__);
+
+ struct gomp_device_descr *device;
+#ifdef HAVE_SYNC_BUILTINS
+ device
+ = __atomic_load_n (&device_for_register_page_locked, MEMMODEL_RELAXED);
+ if (device == (void *) -1)
+ {
+ gomp_debug (0, " init\n");
+
+ gomp_init_targets_once ();
+
+ device = NULL;
+ for (int i = 0; i < num_devices; ++i)
+ {
+ gomp_debug (0, " i=%d, target_id=%d\n",
+ i, devices[i].target_id);
+
+ /* We consider only the first device of potentially several of the
+ same type as this functionality is not specific to an individual
+ offloading device, but instead relates to the host-side
+ implementation of the respective offloading implementation. */
+ if (devices[i].target_id != 0)
+ continue;
+
+ if (!devices[i].register_page_locked_func)
+ continue;
+
+ gomp_debug (0, " found device: %p (%s)\n",
+ &devices[i], devices[i].name);
+ if (device)
+ gomp_fatal ("Unclear how %s and %s libgomp plugins may"
+ " simultaneously provide functionality"
+ " to register page-locked memory",
+ device->name, devices[i].name);
+ else
+ device = &devices[i];
+ }
+
+ struct gomp_device_descr *device_old
+ = __atomic_exchange_n (&device_for_register_page_locked, device,
+ MEMMODEL_RELAXED);
+ gomp_debug (0, " old device_for_register_page_locked: %p\n",
+ device_old);
+ assert (device_old == (void *) -1
+ /* We shouldn't have concurrently found a different or no
+ device. */
+ || device_old == device);
+ }
+#else /* !HAVE_SYNC_BUILTINS */
+ gomp_debug (0, " not implemented for '!HAVE_SYNC_BUILTINS'\n");
+ (void) &device_for_register_page_locked;
+ device = NULL;
+#endif /* HAVE_SYNC_BUILTINS */
+
+ gomp_debug (0, " -> device=%p (%s)\n",
+ device, device ? device->name : "[none]");
+ return device;
+}
+
+/* Register page-locked memory region.
+ Returns whether we have a device capable of that. */
+
+attribute_hidden bool
+gomp_register_page_locked (void *ptr, size_t size)
+{
+ gomp_debug (0, "%s: ptr=%p, size=%llu\n",
+ __FUNCTION__, ptr, (unsigned long long) size);
+
+ struct gomp_device_descr *device = get_device_for_register_page_locked ();
+ gomp_debug (0, " device=%p (%s)\n",
+ device, device ? device->name : "[none]");
+ if (device)
+ {
+ gomp_mutex_lock (&device->lock);
+ if (device->state == GOMP_DEVICE_UNINITIALIZED)
+ gomp_init_device (device);
+ else if (device->state == GOMP_DEVICE_FINALIZED)
+ {
+ gomp_mutex_unlock (&device->lock);
+ gomp_fatal ("Device %s for registering page-locked memory"
+ " is finalized", device->name);
+ }
+ gomp_mutex_unlock (&device->lock);
+
+ if (!device->register_page_locked_func (ptr, size))
+ gomp_fatal ("Failed to register page-locked memory"
+ " via %s libgomp plugin",
+ device->name);
+ }
+ return device != NULL;
+}
+
+/* Unregister page-locked memory region.
+ This must only be called if 'gomp_register_page_locked' returned 'true'. */
+
+attribute_hidden void
+gomp_unregister_page_locked (void *ptr, size_t size)
+{
+ gomp_debug (0, "%s: ptr=%p\n",
+ __FUNCTION__, ptr);
+
+ struct gomp_device_descr *device = get_device_for_register_page_locked ();
+ gomp_debug (0, " device=%p (%s)\n",
+ device, device ? device->name : "[none]");
+ assert (device);
+
+ gomp_mutex_lock (&device->lock);
+ assert (device->state != GOMP_DEVICE_UNINITIALIZED);
+ if (device->state == GOMP_DEVICE_FINALIZED)
+ {
+ gomp_mutex_unlock (&device->lock);
+ return;
+ }
+ gomp_mutex_unlock (&device->lock);
+
+ if (!device->unregister_page_locked_func (ptr, size))
+ gomp_fatal ("Failed to unregister page-locked memory"
+ " via %s libgomp plugin",
+ device->name);
+}
+
+
int
omp_target_is_present (const void *ptr, int device_num)
{
@@ -5268,6 +5403,8 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device,
DLSYM_OPT (usm_alloc, usm_alloc);
DLSYM_OPT (usm_free, usm_free);
DLSYM_OPT (is_usm_ptr, is_usm_ptr);
+ DLSYM_OPT (register_page_locked, register_page_locked);
+ DLSYM_OPT (unregister_page_locked, unregister_page_locked);
DLSYM (dev2host);
DLSYM (host2dev);
DLSYM (evaluate_device);
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-1.c b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c
index fb7ac8b0080..bd71e22b003 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-1.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c
@@ -2,6 +2,8 @@
/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that pinned memory works. */
#include <stdio.h>
@@ -67,9 +69,14 @@ verify0 (char *p, size_t s)
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* Allocate at least a page each time, but stay within the ulimit. */
const int SIZE = PAGE_SIZE;
CHECK_SIZE (SIZE*3);
+#endif
const omp_alloctrait_t traits[] = {
{ omp_atk_pinned, 1 }
@@ -85,19 +92,37 @@ main ()
abort ();
int amount = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount != 0)
+ abort ();
+#else
if (amount == 0)
abort ();
+#endif
p = omp_realloc (p, SIZE*2, allocator, allocator);
int amount2 = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount2 != 0)
+ abort ();
+#else
if (amount2 <= amount)
abort ();
+#endif
p = omp_calloc (1, SIZE, allocator);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (get_pinned_mem () != 0)
+ abort ();
+#else
if (get_pinned_mem () <= amount2)
abort ();
+#endif
verify0 (p, SIZE);
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-2.c b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c
index 651b89fb42f..c71248b046d 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-2.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c
@@ -2,6 +2,8 @@
/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that pinned memory works (pool_size code path). */
#include <stdio.h>
@@ -67,9 +69,14 @@ verify0 (char *p, size_t s)
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* Allocate at least a page each time, but stay within the ulimit. */
const int SIZE = PAGE_SIZE;
CHECK_SIZE (SIZE*3);
+#endif
const omp_alloctrait_t traits[] = {
{ omp_atk_pinned, 1 },
@@ -87,23 +94,41 @@ main ()
abort ();
int amount = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount != 0)
+ abort ();
+#else
if (amount == 0)
abort ();
+#endif
p = omp_realloc (p, SIZE*2, allocator, allocator);
if (!p)
abort ();
int amount2 = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount2 != 0)
+ abort ();
+#else
if (amount2 <= amount)
abort ();
+#endif
p = omp_calloc (1, SIZE, allocator);
if (!p)
abort ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (get_pinned_mem () != 0)
+ abort ();
+#else
if (get_pinned_mem () <= amount2)
abort ();
+#endif
verify0 (p, SIZE);
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-3.c b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c
index f41797881ef..26b0c352d85 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-3.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c
@@ -1,5 +1,7 @@
/* { dg-do run } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that pinned memory fails correctly. */
#include <stdio.h>
@@ -74,8 +76,14 @@ verify0 (char *p, size_t s)
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* This needs to be large enough to cover multiple pages. */
const int SIZE = PAGE_SIZE*4;
+#endif
+ const int PIN_LIMIT = PAGE_SIZE*2;
/* Pinned memory, no fallback. */
const omp_alloctrait_t traits1[] = {
@@ -92,21 +100,33 @@ main ()
omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space, 2, traits2);
/* Ensure that the limit is smaller than the allocation. */
- set_pin_limit (SIZE/2);
+ set_pin_limit (PIN_LIMIT);
// Sanity check
if (get_pinned_mem () != 0)
abort ();
- // Should fail
void *p = omp_alloc (SIZE, allocator1);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
- // Should fail
p = omp_calloc (1, SIZE, allocator1);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
// Should fall back
p = omp_alloc (SIZE, allocator2);
@@ -119,16 +139,29 @@ main ()
abort ();
verify0 (p, SIZE);
- // Should fail to realloc
void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'; does reallocate.
+ if (!notpinned || !p || p == notpinned)
+ abort ();
+#else
+ // Should fail to realloc
if (!notpinned || p)
abort ();
+#endif
- // Should fall back to no realloc needed
+#ifdef OFFLOAD_DEVICE_NVPTX
+ void *p_ = omp_realloc (p, SIZE, allocator2, allocator1);
+ // Does reallocate.
+ if (p_ == p)
+ abort ();
+#else
p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc);
+ // Should fall back to no realloc needed
if (p != notpinned)
abort ();
+#endif
// No memory should have been pinned
int amount = get_pinned_mem ();
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-4.c b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c
index a878da8c558..0bd6a552d94 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-4.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c
@@ -1,5 +1,7 @@
/* { dg-do run } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that pinned memory fails correctly, pool_size code path. */
#include <stdio.h>
@@ -74,8 +76,14 @@ verify0 (char *p, size_t s)
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* This needs to be large enough to cover multiple pages. */
const int SIZE = PAGE_SIZE*4;
+#endif
+ const int PIN_LIMIT = PAGE_SIZE*2;
/* Pinned memory, no fallback. */
const omp_alloctrait_t traits1[] = {
@@ -94,21 +102,33 @@ main ()
omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space, 3, traits2);
/* Ensure that the limit is smaller than the allocation. */
- set_pin_limit (SIZE/2);
+ set_pin_limit (PIN_LIMIT);
// Sanity check
if (get_pinned_mem () != 0)
abort ();
- // Should fail
void *p = omp_alloc (SIZE, allocator1);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
- // Should fail
p = omp_calloc (1, SIZE, allocator1);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
// Should fall back
p = omp_alloc (SIZE, allocator2);
@@ -121,16 +141,29 @@ main ()
abort ();
verify0 (p, SIZE);
- // Should fail to realloc
void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'; does reallocate.
+ if (!notpinned || !p || p == notpinned)
+ abort ();
+#else
+ // Should fail to realloc
if (!notpinned || p)
abort ();
+#endif
- // Should fall back to no realloc needed
+#ifdef OFFLOAD_DEVICE_NVPTX
+ void *p_ = omp_realloc (p, SIZE, allocator2, allocator1);
+ // Does reallocate.
+ if (p_ == p)
+ abort ();
+#else
p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc);
+ // Should fall back to no realloc needed
if (p != notpinned)
abort ();
+#endif
// No memory should have been pinned
int amount = get_pinned_mem ();
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-5.c b/libgomp/testsuite/libgomp.c/alloc-pinned-5.c
index 65983b3d03d..623c96a78e3 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-5.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-5.c
@@ -2,6 +2,8 @@
/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that ompx_pinned_mem_alloc works. */
#include <stdio.h>
@@ -67,9 +69,14 @@ verify0 (char *p, size_t s)
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* Allocate at least a page each time, but stay within the ulimit. */
const int SIZE = PAGE_SIZE;
CHECK_SIZE (SIZE*3);
+#endif
// Sanity check
if (get_pinned_mem () != 0)
@@ -80,19 +87,37 @@ main ()
abort ();
int amount = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount != 0)
+ abort ();
+#else
if (amount == 0)
abort ();
+#endif
p = omp_realloc (p, SIZE*2, ompx_pinned_mem_alloc, ompx_pinned_mem_alloc);
int amount2 = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount2 != 0)
+ abort ();
+#else
if (amount2 <= amount)
abort ();
+#endif
p = omp_calloc (1, SIZE, ompx_pinned_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (get_pinned_mem () != 0)
+ abort ();
+#else
if (get_pinned_mem () <= amount2)
abort ();
+#endif
verify0 (p, SIZE);
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-6.c b/libgomp/testsuite/libgomp.c/alloc-pinned-6.c
index bbe20c04875..c0f8b260e37 100644
--- a/libgomp/testsuite/libgomp.c/alloc-pinned-6.c
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-6.c
@@ -1,5 +1,7 @@
/* { dg-do run } */
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
/* Test that ompx_pinned_mem_alloc fails correctly. */
#include <stdio.h>
@@ -66,31 +68,55 @@ set_pin_limit ()
int
main ()
{
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* Go big or go home. */
+ const int SIZE = 40 * 1024 * 1024;
+#else
/* Allocate at least a page each time, but stay within the ulimit. */
const int SIZE = PAGE_SIZE*4;
+#endif
+ const int PIN_LIMIT = PAGE_SIZE*2;
/* Ensure that the limit is smaller than the allocation. */
- set_pin_limit (SIZE/2);
+ set_pin_limit (PIN_LIMIT);
// Sanity check
if (get_pinned_mem () != 0)
abort ();
- // Should fail
void *p = omp_alloc (SIZE, ompx_pinned_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
- // Should fail
p = omp_calloc (1, SIZE, ompx_pinned_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'.
+ if (!p)
+ abort ();
+#else
+ // Should fail
if (p)
abort ();
+#endif
- // Should fail to realloc
void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
p = omp_realloc (notpinned, SIZE, ompx_pinned_mem_alloc, omp_default_mem_alloc);
+#ifdef OFFLOAD_DEVICE_NVPTX
+ // Doesn't care about 'set_pin_limit'; does reallocate.
+ if (!notpinned || !p || p == notpinned)
+ abort ();
+#else
+ // Should fail to realloc
if (!notpinned || p)
abort ();
+#endif
// No memory should have been pinned
int amount = get_pinned_mem ();
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2023-02-16 21:49 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-02-16 21:49 [gcc/devel/omp/gcc-12] Attempt to register OpenMP pinned memory using a device instead of 'mlock' Thomas Schwinge
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).