libgomp: pinned memory Implement the OpenMP pinned memory trait on Linux hosts using the mlock syscall. libgomp/ChangeLog: * allocator.c (MEMSPACE_PIN): New macro. (xmlock): New function. (omp_init_allocator): Don't disallow the pinned trait. (omp_aligned_alloc): Add pinning via MEMSPACE_PIN. (omp_aligned_calloc): Likewise. (omp_realloc): Likewise. * testsuite/libgomp.c/alloc-pinned-1.c: New test. * testsuite/libgomp.c/alloc-pinned-2.c: New test. diff --git a/libgomp/allocator.c b/libgomp/allocator.c index b1f5fe0a5e2..671b91e7ff8 100644 --- a/libgomp/allocator.c +++ b/libgomp/allocator.c @@ -51,6 +51,25 @@ #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \ ((void)MEMSPACE, (void)SIZE, free (ADDR)) #endif +#ifndef MEMSPACE_PIN +/* Only define this on supported host platforms. */ +#ifdef __linux__ +#define MEMSPACE_PIN(MEMSPACE, ADDR, SIZE) \ + ((void)MEMSPACE, xmlock (ADDR, SIZE)) + +#include +#include +void +xmlock (void *addr, size_t size) +{ + if (mlock (addr, size)) + perror ("libgomp: failed to pin memory (ulimit too low?)"); +} +#else +#define MEMSPACE_PIN(MEMSPACE, ADDR, SIZE) \ + ((void)MEMSPACE, (void)ADDR, (void)SIZE) +#endif +#endif /* Map the predefined allocators to the correct memory space. The index to this table is the omp_allocator_handle_t enum value. */ @@ -212,7 +231,7 @@ omp_init_allocator (omp_memspace_handle_t memspace, int ntraits, data.alignment = sizeof (void *); /* No support for these so far (for hbw will use memkind). */ - if (data.pinned || data.memspace == omp_high_bw_mem_space) + if (data.memspace == omp_high_bw_mem_space) return omp_null_allocator; ret = gomp_malloc (sizeof (struct omp_allocator_data)); @@ -326,6 +345,9 @@ retry: #endif goto fail; } + + if (allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, ptr, new_size); } else { @@ -335,6 +357,9 @@ retry: ptr = MEMSPACE_ALLOC (memspace, new_size); if (ptr == NULL) goto fail; + + if (allocator_data && allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, ptr, new_size); } if (new_alignment > sizeof (void *)) @@ -539,6 +564,9 @@ retry: #endif goto fail; } + + if (allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, ptr, new_size); } else { @@ -548,6 +576,9 @@ retry: ptr = MEMSPACE_CALLOC (memspace, new_size); if (ptr == NULL) goto fail; + + if (allocator_data && allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, ptr, new_size); } if (new_alignment > sizeof (void *)) @@ -727,7 +758,11 @@ retry: #endif goto fail; } - else if (prev_size) + + if (allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, new_ptr, new_size); + + if (prev_size) { ret = (char *) new_ptr + sizeof (struct omp_mem_header); ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; @@ -747,6 +782,10 @@ retry: new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size); if (new_ptr == NULL) goto fail; + + if (allocator_data && allocator_data->pinned) + MEMSPACE_PIN (allocator_data->memspace, ptr, new_size); + ret = (char *) new_ptr + sizeof (struct omp_mem_header); ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; ((struct omp_mem_header *) ret)[-1].size = new_size; diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-1.c b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c new file mode 100644 index 00000000000..0a6360cda29 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c @@ -0,0 +1,81 @@ +/* { dg-do run } */ + +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +/* Test that pinned memory works. */ + +#ifdef __linux__ +#include +#include +#include +#include + +#include + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} +#else +int +get_pinned_mem () +{ + return 0; +} +#endif + +#include + +/* Allocate more than a page each time, but stay within the ulimit. */ +#define SIZE 10*1024 + +int +main () +{ + const omp_alloctrait_t traits[] = { + { omp_atk_pinned, 1 } + }; + omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space, 1, traits); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + void *p = omp_alloc (SIZE, allocator); + if (!p) + abort (); + + int amount = get_pinned_mem (); + if (amount == 0) + abort (); + + p = omp_realloc (p, SIZE*2, allocator, allocator); + + int amount2 = get_pinned_mem (); + if (amount2 <= amount) + abort (); + + p = omp_calloc (1, SIZE, allocator); + + if (get_pinned_mem () <= amount2) + abort (); + + return 0; +} diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-2.c b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c new file mode 100644 index 00000000000..8fdb4ff5cfd --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c @@ -0,0 +1,87 @@ +/* { dg-do run } */ + +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +/* Test that pinned memory works (pool_size code path). */ + +#ifdef __linux__ +#include +#include +#include +#include + +#include + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} +#else +int +get_pinned_mem () +{ + return 0; +} +#endif + +#include + +/* Allocate more than a page each time, but stay within the ulimit. */ +#define SIZE 10*1024 + +int +main () +{ + const omp_alloctrait_t traits[] = { + { omp_atk_pinned, 1 }, + { omp_atk_pool_size, SIZE*8 } + }; + omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space, + 2, traits); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + void *p = omp_alloc (SIZE, allocator); + if (!p) + abort (); + + int amount = get_pinned_mem (); + if (amount == 0) + abort (); + + p = omp_realloc (p, SIZE*2, allocator, allocator); + if (!p) + abort (); + + int amount2 = get_pinned_mem (); + if (amount2 <= amount) + abort (); + + p = omp_calloc (1, SIZE, allocator); + if (!p) + abort (); + + if (get_pinned_mem () <= amount2) + abort (); + + return 0; +}