diff --git a/libgomp/allocator.c b/libgomp/allocator.c index 9b33bcf529b..54310ab93ca 100644 --- a/libgomp/allocator.c +++ b/libgomp/allocator.c @@ -39,16 +39,20 @@ /* These macros may be overridden in config//allocator.c. */ #ifndef MEMSPACE_ALLOC -#define MEMSPACE_ALLOC(MEMSPACE, SIZE) malloc (SIZE) +#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \ + (PIN ? NULL : malloc (SIZE)) #endif #ifndef MEMSPACE_CALLOC -#define MEMSPACE_CALLOC(MEMSPACE, SIZE) calloc (1, SIZE) +#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \ + (PIN ? NULL : calloc (1, SIZE)) #endif #ifndef MEMSPACE_REALLOC -#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) realloc (ADDR, SIZE) +#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \ + ((PIN) || (OLDPIN) ? NULL : realloc (ADDR, SIZE)) #endif #ifndef MEMSPACE_FREE -#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) free (ADDR) +#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \ + (PIN ? NULL : free (ADDR)) #endif /* Map the predefined allocators to the correct memory space. @@ -351,10 +355,6 @@ omp_init_allocator (omp_memspace_handle_t memspace, int ntraits, break; } - /* No support for this so far. */ - if (data.pinned) - return omp_null_allocator; - ret = gomp_malloc (sizeof (struct omp_allocator_data)); *ret = data; #ifndef HAVE_SYNC_BUILTINS @@ -481,7 +481,8 @@ retry: } else #endif - ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size); + ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size, + allocator_data->pinned); if (ptr == NULL) { #ifdef HAVE_SYNC_BUILTINS @@ -511,7 +512,8 @@ retry: = (allocator_data ? allocator_data->memspace : predefined_alloc_mapping[allocator]); - ptr = MEMSPACE_ALLOC (memspace, new_size); + ptr = MEMSPACE_ALLOC (memspace, new_size, + allocator_data && allocator_data->pinned); } if (ptr == NULL) goto fail; @@ -542,9 +544,9 @@ fail: #ifdef LIBGOMP_USE_MEMKIND || memkind #endif - || (allocator_data - && allocator_data->pool_size < ~(uintptr_t) 0) - || !allocator_data) + || !allocator_data + || allocator_data->pool_size < ~(uintptr_t) 0 + || allocator_data->pinned) { allocator = omp_default_mem_alloc; goto retry; @@ -596,6 +598,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator) struct omp_mem_header *data; omp_memspace_handle_t memspace __attribute__((unused)) = omp_default_mem_space; + int pinned __attribute__((unused)) = false; if (ptr == NULL) return; @@ -627,6 +630,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator) #endif memspace = allocator_data->memspace; + pinned = allocator_data->pinned; } else { @@ -651,7 +655,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator) memspace = predefined_alloc_mapping[data->allocator]; } - MEMSPACE_FREE (memspace, data->ptr, data->size); + MEMSPACE_FREE (memspace, data->ptr, data->size, pinned); } ialias (omp_free) @@ -767,7 +771,8 @@ retry: } else #endif - ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size); + ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size, + allocator_data->pinned); if (ptr == NULL) { #ifdef HAVE_SYNC_BUILTINS @@ -797,7 +802,8 @@ retry: = (allocator_data ? allocator_data->memspace : predefined_alloc_mapping[allocator]); - ptr = MEMSPACE_CALLOC (memspace, new_size); + ptr = MEMSPACE_CALLOC (memspace, new_size, + allocator_data && allocator_data->pinned); } if (ptr == NULL) goto fail; @@ -828,9 +834,9 @@ fail: #ifdef LIBGOMP_USE_MEMKIND || memkind #endif - || (allocator_data - && allocator_data->pool_size < ~(uintptr_t) 0) - || !allocator_data) + || !allocator_data + || allocator_data->pool_size < ~(uintptr_t) 0 + || allocator_data->pinned) { allocator = omp_default_mem_alloc; goto retry; @@ -1021,9 +1027,13 @@ retry: #endif if (prev_size) new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr, - data->size, new_size); + data->size, new_size, + (free_allocator_data + && free_allocator_data->pinned), + allocator_data->pinned); else - new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size); + new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size, + allocator_data->pinned); if (new_ptr == NULL) { #ifdef HAVE_SYNC_BUILTINS @@ -1069,10 +1079,14 @@ retry: = (allocator_data ? allocator_data->memspace : predefined_alloc_mapping[allocator]); - new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size); + new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size, + (free_allocator_data + && free_allocator_data->pinned), + allocator_data && allocator_data->pinned); } if (new_ptr == NULL) goto fail; + ret = (char *) new_ptr + sizeof (struct omp_mem_header); ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; ((struct omp_mem_header *) ret)[-1].size = new_size; @@ -1095,7 +1109,8 @@ retry: = (allocator_data ? allocator_data->memspace : predefined_alloc_mapping[allocator]); - new_ptr = MEMSPACE_ALLOC (memspace, new_size); + new_ptr = MEMSPACE_ALLOC (memspace, new_size, + allocator_data && allocator_data->pinned); } if (new_ptr == NULL) goto fail; @@ -1151,9 +1166,9 @@ fail: #ifdef LIBGOMP_USE_MEMKIND || memkind #endif - || (allocator_data - && allocator_data->pool_size < ~(uintptr_t) 0) - || !allocator_data) + || !allocator_data + || allocator_data->pool_size < ~(uintptr_t) 0 + || allocator_data->pinned) { allocator = omp_default_mem_alloc; goto retry; diff --git a/libgomp/config/linux/allocator.c b/libgomp/config/linux/allocator.c index b73acce9121..1496e41875c 100644 --- a/libgomp/config/linux/allocator.c +++ b/libgomp/config/linux/allocator.c @@ -33,4 +33,103 @@ #define LIBGOMP_USE_MEMKIND #endif +/* Implement malloc routines that can handle pinned memory on Linux. + + It's possible to use mlock on any heap memory, but using munlock is + problematic if there are multiple pinned allocations on the same page. + Tracking all that manually would be possible, but adds overhead. This may + be worth it if there are a lot of small allocations getting pinned, but + this seems less likely in a HPC application. + + Instead we optimize for large pinned allocations, and use mmap to ensure + that two pinned allocations don't share the same page. This also means + that large allocations don't pin extra pages by being poorly aligned. */ + +#define _GNU_SOURCE +#include +#include +#include "libgomp.h" + +static void * +linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin) +{ + (void)memspace; + + if (pin) + { + void *addr = mmap (NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) + return NULL; + + if (mlock (addr, size)) + { + gomp_debug (0, "libgomp: failed to pin memory (ulimit too low?)\n"); + munmap (addr, size); + return NULL; + } + + return addr; + } + else + return malloc (size); +} + +static void * +linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin) +{ + if (pin) + return linux_memspace_alloc (memspace, size, pin); + else + return calloc (1, size); +} + +static void +linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size, + int pin) +{ + (void)memspace; + + if (pin) + munmap (addr, size); + else + free (addr); +} + +static void * +linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr, + size_t oldsize, size_t size, int oldpin, int pin) +{ + if (oldpin && pin) + { + void *newaddr = mremap (addr, oldsize, size, MREMAP_MAYMOVE); + if (newaddr == MAP_FAILED) + return NULL; + + return newaddr; + } + else if (oldpin || pin) + { + void *newaddr = linux_memspace_alloc (memspace, size, pin); + if (newaddr) + { + memcpy (newaddr, addr, oldsize < size ? oldsize : size); + linux_memspace_free (memspace, addr, oldsize, oldpin); + } + + return newaddr; + } + else + return realloc (addr, size); +} + +#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \ + linux_memspace_alloc (MEMSPACE, SIZE, PIN) +#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \ + linux_memspace_calloc (MEMSPACE, SIZE, PIN) +#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \ + linux_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) +#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \ + linux_memspace_free (MEMSPACE, ADDR, SIZE, PIN) + #include "../../allocator.c" diff --git a/libgomp/config/nvptx/allocator.c b/libgomp/config/nvptx/allocator.c index 6bc2ea48043..f740b97f6ac 100644 --- a/libgomp/config/nvptx/allocator.c +++ b/libgomp/config/nvptx/allocator.c @@ -358,13 +358,13 @@ nvptx_memspace_realloc (omp_memspace_handle_t memspace, void *addr, return realloc (addr, size); } -#define MEMSPACE_ALLOC(MEMSPACE, SIZE) \ +#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \ nvptx_memspace_alloc (MEMSPACE, SIZE) -#define MEMSPACE_CALLOC(MEMSPACE, SIZE) \ +#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \ nvptx_memspace_calloc (MEMSPACE, SIZE) -#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \ +#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \ nvptx_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE) -#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \ +#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \ nvptx_memspace_free (MEMSPACE, ADDR, SIZE) #include "../../allocator.c" diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-1.c b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c new file mode 100644 index 00000000000..79792b16d83 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c @@ -0,0 +1,95 @@ +/* { dg-do run } */ + +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +/* Test that pinned memory works. */ + +#include +#include + +#ifdef __linux__ +#include +#include + +#include +#include + +#define PAGE_SIZE sysconf(_SC_PAGESIZE) +#define CHECK_SIZE(SIZE) { \ + struct rlimit limit; \ + if (getrlimit (RLIMIT_MEMLOCK, &limit) \ + || limit.rlim_cur <= SIZE) \ + fprintf (stderr, "unsufficient lockable memory; please increase ulimit\n"); \ + } + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} +#else +#define PAGE_SIZE 1 /* unknown */ +#define CHECK_SIZE(SIZE) fprintf (stderr, "OS unsupported\n"); + +int +get_pinned_mem () +{ + return 0; +} +#endif + +#include + +int +main () +{ + /* Allocate at least a page each time, but stay within the ulimit. */ + const int SIZE = PAGE_SIZE; + CHECK_SIZE (SIZE*3); + + const omp_alloctrait_t traits[] = { + { omp_atk_pinned, 1 } + }; + omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space, 1, traits); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + void *p = omp_alloc (SIZE, allocator); + if (!p) + abort (); + + int amount = get_pinned_mem (); + if (amount == 0) + abort (); + + p = omp_realloc (p, SIZE*2, allocator, allocator); + + int amount2 = get_pinned_mem (); + if (amount2 <= amount) + abort (); + + p = omp_calloc (1, SIZE, allocator); + + if (get_pinned_mem () <= amount2) + abort (); + + return 0; +} diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-2.c b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c new file mode 100644 index 00000000000..228c656b715 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c @@ -0,0 +1,101 @@ +/* { dg-do run } */ + +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +/* Test that pinned memory works (pool_size code path). */ + +#include +#include + +#ifdef __linux__ +#include +#include + +#include +#include + +#define PAGE_SIZE sysconf(_SC_PAGESIZE) +#define CHECK_SIZE(SIZE) { \ + struct rlimit limit; \ + if (getrlimit (RLIMIT_MEMLOCK, &limit) \ + || limit.rlim_cur <= SIZE) \ + fprintf (stderr, "unsufficient lockable memory; please increase ulimit\n"); \ + } + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} +#else +#define PAGE_SIZE 1 /* unknown */ +#define CHECK_SIZE(SIZE) fprintf (stderr, "OS unsupported\n"); + +int +get_pinned_mem () +{ + return 0; +} +#endif + +#include + +int +main () +{ + /* Allocate at least a page each time, but stay within the ulimit. */ + const int SIZE = PAGE_SIZE; + CHECK_SIZE (SIZE*3); + + const omp_alloctrait_t traits[] = { + { omp_atk_pinned, 1 }, + { omp_atk_pool_size, SIZE*8 } + }; + omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space, + 2, traits); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + void *p = omp_alloc (SIZE, allocator); + if (!p) + abort (); + + int amount = get_pinned_mem (); + if (amount == 0) + abort (); + + p = omp_realloc (p, SIZE*2, allocator, allocator); + if (!p) + abort (); + + int amount2 = get_pinned_mem (); + if (amount2 <= amount) + abort (); + + p = omp_calloc (1, SIZE, allocator); + if (!p) + abort (); + + if (get_pinned_mem () <= amount2) + abort (); + + return 0; +} diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-3.c b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c new file mode 100644 index 00000000000..90539ffe3e0 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c @@ -0,0 +1,130 @@ +/* { dg-do run } */ + +/* Test that pinned memory fails correctly. */ + +#include +#include + +#ifdef __linux__ +#include +#include + +#include +#include + +#define PAGE_SIZE sysconf(_SC_PAGESIZE) + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} + +void +set_pin_limit (int size) +{ + struct rlimit limit; + if (getrlimit (RLIMIT_MEMLOCK, &limit)) + abort (); + limit.rlim_cur = (limit.rlim_max < size ? limit.rlim_max : size); + if (setrlimit (RLIMIT_MEMLOCK, &limit)) + abort (); +} +#else +int +#define PAGE_SIZE 10000*1024 /* unknown */ + +get_pinned_mem () +{ + return 0; +} + +void +set_pin_limit () +{ +} +#endif + +#include + +int +main () +{ + /* This needs to be large enough to cover multiple pages. */ + const int SIZE = PAGE_SIZE*4; + + /* Pinned memory, no fallback. */ + const omp_alloctrait_t traits1[] = { + { omp_atk_pinned, 1 }, + { omp_atk_fallback, omp_atv_null_fb } + }; + omp_allocator_handle_t allocator1 = omp_init_allocator (omp_default_mem_space, 2, traits1); + + /* Pinned memory, plain memory fallback. */ + const omp_alloctrait_t traits2[] = { + { omp_atk_pinned, 1 }, + { omp_atk_fallback, omp_atv_default_mem_fb } + }; + omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space, 2, traits2); + + /* Ensure that the limit is smaller than the allocation. */ + set_pin_limit (SIZE/2); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + // Should fail + void *p = omp_alloc (SIZE, allocator1); + if (p) + abort (); + + // Should fail + p = omp_calloc (1, SIZE, allocator1); + if (p) + abort (); + + // Should fall back + p = omp_alloc (SIZE, allocator2); + if (!p) + abort (); + + // Should fall back + p = omp_calloc (1, SIZE, allocator2); + if (!p) + abort (); + + // Should fail to realloc + void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc); + p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc); + if (!notpinned || p) + abort (); + + // Should fall back to no realloc needed + p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc); + if (p != notpinned) + abort (); + + // No memory should have been pinned + int amount = get_pinned_mem (); + if (amount != 0) + abort (); + + return 0; +} diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-4.c b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c new file mode 100644 index 00000000000..534e49eefc4 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c @@ -0,0 +1,132 @@ +/* { dg-do run } */ + +/* Test that pinned memory fails correctly, pool_size code path. */ + +#include +#include + +#ifdef __linux__ +#include +#include + +#include +#include + +#define PAGE_SIZE sysconf(_SC_PAGESIZE) + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + fclose (proc); + return val; + } + } + abort (); +} + +void +set_pin_limit (int size) +{ + struct rlimit limit; + if (getrlimit (RLIMIT_MEMLOCK, &limit)) + abort (); + limit.rlim_cur = (limit.rlim_max < size ? limit.rlim_max : size); + if (setrlimit (RLIMIT_MEMLOCK, &limit)) + abort (); +} +#else +int +#define PAGE_SIZE 10000*1024 /* unknown */ + +get_pinned_mem () +{ + return 0; +} + +void +set_pin_limit () +{ +} +#endif + +#include + +int +main () +{ + /* This needs to be large enough to cover multiple pages. */ + const int SIZE = PAGE_SIZE*4; + + /* Pinned memory, no fallback. */ + const omp_alloctrait_t traits1[] = { + { omp_atk_pinned, 1 }, + { omp_atk_fallback, omp_atv_null_fb }, + { omp_atk_pool_size, SIZE*8 } + }; + omp_allocator_handle_t allocator1 = omp_init_allocator (omp_default_mem_space, 3, traits1); + + /* Pinned memory, plain memory fallback. */ + const omp_alloctrait_t traits2[] = { + { omp_atk_pinned, 1 }, + { omp_atk_fallback, omp_atv_default_mem_fb }, + { omp_atk_pool_size, SIZE*8 } + }; + omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space, 3, traits2); + + /* Ensure that the limit is smaller than the allocation. */ + set_pin_limit (SIZE/2); + + // Sanity check + if (get_pinned_mem () != 0) + abort (); + + // Should fail + void *p = omp_alloc (SIZE, allocator1); + if (p) + abort (); + + // Should fail + p = omp_calloc (1, SIZE, allocator1); + if (p) + abort (); + + // Should fall back + p = omp_alloc (SIZE, allocator2); + if (!p) + abort (); + + // Should fall back + p = omp_calloc (1, SIZE, allocator2); + if (!p) + abort (); + + // Should fail to realloc + void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc); + p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc); + if (!notpinned || p) + abort (); + + // Should fall back to no realloc needed + p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc); + if (p != notpinned) + abort (); + + // No memory should have been pinned + int amount = get_pinned_mem (); + if (amount != 0) + abort (); + + return 0; +}