From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1820) id EE355385DC00; Mon, 14 Mar 2022 10:27:02 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org EE355385DC00 Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Hafiz Abid Qadeer To: gcc-cvs@gcc.gnu.org Subject: [gcc/devel/omp/gcc-11] openmp: -foffload-memory=pinned X-Act-Checkin: gcc X-Git-Author: Andrew Stubbs X-Git-Refname: refs/heads/devel/omp/gcc-11 X-Git-Oldrev: 37e77cae79aeb42052453c62d8ea95076f1e22d9 X-Git-Newrev: 8ab67f22c79037dc83fdec2412a0ad033e327e48 Message-Id: <20220314102702.EE355385DC00@sourceware.org> Date: Mon, 14 Mar 2022 10:27:02 +0000 (GMT) X-BeenThere: gcc-cvs@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-cvs mailing list List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 14 Mar 2022 10:27:03 -0000 https://gcc.gnu.org/g:8ab67f22c79037dc83fdec2412a0ad033e327e48 commit 8ab67f22c79037dc83fdec2412a0ad033e327e48 Author: Andrew Stubbs Date: Fri Mar 11 12:58:38 2022 +0000 openmp: -foffload-memory=pinned Implement the -foffload-memory=pinned option such that libgomp is instructed to enable fully-pinned memory at start-up. The option is intended to provide a performance boost to certain offload programs without modifying the code. This feature only works on Linux, at present, and simply calls mlockall to enable always-on memory pinning. It requires that the ulimit feature is set high enough to accommodate all the program's memory usage. In this mode the ompx_pinned_memory_alloc feature is disabled as it is not needed and may conflict. Backport of the patch posted at https://gcc.gnu.org/pipermail/gcc-patches/2022-March/591354.html gcc/ChangeLog: * omp-low.c (omp_enable_pinned_mode): New function. (execute_lower_omp): Call omp_enable_pinned_mode. libgomp/ChangeLog: * config/linux/allocator.c (always_pinned_mode): New variable. (GOMP_enable_pinned_mode): New function. (linux_memspace_alloc): Disable pinning when always_pinned_mode set. (linux_memspace_calloc): Likewise. (linux_memspace_free): Likewise. (linux_memspace_realloc): Likewise. * libgomp.map (GOMP_5.1.1): New version space with GOMP_enable_pinned_mode. * testsuite/libgomp.c/alloc-pinned-7.c: New test. gcc/testsuite/ChangeLog: * c-c++-common/gomp/alloc-pinned-1.c: New test. Diff: --- gcc/ChangeLog.omp | 8 +++ gcc/omp-low.c | 68 ++++++++++++++++++++++++ gcc/testsuite/ChangeLog.omp | 7 +++ gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c | 28 ++++++++++ libgomp/ChangeLog.omp | 15 ++++++ libgomp/config/linux/allocator.c | 26 +++++++++ libgomp/libgomp.map | 5 ++ libgomp/testsuite/libgomp.c/alloc-pinned-7.c | 66 +++++++++++++++++++++++ 8 files changed, 223 insertions(+) diff --git a/gcc/ChangeLog.omp b/gcc/ChangeLog.omp index d740c198851..d44c3b5df7b 100644 --- a/gcc/ChangeLog.omp +++ b/gcc/ChangeLog.omp @@ -1,3 +1,11 @@ +2022-03-11 Andrew Stubbs + + Backport of the patch posted at + https://gcc.gnu.org/pipermail/gcc-patches/2022-March/591354.html + + * omp-low.c (omp_enable_pinned_mode): New function. + (execute_lower_omp): Call omp_enable_pinned_mode. + 2022-03-11 Abid Qadeer Backport of a patch posted at diff --git a/gcc/omp-low.c b/gcc/omp-low.c index 2adee3f4caa..f7ecfb52c73 100644 --- a/gcc/omp-low.c +++ b/gcc/omp-low.c @@ -15383,6 +15383,70 @@ lower_omp (gimple_seq *body, omp_context *ctx) input_location = saved_location; } +/* Emit a constructor function to enable -foffload-memory=pinned + at runtime. Libgomp handles the OS mode setting, but we need to trigger + it by calling GOMP_enable_pinned mode before the program proper runs. */ + +static void +omp_enable_pinned_mode () +{ + static bool visited = false; + if (visited) + return; + visited = true; + + /* Create a new function like this: + + static void __attribute__((constructor)) + __set_pinned_mode () + { + GOMP_enable_pinned_mode (); + } + */ + + tree name = get_identifier ("__set_pinned_mode"); + tree voidfntype = build_function_type_list (void_type_node, NULL_TREE); + tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, name, voidfntype); + + TREE_STATIC (decl) = 1; + TREE_USED (decl) = 1; + DECL_ARTIFICIAL (decl) = 1; + DECL_IGNORED_P (decl) = 0; + TREE_PUBLIC (decl) = 0; + DECL_UNINLINABLE (decl) = 1; + DECL_EXTERNAL (decl) = 0; + DECL_CONTEXT (decl) = NULL_TREE; + DECL_INITIAL (decl) = make_node (BLOCK); + BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl; + DECL_STATIC_CONSTRUCTOR (decl) = 1; + DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("constructor"), + NULL_TREE, NULL_TREE); + + tree t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, + void_type_node); + DECL_ARTIFICIAL (t) = 1; + DECL_IGNORED_P (t) = 1; + DECL_CONTEXT (t) = decl; + DECL_RESULT (decl) = t; + + push_struct_function (decl); + init_tree_ssa (cfun); + + tree callname = get_identifier ("GOMP_enable_pinned_mode"); + tree calldecl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, callname, + voidfntype); + gcall *call = gimple_build_call (calldecl, 0); + + gimple_seq seq = NULL; + gimple_seq_add_stmt (&seq, call); + gimple_set_body (decl, gimple_build_bind (NULL_TREE, seq, NULL)); + + cfun->function_end_locus = UNKNOWN_LOCATION; + cfun->curr_properties |= PROP_gimple_any; + pop_cfun (); + cgraph_node::add_new_function (decl, true); +} + /* Main entry point. */ static unsigned int @@ -15435,6 +15499,10 @@ execute_lower_omp (void) && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl))) == POINTER_TYPE)) remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl)); + + if (flag_offload_memory == OFFLOAD_MEMORY_PINNED) + omp_enable_pinned_mode (); + return 0; } diff --git a/gcc/testsuite/ChangeLog.omp b/gcc/testsuite/ChangeLog.omp index 067133e6b89..5d84b27f261 100644 --- a/gcc/testsuite/ChangeLog.omp +++ b/gcc/testsuite/ChangeLog.omp @@ -1,3 +1,10 @@ +2022-03-11 Andrew Stubbs + + Backport of the patch posted at + https://gcc.gnu.org/pipermail/gcc-patches/2022-March/591354.html + + * c-c++-common/gomp/alloc-pinned-1.c: New test. + 2022-03-11 Abid Qadeer Backport of a patch posted at diff --git a/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c b/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c new file mode 100644 index 00000000000..e0e08019bff --- /dev/null +++ b/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c @@ -0,0 +1,28 @@ +/* { dg-do run } */ +/* { dg-additional-options "-foffload-memory=pinned" } */ +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +#if __cplusplus +#define EXTERNC extern "C" +#else +#define EXTERNC +#endif + +/* Intercept the libgomp initialization call to check it happens. */ + +int good = 0; + +EXTERNC void +GOMP_enable_pinned_mode () +{ + good = 1; +} + +int +main () +{ + if (!good) + __builtin_exit (1); + + return 0; +} diff --git a/libgomp/ChangeLog.omp b/libgomp/ChangeLog.omp index 7cdb6d44f4c..328a88c75f3 100644 --- a/libgomp/ChangeLog.omp +++ b/libgomp/ChangeLog.omp @@ -1,3 +1,18 @@ +2022-03-11 Andrew Stubbs + + Backport of the patch posted at + https://gcc.gnu.org/pipermail/gcc-patches/2022-March/591354.html + + * config/linux/allocator.c (always_pinned_mode): New variable. + (GOMP_enable_pinned_mode): New function. + (linux_memspace_alloc): Disable pinning when always_pinned_mode set. + (linux_memspace_calloc): Likewise. + (linux_memspace_free): Likewise. + (linux_memspace_realloc): Likewise. + * libgomp.map (GOMP_5.1.1): New version space with + GOMP_enable_pinned_mode. + * testsuite/libgomp.c/alloc-pinned-7.c: New test. + 2022-03-11 Abid Qadeer Backport of a patch posted at diff --git a/libgomp/config/linux/allocator.c b/libgomp/config/linux/allocator.c index 032922cc068..07af3a2821a 100644 --- a/libgomp/config/linux/allocator.c +++ b/libgomp/config/linux/allocator.c @@ -39,9 +39,26 @@ #include #include "libgomp.h" +static bool always_pinned_mode = false; + +/* This function is called by the compiler when -foffload-memory=pinned + is used. */ + +void +GOMP_enable_pinned_mode () +{ + if (mlockall (MCL_CURRENT | MCL_FUTURE) != 0) + gomp_error ("failed to pin all memory (ulimit too low?)"); + else + always_pinned_mode = true; +} + static void * linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin) { + /* Explicit pinning may not be required. */ + pin = pin && !always_pinned_mode; + if (memspace == ompx_unified_shared_mem_space) { return gomp_usm_alloc (size, GOMP_DEVICE_ICV); @@ -69,6 +86,9 @@ linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin) static void * linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin) { + /* Explicit pinning may not be required. */ + pin = pin && !always_pinned_mode; + if (memspace == ompx_unified_shared_mem_space) { void *ret = gomp_usm_alloc (size, GOMP_DEVICE_ICV); @@ -86,6 +106,9 @@ static void linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size, int pin) { + /* Explicit pinning may not be required. */ + pin = pin && !always_pinned_mode; + if (memspace == ompx_unified_shared_mem_space) gomp_usm_free (addr, GOMP_DEVICE_ICV); else if (pin) @@ -98,6 +121,9 @@ static void * linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr, size_t oldsize, size_t size, int oldpin, int pin) { + /* Explicit pinning may not be required. */ + pin = pin && !always_pinned_mode; + if (memspace == ompx_unified_shared_mem_space) goto manual_realloc; else if (oldpin && pin) diff --git a/libgomp/libgomp.map b/libgomp/libgomp.map index 7ce457ba1d0..fcab0ed855e 100644 --- a/libgomp/libgomp.map +++ b/libgomp/libgomp.map @@ -402,6 +402,11 @@ GOMP_5.1 { GOMP_evaluate_target_device; } GOMP_5.0.1; +GOMP_5.1.1 { + global: + GOMP_enable_pinned_mode; +} GOMP_5.1; + OACC_2.0 { global: acc_get_num_devices; diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-7.c b/libgomp/testsuite/libgomp.c/alloc-pinned-7.c new file mode 100644 index 00000000000..6fd19b46a5c --- /dev/null +++ b/libgomp/testsuite/libgomp.c/alloc-pinned-7.c @@ -0,0 +1,66 @@ +/* { dg-do run } */ +/* { dg-additional-options "-foffload-memory=pinned" } */ + +/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */ + +/* Test that pinned memory works. */ + +#ifdef __linux__ +#include +#include +#include +#include + +#include + +int +get_pinned_mem () +{ + int pid = getpid (); + char buf[100]; + sprintf (buf, "/proc/%d/status", pid); + + FILE *proc = fopen (buf, "r"); + if (!proc) + abort (); + while (fgets (buf, 100, proc)) + { + int val; + if (sscanf (buf, "VmLck: %d", &val)) + { + printf ("lock %d\n", val); + fclose (proc); + return val; + } + } + abort (); +} +#else +int +get_pinned_mem () +{ + return 0; +} + +#define mlockall(...) 0 +#endif + +#include + +/* Allocate more than a page each time, but stay within the ulimit. */ +#define SIZE 10*1024 + +int +main () +{ + // Sanity check + if (get_pinned_mem () == 0) + { + /* -foffload-memory=pinned has failed, but maybe that's because + isufficient pinned memory was available. */ + if (mlockall (MCL_CURRENT | MCL_FUTURE) == 0) + abort (); + } + + return 0; +}