From: Andrew Stubbs <ams@codesourcery.com>
To: <gcc-patches@gcc.gnu.org>
Subject: [PATCH 08/17] openmp: -foffload-memory=pinned
Date: Thu, 7 Jul 2022 11:34:39 +0100 [thread overview]
Message-ID: <8011a994bb38db60f37127880b0fc682564f6e8d.1657188329.git.ams@codesourcery.com> (raw)
In-Reply-To: <cover.1657188329.git.ams@codesourcery.com>
[-- Attachment #1: Type: text/plain, Size: 1822 bytes --]
Implement the -foffload-memory=pinned option such that libgomp is
instructed to enable fully-pinned memory at start-up. The option is
intended to provide a performance boost to certain offload programs without
modifying the code.
This feature only works on Linux, at present, and simply calls mlockall to
enable always-on memory pinning. It requires that the ulimit feature is
set high enough to accommodate all the program's memory usage.
In this mode the ompx_pinned_memory_alloc feature is disabled as it is not
needed and may conflict.
gcc/ChangeLog:
* omp-builtins.def (BUILT_IN_GOMP_ENABLE_PINNED_MODE): New.
* omp-low.cc (omp_enable_pinned_mode): New function.
(execute_lower_omp): Call omp_enable_pinned_mode.
libgomp/ChangeLog:
* config/linux/allocator.c (always_pinned_mode): New variable.
(GOMP_enable_pinned_mode): New function.
(linux_memspace_alloc): Disable pinning when always_pinned_mode set.
(linux_memspace_calloc): Likewise.
(linux_memspace_free): Likewise.
(linux_memspace_realloc): Likewise.
* libgomp.map: Add GOMP_enable_pinned_mode.
* testsuite/libgomp.c/alloc-pinned-7.c: New test.
gcc/testsuite/ChangeLog:
* c-c++-common/gomp/alloc-pinned-1.c: New test.
---
gcc/omp-builtins.def | 3 +
gcc/omp-low.cc | 66 +++++++++++++++++++
.../c-c++-common/gomp/alloc-pinned-1.c | 28 ++++++++
libgomp/config/linux/allocator.c | 26 ++++++++
libgomp/libgomp.map | 1 +
libgomp/target.c | 4 +-
libgomp/testsuite/libgomp.c/alloc-pinned-7.c | 63 ++++++++++++++++++
7 files changed, 190 insertions(+), 1 deletion(-)
create mode 100644 gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c
create mode 100644 libgomp/testsuite/libgomp.c/alloc-pinned-7.c
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0008-openmp-foffload-memory-pinned.patch --]
[-- Type: text/x-patch; name="0008-openmp-foffload-memory-pinned.patch", Size: 7749 bytes --]
diff --git a/gcc/omp-builtins.def b/gcc/omp-builtins.def
index ee5213eedcf..276dd7812f2 100644
--- a/gcc/omp-builtins.def
+++ b/gcc/omp-builtins.def
@@ -470,3 +470,6 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_WARNING, "GOMP_warning",
BT_FN_VOID_CONST_PTR_SIZE, ATTR_NOTHROW_LEAF_LIST)
DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ERROR, "GOMP_error",
BT_FN_VOID_CONST_PTR_SIZE, ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ENABLE_PINNED_MODE,
+ "GOMP_enable_pinned_mode",
+ BT_FN_VOID, ATTR_NOTHROW_LIST)
diff --git a/gcc/omp-low.cc b/gcc/omp-low.cc
index d73c165f029..ba612e5c67d 100644
--- a/gcc/omp-low.cc
+++ b/gcc/omp-low.cc
@@ -14620,6 +14620,68 @@ lower_omp (gimple_seq *body, omp_context *ctx)
input_location = saved_location;
}
+/* Emit a constructor function to enable -foffload-memory=pinned
+ at runtime. Libgomp handles the OS mode setting, but we need to trigger
+ it by calling GOMP_enable_pinned mode before the program proper runs. */
+
+static void
+omp_enable_pinned_mode ()
+{
+ static bool visited = false;
+ if (visited)
+ return;
+ visited = true;
+
+ /* Create a new function like this:
+
+ static void __attribute__((constructor))
+ __set_pinned_mode ()
+ {
+ GOMP_enable_pinned_mode ();
+ }
+ */
+
+ tree name = get_identifier ("__set_pinned_mode");
+ tree voidfntype = build_function_type_list (void_type_node, NULL_TREE);
+ tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, name, voidfntype);
+
+ TREE_STATIC (decl) = 1;
+ TREE_USED (decl) = 1;
+ DECL_ARTIFICIAL (decl) = 1;
+ DECL_IGNORED_P (decl) = 0;
+ TREE_PUBLIC (decl) = 0;
+ DECL_UNINLINABLE (decl) = 1;
+ DECL_EXTERNAL (decl) = 0;
+ DECL_CONTEXT (decl) = NULL_TREE;
+ DECL_INITIAL (decl) = make_node (BLOCK);
+ BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
+ DECL_STATIC_CONSTRUCTOR (decl) = 1;
+ DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("constructor"),
+ NULL_TREE, NULL_TREE);
+
+ tree t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE,
+ void_type_node);
+ DECL_ARTIFICIAL (t) = 1;
+ DECL_IGNORED_P (t) = 1;
+ DECL_CONTEXT (t) = decl;
+ DECL_RESULT (decl) = t;
+
+ push_struct_function (decl);
+ init_tree_ssa (cfun);
+
+ tree calldecl = builtin_decl_explicit (BUILT_IN_GOMP_ENABLE_PINNED_MODE);
+ gcall *call = gimple_build_call (calldecl, 0);
+
+ gimple_seq seq = NULL;
+ gimple_seq_add_stmt (&seq, call);
+ gimple_set_body (decl, gimple_build_bind (NULL_TREE, seq, NULL));
+
+ cfun->function_end_locus = UNKNOWN_LOCATION;
+ cfun->curr_properties |= PROP_gimple_any;
+ pop_cfun ();
+ cgraph_node::add_new_function (decl, true);
+}
+
/* Main entry point. */
static unsigned int
@@ -14676,6 +14738,10 @@ execute_lower_omp (void)
for (auto task_stmt : task_cpyfns)
finalize_task_copyfn (task_stmt);
task_cpyfns.release ();
+
+ if (flag_offload_memory == OFFLOAD_MEMORY_PINNED)
+ omp_enable_pinned_mode ();
+
return 0;
}
diff --git a/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c b/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c
new file mode 100644
index 00000000000..e0e08019bff
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/alloc-pinned-1.c
@@ -0,0 +1,28 @@
+/* { dg-do run } */
+/* { dg-additional-options "-foffload-memory=pinned" } */
+/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+
+#if __cplusplus
+#define EXTERNC extern "C"
+#else
+#define EXTERNC
+#endif
+
+/* Intercept the libgomp initialization call to check it happens. */
+
+int good = 0;
+
+EXTERNC void
+GOMP_enable_pinned_mode ()
+{
+ good = 1;
+}
+
+int
+main ()
+{
+ if (!good)
+ __builtin_exit (1);
+
+ return 0;
+}
diff --git a/libgomp/config/linux/allocator.c b/libgomp/config/linux/allocator.c
index 18235f59775..e7fe6c3c49a 100644
--- a/libgomp/config/linux/allocator.c
+++ b/libgomp/config/linux/allocator.c
@@ -50,9 +50,26 @@
#include <string.h>
#include "libgomp.h"
+static bool always_pinned_mode = false;
+
+/* This function is called by the compiler when -foffload-memory=pinned
+ is used. */
+
+void
+GOMP_enable_pinned_mode ()
+{
+ if (mlockall (MCL_CURRENT | MCL_FUTURE) != 0)
+ gomp_error ("failed to pin all memory (ulimit too low?)");
+ else
+ always_pinned_mode = true;
+}
+
static void *
linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
{
+ /* Explicit pinning may not be required. */
+ pin = pin && !always_pinned_mode;
+
if (memspace == ompx_unified_shared_mem_space)
{
return gomp_usm_alloc (size);
@@ -80,6 +97,9 @@ linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
static void *
linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin)
{
+ /* Explicit pinning may not be required. */
+ pin = pin && !always_pinned_mode;
+
if (memspace == ompx_unified_shared_mem_space)
{
void *ret = gomp_usm_alloc (size);
@@ -97,6 +117,9 @@ static void
linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size,
int pin)
{
+ /* Explicit pinning may not be required. */
+ pin = pin && !always_pinned_mode;
+
if (memspace == ompx_unified_shared_mem_space)
gomp_usm_free (addr);
else if (pin)
@@ -109,6 +132,9 @@ static void *
linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
size_t oldsize, size_t size, int oldpin, int pin)
{
+ /* Explicit pinning may not be required. */
+ pin = pin && !always_pinned_mode;
+
if (memspace == ompx_unified_shared_mem_space)
goto manual_realloc;
else if (oldpin && pin)
diff --git a/libgomp/libgomp.map b/libgomp/libgomp.map
index 46d5f10f3e1..c86734f15e2 100644
--- a/libgomp/libgomp.map
+++ b/libgomp/libgomp.map
@@ -400,6 +400,7 @@ GOMP_5.0.1 {
global:
GOMP_alloc;
GOMP_free;
+ GOMP_enable_pinned_mode;
} GOMP_5.0;
GOMP_5.1 {
diff --git a/libgomp/target.c b/libgomp/target.c
index 4e203ae3c06..3dd09b7afbd 100644
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -1,3 +1,4 @@
+#include <stdio.h>
/* Copyright (C) 2013-2022 Free Software Foundation, Inc.
Contributed by Jakub Jelinek <jakub@redhat.com>.
@@ -1533,7 +1534,8 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
continue;
}
default:
- if (tgt->list[i].offset == OFFSET_INLINED)
+ if (tgt->list[i].offset == OFFSET_INLINED
+ && !array)
continue;
break;
}
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-7.c b/libgomp/testsuite/libgomp.c/alloc-pinned-7.c
new file mode 100644
index 00000000000..8dc19055038
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-7.c
@@ -0,0 +1,63 @@
+/* { dg-do run } */
+/* { dg-additional-options "-foffload-memory=pinned" } */
+
+/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+
+/* Test that pinned memory works. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+#else
+int
+get_pinned_mem ()
+{
+ return 0;
+}
+
+#define mlockall(...) 0
+#endif
+
+#include <omp.h>
+
+int
+main ()
+{
+ // Sanity check
+ if (get_pinned_mem () == 0)
+ {
+ /* -foffload-memory=pinned has failed, but maybe that's because
+ isufficient pinned memory was available. */
+ if (mlockall (MCL_CURRENT | MCL_FUTURE) == 0)
+ abort ();
+ }
+
+ return 0;
+}
next prev parent reply other threads:[~2022-07-07 10:36 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-07-07 10:34 [PATCH 00/17] openmp, nvptx, amdgcn: 5.0 Memory Allocators Andrew Stubbs
2022-07-07 10:34 ` [PATCH 01/17] libgomp, nvptx: low-latency memory allocator Andrew Stubbs
2022-12-08 11:40 ` Jakub Jelinek
2022-07-07 10:34 ` [PATCH 02/17] libgomp: pinned memory Andrew Stubbs
2022-12-08 12:11 ` Jakub Jelinek
2022-12-08 12:51 ` Andrew Stubbs
2022-12-08 14:02 ` Tobias Burnus
2022-12-08 14:35 ` Andrew Stubbs
2022-12-08 15:02 ` Tobias Burnus
2022-07-07 10:34 ` [PATCH 03/17] libgomp, openmp: Add ompx_pinned_mem_alloc Andrew Stubbs
2022-07-07 10:34 ` [PATCH 04/17] openmp, nvptx: low-lat memory access traits Andrew Stubbs
2022-07-07 10:34 ` [PATCH 05/17] openmp, nvptx: ompx_unified_shared_mem_alloc Andrew Stubbs
2022-07-07 10:34 ` [PATCH 06/17] openmp: Add -foffload-memory Andrew Stubbs
2022-07-07 10:34 ` [PATCH 07/17] openmp: allow requires unified_shared_memory Andrew Stubbs
2022-07-07 10:34 ` Andrew Stubbs [this message]
2022-07-07 11:54 ` [PATCH 08/17] openmp: -foffload-memory=pinned Tobias Burnus
2022-07-07 22:18 ` Andrew Stubbs
2022-07-08 9:00 ` Tobias Burnus
2022-07-08 9:55 ` Andrew Stubbs
2022-07-08 9:57 ` Tobias Burnus
2023-02-20 14:59 ` Prototype 'GOMP_enable_pinned_mode' (was: [PATCH 08/17] openmp: -foffload-memory=pinned) Thomas Schwinge
2022-07-07 10:34 ` [PATCH 09/17] openmp: Use libgomp memory allocation functions with unified shared memory Andrew Stubbs
2022-07-07 10:34 ` [PATCH 10/17] Add parsing support for allocate directive (OpenMP 5.0) Andrew Stubbs
2022-07-07 10:34 ` [PATCH 11/17] Translate " Andrew Stubbs
2022-07-07 10:34 ` [PATCH 12/17] Handle cleanup of omp allocated variables " Andrew Stubbs
2022-07-07 10:34 ` [PATCH 13/17] Gimplify allocate directive " Andrew Stubbs
2022-07-07 10:34 ` [PATCH 14/17] Lower " Andrew Stubbs
2022-07-07 10:34 ` [PATCH 15/17] amdgcn: Support XNACK mode Andrew Stubbs
2022-07-07 10:34 ` [PATCH 16/17] amdgcn, openmp: Auto-detect USM mode and set HSA_XNACK Andrew Stubbs
2022-07-07 10:34 ` [PATCH 17/17] amdgcn: libgomp plugin USM implementation Andrew Stubbs
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=8011a994bb38db60f37127880b0fc682564f6e8d.1657188329.git.ams@codesourcery.com \
--to=ams@codesourcery.com \
--cc=gcc-patches@gcc.gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).