From: "Guillermo E. Martinez" <guillermo.e.martinez@oracle.com>
To: gcc-patches@gcc.gnu.org
Cc: "Guillermo E. Martinez" <guillermo.e.martinez@oracle.com>
Subject: [PATCH] bpf: Add support to eBPF atomic operations
Date: Thu, 9 Sep 2021 12:06:36 -0500 [thread overview]
Message-ID: <20210909170636.365036-1-guillermo.e.martinez@oracle.com> (raw)
Hello people,
This patch add support for atomics operations in eBPF target
using the gcc built-in functions:
__atomic_<operation>_fetch
__atomic_fetch_<operation>
Please if you have comments, don't hesitate to let me know.
Kinds Regards,
Guillermo
eBPF add support for basic atomic operations, the following
gcc built-in functions are implemented for bpf target using
both: 32 and 64 bits data types:
__atomic_fetch_add
__atomic_fetch_sub
__atomic_fetch_and
__atomic_fetch_xor
__atomic_fetch_or
__atomic_exchange
__atomic_compare_exchange_n
Also calls to __atomic_<operation>_fetch are fully supported.
New define instructions were added to bpf.md along with its
tests for gcc atomic built-in functions.
Those operations are fully compliant with:
https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
https://www.kernel.org/doc/Documentation/networking/filter.rst
This support depends of two previous submissions in CGEN and
binutils-gdb projects:
https://sourceware.org/pipermail/cgen/2021q3/002774.html
https://sourceware.org/pipermail/binutils/2021-August/117798.html
gcc/
* config/bpf/bpf.md: Add defines for atomic instructions.
gcc/testsuite/
* gcc.target/bpf/atomic-compare-exchange.c: New test.
* gcc.target/bpf/atomic-exchange.c: Likewise.
* gcc.target/bpf/atomic-fetch-add.c: Likewise.
* gcc.target/bpf/atomic-fetch-and.c: Likewise.
* gcc.target/bpf/atomic-fetch-or.c: Likewise.
* gcc.target/bpf/atomic-fetch-sub.c: Likewise.
* gcc.target/bpf/atomic-fetch-xor.c: Likewise.
---
gcc/config/bpf/bpf.h | 8 +-
gcc/config/bpf/bpf.md | 122 ++++++++++++++++--
gcc/config/bpf/constraints.md | 3 +
.../gcc.target/bpf/atomic-compare-exchange.c | 28 ++++
.../gcc.target/bpf/atomic-exchange.c | 19 +++
.../gcc.target/bpf/atomic-fetch-add.c | 26 ++++
.../gcc.target/bpf/atomic-fetch-and.c | 25 ++++
.../gcc.target/bpf/atomic-fetch-or.c | 25 ++++
.../gcc.target/bpf/atomic-fetch-sub.c | 25 ++++
.../gcc.target/bpf/atomic-fetch-xor.c | 25 ++++
10 files changed, 292 insertions(+), 14 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..f8d0f7fb7dd 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -176,6 +176,7 @@
enum reg_class
{
NO_REGS, /* no registers in set. */
+ R0, /* register r0. */
ALL_REGS, /* all registers. */
LIM_REG_CLASSES /* max value + 1. */
};
@@ -189,6 +190,7 @@ enum reg_class
#define REG_CLASS_NAMES \
{ \
"NO_REGS", \
+ "R0", \
"ALL_REGS" \
}
@@ -202,14 +204,16 @@ enum reg_class
#define REG_CLASS_CONTENTS \
{ \
0x00000000, /* NO_REGS */ \
- 0x00000fff, /* ALL_REGS */ \
+ 0x00000001, /* R0 */ \
+ 0x00000fff, /* ALL_REGS */ \
}
/* A C expression whose value is a register class containing hard
register REGNO. In general there is more that one such class;
choose a class which is "minimal", meaning that no smaller class
also contains the register. */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO) \
+ ((REGNO) == 0 ? R0 : GENERAL_REGS)
/* A macro whose definition is the name of the class to which a
valid base register must belong. A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 03830cc250e..f5a4d2d02b4 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -25,6 +25,11 @@
(define_c_enum "unspec" [
UNSPEC_LDINDABS
UNSPEC_XADD
+ UNSPEC_XAND
+ UNSPEC_XOR
+ UNSPEC_XXOR
+ UNSPEC_XCHG
+ UNSPEC_CMPXCHG
])
;;;; Constants
@@ -56,11 +61,10 @@
;; st generic store instructions for immediates.
;; stx generic store instructions.
;; jmp jump instructions.
-;; xadd atomic exchange-and-add instructions.
;; multi multiword sequence (or user asm statements).
(define_attr "type"
- "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+ "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
(const_string "unknown"))
;; Length of instruction in bytes.
@@ -506,17 +510,111 @@
"ldabs<ldop>\t%0"
[(set_attr "type" "ld")])
-;;;; Atomic increments
+;;;; Atomic operations
(define_mode_iterator AMO [SI DI])
-(define_insn "atomic_add<AMO:mode>"
- [(set (match_operand:AMO 0 "memory_operand" "+m")
- (unspec_volatile:AMO
- [(plus:AMO (match_dup 0)
- (match_operand:AMO 1 "register_operand" "r"))
- (match_operand:SI 2 "const_int_operand")] ;; Memory model.
- UNSPEC_XADD))]
+(define_insn "atomic_fetch_add<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o")
+ (match_operand:AMO 2 "nonmemory_operand" "0")
+ (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+ UNSPEC_XADD))]
+ ""
+ "xadd<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o")
+ (match_operand:AMO 2 "nonmemory_operand" "0")
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_XAND))]
+ ""
+ "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o")
+ (match_operand:AMO 2 "nonmemory_operand" "0")
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_XOR))]
+ ""
+ "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o")
+ (match_operand:AMO 2 "nonmemory_operand" "0")
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_XXOR))]
+ ""
+ "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o")
+ (match_operand:AMO 2 "nonmemory_operand" "0")
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_XCHG))]
+ ""
+ "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+ [(match_operand:SI 0 "register_operand") ;; bool success
+ (match_operand:AMO 1 "register_operand") ;; old value
+ (match_operand:AMO 2 "memory_operand") ;; memory
+ (match_operand:AMO 3 "register_operand") ;; expected
+ (match_operand:AMO 4 "register_operand") ;; desired
+ (match_operand:SI 5 "const_int_operand") ;; is_weak (unused)
+ (match_operand:SI 6 "const_int_operand") ;; success model (unused)
+ (match_operand:SI 7 "const_int_operand")] ;; failure model (unused)
""
- "xadd<mop>\t%0,%1"
- [(set_attr "type" "xadd")])
+{
+ emit_insn
+ (gen_atomic_compare_and_swap<AMO:mode>_1
+ (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+ /* Assume success operation, i.e memory operand
+ is matched with expected value.
+ */
+ emit_move_insn (operands[0], const1_rtx);
+ rtx_code_label *success_label = gen_label_rtx ();
+
+ /* At this point eBPF xcmp was executed, now we can ask
+ * for success exchange, and set suitable value for bool
+ * operand(%0)
+ */
+ emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+ GET_MODE (operands[1]), 1, success_label);
+ emit_move_insn (operands[0], const0_rtx);
+
+ if (success_label)
+ {
+ emit_label (success_label);
+ LABEL_NUSES (success_label) = 1;
+ }
+ DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+ [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+o") ;; memory
+ (match_operand:AMO 2 "register_operand" "0") ;; expected
+ (match_operand:AMO 3 "register_operand" "r") ;; desired
+ (match_operand:SI 4 "const_int_operand")] ;; success (unused)
+ UNSPEC_CMPXCHG))]
+ ""
+ "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@
(define_constraint "S"
"A constant call address."
(match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+ "Register r0")
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+ int done;
+
+ done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+ 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+ done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+ (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+ done = __atomic_compare_exchange (&ptr, &expected, &desired,
+ 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+ done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+ (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+ done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+ done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+ long prev;
+
+ __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+ prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+ __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+ prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
new file mode 100644
index 00000000000..2547396d2a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+ long k;
+
+ k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+ k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+ k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+ k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+ k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+ long k;
+
+ k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+ k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+ k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+ k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+ k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+ k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+ k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+ k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+ long k;
+
+ k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+ k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+ k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+ k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+ k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+ k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+ k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+ k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
new file mode 100644
index 00000000000..6e6a16caa3c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+ long k;
+
+ k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+ k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+ k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+ k = __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+ k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+ k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+ long k;
+
+ k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+ k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+ k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+ k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+ k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+ k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+ k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+ k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */
--
2.30.2
next reply other threads:[~2021-09-09 17:06 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-09-09 17:06 Guillermo E. Martinez [this message]
2021-10-26 1:28 ` [PATCH v2] bpf: Add support to eBPF atomic instructions Guillermo E. Martinez
2021-10-26 5:55 ` Andrew Pinski
2021-10-26 12:06 ` [PATCH v3] " Guillermo E. Martinez
2021-10-26 12:25 ` Guillermo E. Martinez
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210909170636.365036-1-guillermo.e.martinez@oracle.com \
--to=guillermo.e.martinez@oracle.com \
--cc=gcc-patches@gcc.gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).