[PATCH] bpf: Add support to eBPF atomic operations

[PATCH] bpf: Add support to eBPF atomic operations

[Guillermo E. Martinez]

Hello people,

This patch add support for atomics operations in eBPF target
using the gcc built-in functions:

  __atomic_<operation>_fetch
  __atomic_fetch_<operation>

Please if you have comments, don't hesitate to let me know.

Kinds Regards,
Guillermo


eBPF add support for basic atomic operations, the following
gcc built-in functions are implemented for bpf target using
both: 32 and 64 bits data types:

 __atomic_fetch_add
 __atomic_fetch_sub
 __atomic_fetch_and
 __atomic_fetch_xor
 __atomic_fetch_or
 __atomic_exchange
 __atomic_compare_exchange_n

Also calls to __atomic_<operation>_fetch are fully supported.

New define instructions were added to bpf.md along with its
tests for gcc atomic built-in functions.

Those operations are fully compliant with:
  https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
  https://www.kernel.org/doc/Documentation/networking/filter.rst

This support depends of two previous submissions in CGEN and
binutils-gdb projects:
   https://sourceware.org/pipermail/cgen/2021q3/002774.html
   https://sourceware.org/pipermail/binutils/2021-August/117798.html

gcc/
  * config/bpf/bpf.md: Add defines for atomic instructions.

gcc/testsuite/
   * gcc.target/bpf/atomic-compare-exchange.c: New test.
   * gcc.target/bpf/atomic-exchange.c: Likewise.
   * gcc.target/bpf/atomic-fetch-add.c: Likewise.
   * gcc.target/bpf/atomic-fetch-and.c: Likewise.
   * gcc.target/bpf/atomic-fetch-or.c: Likewise.
   * gcc.target/bpf/atomic-fetch-sub.c: Likewise.
   * gcc.target/bpf/atomic-fetch-xor.c: Likewise.
---
 gcc/config/bpf/bpf.h                          |   8 +-
 gcc/config/bpf/bpf.md                         | 122 ++++++++++++++++--
 gcc/config/bpf/constraints.md                 |   3 +
 .../gcc.target/bpf/atomic-compare-exchange.c  |  28 ++++
 .../gcc.target/bpf/atomic-exchange.c          |  19 +++
 .../gcc.target/bpf/atomic-fetch-add.c         |  26 ++++
 .../gcc.target/bpf/atomic-fetch-and.c         |  25 ++++
 .../gcc.target/bpf/atomic-fetch-or.c          |  25 ++++
 .../gcc.target/bpf/atomic-fetch-sub.c         |  25 ++++
 .../gcc.target/bpf/atomic-fetch-xor.c         |  25 ++++
 10 files changed, 292 insertions(+), 14 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c

diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..f8d0f7fb7dd 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -176,6 +176,7 @@
 enum reg_class
 {
   NO_REGS,		/* no registers in set.  */
+  R0,		        /* register r0.  */
   ALL_REGS,		/* all registers.  */
   LIM_REG_CLASSES	/* max value + 1.  */
 };
@@ -189,6 +190,7 @@ enum reg_class
 #define REG_CLASS_NAMES				\
 {						\
   "NO_REGS",					\
+  "R0",					\
   "ALL_REGS"					\
 }
 
@@ -202,14 +204,16 @@ enum reg_class
 #define REG_CLASS_CONTENTS			\
 {						\
    0x00000000, /* NO_REGS */			\
-   0x00000fff, /* ALL_REGS */		        \
+   0x00000001, /* R0 */		        \
+   0x00000fff, /* ALL_REGS */		  \
 }
 
 /* A C expression whose value is a register class containing hard
    register REGNO.  In general there is more that one such class;
    choose a class which is "minimal", meaning that no smaller class
    also contains the register.  */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO)          \
+  ((REGNO) == 0 ? R0 : GENERAL_REGS)
 
 /* A macro whose definition is the name of the class to which a
    valid base register must belong.  A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 03830cc250e..f5a4d2d02b4 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -25,6 +25,11 @@
 (define_c_enum "unspec" [
   UNSPEC_LDINDABS
   UNSPEC_XADD
+  UNSPEC_XAND
+  UNSPEC_XOR
+  UNSPEC_XXOR
+  UNSPEC_XCHG
+  UNSPEC_CMPXCHG
 ])
 
 ;;;; Constants
@@ -56,11 +61,10 @@
 ;; st		generic store instructions for immediates.
 ;; stx		generic store instructions.
 ;; jmp		jump instructions.
-;; xadd		atomic exchange-and-add instructions.
 ;; multi	multiword sequence (or user asm statements).
 
 (define_attr "type"
-  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
   (const_string "unknown"))
 
 ;; Length of instruction in bytes.
@@ -506,17 +510,111 @@
   "ldabs<ldop>\t%0"
   [(set_attr "type" "ld")])
 
-;;;; Atomic increments
+;;;; Atomic operations
 
 (define_mode_iterator AMO [SI DI])
 
-(define_insn "atomic_add<AMO:mode>"
-  [(set (match_operand:AMO 0 "memory_operand" "+m")
-        (unspec_volatile:AMO
-         [(plus:AMO (match_dup 0)
-                    (match_operand:AMO 1 "register_operand" "r"))
-          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
-         UNSPEC_XADD))]
+(define_insn "atomic_fetch_add<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+    UNSPEC_XADD))]
+  ""
+  "xadd<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XAND))]
+  ""
+  "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XOR))]
+  ""
+  "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XXOR))]
+  ""
+  "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XCHG))]
+  ""
+  "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+  [(match_operand:SI 0 "register_operand")    ;; bool success
+   (match_operand:AMO 1 "register_operand")   ;; old value
+   (match_operand:AMO 2 "memory_operand")     ;; memory
+   (match_operand:AMO 3 "register_operand")   ;; expected
+   (match_operand:AMO 4 "register_operand")   ;; desired
+   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
+   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
+   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
   ""
-  "xadd<mop>\t%0,%1"
-  [(set_attr "type" "xadd")])
+{
+  emit_insn
+    (gen_atomic_compare_and_swap<AMO:mode>_1
+      (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+  /* Assume success operation, i.e memory operand
+     is matched with expected value.
+   */
+  emit_move_insn (operands[0], const1_rtx);
+  rtx_code_label *success_label = gen_label_rtx ();
+
+  /* At this point eBPF xcmp was executed, now we can ask
+   * for success exchange, and set suitable value for bool
+   * operand(%0)
+   */
+  emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+                           GET_MODE (operands[1]), 1, success_label);
+  emit_move_insn (operands[0], const0_rtx);
+
+  if (success_label)
+    {
+       emit_label (success_label);
+       LABEL_NUSES (success_label) = 1;
+    }
+  DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
+     (match_operand:AMO 2 "register_operand" "0") ;; expected
+     (match_operand:AMO 3 "register_operand" "r") ;; desired
+     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
+    UNSPEC_CMPXCHG))]
+  ""
+  "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@
 (define_constraint "S"
   "A constant call address."
   (match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+  "Register r0")
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+  int done;
+
+  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __atomic_compare_exchange (&ptr, &expected, &desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+  long prev;
+
+  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
new file mode 100644
index 00000000000..2547396d2a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
new file mode 100644
index 00000000000..6e6a16caa3c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */
-- 
2.30.2

[PATCH v2] bpf: Add support to eBPF atomic instructions

[Guillermo E. Martinez]

Hello people,

This patch v2 to add support for atomics operations in eBPF target
using the gcc built-in functions:

  __atomic_<operation>_fetch
  __atomic_fetch_<operation>

This new version restrict/enable the use of `add + fetch' and the
rest of atomic instructions using the -m[no-]atomics option to
generate code running in old/new kernel versions.

Please if you have comments, don't hesitate to let me know.

Kinds Regards,
Guillermo

    eBPF add support for atomic instructions, the following
    gcc built-in functions are implemented for bpf target using
    both: 32 and 64 bits data types:

     __atomic_fetch_add
     __atomic_fetch_sub
     __atomic_fetch_and
     __atomic_fetch_xor
     __atomic_fetch_or
     __atomic_exchange
     __atomic_compare_exchange_n

    Also calls to __atomic_<instruction>_fetch are fully supported.

    New define instructions were added to bpf.md along with its
    tests for gcc atomic built-in functions.

    In order to restrict/enable the use of `add + fetch' and the
    rest of atomic instructions the -m[no-]atomics was added.

    Those instructions are fully compliant with:
      https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
      https://www.kernel.org/doc/Documentation/networking/filter.rst

    This support depends of two previous submissions in CGEN and
    binutils-gdb projects:
       https://sourceware.org/pipermail/cgen/2021q3/002774.html
       https://sourceware.org/pipermail/binutils/2021-August/117798.html

    gcc/
      * config/bpf/bpf.md: Add defines for atomic instructions.
      * config/bpf/bpf.c: Enable atomics by default in ISA v3.
      * config/bpf/bpf.h: Pass option to gas to disabled use of
      atomics (-mno-atomics).
      * config/bpf/bpf.opt: Add -m[no-]atomics option.
      * doc/invoke.texi: Add documentation for -m[no-a]tomics.

    gcc/testsuite/
       * gcc.target/bpf/atomic-compare-exchange.c: New test.
       * gcc.target/bpf/atomic-exchange.c: Likewise.
       * gcc.target/bpf/atomic-add.c: Likewise.
       * gcc.target/bpf/atomic-and.c: Likewise.
       * gcc.target/bpf/atomic-or.c: Likewise.
       * gcc.target/bpf/atomic-sub.c: Likewise.
       * gcc.target/bpf/atomic-xor.c: Likewise.
       * gcc.target/bpf/atomics-disabled.c: Likewise.
       * gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
       * gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
---
 gcc/ChangeLog                                 |   8 +
 gcc/config/bpf/bpf.c                          |   2 +
 gcc/config/bpf/bpf.h                          |  12 +-
 gcc/config/bpf/bpf.md                         | 155 ++++++++++++++++--
 gcc/config/bpf/bpf.opt                        |   4 +
 gcc/config/bpf/constraints.md                 |   3 +
 gcc/doc/invoke.texi                           |  12 +-
 gcc/testsuite/ChangeLog                       |  12 ++
 .../gcc.target/bpf/atomic-add-fetch.c         |  29 ++++
 gcc/testsuite/gcc.target/bpf/atomic-and.c     |  25 +++
 .../gcc.target/bpf/atomic-compare-exchange.c  |  28 ++++
 .../gcc.target/bpf/atomic-exchange.c          |  19 +++
 gcc/testsuite/gcc.target/bpf/atomic-or.c      |  25 +++
 gcc/testsuite/gcc.target/bpf/atomic-sub.c     |  27 +++
 gcc/testsuite/gcc.target/bpf/atomic-xor.c     |  25 +++
 .../gcc.target/bpf/atomics-disabled.c         |  28 ++++
 .../gcc.target/bpf/ftest-mcpuv3-atomics.c     |  36 ++++
 .../gcc.target/bpf/ftest-no-atomics-add.c     |  23 +++
 18 files changed, 458 insertions(+), 15 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-and.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-or.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-sub.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xor.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomics-disabled.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 115f32e5061..782d33908ba 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,11 @@
+2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* config/bpf/bpf.md: Add defines for atomic instructions.
+	* config/bpf/bpf.c: Enable atomics by default in ISA v3.
+	* config/bpf/bpf.h: Pass option to gas to disable use of
+	atomics (-mno-atomics).
+	* config/bpf/bpf.opt: Add -m[no-]atomics option.
+	* doc/invoke.texi: Add documentation for -m[no-]atomics.
+
 2021-10-20  Alex Coplan  <alex.coplan@arm.com>
 
 	* calls.c (initialize_argument_information): Remove some dead
diff --git a/gcc/config/bpf/bpf.c b/gcc/config/bpf/bpf.c
index 82bb698bd91..5f489c829cc 100644
--- a/gcc/config/bpf/bpf.c
+++ b/gcc/config/bpf/bpf.c
@@ -253,6 +253,8 @@ bpf_option_override (void)
   if (bpf_has_jmp32 == -1)
     bpf_has_jmp32 = (bpf_isa >= ISA_V3);
 
+  if (bpf_has_atomics == -1)
+    bpf_has_atomics = (bpf_isa >= ISA_V3);
 }
 
 #undef TARGET_OPTION_OVERRIDE
diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..d886e7e98ce 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -22,7 +22,9 @@
 
 /**** Controlling the Compilation Driver.  */
 
-#define ASM_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf}"
+#define ASM_SPEC \
+  "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf} " \
+  "%{mno-atomics:-mno-atomics} "
 #define LINK_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL}"
 #define LIB_SPEC ""
 #define STARTFILE_SPEC ""
@@ -176,6 +178,7 @@
 enum reg_class
 {
   NO_REGS,		/* no registers in set.  */
+  R0,		        /* register r0.  */
   ALL_REGS,		/* all registers.  */
   LIM_REG_CLASSES	/* max value + 1.  */
 };
@@ -189,6 +192,7 @@ enum reg_class
 #define REG_CLASS_NAMES				\
 {						\
   "NO_REGS",					\
+  "R0",					\
   "ALL_REGS"					\
 }
 
@@ -202,14 +206,16 @@ enum reg_class
 #define REG_CLASS_CONTENTS			\
 {						\
    0x00000000, /* NO_REGS */			\
-   0x00000fff, /* ALL_REGS */		        \
+   0x00000001, /* R0 */		        \
+   0x00000fff, /* ALL_REGS */		  \
 }
 
 /* A C expression whose value is a register class containing hard
    register REGNO.  In general there is more that one such class;
    choose a class which is "minimal", meaning that no smaller class
    also contains the register.  */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO)          \
+  ((REGNO) == 0 ? R0 : GENERAL_REGS)
 
 /* A macro whose definition is the name of the class to which a
    valid base register must belong.  A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 436c8dfa059..84005007e51 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -25,6 +25,11 @@
 (define_c_enum "unspec" [
   UNSPEC_LDINDABS
   UNSPEC_XADD
+  UNSPEC_XAND
+  UNSPEC_XOR
+  UNSPEC_XXOR
+  UNSPEC_XCHG
+  UNSPEC_CMPXCHG
 ])
 
 ;;;; Constants
@@ -56,11 +61,10 @@
 ;; st		generic store instructions for immediates.
 ;; stx		generic store instructions.
 ;; jmp		jump instructions.
-;; xadd		atomic exchange-and-add instructions.
 ;; multi	multiword sequence (or user asm statements).
 
 (define_attr "type"
-  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
   (const_string "unknown"))
 
 ;; Length of instruction in bytes.
@@ -512,17 +516,146 @@
   "ldabs<ldop>\t%0"
   [(set_attr "type" "ld")])
 
-;;;; Atomic increments
+;;;; Atomic instructions
 
 (define_mode_iterator AMO [SI DI])
 
 (define_insn "atomic_add<AMO:mode>"
   [(set (match_operand:AMO 0 "memory_operand" "+m")
-        (unspec_volatile:AMO
-         [(plus:AMO (match_dup 0)
-                    (match_operand:AMO 1 "register_operand" "r"))
-          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
-         UNSPEC_XADD))]
-  ""
-  "xadd<mop>\t%0,%1"
-  [(set_attr "type" "xadd")])
+    (unspec_volatile:AMO
+      [(plus:AMO (match_dup 0)
+        (match_operand:AMO 1 "register_operand" "r"))
+        (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+   UNSPEC_XADD))]
+  ""
+  "xadd<mop>\t%0,%1")
+
+(define_expand "atomic_fetch_add<AMO:mode>"
+  [(match_operand:AMO 0 "register_operand")
+   (match_operand:AMO 1 "memory_operand")
+   (match_operand:AMO 2 "nonmemory_operand")
+   (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+  ""
+  {
+    if (bpf_has_atomics)
+      {
+        emit_insn
+          (gen_atomic_fetch_add<AMO:mode>_1
+            (operands[0], operands[1], operands[2], operands[3]));
+      }
+    else
+        error ("invalid usage of the xadd return value");
+
+    DONE;
+  })
+
+ (define_insn "atomic_fetch_add<AMO:mode>_1"
+   [(set (match_operand:AMO 0 "register_operand" "=r")
+   (unspec_volatile:AMO
+     [(match_operand:AMO 1 "memory_operand" "+o")
+      (match_operand:AMO 2 "nonmemory_operand" "0")
+      (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+     UNSPEC_XADD))]
+   ""
+   "xaddf<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XAND))]
+  "bpf_has_atomics"
+  "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XOR))]
+  "bpf_has_atomics"
+  "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XXOR))]
+  "bpf_has_atomics"
+  "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XCHG))]
+  "bpf_has_atomics"
+  "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+  [(match_operand:SI 0 "register_operand")    ;; bool success
+   (match_operand:AMO 1 "register_operand")   ;; old value
+   (match_operand:AMO 2 "memory_operand")     ;; memory
+   (match_operand:AMO 3 "register_operand")   ;; expected
+   (match_operand:AMO 4 "register_operand")   ;; desired
+   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
+   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
+   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
+  ""
+{
+  if (bpf_has_atomics)
+    {
+      emit_insn
+        (gen_atomic_compare_and_swap<AMO:mode>_1
+          (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+      /* Assume success operation, i.e memory operand
+         is matched with expected value.
+       */
+      emit_move_insn (operands[0], const1_rtx);
+      rtx_code_label *success_label = gen_label_rtx ();
+
+      /* At this point eBPF xcmp was executed, now we can ask
+       * for success exchange, and set suitable value for bool
+       * operand(%0)
+       */
+      emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+                               GET_MODE (operands[1]), 1, success_label);
+      emit_move_insn (operands[0], const0_rtx);
+
+      if (success_label)
+        {
+           emit_label (success_label);
+           LABEL_NUSES (success_label) = 1;
+        }
+      }
+      else
+        error ("unsupported atomic instruction");
+
+  DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
+     (match_operand:AMO 2 "register_operand" "0") ;; expected
+     (match_operand:AMO 3 "register_operand" "r") ;; desired
+     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
+    UNSPEC_CMPXCHG))]
+  "bpf_has_atomics"
+  "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/bpf.opt b/gcc/config/bpf/bpf.opt
index e8b728ca69f..3eea3a75b62 100644
--- a/gcc/config/bpf/bpf.opt
+++ b/gcc/config/bpf/bpf.opt
@@ -146,6 +146,10 @@ mjmp32
 Target Var(bpf_has_jmp32) Init(-1)
 Enable 32-bit jump instructions.
 
+matomics
+Target Var(bpf_has_atomics) Init(-1)
+Enable eBPF atomic instructions.
+
 mcpu=
 Target RejectNegative Joined Var(bpf_isa) Enum(bpf_isa) Init(ISA_V3)
 
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@
 (define_constraint "S"
   "A constant call address."
   (match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+  "Register r0")
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 6d1e328571a..136a1e87fec 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -905,7 +905,7 @@ Objective-C and Objective-C++ Dialects}.
 
 @emph{eBPF Options}
 @gccoptlist{-mbig-endian -mlittle-endian -mkernel=@var{version}
--mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re
+-mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re -matomics
 -mjmpext -mjmp32 -malu32 -mcpu=@var{version}}
 
 @emph{FR30 Options}
@@ -22824,6 +22824,7 @@ All features of v2, plus:
 @itemize @minus
 @item 32-bit jump operations, as in @option{-mjmp32}
 @item 32-bit ALU operations, as in @option{-malu32}
+@item Atomics instructions
 @end itemize
 
 @end table
@@ -22846,6 +22847,15 @@ the restrictions imposed by the BPF architecture:
 @item Save and restore callee-saved registers at function entry and
 exit, respectively.
 @end itemize
+
+@item -matomics
+Enable use of eBPF atomic instructions. This is set by default when
+eBPF ISA is greater or equal to @samp{v3}. If this option is set,
+then  GCC for eBPF can emit a full set of atomic instructions, those
+atomics may not be executed by all kernel versions. This option is not
+enabled by default when eBPF ISA is lower than @samp{v3}, so atomics are
+disabled with the exception of @samp{add} instruction, allowing
+backward compatibility in older kernel versions.
 @end table
 
 @node FR30 Options
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 602b727f1a5..10ac40d88be 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,15 @@
+2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* gcc.target/bpf/atomic-compare-exchange.c: New test.
+	* gcc.target/bpf/atomic-exchange.c: Likewise.
+	* gcc.target/bpf/atomic-add.c: Likewise.
+	* gcc.target/bpf/atomic-and.c: Likewise.
+	* gcc.target/bpf/atomic-or.c: Likewise.
+	* gcc.target/bpf/atomic-sub.c: Likewise.
+	* gcc.target/bpf/atomic-xor.c: Likewise.
+	* gcc.target/bpf/atomics-disabled.c: Likewise.
+	* gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
+	* gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
+
 2021-10-20  Tamar Christina  <tamar.christina@arm.com>
 
 	* gcc.target/aarch64/mvn-cmeq0-1.c: New test.
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
new file mode 100644
index 00000000000..047a0688b00
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+/*
+ * fetch atomic add is only available with -matomics or
+ * -mcpu=v3 option.
+ */
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-and.c b/gcc/testsuite/gcc.target/bpf/atomic-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-and.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+  int done;
+
+  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __atomic_compare_exchange (&ptr, &expected, &desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+  long prev;
+
+  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-or.c b/gcc/testsuite/gcc.target/bpf/atomic-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-or.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
new file mode 100644
index 00000000000..92b95f6f368
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+  __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomics-disabled.c b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
new file mode 100644
index 00000000000..b2bf040a35c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v2" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are disabled when eBPF ISA version
+ * is lower than v3 or -mno-atomics is set.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
new file mode 100644
index 00000000000..4258c8fa300
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v3" } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are enabled by default in eBPF ISA v3,
+ * but those instructions can be removed from ISA v3 adding
+ * -mno-atomics option.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
new file mode 100644
index 00000000000..6cef3bc3bc7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-mno-atomics" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long *val;
+
+/* Assignment of the return value for xadd instruction
+ * is not allowed when atomic instructions are disabled:
+ * -mno-atomics or mcpu=v{1|2}.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
-- 
2.33.0

Re: [PATCH v2] bpf: Add support to eBPF atomic instructions

[Andrew Pinski]

On Mon, Oct 25, 2021 at 6:29 PM Guillermo E. Martinez via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> Hello people,
>
> This patch v2 to add support for atomics operations in eBPF target
> using the gcc built-in functions:
>
>   __atomic_<operation>_fetch
>   __atomic_fetch_<operation>
>
> This new version restrict/enable the use of `add + fetch' and the
> rest of atomic instructions using the -m[no-]atomics option to
> generate code running in old/new kernel versions.
>
> Please if you have comments, don't hesitate to let me know.

Just a small comment, I see you placed the atomics rtl into bpf.md,
most other targets use atomics.md (or sync.md for targets which __sync
was implemented before the __atomic where in) where they place them so
it would be easier to be found.  It might make sense to do the same
here.
Changing that should be easy really.

Thanks,
Andrew Pinski

>
> Kinds Regards,
> Guillermo
>
>     eBPF add support for atomic instructions, the following
>     gcc built-in functions are implemented for bpf target using
>     both: 32 and 64 bits data types:
>
>      __atomic_fetch_add
>      __atomic_fetch_sub
>      __atomic_fetch_and
>      __atomic_fetch_xor
>      __atomic_fetch_or
>      __atomic_exchange
>      __atomic_compare_exchange_n
>
>     Also calls to __atomic_<instruction>_fetch are fully supported.
>
>     New define instructions were added to bpf.md along with its
>     tests for gcc atomic built-in functions.
>
>     In order to restrict/enable the use of `add + fetch' and the
>     rest of atomic instructions the -m[no-]atomics was added.
>
>     Those instructions are fully compliant with:
>       https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
>       https://www.kernel.org/doc/Documentation/networking/filter.rst
>
>     This support depends of two previous submissions in CGEN and
>     binutils-gdb projects:
>        https://sourceware.org/pipermail/cgen/2021q3/002774.html
>        https://sourceware.org/pipermail/binutils/2021-August/117798.html
>
>     gcc/
>       * config/bpf/bpf.md: Add defines for atomic instructions.
>       * config/bpf/bpf.c: Enable atomics by default in ISA v3.
>       * config/bpf/bpf.h: Pass option to gas to disabled use of
>       atomics (-mno-atomics).
>       * config/bpf/bpf.opt: Add -m[no-]atomics option.
>       * doc/invoke.texi: Add documentation for -m[no-a]tomics.
>
>     gcc/testsuite/
>        * gcc.target/bpf/atomic-compare-exchange.c: New test.
>        * gcc.target/bpf/atomic-exchange.c: Likewise.
>        * gcc.target/bpf/atomic-add.c: Likewise.
>        * gcc.target/bpf/atomic-and.c: Likewise.
>        * gcc.target/bpf/atomic-or.c: Likewise.
>        * gcc.target/bpf/atomic-sub.c: Likewise.
>        * gcc.target/bpf/atomic-xor.c: Likewise.
>        * gcc.target/bpf/atomics-disabled.c: Likewise.
>        * gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
>        * gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
> ---
>  gcc/ChangeLog                                 |   8 +
>  gcc/config/bpf/bpf.c                          |   2 +
>  gcc/config/bpf/bpf.h                          |  12 +-
>  gcc/config/bpf/bpf.md                         | 155 ++++++++++++++++--
>  gcc/config/bpf/bpf.opt                        |   4 +
>  gcc/config/bpf/constraints.md                 |   3 +
>  gcc/doc/invoke.texi                           |  12 +-
>  gcc/testsuite/ChangeLog                       |  12 ++
>  .../gcc.target/bpf/atomic-add-fetch.c         |  29 ++++
>  gcc/testsuite/gcc.target/bpf/atomic-and.c     |  25 +++
>  .../gcc.target/bpf/atomic-compare-exchange.c  |  28 ++++
>  .../gcc.target/bpf/atomic-exchange.c          |  19 +++
>  gcc/testsuite/gcc.target/bpf/atomic-or.c      |  25 +++
>  gcc/testsuite/gcc.target/bpf/atomic-sub.c     |  27 +++
>  gcc/testsuite/gcc.target/bpf/atomic-xor.c     |  25 +++
>  .../gcc.target/bpf/atomics-disabled.c         |  28 ++++
>  .../gcc.target/bpf/ftest-mcpuv3-atomics.c     |  36 ++++
>  .../gcc.target/bpf/ftest-no-atomics-add.c     |  23 +++
>  18 files changed, 458 insertions(+), 15 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-and.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-or.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-sub.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xor.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/atomics-disabled.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
>  create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
>
> diff --git a/gcc/ChangeLog b/gcc/ChangeLog
> index 115f32e5061..782d33908ba 100644
> --- a/gcc/ChangeLog
> +++ b/gcc/ChangeLog
> @@ -1,3 +1,11 @@
> +2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
> +       * config/bpf/bpf.md: Add defines for atomic instructions.
> +       * config/bpf/bpf.c: Enable atomics by default in ISA v3.
> +       * config/bpf/bpf.h: Pass option to gas to disable use of
> +       atomics (-mno-atomics).
> +       * config/bpf/bpf.opt: Add -m[no-]atomics option.
> +       * doc/invoke.texi: Add documentation for -m[no-]atomics.
> +
>  2021-10-20  Alex Coplan  <alex.coplan@arm.com>
>
>         * calls.c (initialize_argument_information): Remove some dead
> diff --git a/gcc/config/bpf/bpf.c b/gcc/config/bpf/bpf.c
> index 82bb698bd91..5f489c829cc 100644
> --- a/gcc/config/bpf/bpf.c
> +++ b/gcc/config/bpf/bpf.c
> @@ -253,6 +253,8 @@ bpf_option_override (void)
>    if (bpf_has_jmp32 == -1)
>      bpf_has_jmp32 = (bpf_isa >= ISA_V3);
>
> +  if (bpf_has_atomics == -1)
> +    bpf_has_atomics = (bpf_isa >= ISA_V3);
>  }
>
>  #undef TARGET_OPTION_OVERRIDE
> diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
> index 82be0c3e190..d886e7e98ce 100644
> --- a/gcc/config/bpf/bpf.h
> +++ b/gcc/config/bpf/bpf.h
> @@ -22,7 +22,9 @@
>
>  /**** Controlling the Compilation Driver.  */
>
> -#define ASM_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf}"
> +#define ASM_SPEC \
> +  "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf} " \
> +  "%{mno-atomics:-mno-atomics} "
>  #define LINK_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL}"
>  #define LIB_SPEC ""
>  #define STARTFILE_SPEC ""
> @@ -176,6 +178,7 @@
>  enum reg_class
>  {
>    NO_REGS,             /* no registers in set.  */
> +  R0,                  /* register r0.  */
>    ALL_REGS,            /* all registers.  */
>    LIM_REG_CLASSES      /* max value + 1.  */
>  };
> @@ -189,6 +192,7 @@ enum reg_class
>  #define REG_CLASS_NAMES                                \
>  {                                              \
>    "NO_REGS",                                   \
> +  "R0",                                        \
>    "ALL_REGS"                                   \
>  }
>
> @@ -202,14 +206,16 @@ enum reg_class
>  #define REG_CLASS_CONTENTS                     \
>  {                                              \
>     0x00000000, /* NO_REGS */                   \
> -   0x00000fff, /* ALL_REGS */                  \
> +   0x00000001, /* R0 */                        \
> +   0x00000fff, /* ALL_REGS */            \
>  }
>
>  /* A C expression whose value is a register class containing hard
>     register REGNO.  In general there is more that one such class;
>     choose a class which is "minimal", meaning that no smaller class
>     also contains the register.  */
> -#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
> +#define REGNO_REG_CLASS(REGNO)          \
> +  ((REGNO) == 0 ? R0 : GENERAL_REGS)
>
>  /* A macro whose definition is the name of the class to which a
>     valid base register must belong.  A base register is one used in
> diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
> index 436c8dfa059..84005007e51 100644
> --- a/gcc/config/bpf/bpf.md
> +++ b/gcc/config/bpf/bpf.md
> @@ -25,6 +25,11 @@
>  (define_c_enum "unspec" [
>    UNSPEC_LDINDABS
>    UNSPEC_XADD
> +  UNSPEC_XAND
> +  UNSPEC_XOR
> +  UNSPEC_XXOR
> +  UNSPEC_XCHG
> +  UNSPEC_CMPXCHG
>  ])
>
>  ;;;; Constants
> @@ -56,11 +61,10 @@
>  ;; st          generic store instructions for immediates.
>  ;; stx         generic store instructions.
>  ;; jmp         jump instructions.
> -;; xadd                atomic exchange-and-add instructions.
>  ;; multi       multiword sequence (or user asm statements).
>
>  (define_attr "type"
> -  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
> +  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
>    (const_string "unknown"))
>
>  ;; Length of instruction in bytes.
> @@ -512,17 +516,146 @@
>    "ldabs<ldop>\t%0"
>    [(set_attr "type" "ld")])
>
> -;;;; Atomic increments
> +;;;; Atomic instructions
>
>  (define_mode_iterator AMO [SI DI])
>
>  (define_insn "atomic_add<AMO:mode>"
>    [(set (match_operand:AMO 0 "memory_operand" "+m")
> -        (unspec_volatile:AMO
> -         [(plus:AMO (match_dup 0)
> -                    (match_operand:AMO 1 "register_operand" "r"))
> -          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
> -         UNSPEC_XADD))]
> -  ""
> -  "xadd<mop>\t%0,%1"
> -  [(set_attr "type" "xadd")])
> +    (unspec_volatile:AMO
> +      [(plus:AMO (match_dup 0)
> +        (match_operand:AMO 1 "register_operand" "r"))
> +        (match_operand:SI 2 "const_int_operand")] ;; Memory model.
> +   UNSPEC_XADD))]
> +  ""
> +  "xadd<mop>\t%0,%1")
> +
> +(define_expand "atomic_fetch_add<AMO:mode>"
> +  [(match_operand:AMO 0 "register_operand")
> +   (match_operand:AMO 1 "memory_operand")
> +   (match_operand:AMO 2 "nonmemory_operand")
> +   (match_operand:AMO 3 "const_int_operand")] ;; Memory model
> +  ""
> +  {
> +    if (bpf_has_atomics)
> +      {
> +        emit_insn
> +          (gen_atomic_fetch_add<AMO:mode>_1
> +            (operands[0], operands[1], operands[2], operands[3]));
> +      }
> +    else
> +        error ("invalid usage of the xadd return value");
> +
> +    DONE;
> +  })
> +
> + (define_insn "atomic_fetch_add<AMO:mode>_1"
> +   [(set (match_operand:AMO 0 "register_operand" "=r")
> +   (unspec_volatile:AMO
> +     [(match_operand:AMO 1 "memory_operand" "+o")
> +      (match_operand:AMO 2 "nonmemory_operand" "0")
> +      (match_operand:AMO 3 "const_int_operand")] ;; Memory model
> +     UNSPEC_XADD))]
> +   ""
> +   "xaddf<mop>\t%1,%0")
> +
> +(define_insn "atomic_fetch_and<AMO:mode>"
> +  [(set (match_operand:AMO 0 "register_operand" "=r")
> +  (unspec_volatile:AMO
> +    [(match_operand:AMO 1 "memory_operand" "+o")
> +     (match_operand:AMO 2 "nonmemory_operand" "0")
> +     (match_operand:AMO 3 "const_int_operand")]
> +    UNSPEC_XAND))]
> +  "bpf_has_atomics"
> +  "xand<mop>\t%1,%0")
> +
> +(define_insn "atomic_fetch_or<AMO:mode>"
> +  [(set (match_operand:AMO 0 "register_operand" "=r")
> +  (unspec_volatile:AMO
> +    [(match_operand:AMO 1 "memory_operand" "+o")
> +     (match_operand:AMO 2 "nonmemory_operand" "0")
> +     (match_operand:AMO 3 "const_int_operand")]
> +    UNSPEC_XOR))]
> +  "bpf_has_atomics"
> +  "xor<mop>\t%1,%0")
> +
> +(define_insn "atomic_fetch_xor<AMO:mode>"
> +  [(set (match_operand:AMO 0 "register_operand" "=r")
> +  (unspec_volatile:AMO
> +    [(match_operand:AMO 1 "memory_operand" "+o")
> +     (match_operand:AMO 2 "nonmemory_operand" "0")
> +     (match_operand:AMO 3 "const_int_operand")]
> +    UNSPEC_XXOR))]
> +  "bpf_has_atomics"
> +  "xxor<mop>\t%1,%0")
> +
> +(define_insn "atomic_exchange<AMO:mode>"
> +  [(set (match_operand:AMO 0 "register_operand" "=r")
> +  (unspec_volatile:AMO
> +    [(match_operand:AMO 1 "memory_operand" "+o")
> +     (match_operand:AMO 2 "nonmemory_operand" "0")
> +     (match_operand:AMO 3 "const_int_operand")]
> +    UNSPEC_XCHG))]
> +  "bpf_has_atomics"
> +  "xchg<mop>\t%1,%0")
> +
> +;; eBPF compare and exchange operation atomically compares the
> +;; value addressed by memory operand(%0) with _R0_(%1), so
> +;; there is a constraint to load the expected value in mentioned
> +;; register. If they match it is replaced with desired value(%3).
> +;; In either case, the value that was there before in %0 is
> +;; zero-extended and loaded back to R0.
> +
> +(define_expand "atomic_compare_and_swap<AMO:mode>"
> +  [(match_operand:SI 0 "register_operand")    ;; bool success
> +   (match_operand:AMO 1 "register_operand")   ;; old value
> +   (match_operand:AMO 2 "memory_operand")     ;; memory
> +   (match_operand:AMO 3 "register_operand")   ;; expected
> +   (match_operand:AMO 4 "register_operand")   ;; desired
> +   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
> +   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
> +   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
> +  ""
> +{
> +  if (bpf_has_atomics)
> +    {
> +      emit_insn
> +        (gen_atomic_compare_and_swap<AMO:mode>_1
> +          (operands[1], operands[2], operands[3], operands[4], operands[6]));
> +
> +      /* Assume success operation, i.e memory operand
> +         is matched with expected value.
> +       */
> +      emit_move_insn (operands[0], const1_rtx);
> +      rtx_code_label *success_label = gen_label_rtx ();
> +
> +      /* At this point eBPF xcmp was executed, now we can ask
> +       * for success exchange, and set suitable value for bool
> +       * operand(%0)
> +       */
> +      emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
> +                               GET_MODE (operands[1]), 1, success_label);
> +      emit_move_insn (operands[0], const0_rtx);
> +
> +      if (success_label)
> +        {
> +           emit_label (success_label);
> +           LABEL_NUSES (success_label) = 1;
> +        }
> +      }
> +      else
> +        error ("unsupported atomic instruction");
> +
> +  DONE;
> +})
> +
> +(define_insn "atomic_compare_and_swap<AMO:mode>_1"
> +  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
> +  (unspec_volatile:AMO
> +    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
> +     (match_operand:AMO 2 "register_operand" "0") ;; expected
> +     (match_operand:AMO 3 "register_operand" "r") ;; desired
> +     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
> +    UNSPEC_CMPXCHG))]
> +  "bpf_has_atomics"
> +  "xcmp<mop>\t%1,%3")
> diff --git a/gcc/config/bpf/bpf.opt b/gcc/config/bpf/bpf.opt
> index e8b728ca69f..3eea3a75b62 100644
> --- a/gcc/config/bpf/bpf.opt
> +++ b/gcc/config/bpf/bpf.opt
> @@ -146,6 +146,10 @@ mjmp32
>  Target Var(bpf_has_jmp32) Init(-1)
>  Enable 32-bit jump instructions.
>
> +matomics
> +Target Var(bpf_has_atomics) Init(-1)
> +Enable eBPF atomic instructions.
> +
>  mcpu=
>  Target RejectNegative Joined Var(bpf_isa) Enum(bpf_isa) Init(ISA_V3)
>
> diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
> index 66b7764d775..9606b6e150e 100644
> --- a/gcc/config/bpf/constraints.md
> +++ b/gcc/config/bpf/constraints.md
> @@ -29,3 +29,6 @@
>  (define_constraint "S"
>    "A constant call address."
>    (match_code "const,symbol_ref,label_ref,const_int"))
> +
> +(define_register_constraint "t" "R0"
> +  "Register r0")
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index 6d1e328571a..136a1e87fec 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -905,7 +905,7 @@ Objective-C and Objective-C++ Dialects}.
>
>  @emph{eBPF Options}
>  @gccoptlist{-mbig-endian -mlittle-endian -mkernel=@var{version}
> --mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re
> +-mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re -matomics
>  -mjmpext -mjmp32 -malu32 -mcpu=@var{version}}
>
>  @emph{FR30 Options}
> @@ -22824,6 +22824,7 @@ All features of v2, plus:
>  @itemize @minus
>  @item 32-bit jump operations, as in @option{-mjmp32}
>  @item 32-bit ALU operations, as in @option{-malu32}
> +@item Atomics instructions
>  @end itemize
>
>  @end table
> @@ -22846,6 +22847,15 @@ the restrictions imposed by the BPF architecture:
>  @item Save and restore callee-saved registers at function entry and
>  exit, respectively.
>  @end itemize
> +
> +@item -matomics
> +Enable use of eBPF atomic instructions. This is set by default when
> +eBPF ISA is greater or equal to @samp{v3}. If this option is set,
> +then  GCC for eBPF can emit a full set of atomic instructions, those
> +atomics may not be executed by all kernel versions. This option is not
> +enabled by default when eBPF ISA is lower than @samp{v3}, so atomics are
> +disabled with the exception of @samp{add} instruction, allowing
> +backward compatibility in older kernel versions.
>  @end table
>
>  @node FR30 Options
> diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
> index 602b727f1a5..10ac40d88be 100644
> --- a/gcc/testsuite/ChangeLog
> +++ b/gcc/testsuite/ChangeLog
> @@ -1,3 +1,15 @@
> +2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
> +       * gcc.target/bpf/atomic-compare-exchange.c: New test.
> +       * gcc.target/bpf/atomic-exchange.c: Likewise.
> +       * gcc.target/bpf/atomic-add.c: Likewise.
> +       * gcc.target/bpf/atomic-and.c: Likewise.
> +       * gcc.target/bpf/atomic-or.c: Likewise.
> +       * gcc.target/bpf/atomic-sub.c: Likewise.
> +       * gcc.target/bpf/atomic-xor.c: Likewise.
> +       * gcc.target/bpf/atomics-disabled.c: Likewise.
> +       * gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
> +       * gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
> +
>  2021-10-20  Tamar Christina  <tamar.christina@arm.com>
>
>         * gcc.target/aarch64/mvn-cmeq0-1.c: New test.
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
> new file mode 100644
> index 00000000000..047a0688b00
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
> @@ -0,0 +1,29 @@
> +/* { dg-do compile } */
> +
> +long delta;
> +long *val;
> +
> +/*
> + * fetch atomic add is only available with -matomics or
> + * -mcpu=v3 option.
> + */
> +void
> +foo ()
> +{
> +  volatile long k;
> +
> +  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
> +  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
> +  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
> +/* { dg-final { scan-assembler "xaddfw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-and.c b/gcc/testsuite/gcc.target/bpf/atomic-and.c
> new file mode 100644
> index 00000000000..6cc05a824f6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-and.c
> @@ -0,0 +1,25 @@
> +/* { dg-do compile } */
> +
> +long mask;
> +long *val;
> +
> +void
> +foo ()
> +{
> +  long k;
> +
> +  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
> +
> +  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
> +  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
> +
> +  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
> +  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
> +
> +  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
> +  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xanddw\t.*" } } */
> +/* { dg-final { scan-assembler "xandw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
> new file mode 100644
> index 00000000000..d522697ec9e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
> @@ -0,0 +1,28 @@
> +/* { dg-do compile } */
> +
> +long val;
> +long ptr;
> +long expected;
> +long desired;
> +
> +void
> +foo ()
> +{
> +  int done;
> +
> +  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
> +                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
> +                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +
> +  done = __atomic_compare_exchange (&ptr, &expected, &desired,
> +                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
> +                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +
> +  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
> +  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
> +}
> +
> +/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
> +/* { dg-final { scan-assembler "xcmpw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
> new file mode 100644
> index 00000000000..f4e60568f7f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
> @@ -0,0 +1,19 @@
> +/* { dg-do compile } */
> +
> +long val;
> +long ptr;
> +
> +void
> +foo ()
> +{
> +  long prev;
> +
> +  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
> +  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
> +
> +  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
> +  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xchgdw\t.*" } } */
> +/* { dg-final { scan-assembler "xchgw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-or.c b/gcc/testsuite/gcc.target/bpf/atomic-or.c
> new file mode 100644
> index 00000000000..af9a999e02a
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-or.c
> @@ -0,0 +1,25 @@
> +/* { dg-do compile } */
> +
> +long bits;
> +long *val;
> +
> +void
> +foo ()
> +{
> +  long k;
> +
> +  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
> +  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
> +  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xordw\t.*" } } */
> +/* { dg-final { scan-assembler "xorw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
> new file mode 100644
> index 00000000000..92b95f6f368
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
> @@ -0,0 +1,27 @@
> +/* { dg-do compile } */
> +
> +long delta;
> +long *val;
> +
> +void
> +foo ()
> +{
> +  volatile long k;
> +
> +  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
> +  __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
> +  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xadddw\t.*" } } */
> +/* { dg-final { scan-assembler "xaddw\t.*" } } */
> +/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
> +/* { dg-final { scan-assembler "xaddfw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
> new file mode 100644
> index 00000000000..433600395a6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
> @@ -0,0 +1,25 @@
> +/* { dg-do compile } */
> +
> +long bits;
> +long *val;
> +
> +void
> +foo ()
> +{
> +  long k;
> +
> +  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
> +  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
> +
> +  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
> +  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xxordw\t.*" } } */
> +/* { dg-final { scan-assembler "xxorw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/atomics-disabled.c b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
> new file mode 100644
> index 00000000000..b2bf040a35c
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
> @@ -0,0 +1,28 @@
> +/* { dg-do compile } */
> +/* { dg-options "-mcpu=v2" } */
> +/* { dg-xfail-if "" { bpf-*-* } } */
> +
> +long delta;
> +long bits;
> +long *val;
> +long ptr;
> +
> +/* Atomic instructions are disabled when eBPF ISA version
> + * is lower than v3 or -mno-atomics is set.
> + */
> +
> +void
> +foo ()
> +{
> +  volatile long k;
> +
> +  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
> +
> +  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
> +}
> diff --git a/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
> new file mode 100644
> index 00000000000..4258c8fa300
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
> @@ -0,0 +1,36 @@
> +/* { dg-do compile } */
> +/* { dg-options "-mcpu=v3" } */
> +
> +long delta;
> +long bits;
> +long *val;
> +long ptr;
> +
> +/* Atomic instructions are enabled by default in eBPF ISA v3,
> + * but those instructions can be removed from ISA v3 adding
> + * -mno-atomics option.
> + */
> +
> +void
> +foo ()
> +{
> +  volatile long k;
> +
> +  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
> +  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
> +
> +  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
> +}
> +
> +/* { dg-final { scan-assembler "xadddw\t.*" } } */
> +/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
> +/* { dg-final { scan-assembler "xandw\t.*" } } */
> +/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
> +/* { dg-final { scan-assembler "xchgdw\t.*" } } */
> +/* { dg-final { scan-assembler "xordw\t.*" } } */
> +/* { dg-final { scan-assembler "xxordw\t.*" } } */
> diff --git a/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
> new file mode 100644
> index 00000000000..6cef3bc3bc7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
> @@ -0,0 +1,23 @@
> +/* { dg-do compile } */
> +/* { dg-options "-mno-atomics" } */
> +/* { dg-xfail-if "" { bpf-*-* } } */
> +
> +long delta;
> +long *val;
> +
> +/* Assignment of the return value for xadd instruction
> + * is not allowed when atomic instructions are disabled:
> + * -mno-atomics or mcpu=v{1|2}.
> + */
> +
> +void
> +foo ()
> +{
> +  volatile long k;
> +
> +  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
> +
> +  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
> +  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
> +}
> --
> 2.33.0
>

[PATCH v3] bpf: Add support to eBPF atomic instructions

[Guillermo E. Martinez]

Hello people,

This patch v3 to add support for atomics instructions in eBPF target,
the rtl for eBPF atomics was splitted in a new file: atomic.md,

Thanks Andrew for the comments,

Kind Regards,
Guillermo

    eBPF add support for atomic instructions, the following
    gcc built-in functions are implemented for bpf target using
    both: 32 and 64 bits data types:

     __atomic_fetch_add
     __atomic_fetch_sub
     __atomic_fetch_and
     __atomic_fetch_xor
     __atomic_fetch_or
     __atomic_exchange
     __atomic_compare_exchange_n

    Also calls to __atomic_<instruction>_fetch are fully supported.

    New define instructions were added to bpf.md along with its
    tests for gcc atomic built-in functions.

    In order to restrict/enable the use of `add + fetch' and the
    rest of atomic instructions the -m[no-]atomics was added.

    Those instructions are fully compliant with:
      https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
      https://www.kernel.org/doc/Documentation/networking/filter.rst

    This support depends of two previous submissions in CGEN and
    binutils-gdb projects:
       https://sourceware.org/pipermail/cgen/2021q3/002774.html
       https://sourceware.org/pipermail/binutils/2021-August/117798.html

    gcc/
      * config/bpf/bpf.md: Add defines for atomic instructions.
      * config/bpf/bpf.c: Enable atomics by default in ISA v3.
      * config/bpf/bpf.h: Pass option to gas to disabled use of
      atomics (-mno-atomics).
      * config/bpf/bpf.opt: Add -m[no-]atomics option.
      * doc/invoke.texi: Add documentation for -m[no-a]tomics.

    gcc/testsuite/
       * gcc.target/bpf/atomic-compare-exchange.c: New test.
       * gcc.target/bpf/atomic-exchange.c: Likewise.
       * gcc.target/bpf/atomic-add.c: Likewise.
       * gcc.target/bpf/atomic-and.c: Likewise.
       * gcc.target/bpf/atomic-or.c: Likewise.
       * gcc.target/bpf/atomic-sub.c: Likewise.
       * gcc.target/bpf/atomic-xor.c: Likewise.
       * gcc.target/bpf/atomics-disabled.c: Likewise.
       * gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
       * gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
---
 gcc/ChangeLog                                 |   8 +
 gcc/config/bpf/atomic.md                      | 160 ++++++++++++++++++
 gcc/config/bpf/bpf.c                          |   2 +
 gcc/config/bpf/bpf.h                          |  12 +-
 gcc/config/bpf/bpf.md                         |  24 +--
 gcc/config/bpf/bpf.opt                        |   4 +
 gcc/config/bpf/constraints.md                 |   3 +
 gcc/doc/invoke.texi                           |  12 +-
 gcc/testsuite/ChangeLog                       |  12 ++
 .../gcc.target/bpf/atomic-add-fetch.c         |  29 ++++
 gcc/testsuite/gcc.target/bpf/atomic-and.c     |  25 +++
 .../gcc.target/bpf/atomic-compare-exchange.c  |  28 +++
 .../gcc.target/bpf/atomic-exchange.c          |  19 +++
 gcc/testsuite/gcc.target/bpf/atomic-or.c      |  25 +++
 gcc/testsuite/gcc.target/bpf/atomic-sub.c     |  27 +++
 gcc/testsuite/gcc.target/bpf/atomic-xor.c     |  25 +++
 .../gcc.target/bpf/atomics-disabled.c         |  28 +++
 .../gcc.target/bpf/ftest-mcpuv3-atomics.c     |  36 ++++
 .../gcc.target/bpf/ftest-no-atomics-add.c     |  23 +++
 19 files changed, 482 insertions(+), 20 deletions(-)
 create mode 100644 gcc/config/bpf/atomic.md
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-and.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-or.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-sub.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xor.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomics-disabled.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 115f32e5061..782d33908ba 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,11 @@
+2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* config/bpf/bpf.md: Add defines for atomic instructions.
+	* config/bpf/bpf.c: Enable atomics by default in ISA v3.
+	* config/bpf/bpf.h: Pass option to gas to disable use of
+	atomics (-mno-atomics).
+	* config/bpf/bpf.opt: Add -m[no-]atomics option.
+	* doc/invoke.texi: Add documentation for -m[no-]atomics.
+
 2021-10-20  Alex Coplan  <alex.coplan@arm.com>
 
 	* calls.c (initialize_argument_information): Remove some dead
diff --git a/gcc/config/bpf/atomic.md b/gcc/config/bpf/atomic.md
new file mode 100644
index 00000000000..9afc38f01ae
--- /dev/null
+++ b/gcc/config/bpf/atomic.md
@@ -0,0 +1,160 @@
+;; Machine description for eBPF atomic Instructions.
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_mode_iterator AMO [SI DI])
+
+(define_insn "atomic_add<AMO:mode>"
+  [(set (match_operand:AMO 0 "memory_operand" "+m")
+    (unspec_volatile:AMO
+      [(plus:AMO (match_dup 0)
+        (match_operand:AMO 1 "register_operand" "r"))
+        (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+   UNSPEC_XADD))]
+  ""
+  "xadd<mop>\t%0,%1")
+
+(define_expand "atomic_fetch_add<AMO:mode>"
+  [(match_operand:AMO 0 "register_operand")
+   (match_operand:AMO 1 "memory_operand")
+   (match_operand:AMO 2 "nonmemory_operand")
+   (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+  ""
+  {
+    if (bpf_has_atomics)
+      {
+        emit_insn
+          (gen_atomic_fetch_add<AMO:mode>_1
+            (operands[0], operands[1], operands[2], operands[3]));
+      }
+    else
+        error ("invalid usage of the xadd return value");
+
+    DONE;
+  })
+
+ (define_insn "atomic_fetch_add<AMO:mode>_1"
+   [(set (match_operand:AMO 0 "register_operand" "=r")
+   (unspec_volatile:AMO
+     [(match_operand:AMO 1 "memory_operand" "+o")
+      (match_operand:AMO 2 "nonmemory_operand" "0")
+      (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+     UNSPEC_XADD))]
+   ""
+   "xaddf<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XAND))]
+  "bpf_has_atomics"
+  "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XOR))]
+  "bpf_has_atomics"
+  "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XXOR))]
+  "bpf_has_atomics"
+  "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XCHG))]
+  "bpf_has_atomics"
+  "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+  [(match_operand:SI 0 "register_operand")    ;; bool success
+   (match_operand:AMO 1 "register_operand")   ;; old value
+   (match_operand:AMO 2 "memory_operand")     ;; memory
+   (match_operand:AMO 3 "register_operand")   ;; expected
+   (match_operand:AMO 4 "register_operand")   ;; desired
+   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
+   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
+   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
+  ""
+{
+  if (bpf_has_atomics)
+    {
+      emit_insn
+        (gen_atomic_compare_and_swap<AMO:mode>_1
+          (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+      /* Assume success operation, i.e memory operand
+         is matched with expected value.
+       */
+      emit_move_insn (operands[0], const1_rtx);
+      rtx_code_label *success_label = gen_label_rtx ();
+
+      /* At this point eBPF xcmp was executed, now we can ask
+       * for success exchange, and set suitable value for bool
+       * operand(%0)
+       */
+      emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+                               GET_MODE (operands[1]), 1, success_label);
+      emit_move_insn (operands[0], const0_rtx);
+
+      if (success_label)
+        {
+           emit_label (success_label);
+           LABEL_NUSES (success_label) = 1;
+        }
+      }
+      else
+        error ("unsupported atomic instruction");
+
+  DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
+     (match_operand:AMO 2 "register_operand" "0") ;; expected
+     (match_operand:AMO 3 "register_operand" "r") ;; desired
+     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
+    UNSPEC_CMPXCHG))]
+  "bpf_has_atomics"
+  "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/bpf.c b/gcc/config/bpf/bpf.c
index 82bb698bd91..5f489c829cc 100644
--- a/gcc/config/bpf/bpf.c
+++ b/gcc/config/bpf/bpf.c
@@ -253,6 +253,8 @@ bpf_option_override (void)
   if (bpf_has_jmp32 == -1)
     bpf_has_jmp32 = (bpf_isa >= ISA_V3);
 
+  if (bpf_has_atomics == -1)
+    bpf_has_atomics = (bpf_isa >= ISA_V3);
 }
 
 #undef TARGET_OPTION_OVERRIDE
diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..d886e7e98ce 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -22,7 +22,9 @@
 
 /**** Controlling the Compilation Driver.  */
 
-#define ASM_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf}"
+#define ASM_SPEC \
+  "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf} " \
+  "%{mno-atomics:-mno-atomics} "
 #define LINK_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL}"
 #define LIB_SPEC ""
 #define STARTFILE_SPEC ""
@@ -176,6 +178,7 @@
 enum reg_class
 {
   NO_REGS,		/* no registers in set.  */
+  R0,		        /* register r0.  */
   ALL_REGS,		/* all registers.  */
   LIM_REG_CLASSES	/* max value + 1.  */
 };
@@ -189,6 +192,7 @@ enum reg_class
 #define REG_CLASS_NAMES				\
 {						\
   "NO_REGS",					\
+  "R0",					\
   "ALL_REGS"					\
 }
 
@@ -202,14 +206,16 @@ enum reg_class
 #define REG_CLASS_CONTENTS			\
 {						\
    0x00000000, /* NO_REGS */			\
-   0x00000fff, /* ALL_REGS */		        \
+   0x00000001, /* R0 */		        \
+   0x00000fff, /* ALL_REGS */		  \
 }
 
 /* A C expression whose value is a register class containing hard
    register REGNO.  In general there is more that one such class;
    choose a class which is "minimal", meaning that no smaller class
    also contains the register.  */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO)          \
+  ((REGNO) == 0 ? R0 : GENERAL_REGS)
 
 /* A macro whose definition is the name of the class to which a
    valid base register must belong.  A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 436c8dfa059..97b1b02313d 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -25,6 +25,11 @@
 (define_c_enum "unspec" [
   UNSPEC_LDINDABS
   UNSPEC_XADD
+  UNSPEC_XAND
+  UNSPEC_XOR
+  UNSPEC_XXOR
+  UNSPEC_XCHG
+  UNSPEC_CMPXCHG
 ])
 
 ;;;; Constants
@@ -56,11 +61,10 @@
 ;; st		generic store instructions for immediates.
 ;; stx		generic store instructions.
 ;; jmp		jump instructions.
-;; xadd		atomic exchange-and-add instructions.
 ;; multi	multiword sequence (or user asm statements).
 
 (define_attr "type"
-  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
   (const_string "unknown"))
 
 ;; Length of instruction in bytes.
@@ -512,17 +516,5 @@
   "ldabs<ldop>\t%0"
   [(set_attr "type" "ld")])
 
-;;;; Atomic increments
-
-(define_mode_iterator AMO [SI DI])
-
-(define_insn "atomic_add<AMO:mode>"
-  [(set (match_operand:AMO 0 "memory_operand" "+m")
-        (unspec_volatile:AMO
-         [(plus:AMO (match_dup 0)
-                    (match_operand:AMO 1 "register_operand" "r"))
-          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
-         UNSPEC_XADD))]
-  ""
-  "xadd<mop>\t%0,%1"
-  [(set_attr "type" "xadd")])
+;; include atomic instructions
+(include "atomic.md")
diff --git a/gcc/config/bpf/bpf.opt b/gcc/config/bpf/bpf.opt
index e8b728ca69f..3eea3a75b62 100644
--- a/gcc/config/bpf/bpf.opt
+++ b/gcc/config/bpf/bpf.opt
@@ -146,6 +146,10 @@ mjmp32
 Target Var(bpf_has_jmp32) Init(-1)
 Enable 32-bit jump instructions.
 
+matomics
+Target Var(bpf_has_atomics) Init(-1)
+Enable eBPF atomic instructions.
+
 mcpu=
 Target RejectNegative Joined Var(bpf_isa) Enum(bpf_isa) Init(ISA_V3)
 
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@
 (define_constraint "S"
   "A constant call address."
   (match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+  "Register r0")
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 6d1e328571a..136a1e87fec 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -905,7 +905,7 @@ Objective-C and Objective-C++ Dialects}.
 
 @emph{eBPF Options}
 @gccoptlist{-mbig-endian -mlittle-endian -mkernel=@var{version}
--mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re
+-mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re -matomics
 -mjmpext -mjmp32 -malu32 -mcpu=@var{version}}
 
 @emph{FR30 Options}
@@ -22824,6 +22824,7 @@ All features of v2, plus:
 @itemize @minus
 @item 32-bit jump operations, as in @option{-mjmp32}
 @item 32-bit ALU operations, as in @option{-malu32}
+@item Atomics instructions
 @end itemize
 
 @end table
@@ -22846,6 +22847,15 @@ the restrictions imposed by the BPF architecture:
 @item Save and restore callee-saved registers at function entry and
 exit, respectively.
 @end itemize
+
+@item -matomics
+Enable use of eBPF atomic instructions. This is set by default when
+eBPF ISA is greater or equal to @samp{v3}. If this option is set,
+then  GCC for eBPF can emit a full set of atomic instructions, those
+atomics may not be executed by all kernel versions. This option is not
+enabled by default when eBPF ISA is lower than @samp{v3}, so atomics are
+disabled with the exception of @samp{add} instruction, allowing
+backward compatibility in older kernel versions.
 @end table
 
 @node FR30 Options
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 602b727f1a5..10ac40d88be 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,15 @@
+2021-10-25  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* gcc.target/bpf/atomic-compare-exchange.c: New test.
+	* gcc.target/bpf/atomic-exchange.c: Likewise.
+	* gcc.target/bpf/atomic-add.c: Likewise.
+	* gcc.target/bpf/atomic-and.c: Likewise.
+	* gcc.target/bpf/atomic-or.c: Likewise.
+	* gcc.target/bpf/atomic-sub.c: Likewise.
+	* gcc.target/bpf/atomic-xor.c: Likewise.
+	* gcc.target/bpf/atomics-disabled.c: Likewise.
+	* gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
+	* gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
+
 2021-10-20  Tamar Christina  <tamar.christina@arm.com>
 
 	* gcc.target/aarch64/mvn-cmeq0-1.c: New test.
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
new file mode 100644
index 00000000000..047a0688b00
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+/*
+ * fetch atomic add is only available with -matomics or
+ * -mcpu=v3 option.
+ */
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-and.c b/gcc/testsuite/gcc.target/bpf/atomic-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-and.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+  int done;
+
+  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __atomic_compare_exchange (&ptr, &expected, &desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+  long prev;
+
+  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-or.c b/gcc/testsuite/gcc.target/bpf/atomic-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-or.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
new file mode 100644
index 00000000000..92b95f6f368
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+  __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomics-disabled.c b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
new file mode 100644
index 00000000000..b2bf040a35c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v2" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are disabled when eBPF ISA version
+ * is lower than v3 or -mno-atomics is set.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
new file mode 100644
index 00000000000..4258c8fa300
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v3" } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are enabled by default in eBPF ISA v3,
+ * but those instructions can be removed from ISA v3 adding
+ * -mno-atomics option.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
new file mode 100644
index 00000000000..6cef3bc3bc7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-mno-atomics" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long *val;
+
+/* Assignment of the return value for xadd instruction
+ * is not allowed when atomic instructions are disabled:
+ * -mno-atomics or mcpu=v{1|2}.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
-- 
2.33.0

[PATCH v3] bpf: Add support to eBPF atomic instructions

[Guillermo E. Martinez]

Apologies, I forgot update the ChangeLog(s)

Kind Regards,
Guillermo


    eBPF add support for atomic instructions, the following
    gcc built-in functions are implemented for bpf target using
    both: 32 and 64 bits data types:

     __atomic_fetch_add
     __atomic_fetch_sub
     __atomic_fetch_and
     __atomic_fetch_xor
     __atomic_fetch_or
     __atomic_exchange
     __atomic_compare_exchange_n

    Also calls to __atomic_<instruction>_fetch are fully supported.

    New define instructions were added to bpf.md along with its
    tests for gcc atomic built-in functions.

    In order to restrict/enable the use of `add + fetch' and the
    rest of atomic instructions the -m[no-]atomics was added.

    Those instructions are fully compliant with:
      https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
      https://www.kernel.org/doc/Documentation/networking/filter.rst

    This support depends of two previous submissions in CGEN and
    binutils-gdb projects:
       https://sourceware.org/pipermail/cgen/2021q3/002774.html
       https://sourceware.org/pipermail/binutils/2021-August/117798.html

    gcc/
      * config/bpf/atomic.md: New file for atomic instructions.
      * config/bpf/bpf.md: Move rtl atomic add definition to atomic.md
      * config/bpf/bpf.c: Enable atomics by default in ISA v3.
      * config/bpf/bpf.h: Pass option to gas to disabled use of
      atomics (-mno-atomics).
      * config/bpf/bpf.opt: Add -m[no-]atomics option.
      * doc/invoke.texi: Add documentation for -m[no-a]tomics.

    gcc/testsuite/
       * gcc.target/bpf/atomic-compare-exchange.c: New test.
       * gcc.target/bpf/atomic-exchange.c: Likewise.
       * gcc.target/bpf/atomic-add.c: Likewise.
       * gcc.target/bpf/atomic-and.c: Likewise.
       * gcc.target/bpf/atomic-or.c: Likewise.
       * gcc.target/bpf/atomic-sub.c: Likewise.
       * gcc.target/bpf/atomic-xor.c: Likewise.
       * gcc.target/bpf/atomics-disabled.c: Likewise.
       * gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
       * gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
---
 gcc/ChangeLog                                 |   9 +
 gcc/config/bpf/atomic.md                      | 169 ++++++++++++++++++
 gcc/config/bpf/bpf.c                          |   2 +
 gcc/config/bpf/bpf.h                          |  12 +-
 gcc/config/bpf/bpf.md                         |  20 +--
 gcc/config/bpf/bpf.opt                        |   4 +
 gcc/config/bpf/constraints.md                 |   3 +
 gcc/doc/invoke.texi                           |  12 +-
 gcc/testsuite/ChangeLog                       |  12 ++
 .../gcc.target/bpf/atomic-add-fetch.c         |  29 +++
 gcc/testsuite/gcc.target/bpf/atomic-and.c     |  25 +++
 .../gcc.target/bpf/atomic-compare-exchange.c  |  28 +++
 .../gcc.target/bpf/atomic-exchange.c          |  19 ++
 gcc/testsuite/gcc.target/bpf/atomic-or.c      |  25 +++
 gcc/testsuite/gcc.target/bpf/atomic-sub.c     |  27 +++
 gcc/testsuite/gcc.target/bpf/atomic-xor.c     |  25 +++
 .../gcc.target/bpf/atomics-disabled.c         |  28 +++
 .../gcc.target/bpf/ftest-mcpuv3-atomics.c     |  36 ++++
 .../gcc.target/bpf/ftest-no-atomics-add.c     |  23 +++
 19 files changed, 487 insertions(+), 21 deletions(-)
 create mode 100644 gcc/config/bpf/atomic.md
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-and.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-or.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-sub.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xor.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomics-disabled.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 115f32e5061..af52f9fcc03 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,12 @@
+2021-10-26  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* config/bpf/atomic.md: New file for atomic instructions.
+	* config/bpf/bpf.md: Move rtl atomic add definition to atomic.md
+	* config/bpf/bpf.c: Enable atomics by default in ISA v3.
+	* config/bpf/bpf.h: Pass option to gas to disable use of
+	atomics (-mno-atomics).
+	* config/bpf/bpf.opt: Add -m[no-]atomics option.
+	* doc/invoke.texi: Add documentation for -m[no-]atomics.
+
 2021-10-20  Alex Coplan  <alex.coplan@arm.com>
 
 	* calls.c (initialize_argument_information): Remove some dead
diff --git a/gcc/config/bpf/atomic.md b/gcc/config/bpf/atomic.md
new file mode 100644
index 00000000000..f8f9a6b96c1
--- /dev/null
+++ b/gcc/config/bpf/atomic.md
@@ -0,0 +1,169 @@
+;; Machine description for eBPF atomic Instructions.
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_c_enum "unspec" [
+  UNSPEC_XADD
+  UNSPEC_XAND
+  UNSPEC_XOR
+  UNSPEC_XXOR
+  UNSPEC_XCHG
+  UNSPEC_CMPXCHG
+])
+
+(define_mode_iterator AMO [SI DI])
+
+(define_insn "atomic_add<AMO:mode>"
+  [(set (match_operand:AMO 0 "memory_operand" "+m")
+    (unspec_volatile:AMO
+      [(plus:AMO (match_dup 0)
+        (match_operand:AMO 1 "register_operand" "r"))
+        (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+   UNSPEC_XADD))]
+  ""
+  "xadd<mop>\t%0,%1")
+
+(define_expand "atomic_fetch_add<AMO:mode>"
+  [(match_operand:AMO 0 "register_operand")
+   (match_operand:AMO 1 "memory_operand")
+   (match_operand:AMO 2 "nonmemory_operand")
+   (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+  ""
+  {
+    if (bpf_has_atomics)
+      {
+        emit_insn
+          (gen_atomic_fetch_add<AMO:mode>_1
+            (operands[0], operands[1], operands[2], operands[3]));
+      }
+    else
+        error ("invalid usage of the xadd return value");
+
+    DONE;
+  })
+
+ (define_insn "atomic_fetch_add<AMO:mode>_1"
+   [(set (match_operand:AMO 0 "register_operand" "=r")
+   (unspec_volatile:AMO
+     [(match_operand:AMO 1 "memory_operand" "+o")
+      (match_operand:AMO 2 "nonmemory_operand" "0")
+      (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+     UNSPEC_XADD))]
+   ""
+   "xaddf<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XAND))]
+  "bpf_has_atomics"
+  "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XOR))]
+  "bpf_has_atomics"
+  "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XXOR))]
+  "bpf_has_atomics"
+  "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XCHG))]
+  "bpf_has_atomics"
+  "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+  [(match_operand:SI 0 "register_operand")    ;; bool success
+   (match_operand:AMO 1 "register_operand")   ;; old value
+   (match_operand:AMO 2 "memory_operand")     ;; memory
+   (match_operand:AMO 3 "register_operand")   ;; expected
+   (match_operand:AMO 4 "register_operand")   ;; desired
+   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
+   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
+   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
+  ""
+{
+  if (bpf_has_atomics)
+    {
+      emit_insn
+        (gen_atomic_compare_and_swap<AMO:mode>_1
+          (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+      /* Assume success operation, i.e memory operand
+         is matched with expected value.
+       */
+      emit_move_insn (operands[0], const1_rtx);
+      rtx_code_label *success_label = gen_label_rtx ();
+
+      /* At this point eBPF xcmp was executed, now we can ask
+       * for success exchange, and set suitable value for bool
+       * operand(%0)
+       */
+      emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+                               GET_MODE (operands[1]), 1, success_label);
+      emit_move_insn (operands[0], const0_rtx);
+
+      if (success_label)
+        {
+           emit_label (success_label);
+           LABEL_NUSES (success_label) = 1;
+        }
+      }
+      else
+        error ("unsupported atomic instruction");
+
+  DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
+     (match_operand:AMO 2 "register_operand" "0") ;; expected
+     (match_operand:AMO 3 "register_operand" "r") ;; desired
+     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
+    UNSPEC_CMPXCHG))]
+  "bpf_has_atomics"
+  "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/bpf.c b/gcc/config/bpf/bpf.c
index 82bb698bd91..5f489c829cc 100644
--- a/gcc/config/bpf/bpf.c
+++ b/gcc/config/bpf/bpf.c
@@ -253,6 +253,8 @@ bpf_option_override (void)
   if (bpf_has_jmp32 == -1)
     bpf_has_jmp32 = (bpf_isa >= ISA_V3);
 
+  if (bpf_has_atomics == -1)
+    bpf_has_atomics = (bpf_isa >= ISA_V3);
 }
 
 #undef TARGET_OPTION_OVERRIDE
diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..d886e7e98ce 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -22,7 +22,9 @@
 
 /**** Controlling the Compilation Driver.  */
 
-#define ASM_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf}"
+#define ASM_SPEC \
+  "%{mbig-endian:-EB} %{!mbig-endian:-EL} %{mxbpf:-mxbpf} " \
+  "%{mno-atomics:-mno-atomics} "
 #define LINK_SPEC "%{mbig-endian:-EB} %{!mbig-endian:-EL}"
 #define LIB_SPEC ""
 #define STARTFILE_SPEC ""
@@ -176,6 +178,7 @@
 enum reg_class
 {
   NO_REGS,		/* no registers in set.  */
+  R0,		        /* register r0.  */
   ALL_REGS,		/* all registers.  */
   LIM_REG_CLASSES	/* max value + 1.  */
 };
@@ -189,6 +192,7 @@ enum reg_class
 #define REG_CLASS_NAMES				\
 {						\
   "NO_REGS",					\
+  "R0",					\
   "ALL_REGS"					\
 }
 
@@ -202,14 +206,16 @@ enum reg_class
 #define REG_CLASS_CONTENTS			\
 {						\
    0x00000000, /* NO_REGS */			\
-   0x00000fff, /* ALL_REGS */		        \
+   0x00000001, /* R0 */		        \
+   0x00000fff, /* ALL_REGS */		  \
 }
 
 /* A C expression whose value is a register class containing hard
    register REGNO.  In general there is more that one such class;
    choose a class which is "minimal", meaning that no smaller class
    also contains the register.  */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO)          \
+  ((REGNO) == 0 ? R0 : GENERAL_REGS)
 
 /* A macro whose definition is the name of the class to which a
    valid base register must belong.  A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 436c8dfa059..fc9f807d1f5 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -24,7 +24,6 @@
 
 (define_c_enum "unspec" [
   UNSPEC_LDINDABS
-  UNSPEC_XADD
 ])
 
 ;;;; Constants
@@ -56,11 +55,10 @@
 ;; st		generic store instructions for immediates.
 ;; stx		generic store instructions.
 ;; jmp		jump instructions.
-;; xadd		atomic exchange-and-add instructions.
 ;; multi	multiword sequence (or user asm statements).
 
 (define_attr "type"
-  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
   (const_string "unknown"))
 
 ;; Length of instruction in bytes.
@@ -512,17 +510,5 @@
   "ldabs<ldop>\t%0"
   [(set_attr "type" "ld")])
 
-;;;; Atomic increments
-
-(define_mode_iterator AMO [SI DI])
-
-(define_insn "atomic_add<AMO:mode>"
-  [(set (match_operand:AMO 0 "memory_operand" "+m")
-        (unspec_volatile:AMO
-         [(plus:AMO (match_dup 0)
-                    (match_operand:AMO 1 "register_operand" "r"))
-          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
-         UNSPEC_XADD))]
-  ""
-  "xadd<mop>\t%0,%1"
-  [(set_attr "type" "xadd")])
+;; include atomic instructions
+(include "atomic.md")
diff --git a/gcc/config/bpf/bpf.opt b/gcc/config/bpf/bpf.opt
index e8b728ca69f..3eea3a75b62 100644
--- a/gcc/config/bpf/bpf.opt
+++ b/gcc/config/bpf/bpf.opt
@@ -146,6 +146,10 @@ mjmp32
 Target Var(bpf_has_jmp32) Init(-1)
 Enable 32-bit jump instructions.
 
+matomics
+Target Var(bpf_has_atomics) Init(-1)
+Enable eBPF atomic instructions.
+
 mcpu=
 Target RejectNegative Joined Var(bpf_isa) Enum(bpf_isa) Init(ISA_V3)
 
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@
 (define_constraint "S"
   "A constant call address."
   (match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+  "Register r0")
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 6d1e328571a..136a1e87fec 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -905,7 +905,7 @@ Objective-C and Objective-C++ Dialects}.
 
 @emph{eBPF Options}
 @gccoptlist{-mbig-endian -mlittle-endian -mkernel=@var{version}
--mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re
+-mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re -matomics
 -mjmpext -mjmp32 -malu32 -mcpu=@var{version}}
 
 @emph{FR30 Options}
@@ -22824,6 +22824,7 @@ All features of v2, plus:
 @itemize @minus
 @item 32-bit jump operations, as in @option{-mjmp32}
 @item 32-bit ALU operations, as in @option{-malu32}
+@item Atomics instructions
 @end itemize
 
 @end table
@@ -22846,6 +22847,15 @@ the restrictions imposed by the BPF architecture:
 @item Save and restore callee-saved registers at function entry and
 exit, respectively.
 @end itemize
+
+@item -matomics
+Enable use of eBPF atomic instructions. This is set by default when
+eBPF ISA is greater or equal to @samp{v3}. If this option is set,
+then  GCC for eBPF can emit a full set of atomic instructions, those
+atomics may not be executed by all kernel versions. This option is not
+enabled by default when eBPF ISA is lower than @samp{v3}, so atomics are
+disabled with the exception of @samp{add} instruction, allowing
+backward compatibility in older kernel versions.
 @end table
 
 @node FR30 Options
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 602b727f1a5..2fe56047263 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,15 @@
+2021-10-26  Guillermo E. Martinez  <guillermo.e.martinez@oracle.com>
+	* gcc.target/bpf/atomic-compare-exchange.c: New test.
+	* gcc.target/bpf/atomic-exchange.c: Likewise.
+	* gcc.target/bpf/atomic-add.c: Likewise.
+	* gcc.target/bpf/atomic-and.c: Likewise.
+	* gcc.target/bpf/atomic-or.c: Likewise.
+	* gcc.target/bpf/atomic-sub.c: Likewise.
+	* gcc.target/bpf/atomic-xor.c: Likewise.
+	* gcc.target/bpf/atomics-disabled.c: Likewise.
+	* gcc.target/bpf/ftest-mcpuv3-atomics.c: Likewise.
+	* gcc.target/bpf/ftest-no-atomics-add.c: Likewise.
+
 2021-10-20  Tamar Christina  <tamar.christina@arm.com>
 
 	* gcc.target/aarch64/mvn-cmeq0-1.c: New test.
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
new file mode 100644
index 00000000000..047a0688b00
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-add-fetch.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+/*
+ * fetch atomic add is only available with -matomics or
+ * -mcpu=v3 option.
+ */
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-and.c b/gcc/testsuite/gcc.target/bpf/atomic-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-and.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+  int done;
+
+  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __atomic_compare_exchange (&ptr, &expected, &desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+  long prev;
+
+  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-or.c b/gcc/testsuite/gcc.target/bpf/atomic-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-or.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
new file mode 100644
index 00000000000..92b95f6f368
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-sub.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+  __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-xor.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomics-disabled.c b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
new file mode 100644
index 00000000000..b2bf040a35c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomics-disabled.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v2" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are disabled when eBPF ISA version
+ * is lower than v3 or -mno-atomics is set.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
new file mode 100644
index 00000000000..4258c8fa300
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-mcpuv3-atomics.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=v3" } */
+
+long delta;
+long bits;
+long *val;
+long ptr;
+
+/* Atomic instructions are enabled by default in eBPF ISA v3,
+ * but those instructions can be removed from ISA v3 adding
+ * -mno-atomics option.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, delta, __ATOMIC_RELAXED);
+
+  __atomic_compare_exchange_n (&ptr, &delta, bits, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  __atomic_exchange(&ptr, &val, &delta, __ATOMIC_RELAXED);
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddfdw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
new file mode 100644
index 00000000000..6cef3bc3bc7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/ftest-no-atomics-add.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-mno-atomics" } */
+/* { dg-xfail-if "" { bpf-*-* } } */
+
+long delta;
+long *val;
+
+/* Assignment of the return value for xadd instruction
+ * is not allowed when atomic instructions are disabled:
+ * -mno-atomics or mcpu=v{1|2}.
+ */
+
+void
+foo ()
+{
+  volatile long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
-- 
2.33.0