[gcc r12-3398] bpf: BPF CO-RE support

[gcc r12-3398] bpf: BPF CO-RE support

[David Faust]

https://gcc.gnu.org/g:8bdabb37549f12ce727800a1c8aa182c0b1dd42a

commit r12-3398-g8bdabb37549f12ce727800a1c8aa182c0b1dd42a
Author: David Faust <david.faust@oracle.com>
Date:   Tue Aug 3 10:27:44 2021 -0700

    bpf: BPF CO-RE support
    
    This commit introduces support for BPF Compile Once - Run
    Everywhere (CO-RE) in GCC.
    
    gcc/ChangeLog:
    
            * config/bpf/bpf.c: Adjust includes.
            (bpf_handle_preserve_access_index_attribute): New function.
            (bpf_attribute_table): Use it here.
            (bpf_builtins): Add BPF_BUILTIN_PRESERVE_ACCESS_INDEX.
            (bpf_option_override): Handle "-mco-re" option.
            (bpf_asm_init_sections): New.
            (TARGET_ASM_INIT_SECTIONS): Redefine.
            (bpf_file_end): New.
            (TARGET_ASM_FILE_END): Redefine.
            (bpf_init_builtins): Add "__builtin_preserve_access_index".
            (bpf_core_compute, bpf_core_get_index): New.
            (is_attr_preserve_access): New.
            (bpf_expand_builtin): Handle new builtins.
            (bpf_core_newdecl, bpf_core_is_maybe_aggregate_access): New.
            (bpf_core_walk): New.
            (bpf_resolve_overloaded_builtin): New.
            (TARGET_RESOLVE_OVERLOADED_BUILTIN): Redefine.
            (handle_attr): New.
            (pass_bpf_core_attr): New RTL pass.
            * config/bpf/bpf-passes.def: New file.
            * config/bpf/bpf-protos.h (make_pass_bpf_core_attr): New.
            * config/bpf/coreout.c: New file.
            * config/bpf/coreout.h: Likewise.
            * config/bpf/t-bpf (TM_H): Add $(srcdir)/config/bpf/coreout.h.
            (coreout.o): New rule.
            (PASSES_EXTRA): Add $(srcdir)/config/bpf/bpf-passes.def.
            * config.gcc (bpf): Add coreout.h to extra_headers.
            Add coreout.o to extra_objs.
            Add $(srcdir)/config/bpf/coreout.c to target_gtfiles.

Diff:
---
 gcc/config.gcc                |   3 +
 gcc/config/bpf/bpf-passes.def |  20 ++
 gcc/config/bpf/bpf-protos.h   |   2 +
 gcc/config/bpf/bpf.c          | 591 ++++++++++++++++++++++++++++++++++++++++++
 gcc/config/bpf/coreout.c      | 356 +++++++++++++++++++++++++
 gcc/config/bpf/coreout.h      | 114 ++++++++
 gcc/config/bpf/t-bpf          |   8 +
 7 files changed, 1094 insertions(+)

diff --git a/gcc/config.gcc b/gcc/config.gcc
index e553ef34bc7..e3e9d8f676f 100644
--- a/gcc/config.gcc
+++ b/gcc/config.gcc
@@ -1525,6 +1525,9 @@ bpf-*-*)
         use_collect2=no
         extra_headers="bpf-helpers.h"
         use_gcc_stdint=provide
+        extra_headers="coreout.h"
+        extra_objs="coreout.o"
+        target_gtfiles="$target_gtfiles \$(srcdir)/config/bpf/coreout.c"
         ;;
 cr16-*-elf)
         tm_file="elfos.h ${tm_file} newlib-stdint.h"
diff --git a/gcc/config/bpf/bpf-passes.def b/gcc/config/bpf/bpf-passes.def
new file mode 100644
index 00000000000..3e961659411
--- /dev/null
+++ b/gcc/config/bpf/bpf-passes.def
@@ -0,0 +1,20 @@
+/* Declaration of target-specific passes for eBPF.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+INSERT_PASS_AFTER (pass_df_initialize_opt, 1, pass_bpf_core_attr);
diff --git a/gcc/config/bpf/bpf-protos.h b/gcc/config/bpf/bpf-protos.h
index aeb512665ed..7ce3386ffda 100644
--- a/gcc/config/bpf/bpf-protos.h
+++ b/gcc/config/bpf/bpf-protos.h
@@ -30,4 +30,6 @@ extern void bpf_print_operand_address (FILE *, rtx);
 extern void bpf_expand_prologue (void);
 extern void bpf_expand_epilogue (void);
 
+rtl_opt_pass * make_pass_bpf_core_attr (gcc::context *);
+
 #endif /* ! GCC_BPF_PROTOS_H */
diff --git a/gcc/config/bpf/bpf.c b/gcc/config/bpf/bpf.c
index 7228978a3a9..01d9c03479e 100644
--- a/gcc/config/bpf/bpf.c
+++ b/gcc/config/bpf/bpf.c
@@ -56,6 +56,24 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 #include "flags.h"
 
+#include "cfg.h" /* needed for struct control_flow_graph used in BB macros */
+#include "gimple.h"
+#include "gimple-iterator.h"
+#include "gimple-walk.h"
+#include "tree-pass.h"
+#include "tree-iterator.h"
+
+#include "context.h"
+#include "pass_manager.h"
+
+#include "gimplify.h"
+#include "gimplify-me.h"
+
+#include "ctfc.h"
+#include "btf.h"
+
+#include "coreout.h"
+
 /* Per-function machine data.  */
 struct GTY(()) machine_function
 {
@@ -105,6 +123,27 @@ bpf_handle_fndecl_attribute (tree *node, tree name,
   return NULL_TREE;
 }
 
+/* Handle preserve_access_index attribute, which can be applied to structs,
+   unions and classes. Actually adding the attribute to the TYPE_DECL is
+   taken care of for us, so just warn for types that aren't supported.  */
+
+static tree
+bpf_handle_preserve_access_index_attribute (tree *node, tree name,
+					    tree args,
+					    int flags,
+					    bool *no_add_attrs)
+{
+  if (TREE_CODE (*node) != RECORD_TYPE && TREE_CODE (*node) != UNION_TYPE)
+    {
+      warning (OPT_Wattributes,
+	       "%qE attribute only applies to structure, union and class types",
+	       name);
+      *no_add_attrs = true;
+    }
+
+  return NULL_TREE;
+}
+
 /* Target-specific attributes.  */
 
 static const struct attribute_spec bpf_attribute_table[] =
@@ -117,6 +156,11 @@ static const struct attribute_spec bpf_attribute_table[] =
  { "kernel_helper", 1, 1, true, false, false, false,
    bpf_handle_fndecl_attribute, NULL },
 
+ /* CO-RE support: attribute to mark that all accesses to the declared
+    struct/union/array should be recorded.  */
+ { "preserve_access_index", 0, -1, false, true, false, true,
+   bpf_handle_preserve_access_index_attribute, NULL },
+
  /* The last attribute spec is set to be NULL.  */
  { NULL,	0,  0, false, false, false, false, NULL, NULL }
 };
@@ -137,11 +181,18 @@ enum bpf_builtins
   BPF_BUILTIN_LOAD_BYTE,
   BPF_BUILTIN_LOAD_HALF,
   BPF_BUILTIN_LOAD_WORD,
+
+  /* Compile Once - Run Everywhere (CO-RE) support.  */
+  BPF_BUILTIN_PRESERVE_ACCESS_INDEX,
+
   BPF_BUILTIN_MAX,
 };
 
 static GTY (()) tree bpf_builtins[(int) BPF_BUILTIN_MAX];
 
+
+void bpf_register_coreattr_pass (void);
+
 /* Initialize the per-function machine status.  */
 
 static struct machine_function *
@@ -183,11 +234,57 @@ bpf_option_override (void)
 
   if (flag_lto && TARGET_BPF_CORE)
     sorry ("BPF CO-RE does not support LTO");
+
+  /* -gbtf implies -mcore when using the BPF backend, unless -mno-co-re
+     is specified.  */
+  if (btf_debuginfo_p () && !(target_flags_explicit & MASK_BPF_CORE))
+    {
+      target_flags |= MASK_BPF_CORE;
+      write_symbols |= BTF_WITH_CORE_DEBUG;
+    }
 }
 
 #undef TARGET_OPTION_OVERRIDE
 #define TARGET_OPTION_OVERRIDE bpf_option_override
 
+/* Return FALSE iff -mcore has been specified.  */
+
+static bool
+ctfc_debuginfo_early_finish_p (void)
+{
+  if (TARGET_BPF_CORE)
+    return false;
+  else
+    return true;
+}
+
+#undef TARGET_CTFC_DEBUGINFO_EARLY_FINISH_P
+#define TARGET_CTFC_DEBUGINFO_EARLY_FINISH_P ctfc_debuginfo_early_finish_p
+
+/* Implement TARGET_ASM_INIT_SECTIONS.  */
+
+static void
+bpf_asm_init_sections (void)
+{
+  if (TARGET_BPF_CORE)
+    btf_ext_init ();
+}
+
+#undef TARGET_ASM_INIT_SECTIONS
+#define TARGET_ASM_INIT_SECTIONS bpf_asm_init_sections
+
+/* Implement TARGET_ASM_FILE_END.  */
+
+static void
+bpf_file_end (void)
+{
+  if (TARGET_BPF_CORE)
+    btf_ext_output ();
+}
+
+#undef TARGET_ASM_FILE_END
+#define TARGET_ASM_FILE_END bpf_file_end
+
 /* Define target-specific CPP macros.  This function in used in the
    definition of TARGET_CPU_CPP_BUILTINS in bpf.h */
 
@@ -837,11 +934,18 @@ bpf_init_builtins (void)
 	       build_function_type_list (ullt, ullt, 0));
   def_builtin ("__builtin_bpf_load_word", BPF_BUILTIN_LOAD_WORD,
 	       build_function_type_list (ullt, ullt, 0));
+  def_builtin ("__builtin_preserve_access_index",
+	       BPF_BUILTIN_PRESERVE_ACCESS_INDEX,
+	       build_function_type_list (ptr_type_node, ptr_type_node, 0));
 }
 
 #undef TARGET_INIT_BUILTINS
 #define TARGET_INIT_BUILTINS bpf_init_builtins
 
+static tree bpf_core_compute (tree, vec<unsigned int> *);
+static int bpf_core_get_index (const tree);
+static bool is_attr_preserve_access (tree);
+
 /* Expand a call to a BPF-specific built-in function that was set up
    with bpf_init_builtins.  */
 
@@ -892,7 +996,75 @@ bpf_expand_builtin (tree exp, rtx target ATTRIBUTE_UNUSED,
       /* The result of the load is in R0.  */
       return gen_rtx_REG (ops[0].mode, BPF_R0);
     }
+  else if (code == -1)
+    {
+      /* A resolved overloaded builtin, e.g. __bpf_preserve_access_index_si */
+      tree arg = CALL_EXPR_ARG (exp, 0);
+
+      if (arg == NULL_TREE)
+	return NULL_RTX;
+
+      auto_vec<unsigned int, 16> accessors;
+      tree container;
 
+      if (TREE_CODE (arg) == SSA_NAME)
+	{
+	  gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
+
+	  if (is_gimple_assign (def_stmt))
+	    arg = gimple_assign_rhs1 (def_stmt);
+	  else
+	    return expand_normal (arg);
+	}
+
+      /* Avoid double-recording information if the argument is an access to
+	 a struct/union marked __attribute__((preserve_access_index)). This
+	 Will be handled by the attribute handling pass.  */
+      if (is_attr_preserve_access (arg))
+	return expand_normal (arg);
+
+      container = bpf_core_compute (arg, &accessors);
+
+      /* Any valid use of the builtin must have at least one access. Otherwise,
+	 there is nothing to record and nothing to do. This is primarily a
+	 guard against optimizations leading to unexpected expressions in the
+	 argument of the builtin. For example, if the builtin is used to read
+	 a field of a structure which can be statically determined to hold a
+	 constant value, the argument to the builtin will be optimized to that
+	 constant. This is OK, and means the builtin call is superfluous.
+	 e.g.
+	   struct S foo;
+	   foo.a = 5;
+	   int x = __preserve_access_index (foo.a);
+	   ... do stuff with x
+	 'foo.a' in the builtin argument will be optimized to '5' with -01+.
+	 This sequence does not warrant recording a CO-RE relocation.  */
+
+      if (accessors.length () < 1)
+	return expand_normal (arg);
+
+      accessors.reverse ();
+
+      container = TREE_TYPE (container);
+
+      rtx_code_label *label = gen_label_rtx ();
+      LABEL_PRESERVE_P (label) = 1;
+      emit_label (label);
+
+      /* Determine what output section this relocation will apply to.
+	 If this function is associated with a section, use that. Otherwise,
+	 fall back on '.text'.  */
+      const char * section_name;
+      if (current_function_decl && DECL_SECTION_NAME (current_function_decl))
+	section_name = DECL_SECTION_NAME (current_function_decl);
+      else
+	section_name = ".text";
+
+      /* Add the CO-RE relocation information to the BTF container.  */
+      bpf_core_reloc_add (container, section_name, &accessors, label);
+
+      return expand_normal (arg);
+    }
   gcc_unreachable ();
 }
 
@@ -946,6 +1118,425 @@ bpf_debug_unwind_info ()
 #undef TARGET_ASM_ALIGNED_DI_OP
 #define TARGET_ASM_ALIGNED_DI_OP "\t.dword\t"
 
+
+/* BPF Compile Once - Run Everywhere (CO-RE) support routines.
+
+   BPF CO-RE is supported in two forms:
+   - A target builtin, __builtin_preserve_access_index
+
+     This builtin accepts a single argument. Any access to an aggregate data
+     structure (struct, union or array) within the argument will be recorded by
+     the CO-RE machinery, resulting in a relocation record being placed in the
+     .BTF.ext section of the output.
+
+     It is implemented in bpf_resolve_overloaded_builtin () and
+     bpf_expand_builtin (), using the supporting routines below.
+
+   - An attribute, __attribute__((preserve_access_index))
+
+     This attribute can be applied to struct and union types. Any access to a
+     type with this attribute will be recorded by the CO-RE machinery.
+
+     The pass pass_bpf_core_attr, below, implements support for
+     this attribute.  */
+
+/* Traverse the subtree under NODE, which is expected to be some form of
+   aggregate access the CO-RE machinery cares about (like a read of a member of
+   a struct or union), collecting access indices for the components and storing
+   them in the vector referenced by ACCESSORS.
+
+   Return the ultimate (top-level) container of the aggregate access. In general,
+   this will be a VAR_DECL or some kind of REF.
+
+   Note that the accessors are computed *in reverse order* of how the BPF
+   CO-RE machinery defines them. The vector needs to be reversed (or simply
+   output in reverse order) for the .BTF.ext relocation information.  */
+
+static tree
+bpf_core_compute (tree node, vec<unsigned int> *accessors)
+{
+
+  if (TREE_CODE (node) == ADDR_EXPR)
+    node = TREE_OPERAND (node, 0);
+
+  else if (TREE_CODE (node) == INDIRECT_REF
+	   || TREE_CODE (node) == POINTER_PLUS_EXPR)
+    {
+      accessors->safe_push (0);
+      return TREE_OPERAND (node, 0);
+    }
+
+  while (1)
+    {
+      switch (TREE_CODE (node))
+	{
+	case COMPONENT_REF:
+	  accessors->safe_push (bpf_core_get_index (TREE_OPERAND (node, 1)));
+	  break;
+
+	case ARRAY_REF:
+	case ARRAY_RANGE_REF:
+	  accessors->safe_push (bpf_core_get_index (node));
+	  break;
+
+	case MEM_REF:
+	  accessors->safe_push (bpf_core_get_index (node));
+	  if (TREE_CODE (TREE_OPERAND (node, 0)) == ADDR_EXPR)
+	    node = TREE_OPERAND (TREE_OPERAND (node, 0), 0);
+	  goto done;
+
+	default:
+	  goto done;
+	}
+      node = TREE_OPERAND (node, 0);
+    }
+ done:
+  return node;
+
+}
+
+/* Compute the index of the NODE in its immediate container.
+   NODE should be a FIELD_DECL (i.e. of struct or union), or an ARRAY_REF. */
+static int
+bpf_core_get_index (const tree node)
+{
+  enum tree_code code = TREE_CODE (node);
+
+  if (code == FIELD_DECL)
+    {
+      /* Lookup the index from the BTF information.  Some struct/union members
+	 may not be emitted in BTF; only the BTF container has enough
+	 information to compute the correct index.  */
+      int idx = bpf_core_get_sou_member_index (ctf_get_tu_ctfc (), node);
+      if (idx >= 0)
+	return idx;
+    }
+
+  else if (code == ARRAY_REF || code == ARRAY_RANGE_REF || code == MEM_REF)
+    {
+      /* For array accesses, the index is operand 1.  */
+      tree index = TREE_OPERAND (node, 1);
+
+      /* If the indexing operand is a constant, extracting is trivial.  */
+      if (TREE_CODE (index) == INTEGER_CST && tree_fits_shwi_p (index))
+	return tree_to_shwi (index);
+    }
+
+  return -1;
+}
+
+/* Synthesize a new builtin function declaration at LOC with signature TYPE.
+   Used by bpf_resolve_overloaded_builtin to resolve calls to
+   __builtin_preserve_access_index.  */
+
+static tree
+bpf_core_newdecl (location_t loc, tree type)
+{
+  tree rettype = build_function_type_list (type, type, NULL);
+  tree newdecl = NULL_TREE;
+  char name[80];
+  int len = snprintf (name, sizeof (name), "%s", "__builtin_pai_");
+
+  static unsigned long cnt = 0;
+  len = snprintf (name + len, sizeof (name) - len, "%lu", cnt++);
+
+  return add_builtin_function_ext_scope (name, rettype, -1, BUILT_IN_MD, NULL,
+					 NULL_TREE);
+}
+
+/* Return whether EXPR could access some aggregate data structure that
+   BPF CO-RE support needs to know about.  */
+
+static int
+bpf_core_is_maybe_aggregate_access (tree expr)
+{
+  enum tree_code code = TREE_CODE (expr);
+  if (code == COMPONENT_REF || code == ARRAY_REF)
+    return 1;
+
+  if (code == ADDR_EXPR)
+      return bpf_core_is_maybe_aggregate_access (TREE_OPERAND (expr, 0));
+
+  return 0;
+}
+
+/* Callback function used with walk_tree from bpf_resolve_overloaded_builtin.  */
+
+static tree
+bpf_core_walk (tree *tp, int *walk_subtrees, void *data)
+{
+  location_t loc = *((location_t *) data);
+
+  /* If this is a type, don't do anything. */
+  if (TYPE_P (*tp))
+    {
+      *walk_subtrees = 0;
+      return NULL_TREE;
+    }
+
+  if (bpf_core_is_maybe_aggregate_access (*tp))
+    {
+      tree newdecl = bpf_core_newdecl (loc, TREE_TYPE (*tp));
+      tree newcall = build_call_expr_loc (loc, newdecl, 1, *tp);
+      *tp = newcall;
+      *walk_subtrees = 0;
+    }
+
+  return NULL_TREE;
+}
+
+
+/* Implement TARGET_RESOLVE_OVERLOADED_BUILTIN (see gccint manual section
+   Target Macros::Misc.).
+   We use this for the __builtin_preserve_access_index builtin for CO-RE
+   support.
+
+   FNDECL is the declaration of the builtin, and ARGLIST is the list of
+   arguments passed to it, and is really a vec<tree,_> *.
+
+   In this case, the 'operation' implemented by the builtin is a no-op;
+   the builtin is just a marker. So, the result is simply the argument.  */
+
+static tree
+bpf_resolve_overloaded_builtin (location_t loc, tree fndecl, void *arglist)
+{
+  if (DECL_MD_FUNCTION_CODE (fndecl) != BPF_BUILTIN_PRESERVE_ACCESS_INDEX)
+    return NULL_TREE;
+
+  /* We only expect one argument, but it may be an arbitrarily-complicated
+     statement-expression. */
+  vec<tree, va_gc> *params = static_cast<vec<tree, va_gc> *> (arglist);
+  unsigned n_params = params ? params->length() : 0;
+
+  if (n_params != 1)
+    {
+      error_at (loc, "expected exactly 1 argument");
+      return NULL_TREE;
+    }
+
+  tree param = (*params)[0];
+
+  /* If not generating BPF_CORE information, the builtin does nothing.  */
+  if (!TARGET_BPF_CORE)
+    return param;
+
+  /* Do remove_c_maybe_const_expr for the arg.
+     TODO: WHY do we have to do this here? Why doesn't c-typeck take care
+     of it before or after this hook? */
+  if (TREE_CODE (param) == C_MAYBE_CONST_EXPR)
+    param = C_MAYBE_CONST_EXPR_EXPR (param);
+
+  /* Construct a new function declaration with the correct type, and return
+     a call to it.
+
+     Calls with statement-expressions, for example:
+     _(({ foo->a = 1; foo->u[2].b = 2; }))
+     require special handling.
+
+     We rearrange this into a new block scope in which each statement
+     becomes a unique builtin call:
+     {
+       _ ({ foo->a = 1;});
+       _ ({ foo->u[2].b = 2;});
+     }
+
+     This ensures that all the relevant information remains within the
+     expression trees the builtin finally gets.  */
+
+  walk_tree (&param, bpf_core_walk, (void *) &loc, NULL);
+
+  return param;
+}
+
+#undef TARGET_RESOLVE_OVERLOADED_BUILTIN
+#define TARGET_RESOLVE_OVERLOADED_BUILTIN bpf_resolve_overloaded_builtin
+
+
+/* Handling for __attribute__((preserve_access_index)) for BPF CO-RE support.
+
+   This attribute marks a structure/union/array type as "preseve", so that
+   every access to that type should be recorded and replayed by the BPF loader;
+   this is just the same functionality as __builtin_preserve_access_index,
+   but in the form of an attribute for an entire aggregate type.
+
+   Note also that nested structs behave as though they all have the attribute.
+   For example:
+     struct X { int a; };
+     struct Y { struct X bar} __attribute__((preserve_access_index));
+     struct Y foo;
+     foo.bar.a;
+   will record access all the way to 'a', even though struct X does not have
+   the preserve_access_index attribute.
+
+   This is to follow LLVM behavior.
+
+   This pass finds all accesses to objects of types marked with the attribute,
+   and wraps them in the same "low-level" builtins used by the builtin version.
+   All logic afterwards is therefore identical to the builtin version of
+   preserve_access_index.  */
+
+/* True iff tree T accesses any member of a struct/union/class which is marked
+   with the PRESERVE_ACCESS_INDEX attribute.  */
+
+static bool
+is_attr_preserve_access (tree t)
+{
+  if (t == NULL_TREE)
+    return false;
+
+  poly_int64 bitsize, bitpos;
+  tree var_off;
+  machine_mode mode;
+  int sign, reverse, vol;
+
+  tree base = get_inner_reference (t, &bitsize, &bitpos, &var_off, &mode,
+				   &sign, &reverse, &vol);
+
+  if (TREE_CODE (base) == MEM_REF)
+    {
+      return lookup_attribute ("preserve_access_index",
+			       TYPE_ATTRIBUTES (TREE_TYPE (base)));
+    }
+
+  if (TREE_CODE (t) == COMPONENT_REF)
+    {
+      /* preserve_access_index propegates into nested structures,
+	 so check whether this is a component of another component
+	 which in turn is part of such a struct.  */
+
+      const tree op = TREE_OPERAND (t, 0);
+
+      if (TREE_CODE (op) == COMPONENT_REF)
+	return is_attr_preserve_access (op);
+
+      const tree container = DECL_CONTEXT (TREE_OPERAND (t, 1));
+
+      return lookup_attribute ("preserve_access_index",
+			       TYPE_ATTRIBUTES (container));
+    }
+
+  else if (TREE_CODE (t) == ADDR_EXPR)
+    return is_attr_preserve_access (TREE_OPERAND (t, 0));
+
+  return false;
+}
+
+/* The body of pass_bpf_core_attr. Scan RTL for accesses to structs/unions
+   marked with __attribute__((preserve_access_index)) and generate a CO-RE
+   relocation for any such access.  */
+
+static void
+handle_attr_preserve (function *fn)
+{
+  basic_block bb;
+  rtx_insn *insn;
+  rtx_code_label *label;
+  FOR_EACH_BB_FN (bb, fn)
+    {
+      FOR_BB_INSNS (bb, insn)
+	{
+	  if (!NONJUMP_INSN_P (insn))
+	    continue;
+	  rtx pat = PATTERN (insn);
+	  if (GET_CODE (pat) != SET)
+	    continue;
+
+	  start_sequence();
+
+	  for (int i = 0; i < 2; i++)
+	    {
+	      rtx mem = XEXP (pat, i);
+	      if (MEM_P (mem))
+		{
+		  tree expr = MEM_EXPR (mem);
+		  if (!expr)
+		    continue;
+
+		  if (TREE_CODE (expr) == MEM_REF
+		      && TREE_CODE (TREE_OPERAND (expr, 0)) == SSA_NAME)
+		    {
+		      gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (expr, 0));
+		      if (is_gimple_assign (def_stmt))
+			expr = gimple_assign_rhs1 (def_stmt);
+		    }
+
+		  if (is_attr_preserve_access (expr))
+		    {
+		      auto_vec<unsigned int, 16> accessors;
+		      tree container = bpf_core_compute (expr, &accessors);
+		      if (accessors.length () < 1)
+			continue;
+		      accessors.reverse ();
+
+		      container = TREE_TYPE (container);
+		      const char * section_name;
+		      if (DECL_SECTION_NAME (fn->decl))
+			section_name = DECL_SECTION_NAME (fn->decl);
+		      else
+			section_name = ".text";
+
+		      label = gen_label_rtx ();
+		      LABEL_PRESERVE_P (label) = 1;
+		      emit_label (label);
+
+		      /* Add the CO-RE relocation information to the BTF container.  */
+		      bpf_core_reloc_add (container, section_name, &accessors, label);
+		    }
+		}
+	    }
+	  rtx_insn *seq = get_insns ();
+	  end_sequence ();
+	  emit_insn_before (seq, insn);
+	}
+    }
+}
+
+
+/* This pass finds accesses to structures marked with the BPF target attribute
+   __attribute__((preserve_access_index)). For every such access, a CO-RE
+   relocation record is generated, to be output in the .BTF.ext section.  */
+
+namespace {
+
+const pass_data pass_data_bpf_core_attr =
+{
+  RTL_PASS, /* type */
+  "bpf_core_attr", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  0, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_bpf_core_attr : public rtl_opt_pass
+{
+public:
+  pass_bpf_core_attr (gcc::context *ctxt)
+    : rtl_opt_pass (pass_data_bpf_core_attr, ctxt)
+  {}
+
+  virtual bool gate (function *) { return TARGET_BPF_CORE; }
+  virtual unsigned int execute (function *);
+};
+
+unsigned int
+pass_bpf_core_attr::execute (function *fn)
+{
+  handle_attr_preserve (fn);
+  return 0;
+}
+
+} /* Anonymous namespace.  */
+
+rtl_opt_pass *
+make_pass_bpf_core_attr (gcc::context *ctxt)
+{
+  return new pass_bpf_core_attr (ctxt);
+}
+
 /* Finally, build the GCC target.  */
 
 struct gcc_target targetm = TARGET_INITIALIZER;
diff --git a/gcc/config/bpf/coreout.c b/gcc/config/bpf/coreout.c
new file mode 100644
index 00000000000..d5486b463cf
--- /dev/null
+++ b/gcc/config/bpf/coreout.c
@@ -0,0 +1,356 @@
+/* BPF Compile Once - Run Everywhere (CO-RE) support.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#define IN_TARGET_CODE 1
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "target.h"
+#include "memmodel.h"
+#include "tm_p.h"
+#include "output.h"
+#include "dwarf2asm.h"
+#include "ctfc.h"
+#include "btf.h"
+#include "rtl.h"
+
+#include "coreout.h"
+
+/* This file contains data structures and routines for construction and output
+   of BPF Compile Once - Run Everywhere (BPF CO-RE) information.
+
+   eBPF programs written in C usually include Linux kernel headers, so that
+   they may interact with kernel data structures in a useful way. This
+   intrudces two major portability issues:
+
+   1. Kernel data structures regularly change, with fields added, moved or
+      deleted between versions. An eBPF program cannot in general be expected
+      to run on any systems which does not share an identical kernel version to
+      the system on which it was compiled.
+
+   2. Included kernel headers (and used data structures) may be internal, not
+      exposed in an userspace API, and therefore target-specific. An eBPF
+      program compiled on an x86_64 machine will include x86_64 kernel headers.
+      The resulting program may not run well (or at all) in machines of
+      another architecture.
+
+   BPF CO-RE is designed to solve the first issue by leveraging the BPF loader
+   to adjust references to kernel data structures made by the program as-needed
+   according to versions of structures actually present on the host kernel.
+
+   To achieve this, additional information is placed in a ".BTF.ext" section.
+   This information tells the loader which references will require adjusting,
+   and how to perform each necessary adjustment.
+
+   For any access to a data structure which may require load-time adjustment,
+   the following information is recorded (making up a CO-RE relocation record):
+   - The BTF type ID of the outermost structure which is accessed.
+   - An access string encoding the accessed member via a series of member and
+     array indexes. These indexes are used to look up detailed BTF information
+     about the member.
+   - The offset of the appropriate instruction to patch in the BPF program.
+   - An integer specifying what kind of relocation to perform.
+
+   A CO-RE-capable BPF loader reads this information together with the BTF
+   information of the program, compares it against BTF information of the host
+   kernel, and determines the appropriate way to patch the specified
+   instruction.
+
+   Once all CO-RE relocations are resolved, the program is loaded and verified
+   as usual. The process can be summarized with the following diagram:
+
+              +------------+
+              | C compiler |
+              +-----+------+
+                    | BPF + BTF + CO-RE relocations
+                    v
+              +------------+
+         +--->| BPF loader |
+         |    +-----+------+
+         |          | BPF (adapted)
+     BTF |          v
+         |    +------------+
+         +----+   Kernel   |
+              +------------+
+
+   Note that a single ELF object may contain multiple eBPF programs. As a
+   result, a single .BTF.ext section can contain CO-RE relocations for multiple
+   programs in distinct sections.  */
+
+/* Internal representation of a BPF CO-RE relocation record.  */
+
+typedef struct GTY (()) bpf_core_reloc {
+  unsigned int bpfcr_type;		/* BTF type ID of container.  */
+  unsigned int  bpfcr_astr_off;		/* Offset of access string in .BTF
+					   string table.  */
+  rtx_code_label * bpfcr_insn_label;	/* RTX label attached to instruction
+					   to patch.  */
+  enum btf_core_reloc_kind bpfcr_kind;	/* Kind of relocation to perform.  */
+} bpf_core_reloc_t;
+
+typedef bpf_core_reloc_t * bpf_core_reloc_ref;
+
+/* Internal representation of a CO-RE relocation (sub)section of the
+   .BTF.ext information. One such section is generated for each ELF section
+   in the output object having relocations that a BPF loader must resolve.  */
+
+typedef struct GTY (()) bpf_core_section {
+  /* Name of ELF section to which these CO-RE relocations apply.  */
+  const char * name;
+
+  /* Offset of section name in .BTF string table.  */
+  uint32_t name_offset;
+
+  /* Relocations in the section.  */
+  vec <bpf_core_reloc_ref, va_gc> * GTY (()) relocs;
+} bpf_core_section_t;
+
+typedef bpf_core_section_t * bpf_core_section_ref;
+
+/* BTF.ext debug info section.  */
+
+static GTY (()) section * btf_ext_info_section;
+
+static int btf_ext_label_num;
+
+#ifndef BTF_EXT_INFO_SECTION_NAME
+#define BTF_EXT_INFO_SECTION_NAME ".BTF.ext"
+#endif
+
+#define BTF_EXT_INFO_SECTION_FLAGS (SECTION_DEBUG)
+
+#define MAX_BTF_EXT_LABEL_BYTES 40
+
+static char btf_ext_info_section_label[MAX_BTF_EXT_LABEL_BYTES];
+
+#ifndef BTF_EXT_INFO_SECTION_LABEL
+#define BTF_EXT_INFO_SECTION_LABEL "Lbtfext"
+#endif
+
+static GTY (()) vec<bpf_core_section_ref, va_gc> *bpf_core_sections;
+
+
+/* Create a new BPF CO-RE relocation record, and add it to the appropriate
+   CO-RE section.  */
+
+void
+bpf_core_reloc_add (const tree type, const char * section_name,
+		    vec<unsigned int> *accessors, rtx_code_label *label)
+{
+  char buf[40];
+  unsigned int i, n = 0;
+
+  /* A valid CO-RE access must have at least one accessor.  */
+  if (accessors->length () < 1)
+    return;
+
+  for (i = 0; i < accessors->length () - 1; i++)
+    n += snprintf (buf + n, sizeof (buf) - n, "%u:", (*accessors)[i]);
+  snprintf (buf + n, sizeof (buf) - n, "%u", (*accessors)[i]);
+
+  bpf_core_reloc_ref bpfcr = ggc_cleared_alloc<bpf_core_reloc_t> ();
+  ctf_container_ref ctfc = ctf_get_tu_ctfc ();
+
+  /* Buffer the access string in the auxiliary strtab. Since the two string
+     tables are concatenated, add the length of the first to the offset.  */
+  size_t strtab_len = ctfc_get_strtab_len (ctfc, CTF_STRTAB);
+  ctf_add_string (ctfc, buf, &(bpfcr->bpfcr_astr_off), CTF_AUX_STRTAB);
+  bpfcr->bpfcr_astr_off += strtab_len;
+
+  bpfcr->bpfcr_type = get_btf_id (ctf_lookup_tree_type (ctfc, type));
+  bpfcr->bpfcr_insn_label = label;
+  bpfcr->bpfcr_kind = BPF_RELO_FIELD_BYTE_OFFSET;
+
+  /* Add the CO-RE reloc to the appropriate section.  */
+  bpf_core_section_ref sec;
+  FOR_EACH_VEC_ELT (*bpf_core_sections, i, sec)
+    if (strcmp (sec->name, section_name) == 0)
+      {
+	vec_safe_push (sec->relocs, bpfcr);
+	return;
+      }
+
+  /* If the CO-RE section does not yet exist, create it.  */
+  sec = ggc_cleared_alloc<bpf_core_section_t> ();
+
+  ctf_add_string (ctfc, section_name, &sec->name_offset, CTF_AUX_STRTAB);
+  sec->name_offset += strtab_len;
+  if (strcmp (section_name, ""))
+    ctfc->ctfc_aux_strlen += strlen (section_name) + 1;
+
+  sec->name = section_name;
+  vec_alloc (sec->relocs, 1);
+  vec_safe_push (sec->relocs, bpfcr);
+
+  vec_safe_push (bpf_core_sections, sec);
+}
+
+/* Return the 0-based index of the field NODE in its containing struct or union
+   type.  */
+
+int
+bpf_core_get_sou_member_index (ctf_container_ref ctfc, const tree node)
+{
+  if (TREE_CODE (node) == FIELD_DECL)
+    {
+      const tree container = DECL_CONTEXT (node);
+      const char * name = IDENTIFIER_POINTER (DECL_NAME (node));
+
+      /* Lookup the CTF type info for the containing type.  */
+      dw_die_ref die = lookup_type_die (container);
+      if (die == NULL)
+        return -1;
+
+      ctf_dtdef_ref dtd = ctf_dtd_lookup (ctfc, die);
+      if (dtd == NULL)
+        return -1;
+
+      unsigned int kind = CTF_V2_INFO_KIND (dtd->dtd_data.ctti_info);
+      if (kind != CTF_K_STRUCT && kind != CTF_K_UNION)
+        return -1;
+
+      int i = 0;
+      ctf_dmdef_t * dmd;
+      for (dmd = dtd->dtd_u.dtu_members;
+           dmd != NULL; dmd = (ctf_dmdef_t *) ctf_dmd_list_next (dmd))
+        {
+          if (get_btf_id (dmd->dmd_type) > BTF_MAX_TYPE)
+            continue;
+          if (strcmp (dmd->dmd_name, name) == 0)
+            return i;
+          i++;
+        }
+    }
+  return -1;
+}
+
+/* Compute and output the header of a .BTF.ext debug info section.  */
+
+static void
+output_btfext_header (void)
+{
+  switch_to_section (btf_ext_info_section);
+  ASM_OUTPUT_LABEL (asm_out_file, btf_ext_info_section_label);
+
+  dw2_asm_output_data (2, BTF_MAGIC, "btf_magic");
+  dw2_asm_output_data (1, BTF_VERSION, "btfext_version");
+  dw2_asm_output_data (1, 0, "btfext_flags");
+  dw2_asm_output_data (4, sizeof (struct btf_ext_header), "btfext_hdr_len");
+
+  uint32_t func_info_off = 0, func_info_len = 0;
+  uint32_t line_info_off = 0, line_info_len = 0;
+  uint32_t core_relo_off = 0, core_relo_len = 0;
+
+  /* Header core_relo_len is the sum total length in bytes of all CO-RE
+     relocation sections.  */
+  size_t i;
+  bpf_core_section_ref sec;
+  core_relo_len += vec_safe_length (bpf_core_sections)
+    * sizeof (struct btf_ext_section_header);
+
+  FOR_EACH_VEC_ELT (*bpf_core_sections, i, sec)
+    core_relo_len +=
+      vec_safe_length (sec->relocs) * sizeof (struct btf_ext_reloc);
+
+  dw2_asm_output_data (4, func_info_off, "func_info_offset");
+  dw2_asm_output_data (4, func_info_len, "func_info_len");
+
+  dw2_asm_output_data (4, line_info_off, "line_info_offset");
+  dw2_asm_output_data (4, line_info_len, "line_info_len");
+
+  dw2_asm_output_data (4, core_relo_off, "core_relo_offset");
+  dw2_asm_output_data (4, core_relo_len, "core_relo_len");
+}
+
+/* Output a single CO-RE relocation record.  */
+
+static void
+output_asm_btfext_core_reloc (bpf_core_reloc_ref bpfcr)
+{
+  dw2_assemble_integer (4, gen_rtx_LABEL_REF (Pmode, bpfcr->bpfcr_insn_label));
+  fprintf (asm_out_file, "\t%s bpfcr_insn\n", ASM_COMMENT_START);
+
+  dw2_asm_output_data (4, bpfcr->bpfcr_type, "bpfcr_type");
+  dw2_asm_output_data (4, bpfcr->bpfcr_astr_off, "bpfcr_astr_off");
+  dw2_asm_output_data (4, bpfcr->bpfcr_kind, "bpfcr_kind");
+}
+
+/* Output all CO-RE relocation records for a section.  */
+
+static void
+output_btfext_core_relocs (bpf_core_section_ref sec)
+{
+  size_t i;
+  bpf_core_reloc_ref bpfcr;
+  FOR_EACH_VEC_ELT (*(sec->relocs), i, bpfcr)
+    output_asm_btfext_core_reloc (bpfcr);
+}
+
+/* Output all CO-RE relocation sections.  */
+
+static void
+output_btfext_core_sections (void)
+{
+  size_t i;
+  bpf_core_section_ref sec;
+  FOR_EACH_VEC_ELT (*bpf_core_sections, i, sec)
+    {
+      /* BTF Ext section info. */
+      dw2_asm_output_data (4, sizeof (struct btf_ext_reloc),
+			   "btfext_secinfo_rec_size");
+
+      /* Section name offset, refers to the offset of a string with the name of
+	 the section to which these CORE relocations refer, e.g. '.text'.
+	 The string is buffered in the BTF strings table.  */
+      dw2_asm_output_data (4, sec->name_offset,  "btfext_secinfo_sec_name_off");
+      dw2_asm_output_data (4, vec_safe_length (sec->relocs),
+			   "btfext_secinfo_num_recs");
+
+      output_btfext_core_relocs (sec);
+    }
+}
+
+/* Initialize sections, labels, and data structures for BTF.ext output.  */
+
+void
+btf_ext_init (void)
+{
+  btf_ext_info_section = get_section (BTF_EXT_INFO_SECTION_NAME,
+				      BTF_EXT_INFO_SECTION_FLAGS, NULL);
+
+  ASM_GENERATE_INTERNAL_LABEL (btf_ext_info_section_label,
+			       BTF_EXT_INFO_SECTION_LABEL,
+			       btf_ext_label_num++);
+
+  vec_alloc (bpf_core_sections, 1);
+}
+
+/* Output the entire .BTF.ext section.  */
+
+void
+btf_ext_output (void)
+{
+  output_btfext_header ();
+  output_btfext_core_sections ();
+
+  bpf_core_sections = NULL;
+}
+
+#include "gt-coreout.h"
diff --git a/gcc/config/bpf/coreout.h b/gcc/config/bpf/coreout.h
new file mode 100644
index 00000000000..82c203df341
--- /dev/null
+++ b/gcc/config/bpf/coreout.h
@@ -0,0 +1,114 @@
+/* coreout.h - Declarations and definitions related to
+   BPF Compile Once - Run Everywhere (CO-RE) support.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+
+#ifndef __COREOUT_H
+#define __COREOUT_H
+
+#include <stdint.h>
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* .BTF.ext information.  */
+
+struct btf_ext_section_header
+{
+  uint32_t kind;
+  uint32_t sec_name_off;
+  uint32_t num_records;
+};
+
+/* A funcinfo record, in the .BTF.ext funcinfo section.  */
+struct btf_ext_funcinfo
+{
+  uint32_t insn_off; /* Offset of the first instruction of the function.  */
+  uint32_t type;     /* Type ID of a BTF_KIND_FUNC type.  */
+};
+
+/* A lineinfo record, in the .BTF.ext lineinfo section.  */
+struct btf_ext_lineinfo
+{
+  uint32_t insn_off;      /* Offset of the instruction.  */
+  uint32_t file_name_off; /* Offset of file name in BTF string table.  */
+  uint32_t line_off;      /* Offset of source line in BTF string table.  */
+  uint32_t line_col;      /* Line number (bits 31-11) and column (11-0).  */
+};
+
+enum btf_core_reloc_kind
+{
+  BPF_RELO_FIELD_BYTE_OFFSET = 0,
+  BPF_RELO_FIELD_BYTE_SIZE = 1,
+  BPF_RELO_FIELD_EXISTS = 2,
+  BPF_RELO_FIELD_SIGNED = 3,
+  BPF_RELO_FIELD_LSHIFT_U64 = 4,
+  BPF_RELO_FIELD_RSHIFT_U64 = 5,
+  BPF_RELO_TYPE_ID_LOCAL = 6,
+  BPF_RELO_TYPE_ID_TARGET = 7,
+  BPF_RELO_TYPE_EXISTS = 8,
+  BPF_RELO_TYPE_SIZE = 9,
+  BPF_RELO_ENUMVAL_EXISTS = 10,
+  BPF_RELO_ENUMVAL_VALUE = 11
+};
+
+struct btf_ext_reloc
+{
+  uint32_t insn_off;       /* Offset of instruction to be patched. A
+			      section-relative label at compile time.  */
+  uint32_t type_id;        /* Type ID of the outermost containing entity, e.g.
+			      the containing structure.  */
+  uint32_t access_str_off; /* Offset of CO-RE accessor string in .BTF strings
+			      section.  */
+  uint32_t kind;           /* An enum btf_core_reloc_kind. Note that it always
+			      takes 32 bits.  */
+};
+
+struct btf_ext_header
+{
+  uint16_t magic;		/* Magic number (BTF_MAGIC).  */
+  uint8_t  version;		/* Data format version (BTF_VERSION).  */
+  uint8_t  flags;		/* Flags. Currently unused.  */
+  uint32_t hdr_len;		/* Length of this header in bytes.  */
+
+  /* Following offsets are relative to the end of this header, in bytes.
+     Following lengths are in bytes.  */
+  uint32_t func_info_off;	/* Offset of funcinfo section.  */
+  uint32_t func_info_len;	/* Length of funcinfo section.  */
+  uint32_t line_info_off;	/* Offset of lineinfo section.  */
+  uint32_t line_info_len;	/* Length of lineinfo section.  */
+
+  uint32_t core_relo_off;	/* Offset of CO-RE relocation section.  */
+  uint32_t core_relo_len;	/* Length of CO-RE relocation section.  */
+};
+
+extern void btf_ext_init (void);
+extern void btf_ext_output (void);
+
+extern void bpf_core_reloc_add (const tree, const char *, vec<unsigned int> *,
+				rtx_code_label *);
+extern int bpf_core_get_sou_member_index (ctf_container_ref, const tree);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* __COREOUT_H */
diff --git a/gcc/config/bpf/t-bpf b/gcc/config/bpf/t-bpf
index e69de29bb2d..b37bf858d8f 100644
--- a/gcc/config/bpf/t-bpf
+++ b/gcc/config/bpf/t-bpf
@@ -0,0 +1,8 @@
+
+TM_H += $(srcdir)/config/bpf/coreout.h
+
+coreout.o: $(srcdir)/config/bpf/coreout.c
+	$(COMPILE) $<
+	$(POSTCOMPILE)
+
+PASSES_EXTRA += $(srcdir)/config/bpf/bpf-passes.def