[RFC][PATCH 10/X][libsanitizer] Colour the shadow stack for each stack variable

[Matthew Malcomson <Matthew.Malcomson@arm.com>]

[-- Attachment #1: Type: text/plain, Size: 14468 bytes --]

On entry to each function we colour the relevant shadow stack region for
each stack variable the colour to match the tag added to each pointer
for that variable.

We currently only use the library option for this, in the future inline
options will be added and configured in the same way as ASAN.

This is the first patch where we use the HWASAN shadow space, so we need
to add in the libhwasan initialisation code that creates this shadow
memory region into the binary we produce.  This instrumentation is  done
in `compile_file`.

gcc/ChangeLog:

2019-09-06  Matthew Malcomson  <matthew.malcomson@arm.com>

	* asan.c (initialize_sanitizer_builtins): Define new builtins.
	(ATTR_NOTHROW_LIST): New macro.
	(hwasan_current_tag): New.
	(hwasan_emit_prologue): New.
	(hwasan_create_untagged_base): New.
	(hwasan_finish_file): New.
	* asan.h (hwasan_finish_file): New.
	(hwasan_current_tag): New.
	(hwasan_create_untagged_base): New.
	(hwasan_emit_prologue): New.
	* builtin-types.def (BT_FN_VOID_PTR_UINT8_SIZE): New.
	* builtins.def (DEF_SANITIZER_BUILTIN): Enable for HWASAN.
	* cfgexpand.c (struct stack_vars_data):
	(expand_stack_vars): Record offsets for hwasan.
	(expand_used_vars): Call function to emit prologue.
	* sanitizer.def (BUILT_IN_HWASAN_INIT): New.
	(BUILT_IN_HWASAN_TAG_MEM): New.
	* toplev.c (compile_file): Emit libhwasan initialisation.



###############     Attachment also inlined for ease of reply    ###############


diff --git a/gcc/asan.h b/gcc/asan.h
index 127c24aa6b0e4e6d0ba332004145ec498034c955..028afdd2e7d16245c6cbbe106b7ccb9c5034d542 100644
--- a/gcc/asan.h
+++ b/gcc/asan.h
@@ -23,10 +23,14 @@ along with GCC; see the file COPYING3.  If not see
 
 extern void asan_function_start (void);
 extern void asan_finish_file (void);
+extern void hwasan_finish_file (void);
 extern void hwasan_record_base (rtx);
+extern uint8_t hwasan_current_tag ();
 extern void hwasan_increment_tag ();
 extern rtx hwasan_with_tag (rtx, poly_int64);
 extern void hwasan_tag_init ();
+extern rtx hwasan_create_untagged_base (rtx);
+extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
 extern bool memory_tagging_p (void);
 extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
 					     HOST_WIDE_INT *, tree *, int);
diff --git a/gcc/asan.c b/gcc/asan.c
index a6ff503ceec294f2c09b0bd723a3d8043e4de6a1..d361b4b562f75cb0c2e081218073eacb3704f8d0 100644
--- a/gcc/asan.c
+++ b/gcc/asan.c
@@ -2895,6 +2895,11 @@ initialize_sanitizer_builtins (void)
     = build_function_type_list (void_type_node, uint64_type_node,
 				ptr_type_node, NULL_TREE);
 
+  tree BT_FN_VOID_PTR_UINT8_SIZE
+    = build_function_type_list (void_type_node, ptr_type_node,
+				unsigned_char_type_node, size_type_node,
+				NULL_TREE);
+
   tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
   tree BT_FN_IX_CONST_VPTR_INT[5];
   tree BT_FN_IX_VPTR_IX_INT[5];
@@ -2945,6 +2950,8 @@ initialize_sanitizer_builtins (void)
 #define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
 #define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
 #define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
+#undef ATTR_NOTHROW_LIST
+#define ATTR_NOTHROW_LIST ECF_NOTHROW
 #undef ATTR_NOTHROW_LEAF_LIST
 #define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
 #undef ATTR_TMPURE_NOTHROW_LEAF_LIST
@@ -3707,6 +3714,8 @@ hwasan_record_base (rtx base)
   hwasan_base_ptr = base;
 }
 
+uint8_t hwasan_current_tag () { return tag_offset; }
+
 void
 hwasan_increment_tag ()
 {
@@ -3760,4 +3769,104 @@ hwasan_tag_init ()
   tag_offset = HWASAN_STACK_BACKGROUND + 1;
 }
 
+void
+hwasan_emit_prologue (rtx *bases,
+		      rtx *untagged_bases,
+		      poly_int64 *offsets,
+		      uint8_t *tags,
+		      size_t length)
+{
+  /*
+    NOTE: bases contains both the tagged and untagged base.
+    This allows us to get both the original frame tag and the untagged variable
+    pointer with a minimal of extra instructions.
+
+    We fetch the untagged variable pointer from the offset to the untagged base
+    and fetch the "base" tag from the tagged base.
+
+    We need the untagged base pointer since libhwasan only accepts untagged
+    pointers in __hwasan_tag_memory.  We need the tagged base pointer to obtain
+    the base tag for an offset.
+
+    We also will need the tagged base pointer for MTE, since the ADDTAG
+    instruction takes a tagged pointer.
+ */
+  for (size_t i = 0; (i * 2) + 1 < length; i++)
+    {
+      poly_int64 start = offsets[i * 2];
+      poly_int64 end = offsets[(i * 2) + 1];
+
+      poly_int64 bot, top;
+      if (known_ge (start, end))
+	{
+	  top = start;
+	  bot = end;
+	}
+      else
+	{
+	  top = end;
+	  bot = start;
+	}
+      poly_int64 size = (top - bot);
+
+      /* Can't check that all poly_int64's are aligned, but still nice
+	 to check when we can.  */
+      HOST_WIDE_INT tmp;
+      if (top.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+      if (bot.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+      if (size.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+
+      /* TODO Other options (i.e. inline options)  */
+      /* TODO At the moment we don't generate a random base tag for each
+         frame.  When that happens we will need to generate the tag by
+         adding tags[i] to the frame tag fetched from `bases[i]`.  */
+      rtx ret = init_one_libfunc ("__hwasan_tag_memory");
+      emit_library_call (ret,
+	  LCT_NORMAL,
+	  VOIDmode,
+	  plus_constant (ptr_mode, untagged_bases[i], bot),
+	  ptr_mode,
+	  const_int_rtx[MAX_SAVED_CONST_INT + tags[i]],
+	  QImode,
+	  gen_int_mode (size, ptr_mode),
+	  ptr_mode);
+    }
+}
+
+rtx
+hwasan_create_untagged_base (rtx orig_base)
+{
+  rtx untagged_base = gen_reg_rtx (Pmode);
+  rtx tag_mask = gen_int_mode ((1ULL << HWASAN_SHIFT) - 1, Pmode);
+  untagged_base = expand_binop (Pmode, and_optab,
+				orig_base, tag_mask,
+				untagged_base, true, OPTAB_DIRECT);
+  gcc_assert (untagged_base);
+  return untagged_base;
+}
+
+/* Needs to be GTY(()), because cgraph_build_static_cdtor may
+   invoke ggc_collect.  */
+static GTY(()) tree hwasan_ctor_statements;
+
+void
+hwasan_finish_file (void)
+{
+  /* Avoid instrumenting code in the hwasan constructors/destructors.  */
+  flag_sanitize &= ~SANITIZE_HWADDRESS;
+  /* TODO Only do this if in userspace.
+      For kernel space will have to look more closely into this.
+      May want to look at `asan_finish_file` for what ASAN does in this
+      situation.  */
+
+  int priority = MAX_RESERVED_INIT_PRIORITY - 1;
+  tree fn = builtin_decl_implicit (BUILT_IN_HWASAN_INIT);
+  append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
+  cgraph_build_static_cdtor ('I', hwasan_ctor_statements, priority);
+  flag_sanitize |= SANITIZE_HWADDRESS;
+}
+
 #include "gt-asan.h"
diff --git a/gcc/builtin-types.def b/gcc/builtin-types.def
index e5c9e063c480d1392b6c2b395ec9d029b6d94209..d05f597b6434f39fe95d4f28dd2ef3ed463dd925 100644
--- a/gcc/builtin-types.def
+++ b/gcc/builtin-types.def
@@ -625,6 +625,8 @@ DEF_FUNCTION_TYPE_3 (BT_FN_VOID_UINT32_UINT32_PTR,
 DEF_FUNCTION_TYPE_3 (BT_FN_VOID_SIZE_SIZE_PTR, BT_VOID, BT_SIZE, BT_SIZE,
 		     BT_PTR)
 DEF_FUNCTION_TYPE_3 (BT_FN_UINT_UINT_PTR_PTR, BT_UINT, BT_UINT, BT_PTR, BT_PTR)
+DEF_FUNCTION_TYPE_3 (BT_FN_VOID_PTR_UINT8_SIZE, BT_VOID, BT_PTR, BT_UINT8,
+		     BT_SIZE)
 
 DEF_FUNCTION_TYPE_4 (BT_FN_SIZE_CONST_PTR_SIZE_SIZE_FILEPTR,
 		     BT_SIZE, BT_CONST_PTR, BT_SIZE, BT_SIZE, BT_FILEPTR)
diff --git a/gcc/builtins.def b/gcc/builtins.def
index c92292aeab9da21cb8268e6483078a6be9e49d95..e067ebc7f7003bcdf7df4d52db18c31762623285 100644
--- a/gcc/builtins.def
+++ b/gcc/builtins.def
@@ -237,6 +237,7 @@ along with GCC; see the file COPYING3.  If not see
   DEF_BUILTIN (ENUM, "__builtin_" NAME, BUILT_IN_NORMAL, TYPE, TYPE,    \
 	       true, true, true, ATTRS, true, \
 	      (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_THREAD \
+				| SANITIZE_HWADDRESS \
 				| SANITIZE_UNDEFINED \
 				| SANITIZE_UNDEFINED_NONDEFAULT) \
 	       || flag_sanitize_coverage))
diff --git a/gcc/cfgexpand.c b/gcc/cfgexpand.c
index 10739bc25940374c686c191ca76b1dfe8f000562..aacf210facc462675a980ee87bd38d4a7d94ad09 100644
--- a/gcc/cfgexpand.c
+++ b/gcc/cfgexpand.c
@@ -1034,9 +1034,24 @@ struct stack_vars_data
      The vector is in reversed, highest offset pairs come first.  */
   auto_vec<HOST_WIDE_INT> asan_vec;
 
+  /* HWASAN records the poly_int64 since it needs to act on everything recorded
+     to the stack (as anything not properly coloured would end up causing a
+     falut of some sort).
+
+     ASAN records HOST_WIDE_INT offsets (that was enough before the
+     introduction of SVE vectors) which  */
+  auto_vec<poly_int64> hwasan_vec;
+  auto_vec<rtx> hwasan_untagged_base_vec;
+  auto_vec<rtx> hwasan_base_vec;
+
   /* Vector of partition representative decls in between the paddings.  */
   auto_vec<tree> asan_decl_vec;
 
+  /* Vector of tag offsets representing the colour of each stack variable.
+     Each offset determines the difference between the randomly generated
+     colour for the current frame and the colour for this stack variable.  */
+  auto_vec<uint8_t> hwasan_colour_vec;
+
   /* Base pseudo register for Address Sanitizer protected automatic vars.  */
   rtx asan_base;
 
@@ -1054,6 +1069,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
   size_t si, i, j, n = stack_vars_num;
   poly_uint64 large_size = 0, large_alloc = 0;
   rtx large_base = NULL;
+  rtx large_untagged_base = NULL;
   unsigned large_align = 0;
   bool large_allocation_done = false;
   tree decl;
@@ -1110,7 +1126,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
     {
       rtx base;
       unsigned base_align, alignb;
-      poly_int64 offset;
+      poly_int64 offset = 0;
 
       i = stack_vars_sorted[si];
 
@@ -1156,7 +1172,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 		 ABI requirements) and these can't share a tag granule with a
 		 tagged variable.  */
 	      gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
-	      alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+	      offset = alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+	      data->hwasan_vec.safe_push (offset);
+	      data->hwasan_untagged_base_vec.safe_push (virtual_stack_vars_rtx);
 	    }
 	  /* ASAN description strings don't yet have a syntax for expressing
 	     polynomial offsets.  */
@@ -1237,6 +1255,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 	      offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
 	      base_align = crtl->max_used_stack_slot_alignment;
 	    }
+
+	  if (memory_tagging_p ())
+	    data->hwasan_vec.safe_push (offset);
 	}
       else
 	{
@@ -1262,6 +1283,18 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 
 	  gcc_assert (large_base != NULL);
 	  large_alloc = aligned_upper_bound (large_alloc, alignb);
+	  if (memory_tagging_p ())
+	    {
+	      /*
+		 For now we just assume that an object with a large alignment
+		 requirement means that the alignment requirement is greater
+		 than the required alignment for tags.
+		*/
+	      if (!large_untagged_base)
+		large_untagged_base = hwasan_create_untagged_base (large_base);
+	      data->hwasan_vec.safe_push (large_alloc);
+	      data->hwasan_untagged_base_vec.safe_push (large_untagged_base);
+	    }
 	  offset = large_alloc;
 	  large_alloc += stack_vars[i].size;
 	  if (memory_tagging_p ())
@@ -1280,6 +1313,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 		 */
 	      poly_int64 align_again =
 		aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
+	      data->hwasan_vec.safe_push (align_again);
 	    }
 
 	  base = large_base;
@@ -1297,8 +1331,15 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 				   ? data->asan_base
 				   : virtual_stack_vars_rtx);
 	}
+
       if (memory_tagging_p ())
-	hwasan_increment_tag ();
+	{
+	  /* Record the tag for this object in `data` so the prologue knows
+	     what colour to put in the shadow memory during cfgexpand.c.  */
+	  data->hwasan_base_vec.safe_push (base);
+	  data->hwasan_colour_vec.safe_push (hwasan_current_tag ());
+	  hwasan_increment_tag ();
+	}
     }
 
   gcc_assert (known_eq (large_alloc, large_size));
@@ -2358,6 +2399,13 @@ expand_used_vars (void)
 	}
 
       expand_stack_vars (NULL, &data);
+
+      if (memory_tagging_p ())
+	hwasan_emit_prologue (data.hwasan_base_vec.address (),
+			      data.hwasan_untagged_base_vec.address (),
+			      data.hwasan_vec.address (),
+			      data.hwasan_colour_vec.address (),
+			      data.hwasan_vec.length ());
     }
 
   if (asan_sanitize_allocas_p () && cfun->calls_alloca)
diff --git a/gcc/sanitizer.def b/gcc/sanitizer.def
index 374d15007d868363d9b4fbf467e1e462abbca61a..7bd50715f24a2cb154b578e2abdea4e8fcdb2107 100644
--- a/gcc/sanitizer.def
+++ b/gcc/sanitizer.def
@@ -180,6 +180,12 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_COMPARE, "__sanitizer_ptr_cmp",
 DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_SUBTRACT, "__sanitizer_ptr_sub",
 		      BT_FN_VOID_PTR_PTRMODE, ATTR_NOTHROW_LEAF_LIST)
 
+/* Hardware Address Sanitizer.  */
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_INIT, "__hwasan_init",
+		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_TAG_MEM, "__hwasan_tag_memory",
+		      BT_FN_VOID_PTR_UINT8_SIZE, ATTR_NOTHROW_LIST)
+
 /* Thread Sanitizer */
 DEF_SANITIZER_BUILTIN(BUILT_IN_TSAN_INIT, "__tsan_init", 
 		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
diff --git a/gcc/toplev.c b/gcc/toplev.c
index d300ac2ec894ee947156616e71796d55d9d04307..93484f87690b8d54d93abe0a67cdf51c4a9a3ee1 100644
--- a/gcc/toplev.c
+++ b/gcc/toplev.c
@@ -508,6 +508,9 @@ compile_file (void)
       if (flag_sanitize & SANITIZE_THREAD)
 	tsan_finish_file ();
 
+      if (gate_hwasan ())
+	hwasan_finish_file ();
+
       omp_finish_file ();
 
       hsa_output_brig ();


[-- Attachment #2: hwasan-implementation09.patch --]
[-- Type: text/plain, Size: 12645 bytes --]

diff --git a/gcc/asan.h b/gcc/asan.h
index 127c24aa6b0e4e6d0ba332004145ec498034c955..028afdd2e7d16245c6cbbe106b7ccb9c5034d542 100644
--- a/gcc/asan.h
+++ b/gcc/asan.h
@@ -23,10 +23,14 @@ along with GCC; see the file COPYING3.  If not see
 
 extern void asan_function_start (void);
 extern void asan_finish_file (void);
+extern void hwasan_finish_file (void);
 extern void hwasan_record_base (rtx);
+extern uint8_t hwasan_current_tag ();
 extern void hwasan_increment_tag ();
 extern rtx hwasan_with_tag (rtx, poly_int64);
 extern void hwasan_tag_init ();
+extern rtx hwasan_create_untagged_base (rtx);
+extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
 extern bool memory_tagging_p (void);
 extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
 					     HOST_WIDE_INT *, tree *, int);
diff --git a/gcc/asan.c b/gcc/asan.c
index a6ff503ceec294f2c09b0bd723a3d8043e4de6a1..d361b4b562f75cb0c2e081218073eacb3704f8d0 100644
--- a/gcc/asan.c
+++ b/gcc/asan.c
@@ -2895,6 +2895,11 @@ initialize_sanitizer_builtins (void)
     = build_function_type_list (void_type_node, uint64_type_node,
 				ptr_type_node, NULL_TREE);
 
+  tree BT_FN_VOID_PTR_UINT8_SIZE
+    = build_function_type_list (void_type_node, ptr_type_node,
+				unsigned_char_type_node, size_type_node,
+				NULL_TREE);
+
   tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
   tree BT_FN_IX_CONST_VPTR_INT[5];
   tree BT_FN_IX_VPTR_IX_INT[5];
@@ -2945,6 +2950,8 @@ initialize_sanitizer_builtins (void)
 #define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
 #define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
 #define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
+#undef ATTR_NOTHROW_LIST
+#define ATTR_NOTHROW_LIST ECF_NOTHROW
 #undef ATTR_NOTHROW_LEAF_LIST
 #define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
 #undef ATTR_TMPURE_NOTHROW_LEAF_LIST
@@ -3707,6 +3714,8 @@ hwasan_record_base (rtx base)
   hwasan_base_ptr = base;
 }
 
+uint8_t hwasan_current_tag () { return tag_offset; }
+
 void
 hwasan_increment_tag ()
 {
@@ -3760,4 +3769,104 @@ hwasan_tag_init ()
   tag_offset = HWASAN_STACK_BACKGROUND + 1;
 }
 
+void
+hwasan_emit_prologue (rtx *bases,
+		      rtx *untagged_bases,
+		      poly_int64 *offsets,
+		      uint8_t *tags,
+		      size_t length)
+{
+  /*
+    NOTE: bases contains both the tagged and untagged base.
+    This allows us to get both the original frame tag and the untagged variable
+    pointer with a minimal of extra instructions.
+
+    We fetch the untagged variable pointer from the offset to the untagged base
+    and fetch the "base" tag from the tagged base.
+
+    We need the untagged base pointer since libhwasan only accepts untagged
+    pointers in __hwasan_tag_memory.  We need the tagged base pointer to obtain
+    the base tag for an offset.
+
+    We also will need the tagged base pointer for MTE, since the ADDTAG
+    instruction takes a tagged pointer.
+ */
+  for (size_t i = 0; (i * 2) + 1 < length; i++)
+    {
+      poly_int64 start = offsets[i * 2];
+      poly_int64 end = offsets[(i * 2) + 1];
+
+      poly_int64 bot, top;
+      if (known_ge (start, end))
+	{
+	  top = start;
+	  bot = end;
+	}
+      else
+	{
+	  top = end;
+	  bot = start;
+	}
+      poly_int64 size = (top - bot);
+
+      /* Can't check that all poly_int64's are aligned, but still nice
+	 to check when we can.  */
+      HOST_WIDE_INT tmp;
+      if (top.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+      if (bot.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+      if (size.is_constant (&tmp))
+	gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+
+      /* TODO Other options (i.e. inline options)  */
+      /* TODO At the moment we don't generate a random base tag for each
+         frame.  When that happens we will need to generate the tag by
+         adding tags[i] to the frame tag fetched from `bases[i]`.  */
+      rtx ret = init_one_libfunc ("__hwasan_tag_memory");
+      emit_library_call (ret,
+	  LCT_NORMAL,
+	  VOIDmode,
+	  plus_constant (ptr_mode, untagged_bases[i], bot),
+	  ptr_mode,
+	  const_int_rtx[MAX_SAVED_CONST_INT + tags[i]],
+	  QImode,
+	  gen_int_mode (size, ptr_mode),
+	  ptr_mode);
+    }
+}
+
+rtx
+hwasan_create_untagged_base (rtx orig_base)
+{
+  rtx untagged_base = gen_reg_rtx (Pmode);
+  rtx tag_mask = gen_int_mode ((1ULL << HWASAN_SHIFT) - 1, Pmode);
+  untagged_base = expand_binop (Pmode, and_optab,
+				orig_base, tag_mask,
+				untagged_base, true, OPTAB_DIRECT);
+  gcc_assert (untagged_base);
+  return untagged_base;
+}
+
+/* Needs to be GTY(()), because cgraph_build_static_cdtor may
+   invoke ggc_collect.  */
+static GTY(()) tree hwasan_ctor_statements;
+
+void
+hwasan_finish_file (void)
+{
+  /* Avoid instrumenting code in the hwasan constructors/destructors.  */
+  flag_sanitize &= ~SANITIZE_HWADDRESS;
+  /* TODO Only do this if in userspace.
+      For kernel space will have to look more closely into this.
+      May want to look at `asan_finish_file` for what ASAN does in this
+      situation.  */
+
+  int priority = MAX_RESERVED_INIT_PRIORITY - 1;
+  tree fn = builtin_decl_implicit (BUILT_IN_HWASAN_INIT);
+  append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
+  cgraph_build_static_cdtor ('I', hwasan_ctor_statements, priority);
+  flag_sanitize |= SANITIZE_HWADDRESS;
+}
+
 #include "gt-asan.h"
diff --git a/gcc/builtin-types.def b/gcc/builtin-types.def
index e5c9e063c480d1392b6c2b395ec9d029b6d94209..d05f597b6434f39fe95d4f28dd2ef3ed463dd925 100644
--- a/gcc/builtin-types.def
+++ b/gcc/builtin-types.def
@@ -625,6 +625,8 @@ DEF_FUNCTION_TYPE_3 (BT_FN_VOID_UINT32_UINT32_PTR,
 DEF_FUNCTION_TYPE_3 (BT_FN_VOID_SIZE_SIZE_PTR, BT_VOID, BT_SIZE, BT_SIZE,
 		     BT_PTR)
 DEF_FUNCTION_TYPE_3 (BT_FN_UINT_UINT_PTR_PTR, BT_UINT, BT_UINT, BT_PTR, BT_PTR)
+DEF_FUNCTION_TYPE_3 (BT_FN_VOID_PTR_UINT8_SIZE, BT_VOID, BT_PTR, BT_UINT8,
+		     BT_SIZE)
 
 DEF_FUNCTION_TYPE_4 (BT_FN_SIZE_CONST_PTR_SIZE_SIZE_FILEPTR,
 		     BT_SIZE, BT_CONST_PTR, BT_SIZE, BT_SIZE, BT_FILEPTR)
diff --git a/gcc/builtins.def b/gcc/builtins.def
index c92292aeab9da21cb8268e6483078a6be9e49d95..e067ebc7f7003bcdf7df4d52db18c31762623285 100644
--- a/gcc/builtins.def
+++ b/gcc/builtins.def
@@ -237,6 +237,7 @@ along with GCC; see the file COPYING3.  If not see
   DEF_BUILTIN (ENUM, "__builtin_" NAME, BUILT_IN_NORMAL, TYPE, TYPE,    \
 	       true, true, true, ATTRS, true, \
 	      (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_THREAD \
+				| SANITIZE_HWADDRESS \
 				| SANITIZE_UNDEFINED \
 				| SANITIZE_UNDEFINED_NONDEFAULT) \
 	       || flag_sanitize_coverage))
diff --git a/gcc/cfgexpand.c b/gcc/cfgexpand.c
index 10739bc25940374c686c191ca76b1dfe8f000562..aacf210facc462675a980ee87bd38d4a7d94ad09 100644
--- a/gcc/cfgexpand.c
+++ b/gcc/cfgexpand.c
@@ -1034,9 +1034,24 @@ struct stack_vars_data
      The vector is in reversed, highest offset pairs come first.  */
   auto_vec<HOST_WIDE_INT> asan_vec;
 
+  /* HWASAN records the poly_int64 since it needs to act on everything recorded
+     to the stack (as anything not properly coloured would end up causing a
+     falut of some sort).
+
+     ASAN records HOST_WIDE_INT offsets (that was enough before the
+     introduction of SVE vectors) which  */
+  auto_vec<poly_int64> hwasan_vec;
+  auto_vec<rtx> hwasan_untagged_base_vec;
+  auto_vec<rtx> hwasan_base_vec;
+
   /* Vector of partition representative decls in between the paddings.  */
   auto_vec<tree> asan_decl_vec;
 
+  /* Vector of tag offsets representing the colour of each stack variable.
+     Each offset determines the difference between the randomly generated
+     colour for the current frame and the colour for this stack variable.  */
+  auto_vec<uint8_t> hwasan_colour_vec;
+
   /* Base pseudo register for Address Sanitizer protected automatic vars.  */
   rtx asan_base;
 
@@ -1054,6 +1069,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
   size_t si, i, j, n = stack_vars_num;
   poly_uint64 large_size = 0, large_alloc = 0;
   rtx large_base = NULL;
+  rtx large_untagged_base = NULL;
   unsigned large_align = 0;
   bool large_allocation_done = false;
   tree decl;
@@ -1110,7 +1126,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
     {
       rtx base;
       unsigned base_align, alignb;
-      poly_int64 offset;
+      poly_int64 offset = 0;
 
       i = stack_vars_sorted[si];
 
@@ -1156,7 +1172,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 		 ABI requirements) and these can't share a tag granule with a
 		 tagged variable.  */
 	      gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
-	      alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+	      offset = alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+	      data->hwasan_vec.safe_push (offset);
+	      data->hwasan_untagged_base_vec.safe_push (virtual_stack_vars_rtx);
 	    }
 	  /* ASAN description strings don't yet have a syntax for expressing
 	     polynomial offsets.  */
@@ -1237,6 +1255,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 	      offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
 	      base_align = crtl->max_used_stack_slot_alignment;
 	    }
+
+	  if (memory_tagging_p ())
+	    data->hwasan_vec.safe_push (offset);
 	}
       else
 	{
@@ -1262,6 +1283,18 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 
 	  gcc_assert (large_base != NULL);
 	  large_alloc = aligned_upper_bound (large_alloc, alignb);
+	  if (memory_tagging_p ())
+	    {
+	      /*
+		 For now we just assume that an object with a large alignment
+		 requirement means that the alignment requirement is greater
+		 than the required alignment for tags.
+		*/
+	      if (!large_untagged_base)
+		large_untagged_base = hwasan_create_untagged_base (large_base);
+	      data->hwasan_vec.safe_push (large_alloc);
+	      data->hwasan_untagged_base_vec.safe_push (large_untagged_base);
+	    }
 	  offset = large_alloc;
 	  large_alloc += stack_vars[i].size;
 	  if (memory_tagging_p ())
@@ -1280,6 +1313,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 		 */
 	      poly_int64 align_again =
 		aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
+	      data->hwasan_vec.safe_push (align_again);
 	    }
 
 	  base = large_base;
@@ -1297,8 +1331,15 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
 				   ? data->asan_base
 				   : virtual_stack_vars_rtx);
 	}
+
       if (memory_tagging_p ())
-	hwasan_increment_tag ();
+	{
+	  /* Record the tag for this object in `data` so the prologue knows
+	     what colour to put in the shadow memory during cfgexpand.c.  */
+	  data->hwasan_base_vec.safe_push (base);
+	  data->hwasan_colour_vec.safe_push (hwasan_current_tag ());
+	  hwasan_increment_tag ();
+	}
     }
 
   gcc_assert (known_eq (large_alloc, large_size));
@@ -2358,6 +2399,13 @@ expand_used_vars (void)
 	}
 
       expand_stack_vars (NULL, &data);
+
+      if (memory_tagging_p ())
+	hwasan_emit_prologue (data.hwasan_base_vec.address (),
+			      data.hwasan_untagged_base_vec.address (),
+			      data.hwasan_vec.address (),
+			      data.hwasan_colour_vec.address (),
+			      data.hwasan_vec.length ());
     }
 
   if (asan_sanitize_allocas_p () && cfun->calls_alloca)
diff --git a/gcc/sanitizer.def b/gcc/sanitizer.def
index 374d15007d868363d9b4fbf467e1e462abbca61a..7bd50715f24a2cb154b578e2abdea4e8fcdb2107 100644
--- a/gcc/sanitizer.def
+++ b/gcc/sanitizer.def
@@ -180,6 +180,12 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_COMPARE, "__sanitizer_ptr_cmp",
 DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_SUBTRACT, "__sanitizer_ptr_sub",
 		      BT_FN_VOID_PTR_PTRMODE, ATTR_NOTHROW_LEAF_LIST)
 
+/* Hardware Address Sanitizer.  */
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_INIT, "__hwasan_init",
+		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_TAG_MEM, "__hwasan_tag_memory",
+		      BT_FN_VOID_PTR_UINT8_SIZE, ATTR_NOTHROW_LIST)
+
 /* Thread Sanitizer */
 DEF_SANITIZER_BUILTIN(BUILT_IN_TSAN_INIT, "__tsan_init", 
 		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
diff --git a/gcc/toplev.c b/gcc/toplev.c
index d300ac2ec894ee947156616e71796d55d9d04307..93484f87690b8d54d93abe0a67cdf51c4a9a3ee1 100644
--- a/gcc/toplev.c
+++ b/gcc/toplev.c
@@ -508,6 +508,9 @@ compile_file (void)
       if (flag_sanitize & SANITIZE_THREAD)
 	tsan_finish_file ();
 
+      if (gate_hwasan ())
+	hwasan_finish_file ();
+
       omp_finish_file ();
 
       hsa_output_brig ();