Re: [Patch, updated] Make emulated TLS lto-friendly.

[Richard Henderson <rth@redhat.com>]

[-- Attachment #1: Type: text/plain, Size: 669 bytes --]

On 07/09/2010 05:10 AM, IainS wrote:
> 1/ a pass behind a gate of ! targetm.have_tls is a whole lot less
> intrusive to non-emutls targets than what we have
> 2/ it's likely more efficient for emutls targets since there's less
> to-ing and fro-ing.
> 3/ probably more maintainable.
> 4/ certainly more transparent.
> 5/ (probably) loses all reference to emutls from varasm, varpool, expr,
> & gimplify..
> 
> mind you, that doesn't mean I know how to write such a pass ...  ;-)

The following almost certainly doesn't work with aliases yet -- there are
too many paths that create them.  But see how far this gets with other 
tests; whether variables get lost etc.


r~

[-- Attachment #2: emutls-1 --]
[-- Type: text/plain, Size: 37899 bytes --]

diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index a807e8e..555f336 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1352,6 +1352,7 @@ OBJS-common = \
 	tree-diagnostic.o \
 	tree-dump.o \
 	tree-eh.o \
+	tree-emutls.o \
 	tree-if-conv.o \
 	tree-into-ssa.o \
 	tree-iterator.o \
@@ -3135,6 +3136,8 @@ tree-switch-conversion.o : tree-switch-conversion.c $(CONFIG_H) $(SYSTEM_H) \
 tree-complex.o : tree-complex.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TREE_H) \
     $(TM_H) $(FLAGS_H) $(TREE_FLOW_H) $(GIMPLE_H) \
     tree-iterator.h $(TREE_PASS_H) tree-ssa-propagate.h $(DIAGNOSTIC_H)
+tree-emutls.o : tree-emutls.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TREE_H) \
+    $(GIMPLE_H) $(TREE_PASS_H) $(TARGET_H) gt-tree-emutls.h
 tree-vect-generic.o : tree-vect-generic.c $(CONFIG_H) $(SYSTEM_H) $(TREE_H) \
     $(TM_H) $(TREE_FLOW_H) $(GIMPLE_H) tree-iterator.h $(TREE_PASS_H) \
     $(FLAGS_H) $(OPTABS_H) $(MACHMODE_H) $(EXPR_H) \
@@ -3734,6 +3737,7 @@ GTFILES = $(CPP_ID_DATA_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(srcdir)/tree-ssanames.c $(srcdir)/tree-eh.c $(srcdir)/tree-ssa-address.c \
   $(srcdir)/tree-cfg.c \
   $(srcdir)/tree-dfa.c \
+  $(srcdir)/tree-emutls.c \
   $(srcdir)/tree-iterator.c $(srcdir)/gimplify.c \
   $(srcdir)/tree-chrec.h \
   $(srcdir)/tree-scalar-evolution.c \
diff --git a/gcc/expr.c b/gcc/expr.c
index 00ebfdc..03b879c 100644
--- a/gcc/expr.c
+++ b/gcc/expr.c
@@ -6829,20 +6829,6 @@ highest_pow2_factor_for_target (const_tree target, const_tree exp)
   return MAX (factor, talign);
 }
 \f
-/* Return &VAR expression for emulated thread local VAR.  */
-
-static tree
-emutls_var_address (tree var)
-{
-  tree emuvar = emutls_decl (var);
-  tree fn = built_in_decls [BUILT_IN_EMUTLS_GET_ADDRESS];
-  tree arg = build_fold_addr_expr_with_type (emuvar, ptr_type_node);
-  tree arglist = build_tree_list (NULL_TREE, arg);
-  tree call = build_function_call_expr (UNKNOWN_LOCATION, fn, arglist);
-  return fold_convert (build_pointer_type (TREE_TYPE (var)), call);
-}
-\f
-
 /* Subroutine of expand_expr.  Expand the two operands of a binary
    expression EXP0 and EXP1 placing the results in OP0 and OP1.
    The value may be stored in TARGET if TARGET is nonzero.  The
@@ -6945,18 +6931,6 @@ expand_expr_addr_expr_1 (tree exp, rtx target, enum machine_mode tmode,
       inner = TREE_OPERAND (exp, 0);
       break;
 
-    case VAR_DECL:
-      /* TLS emulation hook - replace __thread VAR's &VAR with
-	 __emutls_get_address (&_emutls.VAR).  */
-      if (! targetm.have_tls
-	  && TREE_CODE (exp) == VAR_DECL
-	  && DECL_THREAD_LOCAL_P (exp))
-	{
-	  exp = emutls_var_address (exp);
-	  return expand_expr (exp, target, tmode, modifier);
-	}
-      /* Fall through.  */
-
     default:
       /* If the object is a DECL, then expand it for its rtl.  Don't bypass
 	 expand_expr, as that can have various side effects; LABEL_DECLs for
@@ -8394,16 +8368,6 @@ expand_expr_real_1 (tree exp, rtx target, enum machine_mode tmode,
 	  && (TREE_STATIC (exp) || DECL_EXTERNAL (exp)))
 	layout_decl (exp, 0);
 
-      /* TLS emulation hook - replace __thread vars with
-	 *__emutls_get_address (&_emutls.var).  */
-      if (! targetm.have_tls
-	  && TREE_CODE (exp) == VAR_DECL
-	  && DECL_THREAD_LOCAL_P (exp))
-	{
-	  exp = build_fold_indirect_ref_loc (loc, emutls_var_address (exp));
-	  return expand_expr_real_1 (exp, target, tmode, modifier, NULL);
-	}
-
       /* ... fall through ...  */
 
     case FUNCTION_DECL:
diff --git a/gcc/passes.c b/gcc/passes.c
index 8828967..ecd45b2 100644
--- a/gcc/passes.c
+++ b/gcc/passes.c
@@ -746,6 +746,7 @@ init_optimization_passes (void)
   /* Interprocedural optimization passes.  */
   p = &all_small_ipa_passes;
   NEXT_PASS (pass_ipa_free_lang_data);
+  NEXT_PASS (pass_ipa_lower_emutls);
   NEXT_PASS (pass_ipa_function_and_variable_visibility);
   NEXT_PASS (pass_ipa_early_inline);
     {
diff --git a/gcc/tree-emutls.c b/gcc/tree-emutls.c
new file mode 100644
index 0000000..b1b9e4d
--- /dev/null
+++ b/gcc/tree-emutls.c
@@ -0,0 +1,609 @@
+/* Lower TLS operations to emulation functions.
+   Copyright (C) 2006, 2007, 2008, 2009, 2010
+   Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 3, or (at your option) any
+later version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "tree.h"
+#include "gimple.h"
+#include "tree-pass.h"
+#include "tree-flow.h"
+#include "cgraph.h"
+#include "langhooks.h"
+#include "target.h"
+
+/* ??? Should go.  */
+#include "targhooks.h"
+#include "tree-iterator.h"
+#include "output.h"
+
+/* TODO: Get rid of the EMUTLS hooks that no one uses.  */
+
+/* Whenever a target does not support thread-local storage (TLS) natively,
+   we can emulate it with some run-time support in libgcc.  This will in
+   turn rely on "keyed storage" a-la pthread_key_create; essentially all
+   thread libraries provide such functionality.
+
+   In order to coordinate with the libgcc runtime, each TLS variable is
+   described by a "control variable".  This control variable records the
+   required size, alignment, and initial value of the TLS variable for
+   instantiation at runtime.  It also stores an integer token to be used
+   by the runtime to find the address of the variable within each thread.
+
+   On the compiler side, this means that we need to replace all instances
+   of "tls_var" in the code with "*__emutls_get_addr(&control_var)".  We
+   also need to eliminate "tls_var" from the symbol table and introduce
+   "control_var".
+
+   We used to perform all of the transformations during conversion to rtl,
+   and the variable substitutions magically within assemble_variable.
+   However, this late fiddling of the symbol table conflicts with LTO and
+   whole-program compilation.  Therefore we must now make all the changes
+   to the symbol table early in the GIMPLE optimization path, before we
+   write things out to LTO intermediate files.  */
+
+/* These two vectors, once fully populated, are kept in lock-step so that
+   the index of a TLS variable equals the index of its control variable in
+   the other vector.  */
+static GTY (()) varpool_node_set tls_vars;
+static GTY (()) varpool_node_set control_vars;
+
+/* The type of the control structure, shared with the emutls.c runtime.  */
+/* ??? See if we can eliminate the one query via decl_emutls_var_p from
+   varasm.c.  With that gone, this need not be live outside the ipa pass.  */
+static GTY (()) tree emutls_object_type;
+
+/* Emulated TLS objects have the TLS model TLS_MODEL_EMULATED.  This
+   macro can be used on them to distinguish the control variable from
+   the initialization template.  */
+
+bool
+decl_emutls_var_p(tree decl)
+{
+  return TREE_TYPE (decl) == emutls_object_type;
+}
+
+#if !defined (NO_DOT_IN_LABEL)
+# define EMUTLS_SEPARATOR	"."
+#elif !defined (NO_DOLLAR_IN_LABEL)
+# define EMUTLS_SEPARATOR	"$"
+#else
+# define EMUTLS_SEPARATOR	"_"
+#endif
+
+/* Create an IDENTIFIER_NODE by prefixing PREFIX to the
+   IDENTIFIER_NODE NAME's name.  */
+
+static tree
+prefix_name (const char *prefix, tree name)
+{
+  unsigned plen = strlen (prefix);
+  unsigned nlen = strlen (IDENTIFIER_POINTER (name));
+  char *toname = (char *) alloca (plen + nlen + 1);
+
+  memcpy (toname, prefix, plen);
+  memcpy (toname + plen, IDENTIFIER_POINTER (name), nlen + 1);
+
+  return get_identifier (toname);
+}
+
+/* Create an identifier for the struct __emutls_object, given an identifier
+   of the DECL_ASSEMBLY_NAME of the original object.  */
+
+tree
+get_emutls_object_name (tree name)
+{
+  const char *prefix = (targetm.emutls.var_prefix
+			? targetm.emutls.var_prefix
+			: "__emutls_v" EMUTLS_SEPARATOR);
+  return prefix_name (prefix, name);
+}
+
+tree
+default_emutls_var_fields (tree type, tree *name ATTRIBUTE_UNUSED)
+{
+  tree word_type_node, field, next_field;
+
+  field = build_decl (UNKNOWN_LOCATION,
+		      FIELD_DECL, get_identifier ("__templ"), ptr_type_node);
+  DECL_CONTEXT (field) = type;
+  next_field = field;
+
+  field = build_decl (UNKNOWN_LOCATION,
+		      FIELD_DECL, get_identifier ("__offset"),
+		      ptr_type_node);
+  DECL_CONTEXT (field) = type;
+  TREE_CHAIN (field) = next_field;
+  next_field = field;
+
+  word_type_node = lang_hooks.types.type_for_mode (word_mode, 1);
+  field = build_decl (UNKNOWN_LOCATION,
+		      FIELD_DECL, get_identifier ("__align"),
+		      word_type_node);
+  DECL_CONTEXT (field) = type;
+  TREE_CHAIN (field) = next_field;
+  next_field = field;
+
+  field = build_decl (UNKNOWN_LOCATION,
+		      FIELD_DECL, get_identifier ("__size"), word_type_node);
+  DECL_CONTEXT (field) = type;
+  TREE_CHAIN (field) = next_field;
+
+  return field;
+}
+
+/* Initialize emulated tls object TO, which refers to TLS variable
+   DECL and is initialized by PROXY.  */
+
+tree
+default_emutls_var_init (tree to, tree decl, tree proxy)
+{
+  VEC(constructor_elt,gc) *v = VEC_alloc (constructor_elt, gc, 4);
+  constructor_elt *elt;
+  tree type = TREE_TYPE (to);
+  tree field = TYPE_FIELDS (type);
+
+  elt = VEC_quick_push (constructor_elt, v, NULL);
+  elt->index = field;
+  elt->value = fold_convert (TREE_TYPE (field), DECL_SIZE_UNIT (decl));
+
+  elt = VEC_quick_push (constructor_elt, v, NULL);
+  field = TREE_CHAIN (field);
+  elt->index = field;
+  elt->value = build_int_cst (TREE_TYPE (field),
+			      DECL_ALIGN_UNIT (decl));
+
+  elt = VEC_quick_push (constructor_elt, v, NULL);
+  field = TREE_CHAIN (field);
+  elt->index = field;
+  elt->value = null_pointer_node;
+
+  elt = VEC_quick_push (constructor_elt, v, NULL);
+  field = TREE_CHAIN (field);
+  elt->index = field;
+  elt->value = proxy;
+
+  return build_constructor (type, v);
+}
+
+/* Create the structure for struct __emutls_object.  This should match the
+   structure at the top of emutls.c, modulo the union there.  */
+
+static tree
+get_emutls_object_type (void)
+{
+  tree type, type_name, field;
+
+  type = emutls_object_type;
+  if (type)
+    return type;
+
+  emutls_object_type = type = lang_hooks.types.make_type (RECORD_TYPE);
+  type_name = NULL;
+  field = targetm.emutls.var_fields (type, &type_name);
+  if (!type_name)
+    type_name = get_identifier ("__emutls_object");
+  type_name = build_decl (UNKNOWN_LOCATION,
+			  TYPE_DECL, type_name, type);
+  TYPE_NAME (type) = type_name;
+  TYPE_FIELDS (type) = field;
+  layout_type (type);
+
+  return type;
+}
+
+/* Create a read-only variable like DECL, with the same DECL_INITIAL.
+   This will be used for initializing the emulated tls data area.  */
+
+static tree
+get_emutls_init_templ_addr (tree decl)
+{
+  tree name, to;
+
+  if (targetm.emutls.register_common && !DECL_INITIAL (decl)
+      && !DECL_SECTION_NAME (decl))
+    return null_pointer_node;
+
+  name = DECL_ASSEMBLER_NAME (decl);
+  if (!targetm.emutls.tmpl_prefix || targetm.emutls.tmpl_prefix[0])
+    {
+      const char *prefix = (targetm.emutls.tmpl_prefix
+			    ? targetm.emutls.tmpl_prefix
+			    : "__emutls_t" EMUTLS_SEPARATOR);
+      name = prefix_name (prefix, name);
+    }
+
+  to = build_decl (DECL_SOURCE_LOCATION (decl),
+		   VAR_DECL, name, TREE_TYPE (decl));
+  SET_DECL_ASSEMBLER_NAME (to, DECL_NAME (to));
+
+  DECL_ARTIFICIAL (to) = 1;
+  TREE_USED (to) = TREE_USED (decl);
+  TREE_READONLY (to) = 1;
+  DECL_IGNORED_P (to) = 1;
+  DECL_CONTEXT (to) = DECL_CONTEXT (decl);
+  DECL_SECTION_NAME (to) = DECL_SECTION_NAME (decl);
+  DECL_PRESERVE_P (to) = DECL_PRESERVE_P (decl);
+
+  DECL_WEAK (to) = DECL_WEAK (decl);
+  if (DECL_ONE_ONLY (decl))
+    {
+      make_decl_one_only (to, DECL_ASSEMBLER_NAME (to));
+      TREE_STATIC (to) = TREE_STATIC (decl);
+      TREE_PUBLIC (to) = TREE_PUBLIC (decl);
+      DECL_VISIBILITY (to) = DECL_VISIBILITY (decl);
+    }
+  else
+    TREE_STATIC (to) = 1;
+
+  DECL_VISIBILITY_SPECIFIED (to) = DECL_VISIBILITY_SPECIFIED (decl);
+  DECL_INITIAL (to) = DECL_INITIAL (decl);
+  DECL_INITIAL (decl) = NULL;
+
+  varpool_finalize_decl (to);
+  return build_fold_addr_expr (to);
+}
+
+static tree
+new_emutls_decl (tree decl)
+{
+  tree name, to;
+
+  name = DECL_ASSEMBLER_NAME (decl);
+  to = build_decl (DECL_SOURCE_LOCATION (decl), VAR_DECL,
+                   get_emutls_object_name (name),
+                   get_emutls_object_type ());
+
+  SET_DECL_ASSEMBLER_NAME (to, DECL_NAME (to));
+
+  DECL_TLS_MODEL (to) = TLS_MODEL_EMULATED;
+  DECL_ARTIFICIAL (to) = 1;
+  DECL_IGNORED_P (to) = 1;
+  TREE_READONLY (to) = 0;
+  TREE_STATIC (to) = 1;
+
+  DECL_PRESERVE_P (to) = DECL_PRESERVE_P (decl);
+  DECL_CONTEXT (to) = DECL_CONTEXT (decl);
+  TREE_USED (to) = TREE_USED (decl);
+  TREE_PUBLIC (to) = TREE_PUBLIC (decl);
+  DECL_EXTERNAL (to) = DECL_EXTERNAL (decl);
+  DECL_COMMON (to) = DECL_COMMON (decl);
+  DECL_WEAK (to) = DECL_WEAK (decl);
+  DECL_VISIBILITY (to) = DECL_VISIBILITY (decl);
+  DECL_VISIBILITY_SPECIFIED (to) = DECL_VISIBILITY_SPECIFIED (decl);
+  DECL_RESTRICTED_P (to) = DECL_RESTRICTED_P (decl);
+  DECL_DLLIMPORT_P (to) = DECL_DLLIMPORT_P (decl);
+
+  DECL_ATTRIBUTES (to) = targetm.merge_decl_attributes (decl, to);
+
+  if (DECL_ONE_ONLY (decl))
+    make_decl_one_only (to, DECL_ASSEMBLER_NAME (to));
+
+  /* ??? What in the world is this for?  */
+  if (targetm.emutls.var_align_fixed)
+    /* If we're not allowed to change the proxy object's
+       alignment, pretend it's been set by the user.  */
+    DECL_USER_ALIGN (to) = 1;
+
+  /* If this variable is defined locally, then we need to initialize the
+     control structure with size and alignment information.  Initialization
+     of COMMON block variables happens elsewhere via a constructor.  */
+  if (!DECL_EXTERNAL (to)
+      && (!DECL_COMMON (to)
+          || (DECL_INITIAL (decl)
+              && DECL_INITIAL (decl) != error_mark_node)))
+    {
+      tree tmpl = get_emutls_init_templ_addr (decl);
+      DECL_INITIAL (to) = targetm.emutls.var_init (to, decl, tmpl);
+      record_references_in_initializer (to, false);
+    }
+
+  varpool_finalize_decl (to);
+  return to;
+}
+
+tree
+emutls_decl (tree decl)
+{
+  varpool_node_set_iterator i;
+  struct varpool_node *var;
+  
+  i = varpool_node_set_find (tls_vars, varpool_get_node (decl));
+  if (i.index == ~0u)
+    return NULL;
+
+  var = VEC_index (varpool_node_ptr, control_vars->nodes, i.index);
+  return var->decl;
+}
+
+static void
+emutls_common_1 (tree tls_decl, tree control_decl, tree *pstmts)
+{
+  tree args, x;
+  tree word_type_node;
+
+  if (! DECL_COMMON (tls_decl)
+      || (DECL_INITIAL (tls_decl)
+	  && DECL_INITIAL (tls_decl) != error_mark_node))
+    return;
+
+  word_type_node = lang_hooks.types.type_for_mode (word_mode, 1);
+
+  /* ??? The idea was to call get_emutls_init_templ_addr here, but if we
+     do this and there is an initializer, -fanchor_section loses, because
+     it would be too late to ensure the template is output.  */
+  /* ??? Except that if we *can* do this during the IPA pass, then it
+     isn't too late.  */
+  x = null_pointer_node;
+  args = tree_cons (NULL, x, NULL);
+  x = build_int_cst (word_type_node, DECL_ALIGN_UNIT (tls_decl));
+  args = tree_cons (NULL, x, args);
+  x = fold_convert (word_type_node, DECL_SIZE_UNIT (tls_decl));
+  args = tree_cons (NULL, x, args);
+  x = build_fold_addr_expr (control_decl);
+  args = tree_cons (NULL, x, args);
+
+  x = built_in_decls[BUILT_IN_EMUTLS_REGISTER_COMMON];
+  x = build_function_call_expr (UNKNOWN_LOCATION, x, args);
+
+  append_to_statement_list (x, pstmts);
+}
+
+/* Finalize emutls control vars and add a static constructor if required.  */
+/* ??? Can this be done as early as the IPA pass?  */
+
+void
+emutls_finish (void)
+{
+  tree body = NULL_TREE;
+  unsigned int i, n;
+
+  n = VEC_length (varpool_node_ptr, tls_vars->nodes);
+  for (i = 0; i < n; ++i)
+    emutls_common_1 (VEC_index (varpool_node_ptr, tls_vars->nodes, i)->decl,
+		     VEC_index (varpool_node_ptr, control_vars->nodes, i)->decl,
+                     &body);
+
+  if (body == NULL_TREE)
+    return;
+
+  cgraph_build_static_cdtor ('I', body, DEFAULT_INIT_PRIORITY);
+}
+
+\f
+
+struct lower_emutls_data
+{
+  struct cgraph_node *cfun_node;
+  struct cgraph_node *builtin_node;
+  tree builtin_decl;
+  basic_block bb;
+  int bb_freq;
+  gimple_stmt_iterator gsi;
+  gimple stmt;
+};
+
+/* Given an operand *PTR within STMT at *GSI, if the operand references
+   a TLS variable, then lower the reference to a call to the runtime.  */
+
+static bool
+lower_emutls_1 (struct lower_emutls_data *d, tree *ptr)
+{
+  tree t = *ptr;
+  tree control, addr;
+  gimple x;
+
+  /* In the case of "&var" we don't want to generate "&*addr",
+     we'd prefer to simply emit "addr".  */
+  if (TREE_CODE (t) == ADDR_EXPR)
+    {
+      if (lower_emutls_1 (d, &TREE_OPERAND (t, 0)))
+	{
+	  if (TREE_CODE (TREE_OPERAND (t, 0)) == MEM_REF
+	      && integer_zerop (TREE_OPERAND (TREE_OPERAND (t, 0), 1)))
+	    *ptr = TREE_OPERAND (TREE_OPERAND (t, 0), 0);
+	  return true;
+	}
+      return false;
+    }
+
+  /* Look through all components.  */
+  while (handled_component_p (t))
+    {
+      ptr = &TREE_OPERAND (t, 0);
+      t = *ptr;
+    }
+
+  /* If at the end we don't have a TLS variable, nothing to do.  */
+  if (TREE_CODE (t) != VAR_DECL || !DECL_THREAD_LOCAL_P (t))
+    return false;
+
+  /* Compute the address of the TLS variable with help from runtime.  */
+  control = emutls_decl (t);
+  TREE_ADDRESSABLE (control) = 1;
+  addr = create_tmp_var (build_pointer_type (TREE_TYPE (t)), NULL);
+  x = gimple_build_call (d->builtin_decl, 1,
+	                 build_fold_addr_expr (control));
+  gimple_call_set_lhs (x, addr);
+  gsi_insert_before (&d->gsi, x, GSI_SAME_STMT);
+  gimple_set_location (x, gimple_location (d->stmt));
+
+  cgraph_create_edge (d->cfun_node, d->builtin_node, x,
+                      d->bb->count, d->bb_freq, d->bb->loop_depth);
+
+  /* Replace "var" with "*addr" in the statement.  */
+  t = build2 (MEM_REF, TREE_TYPE (t), addr,
+	      build_int_cst (TREE_TYPE (addr), 0));
+  *ptr = t;
+  gimple_set_modified (d->stmt, true);
+
+  return true;
+}
+
+static void
+lower_emutls_function_body (struct cgraph_node *node)
+{
+  struct lower_emutls_data d;
+
+  current_function_decl = node->decl;
+  push_cfun (DECL_STRUCT_FUNCTION (node->decl));
+
+  d.cfun_node = node;
+  d.builtin_decl = built_in_decls[BUILT_IN_EMUTLS_GET_ADDRESS];
+  d.builtin_node = cgraph_node (d.builtin_decl);
+
+  FOR_EACH_BB (d.bb)
+    {
+      d.bb_freq = compute_call_stmt_bb_frequency (current_function_decl, d.bb);
+
+      for (d.gsi = gsi_start_bb (d.bb); !gsi_end_p (d.gsi); gsi_next (&d.gsi))
+	{
+	  gimple stmt = gsi_stmt (d.gsi);
+
+          d.stmt = stmt;
+	  if (gimple_assign_single_p (stmt))
+	    {
+	      lower_emutls_1 (&d, gimple_assign_lhs_ptr (stmt));
+	      lower_emutls_1 (&d, gimple_assign_rhs1_ptr (stmt));
+	    }
+	  else if (is_gimple_call (stmt))
+	    {
+	      unsigned i, n = gimple_call_num_args (stmt);
+	      for (i = 0; i < n; ++i)
+		lower_emutls_1 (&d, gimple_call_arg_ptr (stmt, i));
+	      lower_emutls_1 (&d, gimple_call_lhs_ptr (stmt));
+	    }
+	  else
+	    continue;
+
+	  update_stmt_if_modified (stmt);
+	}
+    }
+
+  pop_cfun ();
+  current_function_decl = NULL;
+}
+
+static unsigned int
+ipa_lower_emutls (void)
+{
+  struct varpool_node *var;
+  struct cgraph_node *func;
+  bool any_aliases = false;
+  unsigned int i;
+
+  tls_vars = varpool_node_set_new ();
+
+  /* Examine all global variables for TLS variables.  */
+  for (var = varpool_nodes; var ; var = var->next)
+    if (DECL_THREAD_LOCAL_P (var->decl))
+      {
+	gcc_checking_assert (TREE_STATIC (var->decl));
+        if (var->alias)
+	  any_aliases = true;
+        else
+	  varpool_node_set_add (tls_vars, var);
+      }
+
+  /* If we found no TLS variables, then there is no further work to do.  */
+  if (tls_vars->nodes == NULL)
+    {
+      tls_vars = NULL;
+      if (dump_file)
+	fprintf (dump_file, "No TLS variables found.\n");
+      return 0;
+    }
+
+  /* If there were any aliases, add them to the set last.  In this way
+     when we create the control variables the control variable for the
+     alias base will have been created first.  */
+  if (any_aliases)
+    for (var = varpool_nodes; var ; var = var->next)
+      if (DECL_THREAD_LOCAL_P (var->decl) && var->alias)
+        varpool_node_set_add (tls_vars, var);
+
+  /* Create the control variables for each TLS variable.  */
+  control_vars = varpool_node_set_new ();
+  for (i = 0; VEC_iterate (varpool_node_ptr, tls_vars->nodes, i, var); ++i)
+    {
+      tree cdecl;
+      struct varpool_node *cvar;
+
+      var = VEC_index (varpool_node_ptr, tls_vars->nodes, i);
+      cdecl = new_emutls_decl (var->decl);
+
+      /* If there was an alias between the TLS variables,
+         mirror that in the control variables.  */
+      /* ??? There appears to be an alias_pairs data structure that
+         holds similar information.  */
+      if (var->alias)
+        {
+	  tree cbase = emutls_decl (var->extra_name->decl);
+          bool ok = varpool_extra_name_alias (cdecl, cbase);
+          gcc_assert (ok);
+        }
+
+      cvar = varpool_get_node (cdecl);
+      varpool_node_set_add (control_vars, cvar);
+
+      /* Indicate that the value of the TLS variable may be found elsewhere.
+	 This also prevents the variable from re-appearing in the GIMPLE.  */
+      /* ??? Unfortuantely, there's no decent actual value to put here;
+	 there's nothing we can emit for the debugger at the moment.  */
+      SET_DECL_VALUE_EXPR (var->decl, error_mark_node);
+      DECL_HAS_VALUE_EXPR_P (var->decl) = 1;
+    }
+
+  /* Adjust all uses of TLS variables within the function bodies.  */
+  for (func = cgraph_nodes; func; func = func->next)
+    if (func->reachable && func->lowered)
+      lower_emutls_function_body (func);
+
+  return TODO_dump_func | TODO_ggc_collect | TODO_verify_stmts;
+}
+
+/* If the target supports TLS natively, we need do nothing here.  */
+
+static bool
+gate_emutls (void)
+{
+  return !targetm.have_tls;
+}
+
+struct simple_ipa_opt_pass pass_ipa_lower_emutls =
+{
+ {
+  SIMPLE_IPA_PASS,
+  "emutls",				/* name */
+  gate_emutls,				/* gate */
+  ipa_lower_emutls,			/* execute */
+  NULL,                                 /* sub */
+  NULL,                                 /* next */
+  0,                                    /* static_pass_number */
+  TV_NONE,				/* tv_id */
+  PROP_cfg,				/* properties_required */
+  0,                                    /* properties_provided */
+  0,                                    /* properties_destroyed */
+  0,                                    /* todo_flags_start */
+  0,					/* todo_flags_finish */
+ }
+};
+
+#include "gt-tree-emutls.h"
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index a4c97b3..b309be8 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -445,6 +445,7 @@ extern struct gimple_opt_pass pass_warn_unused_result;
 extern struct gimple_opt_pass pass_split_functions;
 
 /* IPA Passes */
+extern struct simple_ipa_opt_pass pass_ipa_lower_emutls;
 extern struct simple_ipa_opt_pass pass_ipa_function_and_variable_visibility;
 extern struct simple_ipa_opt_pass pass_ipa_early_inline;
 
diff --git a/gcc/tree.h b/gcc/tree.h
index 960ee7d..28bc47a 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -5257,9 +5257,13 @@ extern void set_user_assembler_name (tree, const char *);
 extern void process_pending_assemble_externals (void);
 extern void finish_aliases_1 (void);
 extern void finish_aliases_2 (void);
-extern tree emutls_decl (tree);
 extern void remove_unreachable_alias_pairs (void);
 
+/* tree-emutls.c */
+extern tree emutls_decl (tree);
+extern tree get_emutls_object_name (tree);
+extern bool decl_emutls_var_p (tree);
+
 /* In stmt.c */
 extern void expand_computed_goto (tree);
 extern bool parse_output_constraint (const char **, int, int, int,
diff --git a/gcc/varasm.c b/gcc/varasm.c
index de78bd0..f15ef0f 100644
--- a/gcc/varasm.c
+++ b/gcc/varasm.c
@@ -186,320 +186,6 @@ static GTY(()) int anchor_labelno;
 /* A pool of constants that can be shared between functions.  */
 static GTY(()) struct rtx_constant_pool *shared_constant_pool;
 
-/* TLS emulation.  */
-
-static GTY ((if_marked ("tree_map_marked_p"), param_is (struct tree_map)))
-     htab_t emutls_htab;
-static GTY (()) tree emutls_object_type;
-/* Emulated TLS objects have the TLS model TLS_MODEL_EMULATED.  This
-   macro can be used on them to distinguish the control variable from
-   the initialization template.  */
-#define DECL_EMUTLS_VAR_P(D)  (TREE_TYPE (D) == emutls_object_type)
-
-#if !defined (NO_DOT_IN_LABEL)
-# define EMUTLS_SEPARATOR	"."
-#elif !defined (NO_DOLLAR_IN_LABEL)
-# define EMUTLS_SEPARATOR	"$"
-#else
-# define EMUTLS_SEPARATOR	"_"
-#endif
-
-/* Create an IDENTIFIER_NODE by prefixing PREFIX to the
-   IDENTIFIER_NODE NAME's name.  */
-
-static tree
-prefix_name (const char *prefix, tree name)
-{
-  unsigned plen = strlen (prefix);
-  unsigned nlen = strlen (IDENTIFIER_POINTER (name));
-  char *toname = (char *) alloca (plen + nlen + 1);
-
-  memcpy (toname, prefix, plen);
-  memcpy (toname + plen, IDENTIFIER_POINTER (name), nlen + 1);
-
-  return get_identifier (toname);
-}
-
-/* Create an identifier for the struct __emutls_object, given an identifier
-   of the DECL_ASSEMBLY_NAME of the original object.  */
-
-static tree
-get_emutls_object_name (tree name)
-{
-  const char *prefix = (targetm.emutls.var_prefix
-			? targetm.emutls.var_prefix
-			: "__emutls_v" EMUTLS_SEPARATOR);
-  return prefix_name (prefix, name);
-}
-
-tree
-default_emutls_var_fields (tree type, tree *name ATTRIBUTE_UNUSED)
-{
-  tree word_type_node, field, next_field;
-
-  field = build_decl (UNKNOWN_LOCATION,
-		      FIELD_DECL, get_identifier ("__templ"), ptr_type_node);
-  DECL_CONTEXT (field) = type;
-  next_field = field;
-
-  field = build_decl (UNKNOWN_LOCATION,
-		      FIELD_DECL, get_identifier ("__offset"),
-		      ptr_type_node);
-  DECL_CONTEXT (field) = type;
-  TREE_CHAIN (field) = next_field;
-  next_field = field;
-
-  word_type_node = lang_hooks.types.type_for_mode (word_mode, 1);
-  field = build_decl (UNKNOWN_LOCATION,
-		      FIELD_DECL, get_identifier ("__align"),
-		      word_type_node);
-  DECL_CONTEXT (field) = type;
-  TREE_CHAIN (field) = next_field;
-  next_field = field;
-
-  field = build_decl (UNKNOWN_LOCATION,
-		      FIELD_DECL, get_identifier ("__size"), word_type_node);
-  DECL_CONTEXT (field) = type;
-  TREE_CHAIN (field) = next_field;
-
-  return field;
-}
-
-/* Create the structure for struct __emutls_object.  This should match the
-   structure at the top of emutls.c, modulo the union there.  */
-
-static tree
-get_emutls_object_type (void)
-{
-  tree type, type_name, field;
-
-  type = emutls_object_type;
-  if (type)
-    return type;
-
-  emutls_object_type = type = lang_hooks.types.make_type (RECORD_TYPE);
-  type_name = NULL;
-  field = targetm.emutls.var_fields (type, &type_name);
-  if (!type_name)
-    type_name = get_identifier ("__emutls_object");
-  type_name = build_decl (UNKNOWN_LOCATION,
-			  TYPE_DECL, type_name, type);
-  TYPE_NAME (type) = type_name;
-  TYPE_FIELDS (type) = field;
-  layout_type (type);
-
-  return type;
-}
-
-/* Create a read-only variable like DECL, with the same DECL_INITIAL.
-   This will be used for initializing the emulated tls data area.  */
-
-static tree
-get_emutls_init_templ_addr (tree decl)
-{
-  tree name, to;
-
-  if (targetm.emutls.register_common && !DECL_INITIAL (decl)
-      && !DECL_SECTION_NAME (decl))
-    return null_pointer_node;
-
-  name = DECL_ASSEMBLER_NAME (decl);
-  if (!targetm.emutls.tmpl_prefix || targetm.emutls.tmpl_prefix[0])
-    {
-      const char *prefix = (targetm.emutls.tmpl_prefix
-			    ? targetm.emutls.tmpl_prefix
-			    : "__emutls_t" EMUTLS_SEPARATOR);
-      name = prefix_name (prefix, name);
-    }
-
-  to = build_decl (DECL_SOURCE_LOCATION (decl),
-		   VAR_DECL, name, TREE_TYPE (decl));
-  SET_DECL_ASSEMBLER_NAME (to, DECL_NAME (to));
-
-  DECL_ARTIFICIAL (to) = 1;
-  TREE_USED (to) = TREE_USED (decl);
-  TREE_READONLY (to) = 1;
-  DECL_IGNORED_P (to) = 1;
-  DECL_CONTEXT (to) = DECL_CONTEXT (decl);
-  DECL_SECTION_NAME (to) = DECL_SECTION_NAME (decl);
-  DECL_PRESERVE_P (to) = DECL_PRESERVE_P (decl);
-
-  DECL_WEAK (to) = DECL_WEAK (decl);
-  if (DECL_ONE_ONLY (decl))
-    {
-      make_decl_one_only (to, DECL_ASSEMBLER_NAME (to));
-      TREE_STATIC (to) = TREE_STATIC (decl);
-      TREE_PUBLIC (to) = TREE_PUBLIC (decl);
-      DECL_VISIBILITY (to) = DECL_VISIBILITY (decl);
-    }
-  else
-    TREE_STATIC (to) = 1;
-
-  DECL_VISIBILITY_SPECIFIED (to) = DECL_VISIBILITY_SPECIFIED (decl);
-  DECL_INITIAL (to) = DECL_INITIAL (decl);
-  DECL_INITIAL (decl) = NULL;
-
-  varpool_finalize_decl (to);
-  return build_fold_addr_expr (to);
-}
-
-/* When emulating tls, we use a control structure for use by the runtime.
-   Create and return this structure.  */
-
-tree
-emutls_decl (tree decl)
-{
-  tree name, to;
-  struct tree_map *h, in;
-  void **loc;
-
-  if (targetm.have_tls || decl == NULL || decl == error_mark_node
-      || TREE_CODE (decl) != VAR_DECL || ! DECL_THREAD_LOCAL_P (decl))
-    return decl;
-
-  /* Look up the object in the hash; return the control structure if
-     it has already been created.  */
-  if (! emutls_htab)
-    emutls_htab = htab_create_ggc (512, tree_map_hash, tree_map_eq, 0);
-
-  name = DECL_ASSEMBLER_NAME (decl);
-
-  /* Note that we use the hash of the decl's name, rather than a hash
-     of the decl's pointer.  In emutls_finish we iterate through the
-     hash table, and we want this traversal to be predictable.  */
-  in.hash = IDENTIFIER_HASH_VALUE (name);
-  in.base.from = decl;
-  loc = htab_find_slot_with_hash (emutls_htab, &in, in.hash, INSERT);
-  h = (struct tree_map *) *loc;
-  if (h != NULL)
-    to = h->to;
-  else
-    {
-      to = build_decl (DECL_SOURCE_LOCATION (decl),
-		       VAR_DECL, get_emutls_object_name (name),
-		       get_emutls_object_type ());
-
-      h = ggc_alloc_tree_map ();
-      h->hash = in.hash;
-      h->base.from = decl;
-      h->to = to;
-      *(struct tree_map **) loc = h;
-
-      DECL_TLS_MODEL (to) = TLS_MODEL_EMULATED;
-      DECL_ARTIFICIAL (to) = 1;
-      DECL_IGNORED_P (to) = 1;
-      /* FIXME: work around PR44132.  */
-      DECL_PRESERVE_P (to) = 1;
-      TREE_READONLY (to) = 0;
-      SET_DECL_ASSEMBLER_NAME (to, DECL_NAME (to));
-      if (DECL_ONE_ONLY (decl))
-	make_decl_one_only (to, DECL_ASSEMBLER_NAME (to));
-      DECL_CONTEXT (to) = DECL_CONTEXT (decl);
-      if (targetm.emutls.var_align_fixed)
-	/* If we're not allowed to change the proxy object's
-	   alignment, pretend it's been set by the user.  */
-	DECL_USER_ALIGN (to) = 1;
-    }
-
-  /* Note that these fields may need to be updated from time to time from
-     the original decl.  Consider:
-	extern __thread int i;
-	int foo() { return i; }
-	__thread int i = 1;
-     in which I goes from external to locally defined and initialized.  */
-  DECL_DLLIMPORT_P (to) = DECL_DLLIMPORT_P (decl);
-  DECL_ATTRIBUTES (to) = targetm.merge_decl_attributes (decl, to);
-
-  TREE_STATIC (to) = TREE_STATIC (decl);
-  TREE_USED (to) = TREE_USED (decl);
-  TREE_PUBLIC (to) = TREE_PUBLIC (decl);
-  DECL_EXTERNAL (to) = DECL_EXTERNAL (decl);
-  DECL_COMMON (to) = DECL_COMMON (decl);
-  DECL_WEAK (to) = DECL_WEAK (decl);
-  DECL_VISIBILITY (to) = DECL_VISIBILITY (decl);
-  DECL_VISIBILITY_SPECIFIED (to) = DECL_VISIBILITY_SPECIFIED (decl);
-  
-  /* Fortran might pass this to us.  */
-  DECL_RESTRICTED_P (to) = DECL_RESTRICTED_P (decl);
-
-  return to;
-}
-
-static int
-emutls_common_1 (void **loc, void *xstmts)
-{
-  struct tree_map *h = *(struct tree_map **) loc;
-  tree args, x, *pstmts = (tree *) xstmts;
-  tree word_type_node;
-
-  if (! DECL_COMMON (h->base.from)
-      || (DECL_INITIAL (h->base.from)
-	  && DECL_INITIAL (h->base.from) != error_mark_node))
-    return 1;
-
-  word_type_node = lang_hooks.types.type_for_mode (word_mode, 1);
-
-  /* The idea was to call get_emutls_init_templ_addr here, but if we
-     do this and there is an initializer, -fanchor_section loses,
-     because it would be too late to ensure the template is
-     output.  */
-  x = null_pointer_node;
-  args = tree_cons (NULL, x, NULL);
-  x = build_int_cst (word_type_node, DECL_ALIGN_UNIT (h->base.from));
-  args = tree_cons (NULL, x, args);
-  x = fold_convert (word_type_node, DECL_SIZE_UNIT (h->base.from));
-  args = tree_cons (NULL, x, args);
-  x = build_fold_addr_expr (h->to);
-  args = tree_cons (NULL, x, args);
-
-  x = built_in_decls[BUILT_IN_EMUTLS_REGISTER_COMMON];
-  x = build_function_call_expr (UNKNOWN_LOCATION, x, args);
-
-  append_to_statement_list (x, pstmts);
-  return 1;
-}
-
-/* Callback to finalize one emutls control variable.  */
-
-static int
-emutls_finalize_control_var (void **loc, 
-				void *unused ATTRIBUTE_UNUSED)
-{
-  struct tree_map *h = *(struct tree_map **) loc;
-  if (h != NULL) 
-    {
-      struct varpool_node *node = varpool_node (h->to);
-      /* Because varpool_finalize_decl () has side-effects,
-         only apply to un-finalized vars.  */
-      if (node && !node->finalized) 
-	varpool_finalize_decl (h->to);
-    }
-  return 1;
-}
-
-/* Finalize emutls control vars and add a static constructor if
-   required.  */
-
-void
-emutls_finish (void)
-{
-  if (emutls_htab == NULL)
-    return;
-  htab_traverse_noresize (emutls_htab, 
-			  emutls_finalize_control_var, NULL);
-
-  if (targetm.emutls.register_common)
-    {
-      tree body = NULL_TREE;
-
-      htab_traverse_noresize (emutls_htab, emutls_common_1, &body);
-      if (body == NULL_TREE)
-	return;
-
-      cgraph_build_static_cdtor ('I', body, DEFAULT_INIT_PRIORITY);
-    }
-}
-
 /* Helper routines for maintaining section_htab.  */
 
 static int
@@ -1213,11 +899,6 @@ get_variable_section (tree decl, bool prefer_noswitch_p)
 		  && ADDR_SPACE_GENERIC_P (as));
       if (DECL_THREAD_LOCAL_P (decl))
 	return tls_comm_section;
-      /* This cannot be common bss for an emulated TLS object without
-	 a register_common hook.  */
-      else if (DECL_TLS_MODEL (decl) == TLS_MODEL_EMULATED
-	       && !targetm.emutls.register_common)
-	;
       else if (TREE_PUBLIC (decl) && bss_initializer_p (decl))
 	return comm_section;
     }
@@ -2101,40 +1782,6 @@ assemble_variable_contents (tree decl, const char *name,
     }
 }
 
-/* Initialize emulated tls object TO, which refers to TLS variable
-   DECL and is initialized by PROXY.  */
-
-tree
-default_emutls_var_init (tree to, tree decl, tree proxy)
-{
-  VEC(constructor_elt,gc) *v = VEC_alloc (constructor_elt, gc, 4);
-  constructor_elt *elt;
-  tree type = TREE_TYPE (to);
-  tree field = TYPE_FIELDS (type);
-
-  elt = VEC_quick_push (constructor_elt, v, NULL);
-  elt->index = field;
-  elt->value = fold_convert (TREE_TYPE (field), DECL_SIZE_UNIT (decl));
-
-  elt = VEC_quick_push (constructor_elt, v, NULL);
-  field = TREE_CHAIN (field);
-  elt->index = field;
-  elt->value = build_int_cst (TREE_TYPE (field),
-			      DECL_ALIGN_UNIT (decl));
-
-  elt = VEC_quick_push (constructor_elt, v, NULL);
-  field = TREE_CHAIN (field);
-  elt->index = field;
-  elt->value = null_pointer_node;
-
-  elt = VEC_quick_push (constructor_elt, v, NULL);
-  field = TREE_CHAIN (field);
-  elt->index = field;
-  elt->value = proxy;
-
-  return build_constructor (type, v);
-}
-
 /* Assemble everything that is needed for a variable or function declaration.
    Not used for automatic variables, and not used for function definitions.
    Should not be called for variables of incomplete structure type.
@@ -2153,35 +1800,9 @@ assemble_variable (tree decl, int top_level ATTRIBUTE_UNUSED,
   rtx decl_rtl, symbol;
   section *sect;
 
-  if (! targetm.have_tls
-      && TREE_CODE (decl) == VAR_DECL
-      && DECL_THREAD_LOCAL_P (decl))
-    {
-      tree to = emutls_decl (decl);
-
-      /* If this variable is defined locally, then we need to initialize the
-         control structure with size and alignment information.  We do this
-	 at the last moment because tentative definitions can take a locally
-	 defined but uninitialized variable and initialize it later, which
-	 would result in incorrect contents.  */
-      if (! DECL_EXTERNAL (to)
-	  && (! DECL_COMMON (to)
-	      || (DECL_INITIAL (decl)
-		  && DECL_INITIAL (decl) != error_mark_node)))
-	{
-	  DECL_INITIAL (to) = targetm.emutls.var_init
-	    (to, decl, get_emutls_init_templ_addr (decl));
-
-	  /* Make sure the template is marked as needed early enough.
-	     Without this, if the variable is placed in a
-	     section-anchored block, the template will only be marked
-	     when it's too late.  */
-	  record_references_in_initializer (to, false);
-	}
-
-      decl = to;
-    }
-
+  /* Emulated TLS had better not get this far.  */
+  gcc_assert (targetm.have_tls || !DECL_THREAD_LOCAL_P (decl));
+              
   last_assemble_variable_decl = 0;
 
   /* Normally no need to say anything here for external references,
@@ -6410,7 +6031,7 @@ categorize_decl_for_section (const_tree decl, int reloc)
     {
       if (DECL_TLS_MODEL (decl) == TLS_MODEL_EMULATED)
 	{
-	  if (DECL_EMUTLS_VAR_P (decl))
+	  if (decl_emutls_var_p (decl))
 	    {
 	      if (targetm.emutls.var_section)
 		ret = SECCAT_EMUTLS_VAR;