Re: Updating patch regarding bypassing assembler when generating LTO object files

[Iain Sandoe <iains.gcc@gmail.com>]

Hi Ankur,

> On 8 May 2022, at 15:29, Ankur Saini via Gcc <gcc@gcc.gnu.org> wrote:
> 

> I have been fiddling around with the patch regarding "bypassing assmebler
> while generating slim lto files" and managed to make it build with the
> current trunk. Though the patch seems to be working on Linux machine, it
> causes an ICE on macOS (it crashes in langhooks.cc while executing
> `lhd_begin_section ()`).

This is probably because Darwin (macOS) switches to a secondary output
when emitting LTO already, you will probably need to account for that in
the changes.

see gcc/config/darwin.cc: darwin_asm_lto_start() etc.

Iain

> 
> here is the patch ( not fully tested and is still a prototype ), I
> basically mimicked the entire previous patch (
> https://gcc.gnu.org/legacy-ml/gcc/2014-09/msg00340.html ) but on current
> source.
> 
> - Ankur
> 
> --- 
> 
> From e5d72a73962c34dcd818c17d440eb98ddd94e624 Mon Sep 17 00:00:00 2001
> From: Ankur Saini <arsenic@sourceware.org>
> Date: Sun, 8 May 2022 07:03:51 +0530
> Subject: [PATCH] bypass-asm prototype: 1
> 
> ---
> gcc/Makefile.in             |  1 +
> gcc/common.opt              |  3 +++
> gcc/langhooks.cc            | 29 +++++++++++++++++++++++++++-
> gcc/{lto => }/lto-object.cc | 29 +++++++++++++++++++++-------
> gcc/lto-streamer.h          | 35 ++++++++++++++++++++++++++++++++++
> gcc/lto/Make-lang.in        |  4 ++--
> gcc/lto/lto-common.cc       |  3 ++-
> gcc/lto/lto-lang.cc         |  1 +
> gcc/lto/lto.h               | 38 -------------------------------------
> gcc/toplev.cc               | 19 ++++++++++++++++---
> 10 files changed, 110 insertions(+), 52 deletions(-)
> rename gcc/{lto => }/lto-object.cc (94%)
> 
> diff --git a/gcc/Makefile.in b/gcc/Makefile.in
> index 31ff95500c9..3d1241d5819 100644
> --- a/gcc/Makefile.in
> +++ b/gcc/Makefile.in
> @@ -1509,6 +1509,7 @@ OBJS = \
> 	lto-section-out.o \
> 	lto-opts.o \
> 	lto-compress.o \
> +	lto-object.o \
> 	mcf.o \
> 	mode-switching.o \
> 	modulo-sched.o \
> diff --git a/gcc/common.opt b/gcc/common.opt
> index 8a0dafc522d..6348197380d 100644
> --- a/gcc/common.opt
> +++ b/gcc/common.opt
> @@ -1158,6 +1158,9 @@ fbtr-bb-exclusive
> Common Ignore
> Does nothing.  Preserved for backward compatibility.
> 
> +fbypass-asm=
> +Common Joined Var(flag_bypass_asm)
> +
> fcallgraph-info
> Common RejectNegative Var(flag_callgraph_info) Init(NO_CALLGRAPH_INFO);
> Output callgraph information on a per-file basis.
> diff --git a/gcc/langhooks.cc b/gcc/langhooks.cc
> index df970678a08..abfab7a27e0 100644
> --- a/gcc/langhooks.cc
> +++ b/gcc/langhooks.cc
> @@ -38,6 +38,10 @@ along with GCC; see the file COPYING3.  If not see
> #include "stor-layout.h"
> #include "cgraph.h"
> #include "debug.h"
> +#include "function.h"
> +#include "basic-block.h"
> +#include "gimple.h"
> +#include "lto-streamer.h"
> 
> /* Do nothing; in many cases the default hook.  */
> 
> @@ -820,6 +824,19 @@ lhd_begin_section (const char *name)
> {
>   section *section;
> 
> +  if (flag_bypass_asm)
> +    {
> +      static int initialized = false;
> +      if (!initialized)
> +	{
> +	  gcc_assert (asm_out_file == NULL);
> +          lto_set_current_out_file (lto_obj_file_open (asm_file_name, true));
> +	  initialized = true;
> +	}
> +      lto_obj_begin_section (name);
> +      return;
> +    }
> +
>   /* Save the old section so we can restore it in lto_end_asm_section.  */
>   gcc_assert (!saved_section);
>   saved_section = in_section;
> @@ -836,8 +853,13 @@ lhd_begin_section (const char *name)
>    implementation just calls assemble_string.  */
> 
> void
> -lhd_append_data (const void *data, size_t len, void *)
> +lhd_append_data (const void *data, size_t len, void *v)
> {
> +  if (flag_bypass_asm)
> +    {
> +      lto_obj_append_data (data, len, v);
> +      return;
> +    }
>   if (data)
>     {
>       timevar_push (TV_IPA_LTO_OUTPUT);
> @@ -854,6 +876,11 @@ lhd_append_data (const void *data, size_t len, void *)
> void
> lhd_end_section (void)
> {
> +  if (flag_bypass_asm)
> +    {
> +      lto_obj_end_section ();
> +      return;
> +    }
>   if (saved_section)
>     {
>       switch_to_section (saved_section);
> diff --git a/gcc/lto/lto-object.cc b/gcc/lto-object.cc
> similarity index 94%
> rename from gcc/lto/lto-object.cc
> rename to gcc/lto-object.cc
> index 329bbc71fd6..8ccc917a4eb 100644
> --- a/gcc/lto/lto-object.cc
> +++ b/gcc/lto-object.cc
> @@ -21,9 +21,17 @@ along with GCC; see the file COPYING3.  If not see
> #include "config.h"
> #include "system.h"
> #include "coretypes.h"
> -#include "tm.h"
> +#include "tree.h"
> +#include "basic-block.h"
> +#include "tree-ssa-alias.h"
> +#include "internal-fn.h"
> +#include "gimple-expr.h"
> +#include "is-a.h"
> +#include "function.h"
> +#include "gimple.h"
> #include "diagnostic-core.h"
> -#include "lto.h"
> +#include "tm.h"
> +#include "lto-streamer.h"
> #include "lto-section-names.h"
> #include "simple-object.h"
> 
> @@ -133,7 +141,13 @@ lto_obj_file_open (const char *filename, bool writable)
>     }
>   else
>     {
> -      gcc_assert (saved_attributes != NULL);
> +      if (!saved_attributes)
> +        {
> +          lto_file *tmp = lto_obj_file_open (flag_bypass_asm, false);
> +          if (!tmp)
> +            goto fail;
> +          lto_obj_file_close (tmp);
> +        }
>       lo->sobj_w = simple_object_start_write (saved_attributes,
> 					      LTO_SEGMENT_NAME,
> 					      &errmsg, &err);
> @@ -148,7 +162,8 @@ fail_errmsg:
>     error ("%s: %s", fname, errmsg);
>   else
>     error ("%s: %s: %s", fname, errmsg, xstrerror (err));
> -					 
> +
> +fail:
>   if (lo->fd != -1)
>     lto_obj_file_close ((lto_file *) lo);
>   free (lo);
> @@ -255,15 +270,15 @@ lto_obj_add_section (void *data, const char *name, off_t offset,
>    the start and size of each section in the .o file.  */
> 
> htab_t
> -lto_obj_build_section_table (lto_file *lto_file, struct lto_section_list *list)
> +lto_obj_build_section_table (lto_file *lto_file, struct lto_section_list *list,
> +                             htab_t section_hash_table)
> {
>   struct lto_simple_object *lo = (struct lto_simple_object *) lto_file;
> -  htab_t section_hash_table;
> +
>   struct lto_obj_add_section_data loasd;
>   const char *errmsg;
>   int err;
> 
> -  section_hash_table = lto_obj_create_section_hash_table ();
> 
>   gcc_assert (lo->sobj_r != NULL && lo->sobj_w == NULL);
>   loasd.section_hash_table = section_hash_table;
> diff --git a/gcc/lto-streamer.h b/gcc/lto-streamer.h
> index 597e9e405ec..4556583a6a8 100644
> --- a/gcc/lto-streamer.h
> +++ b/gcc/lto-streamer.h
> @@ -759,6 +759,30 @@ public:
>   lto_location_cache location_cache;
> };
> 
> +/* Hash table entry to hold the start offset and length of an LTO
> +   section in a .o file.  */
> +struct lto_section_slot
> +{
> +  const char *name;
> +  intptr_t start;
> +  size_t len;
> +  struct lto_section_slot *next;
> +};
> +
> +/* A list of section slots */
> +struct lto_section_list
> +{
> +  struct lto_section_slot *first, *last;
> +};
> +
> +/* A file.  */
> +struct lto_file
> +{
> +  /* The name of the file.  */
> +  const char *filename;
> +  /* The offset for the object inside an ar archive file (or zero).  */
> +  off_t offset;
> +};
> 
> /* In lto-section-in.cc  */
> extern class lto_input_block * lto_create_simple_input_block (
> @@ -898,6 +922,17 @@ void lto_output_location_and_block (struct output_block *, struct bitpack_d *,
> void lto_output_init_mode_table (void);
> void lto_prepare_function_for_streaming (cgraph_node *);
> 
> +/* In lto-elf.c or lto-coff.c  */
> +extern lto_file *lto_obj_file_open (const char *filename, bool writable);
> +extern void lto_obj_file_close (lto_file *file);
> +struct lto_section_list;
> +extern htab_t lto_obj_build_section_table (lto_file *file, struct lto_section_list *list, htab_t section_hash_table);
> +extern htab_t lto_obj_create_section_hash_table (void);
> +extern void lto_obj_begin_section (const char *name);
> +extern void lto_obj_append_data (const void *data, size_t len, void *block);
> +extern void lto_obj_end_section (void);
> +extern lto_file *lto_set_current_out_file (lto_file *file);
> +extern lto_file *lto_get_current_out_file (void);
> 
> /* In lto-cgraph.cc  */
> extern bool asm_nodes_output;
> diff --git a/gcc/lto/Make-lang.in b/gcc/lto/Make-lang.in
> index a2dcf0dfc12..41a822a317e 100644
> --- a/gcc/lto/Make-lang.in
> +++ b/gcc/lto/Make-lang.in
> @@ -24,9 +24,9 @@ LTO_EXE = lto1$(exeext)
> LTO_DUMP_EXE = lto-dump$(exeext)
> LTO_DUMP_INSTALL_NAME := $(shell echo lto-dump|sed '$(program_transform_name)')
> # The LTO-specific object files inclued in $(LTO_EXE).
> -LTO_OBJS = lto/lto-lang.o lto/lto.o lto/lto-object.o attribs.o lto/lto-partition.o lto/lto-symtab.o lto/lto-common.o
> +LTO_OBJS = lto/lto-lang.o lto/lto.o attribs.o lto/lto-partition.o lto/lto-symtab.o lto/lto-common.o
> lto_OBJS = $(LTO_OBJS)
> -LTO_DUMP_OBJS = lto/lto-lang.o lto/lto-object.o attribs.o lto/lto-partition.o lto/lto-symtab.o lto/lto-dump.o lto/lto-common.o
> +LTO_DUMP_OBJS = lto/lto-lang.o attribs.o lto/lto-partition.o lto/lto-symtab.o lto/lto-dump.o lto/lto-common.o
> lto_dump_OBJS = $(LTO_DUMP_OBJS)
> 
> # this is only useful in a LTO bootstrap, but this does not work right
> diff --git a/gcc/lto/lto-common.cc b/gcc/lto/lto-common.cc
> index ca28586a1e8..4a0c58cdbcd 100644
> --- a/gcc/lto/lto-common.cc
> +++ b/gcc/lto/lto-common.cc
> @@ -2306,7 +2306,8 @@ lto_file_read (lto_file *file, FILE *resolution_file, int *count)
>   struct lto_section_list section_list;
> 
>   memset (&section_list, 0, sizeof (struct lto_section_list));
> -  section_hash_table = lto_obj_build_section_table (file, &section_list);
> +  section_hash_table = lto_obj_create_section_hash_table ();
> +  section_hash_table = lto_obj_build_section_table (file, &section_list, section_hash_table);
> 
>   /* Dump the details of LTO objects.  */
>   if (flag_lto_dump_objects)
> diff --git a/gcc/lto/lto-lang.cc b/gcc/lto/lto-lang.cc
> index 8d58d924dff..64446fc9085 100644
> --- a/gcc/lto/lto-lang.cc
> +++ b/gcc/lto/lto-lang.cc
> @@ -35,6 +35,7 @@ along with GCC; see the file COPYING3.  If not see
> #include "lto-tree.h"
> #include "lto.h"
> #include "lto-common.h"
> +#include "lto-streamer.h"
> #include "stringpool.h"
> #include "attribs.h"
> 
> diff --git a/gcc/lto/lto.h b/gcc/lto/lto.h
> index 9f7fae95bc4..bb0a2852250 100644
> --- a/gcc/lto/lto.h
> +++ b/gcc/lto/lto.h
> @@ -21,16 +21,6 @@ along with GCC; see the file COPYING3.  If not see
> #ifndef LTO_H
> #define LTO_H
> 
> -
> -/* A file.  */
> -struct lto_file
> -{
> -  /* The name of the file.  */
> -  const char *filename;
> -  /* The offset for the object inside an ar archive file (or zero).  */
> -  off_t offset;
> -};
> -
> /* In lto-lang.cc  */
> extern const char *resolution_file_name;
> 
> @@ -39,36 +29,8 @@ extern tree lto_eh_personality (void);
> extern void lto_main (void);
> extern void lto_read_all_file_options (void);
> 
> -/* In lto-elf.c or lto-coff.c  */
> -extern lto_file *lto_obj_file_open (const char *filename, bool writable);
> -extern void lto_obj_file_close (lto_file *file);
> -struct lto_section_list;
> -extern htab_t lto_obj_build_section_table (lto_file *file, struct lto_section_list *list);
> -extern htab_t lto_obj_create_section_hash_table (void);
> -extern void lto_obj_begin_section (const char *name);
> -extern void lto_obj_append_data (const void *data, size_t len, void *block);
> -extern void lto_obj_end_section (void);
> -extern lto_file *lto_set_current_out_file (lto_file *file);
> -extern lto_file *lto_get_current_out_file (void);
> -
> extern int lto_link_dump_id, decl_merge_dump_id, partition_dump_id;
> 
> -/* Hash table entry to hold the start offset and length of an LTO
> -   section in a .o file.  */
> -struct lto_section_slot
> -{
> -  const char *name;
> -  intptr_t start;
> -  size_t len;
> -  struct lto_section_slot *next;
> -};
> -
> -/* A list of section slots */
> -struct lto_section_list
> -{
> -  struct lto_section_slot *first, *last;
> -};
> -
> extern unsigned int lto_option_lang_mask (void);
> 
> #endif /* LTO_H */
> diff --git a/gcc/toplev.cc b/gcc/toplev.cc
> index ed546b2cad8..4464c5d6773 100644
> --- a/gcc/toplev.cc
> +++ b/gcc/toplev.cc
> @@ -88,6 +88,7 @@ along with GCC; see the file COPYING3.  If not see
> #include "ipa-modref.h"
> #include "ipa-param-manipulation.h"
> #include "dbgcnt.h"
> +#include "lto-streamer.h"
> 
> #if defined(DBX_DEBUGGING_INFO) || defined(XCOFF_DEBUGGING_INFO)
> #include "dbxout.h"
> @@ -547,6 +548,12 @@ compile_file (void)
>      even when user did not ask for it.  */
>   if (flag_generate_lto && !flag_fat_lto_objects)
>     {
> +      if (flag_bypass_asm)
> +        {
> +          /* TODO  */
> +        }
> +      else
> +        {
> #if defined ASM_OUTPUT_ALIGNED_DECL_COMMON
>       ASM_OUTPUT_ALIGNED_DECL_COMMON (asm_out_file, NULL_TREE, "__gnu_lto_slim",
> 				      HOST_WIDE_INT_1U, 8);
> @@ -558,12 +565,13 @@ compile_file (void)
> 			 HOST_WIDE_INT_1U,
> 			 HOST_WIDE_INT_1U);
> #endif
> +        }
>     }
> 
>   /* Attach a special .ident directive to the end of the file to identify
>      the version of GCC which compiled this code.  The format of the .ident
>      string is patterned after the ones produced by native SVR4 compilers.  */
> -  if (!flag_no_ident)
> +  if (!flag_no_ident && !flag_bypass_asm)
>     {
>       const char *pkg_version = "(GNU) ";
>       char *ident_str;
> @@ -585,7 +593,11 @@ compile_file (void)
>   /* This must be at the end.  Some target ports emit end of file directives
>      into the assembly file here, and hence we cannot output anything to the
>      assembly file after this point.  */
> -  targetm.asm_out.file_end ();
> +
> +  if (flag_bypass_asm)
> +    lto_obj_file_close (lto_get_current_out_file ());
> +  else
> +    targetm.asm_out.file_end ();
> 
>   timevar_stop (TV_PHASE_LATE_ASM);
> }
> @@ -1861,7 +1873,8 @@ lang_dependent_init (const char *name)
> 
>   if (!flag_wpa)
>     {
> -      init_asm_output (name);
> +      if (!flag_bypass_asm)
> +        init_asm_output (name);
> 
>       if (!flag_generate_lto && !flag_compare_debug)
> 	{
> -- 
> 2.35.0
> 
>