* Allow more WHOPR partitions than souce files
@ 2010-09-03 22:58 Jan Hubicka
2010-09-04 9:30 ` Richard Guenther
0 siblings, 1 reply; 6+ messages in thread
From: Jan Hubicka @ 2010-09-03 22:58 UTC (permalink / raw)
To: gcc-patches, dnovillo, rguenther
Hi,
this patch makes collect2 to accept more partition files then number of original .o files
on the command line. This is needed for my partitioning patch that sometimes (for very
large .o files) decide to break up single .o file into multiple.
Bootstrapped/regtested x86_64-linux, OK?
Honza
* collect2.c (maybe_run_lto_and_relink): Allow number of partition files
to exceed number of files.
Index: collect2.c
===================================================================
--- collect2.c (revision 163809)
+++ collect2.c (working copy)
@@ -942,12 +942,15 @@ maybe_run_lto_and_relink (char **lto_ld_
{
char **lto_c_argv;
const char **lto_c_ptr;
- const char **p, **q, **r;
+ const char **p, **q, **r, **last_entry = NULL;
const char **lto_o_ptr;
struct lto_object *list;
char *lto_wrapper = getenv ("COLLECT_LTO_WRAPPER");
struct pex_obj *pex;
const char *prog = "lto-wrapper";
+ int lto_ld_argv_size = 0;
+ char **out_lto_ld_argv;
+ size_t num_files;
if (!lto_wrapper)
fatal ("COLLECT_LTO_WRAPPER must be set.");
@@ -973,7 +976,7 @@ maybe_run_lto_and_relink (char **lto_ld_
{
int c;
FILE *stream;
- size_t i, num_files;
+ size_t i;
char *start, *end;
stream = pex_read_output (pex, 0);
@@ -1007,10 +1010,14 @@ maybe_run_lto_and_relink (char **lto_ld_
do_wait (prog, pex);
pex = NULL;
+ for (lto_ld_argv_size = 0; lto_ld_argv[lto_ld_argv_size]; lto_ld_argv_size++)
+ ;
+ out_lto_ld_argv = XCNEWVEC(char *, num_files + lto_ld_argv_size + 1);
+ memcpy (out_lto_ld_argv, lto_ld_argv, sizeof (char *) * lto_ld_argv_size);
/* After running the LTO back end, we will relink, substituting
the LTO output for the object files that we submitted to the
LTO. Here, we modify the linker command line for the relink. */
- p = CONST_CAST2 (const char **, char **, lto_ld_argv);
+ p = CONST_CAST2 (const char **, char **, out_lto_ld_argv);
lto_o_ptr = CONST_CAST2 (const char **, char **, lto_o_files);
while (*p != NULL)
@@ -1023,6 +1030,7 @@ maybe_run_lto_and_relink (char **lto_ld_
if (*lto_o_ptr)
{
/* Replace first argument with LTO output file. */
+ last_entry = p;
*p++ = *lto_o_ptr++;
}
else
@@ -1047,15 +1055,27 @@ maybe_run_lto_and_relink (char **lto_ld_
if (!list) ++p;
}
- /* The code above assumes we will never have more lto output files than
- input files. Otherwise, we need to resize lto_ld_argv. Check this
- assumption. */
+ /* Insert remaining .o files just after last file appearing in the argument
+ sequence. This way relative position of crtend and friends will be right. */
if (*lto_o_ptr)
- fatal ("too many lto output files");
+ {
+ size_t remaining_files = 0;
+ size_t tail_size = 0;
+
+ for (p = lto_o_ptr; *p; p++)
+ remaining_files++;
+ for (p = last_entry + 1; *p; p++)
+ tail_size++;
+ memmove (last_entry + 1 + remaining_files,
+ last_entry + 1,
+ tail_size * sizeof (char *));
+ memcpy (last_entry + 1, lto_o_ptr, remaining_files * sizeof (char *));
+ }
/* Run the linker again, this time replacing the object files
optimized by the LTO with the temporary file generated by the LTO. */
- fork_execute ("ld", lto_ld_argv);
+ fork_execute ("ld", out_lto_ld_argv);
+ free (lto_ld_argv);
maybe_unlink_list (lto_o_files);
}
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: Allow more WHOPR partitions than souce files
2010-09-03 22:58 Allow more WHOPR partitions than souce files Jan Hubicka
@ 2010-09-04 9:30 ` Richard Guenther
2010-09-04 13:30 ` Jan Hubicka
0 siblings, 1 reply; 6+ messages in thread
From: Richard Guenther @ 2010-09-04 9:30 UTC (permalink / raw)
To: Jan Hubicka; +Cc: gcc-patches, dnovillo
On Fri, 3 Sep 2010, Jan Hubicka wrote:
> Hi,
> this patch makes collect2 to accept more partition files then number of original .o files
> on the command line. This is needed for my partitioning patch that sometimes (for very
> large .o files) decide to break up single .o file into multiple.
>
> Bootstrapped/regtested x86_64-linux, OK?
Ick. Somebody ought to replace these arg arrays with a sane
datastructure when we're doing this kind of funny stuff.
Can you give that a try please? You might need to enhance
VEC_splice to allow splicing in the middle of a vector.
Thanks,
Richard.
> Honza
>
> * collect2.c (maybe_run_lto_and_relink): Allow number of partition files
> to exceed number of files.
> Index: collect2.c
> ===================================================================
> --- collect2.c (revision 163809)
> +++ collect2.c (working copy)
> @@ -942,12 +942,15 @@ maybe_run_lto_and_relink (char **lto_ld_
> {
> char **lto_c_argv;
> const char **lto_c_ptr;
> - const char **p, **q, **r;
> + const char **p, **q, **r, **last_entry = NULL;
> const char **lto_o_ptr;
> struct lto_object *list;
> char *lto_wrapper = getenv ("COLLECT_LTO_WRAPPER");
> struct pex_obj *pex;
> const char *prog = "lto-wrapper";
> + int lto_ld_argv_size = 0;
> + char **out_lto_ld_argv;
> + size_t num_files;
>
> if (!lto_wrapper)
> fatal ("COLLECT_LTO_WRAPPER must be set.");
> @@ -973,7 +976,7 @@ maybe_run_lto_and_relink (char **lto_ld_
> {
> int c;
> FILE *stream;
> - size_t i, num_files;
> + size_t i;
> char *start, *end;
>
> stream = pex_read_output (pex, 0);
> @@ -1007,10 +1010,14 @@ maybe_run_lto_and_relink (char **lto_ld_
> do_wait (prog, pex);
> pex = NULL;
>
> + for (lto_ld_argv_size = 0; lto_ld_argv[lto_ld_argv_size]; lto_ld_argv_size++)
> + ;
> + out_lto_ld_argv = XCNEWVEC(char *, num_files + lto_ld_argv_size + 1);
> + memcpy (out_lto_ld_argv, lto_ld_argv, sizeof (char *) * lto_ld_argv_size);
> /* After running the LTO back end, we will relink, substituting
> the LTO output for the object files that we submitted to the
> LTO. Here, we modify the linker command line for the relink. */
> - p = CONST_CAST2 (const char **, char **, lto_ld_argv);
> + p = CONST_CAST2 (const char **, char **, out_lto_ld_argv);
> lto_o_ptr = CONST_CAST2 (const char **, char **, lto_o_files);
>
> while (*p != NULL)
> @@ -1023,6 +1030,7 @@ maybe_run_lto_and_relink (char **lto_ld_
> if (*lto_o_ptr)
> {
> /* Replace first argument with LTO output file. */
> + last_entry = p;
> *p++ = *lto_o_ptr++;
> }
> else
> @@ -1047,15 +1055,27 @@ maybe_run_lto_and_relink (char **lto_ld_
> if (!list) ++p;
> }
>
> - /* The code above assumes we will never have more lto output files than
> - input files. Otherwise, we need to resize lto_ld_argv. Check this
> - assumption. */
> + /* Insert remaining .o files just after last file appearing in the argument
> + sequence. This way relative position of crtend and friends will be right. */
> if (*lto_o_ptr)
> - fatal ("too many lto output files");
> + {
> + size_t remaining_files = 0;
> + size_t tail_size = 0;
> +
> + for (p = lto_o_ptr; *p; p++)
> + remaining_files++;
> + for (p = last_entry + 1; *p; p++)
> + tail_size++;
> + memmove (last_entry + 1 + remaining_files,
> + last_entry + 1,
> + tail_size * sizeof (char *));
> + memcpy (last_entry + 1, lto_o_ptr, remaining_files * sizeof (char *));
> + }
>
> /* Run the linker again, this time replacing the object files
> optimized by the LTO with the temporary file generated by the LTO. */
> - fork_execute ("ld", lto_ld_argv);
> + fork_execute ("ld", out_lto_ld_argv);
> + free (lto_ld_argv);
>
> maybe_unlink_list (lto_o_files);
> }
>
>
--
Richard Guenther <rguenther@suse.de>
Novell / SUSE Labs
SUSE LINUX Products GmbH - Nuernberg - AG Nuernberg - HRB 16746 - GF: Markus Rex
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: Allow more WHOPR partitions than souce files
2010-09-04 9:30 ` Richard Guenther
@ 2010-09-04 13:30 ` Jan Hubicka
2010-09-04 13:42 ` Richard Guenther
0 siblings, 1 reply; 6+ messages in thread
From: Jan Hubicka @ 2010-09-04 13:30 UTC (permalink / raw)
To: Richard Guenther; +Cc: Jan Hubicka, gcc-patches, dnovillo
> On Fri, 3 Sep 2010, Jan Hubicka wrote:
>
> > Hi,
> > this patch makes collect2 to accept more partition files then number of original .o files
> > on the command line. This is needed for my partitioning patch that sometimes (for very
> > large .o files) decide to break up single .o file into multiple.
> >
> > Bootstrapped/regtested x86_64-linux, OK?
>
> Ick. Somebody ought to replace these arg arrays with a sane
> datastructure when we're doing this kind of funny stuff.
>
> Can you give that a try please? You might need to enhance
> VEC_splice to allow splicing in the middle of a vector.
Hmm, I guess I can just have one loop pushing into out_lto_ld_argv arguments
form lto_ld_argv until first LTO object is found. Then i can push all the
ltrans partitions and continue copying lto_ld_argv ignoring all other LTO
object. This way I won't need to play with inplace updating of the array?
Reorganizing everything to VECtors would be some work as i.e. fork_and_execute
is freeing the array so I would need to update all at aonce...
Honza
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: Allow more WHOPR partitions than souce files
2010-09-04 13:30 ` Jan Hubicka
@ 2010-09-04 13:42 ` Richard Guenther
2010-09-09 0:52 ` Jan Hubicka
0 siblings, 1 reply; 6+ messages in thread
From: Richard Guenther @ 2010-09-04 13:42 UTC (permalink / raw)
To: Jan Hubicka; +Cc: gcc-patches, dnovillo
On Sat, 4 Sep 2010, Jan Hubicka wrote:
> > On Fri, 3 Sep 2010, Jan Hubicka wrote:
> >
> > > Hi,
> > > this patch makes collect2 to accept more partition files then number of original .o files
> > > on the command line. This is needed for my partitioning patch that sometimes (for very
> > > large .o files) decide to break up single .o file into multiple.
> > >
> > > Bootstrapped/regtested x86_64-linux, OK?
> >
> > Ick. Somebody ought to replace these arg arrays with a sane
> > datastructure when we're doing this kind of funny stuff.
> >
> > Can you give that a try please? You might need to enhance
> > VEC_splice to allow splicing in the middle of a vector.
>
> Hmm, I guess I can just have one loop pushing into out_lto_ld_argv arguments
> form lto_ld_argv until first LTO object is found. Then i can push all the
> ltrans partitions and continue copying lto_ld_argv ignoring all other LTO
> object. This way I won't need to play with inplace updating of the array?
That works for me as well.
Richard.
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: Allow more WHOPR partitions than souce files
2010-09-04 13:42 ` Richard Guenther
@ 2010-09-09 0:52 ` Jan Hubicka
2010-09-09 9:00 ` Richard Guenther
0 siblings, 1 reply; 6+ messages in thread
From: Jan Hubicka @ 2010-09-09 0:52 UTC (permalink / raw)
To: Richard Guenther; +Cc: Jan Hubicka, gcc-patches, dnovillo
Hi,
there is updated patch...
Bootstrapped/regested x86_64-linux, OK?
Honza
* collect2.c (maybe_run_lto_and_relink): Rewrite code producing ld
command line to allow more partitions than input files.
Index: collect2.c
===================================================================
--- collect2.c (revision 164002)
+++ collect2.c (working copy)
@@ -942,12 +942,16 @@ maybe_run_lto_and_relink (char **lto_ld_
{
char **lto_c_argv;
const char **lto_c_ptr;
- const char **p, **q, **r;
- const char **lto_o_ptr;
+ char **p;
+ char **lto_o_ptr;
struct lto_object *list;
char *lto_wrapper = getenv ("COLLECT_LTO_WRAPPER");
struct pex_obj *pex;
const char *prog = "lto-wrapper";
+ int lto_ld_argv_size = 0;
+ char **out_lto_ld_argv;
+ int out_lto_ld_argv_size;
+ size_t num_files;
if (!lto_wrapper)
fatal ("COLLECT_LTO_WRAPPER must be set.");
@@ -955,7 +959,11 @@ maybe_run_lto_and_relink (char **lto_ld_
num_lto_c_args++;
/* There is at least one object file containing LTO info,
- so we need to run the LTO back end and relink. */
+ so we need to run the LTO back end and relink.
+
+ To do so we build updated ld arguments with first
+ LTO object replaced by all partitions and other LTO
+ objects removed. */
lto_c_argv = (char **) xcalloc (sizeof (char *), num_lto_c_args);
lto_c_ptr = CONST_CAST2 (const char **, char **, lto_c_argv);
@@ -973,7 +981,7 @@ maybe_run_lto_and_relink (char **lto_ld_
{
int c;
FILE *stream;
- size_t i, num_files;
+ size_t i;
char *start, *end;
stream = pex_read_output (pex, 0);
@@ -1007,55 +1015,50 @@ maybe_run_lto_and_relink (char **lto_ld_
do_wait (prog, pex);
pex = NULL;
+ /* Compute memory needed for new LD arguments. At most number of original arguemtns
+ plus number of partitions. */
+ for (lto_ld_argv_size = 0; lto_ld_argv[lto_ld_argv_size]; lto_ld_argv_size++)
+ ;
+ out_lto_ld_argv = XCNEWVEC(char *, num_files + lto_ld_argv_size + 1);
+ out_lto_ld_argv_size = 0;
+
/* After running the LTO back end, we will relink, substituting
the LTO output for the object files that we submitted to the
LTO. Here, we modify the linker command line for the relink. */
- p = CONST_CAST2 (const char **, char **, lto_ld_argv);
- lto_o_ptr = CONST_CAST2 (const char **, char **, lto_o_files);
+ /* Copy all arguments until we find first LTO file. */
+ p = lto_ld_argv;
while (*p != NULL)
{
for (list = lto_objects.first; list; list = list->next)
- {
- if (*p == list->name) /* Note test for pointer equality! */
- {
- /* Excise argument from linker command line. */
- if (*lto_o_ptr)
- {
- /* Replace first argument with LTO output file. */
- *p++ = *lto_o_ptr++;
- }
- else
- {
- /* Move following arguments one position earlier,
- overwriting the current argument. */
- q = p;
- r = p + 1;
- while (*r != NULL)
- *q++ = *r++;
- *q = NULL;
- }
-
- /* No need to continue searching the LTO object list. */
- break;
- }
- }
-
- /* If we didn't find a match, move on to the next argument.
- Otherwise, P has been set to the correct argument position
- at which to continue. */
- if (!list) ++p;
+ if (*p == list->name) /* Note test for pointer equality! */
+ break;
+ if (list)
+ break;
+ out_lto_ld_argv[out_lto_ld_argv_size++] = *p++;
}
- /* The code above assumes we will never have more lto output files than
- input files. Otherwise, we need to resize lto_ld_argv. Check this
- assumption. */
- if (*lto_o_ptr)
- fatal ("too many lto output files");
+ /* Now insert all LTO partitions. */
+ lto_o_ptr = lto_o_files;
+ while (*lto_o_ptr)
+ out_lto_ld_argv[out_lto_ld_argv_size++] = *lto_o_ptr++;
+
+ /* ... and copy the rest. */
+ while (*p != NULL)
+ {
+ for (list = lto_objects.first; list; list = list->next)
+ if (*p == list->name) /* Note test for pointer equality! */
+ break;
+ if (!list)
+ out_lto_ld_argv[out_lto_ld_argv_size++] = *p;
+ p++;
+ }
+ out_lto_ld_argv[out_lto_ld_argv_size++] = 0;
/* Run the linker again, this time replacing the object files
optimized by the LTO with the temporary file generated by the LTO. */
- fork_execute ("ld", lto_ld_argv);
+ fork_execute ("ld", out_lto_ld_argv);
+ free (lto_ld_argv);
maybe_unlink_list (lto_o_files);
}
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: Allow more WHOPR partitions than souce files
2010-09-09 0:52 ` Jan Hubicka
@ 2010-09-09 9:00 ` Richard Guenther
0 siblings, 0 replies; 6+ messages in thread
From: Richard Guenther @ 2010-09-09 9:00 UTC (permalink / raw)
To: Jan Hubicka; +Cc: gcc-patches, dnovillo
On Thu, 9 Sep 2010, Jan Hubicka wrote:
> Hi,
> there is updated patch...
>
> Bootstrapped/regested x86_64-linux, OK?
Ok.
Thanks,
Richard.
> Honza
> * collect2.c (maybe_run_lto_and_relink): Rewrite code producing ld
> command line to allow more partitions than input files.
> Index: collect2.c
> ===================================================================
> --- collect2.c (revision 164002)
> +++ collect2.c (working copy)
> @@ -942,12 +942,16 @@ maybe_run_lto_and_relink (char **lto_ld_
> {
> char **lto_c_argv;
> const char **lto_c_ptr;
> - const char **p, **q, **r;
> - const char **lto_o_ptr;
> + char **p;
> + char **lto_o_ptr;
> struct lto_object *list;
> char *lto_wrapper = getenv ("COLLECT_LTO_WRAPPER");
> struct pex_obj *pex;
> const char *prog = "lto-wrapper";
> + int lto_ld_argv_size = 0;
> + char **out_lto_ld_argv;
> + int out_lto_ld_argv_size;
> + size_t num_files;
>
> if (!lto_wrapper)
> fatal ("COLLECT_LTO_WRAPPER must be set.");
> @@ -955,7 +959,11 @@ maybe_run_lto_and_relink (char **lto_ld_
> num_lto_c_args++;
>
> /* There is at least one object file containing LTO info,
> - so we need to run the LTO back end and relink. */
> + so we need to run the LTO back end and relink.
> +
> + To do so we build updated ld arguments with first
> + LTO object replaced by all partitions and other LTO
> + objects removed. */
>
> lto_c_argv = (char **) xcalloc (sizeof (char *), num_lto_c_args);
> lto_c_ptr = CONST_CAST2 (const char **, char **, lto_c_argv);
> @@ -973,7 +981,7 @@ maybe_run_lto_and_relink (char **lto_ld_
> {
> int c;
> FILE *stream;
> - size_t i, num_files;
> + size_t i;
> char *start, *end;
>
> stream = pex_read_output (pex, 0);
> @@ -1007,55 +1015,50 @@ maybe_run_lto_and_relink (char **lto_ld_
> do_wait (prog, pex);
> pex = NULL;
>
> + /* Compute memory needed for new LD arguments. At most number of original arguemtns
> + plus number of partitions. */
> + for (lto_ld_argv_size = 0; lto_ld_argv[lto_ld_argv_size]; lto_ld_argv_size++)
> + ;
> + out_lto_ld_argv = XCNEWVEC(char *, num_files + lto_ld_argv_size + 1);
> + out_lto_ld_argv_size = 0;
> +
> /* After running the LTO back end, we will relink, substituting
> the LTO output for the object files that we submitted to the
> LTO. Here, we modify the linker command line for the relink. */
> - p = CONST_CAST2 (const char **, char **, lto_ld_argv);
> - lto_o_ptr = CONST_CAST2 (const char **, char **, lto_o_files);
>
> + /* Copy all arguments until we find first LTO file. */
> + p = lto_ld_argv;
> while (*p != NULL)
> {
> for (list = lto_objects.first; list; list = list->next)
> - {
> - if (*p == list->name) /* Note test for pointer equality! */
> - {
> - /* Excise argument from linker command line. */
> - if (*lto_o_ptr)
> - {
> - /* Replace first argument with LTO output file. */
> - *p++ = *lto_o_ptr++;
> - }
> - else
> - {
> - /* Move following arguments one position earlier,
> - overwriting the current argument. */
> - q = p;
> - r = p + 1;
> - while (*r != NULL)
> - *q++ = *r++;
> - *q = NULL;
> - }
> -
> - /* No need to continue searching the LTO object list. */
> - break;
> - }
> - }
> -
> - /* If we didn't find a match, move on to the next argument.
> - Otherwise, P has been set to the correct argument position
> - at which to continue. */
> - if (!list) ++p;
> + if (*p == list->name) /* Note test for pointer equality! */
> + break;
> + if (list)
> + break;
> + out_lto_ld_argv[out_lto_ld_argv_size++] = *p++;
> }
>
> - /* The code above assumes we will never have more lto output files than
> - input files. Otherwise, we need to resize lto_ld_argv. Check this
> - assumption. */
> - if (*lto_o_ptr)
> - fatal ("too many lto output files");
> + /* Now insert all LTO partitions. */
> + lto_o_ptr = lto_o_files;
> + while (*lto_o_ptr)
> + out_lto_ld_argv[out_lto_ld_argv_size++] = *lto_o_ptr++;
> +
> + /* ... and copy the rest. */
> + while (*p != NULL)
> + {
> + for (list = lto_objects.first; list; list = list->next)
> + if (*p == list->name) /* Note test for pointer equality! */
> + break;
> + if (!list)
> + out_lto_ld_argv[out_lto_ld_argv_size++] = *p;
> + p++;
> + }
> + out_lto_ld_argv[out_lto_ld_argv_size++] = 0;
>
> /* Run the linker again, this time replacing the object files
> optimized by the LTO with the temporary file generated by the LTO. */
> - fork_execute ("ld", lto_ld_argv);
> + fork_execute ("ld", out_lto_ld_argv);
> + free (lto_ld_argv);
>
> maybe_unlink_list (lto_o_files);
> }
>
>
--
Richard Guenther <rguenther@suse.de>
Novell / SUSE Labs
SUSE LINUX Products GmbH - Nuernberg - AG Nuernberg - HRB 16746 - GF: Markus Rex
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2010-09-09 8:41 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2010-09-03 22:58 Allow more WHOPR partitions than souce files Jan Hubicka
2010-09-04 9:30 ` Richard Guenther
2010-09-04 13:30 ` Jan Hubicka
2010-09-04 13:42 ` Richard Guenther
2010-09-09 0:52 ` Jan Hubicka
2010-09-09 9:00 ` Richard Guenther
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).