RE: [EXTERNAL] Re: [PATCH] Fix autoprofiledbootstrap build

[Eugene Rozenfeld <Eugene.Rozenfeld@microsoft.com>]

[-- Attachment #1: Type: text/plain, Size: 1523 bytes --]

Hi Jeff,

I revived profile_merger tool in http://github.com/google/autofdo and re-worked the patch to merge profiles for compiling the libraries.

Please take a look at the attached patch.

Thanks,

Eugene

-----Original Message-----
From: Jeff Law <jeffreyalaw@gmail.com> 
Sent: Tuesday, November 22, 2022 10:16 PM
To: Eugene Rozenfeld <Eugene.Rozenfeld@microsoft.com>; gcc-patches@gcc.gnu.org; Andi Kleen <ak@linux.intel.com>
Subject: Re: [EXTERNAL] Re: [PATCH] Fix autoprofiledbootstrap build

[You don't often get email from jeffreyalaw@gmail.com. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

On 11/22/22 14:20, Eugene Rozenfeld wrote:
> I took another look at this. We actually collect perf data when building the libraries. So, we have ./prev-gcc/perf.data, ./prev-libcpp/perf.data, ./prev-libiberty/perf.data, etc. But when creating gcov data for  -fauto-profile build of cc1plus or cc1 we only use ./prev-gcc/perf.data . So, a better solution would be either having a single perf.data for all builds (gcc and libraries) or merging perf.data files before attempting autostagefeedback. What would you recommend?

ISTM that if neither approach loses data, then they're functionally equivalent -- meaning that we can select whichever is easier to wire into our build system.

A single perf.data might serialize the build.  So perhaps separate, then merge right before autostagefeedback.


But I'm willing to go with whatever you think is best.

Jeff



[-- Attachment #2: 0001-Fix-autoprofiledbootstrap-build.patch --]
[-- Type: application/octet-stream, Size: 9411 bytes --]

From 1808fe371ab5618b7c0ce22c0dbecdaf593e516d Mon Sep 17 00:00:00 2001
From: Eugene Rozenfeld <erozen@microsoft.com>
Date: Mon, 21 Nov 2022 13:33:38 -0800
Subject: [PATCH] Fix autoprofiledbootstrap build

1. Fix gcov version
2. Merge perf data collected when compiling the compiler and runtime libraries
3. Fix documentation typo

Tested on x86_64-pc-linux-gnu.

gcc/ChangeLog:

	* Makefile.in: Define PROFILE_MERGER
	* Makefile.tpl: Define PROFILE_MERGER
	* c/Make-lang.in: Merge perf data collected when compiling cc1 and runtime libraries
	* cp/Make-lang.in: Merge perf data collected when compiling cc1plus and runtime libraries
	* lto/Make-lang.in: Merge perf data collected when compiling lto1 and runtime libraries
	* doc/install.texi: Fix documentation typo
---
 Makefile.in          |  4 +++-
 Makefile.tpl         |  4 +++-
 gcc/c/Make-lang.in   | 32 +++++++++++++++++++++++++++++---
 gcc/cp/Make-lang.in  | 33 +++++++++++++++++++++++++++++----
 gcc/doc/install.texi |  2 +-
 gcc/lto/Make-lang.in | 32 +++++++++++++++++++++++++++++---
 6 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 06a9398e172..33f3c862557 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -491,6 +491,7 @@ PGO-TRAINING-TARGETS = binutils gas gdb ld sim
 PGO_BUILD_TRAINING = $(addprefix maybe-check-,$(PGO-TRAINING-TARGETS))
 
 CREATE_GCOV = create_gcov
+PROFILE_MERGER = profile_merger
 
 TFLAGS =
 
@@ -971,7 +972,8 @@ EXTRA_HOST_FLAGS = \
 	'STRIP=$(STRIP)' \
 	'WINDRES=$(WINDRES)' \
 	'WINDMC=$(WINDMC)' \
-	'CREATE_GCOV=$(CREATE_GCOV)'
+	'CREATE_GCOV=$(CREATE_GCOV)' \
+	'PROFILE_MERGER=$(PROFILE_MERGER)'
 
 FLAGS_TO_PASS = $(BASE_FLAGS_TO_PASS) $(EXTRA_HOST_FLAGS)
 
diff --git a/Makefile.tpl b/Makefile.tpl
index dfbd74b68f8..9d8ef9cf678 100644
--- a/Makefile.tpl
+++ b/Makefile.tpl
@@ -494,6 +494,7 @@ PGO-TRAINING-TARGETS = binutils gas gdb ld sim
 PGO_BUILD_TRAINING = $(addprefix maybe-check-,$(PGO-TRAINING-TARGETS))
 
 CREATE_GCOV = create_gcov
+PROFILE_MERGER = profile_merger
 
 TFLAGS =
 
@@ -728,7 +729,8 @@ EXTRA_HOST_FLAGS = \
 	'STRIP=$(STRIP)' \
 	'WINDRES=$(WINDRES)' \
 	'WINDMC=$(WINDMC)' \
-	'CREATE_GCOV=$(CREATE_GCOV)'
+	'CREATE_GCOV=$(CREATE_GCOV)' \
+	'PROFILE_MERGER=$(PROFILE_MERGER)'
 
 FLAGS_TO_PASS = $(BASE_FLAGS_TO_PASS) $(EXTRA_HOST_FLAGS)
 
diff --git a/gcc/c/Make-lang.in b/gcc/c/Make-lang.in
index b3b12dcffee..20840aceab6 100644
--- a/gcc/c/Make-lang.in
+++ b/gcc/c/Make-lang.in
@@ -88,9 +88,34 @@ cc1$(exeext): $(C_OBJS) cc1-checksum.o $(BACKEND) $(LIBDEPS)
 	  cc1-checksum.o $(BACKEND) $(LIBS) $(BACKENDLIBS)
 	@$(call LINK_PROGRESS,$(INDEX.c),end)
 
-cc1.fda: ../stage1-gcc/cc1$(exeext) ../prev-gcc/$(PERF_DATA)
-	$(CREATE_GCOV) -binary ../stage1-gcc/cc1$(exeext) -gcov cc1.fda -profile ../prev-gcc/$(PERF_DATA) -gcov_version 1
-
+components_in_prev = "bfd opcodes binutils fixincludes gas gcc gmp mpfr mpc isl gold intl ld libbacktrace libcpp libcody libdecnumber libiberty libiberty-linker-plugin libiconv zlib lto-plugin libctf libsframe"
+components_in_prev_target = "libstdc++-v3 libsanitizer libvtv libgcc libbacktrace libphobos zlib libgomp libatomic"
+
+.PHONY: create_fdas_for_cc1
+
+cc1.fda: create_fdas_for_cc1
+	$(PROFILE_MERGER) $(shell ls -ha cc1_*.fda) --output_file cc1.fda -gcov_version 2
+
+create_fdas_for_cc1: ../stage1-gcc/cc1$(exeext) ../prev-gcc/$(PERF_DATA)
+	for component_in_prev in "$(components_in_prev)"; do \
+	  perf_path=../prev-$$component_in_prev/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=cc1_$$component_in_prev.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/cc1$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
+
+	for component_in_prev_target in "$(components_in_prev_target)"; do \
+	  perf_path=../prev-$(TARGET_SUBDIR)/$$component_in_prev_target/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=cc1_$$component_in_prev_target.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/cc1$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
 #\f
 # Build hooks:
 
@@ -181,6 +206,7 @@ c.mostlyclean:
 	-rm -f c/*$(objext)
 	-rm -f c/*$(coverageexts)
 	-rm -f cc1.fda
+	-rm -f cc1_*.fda
 c.clean:
 c.distclean:
 	-rm -f c/config.status c/Makefile
diff --git a/gcc/cp/Make-lang.in b/gcc/cp/Make-lang.in
index f7ba12ee351..d7117c81293 100644
--- a/gcc/cp/Make-lang.in
+++ b/gcc/cp/Make-lang.in
@@ -177,9 +177,34 @@ endif
 # This is the file that depends on the generated header file.
 cp/name-lookup.o: $(srcdir)/cp/std-name-hint.h
 
-cc1plus.fda: ../stage1-gcc/cc1plus$(exeext) ../prev-gcc/$(PERF_DATA)
-	$(CREATE_GCOV) -binary ../stage1-gcc/cc1plus$(exeext) -gcov cc1plus.fda -profile ../prev-gcc/$(PERF_DATA) -gcov_version 1
-
+components_in_prev = "bfd opcodes binutils fixincludes gas gcc gmp mpfr mpc isl gold intl ld libbacktrace libcpp libcody libdecnumber libiberty libiberty-linker-plugin libiconv zlib lto-plugin libctf libsframe"
+components_in_prev_target = "libstdc++-v3 libsanitizer libvtv libgcc libbacktrace libphobos zlib libgomp libatomic"
+
+.PHONY: create_fdas_for_cc1plus
+
+cc1plus.fda: create_fdas_for_cc1plus
+	$(PROFILE_MERGER) $(shell ls -ha cc1plus_*.fda) --output_file cc1plus.fda -gcov_version 2
+
+create_fdas_for_cc1plus: ../stage1-gcc/cc1plus$(exeext) ../prev-gcc/$(PERF_DATA)
+	for component_in_prev in "$(components_in_prev)"; do \
+	  perf_path=../prev-$$component_in_prev/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=cc1plus_$$component_in_prev.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/cc1plus$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
+
+	for component_in_prev_target in "$(components_in_prev_target)"; do \
+	  perf_path=../prev-$(TARGET_SUBDIR)/$$component_in_prev_target/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=cc1plus_$$component_in_prev_target.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/cc1plus$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
 #\f
 # Build hooks:
 
@@ -326,7 +351,7 @@ c++.mostlyclean:
 	-rm -f doc/g++.1
 	-rm -f cp/*$(objext)
 	-rm -f cp/*$(coverageexts)
-	-rm -f xg++$(exeext) g++-cross$(exeext) cc1plus$(exeext) cc1plus.fda
+	-rm -f xg++$(exeext) g++-cross$(exeext) cc1plus$(exeext) cc1plus*.fda
 c++.clean:
 c++.distclean:
 	-rm -f cp/config.status cp/Makefile
diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 63fc949b447..6b2e15d903c 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -3120,7 +3120,7 @@ It is recommended to only use GCC for this.
 
 On Linux/x86_64 hosts with some restrictions (no virtualization) it is
 also possible to do autofdo build with @samp{make
-autoprofiledback}. This uses Linux perf to sample branches in the
+autoprofiledbootstrap}. This uses Linux perf to sample branches in the
 binary and then rebuild it with feedback derived from the profile.
 Linux perf and the @code{autofdo} toolkit needs to be installed for
 this.
diff --git a/gcc/lto/Make-lang.in b/gcc/lto/Make-lang.in
index b3b965e44f7..4f6025100a3 100644
--- a/gcc/lto/Make-lang.in
+++ b/gcc/lto/Make-lang.in
@@ -74,7 +74,7 @@ lto.srcinfo:
 lto.install-plugin:
 
 lto.mostlyclean:
-	rm -f $(LTO_OBJS) $(LTO_EXE) lto1.fda $(LTO_DUMP_OBJS) $(LTO_DUMP_EXE) lto-dump.fda
+	rm -f $(LTO_OBJS) $(LTO_EXE) lto1*.fda $(LTO_DUMP_OBJS) $(LTO_DUMP_EXE) lto-dump.fda
 
 lto.clean:
 lto.distclean:
@@ -105,8 +105,34 @@ $(LTO_DUMP_EXE): $(LTO_DUMP_OBJS) $(BACKEND) $(LIBDEPS) $(lto2.prev)
 
 lto/lto-dump.o: $(LTO_OBJS)
 
-lto1.fda: ../prev-gcc/lto1$(exeext) ../prev-gcc/$(PERF_DATA)
-	$(CREATE_GCOV) -binary ../prev-gcc/lto1$(exeext) -gcov lto1.fda -profile ../prev-gcc/$(PERF_DATA) -gcov_version 1
+components_in_prev = "bfd opcodes binutils fixincludes gas gcc gmp mpfr mpc isl gold intl ld libbacktrace libcpp libcody libdecnumber libiberty libiberty-linker-plugin libiconv zlib lto-plugin libctf libsframe"
+components_in_prev_target = "libstdc++-v3 libsanitizer libvtv libgcc libbacktrace libphobos zlib libgomp libatomic"
+
+.PHONY: create_fdas_for_lto1
+
+lto1.fda: create_fdas_for_lto1
+	$(PROFILE_MERGER) $(shell ls -ha lto1_*.fda) --output_file lto1.fda -gcov_version 2
+
+create_fdas_for_lto1: ../stage1-gcc/lto1$(exeext) ../prev-gcc/$(PERF_DATA)
+	for component_in_prev in "$(components_in_prev)"; do \
+	  perf_path=../prev-$$component_in_prev/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=lto1_$$component_in_prev.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/lto1$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
+
+	for component_in_prev_target in "$(components_in_prev_target)"; do \
+	  perf_path=../prev-$(TARGET_SUBDIR)/$$component_in_prev_target/$(PERF_DATA); \
+	  echo "Perf path:"; \
+	  echo $$perf_path; \
+	  if [ -f $$perf_path ]; then \
+	    profile_name=lto1_$$component_in_prev_target.fda; \
+	    $(CREATE_GCOV) -binary ../stage1-gcc/lto1$(exeext) -gcov $$profile_name -profile $$perf_path -gcov_version 2; \
+	  fi; \
+	done;
 
 # LTO testing is done as part of C/C++/Fortran etc. testing.
 check-lto:
-- 
2.25.1