[gcc/devel/c++-coroutines] Introduce -fprofile-reproducibility and support it with TOP N.

[gcc/devel/c++-coroutines] Introduce -fprofile-reproducibility and support it with TOP N.

[Iain D Sandoe]

https://gcc.gnu.org/g:ea0b12523d0d9a9059b5173ce9653b92ddfb284f

commit ea0b12523d0d9a9059b5173ce9653b92ddfb284f
Author: Martin Liska <mliska@suse.cz>
Date:   Tue Feb 18 14:28:22 2020 +0100

    Introduce -fprofile-reproducibility and support it with TOP N.
    
    	PR ipa/92924
    	* common.opt: Add -fprofile-reproducibility.
    	* doc/invoke.texi: Document it.
    	* value-prof.c (dump_histogram_value):
    	Document and support behavior for counters[0]
    	being a negative value.
    	(get_nth_most_common_value): Handle negative
    	counters[0] in respect to flag_profile_reproducible.
    	PR ipa/92924
    	* libgcov-merge.c (merge_topn_values_set): Record
    	when a TOP N counter becomes invalid.  When merging
    	remove a smallest value if the space is needed.

Diff:
---
 gcc/ChangeLog          | 11 +++++++++++
 gcc/common.opt         | 16 ++++++++++++++++
 gcc/coretypes.h        |  7 +++++++
 gcc/doc/invoke.texi    | 31 ++++++++++++++++++++++++++++++-
 gcc/value-prof.c       | 29 ++++++++++++++++++++++-------
 libgcc/ChangeLog       |  7 +++++++
 libgcc/libgcov-merge.c | 50 +++++++++++++++++++++++++++++++++++---------------
 7 files changed, 128 insertions(+), 23 deletions(-)

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index efc2116..db6740a 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,14 @@
+2020-02-18  Martin Liska  <mliska@suse.cz>
+
+	PR ipa/92924
+	* common.opt: Add -fprofile-reproducibility.
+	* doc/invoke.texi: Document it.
+	* value-prof.c (dump_histogram_value):
+	Document and support behavior for counters[0]
+	being a negative value.
+	(get_nth_most_common_value): Handle negative
+	counters[0] in respect to flag_profile_reproducible.
+
 2020-02-18  Jakub Jelinek  <jakub@redhat.com>
 
 	PR ipa/93797
diff --git a/gcc/common.opt b/gcc/common.opt
index 5692cd0..fa9da50 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -2169,6 +2169,22 @@ Common Joined RejectNegative Var(flag_profile_exclude_files)
 Instrument only functions from files where names do not match all the regular expressions (separated by a semi-colon).
 
 Enum
+Name(profile_reproducibility) Type(enum profile_reproducibility) UnknownError(unknown profile reproducibility method %qs)
+
+EnumValue
+Enum(profile_reproducibility) String(serial) Value(PROFILE_REPRODUCIBILITY_SERIAL)
+
+EnumValue
+Enum(profile_reproducibility) String(parallel-runs) Value(PROFILE_REPRODUCIBILITY_PARALLEL_RUNS)
+
+EnumValue
+Enum(profile_reproducibility) String(multithreaded) Value(PROFILE_REPRODUCIBILITY_MULTITHREADED)
+
+fprofile-reproducible
+Common Joined RejectNegative Var(flag_profile_reproducible) Enum(profile_reproducibility) Init(PROFILE_REPRODUCIBILITY_SERIAL)
+-fprofile-reproducible=[serial|parallel-runs|multithreaded] Control level of reproducibility of profile gathered by -fprofile-generate.
+
+Enum
 Name(profile_update) Type(enum profile_update) UnknownError(unknown profile update method %qs)
 
 EnumValue
diff --git a/gcc/coretypes.h b/gcc/coretypes.h
index d8fd50d..cda22697 100644
--- a/gcc/coretypes.h
+++ b/gcc/coretypes.h
@@ -212,6 +212,13 @@ enum profile_update {
   PROFILE_UPDATE_PREFER_ATOMIC
 };
 
+/* Type of profile reproducibility methods.  */
+enum profile_reproducibility {
+    PROFILE_REPRODUCIBILITY_SERIAL,
+    PROFILE_REPRODUCIBILITY_PARALLEL_RUNS,
+    PROFILE_REPRODUCIBILITY_MULTITHREADED
+};
+
 /* Types of unwind/exception handling info that can be generated.  */
 
 enum unwind_info_type
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 3e47d06..ba2b7e4 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -562,7 +562,7 @@ Objective-C and Objective-C++ Dialects}.
 -fprofile-abs-path @gol
 -fprofile-dir=@var{path}  -fprofile-generate  -fprofile-generate=@var{path} @gol
 -fprofile-note=@var{path}  -fprofile-update=@var{method} @gol
--fprofile-filter-files=@var{regex}  -fprofile-exclude-files=@var{regex} @gol
+-fprofile-filter-files=@var{regex}  -fprofile-exclude-files=@var{regex} -fprofile-reproducibility @gol
 -fsanitize=@var{style}  -fsanitize-recover  -fsanitize-recover=@var{style} @gol
 -fasan-shadow-offset=@var{number}  -fsanitize-sections=@var{s1},@var{s2},... @gol
 -fsanitize-undefined-trap-on-error  -fbounds-check @gol
@@ -13360,6 +13360,35 @@ all the regular expressions (separated by a semi-colon).
 For example, @option{-fprofile-exclude-files=/usr/*} will prevent instrumentation
 of all files that are located in @file{/usr/} folder.
 
+@item -fprofile-reproducible
+@opindex fprofile-reproducible
+Control level of reproducibility of profile gathered by
+@code{-fprofile-generate}.  This makes it possible to rebuild program
+with same outcome which is useful, for example, for distribution
+packages.
+
+With @option{-fprofile-reproducibility=serial} the profile gathered by
+@option{-fprofile-generate} is reproducible provided the trained program
+behaves the same at each invocation of the train run, it is not
+multi-threaded and profile data streaming is always done in the same
+order.  Note that profile streaming happens at the end of program run but
+also before @code{fork} function is invoked.
+
+Note that it is quite common that execution counts of some part of
+programs depends, for example, on length of temporary file names or
+memory space randomization (that may affect hash-table collision rate).
+Such non-reproducible part of programs may be annotated by
+@code{no_instrument_function} function attribute. @code{gcov-dump} with
+@option{-l} can be used to dump gathered data and verify that they are
+indeed reproducible.
+
+With @option{-fprofile-reproducibility=parallel-runs} collected profile
+stays reproducible regardless the order of streaming of the data into
+gcda files.  This setting makes it possible to run multiple instances of
+instrumented program in parallel (such as with @code{make -j}). This
+reduces quality of gathered data, in particular of indirect call
+profiling.
+
 @item -fsanitize=address
 @opindex fsanitize=address
 Enable AddressSanitizer, a fast memory error detector.
diff --git a/gcc/value-prof.c b/gcc/value-prof.c
index f0456c8..5f940f4 100644
--- a/gcc/value-prof.c
+++ b/gcc/value-prof.c
@@ -265,8 +265,10 @@ dump_histogram_value (FILE *dump_file, histogram_value hist)
 		    ? "Top N value counter" : "Indirect call counter"));
 	  if (hist->hvalue.counters)
 	    {
-	      fprintf (dump_file, " all: %" PRId64 ", values: ",
-		       (int64_t) hist->hvalue.counters[0]);
+	      fprintf (dump_file, " all: %" PRId64 "%s, values: ",
+		       abs ((int64_t) hist->hvalue.counters[0]),
+		       hist->hvalue.counters[0] < 0
+		       ? " (values missing)": "");
 	      for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
 		{
 		  fprintf (dump_file, "[%" PRId64 ":%" PRId64 "]",
@@ -719,26 +721,39 @@ gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob,
 
 /* Return the n-th value count of TOPN_VALUE histogram.  If
    there's a value, return true and set VALUE and COUNT
-   arguments.  */
+   arguments.
+
+   Counters have the following meaning.
+
+   abs (counters[0]) is the number of executions
+   for i in 0 ... TOPN-1
+     counters[2 * i + 1] is target
+     abs (counters[2 * i + 2]) is corresponding hitrate counter.
+
+   Value of counters[0] negative when counter became
+   full during merging and some values are lost.  */
 
 bool
 get_nth_most_common_value (gimple *stmt, const char *counter_type,
 			   histogram_value hist, gcov_type *value,
 			   gcov_type *count, gcov_type *all, unsigned n)
 {
-  if (hist->hvalue.counters[2] == -1)
-    return false;
-
   gcc_assert (n < GCOV_TOPN_VALUES);
 
   *count = 0;
   *value = 0;
 
-  gcov_type read_all = hist->hvalue.counters[0];
+  gcov_type read_all = abs (hist->hvalue.counters[0]);
 
   gcov_type v = hist->hvalue.counters[2 * n + 1];
   gcov_type c = hist->hvalue.counters[2 * n + 2];
 
+  if (hist->hvalue.counters[0] < 0
+      && (flag_profile_reproducible == PROFILE_REPRODUCIBILITY_PARALLEL_RUNS
+	  || (flag_profile_reproducible
+	      == PROFILE_REPRODUCIBILITY_MULTITHREADED)))
+    return false;
+
   /* Indirect calls can't be verified.  */
   if (stmt
       && check_counter (stmt, counter_type, &c, &read_all,
diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog
index 7b46ccb..dedc308 100644
--- a/libgcc/ChangeLog
+++ b/libgcc/ChangeLog
@@ -1,3 +1,10 @@
+2020-02-18  Martin Liska  <mliska@suse.cz>
+
+	PR ipa/92924
+	* libgcov-merge.c (merge_topn_values_set): Record
+	when a TOP N counter becomes invalid.  When merging
+	remove a smallest value if the space is needed.
+
 2020-02-12  Sandra Loosemore  <sandra@codesourcery.com>
 
 	PR libstdc++/79193
diff --git a/libgcc/libgcov-merge.c b/libgcc/libgcov-merge.c
index 19b8ee7..c0785b0 100644
--- a/libgcc/libgcov-merge.c
+++ b/libgcc/libgcov-merge.c
@@ -86,36 +86,47 @@ __gcov_merge_time_profile (gcov_type *counters, unsigned n_counters)
 
 #ifdef L_gcov_merge_topn
 
+/* To merging of TOPN profiles.
+   counters[0] is the number of executions
+   for i in 0 ... TOPN-1
+     counters[2 * i + 1] is target
+     counters[2 * i + 2] is corresponding hitrate counter.
+
+   Because we prune counters only those with probability >= 1/TOPN are
+   present now.
+
+   We use sign of counters[0] to track whether the number of different
+   targets exceeds TOPN.  */
+
 static void
 merge_topn_values_set (gcov_type *counters)
 {
   /* First value is number of total executions of the profiler.  */
-  gcov_type all = gcov_get_counter_ignore_scaling (-1);
-  counters[0] += all;
+  gcov_type all = gcov_get_counter ();
+  gcov_type *total = &counters[0];
   ++counters;
 
+  /* Negative value means that counter is missing some of values.  */
+  if (all < 0)
+    *total = -(*total);
+
+  *total += all;
+
   /* Read all part values.  */
   gcov_type read_counters[2 * GCOV_TOPN_VALUES];
-
   for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
     {
       read_counters[2 * i] = gcov_get_counter_target ();
       read_counters[2 * i + 1] = gcov_get_counter_ignore_scaling (-1);
     }
 
-  if (read_counters[1] == -1)
-    {
-      counters[1] = -1;
-      return;
-    }
-
   for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
     {
       if (read_counters[2 * i + 1] == 0)
 	continue;
 
       unsigned j;
-      int slot = -1;
+      int slot = 0;
 
       for (j = 0; j < GCOV_TOPN_VALUES; j++)
 	{
@@ -124,13 +135,15 @@ merge_topn_values_set (gcov_type *counters)
 	      counters[2 * j + 1] += read_counters[2 * i + 1];
 	      break;
 	    }
-	  else if (counters[2 * j + 1] == 0)
+	  else if (counters[2 * j + 1] < counters[2 * slot + 1])
 	    slot = j;
 	}
 
       if (j == GCOV_TOPN_VALUES)
 	{
-	  if (slot > 0)
+	  gcov_type slot_count = counters[2 * slot + 1];
+	  /* We found an empty slot.  */
+	  if (slot_count == 0)
 	    {
 	      /* If we found empty slot, add the value.  */
 	      counters[2 * slot] = read_counters[2 * i];
@@ -138,9 +151,16 @@ merge_topn_values_set (gcov_type *counters)
 	    }
 	  else
 	    {
-	      /* We haven't found a slot, bail out.  */
-	      counters[1] = -1;
-	      return;
+	      /* Here we are loosing some values.  */
+	      if (*total >= 0)
+		*total = -(*total);
+	      if (read_counters[2 * i + 1] > slot_count)
+		{
+		  counters[2 * slot] = read_counters[2 * i];
+		  counters[2 * slot + 1] = read_counters[2 * i + 1];
+		}
+	      else
+		counters[2 * slot + 1] -= read_counters[2 * i + 1];
 	    }
 	}
     }