[committed] Add --devel-die-count-method {none,estimate}

[committed] Add --devel-die-count-method {none,estimate}

[Tom de Vries]

Hi,

Before parsing a .debug_info section, we don't know how many DIEs it contains.
Knowing the number of DIEs before parsing can be used to:
- know whether we run into the low-mem die-limit, and if so go straight to
  parsing in low-mem mode.
- know whether we run into the max die-limit, and skip parsing altoghether.
- allocate off_htab immediately to its final size, instead of growing to it.

We've recently added code that estimates the number of DIEs, and optimizes
based on that, but estimates can be wrong, so it's good to keep the old
behaviour available for comparison.

Add a devel-only option --devel-die-count-method with argument values 'none'
and 'estimate'.  The default value is 'estimate'.

Behaviour --devel-die-count-method none: No estimates.
- If we run into the low-mem die-limit during parsing in regular mode, we
  restart parsing in low-mem mode.
- If we run into the max die-limit during parsing, we abort parsing.
- The off_htab grows to its final size.

Behaviour --devel-die-count-method estimate: Estimate the number of DIEs
before parsing.
- If the estimated number of DIEs is larger than the low-mem die-limit,
  count the number of DIEs before parsing to decide whether to start parsing
  in low-mem mode or regular mode.
- If the estimated number of DIEs is larger than the max die-limit,
  count the number of DIEs before parsing to decide whether to skip parsing or
  not.
- The off_htab is allocated at its estimated final size.

A comparison of the different arguments for cc1 gives:
...
$ time.sh dwz cc1 -o 1 --devel-die-count-method=none
maxmem: 994576
real: 12.60
user: 12.11
system: 0.48
$ time.sh dwz cc1 -o 1 --devel-die-count-method=estimate
maxmem: 655140
real: 9.51
user: 9.26
system: 0.24
...

And for vmlinux:
...
$ time.sh dwz vmlinux -o 1 --devel-die-count-method=none
maxmem: 1347248
real: 39.47
user: 37.77
system: 0.96
$ time.sh dwz vmlinux -o 1 --devel-die-count-method=estimate
maxmem: 1347424
real: 33.51
user: 32.89
system: 0.61
...

And for clang, while running into the max die-limit:
...
$ time.sh dwz clang -o 1 --devel-die-count-method=none
dwz: clang: Too many DIEs, not optimizing
Command exited with non-zero status 1
maxmem: 1111656
real: 7.30
user: 6.98
system: 0.31
$ time.sh dwz clang -o 1 --devel-die-count-method=estimate
dwz: clang: Too many DIEs, not optimizing
Command exited with non-zero status 1
maxmem: 638916
real: 1.74
user: 1.68
system: 0.05
...

And for clang, without running into any limit:
...
$ time.sh dwz -lnone -Lnone clang -o 1 --devel-die-count-method=none
maxmem: 12945264
real: 105.14
user: 93.21
system: 6.14
$ time.sh dwz -lnone -Lnone clang -o 1 --devel-die-count-method=estimate
maxmem: 13080256
real: 91.78
user: 82.45
system: 5.17
...

And finally, for CheckerOptionHandlingAnalyzerPlugin.so:
...
$ time.sh dwz -lnone -Lnone CheckerOptionHandlingAnalyzerPlugin.so -o 1 \
    --devel-die-count-method=none
maxmem: 5517416
real: 47.26
user: 43.99
system: 2.08
$ time.sh dwz -lnone -Lnone CheckerOptionHandlingAnalyzerPlugin.so -o 1 \
    --devel-die-count-method=estimate
maxmem: 5517240
real: 43.51
user: 41.59
system: 1.91
...

Committed to trunk.

Thanks,
- Tom

Add --devel-die-count-method {none,estimate}

2019-11-26  Tom de Vries  <tdevries@suse.de>

	* dwz.c (enum die_count_methods): New enum.
	(die_count_method): New var.
	(off_htab_add_die, read_debug_info): Handle die_count_method.
	(dwz_options): Add --devel-die-count-method entry.
	(main): Handle --devel-die-count-method.

---
 dwz.c | 46 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/dwz.c b/dwz.c
index 4e42062..8d62444 100644
--- a/dwz.c
+++ b/dwz.c
@@ -164,6 +164,12 @@ static int save_temps = 0;
 static int verify_edges_p = 0;
 static int dump_edges_p = 0;
 static int partition_dups_opt;
+enum die_count_methods
+{
+  none,
+  estimate
+};
+static enum die_count_methods die_count_method = estimate;
 
 typedef struct
 {
@@ -1317,6 +1323,7 @@ off_htab_add_die (dw_cu_ref cu, dw_die_ref die, unsigned int *die_count)
       if (low_mem
 	  || op_multifile
 	  || (multifile_mode == 0
+	      && die_count_method == estimate
 	      && (estimated_nr_dies >= low_mem_die_limit
 		  || estimated_nr_dies >= max_die_limit)))
 	initial_size = default_initial_size;
@@ -1325,11 +1332,21 @@ off_htab_add_die (dw_cu_ref cu, dw_die_ref die, unsigned int *die_count)
 	  size_t nr_dies;
 	  if (die_count && *die_count != 0)
 	    nr_dies = *die_count;
-	  else
+	  else if (die_count_method == none)
+	    nr_dies = 0;
+	  else if (die_count_method == estimate)
 	    nr_dies = estimated_nr_dies;
-	  size_t final_hashtab_size
-	    = emulate_htab (default_initial_size, nr_dies);
-	  initial_size = final_hashtab_size;
+	  else
+	    assert (false);
+
+	  if (nr_dies != 0)
+	    {
+	      size_t final_hashtab_size
+		= emulate_htab (default_initial_size, nr_dies);
+	      initial_size = final_hashtab_size;
+	    }
+	  else
+	    initial_size = default_initial_size;
 	}
       off_htab = htab_try_create (initial_size, off_hash, off_eq, NULL);
       if (tracing)
@@ -5097,6 +5114,7 @@ read_debug_info (DSO *dso, int kind, unsigned int *die_count)
   unsigned int estimated_nr_dies = estimate_nr_dies ();
   if (kind == DEBUG_INFO
       && multifile_mode == 0
+      && die_count_method == estimate
       && (estimated_nr_dies > max_die_limit
 	  || estimated_nr_dies > low_mem_die_limit))
     {
@@ -12936,6 +12954,8 @@ make_temp_file (const char *name)
   return fd;
 }
 
+int die_count_method_parsed;
+
 /* Options for getopt_long.  */
 static struct option dwz_options[] =
 {
@@ -12961,6 +12981,8 @@ static struct option dwz_options[] =
   { "devel-dump-edges",  no_argument,	    &dump_edges_p, 1 },
   { "devel-partition-dups-opt",
 			 no_argument,	    &partition_dups_opt, 1 },
+  { "devel-die-count-method",
+			 required_argument, &die_count_method_parsed, 1 },
 #endif
   { NULL,		 no_argument,	    0, 0 }
 };
@@ -13021,6 +13043,22 @@ main (int argc, char *argv[])
 
 	case 0:
 	  /* Option handled by getopt_long.  */
+	  if (die_count_method_parsed)
+	    {
+	      die_count_method_parsed = 0;
+	      if (strcmp (optarg, "none") == 0)
+		{
+		  die_count_method = none;
+		  break;
+		}
+	      if (strcmp (optarg, "estimate") == 0)
+		{
+		  die_count_method = estimate;
+		  break;
+		}
+	      error (1, 0, "invalid argument --devel-die-count-method %s",
+		     optarg);
+	    }
 	  break;
 
 	case 'o':