[glibc/maskray/grte] Add clang and debug support to conformance scripts

[glibc/maskray/grte] Add clang and debug support to conformance scripts

[Fangrui Song]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=47826951cfc8947d5f10197651f76fb03a8ce91e

commit 47826951cfc8947d5f10197651f76fb03a8ce91e
Author: Stan Shebs <stanshebs@google.com>
Date:   Wed Jan 24 12:42:41 2018 -0800

    Add clang and debug support to conformance scripts

Diff:
---
 conform/GlibcConform.pm        | 23 ++++++++++++++++++-
 conform/Makefile               |  3 +++
 conform/conformtest.pl         | 10 ++++++++-
 conform/linknamespace.pl       | 51 +++++++++++++++++++++++++++++++++++++-----
 conform/list-header-symbols.pl |  4 ++--
 5 files changed, 82 insertions(+), 9 deletions(-)

diff --git a/conform/GlibcConform.pm b/conform/GlibcConform.pm
index ba9c7e822f..4e54685819 100644
--- a/conform/GlibcConform.pm
+++ b/conform/GlibcConform.pm
@@ -39,20 +39,26 @@ $CFLAGS{"POSIX2008"} = "-std=c99 -D_POSIX_C_SOURCE=200809L";
 # Return a list of functions exported by a header, empty if an include
 # of the header does not compile.
 sub list_exported_functions {
-  my ($cc, $standard, $header, $tmpdir) = @_;
+  my ($cc, $standard, $header, $tmpdir, $withclang) = @_;
   my ($cc_all) = "$cc -D_ISOMAC $CFLAGS{$standard}";
   my ($tmpfile) = "$tmpdir/list-$$.c";
   my ($auxfile) = "$tmpdir/list-$$.c.aux";
+  my ($astfile) = "$tmpdir/list-$$.c.ast";
   my ($ret);
   my (%res) = ();
   open (TMPFILE, ">$tmpfile") || die ("open $tmpfile: $!\n");
   print TMPFILE "#include <$header>\n";
   close (TMPFILE) || die ("close $tmpfile: $!\n");
+  if ($withclang ne "yes") {
   $ret = system "$cc_all -c $tmpfile -o /dev/null -aux-info $auxfile > /dev/null";
+  } else {
+    $ret = system "$cc_all -c $tmpfile -o /dev/null -Xclang -ast-dump |grep FunctionDecl > $astfile";
+  }
   unlink ($tmpfile) || die ("unlink $tmpfile: $!\n");
   if ($ret != 0) {
     return;
   }
+  if ($withclang ne "yes") {
   open (AUXFILE, "<$auxfile") || die ("open $auxfile: $!\n");
   while (<AUXFILE>) {
     s|/\*.*?\*/||g;
@@ -70,5 +76,20 @@ sub list_exported_functions {
   }
   close (AUXFILE) || die ("close $auxfile: $!\n");
   unlink ($auxfile) || die ("unlink $auxfile: $!\n");
+  } else {
+  open (ASTFILE, "<$astfile") || die ("open $astfile: $!\n");
+  while (<ASTFILE>) {
+    s/^.*:[0-9][0-9]*:[0-9][0-9]* //g;
+    s/^.*:[0-9][0-9]* implicit //g;
+    s/^.*:[0-9][0-9]* //g;
+    if (/(\w+)\s* /) {
+      $res{$1} = 1;
+    } else {
+      die ("couldn't parse -ast-dump output: $_\n");
+    }
+  }
+  close (ASTFILE) || die ("close $astfile: $!\n");
+  unlink ($astfile) || die ("unlink $astfile: $!\n");
+  }
   return sort keys %res;
 }
diff --git a/conform/Makefile b/conform/Makefile
index 864fdeca21..6a72025f4b 100644
--- a/conform/Makefile
+++ b/conform/Makefile
@@ -178,6 +178,7 @@ $(conformtest-header-tests): $(objpfx)%/conform.out: \
 	 $(PERL) -I. conformtest.pl --tmpdir=$(@D)/scratch --cc='$(CC)' \
 		 --flags='$(conformtest-cc-flags)' --standard=$$std \
 		 --headers=$$hdr $(conformtest-xfail) $(conformtest-cross) \
+		 --withclang='$(with-clang)' \
 		 > $@); \
 	$(evaluate-test)
 
@@ -185,6 +186,7 @@ $(linknamespace-symlists-tests): $(objpfx)symlist-%: list-header-symbols.pl
 	$(PERL) -I. -w $< --tmpdir=$(objpfx) --cc='$(CC)' \
 		--flags='$(conformtest-cc-flags)' --standard=$* \
 		--headers="$(strip $(conformtest-headers-$*))" \
+		--withclang='$(with-clang)' \
 		> $@ 2> $@.err; \
 	$(evaluate-test)
 
@@ -225,6 +227,7 @@ $(linknamespace-header-tests): $(objpfx)%/linknamespace.out: \
 		 --stdsyms=$(objpfx)symlist-$$std --header=$$hdr \
 		 --libsyms=$(objpfx)symlist-stdlibs-$$std \
 		 --readelf='$(READELF)' \
+		 --withclang='$(with-clang)' \
 		 > $@ 2>&1); \
 	$(evaluate-test)
 
diff --git a/conform/conformtest.pl b/conform/conformtest.pl
index cb500f0e76..e12e072278 100644
--- a/conform/conformtest.pl
+++ b/conform/conformtest.pl
@@ -11,7 +11,7 @@ $cross = "";
 $xfail_str = "";
 GetOptions ('headers=s' => \@headers, 'standard=s' => \$standard,
 	    'flags=s' => \$flags, 'cc=s' => \$CC, 'tmpdir=s' => \$tmpdir,
-	    'cross' => \$cross, 'xfail=s' => \$xfail_str);
+	    'cross' => \$cross, 'xfail=s' => \$xfail_str, 'withclang=s' => \$withclang);
 @headers = split(/,/,join(',',@headers));
 
 # List of the headers we are testing.
@@ -270,9 +270,17 @@ sub checknamespace {
   close (TESTFILE);
 
   undef %errors;
+  if ($withclang eq "yes") {
+    open (CONTENT, "$CC $CFLAGS_namespace -E $fnamebase.c -P -Wp,-dM | sed -e '/^# [1-9]/d' -e '/^[[:space:]]*\$/d' |");
+  } else {
   open (CONTENT, "$CC $CFLAGS_namespace -E $fnamebase.c -P -Wp,-dN | sed -e '/^# [1-9]/d' -e '/^[[:space:]]*\$/d' |");
+  }
   loop: while (<CONTENT>) {
     chop;
+    if ($withclang eq "yes") {
+      # Filter extra output coming from -dM
+      s/^(#[^ ]+ [^ (]+).*$/$1/g;
+    }
     if (/^#define (.*)/) {
       newtoken ($1, @allow);
     } elsif (/^#undef (.*)/) {
diff --git a/conform/linknamespace.pl b/conform/linknamespace.pl
index 3fc6aca621..0445b32ae8 100644
--- a/conform/linknamespace.pl
+++ b/conform/linknamespace.pl
@@ -27,7 +27,9 @@ use Getopt::Long;
 GetOptions ('header=s' => \$header, 'standard=s' => \$standard,
 	    'flags=s' => \$flags, 'cc=s' => \$CC, 'tmpdir=s' => \$tmpdir,
 	    'stdsyms=s' => \$stdsyms_file, 'libsyms=s' => \$libsyms_file,
-	    'readelf=s' => \$READELF);
+	    'readelf=s' => \$READELF, 'withclang=s' => \$withclang);
+
+$debug = 1;
 
 # Load the list of symbols that are OK.
 %stdsyms = ();
@@ -162,7 +164,7 @@ foreach my $sym (@sym_data) {
 # detected by this script if the same namespace issue applies for
 # static linking.
 
-@c_syms = list_exported_functions ("$CC $flags", $standard, $header, $tmpdir);
+@c_syms = list_exported_functions ("$CC $flags", $standard, $header, $tmpdir, $withclang);
 $cincfile = "$tmpdir/undef-$$.c";
 $cincfile_o = "$tmpdir/undef-$$.o";
 $cincfile_sym = "$tmpdir/undef-$$.sym";
@@ -177,9 +179,11 @@ system ("$CC $flags -D_ISOMAC $CFLAGS{$standard} -c $cincfile -o $cincfile_o")
 system ("LC_ALL=C $READELF -W -s $cincfile_o > $cincfile_sym")
   && die ("readelf failed\n");
 @elf_syms = list_syms ($cincfile_sym);
-unlink ($cincfile) || die ("unlink $cincfile: $!\n");
-unlink ($cincfile_o) || die ("unlink $cincfile_o: $!\n");
-unlink ($cincfile_sym) || die ("unlink $cincfile_sym: $!\n");
+if (!$debug) {
+  unlink ($cincfile) || die ("unlink $cincfile: $!\n");
+  unlink ($cincfile_o) || die ("unlink $cincfile_o: $!\n");
+  unlink ($cincfile_sym) || die ("unlink $cincfile_sym: $!\n");
+}
 
 %seen_where = ();
 %files_seen = ();
@@ -207,7 +211,31 @@ while (%current_undef) {
 	  $seen_where{$ssym} = "$current_undef{$sym} -> [$file] $ssym";
 	}
       }
+      # A clang build can leave strong undefined symbols in the file,
+      # instead of GC'ing them; filter them out.
       foreach my $usym (@{$strong_undef_syms{$file}}) {
+	  $alsoseen = 0;
+	  foreach my $ssym (@{$seen_syms{$file}}) {
+	      if ($ssym eq $usym) {
+		  if ($debug) {
+		      print "$usym is strong undef also seen in $file, skipping\n";
+		  }
+		  $alsoseen = 1;
+		  last;
+	      }
+	  }
+	  if ($alsoseen) {
+	      next;
+	  }
+	  if ($debug) {
+	      foreach my $file2 (@{$sym_objs{$sym}}) {
+		  foreach my $ssym (@{$seen_syms{$file2}}) {
+		      if ($ssym eq $usym) {
+			  print " seen in $file2";
+		      }
+		  }
+	      }
+	  }
 	if (!defined ($all_undef{$usym})) {
 	  $all_undef{$usym} = "$current_undef{$sym} -> [$file] $usym";
 	  $new_undef{$usym} = "$current_undef{$sym} -> [$file] $usym";
@@ -220,14 +248,27 @@ while (%current_undef) {
 
 $ret = 0;
 foreach my $sym (sort keys %seen_where) {
+  if ($debug) {
+      print "RAW $seen_where{$sym}\n";
+  }
   if ($sym =~ /^_/) {
     next;
   }
   if (defined ($stdsyms{$sym})) {
+    if ($debug) {
+	print "$sym IS IN stdsyms\n";
+    }
     next;
   }
+  if ($debug) {
+      print "FINAL ";
+  }
   print "$seen_where{$sym}\n";
   $ret = 1;
 }
 
+if ($debug) {
+    print "Return result is $ret\n";
+}
+
 exit $ret;
diff --git a/conform/list-header-symbols.pl b/conform/list-header-symbols.pl
index 0db61bfe86..adb282c957 100644
--- a/conform/list-header-symbols.pl
+++ b/conform/list-header-symbols.pl
@@ -24,7 +24,7 @@ use GlibcConform;
 use Getopt::Long;
 
 GetOptions ('headers=s' => \$headers, 'standard=s' => \$standard,
-	    'flags=s' => \$flags, 'cc=s' => \$CC, 'tmpdir=s' => \$tmpdir);
+	    'flags=s' => \$flags, 'cc=s' => \$CC, 'tmpdir=s' => \$tmpdir, 'withclang=s' => \$withclang);
 @headers = split (/\s+/, $headers);
 
 # Extra symbols possibly not found through -aux-info but still
@@ -67,7 +67,7 @@ $extra_syms{"POSIX2008"} = ["errno", "setjmp", "va_end", "environ",
 %user_syms = ();
 
 foreach my $header (@headers) {
-  @syms = list_exported_functions ("$CC $flags", $standard, $header, $tmpdir);
+  @syms = list_exported_functions ("$CC $flags", $standard, $header, $tmpdir, $withclang);
   foreach my $sym (@syms) {
     if ($sym !~ /^_/) {
       $user_syms{$sym} = 1;