[binutils-gdb] libtool.m4: fix nm BSD flag detection

[binutils-gdb] libtool.m4: fix nm BSD flag detection

[Nick Alcock]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=bef9ef8ca0f941d743c77cc55b5fe7985990b2a7

commit bef9ef8ca0f941d743c77cc55b5fe7985990b2a7
Author: Nick Alcock <nick.alcock@oracle.com>
Date:   Mon Sep 27 20:31:21 2021 +0100

    libtool.m4: fix nm BSD flag detection
    
    Libtool needs to get BSD-format (or MS-format) output out of the system
    nm, so that it can scan generated object files for symbol names for
    -export-symbols-regex support.  Some nms need specific flags to turn on
    BSD-formatted output, so libtool checks for this in its AC_PATH_NM.
    Unfortunately the code to do this has a pair of interlocking flaws:
    
     - it runs the test by doing an nm of /dev/null.  Some platforms
       reasonably refuse to do an nm on a device file, but before now this
       has only been worked around by assuming that the error message has a
       specific textual form emitted by Tru64 nm, and that getting this
       error means this is Tru64 nm and that nm -B would work to produce
       BSD-format output, even though the test never actually got anything
       but an error message out of nm -B.  This is fixable by nm'ing *nm
       itself* (since we necessarily have a path to it).
    
     - the test is entirely skipped if NM is set in the environment, on the
       grounds that the user has overridden the test: but the user cannot
       reasonably be expected to know that libtool wants not only nm but
       also flags forcing BSD-format output.  Worse yet, one such "user" is
       the top-level Cygnus configure script, which neither tests for
       nor specifies any BSD-format flags.  So platforms needing BSD-format
       flags always fail to set them when run in a Cygnus tree, breaking
       -export-symbols-regex on such platforms.  Libtool also needs to
       augment $LD on some platforms, but this is done unconditionally,
       augmenting whatever the user specified: the nm check should do the
       same.
    
       One wrinkle: if the user has overridden $NM, a path might have been
       provided: so we use the user-specified path if there was one, and
       otherwise do the path search as usual.  (If the nm specified doesn't
       work, this might lead to a few extra pointless path searches -- but
       the test is going to fail anyway, so that's not a problem.)
    
    (Tested with NM unset, and set to nm, /usr/bin/nm, my-nm where my-nm is a
    symlink to /usr/bin/nm on the PATH, and /not-on-the-path/my-nm where
    *that* is a symlink to /usr/bin/nm.)
    
    ChangeLog
    2021-09-27  Nick Alcock  <nick.alcock@oracle.com>
    
            PR libctf/27967
            * libtool.m4 (LT_PATH_NM): Try BSDization flags with a user-provided
            NM, if there is one.  Run nm on itself, not on /dev/null, to avoid
            errors from nms that refuse to work on non-regular files.  Remove
            other workarounds for this problem.  Strip out blank lines from the
            nm output.

Diff:
---
 ChangeLog  |  9 +++++++
 libtool.m4 | 88 ++++++++++++++++++++++++++++++++------------------------------
 2 files changed, 54 insertions(+), 43 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index dd1ba3d66b5..afa6629abd1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2021-09-27  Nick Alcock  <nick.alcock@oracle.com>
+
+	PR libctf/27967
+	* libtool.m4 (LT_PATH_NM): Try BSDization flags with a user-provided
+	NM, if there is one.  Run nm on itself, not on /dev/null, to avoid
+	errors from nms that refuse to work on non-regular files.  Remove
+	other workarounds for this problem.  Strip out blank lines from the
+	nm output.
+
 2021-09-27  Nick Alcock  <nick.alcock@oracle.com>
 
 	PR libctf/27967
diff --git a/libtool.m4 b/libtool.m4
index 7a711249304..a216bb14e99 100644
--- a/libtool.m4
+++ b/libtool.m4
@@ -3200,53 +3200,55 @@ _LT_DECL([], [file_magic_cmd], [1],
 
 # LT_PATH_NM
 # ----------
-# find the pathname to a BSD- or MS-compatible name lister
+# find the pathname to a BSD- or MS-compatible name lister, and any flags
+# needed to make it compatible
 AC_DEFUN([LT_PATH_NM],
 [AC_REQUIRE([AC_PROG_CC])dnl
 AC_CACHE_CHECK([for BSD- or MS-compatible name lister (nm)], lt_cv_path_NM,
 [if test -n "$NM"; then
-  # Let the user override the test.
-  lt_cv_path_NM="$NM"
-else
-  lt_nm_to_check="${ac_tool_prefix}nm"
-  if test -n "$ac_tool_prefix" && test "$build" = "$host"; then
-    lt_nm_to_check="$lt_nm_to_check nm"
-  fi
-  for lt_tmp_nm in $lt_nm_to_check; do
-    lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
-    for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do
-      IFS="$lt_save_ifs"
-      test -z "$ac_dir" && ac_dir=.
-      tmp_nm="$ac_dir/$lt_tmp_nm"
-      if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then
-	# Check to see if the nm accepts a BSD-compat flag.
-	# Adding the `sed 1q' prevents false positives on HP-UX, which says:
-	#   nm: unknown option "B" ignored
-	# Tru64's nm complains that /dev/null is an invalid object file
-	case `"$tmp_nm" -B /dev/null 2>&1 | sed '1q'` in
-	*/dev/null* | *'Invalid file or object type'*)
-	  lt_cv_path_NM="$tmp_nm -B"
-	  break
-	  ;;
-	*)
-	  case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in
-	  */dev/null*)
-	    lt_cv_path_NM="$tmp_nm -p"
-	    break
-	    ;;
-	  *)
-	    lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but
-	    continue # so that we can try to find one that supports BSD flags
-	    ;;
-	  esac
-	  ;;
-	esac
-      fi
-    done
-    IFS="$lt_save_ifs"
-  done
-  : ${lt_cv_path_NM=no}
-fi])
+   # Let the user override the nm to test.
+   lt_nm_to_check="$NM"
+ else
+   lt_nm_to_check="${ac_tool_prefix}nm"
+   if test -n "$ac_tool_prefix" && test "$build" = "$host"; then
+     lt_nm_to_check="$lt_nm_to_check nm"
+   fi
+ fi
+ for lt_tmp_nm in $lt_nm_to_check; do
+   lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+   for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do
+     IFS="$lt_save_ifs"
+     test -z "$ac_dir" && ac_dir=.
+     case "$lt_tmp_nm" in
+     */*|*\\*) tmp_nm="$lt_tmp_nm";;
+     *) tmp_nm="$ac_dir/$lt_tmp_nm";;
+     esac
+     if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then
+       # Check to see if the nm accepts a BSD-compat flag.
+       # Adding the `sed 1q' prevents false positives on HP-UX, which says:
+       #   nm: unknown option "B" ignored
+       case `"$tmp_nm" -B "$tmp_nm" 2>&1 | grep -v '^ *$' | sed '1q'` in
+       *$tmp_nm*) lt_cv_path_NM="$tmp_nm -B"
+	 break
+	 ;;
+       *)
+	 case `"$tmp_nm" -p "$tmp_nm" 2>&1 | grep -v '^ *$' | sed '1q'` in
+	 *$tmp_nm*)
+	   lt_cv_path_NM="$tmp_nm -p"
+	   break
+	   ;;
+	 *)
+	   lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but
+	   continue # so that we can try to find one that supports BSD flags
+	   ;;
+	 esac
+	 ;;
+       esac
+     fi
+   done
+   IFS="$lt_save_ifs"
+ done
+ : ${lt_cv_path_NM=no}])
 if test "$lt_cv_path_NM" != "no"; then
   NM="$lt_cv_path_NM"
 else