[PATCH 1/3] configure: Fix handling of python versions 2 and 3

[PATCH 1/3] configure: Fix handling of python versions 2 and 3

[David Ward]

When python version 2 is not found, AM_PROG_PYTHON sets the output
variable PYTHON to ":" (which is intentional; see "man 1P colon").
Fix incorrect tests that compared PYTHON to an empty string.

Use the same behavior for python version 3: when it is not found,
set the output variable PYTHON3 to ":" and test that accordingly.

Pass the variables "python3" and "py3execdir" to the subconfigure
unconditionally, just like the variables "python" and "pyexecdir".

When a program named "python" exists, fix a conditional that tests
if it is python version 3.

Do not guess the name of the python-config script. Simply append
"-config" to the program name for the python interpreter.
---
 configure    | 48 +++++++++++++++++++++---------------------------
 configure.ac | 34 ++++++++++++++++------------------
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/configure b/configure
index 48d8906..a0a784c 100755
--- a/configure
+++ b/configure
@@ -1638,7 +1638,7 @@ Optional Packages:
   --with-dracutbindir=DIR Use the dracut binary located in DIR
   --without-avahi         Do not use Avahi even if present
   --with-rpm              query rpm database for missing debuginfos
-  --with-python3          prefer /usr/bin/python3
+  --with-python3          prefer python version 3
   --with-elfutils=DIRECTORY
                           find elfutils source code in DIRECTORY
   --with-dyninst=DIRECTORY
@@ -9164,7 +9164,7 @@ cat >>confdefs.h <<_ACEOF
 #define PYTHON_BASENAME "${python_basename}"
 _ACEOF
 
-if test -n "$PYTHON"; then
+if test "x$PYTHON" != "x:"; then
 
 $as_echo "#define PYTHON_EXISTS /**/" >>confdefs.h
 
@@ -9176,7 +9176,7 @@ fi
 as_fn_append ac_configure_args " python='$PYTHON' pyexecdir='$pyexecdir'"
 
 # Now let's try to find python version 3.
-if test "x$PYTHON_UNKNOWN" != "xno" -a "x$python_unknown_version" = "x3"; then
+if test "x$PYTHON_UNKNOWN" != "xno" -a "x$ac_cv_python_unknown_version" = "x3"; then
    # Extract the first word of "python", so it can be a program name with args.
 set dummy python; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -9218,7 +9218,7 @@ fi
 
 
 else
-   for ac_prog in python3 python3.5 python3.4 python3.3 python3.2 python3.1 python3.0
+   for ac_prog in python3 python3.7 python3.6 python3.5 python3.4 python3.3 python3.2 python3.1 python3.0
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -9262,6 +9262,7 @@ fi
 
   test -n "$PYTHON3" && break
 done
+test -n "$PYTHON3" || PYTHON3=":"
 
 fi
 python3_basename=$(basename "$PYTHON3")
@@ -9273,7 +9274,7 @@ _ACEOF
 
 # If we found python version 3, set up the other variables for python
 # version 3 that AM_PATH_PYTHON sets up for python version 2.
-if test -n "$PYTHON3"; then
+if test "x$PYTHON3" != "x:"; then
 
 $as_echo "#define PYTHON3_EXISTS /**/" >>confdefs.h
 
@@ -9327,12 +9328,12 @@ fi
 $as_echo "$ac_cv_py3execdir" >&6; }
    py3execdir=$ac_cv_py3execdir
 
-
-   # Make sure the python3 and py3execdir variables get sent down to
-   # the subconfigure in the testsuite directory.
-   as_fn_append ac_configure_args " python3='$PYTHON3' py3execdir='$ac_cv_py3execdir'"
 fi
 
+# Make sure the python3 and py3execdir variables get sent down to
+# the subconfigure in the testsuite directory.
+as_fn_append ac_configure_args " python3='$PYTHON3' py3execdir='$ac_cv_py3execdir'"
+
 if test "${enable_prologues+set}" != set; then
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking to see if prologue searching should be the default" >&5
 $as_echo_n "checking to see if prologue searching should be the default... " >&6; }
@@ -11395,7 +11396,7 @@ if test "${with_python3+set}" = set; then :
 fi
 
 if test "x$with_python3" = "xyes"; then :
-  if test -z "$PYTHON3"; then :
+  if test "x$PYTHON3" = "x:"; then :
   as_fn_error $? "python version 3 is required" "$LINENO" 5
 else
   preferred_python=$PYTHON3
@@ -12135,10 +12136,9 @@ fi
 
 have_python2_support=no
 if test "x$with_python2_probes" != "xno"; then :
-     for ac_prog in python2-config python-config
-do
-  # Extract the first word of "$ac_prog", so it can be a program name with args.
-set dummy $ac_prog; ac_word=$2
+     if test "x$PYTHON" != "x:"; then :
+  # Extract the first word of "${python_basename}-config", so it can be a program name with args.
+set dummy ${python_basename}-config; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_PYTHON_CONFIG+:} false; then :
@@ -12177,11 +12177,8 @@ $as_echo "no" >&6; }
 fi
 
 
-  test -n "$PYTHON_CONFIG" && break
-done
-test -n "$PYTHON_CONFIG" || PYTHON_CONFIG="no"
-
-   if test "x$PYTHON_CONFIG" != "xno"; then :
+fi
+   if test "x$PYTHON_CONFIG" != "x"; then :
   	  	  PYTHON_CPPFLAGS=`$PYTHON_CONFIG --includes 2> /dev/null`
 	  PYTHON_CPPFLAGS=$PYTHON_CPPFLAGS
 
@@ -12230,12 +12227,9 @@ fi
 
 have_python3_support=no
 if test "x$with_python3_probes" != "xno"; then :
-     pathless_python3=$(basename "$PYTHON3")
-   if test "x${pathless_python3}" == "x"; then
-	pathless_python3="python3"
-   fi
-   # Extract the first word of "${pathless_python3}-config", so it can be a program name with args.
-set dummy ${pathless_python3}-config; ac_word=$2
+     if test "x$PYTHON3" != "x:" ; then :
+  # Extract the first word of "${python3_basename}-config", so it can be a program name with args.
+set dummy ${python3_basename}-config; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_PYTHON3_CONFIG+:} false; then :
@@ -12261,7 +12255,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_PYTHON3_CONFIG" && ac_cv_path_PYTHON3_CONFIG="no"
   ;;
 esac
 fi
@@ -12275,7 +12268,8 @@ $as_echo "no" >&6; }
 fi
 
 
-   if test "x$PYTHON3_CONFIG" != "xno"; then :
+fi
+   if test "x$PYTHON3_CONFIG" != "x"; then :
   	  	  PYTHON3_CPPFLAGS=`$PYTHON3_CONFIG --includes 2> /dev/null`
 	  PYTHON3_CPPFLAGS=$PYTHON3_CPPFLAGS
 
diff --git a/configure.ac b/configure.ac
index a545dc2..378b1dd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -80,7 +80,7 @@ AM_PATH_PYTHON([2.6], [], [:])
 python_basename=$(basename "$PYTHON")
 AC_DEFINE_UNQUOTED([PYTHON_BASENAME], "${python_basename}",
 		   [Base name of the python2 interpreter binary.])
-if test -n "$PYTHON"; then
+if test "x$PYTHON" != "x:"; then
 AC_DEFINE([PYTHON_EXISTS], [],
 	  [The python2 interpreter binary exists.])
 fi
@@ -91,11 +91,11 @@ fi
 AS_VAR_APPEND([ac_configure_args], [" python='$PYTHON' pyexecdir='$pyexecdir'"])
 
 # Now let's try to find python version 3.
-if test "x$PYTHON_UNKNOWN" != "xno" -a "x$python_unknown_version" = "x3"; then
+if test "x$PYTHON_UNKNOWN" != "xno" -a "x$ac_cv_python_unknown_version" = "x3"; then
    AC_PATH_PROG([PYTHON3], [python])
 else
    AC_PATH_PROGS([PYTHON3],
-		 [python3 python3.5 python3.4 python3.3 python3.2 python3.1 python3.0])
+		 [python3 python3.7 python3.6 python3.5 python3.4 python3.3 python3.2 python3.1 python3.0], [:])
 fi
 python3_basename=$(basename "$PYTHON3")
 AC_DEFINE_UNQUOTED([PYTHON3_BASENAME], "${python3_basename}",
@@ -103,7 +103,7 @@ AC_DEFINE_UNQUOTED([PYTHON3_BASENAME], "${python3_basename}",
 
 # If we found python version 3, set up the other variables for python
 # version 3 that AM_PATH_PYTHON sets up for python version 2.
-if test -n "$PYTHON3"; then
+if test "x$PYTHON3" != "x:"; then
    AC_DEFINE([PYTHON3_EXISTS], [],
 	     [The python3 interpreter binary exists.])
    AC_CACHE_CHECK([for python3 version], [ac_cv_python3_version],
@@ -123,12 +123,12 @@ if test -n "$PYTHON3"; then
    		  [ac_cv_py3execdir],
 		  [ac_cv_py3execdir=`$PYTHON3 -c "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(1,0,prefix='$PYTHON3_EXEC_PREFIX'))"`])
    AC_SUBST([py3execdir], [$ac_cv_py3execdir])
-
-   # Make sure the python3 and py3execdir variables get sent down to
-   # the subconfigure in the testsuite directory.
-   AS_VAR_APPEND([ac_configure_args], [" python3='$PYTHON3' py3execdir='$ac_cv_py3execdir'"])
 fi
 
+# Make sure the python3 and py3execdir variables get sent down to
+# the subconfigure in the testsuite directory.
+AS_VAR_APPEND([ac_configure_args], [" python3='$PYTHON3' py3execdir='$ac_cv_py3execdir'"])
+
 dnl Handle the prologues option.
 dnl
 dnl If the user didn't specify --enable-prologues/--disable-prologues
@@ -561,9 +561,9 @@ LIBS=$LIBS_no_readline
 
 dnl Allow user to choose python3 for /usr/bin/dtrace
 AC_ARG_WITH([python3],
-  AS_HELP_STRING([--with-python3],[prefer /usr/bin/python3]))
+  AS_HELP_STRING([--with-python3],[prefer python version 3]))
 AS_IF([test "x$with_python3" = "xyes"],
-      [AS_IF([test -z "$PYTHON3"],
+      [AS_IF([test "x$PYTHON3" = "x:"],
              [AC_MSG_ERROR([python version 3 is required])],
 	     [AC_SUBST(preferred_python,[$PYTHON3])])],
       [AS_IF([test "x$PYTHON" = "x:"],
@@ -735,8 +735,9 @@ AC_ARG_WITH([python2-probes],
 have_python2_support=no
 AS_IF([test "x$with_python2_probes" != "xno"],
   [dnl First, check to make sure we have the 'python-config' script.
-   AC_PATH_PROGS([PYTHON_CONFIG], [python2-config python-config], [no])
-   AS_IF([test "x$PYTHON_CONFIG" != "xno"],
+   AS_IF([test "x$PYTHON" != "x:"],
+	 [AC_PATH_PROG([PYTHON_CONFIG], [${python_basename}-config])])
+   AS_IF([test "x$PYTHON_CONFIG" != "x"],
 	 [dnl Use the 'python-config' script to find the python
 	  dnl include directories.
 	  PYTHON_CPPFLAGS=`$PYTHON_CONFIG --includes 2> /dev/null`
@@ -765,12 +766,9 @@ AC_ARG_WITH([python3-probes],
 have_python3_support=no
 AS_IF([test "x$with_python3_probes" != "xno"],
   [dnl First, check to make sure we have the 'python3-config' script.
-   pathless_python3=$(basename "$PYTHON3")
-   if test "x${pathless_python3}" == "x"; then
-	pathless_python3="python3"   
-   fi
-   AC_PATH_PROG([PYTHON3_CONFIG], [${pathless_python3}-config], [no])
-   AS_IF([test "x$PYTHON3_CONFIG" != "xno"],
+   AS_IF([test "x$PYTHON3" != "x:" ],
+	 [AC_PATH_PROG([PYTHON3_CONFIG], [${python3_basename}-config])])
+   AS_IF([test "x$PYTHON3_CONFIG" != "x"],
 	 [dnl Use the 'python3-config' script to find the python
 	  dnl include directories.
 	  PYTHON3_CPPFLAGS=`$PYTHON3_CONFIG --includes 2> /dev/null`
-- 
1.8.3.1

[PATCH 2/3] overload.py: Fix python version 2/3 compatibility

[David Ward]

The modified XML tree is outputted either as a bytearray with UTF-8
encoding in python version 3, or as a string in python version 2.
Handle this by writing the bytearray directly to sys.stdout.buffer,
or the string directly to sys.stdout, respectively.

Remove what appears to be "troubleshooting code" that was added in
commit 616ec7a0b, which dumps a large amount of unnecessary output
to stderr.

Call this script using the configured program name for python.
---
 doc/SystemTap_Tapset_Reference/Makefile.am |  2 +-
 doc/SystemTap_Tapset_Reference/Makefile.in |  2 +-
 doc/SystemTap_Tapset_Reference/overload.py | 11 ++++-------
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/doc/SystemTap_Tapset_Reference/Makefile.am b/doc/SystemTap_Tapset_Reference/Makefile.am
index 8a91bd7..6599e52 100644
--- a/doc/SystemTap_Tapset_Reference/Makefile.am
+++ b/doc/SystemTap_Tapset_Reference/Makefile.am
@@ -23,7 +23,7 @@ tapsets.xml: docproc $(shell find $(SRCTREE)/tapset -name '*.stp')
 if BUILD_HTMLDOCS
 	sed -e '/^!Syscalls/{r $(abs_srcdir)/syscalls.xmlpart' -e 'd}' $(abs_srcdir)/tapsets.tmpl > tapsets.tmpl.new
 	SRCTREE=$(SRCTREE) $(DOCPROC) doc tapsets.tmpl.new > tapsets.xml.new
-	python $(srcdir)/overload.py tapsets.xml.new > tapsets.xml.new1
+	$(preferred_python) $(srcdir)/overload.py tapsets.xml.new > tapsets.xml.new1
 	xsltproc $(srcdir)/sort-tapsets.xslt tapsets.xml.new1 > tapsets.xml.new2
 	rm tapsets.xml.new tapsets.xml.new1 tapsets.tmpl.new
 	if test -s tapsets.xml && cmp tapsets.xml.new2 tapsets.xml >/dev/null ; then \
diff --git a/doc/SystemTap_Tapset_Reference/Makefile.in b/doc/SystemTap_Tapset_Reference/Makefile.in
index c441e73..2196b5e 100644
--- a/doc/SystemTap_Tapset_Reference/Makefile.in
+++ b/doc/SystemTap_Tapset_Reference/Makefile.in
@@ -620,7 +620,7 @@ uninstall-am:
 @BUILD_REFDOCS_TRUE@tapsets.xml: docproc $(shell find $(SRCTREE)/tapset -name '*.stp')
 @BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	sed -e '/^!Syscalls/{r $(abs_srcdir)/syscalls.xmlpart' -e 'd}' $(abs_srcdir)/tapsets.tmpl > tapsets.tmpl.new
 @BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	SRCTREE=$(SRCTREE) $(DOCPROC) doc tapsets.tmpl.new > tapsets.xml.new
-@BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	python $(srcdir)/overload.py tapsets.xml.new > tapsets.xml.new1
+@BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	$(preferred_python) $(srcdir)/overload.py tapsets.xml.new > tapsets.xml.new1
 @BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	xsltproc $(srcdir)/sort-tapsets.xslt tapsets.xml.new1 > tapsets.xml.new2
 @BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	rm tapsets.xml.new tapsets.xml.new1 tapsets.tmpl.new
 @BUILD_HTMLDOCS_TRUE@@BUILD_REFDOCS_TRUE@	if test -s tapsets.xml && cmp tapsets.xml.new2 tapsets.xml >/dev/null ; then \
diff --git a/doc/SystemTap_Tapset_Reference/overload.py b/doc/SystemTap_Tapset_Reference/overload.py
index 18d670b..6ddec43 100755
--- a/doc/SystemTap_Tapset_Reference/overload.py
+++ b/doc/SystemTap_Tapset_Reference/overload.py
@@ -1,4 +1,3 @@
-#! /usr/bin/python
 # XML tree transformation for systemtap function overloading
 # This script merges all overloaded tapset function entries
 # into one entry.
@@ -45,7 +44,6 @@ def annotate(entry):
     """
     Numbers all overloaded entries.
     """
-    sys.stderr.write("entry: %s\n" % etree.tostring(entry))
     num_overloads = len(entry.xpath("refsynopsisdiv/programlisting"))
     synopsis = entry.xpath("refsynopsisdiv/programlisting")
     description = entry.xpath("refsect1[2]/para")
@@ -59,8 +57,6 @@ def merge(functions):
     """
     merged = functions[0]
 
-    sys.stderr.write("processing item %s\n" % merged.xpath("refnamediv/refname")[0].text)
-    
     # merge params
     new_params = get_params(functions)
     param_list = merged.xpath("refsect1[1]/variablelist")[0]
@@ -84,7 +80,7 @@ def merge_overloads(functions_list):
         merge(functions)
 
 def usage():
-    print "Usage: ./overload.py <xml>"
+    print("Usage: ./overload.py <xml>")
 
 def main():
     if len(sys.argv) != 2:
@@ -96,8 +92,9 @@ def main():
     refentries = [r for r in root.iter("refentry")]
     functions = collect_overloads(refentries)
     merge_overloads(functions)
-    print etree.tostring(root, encoding='UTF-8', xml_declaration=True,
-            doctype=tree.docinfo.doctype)
+    output_file = getattr(sys.stdout, "buffer", sys.stdout)
+    tree.write(output_file, encoding="UTF-8", xml_declaration=True,
+                 doctype=tree.docinfo.doctype)
 
 if __name__ == '__main__':
     main()
-- 
1.8.3.1

[PATCH 3/3] doc/Makefile: Fix command list syntax

[David Ward]

---
 doc/Makefile.am | 4 ++--
 doc/Makefile.in | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/Makefile.am b/doc/Makefile.am
index 02459d8..9d50cb3 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -34,8 +34,8 @@ SUFFIXES = ps pdf dvi ps tex
 .tex.dvi:
 	pwd=`pwd`; cd $(srcdir); \
 	latex -output-directory=$$pwd $<; \
-	touch $*.glo \
-	makeindex $*.glo -s nomencl.ist -o $*.gls \
+	touch $*.glo; \
+	makeindex $*.glo -s nomencl.ist -o $*.gls; \
 	latex -output-directory=$$pwd $<; \
 	latex -output-directory=$$pwd $<; \
 	latex -output-directory=$$pwd $<
diff --git a/doc/Makefile.in b/doc/Makefile.in
index c4a72b6..3dc73a5 100644
--- a/doc/Makefile.in
+++ b/doc/Makefile.in
@@ -642,8 +642,8 @@ uninstall-am: uninstall-local
 .tex.dvi:
 	pwd=`pwd`; cd $(srcdir); \
 	latex -output-directory=$$pwd $<; \
-	touch $*.glo \
-	makeindex $*.glo -s nomencl.ist -o $*.gls \
+	touch $*.glo; \
+	makeindex $*.glo -s nomencl.ist -o $*.gls; \
 	latex -output-directory=$$pwd $<; \
 	latex -output-directory=$$pwd $<; \
 	latex -output-directory=$$pwd $<
-- 
1.8.3.1

[PATCH 0/3] Fix build issues involving Python

[David Ward]

The following patches fix build issues that arise because of differences
in the way Python is installed across distributions. For example, RHEL 8
will not have a program called "python" [1]. In Arch Linux, the "python"
program is actually Python 3. And in many cases, Python 2 is no longer
installed by default.

Thanks!

[1]
https://developers.redhat.com/blog/2018/11/27/what-no-python-in-rhel-8-beta/

David Ward (3):
  configure: Fix handling of python versions 2 and 3
  overload.py: Fix python version 2/3 compatibility
  doc/Makefile: Fix command list syntax

 configure                                  | 48 +++++++++++++-----------------
 configure.ac                               | 34 ++++++++++-----------
 doc/Makefile.am                            |  4 +--
 doc/Makefile.in                            |  4 +--
 doc/SystemTap_Tapset_Reference/Makefile.am |  2 +-
 doc/SystemTap_Tapset_Reference/Makefile.in |  2 +-
 doc/SystemTap_Tapset_Reference/overload.py | 11 +++----
 7 files changed, 47 insertions(+), 58 deletions(-)

-- 
1.8.3.1

Re: [PATCH 0/3] Fix build issues involving Python

[Serhei Makarov]

On Mon, Feb 11, 2019, at 12:26 PM, David Ward wrote:
> The following patches fix build issues that arise because of differences
> in the way Python is installed across distributions. For example, RHEL 8
> will not have a program called "python" [1]. In Arch Linux, the "python"
> program is actually Python 3. And in many cases, Python 2 is no longer
> installed by default.

Thank you! It looked good to me, so I have pushed these patches to the repo. Sorry about the delay.

I can attest to needing these fixes on RHEL8 to avoid needing to configure --with-python manually.

-- 
All the best,
    Serhei
    https://serhei.io