PATCH: Add capability to contrib/compare_tests to handle directories

PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

[-- Attachment #1: Type: text/plain, Size: 5891 bytes --]

Hi,

Should not change behavior for comparing two files (expect for usage
output), and also should be POSIX compliant.

Tested on x86_64 tests logs and test directories, would be interested
in help testing on other platforms.

Ok to commit?
-- 
Quentin Neill


From 4d4fa9d094745ace0b6e51faadb2f3ea40cb7c7f Mon Sep 17 00:00:00 2001
From: Quentin Neill <quentin.neill@amd.com>
Date: Wed, 7 Sep 2011 12:04:35 -0500
Subject: [PATCH] Add capability to compare test log directories.

---
 contrib/ChangeLog     |    4 ++
 contrib/compare_tests |  107 ++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 101 insertions(+), 10 deletions(-)

diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index 07adb58..e2007e7 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,7 @@
+2011-09-07  Quentin Neill  <quentin.neill@amd.com>
+
+	* compare_tests: Add capability to compare test log directories.
+
 2011-08-25  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>

 	* gcc_update: Determine svn branch from hg convert_revision.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..b0e3321 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,36 @@
 # This script automatically test the given tool with the tool's test cases,
 # reporting anything of interest.

-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+	if [ -n "$1" ] ; then
+		echo "$0: Error: $1" >&2
+		echo >&2
+	fi
+	cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+	Compare the PREVIOUS and CURRENT test case logs, reporting anything
of interest.
+
+	If PREVIOUS and CURRENT are directories, find and compare any *.log files
+	therein (except config.log files).
+
+	Unless -strict is given, these discrepancies are not counted as errors:
+		missing/extra log files when comparing directories
+		tests that failed in PREVIOUS but pass in CURRENT
+		tests that were not in PREVIOUS but appear in CURRENT
+		tests in PREVIOUS that are missing in CURRENT
+
+	Exit with the following values:
+		0 if there is nothing of interest
+		1 if there are errors when comparing single test case files
+		N for the number of errors found when comparing directories
+EOUSAGE
+	exit 2
+}

 # Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>

 tool=gxx

@@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
 tmp2=/tmp/$tool-testing.$$b
 now_s=/tmp/$tool-testing.$$d
 before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0

-if [ "$2" = "" ]; then
-	echo "Usage: $0 previous current" >&2
-	exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+	find "$1" \( ! -name config.log \) -name '*.log' >$lst1
+	find "$2" \( ! -name config.log \) -name '*.log' >$lst2
+	echo "# Comparing directories"
+	echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
+	echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
+	echo
+	# remove leading directory components to compare
+	sed -e "s|^$1/||" $lst1 | sort >$lst3
+	sed -e "s|^$2/||" $lst2 | sort >$lst4
+	comm -23 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra log files in Dir1=$1"
+		sed -e "s|^|< $1/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -13 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra log files in Dir2=$2"
+		sed -e "s|^|> $2/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -12 $lst3 $lst4 | sort -u >$lst5
+	if [ ! -s $lst5 ] ; then
+		echo "# No common log files"
+		exit_status=`expr $exit_status + 1`
+		exit $exit_status
+	fi
+	cmnlogs=`cat $lst5 | wc -l`
+	echo "# Comparing $cmnlogs common log files"
+	for fname in `cat $lst5`
+	do
+		f1="$1/$fname"
+		f2="$2/$fname"
+		echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
+		${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
+		ret=$?
+		if [ $ret -ne 0 ]; then
+			exit_status=`expr $exit_status + 1`
+			echo "## Differences found: $fname"
+		fi
+	done
+	if [ $exit_status -ne 0 ]; then
+		echo "# $exit_status differences in $cmnlogs common log files found"
+	else
+		echo "# No differences found in $cmnlogs common log files"
+	fi
+	exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+	usage "Must specify either two directories or two files"
 fi

 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target /
{target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); };
print $0; }' >$tmp1
@@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk
'/^Running target / {target =
 before=$tmp1
 now=$tmp2

-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15

 if sort -k 2 </dev/null >/dev/null 2>&1; then
   skip1='-k 2'
@@ -60,6 +143,7 @@ if [ $? = 0 ]; then
 	echo "Tests that now work, but didn't before:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi

@@ -83,6 +167,7 @@ if [ $? = 0 ]; then
 	echo "New tests that PASS:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi

@@ -94,6 +179,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that passed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi

@@ -105,6 +191,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that failed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi

-- 
1.7.1

[-- Attachment #2: 1907-20110907-quentin.neill.amd.com-add_dir_compare_to_compare_test-patch.txt --]
[-- Type: text/plain, Size: 5650 bytes --]

From 74fc00e731b235b429226c9d9801bade015363f3 Mon Sep 17 00:00:00 2001
From: Quentin Neill <quentin.neill@amd.com>
Date: Wed, 7 Sep 2011 12:04:35 -0500
Subject: [PATCH] Add capability to compare test log directories.

---
 contrib/ChangeLog     |    4 ++
 contrib/compare_tests |  107 ++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 101 insertions(+), 10 deletions(-)

diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index 07adb58..e2007e7 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,7 @@
+2011-09-07  Quentin Neill  <quentin.neill@amd.com>
+
+	* compare_tests: Add capability to compare test log directories.
+
 2011-08-25  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
 
 	* gcc_update: Determine svn branch from hg convert_revision.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..88e0fa6 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,36 @@
 # This script automatically test the given tool with the tool's test cases,
 # reporting anything of interest.
 
-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+	if [ -n "$1" ] ; then
+		echo "$0: Error: $1" >&2
+		echo >&2
+	fi
+	cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+	Compare the PREVIOUS and CURRENT test case logs, reporting anything of interest.
+
+	If PREVIOUS and CURRENT are directories, find and compare any *.log files
+	therein (except config.log files).
+
+	Unless -strict is given, these discrepancies are not counted as errors:
+		missing/extra log files when comparing directories
+		tests that failed in PREVIOUS but pass in CURRENT
+		tests that were not in PREVIOUS but appear in CURRENT
+		tests in PREVIOUS that are missing in CURRENT
+
+	Exit with the following values:
+		0 if there is nothing of interest
+		1 if there are errors when comparing single test case files
+		N for the number of errors found when comparing directories
+EOUSAGE
+	exit 2
+}
 
 # Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill.gnu@gmail.com>
 
 tool=gxx
 
@@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
 tmp2=/tmp/$tool-testing.$$b
 now_s=/tmp/$tool-testing.$$d
 before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
 
-if [ "$2" = "" ]; then
-	echo "Usage: $0 previous current" >&2
-	exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+	find "$1" \( ! -name config.log \) -name '*.log' >$lst1
+	find "$2" \( ! -name config.log \) -name '*.log' >$lst2
+	echo "# Comparing directories"
+	echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
+	echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
+	echo
+	# remove leading directory components to compare
+	sed -e "s|^$1/||" $lst1 | sort >$lst3
+	sed -e "s|^$2/||" $lst2 | sort >$lst4
+	comm -23 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra log files in Dir1=$1"
+		sed -e "s|^|< $1/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -13 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra log files in Dir2=$2"
+		sed -e "s|^|> $2/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -12 $lst3 $lst4 | sort -u >$lst5
+	if [ ! -s $lst5 ] ; then
+		echo "# No common log files"
+		exit_status=`expr $exit_status + 1`
+		exit $exit_status
+	fi
+	cmnlogs=`cat $lst5 | wc -l`
+	echo "# Comparing $cmnlogs common log files"
+	for fname in `cat $lst5`
+	do
+		f1="$1/$fname"
+		f2="$2/$fname"
+		echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
+		${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
+		ret=$?
+		if [ $ret -ne 0 ]; then
+			exit_status=`expr $exit_status + 1`
+			echo "## Differences found: $fname"
+		fi
+	done
+	if [ $exit_status -ne 0 ]; then
+		echo "# $exit_status differences in $cmnlogs common log files found"
+	else
+		echo "# No differences found in $cmnlogs common log files"
+	fi
+	exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+	usage "Must specify either two directories or two files"
 fi
 
 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
@@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target =
 before=$tmp1
 now=$tmp2
 
-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
 
 if sort -k 2 </dev/null >/dev/null 2>&1; then
   skip1='-k 2'
@@ -60,6 +143,7 @@ if [ $? = 0 ]; then
 	echo "Tests that now work, but didn't before:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -83,6 +167,7 @@ if [ $? = 0 ]; then
 	echo "New tests that PASS:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -94,6 +179,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that passed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -105,6 +191,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that failed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
-- 
1.7.1

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

Ping?

On Wed, Sep 7, 2011 at 12:21 PM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> Hi,
>
> Should not change behavior for comparing two files (expect for usage
> output), and also should be POSIX compliant.
>
> Tested on x86_64 tests logs and test directories, would be interested
> in help testing on other platforms.
>
> Ok to commit?
> --
> Quentin Neill
>
>
> From 4d4fa9d094745ace0b6e51faadb2f3ea40cb7c7f Mon Sep 17 00:00:00 2001
> From: Quentin Neill <quentin.neill@amd.com>
> Date: Wed, 7 Sep 2011 12:04:35 -0500
> Subject: [PATCH] Add capability to compare test log directories.
>
> ---
>  contrib/ChangeLog     |    4 ++
>  contrib/compare_tests |  107 ++++++++++++++++++++++++++++++++++++++++++++-----
>  2 files changed, 101 insertions(+), 10 deletions(-)
>
> diff --git a/contrib/ChangeLog b/contrib/ChangeLog
> index 07adb58..e2007e7 100644
> --- a/contrib/ChangeLog
> +++ b/contrib/ChangeLog
> @@ -1,3 +1,7 @@
> +2011-09-07  Quentin Neill  <quentin.neill@amd.com>
> +
> +       * compare_tests: Add capability to compare test log directories.
> +
>  2011-08-25  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
>
>        * gcc_update: Determine svn branch from hg convert_revision.
> diff --git a/contrib/compare_tests b/contrib/compare_tests
> index bed9742..b0e3321 100755
> --- a/contrib/compare_tests
> +++ b/contrib/compare_tests
> @@ -2,13 +2,36 @@
>  # This script automatically test the given tool with the tool's test cases,
>  # reporting anything of interest.
>
> -# exits with 0 if there is nothing of interest
> -# exits with 1 if there is something interesting
> -# exits with 2 if an error occurred
> -
> -# Give two .sum files to compare them
> +usage()
> +{
> +       if [ -n "$1" ] ; then
> +               echo "$0: Error: $1" >&2
> +               echo >&2
> +       fi
> +       cat >&2 <<EOUSAGE
> +Usage: $0 [-strict] PREVIOUS CURRENT
> +
> +       Compare the PREVIOUS and CURRENT test case logs, reporting anything
> of interest.
> +
> +       If PREVIOUS and CURRENT are directories, find and compare any *.log files
> +       therein (except config.log files).
> +
> +       Unless -strict is given, these discrepancies are not counted as errors:
> +               missing/extra log files when comparing directories
> +               tests that failed in PREVIOUS but pass in CURRENT
> +               tests that were not in PREVIOUS but appear in CURRENT
> +               tests in PREVIOUS that are missing in CURRENT
> +
> +       Exit with the following values:
> +               0 if there is nothing of interest
> +               1 if there are errors when comparing single test case files
> +               N for the number of errors found when comparing directories
> +EOUSAGE
> +       exit 2
> +}
>
>  # Written by Mike Stump <mrs@cygnus.com>
> +# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
>
>  tool=gxx
>
> @@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
>  tmp2=/tmp/$tool-testing.$$b
>  now_s=/tmp/$tool-testing.$$d
>  before_s=/tmp/$tool-testing.$$e
> +lst1=/tmp/$tool-lst1.$$
> +lst2=/tmp/$tool-lst2.$$
> +lst3=/tmp/$tool-lst3.$$
> +lst4=/tmp/$tool-lst4.$$
> +lst5=/tmp/$tool-lst5.$$
> +tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
> +
> +[ "$1" = "-strict" ] && strict=$1 && shift
> +[ "$1" = "-?" ] && usage
> +[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
> +
> +trap "rm -f $tmps" 0 1 2 3 5 9 13 15
> +exit_status=0
>
> -if [ "$2" = "" ]; then
> -       echo "Usage: $0 previous current" >&2
> -       exit 2
> +if [ -d "$1" -a -d "$2" ] ; then
> +       find "$1" \( ! -name config.log \) -name '*.log' >$lst1
> +       find "$2" \( ! -name config.log \) -name '*.log' >$lst2
> +       echo "# Comparing directories"
> +       echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
> +       echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
> +       echo
> +       # remove leading directory components to compare
> +       sed -e "s|^$1/||" $lst1 | sort >$lst3
> +       sed -e "s|^$2/||" $lst2 | sort >$lst4
> +       comm -23 $lst3 $lst4 >$lst5
> +       if [ -s $lst5 ] ; then
> +               echo "# Extra log files in Dir1=$1"
> +               sed -e "s|^|< $1/|" $lst5
> +               echo
> +               [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
> +       fi
> +       comm -13 $lst3 $lst4 >$lst5
> +       if [ -s $lst5 ] ; then
> +               echo "# Extra log files in Dir2=$2"
> +               sed -e "s|^|> $2/|" $lst5
> +               echo
> +               [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
> +       fi
> +       comm -12 $lst3 $lst4 | sort -u >$lst5
> +       if [ ! -s $lst5 ] ; then
> +               echo "# No common log files"
> +               exit_status=`expr $exit_status + 1`
> +               exit $exit_status
> +       fi
> +       cmnlogs=`cat $lst5 | wc -l`
> +       echo "# Comparing $cmnlogs common log files"
> +       for fname in `cat $lst5`
> +       do
> +               f1="$1/$fname"
> +               f2="$2/$fname"
> +               echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
> +               ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
> +               ret=$?
> +               if [ $ret -ne 0 ]; then
> +                       exit_status=`expr $exit_status + 1`
> +                       echo "## Differences found: $fname"
> +               fi
> +       done
> +       if [ $exit_status -ne 0 ]; then
> +               echo "# $exit_status differences in $cmnlogs common log files found"
> +       else
> +               echo "# No differences found in $cmnlogs common log files"
> +       fi
> +       exit $exit_status
> +elif [ -d "$1" -o -d "$2" ] ; then
> +       usage "Must specify either two directories or two files"
>  fi
>
>  sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target /
> {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); };
> print $0; }' >$tmp1
> @@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk
> '/^Running target / {target =
>  before=$tmp1
>  now=$tmp2
>
> -exit_status=0
> -trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
>
>  if sort -k 2 </dev/null >/dev/null 2>&1; then
>   skip1='-k 2'
> @@ -60,6 +143,7 @@ if [ $? = 0 ]; then
>        echo "Tests that now work, but didn't before:"
>        echo
>        cat $tmp2
> +       [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
>        echo
>  fi
>
> @@ -83,6 +167,7 @@ if [ $? = 0 ]; then
>        echo "New tests that PASS:"
>        echo
>        cat $tmp2
> +       [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
>        echo
>  fi
>
> @@ -94,6 +179,7 @@ if [ $? = 0 ]; then
>        echo "Old tests that passed, that have disappeared: (Eeek!)"
>        echo
>        cat $tmp2
> +       [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
>        echo
>  fi
>
> @@ -105,6 +191,7 @@ if [ $? = 0 ]; then
>        echo "Old tests that failed, that have disappeared: (Eeek!)"
>        echo
>        cat $tmp2
> +       [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
>        echo
>  fi
>
> --
> 1.7.1
>

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Mike Stump]

On Oct 4, 2011, at 2:37 PM, Quentin Neill wrote:
> Ping?

The problem with this patch is it reorders the listing so that lower priority things are after higher priority things.  The entire point of the routine is to list the high priority things first, so that a casual user can read the first line or two and not immediately if there is something they care about.

Now, this property can be preserved simply by concatenating all the .sum files found in a directory into an all.sum file, and then running the script on those two files.  If one does that, then the priority order is preserved.

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

On Tue, Oct 4, 2011 at 4:57 PM, Mike Stump <mikestump@comcast.net> wrote:
> On Oct 4, 2011, at 2:37 PM, Quentin Neill wrote:
>> Ping?
>
> The problem with this patch is it reorders the listing so that lower priority things are after higher priority things.  The entire point of the routine is to list the high priority things first, so that a casual user can read the first line or two and not immediately if there is something they care about.
>
> Now, this property can be preserved simply by concatenating all the .sum files found in a directory into an all.sum file, and then running the script on those two files.  If one does that, then the priority order is preserved.

Hi Mike,
Thanks for the feedback.  From the comments, I assumed the usage was
to compare two .log files not .sum files.
Maybe it is a new scenario I'm imagining - comparing two builds to see
if ANY test results changed, regardless of order.
In any case, I will work on your changes when I get time (after stage1 probably)
-- 
Quentin

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

[-- Attachment #1: Type: text/plain, Size: 1336 bytes --]

On Wed, Nov 2, 2011 at 12:32 PM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> On Tue, Oct 4, 2011 at 4:57 PM, Mike Stump <mikestump@comcast.net> wrote:
>> On Oct 4, 2011, at 2:37 PM, Quentin Neill wrote:
>>> Ping?
>>
>> The problem with this patch is it reorders the listing so that lower priority things are after higher priority things.  The entire point of the routine is to list the high priority things first, so that a casual user can read the first line or two and not immediately if there is something they care about.
>>
>> Now, this property can be preserved simply by concatenating all the .sum files found in a directory into an all.sum file, and then running the script on those two files.  If one does that, then the priority order is preserved.
>
> Hi Mike,
> Thanks for the feedback.  From the comments, I assumed the usage was
> to compare two .log files not .sum files.
> Maybe it is a new scenario I'm imagining - comparing two builds to see
> if ANY test results changed, regardless of order.
> In any case, I will work on your changes when I get time (after stage1 probably)
> --
> Quentin

I got to this anyway.
My scenario about "ANY test results changed" is what I added with -strict.
This patch concatenates the common .sum files before comparing.

Okay to commit?
-- 
Quentin

[-- Attachment #2: 1912-20111104-quentin.neill.amd.com-add_dir_compare_to_compare_test2-patch.txt --]
[-- Type: text/plain, Size: 5686 bytes --]

From aaac4ffeedf4b3c9641d593de6526b4e872760d5 Mon Sep 17 00:00:00 2001
From: Quentin Neill <quentin.neill@amd.com>
Date: Fri, 4 Nov 2011 22:13:07 -0500
Subject: [PATCH] 2011-11-04   Quentin Neill  <quentin.neill@amd.com>

	* compare_tests: Add ability to compare all .sum
	files from two build directories.
---
 contrib/ChangeLog     |    5 ++
 contrib/compare_tests |  103 ++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index eddf6ec..b0afb27 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,8 @@
+2011-11-04   Quentin Neill  <quentin.neill@amd.com>
+
+	* compare_tests: Add ability to compare all .sum
+	files from two build directories.
+
 2011-09-13   Diego Novillo  <dnovillo@google.com>
 
 	* testsuite-management: New.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..a23fcf6 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,35 @@
 # This script automatically test the given tool with the tool's test cases,
 # reporting anything of interest.
 
-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+	if [ -n "$1" ] ; then
+		echo "$0: Error: $1" >&2
+		echo >&2
+	fi
+	cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
+
+	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
+
+	Unless -strict is given, these discrepancies are not counted as errors:
+		missing/extra .sum files when comparing directories
+		tests that failed in PREVIOUS but pass in CURRENT
+		tests that were not in PREVIOUS but appear in CURRENT
+		tests in PREVIOUS that are missing in CURRENT
+
+	Exit with the following values:
+		0 if there is nothing of interest
+		1 if there are errors when comparing single test case files
+		N for the number of errors found when comparing directories
+EOUSAGE
+	exit 2
+}
 
 # Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
 
 tool=gxx
 
@@ -16,10 +38,69 @@ tmp1=/tmp/$tool-testing.$$a
 tmp2=/tmp/$tool-testing.$$b
 now_s=/tmp/$tool-testing.$$d
 before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
 
-if [ "$2" = "" ]; then
-	echo "Usage: $0 previous current" >&2
-	exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+	find "$1" -name '*.sum' >$lst1
+	find "$2" -name '*.sum' >$lst2
+	echo "# Comparing directories"
+	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
+	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
+	echo
+	# remove leading directory components to compare
+	sed -e "s|^$1/||" $lst1 | sort >$lst3
+	sed -e "s|^$2/||" $lst2 | sort >$lst4
+	comm -23 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir1=$1"
+		sed -e "s|^|< $1/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -13 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir2=$2"
+		sed -e "s|^|> $2/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -12 $lst3 $lst4 | sort -u >$lst5
+	if [ ! -s $lst5 ] ; then
+		echo "# No common sum files"
+		exit_status=`expr $exit_status + 1`
+		exit $exit_status
+	fi
+	cmnsums=`cat $lst5 | wc -l`
+	echo "# Comparing $cmnsums common sum files"
+	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
+	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
+	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
+	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		exit_status=`expr $exit_status + 1`
+		echo "## Differences found: $fname"
+	fi
+	if [ $exit_status -ne 0 ]; then
+		echo "# $exit_status differences in $cmnsums common sum files found"
+	else
+		echo "# No differences found in $cmnsums common sum files"
+	fi
+	exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+	usage "Must specify either two directories or two files"
 fi
 
 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
@@ -28,8 +109,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target =
 before=$tmp1
 now=$tmp2
 
-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
 
 if sort -k 2 </dev/null >/dev/null 2>&1; then
   skip1='-k 2'
@@ -60,6 +139,7 @@ if [ $? = 0 ]; then
 	echo "Tests that now work, but didn't before:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -83,6 +163,7 @@ if [ $? = 0 ]; then
 	echo "New tests that PASS:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -94,6 +175,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that passed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -105,6 +187,7 @@ if [ $? = 0 ]; then
 	echo "Old tests that failed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
-- 
1.7.1

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Mike Stump]

On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
> This patch concatenates the common .sum files before comparing.
> 
> Okay to commit?

Ok, thanks for the contribution.

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

On Sat, Nov 5, 2011 at 4:26 PM, Mike Stump <mikestump@comcast.net> wrote:
> On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
>> This patch concatenates the common .sum files before comparing.
>>
>> Okay to commit?
>
> Ok, thanks for the contribution.
>

FYI I see my patch was missing these two fixes:
1. fix missing sum1 and sum2 temp variables
2. handle trailing slashes in dir args

Okay to commit?  Should I commit such a patch as trivial?
-- 
Quentin


Index: compare_tests
===================================================================
--- compare_tests       (revision 181166)
+++ compare_tests       (working copy)
@@ -43,7 +43,9 @@ lst2=/tmp/$tool-lst2.$$
 lst3=/tmp/$tool-lst3.$$
 lst4=/tmp/$tool-lst4.$$
 lst5=/tmp/$tool-lst5.$$
-tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+sum1=/tmp/$tool-sum1.$$
+sum2=/tmp/$tool-sum2.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"

 [ "$1" = "-strict" ] && strict=$1 && shift
 [ "$1" = "-?" ] && usage
@@ -60,8 +62,8 @@ if [ -d "$1" -a -d "$2" ] ; then
        echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
        echo
        # remove leading directory components to compare
-       sed -e "s|^$1/||" $lst1 | sort >$lst3
-       sed -e "s|^$2/||" $lst2 | sort >$lst4
+       sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
+       sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
        comm -23 $lst3 $lst4 >$lst5
        if [ -s $lst5 ] ; then
                echo "# Extra sum files in Dir1=$1"

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Mike Stump]

On Nov 8, 2011, at 7:29 AM, Quentin Neill <quentin.neill.gnu@gmail.com> wrote:
> FYI I see my patch was missing these two fixes:

> Okay to commit?

Ok.

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Mike Stump]

On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
> My scenario about "ANY test results changed" is what I added with -strict.
> This patch concatenates the common .sum files before comparing.

So, how exactly does this work for you:

+	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
+	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
+	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
+	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2

sum1 and sum2 appear to be variables that aren't set.

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

On Sat, Feb 11, 2012 at 8:13 AM, Mike Stump <mikestump@comcast.net> wrote:
> On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
>> My scenario about "ANY test results changed" is what I added with -strict.
>> This patch concatenates the common .sum files before comparing.
>
> So, how exactly does this work for you:
>
> +       ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
> +       ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
> +       echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
> +       ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
>
> sum1 and sum2 appear to be variables that aren't set.

Hi Mike,

Thanks for the fix.  This seemed familiar, and upon review it looks
like I never committed this fix:
http://gcc.gnu.org/ml/gcc-patches/2011-11/msg01194.html


Do you prefer this patch with my original intent (declaring sum1/sum2
with other tmps and removing the trap on line 52):

--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -43,7 +43,9 @@ lst2=/tmp/$tool-lst2.$$
 lst3=/tmp/$tool-lst3.$$
 lst4=/tmp/$tool-lst4.$$
 lst5=/tmp/$tool-lst5.$$
-tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+sum1=/tmp/$tool-sum1.$$
+sum2=/tmp/$tool-sum2.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"


 [ "$1" = "-strict" ] && strict=$1 && shift
 [ "$1" = "-?" ] && usage
@@ -60,8 +62,8 @@ if [ -d "$1" -a -d "$2" ] ; then
    echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
    echo
    # remove leading directory components to compare
-   sed -e "s|^$1/||" $lst1 | sort >$lst3
-   sed -e "s|^$2/||" $lst2 | sort >$lst4
+   sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
+   sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
    comm -23 $lst3 $lst4 >$lst5
    if [ -s $lst5 ] ; then
        echo "# Extra sum files in Dir1=$1"
@@ -83,14 +85,11 @@ if [ -d "$1" -a -d "$2" ] ; then
        exit $exit_status
    fi
    cmnsums=`cat $lst5 | wc -l`
-   sum1="/tmp/$tool-sum-1"
-   sum2="/tmp/$tool-sum-2"
    echo "# Comparing $cmnsums common sum files"
    ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
    ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
    echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
    ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
-   rm -f $sum1 $sum2
    ret=$?
    if [ $ret -ne 0 ]; then
        exit_status=`expr $exit_status + 1`



Or would you prefer this minimal fix (to remove trailing directory slashes)?

@@ -60,8 +60,8 @@ if [ -d "$1" -a -d "$2" ] ; then
    echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
    echo
    # remove leading directory components to compare
-   sed -e "s|^$1/||" $lst1 | sort >$lst3
-   sed -e "s|^$2/||" $lst2 | sort >$lst4
+   sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
+   sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
    comm -23 $lst3 $lst4 >$lst5
    if [ -s $lst5 ] ; then
        echo "# Extra sum files in Dir1=$1"


And if so, okay to commit?
-- 
Quentin

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

On Tue, Feb 14, 2012 at 10:39 AM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> On Sat, Feb 11, 2012 at 8:13 AM, Mike Stump <mikestump@comcast.net> wrote:
>> On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
>>> My scenario about "ANY test results changed" is what I added with -strict.
>>> This patch concatenates the common .sum files before comparing.
>>
>> So, how exactly does this work for you:
>>
>> +       ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
>> +       ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
>> +       echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
>> +       ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
>>
>> sum1 and sum2 appear to be variables that aren't set.
>
> Hi Mike,
>
> Thanks for the fix.  This seemed familiar, and upon review it looks
> like I never committed this fix:
> http://gcc.gnu.org/ml/gcc-patches/2011-11/msg01194.html
>
>
> Do you prefer this patch with my original intent (declaring sum1/sum2
> with other tmps and removing the trap on line 52):

Horrible wording for the first patch description.  How about:

  "Do you prefer this patch with my original intent (declaring sum1/sum2
   with other tmps, and removing those files in the trap on line 52):"

-- 
Quentin

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Mike Stump]

On Feb 14, 2012, at 8:39 AM, Quentin Neill wrote:
> Thanks for the fix.  This seemed familiar, and upon review it looks
> like I never committed this fix:
> http://gcc.gnu.org/ml/gcc-patches/2011-11/msg01194.html

Ah, ok, let's go with your version, it is much better.  Thanks.

Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill]

On Thu, Feb 16, 2012 at 8:18 AM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> On Tue, Feb 14, 2012 at 9:01 PM, Mike Stump <mikestump@comcast.net> wrote:
>> On Feb 14, 2012, at 8:39 AM, Quentin Neill wrote:
>>> Thanks for the fix.  This seemed familiar, and upon review it looks
>>> like I never committed this fix:
>>> http://gcc.gnu.org/ml/gcc-patches/2011-11/msg01194.html
>>
>> Ah, ok, let's go with your version, it is much better.  Thanks.
>
> Committed.
> --
> Quentin

While working on this, I realized errors detected in directories with
more than one sum file don't show the sum file with the problem
(instead they show the sum1/sum2 files).  Patch below iterates over
sum files instead of concatenating.  Okay to commit?

(output before the patch)

$ compare_tests -strict /d/qneill/tst.w[io].zeroshift.*
# Comparing directories
## Dir1=/d/qneill/tst.wi.zeroshift.192188f71c30: 2 sum files
## Dir2=/d/qneill/tst.wo.zeroshift.bdcd46972d7d: 2 sum files

# Comparing 2 common sum files
## /bin/sh /d/paqa/gcc/compare_tests -strict /tmp/gxx-sum1.4434
/tmp/gxx-sum2.4434
Old tests that passed, that have disappeared: (Eeek!)

gcc.target/x86_64/abi/callabi/vaarg-5b.c (test for excess errors)
Strict test fails


(output after the patch)
$ ./compare_tests -strict /d/qneill/tst.w[io].zeroshift.*
# Comparing directories
## Dir1=/d/qneill/tst.wi.zeroshift.192188f71c30: 2 sum files
## Dir2=/d/qneill/tst.wo.zeroshift.bdcd46972d7d: 2 sum files

# Comparing 2 common sum files
## /bin/sh ./compare_tests -strict
/d/qneill/tst.wi.zeroshift.192188f71c30/bld/gcc/testsuite/gcc/gcc.sum
/d/qneill/tst.wo.zeroshift.bdcd46972d7d/bld/gcc/testsuite/gcc/gcc.sum
Old tests that passed, that have disappeared: (Eeek!)

gcc.target/x86_64/abi/callabi/vaarg-5b.c (test for excess errors)
Strict test fails

## Differences found: bld/gcc/testsuite/gcc/gcc.sum
## /bin/sh ./compare_tests -strict
/d/qneill/tst.wi.zeroshift.192188f71c30/bld/gcc/testsuite/gcc/g++.sum
/d/qneill/tst.wo.zeroshift.bdcd46972d7d/bld/gcc/testsuite/gcc/g++.sum
# 1 differences in 2 common sum files found



diff --git a/contrib/compare_tests b/contrib/compare_tests
index 2fc6e05..611faab 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -43,9 +43,7 @@ lst2=/tmp/$tool-lst2.$$
 lst3=/tmp/$tool-lst3.$$
 lst4=/tmp/$tool-lst4.$$
 lst5=/tmp/$tool-lst5.$$
-sum1=/tmp/$tool-sum1.$$
-sum2=/tmp/$tool-sum2.$$
-tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"

 [ "$1" = "-strict" ] && strict=$1 && shift
 [ "$1" = "-?" ] && usage
@@ -86,15 +84,16 @@ if [ -d "$1" -a -d "$2" ] ; then
        fi
        cmnsums=`cat $lst5 | wc -l`
        echo "# Comparing $cmnsums common sum files"
-       ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
-       ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
-       echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
-       ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
-       ret=$?
-       if [ $ret -ne 0 ]; then
-               exit_status=`expr $exit_status + 1`
-               echo "## Differences found: $fname"
-       fi
+       for fname in `cat $lst5`
+       do
+               echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $1/$fname $2/$fname"
+               ${CONFIG_SHELL-/bin/sh} $0 $strict $1/$fname $2/$fname
+               ret=$?
+               if [ $ret -ne 0 ]; then
+                       exit_status=`expr $exit_status + 1`
+                       echo "## Differences found: $fname"
+               fi
+       done
        if [ $exit_status -ne 0 ]; then
                echo "# $exit_status differences in $cmnsums common
sum files found"
        else




-- 
Quentin