From: Quentin Neill <quentin.neill.gnu@gmail.com>
To: gcc-patches@gcc.gnu.org
Subject: PATCH: Add capability to contrib/compare_tests to handle directories
Date: Wed, 07 Sep 2011 17:29:00 -0000 [thread overview]
Message-ID: <CAEhygDr2bLu5fGyeAxD=gsvfA9dz7tHE-XnndTDP5N4Augt1Sw@mail.gmail.com> (raw)
[-- Attachment #1: Type: text/plain, Size: 5891 bytes --]
Hi,
Should not change behavior for comparing two files (expect for usage
output), and also should be POSIX compliant.
Tested on x86_64 tests logs and test directories, would be interested
in help testing on other platforms.
Ok to commit?
--
Quentin Neill
From 4d4fa9d094745ace0b6e51faadb2f3ea40cb7c7f Mon Sep 17 00:00:00 2001
From: Quentin Neill <quentin.neill@amd.com>
Date: Wed, 7 Sep 2011 12:04:35 -0500
Subject: [PATCH] Add capability to compare test log directories.
---
contrib/ChangeLog | 4 ++
contrib/compare_tests | 107 ++++++++++++++++++++++++++++++++++++++++++++-----
2 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index 07adb58..e2007e7 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,7 @@
+2011-09-07 Quentin Neill <quentin.neill@amd.com>
+
+ * compare_tests: Add capability to compare test log directories.
+
2011-08-25 Rainer Orth <ro@CeBiTec.Uni-Bielefeld.DE>
* gcc_update: Determine svn branch from hg convert_revision.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..b0e3321 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,36 @@
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.
-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+ if [ -n "$1" ] ; then
+ echo "$0: Error: $1" >&2
+ echo >&2
+ fi
+ cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+ Compare the PREVIOUS and CURRENT test case logs, reporting anything
of interest.
+
+ If PREVIOUS and CURRENT are directories, find and compare any *.log files
+ therein (except config.log files).
+
+ Unless -strict is given, these discrepancies are not counted as errors:
+ missing/extra log files when comparing directories
+ tests that failed in PREVIOUS but pass in CURRENT
+ tests that were not in PREVIOUS but appear in CURRENT
+ tests in PREVIOUS that are missing in CURRENT
+
+ Exit with the following values:
+ 0 if there is nothing of interest
+ 1 if there are errors when comparing single test case files
+ N for the number of errors found when comparing directories
+EOUSAGE
+ exit 2
+}
# Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
tool=gxx
@@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
tmp2=/tmp/$tool-testing.$$b
now_s=/tmp/$tool-testing.$$d
before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
-if [ "$2" = "" ]; then
- echo "Usage: $0 previous current" >&2
- exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+ find "$1" \( ! -name config.log \) -name '*.log' >$lst1
+ find "$2" \( ! -name config.log \) -name '*.log' >$lst2
+ echo "# Comparing directories"
+ echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
+ echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
+ echo
+ # remove leading directory components to compare
+ sed -e "s|^$1/||" $lst1 | sort >$lst3
+ sed -e "s|^$2/||" $lst2 | sort >$lst4
+ comm -23 $lst3 $lst4 >$lst5
+ if [ -s $lst5 ] ; then
+ echo "# Extra log files in Dir1=$1"
+ sed -e "s|^|< $1/|" $lst5
+ echo
+ [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ fi
+ comm -13 $lst3 $lst4 >$lst5
+ if [ -s $lst5 ] ; then
+ echo "# Extra log files in Dir2=$2"
+ sed -e "s|^|> $2/|" $lst5
+ echo
+ [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ fi
+ comm -12 $lst3 $lst4 | sort -u >$lst5
+ if [ ! -s $lst5 ] ; then
+ echo "# No common log files"
+ exit_status=`expr $exit_status + 1`
+ exit $exit_status
+ fi
+ cmnlogs=`cat $lst5 | wc -l`
+ echo "# Comparing $cmnlogs common log files"
+ for fname in `cat $lst5`
+ do
+ f1="$1/$fname"
+ f2="$2/$fname"
+ echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
+ ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
+ ret=$?
+ if [ $ret -ne 0 ]; then
+ exit_status=`expr $exit_status + 1`
+ echo "## Differences found: $fname"
+ fi
+ done
+ if [ $exit_status -ne 0 ]; then
+ echo "# $exit_status differences in $cmnlogs common log files found"
+ else
+ echo "# No differences found in $cmnlogs common log files"
+ fi
+ exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+ usage "Must specify either two directories or two files"
fi
sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target /
{target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); };
print $0; }' >$tmp1
@@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk
'/^Running target / {target =
before=$tmp1
now=$tmp2
-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
if sort -k 2 </dev/null >/dev/null 2>&1; then
skip1='-k 2'
@@ -60,6 +143,7 @@ if [ $? = 0 ]; then
echo "Tests that now work, but didn't before:"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -83,6 +167,7 @@ if [ $? = 0 ]; then
echo "New tests that PASS:"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -94,6 +179,7 @@ if [ $? = 0 ]; then
echo "Old tests that passed, that have disappeared: (Eeek!)"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -105,6 +191,7 @@ if [ $? = 0 ]; then
echo "Old tests that failed, that have disappeared: (Eeek!)"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
--
1.7.1
[-- Attachment #2: 1907-20110907-quentin.neill.amd.com-add_dir_compare_to_compare_test-patch.txt --]
[-- Type: text/plain, Size: 5650 bytes --]
From 74fc00e731b235b429226c9d9801bade015363f3 Mon Sep 17 00:00:00 2001
From: Quentin Neill <quentin.neill@amd.com>
Date: Wed, 7 Sep 2011 12:04:35 -0500
Subject: [PATCH] Add capability to compare test log directories.
---
contrib/ChangeLog | 4 ++
contrib/compare_tests | 107 ++++++++++++++++++++++++++++++++++++++++++++-----
2 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index 07adb58..e2007e7 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,7 @@
+2011-09-07 Quentin Neill <quentin.neill@amd.com>
+
+ * compare_tests: Add capability to compare test log directories.
+
2011-08-25 Rainer Orth <ro@CeBiTec.Uni-Bielefeld.DE>
* gcc_update: Determine svn branch from hg convert_revision.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..88e0fa6 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,36 @@
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.
-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+ if [ -n "$1" ] ; then
+ echo "$0: Error: $1" >&2
+ echo >&2
+ fi
+ cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+ Compare the PREVIOUS and CURRENT test case logs, reporting anything of interest.
+
+ If PREVIOUS and CURRENT are directories, find and compare any *.log files
+ therein (except config.log files).
+
+ Unless -strict is given, these discrepancies are not counted as errors:
+ missing/extra log files when comparing directories
+ tests that failed in PREVIOUS but pass in CURRENT
+ tests that were not in PREVIOUS but appear in CURRENT
+ tests in PREVIOUS that are missing in CURRENT
+
+ Exit with the following values:
+ 0 if there is nothing of interest
+ 1 if there are errors when comparing single test case files
+ N for the number of errors found when comparing directories
+EOUSAGE
+ exit 2
+}
# Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill.gnu@gmail.com>
tool=gxx
@@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
tmp2=/tmp/$tool-testing.$$b
now_s=/tmp/$tool-testing.$$d
before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
-if [ "$2" = "" ]; then
- echo "Usage: $0 previous current" >&2
- exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+ find "$1" \( ! -name config.log \) -name '*.log' >$lst1
+ find "$2" \( ! -name config.log \) -name '*.log' >$lst2
+ echo "# Comparing directories"
+ echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
+ echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
+ echo
+ # remove leading directory components to compare
+ sed -e "s|^$1/||" $lst1 | sort >$lst3
+ sed -e "s|^$2/||" $lst2 | sort >$lst4
+ comm -23 $lst3 $lst4 >$lst5
+ if [ -s $lst5 ] ; then
+ echo "# Extra log files in Dir1=$1"
+ sed -e "s|^|< $1/|" $lst5
+ echo
+ [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ fi
+ comm -13 $lst3 $lst4 >$lst5
+ if [ -s $lst5 ] ; then
+ echo "# Extra log files in Dir2=$2"
+ sed -e "s|^|> $2/|" $lst5
+ echo
+ [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ fi
+ comm -12 $lst3 $lst4 | sort -u >$lst5
+ if [ ! -s $lst5 ] ; then
+ echo "# No common log files"
+ exit_status=`expr $exit_status + 1`
+ exit $exit_status
+ fi
+ cmnlogs=`cat $lst5 | wc -l`
+ echo "# Comparing $cmnlogs common log files"
+ for fname in `cat $lst5`
+ do
+ f1="$1/$fname"
+ f2="$2/$fname"
+ echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
+ ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
+ ret=$?
+ if [ $ret -ne 0 ]; then
+ exit_status=`expr $exit_status + 1`
+ echo "## Differences found: $fname"
+ fi
+ done
+ if [ $exit_status -ne 0 ]; then
+ echo "# $exit_status differences in $cmnlogs common log files found"
+ else
+ echo "# No differences found in $cmnlogs common log files"
+ fi
+ exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+ usage "Must specify either two directories or two files"
fi
sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
@@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target =
before=$tmp1
now=$tmp2
-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
if sort -k 2 </dev/null >/dev/null 2>&1; then
skip1='-k 2'
@@ -60,6 +143,7 @@ if [ $? = 0 ]; then
echo "Tests that now work, but didn't before:"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -83,6 +167,7 @@ if [ $? = 0 ]; then
echo "New tests that PASS:"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -94,6 +179,7 @@ if [ $? = 0 ]; then
echo "Old tests that passed, that have disappeared: (Eeek!)"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
@@ -105,6 +191,7 @@ if [ $? = 0 ]; then
echo "Old tests that failed, that have disappeared: (Eeek!)"
echo
cat $tmp2
+ [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
echo
fi
--
1.7.1
next reply other threads:[~2011-09-07 17:21 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2011-09-07 17:29 Quentin Neill [this message]
2011-10-04 21:45 ` Quentin Neill
2011-10-04 22:08 ` Mike Stump
2011-11-02 17:45 ` Quentin Neill
2011-11-05 4:33 ` Quentin Neill
2011-11-05 21:39 ` Mike Stump
2011-11-08 16:11 ` Quentin Neill
2011-11-08 16:48 ` Mike Stump
2012-02-11 14:15 ` Mike Stump
2012-02-14 16:41 ` Quentin Neill
2012-02-14 16:47 ` Quentin Neill
2012-02-15 7:23 ` Mike Stump
[not found] ` <CAEhygDrP-ywsm8ZZ79YF6modV-GzLtpRpkT9DRC507QLjnSb8Q@mail.gmail.com>
2012-02-16 15:34 ` Quentin Neill
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to='CAEhygDr2bLu5fGyeAxD=gsvfA9dz7tHE-XnndTDP5N4Augt1Sw@mail.gmail.com' \
--to=quentin.neill.gnu@gmail.com \
--cc=gcc-patches@gcc.gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).