Index: Harness.java =================================================================== RCS file: Harness.java diff -N Harness.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ Harness.java 27 Mar 2006 21:33:20 -0000 @@ -0,0 +1,955 @@ +// Copyright (c) 2006 Red Hat, Inc. +// Written by Anthony Balkissoon +// Adapted from gnu.testlet.SimpleTestHarness written by Tom Tromey. +// Copyright (c) 2005 Mark J. Wielaard + +// This file is part of Mauve. + +// Mauve is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2, or (at your option) +// any later version. + +// Mauve is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Mauve; see the file COPYING. If not, write to +// the Free Software Foundation, 59 Temple Place - Suite 330, +// Boston, MA 02111-1307, USA. + +// KNOWN BUGS: +// - should look for /*{ ... }*/ and treat contents as expected +// output of test. In this case we should redirect System.out +// to a temp file we create. + +/* + * See the README.Harness file for information on how to use this + * file and what it is designed to do. + */ + +import gnu.testlet.ResourceNotFoundException; +import gnu.testlet.TestHarness; +import gnu.testlet.TestReport; +import gnu.testlet.TestResult; +import gnu.testlet.TestSecurityManager; +import gnu.testlet.Testlet; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Vector; + +public class Harness + extends TestHarness +{ + private int count = 0; + + private int failures = 0; + + private static Vector expected_xfails = new Vector(); + + private int xfailures = 0; + + private int xpasses = 0; + + private int total = 0; + + private boolean verbose = false; + + private boolean debug = false; + + private boolean results_only = false; + + private boolean exceptions = false; + + private String description; + + private String last_check; + + private TestReport report = null; + + private TestResult currentResult = null; + + private static boolean recursion = true; + + private static boolean showPasses = false; + + private static int total_tests = 0; + + private static int total_test_fails = 0; + + private final String getDescription(String pf) + { + return (pf + ": " + description + + ((last_check == null) ? "" : (": " + last_check)) + " (number " + + (count + 1) + ")"); + } + + protected int getFailures() + { + return failures; + } + + /** + * Removes the "gnu.testlet." from the start of a String. + * @param val the String + * @return the String with "gnu.testlet." removed + */ + private static String stripPrefix(String val) + { + if (val.startsWith("gnu.testlet.")) + val = val.substring(12); + return val; + } + + /** + * A convenience method that sets a checkpoint with the specified name + * then records a failed check. + * + * @param name the checkpoint name. + */ + public void fail(String name) + { + checkPoint(name); + check2(false); + System.out.println ("forced fail"); + } + + /** + * Checks the two objects for equality and records the result of + * the check. + * + * @param result the actual result. + * @param expected the expected result. + */ + public void check(Object result, Object expected) + { + boolean ok = (result == null ? expected == null : result.equals(expected)); + check2(ok); + // This debug message may be misleading, depending on whether + // string conversion produces same results for unequal objects. + if (! ok) + { + String gotString = result == null ? "null" + : result.getClass().getName(); + String expString = expected == null ? "null" + : expected.getClass().getName(); + if (gotString.equals(expString)) + { + if (debug) + { + gotString = result.toString(); + expString = expected.toString(); + System.out.println("\n got " + gotString + + "\n\n but expected " + expString + + "\n\n"); + return; + } + else + { + System.out.println("objects were not equal. " + + "Use -debug for more information."); + return; + } + } + System.out.println("got " + gotString + " but expected " + expString); + } + } + + /** + * Checks two booleans for equality and records the result of the check. + * + * @param result the actual result. + * @param expected the expected result. + */ + public void check(boolean result, boolean expected) + { + boolean ok = (result == expected); + check2(ok); + if (! ok) + System.out.println("got " + result + " but expected " + expected); + } + + /** + * Checks two ints for equality and records the result of the check. + * + * @param result the actual result. + * @param expected the expected result. + */ + public void check(int result, int expected) + { + boolean ok = (result == expected); + check2(ok); + if (! ok) + System.out.println("got " + result + " but expected " + expected); + } + + /** + * Checks two longs for equality and records the result of the check. + * + * @param result the actual result. + * @param expected the expected result. + */ + public void check(long result, long expected) + { + boolean ok = (result == expected); + check2(ok); + if (! ok) + System.out.println("got " + result + " but expected " + expected); + } + + /** + * Checks two doubles for equality and records the result of the check. + * + * @param result the actual result. + * @param expected the expected result. + */ + public void check(double result, double expected) + { + // This triple check overcomes the fact that == does not + // compare NaNs, and cannot tell between 0.0 and -0.0; + // and all without relying on java.lang.Double (which may + // itself be buggy - else why would we be testing it? ;) + // For 0, we switch to infinities, and for NaN, we rely + // on the identity in JLS 15.21.1 that NaN != NaN is true. + boolean ok = (result == expected ? (result != 0) + || (1 / result == 1 / expected) + : (result != result) + && (expected != expected)); + check2(ok); + if (! ok) + System.out.println("got " + result + " but expected " + expected); + } + + public void check(boolean result) + { + check2(result); + if (!result) + System.out.println ("boolean passed to check was false"); + } + + /** + * This method prints out failures and checks the XFAILS file. + * @param result true if the test passed, false if it failed + */ + private void check2(boolean result) + { + if (! result) + { + StackTraceElement[] st = new Throwable().getStackTrace(); + int line = -1; + for (int i = 0; i < st.length; i++) + { + if (st[i].getClassName().equals(description)) + { + line = st[i].getLineNumber(); + break; + } + } + + String desc; + currentResult.addFail((last_check == null ? "" : last_check) + + " (number " + (count + 1) + ")"); + if (! expected_xfails.contains(desc = getDescription("FAIL"))) + { + if (failures == 0) + System.out.println ("\nFAIL: " + stripPrefix(description)); + System.out.print(" line " + line + ": " + + (last_check == null ? "" : last_check) + "[" + + (count + 1) + "] -- "); + ++failures; + } + else if (verbose || results_only) + { + System.out.println("X" + desc); + ++xfailures; + } + } + else + { + currentResult.addPass(); + if (verbose || results_only) + { + if (expected_xfails.contains(getDescription("FAIL"))) + { + System.out.println(getDescription("XPASS")); + ++xpasses; + } + else + { + System.out.println(getDescription("PASS")); + } + } + } + ++count; + ++total; + } + + public Reader getResourceReader(String name) throws ResourceNotFoundException + { + return new BufferedReader(new InputStreamReader(getResourceStream(name))); + } + + public InputStream getResourceStream(String name) + throws ResourceNotFoundException + { + // The following code assumes File.separator is a single character. + if (File.separator.length() > 1) + throw new Error("File.separator length is greater than 1"); + String realName = name.replace('#', File.separator.charAt(0)); + try + { + return new FileInputStream(getSourceDirectory() + File.separator + + realName); + } + catch (FileNotFoundException ex) + { + throw new ResourceNotFoundException(ex.getLocalizedMessage() + ": " + + getSourceDirectory() + + File.separator + realName); + } + } + + public File getResourceFile(String name) throws ResourceNotFoundException + { + // The following code assumes File.separator is a single character. + if (File.separator.length() > 1) + throw new Error("File.separator length is greater than 1"); + String realName = name.replace('#', File.separator.charAt(0)); + File f = new File(getSourceDirectory() + File.separator + realName); + if (! f.exists()) + { + throw new ResourceNotFoundException("cannot find mauve resource file" + + ": " + getSourceDirectory() + + File.separator + realName); + } + return f; + } + + public void checkPoint(String name) + { + last_check = name; + count = 0; + } + + public void verbose(String message) + { + if (verbose) + System.out.println(message); + } + + public void debug(String message) + { + debug(message, true); + } + + public void debug(String message, boolean newline) + { + if (debug) + { + if (newline) + System.out.println(message); + else + System.out.print(message); + } + } + + public void debug(Throwable ex) + { + if (debug) + ex.printStackTrace(System.out); + } + + public void debug(Object[] o, String desc) + { + debug("Dumping Object Array: " + desc); + if (o == null) + { + debug("null"); + return; + } + + for (int i = 0; i < o.length; i++) + { + if (o[i] instanceof Object[]) + debug((Object[]) o[i], desc + " element " + i); + else + debug(" Element " + i + ": " + o[i]); + } + } + + private void removeSecurityManager() + { + SecurityManager m = System.getSecurityManager(); + if (m instanceof TestSecurityManager) + { + TestSecurityManager tsm = (TestSecurityManager) m; + tsm.setRunChecks(false); + System.setSecurityManager(null); + } + } + + /** + * This method runs a single test. If an exception is caught some + * information is printed out so the test can be debugged. + * @param name the name of the test to run + */ + protected void runtest(String name) + { + // Try to ensure we start off with a reasonably clean slate. + System.gc(); + System.runFinalization(); + + currentResult = new TestResult(name); + + checkPoint(null); + + Testlet t = null; + try + { + Class k = Class.forName(name); + + Object o = k.newInstance(); + if (! (o instanceof Testlet)) + return; + + t = (Testlet) o; + } + catch (Throwable ex) + { + String d = "FAIL: " + stripPrefix(name) + + "uncaught exception when loading"; + currentResult.addException(ex, "failed loading class " + name); + if (verbose || exceptions) + d += ": " + ex.toString(); + + if (exceptions) + ex.printStackTrace(System.out); + debug(ex); + if (ex instanceof InstantiationException + || ex instanceof IllegalAccessException) + debug("Hint: is the code we just loaded a public non-abstract " + + "class with a public nullary constructor???"); + ++failures; + ++total; + } + + if (t != null) + { + description = name; + try + { + t.test(this); + removeSecurityManager(); + } + catch (Throwable ex) + { + if (failures == 0) + System.out.println ("\nFAIL: " + stripPrefix(name) + ":"); + removeSecurityManager(); + String s = (last_check == null ? "" : " at " + last_check + " [" + + (count + 1) + "]"); + String d = exceptionDetails(ex, name, exceptions); + currentResult.addException(ex, "uncaught exception" + s); + if (verbose || exceptions) + d += ": " + ex.toString(); + System.out.println(d); + if (exceptions) + ex.printStackTrace(System.out); + debug(ex); + ++failures; + ++total; + } + } + if (report != null) + report.addTestResult(currentResult); + } + + /** + * This method returns some information about uncaught exceptions. + * Nothing is printed if the test was run with the -exceptions flag since in + * that case a full stack trace will be printed. + * @param ex the exception + * @param name the name of the test + * @param exceptions true if a full stack trace will be printed + * @return a String containing some information about the uncaught exception + */ + private static String exceptionDetails(Throwable ex, String name, + boolean exceptions) + { + // If a full stack trace will be printed, this method returns "". + if (exceptions) + return "uncaught exception:"; + + StackTraceElement[] st = ex.getStackTrace(); + if (st == null || st.length == 0) + return "uncaught exception:"; + + // lineOrigin will store the line number in the test method that caused + // the exception. + int lineOrigin = -1; + + // This for loop looks for the line within the test method that caused the + // exception. + for (int i = 0; i < st.length; i++) + { + if (st[i].getClassName().equals(name) + && st[i].getMethodName().equals("test")) + { + lineOrigin = st[i].getLineNumber(); + break; + } + } + + // sb holds all the information we wish to return. + StringBuilder sb = new StringBuilder(" line " + lineOrigin + + ": uncaught exception:\n "); + sb.append(ex.getClass().getName() + " in "); + sb.append(stripPrefix(st[0].getClassName()) + "." + st[0].getMethodName() + + " (line " + st[0].getLineNumber() + ")"); + sb.append("\n Run tests with -exceptions to print exception " + + "stack traces."); + return sb.toString(); + } + + protected int done() + { + if (! results_only) + { + if (failures > 0 && verbose) + { + System.out.print ("TEST FAILED: "); + System.out.println(failures + " of " + total + " checks failed " + + description + "\n"); + } + else if (verbose) + System.out.println ("PASS ("+total+" checks) " + description); + if (xpasses > 0) + System.out.println(xpasses + " of " + total + + " tests unexpectedly passed"); + if (xfailures > 0) + System.out.println(xfailures + " of " + total + + " tests expectedly failed"); + } + return failures > 0 ? 1 : 0; + } + + protected Harness(boolean verbose, boolean debug) + { + this(verbose, debug, false, false, null); + } + + protected Harness(boolean verbose, boolean debug, + boolean results_only, boolean exceptions, + TestReport report) + { + this.verbose = verbose; + this.debug = debug; + this.results_only = results_only; + this.exceptions = exceptions; + this.report = report; + + try + { + BufferedReader xfile = new BufferedReader(new FileReader("xfails")); + String str; + while ((str = xfile.readLine()) != null) + { + expected_xfails.addElement(str); + } + } + catch (FileNotFoundException ex) + { + // Nothing. + } + catch (IOException ex) + { + // Nothing. + } + } + + public static void main(String[] args) + { + boolean verbose = false; + boolean debug = false; + boolean results_only = false; + boolean exceptions = false; + String file = null; + String xmlfile = null; + TestReport report = null; + Vector commandLineTests = null; + Vector excludeTests = new Vector(); + int i; + for (i = 0; i < args.length; i++) + { + if (args[i].equals("-verbose")) + verbose = true; + else if (args[i].equals("-norecursion")) + recursion = false; + else if (args[i].equals("-showpasses")) + showPasses = true; + else if (args[i].equals("-help") || args[i].equals("--help")) + printHelpMessage(); + else if (args[i].equals("-debug")) + debug = true; + else if (args[i].equals("-resultsonly")) + { + results_only = true; + verbose = false; + debug = false; + } + else if (args[i].equals("-exceptions")) + exceptions = true; + else if (args[i].equalsIgnoreCase("-file")) + { + if (++i >= args.length) + throw new RuntimeException("No file path after '-file'. Exit"); + file = args[i]; + } + else if (args[i].equals("-exclude")) + { + if (++i >= args.length) + throw new RuntimeException ("No test or directory " + + "given after '-exclude'. Exit"); + if (args[i].endsWith(".java")) + args[i] = args[i].substring(0, args[i].length() - 5); + excludeTests.add(startingFormat(args[i])); + } + else if (args[i].equals("-xmlout")) + { + if (++i >= args.length) + throw new RuntimeException("No file path after '-xmlout'."); + xmlfile = args[i]; + } + else if (args[i] != null) + { + // This is a command-line (not standard input) test or directory. + if (commandLineTests == null) + commandLineTests = new Vector(); + commandLineTests.add(startingFormat(args[i])); + } + + } + if (xmlfile != null) + { + report = new TestReport(System.getProperties()); + } + + runAllTests(file, verbose, debug, results_only, exceptions, report, + xmlfile, commandLineTests, excludeTests); + + if (total_tests > 1) + System.out.println("\nTEST RESULTS:\n" + total_test_fails + " of " + + total_tests + " tests failed.\n"); + else if (total_tests == 0) + { + if (recursion == false) + { + System.out.println ("No tests were run.\nDid you use -norecursion" + + "and specify a folder that had no tests in it?\n" + + "For example, 'jamvm -norecursion javax.swing' will not " + + "run any tests\nbecause no tests are located directly in " + + "the javax.swing folder.\n\nTry removing the -norecursion " + + "option. Use the -help option for more\ninformation or " + + "read the README.Harness file"); + } + else + printHelpMessage(); + } + else if (total_test_fails == 0 && !showPasses) + System.out.println ("TEST RESULT: pass"); + System.exit(total_test_fails > 0 ? 1 : 0); + } + + /** + * This method takes a String and puts it into a consistent format + * so we can deal with all test names in the same way. It ensures + * that tests start with "gnu.testlet" and that slashes ('/', which + * are file separators) are replaced with dots (for use in class names). + * It also strips the .java or .class extensions if they are present, + * and removes single trailing dots. + * @param val + * @return + */ + private static String startingFormat(String val) + { + if (val != null) + { + val = val.replace(File.separatorChar, '.'); + if (! val.startsWith("gnu.testlet.")) + val = "gnu.testlet." + val; + if (val.endsWith(".")) + val = val.substring(0, val.length() - 1); + if (val.endsWith(".class")) + val = val.substring(0, val.length() - 6); + } + return val; + } + + /** + * This method prints a help screen to the console. + */ + static void printHelpMessage() + { + System.out.println("********************************\n" + + "This Harness runs the tests in the Mauve test suite." + + "\nTests should be run from the top level Mauve source folder.\n" + + "********************************\n\n" + + "Usage: [java] Harness \n" + + " where [java] is the runtime you wish to use to run the test." + + "\n\nExample: 'jamvm Harness -showpasses javax.swing'\n" + + " will use jamvm (if installed) to run all the tests in the\n" + + " gnu.testlet.javax.swing folder and will display PASSES\n" + + " as well as FAILS.\n\nOptions:\n" + + " -verbose: run in noisy mode, displaying extra information\n" + + " -norecursion: if a folder is specified to be run, don't run\n" + + " the tests in its subfolders\n" + + " -showpasses: display passing tests as well as failing ones\n" + + " -exceptions: print stack traces for uncaught exceptions\n" + + " -debug: displays some extra information when tests fail\n" + + " -file [filename]: specifies a file that contains the names of\n" + + " tests to be run\n" + + " -exclude [test|folder]: specifies a test or a folder to exclude\n" + + " from the run\n" + + " -results_only: turns off verbose and debug and only displays\n" + + " pass or fail\n" + + " -xmlout [filename]: specifies a file to use for xml output\n" + + " -help: display this help message\n"); + + System.exit(0); + } + /** + * This method runs all the tests, both from the command line and from + * standard input. This is so the legacy method of running tests by + * echoing the classname and piping it to the Harness works, but so does + * a more natural "jamvm Harness ". + * @param file the file input of testnames to run + * @param verbose true if harness should run in verbose mode + * @param debug true if harness should run in debug mode + * @param results_only true if harness should run in results_only mode + * @param exceptions true if exception stack traces should be printed + * @param report the TestReport to generate + * @param xmlfile the xmlfile to use for the report + * @param commandLineTests the Vector of tests that were specified on the + * command line + */ + static void runAllTests(String file, boolean verbose, boolean debug, + boolean results_only, boolean exceptions, + TestReport report, String xmlfile, + Vector commandLineTests, Vector excludeTests) + { + // Run the commandLine tests. + if (commandLineTests != null) + { + for (int i = 0; i < commandLineTests.size(); i++) + { + String cname = null; + cname = (String) commandLineTests.elementAt(i); + if (cname == null) + break; + if (verbose) + System.out.println(cname + "\n----"); + + processTest(cname, verbose, debug, results_only, exceptions, + report, xmlfile, excludeTests); + } + } + // Now run the standard input tests. + BufferedReader r = null; + if (file != null) + try + { + r = new BufferedReader(new FileReader(file)); + } + catch (FileNotFoundException x) + { + throw new RuntimeException("Cannot find \"" + file + "\". Exit"); + } + else + { + r = new BufferedReader(new InputStreamReader(System.in)); + try + { + if (! r.ready()) + { + if (commandLineTests == null || commandLineTests.size() == 0) + processTest("gnu.testlet.all", verbose, debug, results_only, + exceptions, report, xmlfile, excludeTests); + return; + } + } + catch (IOException ioe) + { + } + } + + while (true) + { + String cname = null; + try + { + cname = r.readLine(); + if (cname == null) + break; + if (verbose) + System.out.println(cname + "\n----"); + } + catch (IOException x) + { + // Nothing. + } + processTest(startingFormat(cname), verbose, debug, results_only, + exceptions, report, xmlfile, excludeTests); + } + } + + /** + * This method runs a single test in a new Harness and increments the + * total tests run and total failures, if the test fails. Prints + * PASS and adds to the report, if the appropriate options are enabled. + * @param testName the name of the test + * @param verbose whether or not verbose mode is on + * @param debug true if debug mode is on + * @param results_only true if results_only mode is on + * @param exceptions true if exception stack traces are printed + * @param report the TestReport to generate + * @param xmlfile the name of the file for xml output + */ + static void runTest(String testName, boolean verbose, boolean debug, + boolean results_only, boolean exceptions, + TestReport report, String xmlfile) + { + Harness harness = new Harness(verbose, debug, results_only, exceptions, + report); + harness.runtest(testName); + int temp = harness.done(); + total_test_fails += temp; + total_tests ++; + if (report != null) + { + File f = new File(xmlfile); + try + { + report.writeXml(f); + } + catch (IOException e) + { + throw new Error("Failed to write data to xml file: " + + e.getMessage()); + } + } + // If the test passed and the user wants to know about passes, tell them. + if (showPasses && temp == 0) + System.out.println ("PASS: "+testName); + } + + /** + * This method handles the input, whether it is a single test or a folder + * and calls runTest on the appropriate .class files. Will also compile + * tests that haven't been compiled or that have been changed since last + * being compiled. + * @param cname the input file name - may be a directory + * @param verbose true if the harness should run in verbose mode + * @param debug true if the harness should run in debug mode + * @param results_only true if the harness should run in results_only mode + * @param exceptions true if the harness should print exception stack traces + * @param report true if the harness should generate a report + * @param xmlfile the xmlfile for the report + */ + static void processTest(String cname, boolean verbose, boolean debug, + boolean results_only, boolean exceptions, + TestReport report, String xmlfile, Vector excludesTests) + { + if (cname.equals("CVS") || cname.endsWith(File.separatorChar + "CVS") + || cname.indexOf("$") != - 1 || excludesTests.contains(cname)) + return; + + if (cname.equals("gnu.testlet.all")) + cname = "gnu.testlet"; + + // Check if cname represents a single test, and if so run it. + boolean single = true; + try + { + if (cname.endsWith(".java")) + { + // FIXME: we need to invoke the compiler here in case the .class + // is not present or is out of date. + + // FIXME: must also check to see if the file is marked not-a-test, + // although this is not a big deal, it will add a constant number + // to both the PASSES and the total tests + cname = cname.substring(0, cname.length() - 5); + if (excludesTests.contains(cname)) + return; + } + Class.forName(cname); + } + catch (Throwable t) + { + // This means it wasn't a single test. + single = false; + } + if (single) + { + runTest(cname, verbose, debug, results_only, exceptions, report, + xmlfile); + return; + } + // If we entered this branch, cname does not correspond to a + // single test, so let's treat it as a directory and run + // all the tests (recursively) within it. + cname = cname.replace('.', File.separatorChar); + File dir = new File(cname); + if (! dir.exists()) + return; + String[] filenames = dir.list(); + for (int k = 0; k < filenames.length; k++) + { + // This loop should compile all .java files that aren't + // already compiled and then run harness.runtest on the + // resulting .class files, and call processTest recursively + // on directories. + + // First, harness.runtest on .class files. + String temp = dir.getPath() + File.separatorChar + filenames[k]; + File f = new File(temp); + // If it's a directory, call this method on it. + if (f.isDirectory() && recursion + && ! excludesTests.contains(startingFormat(temp))) + processTest(temp, verbose, debug, results_only, exceptions, report, + xmlfile, excludesTests); + else if (temp.endsWith(".java")) + { + // FIXME: Here we need to check if the existing .class file + // is uptodate, otherwise we have to recompile. + File tempFile = new File(temp.substring(0, temp.length() - 4) + + "class"); + if (tempFile.exists()) + { + // The corresponding .class file existed. + String testName = tempFile.getName(); + testName = testName.substring(0, testName.length() - 6); + testName = dir + "." + testName; + //harness.runtest(); + runTest(testName.replace(File.separatorChar, '.'), verbose, + debug, results_only, exceptions, report, xmlfile); + } + else + { + // FIXME: here we need to invoke the compiler and then + // run the test. + } + } + } + } +}