Java example source code file (ParallelTestRunner.java)

This example Java source code file (ParallelTestRunner.java) is included in the alvinalexander.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Learn more about this Java project at its project page.

Java - Java tags/keywords

bufferedreader, bytearrayoutputstream, file, fileoutputstream, ioexception, list, outputstream, override, printstream, printwriter, regex, scriptrunnable, string, testfailederror, threading, threads, throwable, util

The ParallelTestRunner.java Java example source code

/*
 * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.  Oracle designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Oracle in the LICENSE file that accompanied this code.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

package jdk.nashorn.internal.test.framework;

import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringReader;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jdk.nashorn.internal.test.framework.TestFinder.TestFactory;

/**
 * Parallel test runner runs tests in multiple threads - but avoids any dependency
 * on third-party test framework library such as TestNG.
 */
public class ParallelTestRunner {

    // ParallelTestRunner-specific
    private static final String    TEST_JS_THREADS     = "test.js.threads";
    private static final String    TEST_JS_REPORT_FILE = "test.js.report.file";
    private static final int       THREADS             = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors());

    private final List<ScriptRunnable> tests    = new ArrayList<>();
    private final Set<String>      orphans  = new TreeSet<>();
    private final ExecutorService  executor = Executors.newFixedThreadPool(THREADS);

    // Ctrl-C handling
    private final CountDownLatch   finishedLatch = new CountDownLatch(1);
    private final Thread           shutdownHook  = new Thread() {
                                                       @Override
                                                       public void run() {
                                                           if (!executor.isTerminated()) {
                                                               executor.shutdownNow();
                                                               try {
                                                                   executor.awaitTermination(25, TimeUnit.SECONDS);
                                                                   finishedLatch.await(5, TimeUnit.SECONDS);
                                                               } catch (final InterruptedException e) {
                                                                   // empty
                                                               }
                                                           }
                                                       }
                                                   };

    public ParallelTestRunner() throws Exception {
        suite();
    }

    private static PrintStream outputStream() {
        final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, "");
        PrintStream output = System.out;

        if (!reportFile.isEmpty()) {
            try {
                output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile)));
            } catch (final IOException e) {
                System.err.println(e);
            }
        }

        return output;
    }

    public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> {
        private final Result                result   = new Result();

        public class Result {
            private boolean  passed = true;
            public String    expected;
            public String    out;
            public String    err;
            public Throwable exception;

            public ScriptRunnable getTest() {
                return ScriptRunnable.this;
            }

            public boolean passed() {
                return passed;
            }

            @Override
            public String toString() {
                return getTest().toString();
            }
        }

        public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map testOptions, final List scriptArguments) {
            super(framework, testFile, engineOptions, testOptions, scriptArguments);
        }

        @Override
        protected void log(String msg) {
            System.err.println(msg);
        }

        @Override
        protected void fail(final String message) {
            throw new TestFailedError(message);
        }

        @Override
        protected void compile() throws IOException {
            final ByteArrayOutputStream out = new ByteArrayOutputStream();
            final ByteArrayOutputStream err = new ByteArrayOutputStream();
            final List<String> args = getCompilerArgs();
            int errors;
            try {
                errors = evaluateScript(out, err, args.toArray(new String[args.size()]));
            } catch (final AssertionError e) {
                final PrintWriter writer = new PrintWriter(err);
                e.printStackTrace(writer);
                writer.flush();
                errors = 1;
            }
            if (errors != 0 || checkCompilerMsg) {
                result.err = err.toString();
                if (expectCompileFailure || checkCompilerMsg) {
                    final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName()));
                    TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray())));
                    outputDest.println("--");
                }
                if (errors != 0 && !expectCompileFailure) {
                    fail(String.format("%d errors compiling %s", errors, testFile));
                }
                if (checkCompilerMsg) {
                    compare(getErrorFileName(), expectedFileName, true);
                }
            }
            if (expectCompileFailure && errors == 0) {
                fail(String.format("No errors encountered compiling negative test %s", testFile));
            }
        }

        @Override
        protected void execute() {
            final List<String> args = getRuntimeArgs();
            final ByteArrayOutputStream out = new ByteArrayOutputStream();
            final ByteArrayOutputStream err = new ByteArrayOutputStream();

            try {
                final int errors = evaluateScript(out, err, args.toArray(new String[args.size()]));

                if (errors != 0 || err.size() > 0) {
                    if (expectRunFailure) {
                        return;
                    }
                    if (!ignoreStdError) {

                        try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
                            outputFile.write(out.toByteArray());
                            errorFile.write(err.toByteArray());
                        }

                        result.out = out.toString();
                        result.err = err.toString();
                        fail(err.toString());
                    }
                }

                if (compare) {
                    final File expectedFile = new File(expectedFileName);
                    try {
                        BufferedReader expected;
                        if (expectedFile.exists()) {
                            expected = new BufferedReader(new FileReader(expectedFile));
                        } else {
                            expected = new BufferedReader(new StringReader(""));
                        }
                        compare(new BufferedReader(new StringReader(out.toString())), expected, false);
                    } catch (final Throwable ex) {
                        if (expectedFile.exists()) {
                            copyExpectedFile();
                        }
                        try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
                            outputFile.write(out.toByteArray());
                            errorFile.write(err.toByteArray());
                        }
                        throw ex;
                    }
                }
            } catch (final IOException e) {
                if (!expectRunFailure) {
                    fail("Failure running test " + testFile + ": " + e.getMessage());
                } // else success
            }
        }

        private void compare(final String outputFileName, final String expected, final boolean compareCompilerMsg) throws IOException {
            final File expectedFile = new File(expected);

            BufferedReader expectedReader;
            if (expectedFile.exists()) {
                expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName)));
            } else {
                expectedReader = new BufferedReader(new StringReader(""));
            }

            final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(outputFileName)));

            compare(actual, expectedReader, compareCompilerMsg);
        }

        private void copyExpectedFile() {
            if (!new File(expectedFileName).exists()) {
                return;
            }
            // copy expected file overwriting existing file and preserving last
            // modified time of source
            try {
                Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
            } catch (final IOException ex) {
                fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage());
            }
        }

        @Override
        public Result call() {
            try {
                runTest();
            } catch (final Throwable ex) {
                result.exception = ex;
                result.passed = false;
            }
            return result;
        }

        private String getOutputFileName() {
            buildDir.mkdirs();
            return outputFileName;
        }

        private String getErrorFileName() {
            buildDir.mkdirs();
            return errorFileName;
        }

        private String getCopyExpectedFileName() {
            buildDir.mkdirs();
            return copyExpectedFileName;
        }
    }

    private void suite() throws Exception {
        Locale.setDefault(new Locale(""));
        System.setOut(outputStream());

        final TestFactory<ScriptRunnable> testFactory = new TestFactory() {
            @Override
            public ScriptRunnable createTest(String framework, File testFile, List<String> engineOptions, Map testOptions, List arguments) {
                return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments);
            }

            @Override
            public void log(String msg) {
                System.err.println(msg);
            }
        };

        TestFinder.findAllTests(tests, orphans, testFactory);

        Collections.sort(tests, new Comparator<ScriptRunnable>() {
            @Override
            public int compare(final ScriptRunnable o1, final ScriptRunnable o2) {
                return o1.testFile.compareTo(o2.testFile);
            }
        });
    }

    @SuppressWarnings("resource")
    public boolean run() throws IOException {
        final int testCount = tests.size();
        int passCount = 0;
        int doneCount = 0;
        System.out.printf("Found %d tests.\n", testCount);
        final long startTime = System.nanoTime();

        Runtime.getRuntime().addShutdownHook(shutdownHook);

        final List<Future futures = new ArrayList<>();
        for (final ScriptRunnable test : tests) {
            futures.add(executor.submit(test));
        }

        executor.shutdown();
        try {
            executor.awaitTermination(60, TimeUnit.MINUTES);
        } catch (final InterruptedException ex) {
            // empty
        }

        final List<ScriptRunnable.Result> results = new ArrayList<>();
        for (final Future<ScriptRunnable.Result> future : futures) {
            if (future.isDone()) {
                try {
                    final ScriptRunnable.Result result = future.get();
                    results.add(result);
                    doneCount++;
                    if (result.passed()) {
                        passCount++;
                    }
                } catch (CancellationException | ExecutionException ex) {
                    ex.printStackTrace();
                } catch (final InterruptedException ex) {
                    assert false : "should not reach here";
                }
            }
        }

        Collections.sort(results, new Comparator<ScriptRunnable.Result>() {
            @Override
            public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) {
                return o1.getTest().testFile.compareTo(o2.getTest().testFile);
            }
        });

        boolean hasFailed = false;
        final String failedList = System.getProperty(TEST_FAILED_LIST_FILE);
        final boolean hasFailedList = failedList != null;
        final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0;
        final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null;
        try {
            final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter);
            for (final ScriptRunnable.Result result : results) {
                if (!result.passed()) {
                    if (hasFailed == false) {
                        hasFailed = true;
                        System.out.println();
                        System.out.println("FAILED TESTS");
                    }

                    System.out.println(result.getTest());
                    if(failedFileWriter != null) {
                        failedListWriter.println(result.getTest().testFile.getPath());
                    }
                    if (result.exception != null) {
                        final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();
                        System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");
                        System.out.print(result.out != null ? result.out : "");
                    }
                }
            }
        } finally {
            if(failedFileWriter != null) {
                failedFileWriter.close();
            }
        }
        final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s]
        System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60);
        System.out.flush();

        finishedLatch.countDown();

        if (hasFailed) {
            throw new AssertionError("TEST FAILED");
        }

        if(hasFailedList) {
            new File(failedList).delete();
        }

        if(hadPreviouslyFailingTests) {
            System.out.println();
            System.out.println("Good job on getting all your previously failing tests pass!");
            System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures.");
            System.out.println();
        }

        return hadPreviouslyFailingTests;
    }

    public static void main(final String[] args) throws Exception {
        parseArgs(args);

        while(new ParallelTestRunner().run());
    }

    private static void parseArgs(final String[] args) {
        if (args.length > 0) {
            String roots = "";
            String reportFile = "";
            for (int i = 0; i < args.length; i++) {
                if (args[i].equals("--roots") && i != args.length - 1) {
                    roots += args[++i] + " ";
                } else if (args[i].equals("--report-file") && i != args.length - 1) {
                    reportFile = args[++i];
                } else if (args[i].equals("--test262")) {
                    try {
                        setTest262Properties();
                    } catch (final IOException ex) {
                        System.err.println(ex);
                    }
                }
            }
            if (!roots.isEmpty()) {
                System.setProperty(TEST_JS_ROOTS, roots.trim());
            }
            if (!reportFile.isEmpty()) {
                System.setProperty(TEST_JS_REPORT_FILE, reportFile);
            }
        }
    }

    private static void setTest262Properties() throws IOException {
        System.setProperty(TEST_JS_ROOTS, "test/test262/test/suite/");
        System.setProperty(TEST_JS_FRAMEWORK, "test/script/test262.js test/test262/test/harness/framework.js test/test262/test/harness/sta.js");
        System.setProperty(TEST_JS_EXCLUDES_FILE, "test/test262/test/config/excludelist.xml");
        System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true");

        final Properties projectProperties = new Properties();
        projectProperties.load(new FileInputStream("project.properties"));
        String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", "");
        final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}");
        for (;;) {
            final Matcher matcher = pattern.matcher(excludeList);
            if (!matcher.find()) {
                break;
            }
            final String propertyValue = projectProperties.getProperty(matcher.group(1), "");
            excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end());
        }
        System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList);
    }

    public static final class OutputStreamDelegator extends OutputStream {
        private final OutputStream[] streams;

        public OutputStreamDelegator(final OutputStream... streams) {
            this.streams = streams;
        }

        @Override
        public void write(final int b) throws IOException {
            for (final OutputStream stream : streams) {
                stream.write(b);
            }
        }

        @Override
        public void flush() throws IOException {
            for (final OutputStream stream : streams) {
                stream.flush();
            }
        }
    }
}

final class TestFailedError extends Error {
    private static final long serialVersionUID = 1L;

    public TestFailedError(final String message) {
        super(message);
    }

    public TestFailedError(final String message, final Throwable cause) {
        super(message, cause);
    }

    public TestFailedError(final Throwable cause) {
        super(cause);
    }
}