user/src/com/google/gwt/junit/benchmarks/BenchmarkReport.java

/*
 * Copyright 2007 Google Inc.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package com.google.gwt.junit.benchmarks;

import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.typeinfo.HasMetaData;
import com.google.gwt.core.ext.typeinfo.JClassType;
import com.google.gwt.core.ext.typeinfo.JMethod;
import com.google.gwt.core.ext.typeinfo.TypeOracle;
import com.google.gwt.dev.util.Util;
import com.google.gwt.junit.client.TestResults;
import com.google.gwt.junit.client.Trial;
import com.google.gwt.junit.rebind.BenchmarkGenerator;
import com.google.gwt.util.tools.Utility;

import junit.framework.TestCase;

import org.eclipse.jdt.internal.compiler.IProblemFactory;
import org.eclipse.jdt.internal.compiler.ISourceElementRequestor;
import org.eclipse.jdt.internal.compiler.SourceElementParser;
import org.eclipse.jdt.internal.compiler.SourceElementRequestorAdapter;
import org.eclipse.jdt.internal.compiler.batch.CompilationUnit;
import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.text.BreakIterator;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

/**
 * Generates a detailed report that contains the results of all of the
 * benchmark-related unit tests executed during a unit test session. The primary
 * user of this class is JUnitShell.
 * 
 * The report is in XML format. To view the XML reports, use benchmarkViewer.
 */
public class BenchmarkReport {

  /**
   * Converts a set of test results for a single benchmark method into XML.
   */
  private class BenchmarkXml {

    private MetaData metaData;

    private List/* <JUnitMessageQueue.TestResult> */results;

    private TestCase test;

    BenchmarkXml(TestCase test,
        List/* <JUnitMessageQueue.TestResult> */results) {
      this.test = test;
      this.results = results;
      Map/* <String,MetaData> */methodMetaData = (Map/* <String,MetaData> */) testMetaData.get(test.getClass().toString());
      metaData = (MetaData) methodMetaData.get(test.getName());
    }

    Element toElement(Document doc) {
      Element benchmark = doc.createElement("benchmark");
      benchmark.setAttribute("class", test.getClass().getName());
      benchmark.setAttribute("name", metaData.getTestName());
      benchmark.setAttribute("description", metaData.getTestDescription());

      String sourceCode = metaData.getSourceCode();
      if (sourceCode != null) {
        Element sourceCodeElement = doc.createElement("source_code");
        sourceCodeElement.appendChild(doc.createTextNode(sourceCode));
        benchmark.appendChild(sourceCodeElement);
      }

      // TODO(tobyr): create target_code element

      for (Iterator it = results.iterator(); it.hasNext();) {
        TestResults result = (TestResults) it.next();
        benchmark.appendChild(toElement(doc, result));
      }

      return benchmark;
    }

    private Element toElement(Document doc, TestResults result) {
      Element resultElement = doc.createElement("result");
      resultElement.setAttribute("host", result.getHost());
      resultElement.setAttribute("agent", result.getAgent());

      List trials = result.getTrials();

      for (Iterator it = trials.iterator(); it.hasNext();) {
        Trial trial = (Trial) it.next();
        Element trialElement = toElement(doc, trial);
        resultElement.appendChild(trialElement);
      }

      return resultElement;
    }

    private Element toElement(Document doc, Trial trial) {
      Element trialElement = doc.createElement("trial");

      Map variables = trial.getVariables();

      for (Iterator it = variables.entrySet().iterator(); it.hasNext();) {
        Map.Entry entry = (Map.Entry) it.next();
        Object name = entry.getKey();
        Object value = entry.getValue();
        Element variableElement = doc.createElement("variable");
        variableElement.setAttribute("name", name.toString());
        variableElement.setAttribute("value", value.toString());
        trialElement.appendChild(variableElement);
      }

      trialElement.setAttribute("timing",
          String.valueOf(trial.getRunTimeMillis()));

      Throwable exception = trial.getException();

      if (exception != null) {
        Element exceptionElement = doc.createElement("exception");
        exceptionElement.appendChild(doc.createTextNode(exception.toString()));
        trialElement.appendChild(exceptionElement);
      }

      return trialElement;
    }
  }

  /**
   * Parses a .java source file to get the source code for methods.
   * 
   * This Parser takes some shortcuts based on the fact that it's only being
   * used to locate test methods for unit tests. (For example, only requiring a
   * method name instead of a full type signature for lookup).
   * 
   * TODO(tobyr) I think that I might be able to replace all this code with a
   * call to the existing metadata interface. Check declEnd/declStart in
   * JAbstractMethod.
   */
  private static class Parser {

    static class MethodBody {

      int declarationEnd; // the character index of the end of the method

      int declarationStart; // the character index of the start of the method

      String source;
    }

    private MethodBody currentMethod; // Only used during the visitor

    // But it's less painful
    private Map/* <String,MethodBody> */methods = new HashMap/* <String,MethodBody> */();

    // Contains the contents of the entire source file
    private char[] sourceContents;

    Parser(JClassType klass) throws IOException {

      Map settings = new HashMap();
      settings.put(CompilerOptions.OPTION_Source, CompilerOptions.VERSION_1_4);
      settings.put(CompilerOptions.OPTION_TargetPlatform,
          CompilerOptions.VERSION_1_4);
      settings.put(CompilerOptions.OPTION_DocCommentSupport,
          CompilerOptions.ENABLED);
      CompilerOptions options = new CompilerOptions(settings);

      IProblemFactory problemFactory = new DefaultProblemFactory(
          Locale.getDefault());

      // Save off the bounds of any method that is a test method
      ISourceElementRequestor requestor = new SourceElementRequestorAdapter() {
        public void enterMethod(MethodInfo methodInfo) {
          String name = new String(methodInfo.name);
          if (name.startsWith("test")) {
            currentMethod = new MethodBody();
            currentMethod.declarationStart = methodInfo.declarationStart;
            methods.put(name, currentMethod);
          }
        }

        public void exitMethod(int declarationEnd, int defaultValueStart,
            int defaultValueEnd) {
          if (currentMethod != null) {
            currentMethod.declarationEnd = declarationEnd;
            currentMethod = null;
          }
        }
      };

      boolean reportLocalDeclarations = true;
      boolean optimizeStringLiterals = true;

      SourceElementParser parser = new SourceElementParser(requestor,
          problemFactory, options, reportLocalDeclarations,
          optimizeStringLiterals);

      File sourceFile = findSourceFile(klass);
      sourceContents = read(sourceFile);
      CompilationUnit unit = new CompilationUnit(sourceContents,
          sourceFile.getName(), null);

      parser.parseCompilationUnit(unit, true);
    }

    /**
     * Returns the source code for the method of the given name.
     * 
     * @return null if the source code for the method can not be located
     */
    public String getMethod(JMethod method) {
      /*
       * MethodBody methodBody = (MethodBody)methods.get( method.getName() ); if (
       * methodBody == null ) { return null; } if ( methodBody.source == null ) {
       * methodBody.source = new String(sourceContents,
       * methodBody.declarationStart, methodBody.declarationEnd - methodBody.
       * declarationStart + 1); } return methodBody.source;
       */
      return new String(sourceContents, method.getDeclStart(),
          method.getDeclEnd() - method.getDeclStart() + 1);
    }
  }

  /**
   * Converts an entire report into XML.
   */
  private class ReportXml {

    private Map/* <String,Element> */categoryElementMap = new HashMap/* <String,Element> */();

    private Date date = new Date();

    private String version = "unknown";

    Element toElement(Document doc) {
      Element report = doc.createElement("gwt_benchmark_report");
      String dateString = DateFormat.getDateTimeInstance().format(date);
      report.setAttribute("date", dateString);
      report.setAttribute("gwt_version", version);

      // - Add each test result into the report.
      // - Add the category for the test result, if necessary.
      for (Iterator entryIt = testResults.entrySet().iterator(); entryIt.hasNext();) {
        Map.Entry entry = (Map.Entry) entryIt.next();
        TestCase test = (TestCase) entry.getKey();
        List/* <JUnitMessageQueue.TestResult> */results = (List/* <JUnitMessageQueue.TestResult> */) entry.getValue();
        BenchmarkXml xml = new BenchmarkXml(test, results);
        Element categoryElement = getCategoryElement(doc, report,
            xml.metaData.getCategory().getClassName());
        categoryElement.appendChild(xml.toElement(doc));
      }

      return report;
    }

    /**
     * Locates or creates the category element by the specified name.
     * 
     * @param doc The document to search
     * @return The matching category element
     */
    private Element getCategoryElement(Document doc, Element report, String name) {
      Element e = (Element) categoryElementMap.get(name);

      if (e != null) {
        return e;
      }

      Element categoryElement = doc.createElement("category");
      categoryElementMap.put(name, categoryElement);
      CategoryImpl category = (CategoryImpl) testCategories.get(name);
      categoryElement.setAttribute("name", category.getName());
      categoryElement.setAttribute("description", category.getDescription());

      report.appendChild(categoryElement);

      return categoryElement;
    }
  }

  private static final String GWT_BENCHMARK_CATEGORY = "gwt.benchmark.category";

  private static final String GWT_BENCHMARK_DESCRIPTION = "gwt.benchmark.description";

  private static final String GWT_BENCHMARK_NAME = "gwt.benchmark.name";

  private static File findSourceFile(JClassType klass) {
    final char separator = File.separator.charAt(0);
    String filePath = klass.getPackage().getName().replace('.', separator)
        + separator + klass.getSimpleSourceName() + ".java";
    String[] paths = getClassPath();

    for (int i = 0; i < paths.length; ++i) {
      File maybeSourceFile = new File(paths[i] + separator + filePath);

      if (maybeSourceFile.exists()) {
        return maybeSourceFile;
      }
    }

    return null;
  }

  private static String[] getClassPath() {
    String path = System.getProperty("java.class.path");
    return path.split(File.pathSeparator);
  }

  private static String getSimpleMetaData(HasMetaData hasMetaData, String name) {
    String[][] allValues = hasMetaData.getMetaData(name);

    if (allValues == null) {
      return null;
    }

    StringBuffer result = new StringBuffer();

    for (int i = 0; i < allValues.length; ++i) {
      String[] values = allValues[i];
      for (int j = 0; j < values.length; ++j) {
        result.append(values[j]);
        result.append(" ");
      }
    }

    String resultString = result.toString().trim();
    return resultString.equals("") ? null : resultString;
  }

  private static char[] read(File f) throws IOException {
    // TODO(tobyr) Can be done oh so much faster by just reading directly into
    // a char[]

    BufferedReader reader = new BufferedReader(new FileReader(f));
    StringBuffer source = new StringBuffer((int) f.length());

    while (true) {
      String line = reader.readLine();
      if (line == null) {
        break;
      }
      source.append(line);
      source.append("\n");
    }

    char[] buf = new char[source.length()];
    source.getChars(0, buf.length, buf, 0);

    return buf;
  }

  private Map/* <String,Map<CategoryImpl> */testCategories = new HashMap/* <String,CategoryImpl> */();

  private Map/* <String,Map<String,MetaData>> */testMetaData = new HashMap/* <String,Map<String,MetaData>> */();

  private Map/* <TestCase,List<JUnitMessageQueue.TestResult>> */testResults = new HashMap/* <TestCase,JUnitMessageQueue.List<TestResult>> */();

  private TypeOracle typeOracle;

  private TreeLogger logger;

  public BenchmarkReport(TreeLogger logger) {
    this.logger = logger;
  }

  /**
   * Adds the Benchmark to the report. All of the metadata about the benchmark
   * (category, name, description, etc...) is recorded from the TypeOracle.
   */
  public void addBenchmark(JClassType benchmarkClass, TypeOracle typeOracle) {

    this.typeOracle = typeOracle;
    String categoryType = getSimpleMetaData(benchmarkClass,
        GWT_BENCHMARK_CATEGORY);

    Map zeroArgMethods = BenchmarkGenerator.getNotOverloadedTestMethods(benchmarkClass);
    Map/* <String,JMethod> */parameterizedMethods = BenchmarkGenerator.getParameterizedTestMethods(
        benchmarkClass, TreeLogger.NULL);
    List/* <JMethod> */testMethods = new ArrayList/* <JMethod> */(
        zeroArgMethods.size() + parameterizedMethods.size());
    testMethods.addAll(zeroArgMethods.values());
    testMethods.addAll(parameterizedMethods.values());

    Map/* <String,MetaData> */metaDataMap = (Map/* <String,MetaData> */) testMetaData.get(benchmarkClass.toString());
    if (metaDataMap == null) {
      metaDataMap = new HashMap/* <String,MetaData> */();
      testMetaData.put(benchmarkClass.toString(), metaDataMap);
    }

    Parser parser = null;

    try {
      parser = new Parser(benchmarkClass);
    } catch (IOException e) {
      // if we fail to create the parser for some reason, we'll have to just
      // deal with a null parser.
      logger.log(TreeLogger.WARN, "Unable to parse the code for "
          + benchmarkClass, e);
    }

    // Add all of the benchmark methods
    for (int i = 0; i < testMethods.size(); ++i) {
      JMethod method = (JMethod) testMethods.get(i);
      String methodName = method.getName();
      String methodCategoryType = getSimpleMetaData(method,
          GWT_BENCHMARK_CATEGORY);
      if (methodCategoryType == null) {
        methodCategoryType = categoryType;
      }
      CategoryImpl methodCategory = getCategory(methodCategoryType);
      StringBuffer sourceCode = parser == null ? null : new StringBuffer(
          parser.getMethod(method));
      StringBuffer summary = new StringBuffer();
      StringBuffer comment = new StringBuffer();
      getComment(sourceCode, summary, comment);

      MetaData metaData = new MetaData(benchmarkClass.toString(), methodName,
          sourceCode != null ? sourceCode.toString() : null, methodCategory,
          methodName, summary.toString());
      metaDataMap.put(methodName, metaData);
    }
  }

  public void addBenchmarkResults(TestCase test, TestResults results) {
    List/* <TestResults> */currentResults = (List/* <TestResults> */) testResults.get(test);
    if (currentResults == null) {
      currentResults = new ArrayList/* <TestResults> */();
      testResults.put(test, currentResults);
    }
    currentResults.add(results);
  }

  /**
   * Generates reports for all of the benchmarks which were added to the
   * generator.
   * 
   * @param outputPath The path to write the reports to.
   * @throws ParserConfigurationException
   * @throws IOException
   * @throws IOException If anything goes wrong writing to outputPath
   */
  public void generate(String outputPath) throws ParserConfigurationException,
      IOException {

    // Don't generate a new report if no tests were actually run.
    if (testResults.size() == 0) {
      return;
    }

    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
    DocumentBuilder builder = factory.newDocumentBuilder();
    Document doc = builder.newDocument();
    doc.appendChild(new ReportXml().toElement(doc));
    byte[] xmlBytes = Util.toXmlUtf8(doc);
    FileOutputStream fos = null;
    try {
      fos = new FileOutputStream(outputPath);
      fos.write(xmlBytes);
    } finally {
      Utility.close(fos);
    }

    // TODO(bruce) The code below is commented out because of GWT Issue 958.

    // // TODO(tobyr) Looks like indenting is busted
    // // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6296446
    // // Not a big deal, since we don't intend to read the XML by hand anyway
    // TransformerFactory transformerFactory = TransformerFactory.newInstance();
    // // Think this can be used with JDK 1.5
    // // transformerFactory.setAttribute( "indent-number", new Integer(2) );
    // Transformer serializer = transformerFactory.newTransformer();
    // serializer.setOutputProperty(OutputKeys.METHOD, "xml");
    // serializer.setOutputProperty(OutputKeys.INDENT, "yes");
    // serializer
    // .setOutputProperty("{ http://xml.apache.org/xslt }indent-amount", "2");
    // BufferedOutputStream docOut = new BufferedOutputStream(
    // new FileOutputStream(outputPath));
    // serializer.transform(new DOMSource(doc), new StreamResult(docOut));
    // docOut.close();
  }

  private CategoryImpl getCategory(String name) {
    CategoryImpl c = (CategoryImpl) testCategories.get(name);

    if (c != null) {
      return c;
    }

    String categoryName = "";
    String categoryDescription = "";

    if (name != null) {
      JClassType categoryType = typeOracle.findType(name);

      if (categoryType != null) {
        categoryName = getSimpleMetaData(categoryType, GWT_BENCHMARK_NAME);
        categoryDescription = getSimpleMetaData(categoryType,
            GWT_BENCHMARK_DESCRIPTION);
      }
    }

    c = new CategoryImpl(name, categoryName, categoryDescription);
    testCategories.put(name, c);
    return c;
  }

  /**
   * Parses out the JavaDoc comment from a string of source code. Returns the
   * first sentence summary in <code>summary</code> and the body of the entire
   * comment (including the summary) in <code>comment</code>.
   */
  private void getComment(StringBuffer sourceCode, StringBuffer summary,
      StringBuffer comment) {

    if (sourceCode == null) {
      return;
    }

    summary.setLength(0);
    comment.setLength(0);

    String regex = "/\\*\\*(.(?!}-\\*/))*\\*/";

    Pattern p = Pattern.compile(regex, Pattern.DOTALL);
    Matcher m = p.matcher(sourceCode);

    // Early out if there is no javadoc comment.
    if (!m.find()) {
      return;
    }

    String commentStr = m.group();

    p = Pattern.compile("(/\\*\\*\\s*)" + // The comment header
        "(((\\s*\\**\\s*)([^\n\r]*)[\n\r]+)*)" // The comment body
    );

    m = p.matcher(commentStr);

    if (!m.find()) {
      return;
    }

    String stripped = m.group(2);

    p = Pattern.compile("^\\p{Blank}*\\**\\p{Blank}*", Pattern.MULTILINE);
    String bareComment = p.matcher(stripped).replaceAll("");

    BreakIterator iterator = BreakIterator.getSentenceInstance();
    iterator.setText(bareComment);
    int firstSentenceEnd = iterator.next();
    if (firstSentenceEnd == BreakIterator.DONE) {
      summary.append(bareComment);
    } else {
      summary.append(bareComment.substring(0, firstSentenceEnd));
    }

    comment.append(bareComment);

    // Measure the indentation width on the second line to infer what the
    // first line indent should be.
    p = Pattern.compile("[^\\r\\n]+[\\r\\n]+(\\s+)\\*", Pattern.MULTILINE);
    m = p.matcher(sourceCode);
    int indentLen = 0;
    if (m.find()) {
      String indent = m.group(1);
      indentLen = indent.length() - 1;
    }
    StringBuffer leadingIndent = new StringBuffer();
    for (int i = 0; i < indentLen; ++i) {
      leadingIndent.append(' ');
    }

    // By inserting at 0 here, we are assuming that sourceCode begins with
    // /**, which is actually a function of how JDT sees a declaration start.
    // If in the future, you see bogus indentation here, it means that this
    // assumption is bad.
    sourceCode.insert(0, leadingIndent);
  }
}