Initial (in progress) version of the benchmark harness.

Change-Id: Ibcd1f3c3b7c9bb66c395b6d3d929b2c39d64c814
diff --git a/src/main/java/com/google/gwt/benchmark/BenchmarkingHarness.java b/src/main/java/com/google/gwt/benchmark/BenchmarkingHarness.java
new file mode 100644
index 0000000..1e48941
--- /dev/null
+++ b/src/main/java/com/google/gwt/benchmark/BenchmarkingHarness.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.gwt.benchmark.artifacts.PermutationInfo;
+import com.google.gwt.benchmark.git.GitException;
+import com.google.gwt.benchmark.git.GitInterface;
+import com.google.gwt.benchmark.git.RevisionInfo;
+import com.google.gwt.benchmark.project.ProjectConfiguration;
+import com.google.gwt.benchmark.runner.ProcessRunner;
+import com.google.gwt.benchmark.runner.ProcessStats;
+import com.google.gwt.benchmark.runner.RunnerException;
+
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.InvalidRemoteException;
+import org.eclipse.jgit.api.errors.TransportException;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Basic benchmarking harness.
+ */
+public class BenchmarkingHarness {
+
+  private static class Parameters {
+    int numberOfRuns = DEFAULT_NUMBER_OF_RUNS;
+    Path remoteGitMirrorPath;
+    String startingRevisionId;
+    Path toolsPath;
+    Path workDirPath;
+  }
+
+  private static final int DEFAULT_NUMBER_OF_RUNS = 3;
+
+  private static final String DEV_JAR = "build/lib/gwt-dev.jar";
+  private static final Logger log = Logger.getLogger(BenchmarkingHarness.class.getName());
+  private static final String REPO_DIR = "gwt";
+
+  private static final String REPORT_DATE_FORMAT = "%1$tY-%1$tm-%1$td";
+  private static final String REPORT_FULL_FORMAT = "%1$tY-%1$tm-%1$td %1$tH:%1$tM";
+  private static final String RESULTS_DIR = "results";
+  private static final String SCRATCH_DIR = "scratch";
+  private static final String TIME_FORMAT = "%1$tY%1$tm%1$td%1$tH%1$tM%1$tS";
+  private static final String USER_JAR = "build/lib/gwt-user.jar";
+
+  /**
+   * Entry point to the benchmarking system.
+   */
+  public static void main(String... args) throws IOException, GitAPIException, GitException,
+      RunnerException, URISyntaxException {
+
+    Parameters parameters = validateParameters(args);
+    benchmark(parameters.workDirPath, parameters.remoteGitMirrorPath, parameters.toolsPath,
+        parameters.numberOfRuns, parameters.startingRevisionId);
+  }
+
+  /**
+   * Runs the benchmarks on all projects.
+   */
+  private static void benchmark(Path workDirPath, Path mainRepoPath, Path svnToolsRepoPath,
+      int numberOfRuns, String startFromHash) throws IOException, GitAPIException, GitException,
+      RunnerException {
+
+    Path scratchDirPath = workDirPath.resolve(SCRATCH_DIR);
+    Path targetRepoPath = scratchDirPath.resolve(REPO_DIR);
+
+    FileUtils.deleteQuietly(scratchDirPath.toFile());
+    scratchDirPath.toFile().mkdirs();
+
+    // Update the tools repository.
+    updateSvnRepository(svnToolsRepoPath);
+
+    // Update the main repository for source and clone into the working repo.
+    GitInterface.updateGitRepository(mainRepoPath);
+    makePristineCheckout(mainRepoPath, targetRepoPath);
+    if (startFromHash != null) {
+      GitInterface.checkout(targetRepoPath, startFromHash);
+    }
+
+    List<RevisionInfo> revisionsToBenchmark =
+        GitInterface.getRevisionsAffectingTargets(targetRepoPath, ImmutableList.of("dev/core"),
+            ImmutableList.of(".java"));
+    for (RevisionInfo revisionInfo : revisionsToBenchmark) {
+      makePristineCheckout(mainRepoPath, targetRepoPath);
+      GitInterface.checkout(targetRepoPath, revisionInfo.getCommitId());
+      benchmarkAtCurrentRevision(workDirPath, svnToolsRepoPath, targetRepoPath, numberOfRuns);
+    }
+  }
+
+  /**
+   * Runs the benchmarks on all projects at the current revision.
+   */
+  private static void benchmarkAtCurrentRevision(Path workDirPath, Path svnToolsRepoPath,
+      Path targetRepoPath, int numberOfRuns) throws GitException, RunnerException {
+    RevisionInfo revInfo = GitInterface.getCurrentCommit(targetRepoPath);
+    Path resultsDirPath = workDirPath.resolve(RESULTS_DIR);
+
+    // Create a directory to hold the results of a compilation.
+    Path newResultsDirPath =
+        resultsDirPath.resolve(String.format(TIME_FORMAT + "-%2$s", revInfo.getCommitTime(),
+            revInfo.getCommitId()));
+
+    Date compileStartTime = new Date();
+    String compileStartTimeStamp = String.format(TIME_FORMAT, compileStartTime);
+
+    newResultsDirPath.toFile().mkdirs();
+    log.info("Results for this compile will be at " + newResultsDirPath);
+
+    Path buildOutFilePath =
+        newResultsDirPath.resolve("compile-gwt-" + compileStartTimeStamp + ".out");
+    Path buildErrFilePath =
+        newResultsDirPath.resolve("compile-gwt-" + compileStartTimeStamp + ".err");
+    // build gwt
+    ProcessRunner.runAndTimeCommand(targetRepoPath, buildOutFilePath, buildErrFilePath,
+        ImmutableMap.of("GWT_TOOLS", svnToolsRepoPath.toString()), "ant", "dist-dev");
+
+    // Compile projects
+    List<ProjectConfiguration> projectConfigurations =
+        ProjectConfiguration.getProjectConfigurations(workDirPath);
+    for (ProjectConfiguration project : projectConfigurations) {
+      Path benchmarkTimingResultsFilePath = resultsDirPath.resolve("timings." + project.getName());
+      Path benchmarkStatsResultsFilePath =
+          resultsDirPath.resolve("artifacts-stats." + project.getName());
+      for (int i = 0; i < numberOfRuns; i++) {
+        try {
+          Date benchStartTime = new Date();
+          Path dirForThisRunPath =
+              newResultsDirPath.resolve(project.getName()).resolve("runs")
+                  .resolve(String.format(TIME_FORMAT, benchStartTime));
+          ProcessStats processStats =
+              benchmarkOneCompilation(targetRepoPath, dirForThisRunPath, project);
+
+          writeTimingStats(revInfo, compileStartTime, benchmarkTimingResultsFilePath,
+              benchStartTime, processStats);
+
+          Path outJarPath = dirForThisRunPath.resolve("out.jar");
+          Path auxJarPath = dirForThisRunPath.resolve("aux.jar");
+          // Process output
+          PermutationInfo.Stats stats =
+              PermutationInfo.computeAverages(PermutationInfo.getAllPemutationInfo(outJarPath,
+                  auxJarPath, project.getOutputPath()));
+
+          writeArtifactSizeStats(revInfo, compileStartTime, benchmarkStatsResultsFilePath,
+              benchStartTime, stats);
+        } catch (Exception e) {
+          log.severe("Benchmark was not successfull:" + e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Benchmark one compilation of a project.
+   */
+  private static ProcessStats benchmarkOneCompilation(Path targetRepoPath, Path dirForThisRunPath,
+      ProjectConfiguration project) throws RunnerException {
+    log.info("Results for this run will be at " + dirForThisRunPath);
+
+    dirForThisRunPath.toFile().mkdirs();
+    List<String> commandLine =
+        project.getCompleteCommandLine(
+            dirForThisRunPath,
+            ImmutableList.of(targetRepoPath.resolve(USER_JAR).toString(),
+                targetRepoPath.resolve(DEV_JAR).toString()));
+    Path compileOutFilePath = dirForThisRunPath.resolve("compile.out");
+    Path compileErrFilePath = dirForThisRunPath.resolve("compile.err");
+
+    return ProcessRunner.runAndTimeCommand(project.getPath(), compileOutFilePath,
+        compileErrFilePath, ImmutableMap.<String, String>of(),
+        commandLine.toArray(new String[commandLine.size()]));
+  }
+
+  /**
+   * Clones a git repository from a git mirror at {@code mainRepoPath} to {@code targetRepoPath}.
+   */
+  private static void makePristineCheckout(Path mainRepoPath, Path targetRepoPath)
+      throws GitAPIException, InvalidRemoteException, TransportException {
+    FileUtils.deleteQuietly(targetRepoPath.toFile());
+    GitInterface.cloneRepository(mainRepoPath, targetRepoPath);
+  }
+
+  /**
+   * Updates the svn tools repository.
+   */
+  private static void updateSvnRepository(Path repoDirPath) throws RunnerException {
+    // Update svn/tools
+    log.info("Updating svn repo at " + repoDirPath);
+    ProcessRunner.runAndTimeCommand(repoDirPath, Paths.get("/dev/null"), Paths.get("/dev/null"),
+        ImmutableMap.<String, String>of(), "svn", "update");
+  }
+
+  /**
+   * Print command line arguments.
+   */
+  private static void usageExit() {
+    System.out
+        .println("usage: benchmark workdir gwt-remote-mirror svn-tools [#runs] [starting commit]");
+    System.exit(1);
+  }
+
+  /**
+   * Validates the command line and extracts parameter values.
+   */
+  private static Parameters validateParameters(String... args) {
+    Parameters parameters = new Parameters();
+    if (args.length < 3 || args.length > 5) {
+      usageExit();
+    }
+
+    parameters.workDirPath = Paths.get(args[0]);
+    parameters.remoteGitMirrorPath = Paths.get(args[1]);
+    parameters.toolsPath = Paths.get(args[2]);
+    try {
+      if (args.length > 3) {
+        parameters.numberOfRuns = Integer.parseInt(args[3]);
+      }
+      if (args.length > 4) {
+        parameters.startingRevisionId = args[4];
+      }
+    } catch (Exception e) {
+      usageExit();
+    }
+    return parameters;
+  }
+
+  private static void writeArtifactSizeStats(RevisionInfo revInfo, Date compileStartTime,
+      Path benchmarkStatsResultsFilePath, Date benchStartTime, PermutationInfo.Stats stats)
+      throws IOException {
+    // Commit, commit time, build time, run-time, avg-total-compressed, avg-total-compressed,
+    // avg-initial-compressed, avg-initial-uncompressed, avg-leftovers-compressed,
+    // avg-leftovers-uncompressed, avg-nbr-fragments, avg-symbolmapitems, avg-symbolmapsize
+    String outputStats =
+        String.format("%s,%s,%s,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d\n", revInfo.getCommitId(),
+            String.format(REPORT_DATE_FORMAT, revInfo.getCommitTime()),
+            String.format(REPORT_FULL_FORMAT, compileStartTime),
+            String.format(REPORT_FULL_FORMAT, benchStartTime), stats.getTotalCompressedSize(),
+            stats.getTotalUncompressedSize(), stats.getInitialCompressedSize(),
+            stats.getInitialUncompressedSize(), stats.getLeftoverCompressedSize(),
+            stats.getLeftoverUncompressedSize(), stats.getNumberofFragments(),
+            stats.getSymbolMapItems(), stats.getSymbolMapSize());
+    log.info("Stats " + outputStats);
+    FileUtils.writeStringToFile(benchmarkStatsResultsFilePath.toFile(), outputStats, true);
+  }
+
+  private static void writeTimingStats(RevisionInfo revInfo, Date compileStartTime,
+      Path benchmarkTimingResultsFilePath, Date benchStartTime, ProcessStats processStats)
+      throws IOException {
+    // Commit, commit time, build time, run-time, elapsed, system, user, memory, exitCode
+    String benchmarkResults =
+        String.format("%s,%s,%s,%s,%.2f,%.2f,%.2f,%d,%d\n", revInfo.getCommitId(),
+            String.format(REPORT_DATE_FORMAT, revInfo.getCommitTime()),
+            String.format(REPORT_FULL_FORMAT, compileStartTime),
+            String.format(REPORT_FULL_FORMAT, benchStartTime), processStats.getElapsedSeconds(),
+            processStats.getSystemSeconds(), processStats.getUserSeconds(),
+            processStats.getMemoryUsed(), processStats.getExitCode());
+    log.info("Results " + benchmarkResults);
+    FileUtils.writeStringToFile(benchmarkTimingResultsFilePath.toFile(), benchmarkResults, true);
+  }
+}
diff --git a/src/main/java/com/google/gwt/benchmark/git/GitInterface.java b/src/main/java/com/google/gwt/benchmark/git/GitInterface.java
index ce5559d..10df338 100644
--- a/src/main/java/com/google/gwt/benchmark/git/GitInterface.java
+++ b/src/main/java/com/google/gwt/benchmark/git/GitInterface.java
@@ -17,7 +17,6 @@
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-import com.google.gwt.benchmark.Benchmark;
 
 import org.eclipse.jgit.api.CheckoutCommand;
 import org.eclipse.jgit.api.CheckoutResult;
@@ -101,7 +100,7 @@
     CloneCommand clone =
         Git.cloneRepository().setDirectory(targetRepoPath.toFile()).setURI(mainRepoPath.toString());
     Git git = clone.call();
-    Benchmark.log.info("Cloned repo from " + mainRepoPath + " to " + targetRepoPath);
+    log.info("Cloned repo from " + mainRepoPath + " to " + targetRepoPath);
   }
 
   /**
@@ -157,7 +156,7 @@
    */
   public static void updateGitRepository(Path repoPath) throws IOException, GitAPIException,
       InvalidRemoteException, TransportException {
-    Benchmark.log.info("Updating git repo at " + repoPath);
+    log.info("Updating git repo at " + repoPath);
     Git repo = Git.open(repoPath.toFile());
     FetchCommand fetchCommand = repo.fetch();
     fetchCommand.call();
diff --git a/src/main/java/com/google/gwt/benchmark/project/ProjectConfiguration.java b/src/main/java/com/google/gwt/benchmark/project/ProjectConfiguration.java
index 4b0ab04..6506734 100644
--- a/src/main/java/com/google/gwt/benchmark/project/ProjectConfiguration.java
+++ b/src/main/java/com/google/gwt/benchmark/project/ProjectConfiguration.java
@@ -17,7 +17,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
+import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 
@@ -137,21 +137,28 @@
   /**
    * Returns the complete command line required to compile this project.
    */
-  public String getCompleteCommandLine(String outDir, List<String> extraJars) {
-    Preconditions.checkState(outDir != null && !outDir.isEmpty(),
+  public List<String> getCompleteCommandLine(Path outDirPath, List<String> extraJars) {
+    Preconditions.checkState(outDirPath != null && !outDirPath.toString().isEmpty(),
         "Output directory should not be null nor empty");
     Preconditions.checkState(compilerEntryPoint != null && !compilerEntryPoint.isEmpty(),
         "Compiler entry point should not be null");
     Preconditions.checkState(targetModule != null && !targetModule.isEmpty(),
         "Target module should not be null");
     String classPath = Joiner.on(":").join(Iterables.concat(extraJars, getInputJars()));
-    return Joiner
-        .on(" ")
-        .skipNulls()
-        .join("java", "-cp", classPath, Strings.emptyToNull(javaCommandLineOpts),
-            compilerEntryPoint, Strings.emptyToNull(gwtCommandLineOpts), targetModule, "-war",
-            outDir + "/" + OUT_JAR, "-extra", outDir + "/" + AUX_JAR, "-deploy",
-            outDir + "/" + AUX_JAR);
+
+    // Build the command line.
+    List<String> commandLine = Lists.newArrayList("java", "-cp", classPath);
+    Iterables.addAll(commandLine,
+        Splitter.on(" ").trimResults().omitEmptyStrings().split(javaCommandLineOpts));
+    commandLine.add(compilerEntryPoint);
+    Iterables.addAll(commandLine,
+        Splitter.on(" ").trimResults().omitEmptyStrings().split(gwtCommandLineOpts));
+    commandLine.add(targetModule);
+    Collections.addAll(commandLine, "-war", outDirPath.resolve(OUT_JAR).toString());
+    Collections.addAll(commandLine, "-extra", outDirPath.resolve(AUX_JAR).toString());
+    Collections.addAll(commandLine, "-deploy", outDirPath.resolve(AUX_JAR).toString());
+
+    return commandLine;
   }
 
   /**
diff --git a/src/test/java/com/google/gwt/benchmark/project/ProjectConfigurationTest.java b/src/test/java/com/google/gwt/benchmark/project/ProjectConfigurationTest.java
index de86146..dbf88f5 100644
--- a/src/test/java/com/google/gwt/benchmark/project/ProjectConfigurationTest.java
+++ b/src/test/java/com/google/gwt/benchmark/project/ProjectConfigurationTest.java
@@ -20,6 +20,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -80,8 +81,9 @@
         "java -cp gwt-user.jar:gwt-dev.jar -Dtest com.dummy.Compiler "
             + "-draftCompile com.target.Target -war outdir/out.jar -extra outdir/aux.jar "
             + "-deploy outdir/aux.jar",
-        projectConfiguration.getCompleteCommandLine("outdir",
-            ImmutableList.of("gwt-user.jar", "gwt-dev.jar")));
+        Joiner.on(" ").join(
+            projectConfiguration.getCompleteCommandLine(Paths.get("outdir"),
+                ImmutableList.of("gwt-user.jar", "gwt-dev.jar"))));
   }
 
   /**
@@ -117,11 +119,12 @@
     assertEquals(
         "java -cp gwt-user.jar:gwt-dev.jar com.google.gwt.dev.Compiler p2target -war outdir/out.jar"
             + " -extra outdir/aux.jar -deploy outdir/aux.jar",
-        projectConfiguration.getCompleteCommandLine("outdir",
-            ImmutableList.of("gwt-user.jar", "gwt-dev.jar")));
+        Joiner.on(" ").join(
+            projectConfiguration.getCompleteCommandLine(Paths.get("outdir"),
+                ImmutableList.of("gwt-user.jar", "gwt-dev.jar"))));
 
     try {
-      projectConfiguration.getCompleteCommandLine("", ImmutableList.<String>of());
+      projectConfiguration.getCompleteCommandLine(Paths.get(""), ImmutableList.<String>of());
       fail("Should have thrown an exception");
     } catch (Exception e) {
     }